Upload main.py

main.py

@@ -28,9 +28,9 @@ import threading
 import difflib
 from starlette.middleware.gzip import GZipMiddleware
 from transformers import pipeline as hf_pipeline
-import os
 os.environ.setdefault("OMP_NUM_THREADS", "1")
-from fastapi import Path
+from fastapi.responses import PlainTextResponse
+
 
 import torch
 torch.set_num_threads(2)
@@ -53,6 +53,17 @@ _local_pipes = {}
 _news_clf = None
 _sbert = None
 
+
+# set a writable cache for tldextract and avoid network PSL fetches
+_TLD_CACHE = os.getenv("TLDEXTRACT_CACHE", "/data/tld_cache")
+try:
+    # suffix_list_urls=None => use cached public suffix list only (no HTTP on startup)
+    _tld = tldextract.TLDExtract(cache_dir=_TLD_CACHE, suffix_list_urls=None)
+except Exception:
+    # safe fallback: still parses domains without PSL refresh
+    _tld = tldextract.extract
+
+
 # --- Translation runtime flags / caches ---
 ALLOW_HF_REMOTE = os.getenv("ALLOW_HF_REMOTE", "0") == "1"  # default OFF
 _hf_bad_models: Set[str] = set()
@@ -796,10 +807,21 @@ def cluster_id(cluster, enriched_articles):
 
 
 # ----------------- NLTK / VADER -----------------
+NLTK_DATA_DIR = os.environ.get("NLTK_DATA", "/app/nltk_data")
+
+# Make sure NLTK looks in the baked, writable dir first
+if NLTK_DATA_DIR not in nltk.data.path:
+    nltk.data.path.insert(0, NLTK_DATA_DIR)
+
 try:
     nltk.data.find("sentiment/vader_lexicon")
 except LookupError:
-    nltk.download("vader_lexicon")  # one-time fetch in a fresh container
+    # As a fallback, try downloading into the writable dir (won't run if already baked)
+    try:
+        os.makedirs(NLTK_DATA_DIR, exist_ok=True)
+        nltk.download("vader_lexicon", download_dir=NLTK_DATA_DIR, quiet=True)
+    except Exception:
+        pass  # don't crash if download is blocked
 
 try:
     _vader = SentimentIntensityAnalyzer()
@@ -875,7 +897,7 @@ def geocode_source(source_text: str, domain: str = "", do_network: bool = False)
     if cache_key in domain_geo_cache:
         return domain_geo_cache[cache_key]
 
-    ext = tldextract.extract(domain or "")
+    ext = _tld(domain or "")
     fqdn = ".".join([p for p in (ext.domain, ext.suffix) if p]) if (ext.domain or ext.suffix) else ""
 
     # 0) Major outlets / domain map
@@ -1456,7 +1478,7 @@ def enrich_article(a, language=None, translate=False, target_lang=None):
     # Canonicalize URL & derive domain
     article_url = _canonical_url(a.get("url") or "")
     try:
-        ext = tldextract.extract(article_url)
+        ext = _tld(article_url)
         domain = ".".join([p for p in (ext.domain, ext.suffix) if p]) if (ext.domain or ext.suffix) else ""
     except Exception:
         domain = ""
@@ -2020,3 +2042,11 @@ def diag_translate():
         "libre_ok": bool(libre),
         "sample": libre or remote or local
     }
+
+@app.get("/", include_in_schema=False)
+def root():
+    return {"ok": True, "service": "newsglobe-backend"}
+
+@app.get("/favicon.ico", include_in_schema=False)
+def favicon():
+    return PlainTextResponse("", status_code=204)