Update app.py

app.py

@@ -7,6 +7,9 @@ import phonenumbers
 import pandas as pd
 import urllib.parse
 from bs4 import BeautifulSoup
+import json
+from pathlib import Path
+
 
 import torch
 from transformers import (
@@ -105,15 +108,96 @@ def dedup_by_url(items):
 # ============================
 # SEARCH & SCRAPING
 # ============================
+CACHE_PATH = Path("tmp/google_cse_cache.json")
+CACHE_TTL = 60 * 60 * 24  # 24 hours cache; adjust as needed
+MAX_GOOGLE_RETRIES = 5
+MIN_SECONDS_BETWEEN_CALLS = 1.0  # throttle: 1 sec between Google calls to avoid bursts
+
+# load cache (simple file-based)
+def _load_cache():
+    try:
+        if CACHE_PATH.exists():
+            return json.loads(CACHE_PATH.read_text(encoding="utf-8"))
+    except Exception:
+        pass
+    return {}
+
+def _save_cache(cache):
+    try:
+        CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
+        CACHE_PATH.write_text(json.dumps(cache), encoding="utf-8")
+    except Exception:
+        pass
+
+# track last call time so we can throttle
+_last_google_call = {"t": 0.0}
+
 def google_search(query, num_results=5):
+    """
+    Robust Google CSE caller with caching, exponential backoff on 429, and simple throttling.
+    Returns list of {"title","link","snippet"}.
+    """
+    global _last_google_call
     if not API_KEY or not CX or "YOUR_GOOGLE_API_KEY" in API_KEY or "YOUR_CSE_ID" in CX:
         raise RuntimeError("Google API key and CSE ID must be set as environment variables.")
+
+    cache = _load_cache()
+    cache_key = f"gse::{query}::n{num_results}"
+    now = time.time()
+
+    # Check cache and TTL
+    if cache_key in cache:
+        entry = cache[cache_key]
+        if now - entry.get("ts", 0) < CACHE_TTL:
+            # cached
+            return entry.get("items", [])
+
+    # throttle to avoid bursts
+    elapsed = now - _last_google_call["t"]
+    if elapsed < MIN_SECONDS_BETWEEN_CALLS:
+        time.sleep(MIN_SECONDS_BETWEEN_CALLS - elapsed)
+
     url = "https://www.googleapis.com/customsearch/v1"
-    params = {"q":query, "key":API_KEY, "cx":CX, "num":num_results}
-    r = requests.get(url, params=params, timeout=20)
-    r.raise_for_status()
-    items = r.json().get("items", []) or []
-    return [{"title":i.get("title",""), "link":i.get("link",""), "snippet":i.get("snippet","")} for i in items]
+    params = {"q": query, "key": API_KEY, "cx": CX, "num": num_results}
+
+    backoff = 1.0
+    for attempt in range(1, MAX_GOOGLE_RETRIES + 1):
+        try:
+            r = requests.get(url, params=params, timeout=15)
+            _last_google_call["t"] = time.time()
+            if r.status_code == 200:
+                items = r.json().get("items", []) or []
+                parsed = [{"title": i.get("title",""), "link": i.get("link",""), "snippet": i.get("snippet","")} for i in items]
+                # write to cache
+                cache[cache_key] = {"ts": time.time(), "items": parsed}
+                _save_cache(cache)
+                return parsed
+            elif r.status_code == 429:
+                # handle Retry-After if present
+                ra = r.headers.get("Retry-After")
+                wait = float(ra) if ra and ra.isdigit() else backoff
+                print(f"[google_search] 429 -> sleeping {wait}s (attempt {attempt})")
+                time.sleep(wait)
+                backoff *= 2
+                continue
+            else:
+                r.raise_for_status()
+        except requests.HTTPError as e:
+            print(f"[google_search HTTPError] {e} (attempt {attempt})")
+            if attempt == MAX_GOOGLE_RETRIES:
+                raise
+            time.sleep(backoff)
+            backoff *= 2
+        except Exception as e:
+            print(f"[google_search error] {e} (attempt {attempt})")
+            if attempt == MAX_GOOGLE_RETRIES:
+                raise
+            time.sleep(backoff)
+            backoff *= 2
+
+    # If we exit loop without return, return empty list
+    return []
+# ----------------- end google_search replacement -----------------
 
 def extract_phones(text, region="GH"):
     phones = []
@@ -345,9 +429,23 @@ def find_professionals_from_story(story, country=DEFAULT_COUNTRY, results_per_qu
     region = get_region_for_country(country)
     queries, profs = build_queries(story, country)
 
-    # Search
-    search_results = []
+        # Build queries earlier as you already do:
+    queries, profs = build_queries(story, country)
+
+    # Deduplicate and limit queries to e.g. top 3
+    unique_queries = []
+    seen_q = set()
     for q in queries:
+        if q not in seen_q:
+            seen_q.add(q)
+            unique_queries.append(q)
+    # limit number of queries to reduce CSE calls
+    MAX_QUERIES_PER_STORY = 3
+    unique_queries = unique_queries[:MAX_QUERIES_PER_STORY]
+
+    # Search (serialized, cached, with error handling)
+    search_results = []
+    for q in unique_queries:
         try:
             items = google_search(q, num_results=results_per_query)
             for it in items:
@@ -356,10 +454,6 @@ def find_professionals_from_story(story, country=DEFAULT_COUNTRY, results_per_qu
         except Exception as e:
             print("[search error]", q, e)
 
-    search_results = dedup_by_url(search_results)
-    if not search_results:
-        return {"summary":"No results found. Try a different country or wording.",
-                "professionals":[], "queries_used":queries}
 
     # NER on titles/snippets
     all_people, all_orgs, all_locs = [], [], []