Add system

system/augmented_searching.py

@@ -6,10 +6,11 @@ import pandas as pd
 from datetime import datetime
 from pathlib import Path
 import spacy
+from date_verifier import is_after_start
 
 def google_search(query, api_key, search_engine_id, start_date, end_date):
     # print(f"[SYSTEM] Calling Google Search API for: {query}")
-    sort = f"date:r:{start_date}:{end_date}"
+    sort = f"date:r:{start_date}:{end_date}" #20241230:20250130
     url = "https://www.googleapis.com/customsearch/v1"
     params = {
         "q": query,
@@ -80,7 +81,7 @@ def run_augmented_searching(qa_file, pipeline_base_dir, suggestion_meta, pledge_
 
     results = google_search(claim_text, api_key, search_engine_id, start_date, end_date)
     for result in results:
-        if result["link"] not in urls and "fullfact.org/government-tracker" not in result["link"]:
+        if result["link"] not in urls and "fullfact.org/government-tracker" not in result["link"] and is_after_start(result["link"], start_date):
             string_values.append("claim")
             urls.append(result["link"])
             queries.append(f"{pledge_author}: {claim_text}")
@@ -88,7 +89,7 @@ def run_augmented_searching(qa_file, pipeline_base_dir, suggestion_meta, pledge_
     for question in questions:
         results = google_search(f"{question}", api_key, search_engine_id, start_date, end_date)
         for result in results:
-            if result["link"] not in urls and "fullfact.org/government-tracker" not in result["link"]:
+            if result["link"] not in urls and "fullfact.org/government-tracker" not in result["link"] and is_after_start(result["link"], start_date):
                 string_values.append("question")
                 urls.append(result["link"])
                 queries.append(f"{question}")

system/date_verifier.py

@@ -0,0 +1,58 @@
+from datetime import datetime
+import re, trafilatura
+from trafilatura.settings import DEFAULT_CONFIG
+
+DEFAULT_CONFIG.MAX_FILE_SIZE = 50000  
+
+_URL_DATE_PATS = [
+    re.compile(r"(?P<y>\d{4})-(?P<m>\d{2})-(?P<d>\d{2})"),  # 2025-07-03
+    re.compile(r"(?P<y>\d{4})/(?P<m>\d{2})/(?P<d>\d{2})"),  # 2025/07/03
+    re.compile(r"(?P<y>\d{4})(?P<m>\d{2})(?P<d>\d{2})"),    # 20250703
+]
+
+def _meta_date(url: str):
+
+    page = trafilatura.fetch_url(url, config=DEFAULT_CONFIG)
+    if not page:
+        return None
+    meta = trafilatura.extract_metadata(page)
+    if not meta or not meta.date:
+        return None
+    try:
+        return datetime.fromisoformat(meta.date)
+    except ValueError:
+
+        try:
+            return datetime.fromisoformat(meta.date.split("T")[0])
+        except Exception:
+            return None
+
+def _regex_date(url: str):
+
+    for pat in _URL_DATE_PATS:
+        m = pat.search(url)
+        if m:
+            try:
+                return datetime(
+                    int(m.group("y")), int(m.group("m")), int(m.group("d"))
+                )
+            except ValueError:
+                pass
+    return None
+
+
+def is_after_start(url: str, start_ymd: str) -> bool:
+    """
+    - start_ymd: 'YYYYMMDD'
+    """
+    t0 = datetime.strptime(start_ymd, "%Y%m%d")
+
+    pub_dt = _meta_date(url)
+
+    if pub_dt is None:
+        pub_dt = _regex_date(url)
+
+    if pub_dt is None:
+        return True
+
+    return pub_dt >= t0