Update webscout.py

webscout.py

@@ -5,11 +5,14 @@ from datetime import datetime, timezone
 from decimal import Decimal
 from functools import cached_property
 from itertools import cycle, islice
+from random import choice
 from threading import Event
 from types import TracebackType
 from typing import Dict, List, Optional, Tuple, Type, Union, cast
 
-import pyreqwest_impersonate as pri  # type: ignore
+import pyreqwest_impersonate as pri
+
+
 
 try:
     from lxml.etree import _Element
@@ -26,29 +29,33 @@ from html import unescape
 from math import atan2, cos, radians, sin, sqrt
 from typing import Any, Dict, List, Union
 from urllib.parse import unquote
-import orjson
-import requests
-import base64
-from typing import List, Dict, Union
-import json
-import requests
-import base64
-from typing import List, Dict, Union
 
+from .exceptions import WebscoutE
+
+try:
+    HAS_ORJSON = True
+    import orjson
+except ImportError:
+    HAS_ORJSON = False
+    import json
 
 REGEX_STRIP_TAGS = re.compile("<.*?>")
 
 
 def json_dumps(obj: Any) -> str:
     try:
-        return orjson.dumps(obj).decode("utf-8")
+        return (
+            orjson.dumps(obj, option=orjson.OPT_INDENT_2).decode()
+            if HAS_ORJSON
+            else json.dumps(obj, ensure_ascii=False, indent=2)
+        )
     except Exception as ex:
         raise WebscoutE(f"{type(ex).__name__}: {ex}") from ex
 
 
 def json_loads(obj: Union[str, bytes]) -> Any:
     try:
-        return orjson.loads(obj)
+        return orjson.loads(obj) if HAS_ORJSON else json.loads(obj)
     except Exception as ex:
         raise WebscoutE(f"{type(ex).__name__}: {ex}") from ex
 
@@ -101,8 +108,6 @@ def _calculate_distance(lat1: Decimal, lon1: Decimal, lat2: Decimal, lon2: Decim
     c = 2 * atan2(sqrt(a), sqrt(1 - a))
     return R * c
 
-logger = logging.getLogger("webscout.WEBS")
-
 class WebscoutE(Exception):
     """Base exception class for search."""
 
@@ -121,14 +126,26 @@ class AllProvidersFailure(Exception):
     """None of the providers generated response successfully"""
 
     pass
+logger = logging.getLogger("webscout.WEBS")
+
+
 class WEBS:
     """webscout class to get search results from duckduckgo.com."""
 
     _executor: ThreadPoolExecutor = ThreadPoolExecutor()
+    _impersonates = (
+        "chrome_99", "chrome_100", "chrome_101", "chrome_104", "chrome_105", "chrome_106", "chrome_108", 
+        "chrome_107", "chrome_109", "chrome_114", "chrome_116", "chrome_117", "chrome_118", "chrome_119", 
+        "chrome_120", #"chrome_123", "chrome_124", "chrome_126",
+        "safari_ios_16.5", "safari_ios_17.2", "safari_ios_17.4.1", "safari_15.3", "safari_15.5", 
+        "safari_15.6.1", "safari_16", "safari_16.5", "safari_17.2.1", "safari_17.4.1", "safari_17.5",
+        #"okhttp_3.9", "okhttp_3.11", "okhttp_3.13", "okhttp_3.14", "okhttp_4.9", "okhttp_4.10", "okhttp_5",
+        "edge_99", "edge_101", "edge_122",
+    )  # fmt: skip
 
     def __init__(
         self,
-        headers: Optional[Dict[str, str]] = {'0': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0', '1': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', '2': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', '3': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', '4': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', '5': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.62', '6': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/111.0', '7': 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Mobile/15E148 Safari/605.1.15', '8': 'Mozilla/5.0 (iPad; CPU OS 16_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Mobile/15E148 Safari/605.1.15', '9': 'Mozilla/5.0 (Android 13; Mobile; rv:109.0) Gecko/109.0 Firefox/109.0', '10': 'Mozilla/5.0 (Linux; Android 13; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Mobile Safari/537.36', '11': 'Mozilla/5.0 (Linux; U; Android 11; en-us; SM-G991U) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/89.0.4387.119 Mobile Safari/537.36', '12': 'Mozilla/5.0 (Linux; Android 12; SM-G998U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Mobile Safari/537.36', '13': 'Mozilla/5.0 (Linux; Android 13; Pixel 7 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Mobile Safari/537.36', '14': 'Mozilla/5.0 (Linux; Android 12; LM-G900V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Mobile Safari/537.36', '15': 'Mozilla/5.0 (Linux; Android 11; SM-G975U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Mobile Safari/537.36', '16': 'Mozilla/5.0 (Linux; Android 11; SM-N975U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Mobile Safari/537.36', '17': 'Mozilla/5.0 (Linux; Android 13; SM-S918U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Mobile Safari/537.36', '18': 'Mozilla/5.0 (Linux; Android 13; SM-F936U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Mobile Safari/537.36'},
+        headers: Optional[Dict[str, str]] = None,
         proxy: Optional[str] = None,
         proxies: Union[Dict[str, str], str, None] = None,  # deprecated
         timeout: Optional[int] = 10,
@@ -152,9 +169,9 @@ class WEBS:
             headers=self.headers,
             proxy=self.proxy,
             timeout=timeout,
-            cookie_store=False,
+            cookie_store=True,
             referer=True,
-            impersonate="chrome_124",
+            impersonate=choice(self._impersonates),
             follow_redirects=False,
             verify=False,
         )
@@ -208,13 +225,14 @@ class WEBS:
         resp_content = self._get_url("POST", "https://duckduckgo.com", data={"q": keywords})
         return _extract_vqd(resp_content, keywords)
 
-    def chat(self, keywords: str, model: str = "gpt-3.5") -> str:
+    def chat(self, keywords: str, model: str = "gpt-3.5", timeout: int = 20) -> str:
         """Initiates a chat session with DuckDuckGo AI.
 
         Args:
             keywords (str): The initial message or question to send to the AI.
             model (str): The model to use: "gpt-3.5", "claude-3-haiku", "llama-3-70b", "mixtral-8x7b".
                 Defaults to "gpt-3.5".
+            timeout (int): Timeout value for the HTTP client. Defaults to 20.
 
         Returns:
             str: The response from the AI.
@@ -237,18 +255,16 @@ class WEBS:
             "messages": self._chat_messages,
         }
         resp = self.client.post(
-            "https://duckduckgo.com/duckchat/v1/chat", headers={"x-vqd-4": self._chat_vqd}, json=json_data
+            "https://duckduckgo.com/duckchat/v1/chat",
+            headers={"x-vqd-4": self._chat_vqd},
+            json=json_data,
+            timeout=timeout,
         )
         self._chat_vqd = resp.headers.get("x-vqd-4", "")
 
-        messages = []
-        for line in resp.text.replace("data: ", "").replace("[DONE]", "").split("\n\n"):
-            x = line.strip()
-            if x:
-                j = json_loads(x)
-                message = j.get("message", "")
-                messages.append(message)
-        result = "".join(messages)
+        data = ",".join(x for line in resp.text.rstrip("[DONE]\n").split("data:") if (x := line.strip()))
+        result = "".join(x.get("message", "") for x in json_loads("[" + data + "]"))
+
         self._chat_messages.append({"role": "assistant", "content": result})
         return result
 
@@ -435,7 +451,7 @@ class WEBS:
             for e in elements:
                 if isinstance(e, _Element):
                     hrefxpath = e.xpath("./a/@href")
-                    href = str(hrefxpath[0]) if isinstance(hrefxpath, List) else None
+                    href = str(hrefxpath[0]) if hrefxpath and isinstance(hrefxpath, List) else None
                     if (
                         href
                         and href not in cache
@@ -445,9 +461,9 @@ class WEBS:
                     ):
                         cache.add(href)
                         titlexpath = e.xpath("./h2/a/text()")
-                        title = str(titlexpath[0]) if isinstance(titlexpath, List) else ""
+                        title = str(titlexpath[0]) if titlexpath and isinstance(titlexpath, List) else ""
                         bodyxpath = e.xpath("./a//text()")
-                        body = "".join(str(x) for x in bodyxpath) if isinstance(bodyxpath, List) else ""
+                        body = "".join(str(x) for x in bodyxpath) if bodyxpath and isinstance(bodyxpath, List) else ""
                         result = {
                             "title": _normalize(title),
                             "href": _normalize_url(href),
@@ -537,10 +553,14 @@ class WEBS:
                         else:
                             cache.add(href)
                             titlexpath = e.xpath(".//a//text()")
-                            title = str(titlexpath[0]) if isinstance(titlexpath, List) else ""
+                            title = str(titlexpath[0]) if titlexpath and isinstance(titlexpath, List) else ""
                     elif i == 2:
                         bodyxpath = e.xpath(".//td[@class='result-snippet']//text()")
-                        body = "".join(str(x) for x in bodyxpath) if isinstance(bodyxpath, List) else ""
+                        body = (
+                            "".join(str(x) for x in bodyxpath).strip()
+                            if bodyxpath and isinstance(bodyxpath, List)
+                            else ""
+                        )
                         if href:
                             result = {
                                 "title": _normalize(title),