Commit
·
050c5eb
1
Parent(s):
7b2b9d3
Added middleware
Browse files- webrify2.py +65 -24
webrify2.py
CHANGED
|
@@ -4,6 +4,7 @@ from pydantic import BaseModel
|
|
| 4 |
from typing import Optional
|
| 5 |
import base64
|
| 6 |
import json
|
|
|
|
| 7 |
from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeoutError
|
| 8 |
|
| 9 |
app = FastAPI(title="Web Analyzer API")
|
|
@@ -19,47 +20,87 @@ class MetadataResponse(BaseModel):
|
|
| 19 |
twitter: dict
|
| 20 |
canonical: Optional[str]
|
| 21 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
|
| 23 |
-
#
|
| 24 |
-
# pw = await async_playwright().start()
|
| 25 |
-
# browser = await pw.chromium.launch(headless=True)
|
| 26 |
-
# page = await browser.new_page()
|
| 27 |
-
# try:
|
| 28 |
-
# await page.goto(url, timeout=30000)
|
| 29 |
-
# except PlaywrightTimeoutError:
|
| 30 |
-
# raise HTTPException(status_code=504, detail="Page load timed out")
|
| 31 |
-
# return page, browser, pw
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
async def get_page(url):
|
|
|
|
|
|
|
| 36 |
pw = await async_playwright().start()
|
| 37 |
browser = await pw.chromium.launch(headless=True)
|
| 38 |
context = await browser.new_context()
|
| 39 |
|
| 40 |
-
# Stealth:
|
| 41 |
await context.add_init_script(
|
| 42 |
"Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
|
| 43 |
)
|
| 44 |
|
| 45 |
page = await context.new_page()
|
| 46 |
-
page.set_default_timeout(
|
| 47 |
|
| 48 |
try:
|
| 49 |
-
# Try networkidle first (wait for full load)
|
| 50 |
-
await page.goto(url, timeout=90000, wait_until="networkidle")
|
| 51 |
-
await page.wait_for_selector("body", timeout=10000) # Ensure DOM is visible
|
| 52 |
-
except PlaywrightTimeoutError:
|
| 53 |
try:
|
| 54 |
-
|
| 55 |
-
await page.goto(url,
|
| 56 |
-
except
|
| 57 |
-
|
| 58 |
-
await
|
| 59 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
return page, browser, pw
|
| 62 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
@app.middleware("http")
|
| 64 |
async def remove_leaky_headers(request: Request, call_next):
|
| 65 |
response: Response = await call_next(request)
|
|
|
|
| 4 |
from typing import Optional
|
| 5 |
import base64
|
| 6 |
import json
|
| 7 |
+
import asyncio
|
| 8 |
from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeoutError
|
| 9 |
|
| 10 |
app = FastAPI(title="Web Analyzer API")
|
|
|
|
| 20 |
twitter: dict
|
| 21 |
canonical: Optional[str]
|
| 22 |
|
| 23 |
+
# Optional timeout wrapper to enforce global timeout
|
| 24 |
+
async def timeout_wrapper(coro, timeout=20):
|
| 25 |
+
try:
|
| 26 |
+
return await asyncio.wait_for(coro, timeout)
|
| 27 |
+
except asyncio.TimeoutError:
|
| 28 |
+
raise HTTPException(status_code=504, detail="Operation timed out")
|
| 29 |
|
| 30 |
+
# More robust get_page() with fallbacks, stealth, and logging
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
async def get_page(url):
|
| 32 |
+
print(f"[INFO] Visiting URL: {url}")
|
| 33 |
+
|
| 34 |
pw = await async_playwright().start()
|
| 35 |
browser = await pw.chromium.launch(headless=True)
|
| 36 |
context = await browser.new_context()
|
| 37 |
|
| 38 |
+
# Stealth mode: prevent simple headless detection
|
| 39 |
await context.add_init_script(
|
| 40 |
"Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
|
| 41 |
)
|
| 42 |
|
| 43 |
page = await context.new_page()
|
| 44 |
+
page.set_default_timeout(20000) # 20s max for waits on elements
|
| 45 |
|
| 46 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
try:
|
| 48 |
+
print("[INFO] Trying to load with 'domcontentloaded'")
|
| 49 |
+
await page.goto(url, wait_until="domcontentloaded", timeout=20000)
|
| 50 |
+
except PlaywrightTimeoutError:
|
| 51 |
+
print("[WARN] domcontentloaded failed, trying 'load'")
|
| 52 |
+
await page.goto(url, wait_until="load", timeout=20000)
|
| 53 |
+
|
| 54 |
+
try:
|
| 55 |
+
await page.wait_for_selector("body", timeout=5000)
|
| 56 |
+
except Exception:
|
| 57 |
+
print("[WARN] <body> not found quickly. May still continue.")
|
| 58 |
|
| 59 |
+
except Exception as e:
|
| 60 |
+
print(f"[ERROR] Page load failed for {url}: {e}")
|
| 61 |
+
await browser.close()
|
| 62 |
+
await pw.stop()
|
| 63 |
+
raise HTTPException(status_code=504, detail=f"Page load failed: {str(e)}")
|
| 64 |
+
|
| 65 |
+
print("[INFO] Page loaded successfully.")
|
| 66 |
return page, browser, pw
|
| 67 |
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
# async def get_page(url):
|
| 75 |
+
# pw = await async_playwright().start()
|
| 76 |
+
# browser = await pw.chromium.launch(headless=True)
|
| 77 |
+
# context = await browser.new_context()
|
| 78 |
+
|
| 79 |
+
# # Stealth: hide headless detection
|
| 80 |
+
# await context.add_init_script(
|
| 81 |
+
# "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
|
| 82 |
+
# )
|
| 83 |
+
|
| 84 |
+
# page = await context.new_page()
|
| 85 |
+
# page.set_default_timeout(90000) # Apply to all waits
|
| 86 |
+
|
| 87 |
+
# try:
|
| 88 |
+
# # Try networkidle first (wait for full load)
|
| 89 |
+
# await page.goto(url, timeout=90000, wait_until="networkidle")
|
| 90 |
+
# await page.wait_for_selector("body", timeout=10000) # Ensure DOM is visible
|
| 91 |
+
# except PlaywrightTimeoutError:
|
| 92 |
+
# try:
|
| 93 |
+
# # Fallback to lighter load event
|
| 94 |
+
# await page.goto(url, timeout=90000, wait_until="load")
|
| 95 |
+
# except Exception as e:
|
| 96 |
+
# await browser.close()
|
| 97 |
+
# await pw.stop()
|
| 98 |
+
# raise HTTPException(status_code=504, detail=f"Page load failed: {str(e)}")
|
| 99 |
+
|
| 100 |
+
# return page, browser, pw
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
|
| 104 |
@app.middleware("http")
|
| 105 |
async def remove_leaky_headers(request: Request, call_next):
|
| 106 |
response: Response = await call_next(request)
|