Upload 5 files

Dockerfile

@@ -0,0 +1,47 @@
+# Use official Python image
+FROM python:3.10-slim
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies for Playwright
+RUN apt-get update && apt-get install -y \
+    wget \
+    curl \
+    gnupg \
+    ca-certificates \
+    fonts-liberation \
+    libasound2 \
+    libatk-bridge2.0-0 \
+    libatk1.0-0 \
+    libcups2 \
+    libdbus-1-3 \
+    libgdk-pixbuf2.0-0 \
+    libnspr4 \
+    libnss3 \
+    libx11-xcb1 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxrandr2 \
+    xdg-utils \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements and install Python dependencies
+COPY requirements.txt .
+RUN pip install --upgrade pip && pip install -r requirements.txt
+
+# Install Playwright browsers
+RUN playwright install --with-deps
+
+# Copy application files
+COPY . .
+
+# Expose port (Hugging Face uses port 7860 or 8000 typically)
+EXPOSE 7860
+
+# Command to run the app
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

main.py

@@ -0,0 +1,126 @@
+from fastapi import FastAPI, HTTPException, Request
+from pydantic import HttpUrl
+from playwright.async_api import async_playwright
+from urllib.parse import urljoin, urlparse
+import logging
+from fastapi.responses import JSONResponse
+from fastapi.templating import Jinja2Templates
+from fastapi.staticfiles import StaticFiles
+from typing import List, Dict
+import asyncio
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+app = FastAPI(title="Website Scraper API (Enhanced for Images)")
+
+# Mount static files
+app.mount("/static", StaticFiles(directory="static"), name="static")
+
+# Set up Jinja2 templates
+templates = Jinja2Templates(directory="templates")
+
+# Maximum number of pages to scrape
+MAX_PAGES = 20
+
+async def scrape_page(url: str, visited: set, base_domain: str) -> tuple[Dict, set]:
+    """Scrape a single page for text, images, and links using Playwright."""
+    try:
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            context = await browser.new_context(
+                user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
+                viewport={"width": 1280, "height": 720}
+            )
+            page = await context.new_page()
+            
+            # Navigate and wait for content
+            await page.goto(url, wait_until="networkidle", timeout=30000)
+            
+            # Scroll to trigger lazy-loaded images
+            await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+            await page.wait_for_timeout(2000)  # Wait for lazy-loaded content
+            
+            # Extract text content
+            text_content = await page.evaluate(
+                """() => document.body.innerText"""
+            )
+            text_content = ' '.join(text_content.split()) if text_content else ""
+
+            # Extract images from src, data-src, and srcset
+            images = await page.evaluate(
+                """() => {
+                    const imgElements = document.querySelectorAll('img');
+                    const imgUrls = new Set();
+                    imgElements.forEach(img => {
+                        if (img.src) imgUrls.add(img.src);
+                        if (img.dataset.src) imgUrls.add(img.dataset.src);
+                        if (img.srcset) {
+                            img.srcset.split(',').forEach(src => {
+                                const url = src.trim().split(' ')[0];
+                                if (url) imgUrls.add(url);
+                            });
+                        }
+                    });
+                    return Array.from(imgUrls);
+                }"""
+            )
+            images = [urljoin(url, img) for img in images if img]
+
+            # Extract links
+            links = await page.evaluate(
+                """() => Array.from(document.querySelectorAll('a')).map(a => a.href)"""
+            )
+            links = set(urljoin(url, link) for link in links if urlparse(urljoin(url, link)).netloc == base_domain and urljoin(url, link) not in visited)
+
+            await browser.close()
+
+        page_data = {
+            "url": url,
+            "text": text_content,
+            "images": images
+        }
+        return page_data, links
+
+    except Exception as e:
+        logging.error(f"Error scraping {url}: {e}")
+        return {}, set()
+
+@app.get("/scrape")
+async def crawl_website(url: HttpUrl):
+    """Crawl the website starting from the given URL and return scraped data for up to 10 pages as JSON."""
+    try:
+        visited = set()
+        to_visit = {str(url)}
+        base_domain = urlparse(str(url)).netloc
+        results = []
+
+        while to_visit and len(visited) < MAX_PAGES:
+            current_url = to_visit.pop()
+            if current_url in visited:
+                continue
+
+            logging.info(f"Scraping: {current_url}")
+            visited.add(current_url)
+
+            page_data, new_links = await scrape_page(current_url, visited, base_domain)
+            if page_data:
+                results.append(page_data)
+            to_visit.update(new_links)
+            
+            # Small delay to avoid overwhelming the server
+            await asyncio.sleep(0.5)
+
+        return JSONResponse(content={"pages": results})
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Scraping failed: {str(e)}")
+
+@app.get("/")
+async def serve_home(request: Request):
+    """Serve the frontend HTML page."""
+    return templates.TemplateResponse("index.html", {"request": request})
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8001)

static/styles.css

@@ -0,0 +1,14 @@
+textarea {
+    width: 100%;
+    min-height: 400px;
+    max-height: 600px;
+    overflow-y: auto;
+    background-color: #f8fafc;
+    padding: 1rem;
+    border-radius: 0.5rem;
+    border: 1px solid #e2e8f0;
+    resize: vertical;
+    font-family: monospace;
+    font-size: 14px;
+    box-sizing: border-box;
+}

templates/index.html

@@ -0,0 +1,70 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Web Scraper</title>
+    <script src="https://cdn.tailwindcss.com"></script>
+    <link rel="stylesheet" href="/static/styles.css">
+</head>
+<body class="min-h-screen bg-gray-100 flex flex-col items-center p-4">
+    <div class="w-full bg-white rounded-lg shadow-md p-6">
+        <h1 class="text-2xl font-bold text-gray-800 mb-4">Web Scraper</h1>
+        <form id="scrapeForm" class="flex flex-col space-y-4">
+            <input
+                type="url"
+                id="urlInput"
+                name="url"
+                placeholder="Enter URL to scrape (e.g., https://example.com)"
+                class="w-full p-2 border rounded focus:outline-none focus:ring-2 focus:ring-blue-500"
+                required
+            />
+            <button
+                type="submit"
+                id="scrapeButton"
+                class="w-full bg-blue-500 text-white p-2 rounded hover:bg-blue-600 disabled:bg-blue-300"
+            >
+                Scrape Website
+            </button>
+        </form>
+        <div id="error" class="hidden mt-4 p-2 bg-red-100 text-red-700 rounded"></div>
+        <div id="result" class="hidden mt-4 p-4 bg-gray-50 rounded border">
+            <h2 class="text-lg font-semibold text-gray-700">Scraped Data</h2>
+            <textarea id="jsonOutput" class="mt-2 text-sm text-gray-600" readonly></textarea>
+        </div>
+    </div>
+
+    <script>
+        document.getElementById('scrapeForm').addEventListener('submit', async (e) => {
+            e.preventDefault();
+            const url = document.getElementById('urlInput').value;
+            const errorDiv = document.getElementById('error');
+            const resultDiv = document.getElementById('result');
+            const jsonOutput = document.getElementById('jsonOutput');
+            const button = document.getElementById('scrapeButton');
+
+            // Reset UI
+            errorDiv.classList.add('hidden');
+            resultDiv.classList.add('hidden');
+            button.disabled = true;
+            button.textContent = 'Scraping...';
+
+            try {
+                const response = await fetch(`/scrape?url=${encodeURIComponent(url)}`);
+                if (!response.ok) {
+                    throw new Error(`HTTP error! Status: ${response.status}`);
+                }
+                const data = await response.json();
+                jsonOutput.value = JSON.stringify(data, null, 2);
+                resultDiv.classList.remove('hidden');
+            } catch (err) {
+                errorDiv.textContent = `Error: ${err.message}`;
+                errorDiv.classList.remove('hidden');
+            } finally {
+                button.disabled = false;
+                button.textContent = 'Scrape Website';
+            }
+        });
+    </script>
+</body>
+</html>