File size: 4,310 Bytes
6997817 0e831b4 6997817 0570e44 5e9a370 0e831b4 0570e44 6997817 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 |
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re, tempfile, requests, os
from bs4 import BeautifulSoup
from markdownify import markdownify as md
from readability import Document
import gradio as gr
def clean_text_for_rag(text: str) -> str:
text = re.sub(
r"[’‘“”«»–—\u00A0\u202F…œŒæÆ©®™§°±×÷]",
lambda m: {
"’": "'", "‘": "'", "“": '"', "”": '"',
"«": '"', "»": '"', "–": "-", "—": "-",
"…": "...", "œ": "oe", "Œ": "OE",
"æ": "ae", "Æ": "AE", "©": "(c)", "®": "(R)",
"™": "TM", "§": "§", "°": "°", "±": "+/-",
"×": "x", "÷": "/"
}.get(m.group(0), m.group(0)),
text,
)
text = re.sub(r'[^a-zA-ZÀ-ÿæ-œ0-9\s\.\,\:\;\!\?\-\_\'\"\\\(\)]', '', text)
return re.sub(r'\s+', ' ', text).strip()
def fetch_html(url: str) -> str:
hdr = {"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/124.0 Safari/537.36"}
r = requests.get(url, headers=hdr, timeout=20)
r.raise_for_status()
if r.encoding == "ISO-8859-1":
r.encoding = r.apparent_encoding
return r.text
def extract_main(html: str) -> str:
"""
Extraction du contenu principal d'une page web en éliminant les éléments parasites.
- Utilise readability.Document pour identifier la zone de l'article.
- Supprime les éléments de navigation, commentaires, widgets, publicités, etc.
- Ne garde que les balises sémantiquement pertinentes (article, p, h1-6, li, etc.).
"""
doc = Document(html)
article_html = doc.content()
soup = BeautifulSoup(article_html, "html.parser")
# 🔹 Supprimer les éléments non pertinents (pubs, menus, scripts, etc.)
for tag in soup(["script", "style", "noscript", "footer", "header", "nav", "form", "iframe", "aside", "button"]):
tag.decompose()
# 🔹 Supprimer les div ou sections trop petites ou sans texte utile
for div in soup.find_all(["div", "section"]):
text = div.get_text(strip=True)
if len(text) < 80: # seuil ajustable
div.decompose()
# 🔹 Conserver uniquement les éléments textuels pertinents
keep_tags = ["article", "p", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote", "ul", "ol", "li", "pre", "code"]
clean_parts = []
for tag in soup.find_all(keep_tags):
text = tag.get_text(" ", strip=True)
if text and not re.match(r"^(Partager|Suivre|Commentaires|Lire aussi|Publicité)", text, re.I):
clean_parts.append(text)
clean_text = "\n\n".join(clean_parts).strip()
return clean_text
def to_markdown(text: str) -> str:
md_raw = md(f"<div>{text}</div>", heading_style="ATX")
return clean_text_for_rag(md_raw)
def process(url: str, out_name: str) -> str:
html = fetch_html(url)
main_text = extract_main(html) # **seul le contenu pertinent**
markdown = to_markdown(main_text)
out_name = out_name.strip()
if not out_name.lower().endswith(".md"):
out_name += ".md"
tmp_dir = tempfile.mkdtemp()
out_path = os.path.join(tmp_dir, out_name)
with open(out_path, "w", encoding="utf-8") as f:
f.write(markdown)
return out_path
with gr.Blocks(title="Web → Markdown") as demo:
gr.Markdown("# 🌐 Web Scraping — du web vers Markdown")
with gr.Row():
with gr.Column():
url_in = gr.Textbox(label="URL à scraper")
out_name = gr.Textbox(label="Nom du fichier (.md)", value="output.md")
btn = gr.Button("🛠️ Générer", variant="primary")
with gr.Column():
file_out = gr.File(label="Fichier Markdown")
btn.click(fn=process, inputs=[url_in, out_name], outputs=file_out)
gr.Markdown(
"""
---
**Opérations effectuées :**
- Extraction ciblée du contenu utile (titre, texte, articles, etc.)
- Suppression automatique des menus, pubs et scripts indésirables
- Mise en forme fluide en Markdown lisible et propre
- Téléchargement immédiat du résultat
"""
)
if __name__ == "__main__":
demo.launch(share=True) |