✨ Add HTML purification functionality: integrate PurifyHtml in app and create Gradio interface for user input; update requirements to include transformers.

Purify.py

@@ -1,10 +1,9 @@
+from transformers import AutoTokenizer, AutoModelForCausalLM
 from bs4 import BeautifulSoup, Tag
 import datetime
 import requests
 import re
 
-Url = 'https://huggingface.co'
-
 NoisePatterns = {
 	'(No)Script':          r'<[ ]*(script|noscript)[^>]*?>.*?<\/[ ]*\1[ ]*>',
 	'Style':               r'<[ ]*(style)[^>]*?>.*?<\/[ ]*\1[ ]*>',
@@ -32,14 +31,11 @@ def RemoveNoise(RawHtml: str) -> str:
 		str: Cleaned HTML content without noise.
 	'''
 	CleanedHtml = RawHtml
-	OriginalCharCount = len(RawHtml)
 	for PatternName, Pattern in NoisePatterns.items():
 		if PatternName in ['EmptyLines', 'EmptyTags']: # These patterns are line-based
 			CleanedHtml = re.sub(Pattern, '', CleanedHtml, flags=re.MULTILINE)
 		else:
 			CleanedHtml = re.sub(Pattern, '', CleanedHtml, flags=re.DOTALL | re.IGNORECASE | re.MULTILINE)
-		print(f'• Removed {PatternName} noise. Removed {OriginalCharCount - len(CleanedHtml)} characters.')
-		OriginalCharCount = len(CleanedHtml)
 	return CleanedHtml
 
 def FetchHtmlContent(Url: str) -> str | int:
@@ -58,43 +54,46 @@ def FetchHtmlContent(Url: str) -> str | int:
 	else:
 		return Response.status_code
 
-Start = datetime.datetime.now()
-RawHtml = FetchHtmlContent(Url)
-if isinstance(RawHtml, str):
-	RawCharCount = len(RawHtml)
-
-	print('Prettifying HTML content...')
-
-	Soup = BeautifulSoup(RawHtml, 'html.parser')
-	PrettifiedHtml = str(Soup.prettify())
-
-	Title = Soup.title.string if Soup.title else 'No title found'
-	MetaDesc = Soup.find('meta', attrs={'name': 'description'})
-	Description = MetaDesc.get('content', 'No description found') if isinstance(MetaDesc, Tag) else 'No description found'
-
-	print('Purifying HTML content...')
-
-	CleanedHtml = RemoveNoise(PrettifiedHtml)
+def PurifyHtml(Url: str) -> str: # type: ignore
+	Start = datetime.datetime.now()
+	RawHtml = FetchHtmlContent(Url)
+	if isinstance(RawHtml, str):
+		RawCharCount = len(RawHtml)
+
+		Soup = BeautifulSoup(RawHtml, 'html.parser')
+		PrettifiedHtml = str(Soup.prettify())
+
+		Title = Soup.title.string if Soup.title else 'No title found'
+		MetaDesc = Soup.find('meta', attrs={'name': 'description'})
+		Description = MetaDesc.get('content', 'No description found') if isinstance(MetaDesc, Tag) else 'No description found'
+
+		CleanedHtml = RemoveNoise(PrettifiedHtml)
+
+		CleanedCharCount = len(CleanedHtml)
+		Ratio = CleanedCharCount / RawCharCount if RawCharCount > 0 else 0
+
+		Summary = [
+			'<!-- --- Purification Summary ---',
+			f'URL: {Url}',
+			f'Title: {Title}',
+			f'Description: {Description}',
+			f'Time of Fetch: {datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")} (Took {datetime.datetime.now() - Start})',
+			f'Noise Removal Ratio: {Ratio:.2%} (lower is better)',
+			f'Characters: {RawCharCount} -> {CleanedCharCount} ({RawCharCount - CleanedCharCount} characters removed)',
+			'----------------------------- -->'
+		]
+		for Line in Summary:
+			print(Line)
+		
+		Tokenizer = AutoTokenizer.from_pretrained('jinaai/ReaderLM-v2')
+		Model = AutoModelForCausalLM.from_pretrained('jinaai/ReaderLM-v2')
 
-	CleanedCharCount = len(CleanedHtml)
-	Ratio = CleanedCharCount / RawCharCount if RawCharCount > 0 else 0
+		Message = [
+			{'role': 'user', 'content': f'Please summarize the following HTML content in clean markdown:\n\n{CleanedHtml}'},
+		]
+		SummaryOutput = Model.chat(Message, tokenizer=Tokenizer, max_new_tokens=500, do_sample=False)
 
-	Summary = [
-		'<!-- --- Purification Summary ---',
-		f'URL: {Url}',
-		f'Title: {Title}',
-		f'Description: {Description}',
-		f'Time of Fetch: {datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")} (Took {datetime.datetime.now() - Start})',
-		f'Noise Removal Ratio: {Ratio:.2%} (lower is better)',
-		f'Characters: {RawCharCount} -> {CleanedCharCount} ({RawCharCount - CleanedCharCount} characters removed)',
-		'----------------------------- -->'
-	]
-	for Line in Summary:
-		print(Line)
+		return str(SummaryOutput)
 
-	with open('CleanedHtml.html', 'w', encoding='utf-8') as File:
-		for Line in Summary:
-			File.write(Line + '\n')
-		File.write(CleanedHtml)
-else:
-	print(f'Failed to fetch HTML content. Status code: {RawHtml}')
+	else:
+		print(f'Failed to fetch HTML content. Status code: {RawHtml}')

app.py

@@ -1,5 +1,6 @@
 from pymongo import MongoClient
 from dotenv import load_dotenv
+from Purify import PurifyHtml
 from typing import Literal
 from bson import ObjectId
 from io import StringIO
@@ -143,6 +144,16 @@ def Ping(Host: str, Count: int = 8) -> str:
 	else:
 		return f'Ping to {Host} failed: No successful responses'
 
+def Purify(Url: str) -> str:
+	'''Purify HTML content from a URL.
+	Args:
+		Url (str): The URL to fetch and purify HTML content from.
+	Returns:
+		str: The purified HTML content or an error message.
+	'''
+
+	return PurifyHtml(Url)
+
 # ╭───────────────────────────────────╮
 # │    Fun and Entertainment Tools    │
 # ╰───────────────────────────────────╯
@@ -163,15 +174,22 @@ def Fact() -> str:
 	'''
 	return requests.get('https://uselessfacts.jsph.pl/random.json?language=en').json()['text']
 
-def Plot() -> str:
+def Plot(GiveExamplePrompt: bool = True) -> list[str]:
 	'''Generate a random plot for a movie or story.
+	Args:
+		GiveExamplePrompt (bool): If True, returns a random plot prompt from a predefined dataset.
 	Returns:
 		str: A random plot description.
 	'''
 	with open(r'Data/human-writing-dpo.json', 'r', encoding='utf-8') as PlotFile:
 		Data = json.load(PlotFile)
 		Plot = random.choice(Data)
-		return Plot['prompt'] if isinstance(Plot, dict) and 'prompt' in Plot else 'Error: Invalid plot data format'
+		Prompt = Plot['prompt']
+		Chosen = Plot['chosen']
+		if GiveExamplePrompt:
+			return [Prompt, Chosen]
+		else:
+			return [Prompt, '']
 
 # ╭─────────────────────────────╮
 # │    Text Processing Tools    │
@@ -398,6 +416,13 @@ with gradio.Blocks(
 				PingOutput = gradio.Text(label='Ping Result 📡', interactive=False)
 				PingBtn = gradio.Button('Ping Host 📡', variant='primary')
 				PingBtn.click(Ping, inputs=[PingInput, PingCount], outputs=PingOutput)
+		
+		with gradio.TabItem('Web Scraping & Purification 🌐'):
+			with gradio.Group():
+				PurifyInput = gradio.Textbox(label='URL to Purify 🌐', placeholder='Enter URL to fetch and purify HTML', lines=1, max_lines=1)
+				PurifyOutput = gradio.Text(label='Purified HTML Content 📝', interactive=False)
+				PurifyBtn = gradio.Button('Purify HTML 🧹', variant='primary')
+				PurifyBtn.click(Purify, inputs=PurifyInput, outputs=PurifyOutput)
 
 	with gradio.TabItem('Fun & Entertainment 🎭'):
 		with gradio.TabItem('Random Joke 😂'):
@@ -416,8 +441,10 @@ with gradio.Blocks(
 		with gradio.TabItem('Random Plot 🎬'):
 			with gradio.Group():
 				PlotOutput = gradio.Text(label='Random Plot 🎬', interactive=False)
+				PlotExample = gradio.Checkbox(label='Give Example Plot Prompt 📜', value=True, interactive=True)
+				PlotExampleOutput = gradio.Text(label='Example Plot Prompt 📜', interactive=False)
 				PlotBtn = gradio.Button('Get Plot 🎥', variant='primary')
-				PlotBtn.click(Plot, outputs=PlotOutput)
+				PlotBtn.click(Plot, inputs=[PlotExample], outputs=[PlotOutput, PlotExampleOutput])
 
 	with gradio.TabItem('Text Processing 📝'):
 		with gradio.TabItem('Text Reversal 🔄'):

requirements.txt

@@ -1 +1,2 @@
+transformers
 pymongo