Add all files

jsonToText.py

@@ -0,0 +1,12 @@
+import json
+
+def convert_json_to_text(json_path, text_path):
+    # Read JSON data from file
+    with open(json_path, 'r') as json_file:
+        data = json.load(json_file)
+    
+    # Open a text file for writing
+    with open(text_path, 'w') as text_file:
+        for item in data:
+            # Write only the extracted_content to the text file with a newline character
+            text_file.write(f"{item['extracted_content']}\n")

llm.py

@@ -0,0 +1,102 @@
+import os
+import re
+import json
+from groq import Groq
+
+def process_and_save_json(input_file_path, output_file_path, api_key, chunk_size=2048, overlap_size=256, model="llama3-8b-8192", about='Events', details=['name', 'details']):
+    """
+    Processes the input file in overlapping chunks, interacts with the model, and saves the JSON output to a file.
+
+    Parameters:
+        input_file_path (str): Path to the input file containing data.
+        output_file_path (str): Path to the output JSON file.
+        api_key (str): Groq API key for authentication.
+        chunk_size (int): Size of each chunk of data.
+        overlap_size (int): Number of characters to overlap between chunks.
+        model (str): Model identifier to use for processing.
+        about (str): Description of the data for the model.
+        details (list or str): List of column names expected in the output JSON or a specific condition.
+    """
+    # Initialize the Groq client with the provided API key
+    client = Groq(api_key=api_key)
+
+    def read_file_in_chunks(file_path, chunk_size, overlap_size):
+        """Reads the file in chunks of a specified size with overlapping."""
+        with open(file_path, 'r', encoding='utf-8') as f:
+            buffer = f.read()
+            start = 0
+            while start < len(buffer):
+                end = start + chunk_size
+                yield buffer[start:end]
+                start = end - overlap_size  # Move the start point for the next chunk
+
+    def extract_text_between_braces(text):
+        """Extracts and returns all text between curly braces."""
+        matches = re.findall(r'\{.*?\}', text, re.DOTALL)
+        return matches
+
+    def ensure_strings_in_json(data):
+        """Ensure that all values in JSON are strings."""
+        if isinstance(data, dict):
+            return {k: str(v) if not isinstance(v, (dict, list)) else ensure_strings_in_json(v) for k, v in data.items()}
+        elif isinstance(data, list):
+            return [ensure_strings_in_json(item) for item in data]
+        return str(data)
+
+    def process_chunk(client, chunk, model, about, details):
+        """Sends a chunk to the model and returns the completion."""
+        
+        system_message = (
+            f"You are a helpful assistant for cleaning and organizing the data. \n"
+            f"This data is about {about}. \n"
+            f"Output should be well organized in the form of JSON with the following columns: {', '.join(details)}.\n"
+            f"Do not add extra details apart from JSON.\n"
+            f"If there is no such data, return an empty list.\n"
+        )
+        
+        completion = client.chat.completions.create(
+            model=model,
+            messages=[
+                {
+                    "role": "system",
+                    "content": system_message
+                },
+                {
+                    "role": "user",
+                    "content": chunk
+                }
+            ],
+            temperature=1,
+            max_tokens=8192,
+            top_p=1,
+            stream=False,
+            stop=None,
+        )
+        
+        # Accessing the message content using dot notation
+        return completion.choices[0].message.content
+
+    combined_output = []
+
+    # Read and process the file in chunks
+    for chunk in read_file_in_chunks(input_file_path, chunk_size, overlap_size):
+        output = process_chunk(client, chunk, model, about, details)
+        # Extract all text between curly braces from each chunk output
+        brace_texts = extract_text_between_braces(output)
+        for brace_text in brace_texts:
+            try:
+                # Parse JSON and ensure all values are strings
+                json_data = json.loads(brace_text)
+                json_data = ensure_strings_in_json(json_data)
+                combined_output.append(json_data)
+            except json.JSONDecodeError:
+                print("+++++++++++++++++++++++++++++++++++++++++++++++++")
+                print("Invalid JSON format in extracted text:")
+                print(brace_text)
+                print("+++++++++++++++++++++++++++++++++++++++++++++++++")
+
+    # Output the combined result to a JSON file
+    with open(output_file_path, 'w', encoding='utf-8') as f:
+        json.dump(combined_output, f, indent=4)
+
+    print(f"Processing complete. Output saved to '{output_file_path}'.")

main.py

@@ -0,0 +1,124 @@
+import streamlit as st
+import subprocess
+import os
+
+from jsonToText import convert_json_to_text
+from llm import process_and_save_json
+
+# Default XPath if not provided by the user
+DEFAULT_XPATH = '//body'  # You can change this to whatever default XPath you prefer
+
+# Function to update the spider with user inputs and run it
+def run_spider(website_url, xpath):
+    # Extract domain from the website URL
+    domain = website_url.split("//")[-1].split("/")[0]
+    
+    # Update the spider file with user input (start_urls, custom_xpath, and allowed_domains)
+    spider_path = 'webscraper/webscraper/spiders/websiteSpider.py'
+    
+    # Read the spider file
+    with open(spider_path, 'r') as file:
+        spider_code = file.readlines()
+
+    # Modify start_urls, custom_xpath, and allowed_domains
+    for idx, line in enumerate(spider_code):
+        if line.strip().startswith('start_urls ='):
+            spider_code[idx] = f'    start_urls = ["{website_url}"]\n'
+        if line.strip().startswith('custom_xpath ='):
+            spider_code[idx] = f'    custom_xpath = "{xpath}"\n'
+        if line.strip().startswith('allowed_domains ='):
+            spider_code[idx] = f'    allowed_domains = ["{domain}"]\n'
+
+    # Write back the modified spider code
+    with open(spider_path, 'w') as file:
+        file.writelines(spider_code)
+
+    # Run the Scrapy spider using subprocess
+    scrapy_command = f'scrapy crawl websiteSpider'
+    spider_dir = 'webscraper/webscraper'  # Set the directory where the spider is located
+    subprocess.run(scrapy_command, cwd=spider_dir, shell=True)
+
+
+# Streamlit UI
+st.title('Web Scraper Interface')
+
+# User input for website link and XPath
+website_url = st.text_input('Enter the website URL:', '')
+xpath = st.text_input('Enter the XPath:', DEFAULT_XPATH)  # Default XPath if not provided
+
+# Variable to check if spider has been run
+spider_ran = False
+
+# Button to run the spider
+if st.button('Run Spider'):
+    if website_url:
+        st.write(f'Running the spider on {website_url} using XPath: {xpath}')
+        run_spider(website_url, xpath)
+        st.success('Spider finished running!')
+        convert_json_to_text("webscraper/webscraper/scraped.json", "output.txt")
+        spider_ran = True
+    else:
+        st.error('Please provide a website URL.')
+
+# If spider has been run, show download buttons
+if spider_ran or os.path.exists("webscraper/webscraper/scraped.json"):
+    # Add an option to download the output.txt file
+    with open("output.txt", "r") as file:
+        st.download_button(
+            label="Download Output Text",
+            data=file,
+            file_name="output.txt",
+            mime="text/plain"
+        )
+    
+    # Add an option to download the scraped.json file
+    with open("webscraper/webscraper/scraped.json", "r") as json_file:
+        st.download_button(
+            label="Download Scraped JSON",
+            data=json_file,
+            file_name="scraped.json",
+            mime="application/json"
+        )
+
+# Title for organizing section
+st.title("Do you want to organize the scraped data?")
+
+# Use session state to track if the user has clicked "Yes"
+if "organize_requested" not in st.session_state:
+    st.session_state.organize_requested = False
+
+# Button to toggle the organize section
+if st.button("Yes"):
+    st.session_state.organize_requested = True
+
+# If user clicked "Yes", show input for 'about', 'API key', and organize button
+if st.session_state.organize_requested:
+    # Input for 'about' to describe the data for organizing
+    about = st.text_input('Enter one or two words that describe the data like "books" or "events":', '')
+
+    # Input for custom details to organize the data (e.g., name, price, stock)
+    details = st.text_input('Enter the details to extract (comma separated) like name, date', '')
+
+    # Input for the API key
+    api_key = st.text_input('Enter your Groq API key:', type="password")
+
+    # Button to organize and save JSON
+    if st.button("Organize"):
+        if about and details and api_key:
+            # Convert comma-separated details into a list
+            details_list = [detail.strip() for detail in details.split(',')]
+            
+            # Process and save the JSON with the provided details and API key
+            process_and_save_json("output.txt", "organize.json", api_key=api_key, about=about, details=details_list)
+            st.success('Data has been organized and saved to organize.json.')
+            
+            # Add an option to download the organized JSON file
+            with open("organize.json", "r") as organized_json_file:
+                st.download_button(
+                    label="Download Organized JSON",
+                    data=organized_json_file,
+                    file_name="organize.json",
+                    mime="application/json"
+                )
+        else:
+            st.error('Please provide a description, details, and your API key before organizing.')

organize.json

@@ -0,0 +1,1242 @@
+[
+    {
+        "booksName": "A Light in the ...",
+        "Price": "51.77",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Tipping the Velvet",
+        "Price": "53.74",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Soumission",
+        "Price": "50.1",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Sharp Objects",
+        "Price": "47.82",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Sapiens: A Brief History ...",
+        "Price": "54.23",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Requiem Red",
+        "Price": "22.65",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Dirty Little Secrets ...",
+        "Price": "33.34",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Coming Woman: A ...",
+        "Price": "17.93",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Boys in the ...",
+        "Price": "22.6",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Black Maria",
+        "Price": "52.15",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Starving Hearts (Triangular Trade ...",
+        "Price": "13.99",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Shakespeare's Sonnets",
+        "Price": "20.66",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Set Me Free",
+        "Price": "17.46",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Scott Pilgrim's Precious Little ...",
+        "Price": "52.29",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Rip it Up and ...",
+        "Price": "35.02",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Our Band Could Be ...",
+        "Price": "57.25",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Olio",
+        "Price": "23.88",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Mesaerion: The Best Science ...",
+        "Price": "37.59",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Libertarianism for Beginners",
+        "Price": "51.33",
+        "InStock": "True"
+    },
+    {
+        "booksName": "It's Only the Himalayas",
+        "Price": "45.17",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Bulletproof Diet: Lose ...",
+        "Price": "49.05",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Eat Fat, Get Thin",
+        "Price": "54.07",
+        "InStock": "True"
+    },
+    {
+        "booksName": "10-Day Green Smoothie Cleanse: ...",
+        "Price": "49.71",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Art and Science ...",
+        "Price": "52.98",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Suzie Snowflake: One beautiful ...",
+        "Price": "54.81",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Libertarianism for Beginners",
+        "Price": "51.33",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Art and Science ...",
+        "Price": "52.98",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Suzie Snowflake: One beautiful ...",
+        "Price": "54.81",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Libertarianism for Beginners",
+        "Price": "51.33",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Why the Right Went ...",
+        "Price": "52.65",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Equal Is Unfair: America's ...",
+        "Price": "56.86",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Dark Notes",
+        "Price": "19.19",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Amid the Chaos",
+        "Price": "36.58",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Grownup",
+        "Price": "35.88",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Long Shadow of ...",
+        "Price": "10.97",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Silence in the Dark ...",
+        "Price": "58.33",
+        "InStock": "True"
+    },
+    {
+        "booksName": "All the Light We ...",
+        "Price": "29.87",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Girl You Left ...",
+        "Price": "15.79",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Logan Kade (Fallen Crest ...",
+        "Price": "13.12",
+        "InStock": "True"
+    },
+    {
+        "booksName": "(Un)Qualified: How God Uses ...",
+        "Price": "54.0",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Crazy Love: Overwhelmed by ...",
+        "Price": "47.72",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Blue Like Jazz: Nonreligious ...",
+        "Price": "25.77",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Online Marketing for Busy ...",
+        "Price": "46.35",
+        "InStock": "True"
+    },
+    {
+        "booksName": "How to Be Miserable: ...",
+        "Price": "46.03",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Overload: How to Unplug, ...",
+        "Price": "52.15",
+        "InStock": "True"
+    },
+    {
+        "booksName": "You Are a Badass: ...",
+        "Price": "12.08",
+        "InStock": "True"
+    },
+    {
+        "booksName": "How to Stop Worrying ...",
+        "Price": "46.49",
+        "InStock": "True"
+    },
+    {
+        "booksName": "In Her Wake",
+        "Price": "12.84",
+        "InStock": "True"
+    },
+    {
+        "booksName": "How Music Works",
+        "Price": "37.32",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Foolproof Preserving: A Guide ...",
+        "Price": "30.52",
+        "InStock": "True"
+    },
+    {
+        "booksName": "In Her Wake",
+        "Price": "12.84",
+        "InStock": "True"
+    },
+    {
+        "booksName": "How Music Works",
+        "Price": "37.32",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Foolproof Preserving: A Guide ...",
+        "Price": "30.52",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Chase Me (Paris Nights ...",
+        "Price": "25.27",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Black Dust",
+        "Price": "34.53",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Birdsong: A Story in ...",
+        "Price": "54.64",
+        "InStock": "True"
+    },
+    {
+        "booksName": "America's Cradle of Quarterbacks: ...",
+        "Price": "22.5",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Aladdin and His Wonderful ...",
+        "Price": "53.13",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Worlds Elsewhere: Journeys Around ...",
+        "Price": "40.3",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Wall and Piece",
+        "Price": "44.18",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Four Agreements: A ...",
+        "Price": "17.66",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Five Love Languages: ...",
+        "Price": "31.05",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Elephant Tree",
+        "Price": "23.82",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Bear and the ...",
+        "Price": "36.89",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Sophie's World",
+        "Price": "15.94",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Penny Maybe",
+        "Price": "33.29",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Maude (1883-1993):She Grew Up ...",
+        "Price": "18.02",
+        "InStock": "True"
+    },
+    {
+        "booksName": "In a Dark, Dark ...",
+        "Price": "19.63",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Behind Closed Doors",
+        "Price": "52.22",
+        "InStock": "True"
+    },
+    {
+        "booksName": "You can't bury them ...",
+        "Price": "33.63",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Slow States of Collapse: ...",
+        "Price": "57.31",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Reasons to Stay Alive",
+        "Price": "26.41",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Private Paris (Private #10)",
+        "Price": "47.61",
+        "InStock": "True"
+    },
+    {
+        "booksName": "#HigherSelfie: Wake Up Your ...",
+        "Price": "23.11",
+        "InStock": "True"
+    },
+    {
+        "bookName": "Reasons to Stay Alive",
+        "Price": "26.41",
+        "InStock": "True"
+    },
+    {
+        "bookName": "Private Paris (Private #10)",
+        "Price": "47.61",
+        "InStock": "True"
+    },
+    {
+        "bookName": "#HigherSelfie: Wake Up Your ...",
+        "Price": "23.11",
+        "InStock": "True"
+    },
+    {
+        "bookName": "Without Borders (Wanderlove #1)",
+        "Price": "45.07",
+        "InStock": "True"
+    },
+    {
+        "bookName": "When We Collided",
+        "Price": "31.77",
+        "InStock": "True"
+    },
+    {
+        "bookName": "We Love You, Charlie ...",
+        "Price": "50.27",
+        "InStock": "True"
+    },
+    {
+        "bookName": "Untitled Collection: Sabbath Poems ...",
+        "Price": "14.27",
+        "InStock": "True"
+    },
+    {
+        "bookName": "Unseen City: The Majesty ...",
+        "Price": "44.18",
+        "InStock": "True"
+    },
+    {
+        "bookName": "Unicorn Tracks",
+        "Price": "18.78",
+        "InStock": "True"
+    },
+    {
+        "bookName": "Unbound: How Eight Technologies ...",
+        "Price": "25.52",
+        "InStock": "True"
+    },
+    {
+        "bookName": "Tsubasa: WoRLD CHRoNiCLE 2 ...",
+        "Price": "16.28",
+        "InStock": "True"
+    },
+    {
+        "bookName": "Throwing Rocks at the ...",
+        "Price": "31.12",
+        "InStock": "True"
+    },
+    {
+        "bookName": "This One Summer",
+        "Price": "19.49",
+        "InStock": "True"
+    },
+    {
+        "bookName": "Thirst",
+        "Price": "17.27",
+        "InStock": "True"
+    },
+    {
+        "bookName": "The Torch Is Passed: ...",
+        "Price": "19.09",
+        "InStock": "True"
+    },
+    {
+        "bookName": "The Secret of Dreadwillow ...",
+        "Price": "56.13",
+        "InStock": "True"
+    },
+    {
+        "bookName": "The Pioneer Woman Cooks: ...",
+        "Price": "56.41",
+        "InStock": "True"
+    },
+    {
+        "bookName": "The Past Never Ends",
+        "Price": "56.5",
+        "InStock": "True"
+    },
+    {
+        "bookName": "The Natural History of ...",
+        "Price": "45.22",
+        "InStock": "True"
+    },
+    {
+        "bookName": "A Light in the ...",
+        "Price": "51.77",
+        "InStock": "True"
+    },
+    {
+        "bookName": "Tipping the Velvet",
+        "Price": "53.74",
+        "InStock": "True"
+    },
+    {
+        "bookName": "Soumission",
+        "Price": "50.1",
+        "InStock": "True"
+    },
+    {
+        "bookName": "Sharp Objects",
+        "Price": "47.82",
+        "InStock": "True"
+    },
+    {
+        "bookName": "Sapiens: A Brief History ...",
+        "Price": "54.23",
+        "InStock": "True"
+    },
+    {
+        "bookName": "The Requiem Red",
+        "Price": "22.65",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Sharp Objects",
+        "Price": "47.82",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Sapiens: A Brief History ...",
+        "Price": "54.23",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Requiem Red",
+        "Price": "22.65",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Dirty Little Secrets ...",
+        "Price": "33.34",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Coming Woman: A ...",
+        "Price": "17.93",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Boys in the ...",
+        "Price": "22.6",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Black Maria",
+        "Price": "52.15",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Starving Hearts (Triangular Trade ...",
+        "Price": "13.99",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Shakespeare's Sonnets",
+        "Price": "20.66",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Set Me Free",
+        "Price": "17.46",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Scott Pilgrim's Precious Little ...",
+        "Price": "52.29",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Rip it Up and ...",
+        "Price": "35.02",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Our Band Could Be ...",
+        "Price": "57.25",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Olio",
+        "Price": "23.88",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Mesaerion: The Best Science ...",
+        "Price": "37.59",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Libertarianism for Beginners",
+        "Price": "51.33",
+        "InStock": "True"
+    },
+    {
+        "booksName": "It's Only the Himalayas",
+        "Price": "45.17",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Nameless City (The ...",
+        "Price": "38.16",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Murder That Never ...",
+        "Price": "54.11",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Most Perfect Thing: ...",
+        "Price": "42.96",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Mindfulness and Acceptance ...",
+        "Price": "23.89",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Life-Changing Magic of ...",
+        "Price": "16.77",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Inefficiency Assassin: Time ...",
+        "Price": "20.59",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Gutsy Girl: Escapades ...",
+        "Price": "37.0",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Life-Changing Magic of ...",
+        "Price": "16.77",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Inefficiency Assassin: Time ...",
+        "Price": "20.59",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Gutsy Girl: Escapades ...",
+        "Price": "37.13",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Electric Pencil: Drawings ...",
+        "Price": "56.06",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Death of Humanity: ...",
+        "Price": "58.11",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Bulletproof Diet: Lose ...",
+        "Price": "49.05",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Art Forger",
+        "Price": "40.76",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Age of Genius: ...",
+        "Price": "19.73",
+        "InStock": "True"
+    },
+    {
+        "booksName": "The Activist's Tao Te ...",
+        "Price": "32.24",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Spark Joy: An Illustrated ...",
+        "Price": "41.83",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Soul Reader",
+        "Price": "39.58",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Security",
+        "Price": "39.25",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Saga, Volume 6 (Saga ...",
+        "Price": "25.02",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Saga, Volume 5 (Saga ...",
+        "Price": "51.04",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Reskilling America: Learning to ...",
+        "Price": "19.83",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Rat Queens, Vol. 3: ...",
+        "Price": "50.4",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Princess Jellyfish 2-in-1 Omnibus, ...",
+        "Price": "13.61",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Princess Between Worlds (Wide-Awake ...",
+        "Price": "13.34",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Pop Gun War, Volume ...",
+        "Price": "18.97",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Political Suicide: Missteps, Peccadilloes, ...",
+        "Price": "36.28",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Patience",
+        "Price": "10.16",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Outcast, Vol. 1: A ...",
+        "Price": "15.44",
+        "InStock": "True"
+    },
+    {
+        "booksName": "orange: The Complete Col",
+        "Price": "0.0",
+        "InStock": "False"
+    },
+    {
+        "booksName": "Missteps, Peccadilloes, ...",
+        "Price": "36.28",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Patience",
+        "Price": "10.16",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Outcast, Vol. 1: A ...",
+        "Price": "15.44",
+        "InStock": "True"
+    },
+    {
+        "booksName": "orange: The Complete Collection ...",
+        "Price": "48.41",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Online Marketing for Busy ...",
+        "Price": "46.35",
+        "InStock": "True"
+    },
+    {
+        "booksName": "On a Midnight Clear",
+        "Price": "14.07",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Obsidian (Lux #1)",
+        "Price": "14.86",
+        "InStock": "True"
+    },
+    {
+        "booksName": "My Paris Kitchen: Recipes ...",
+        "Price": "33.37",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Masks and Shadows",
+        "Price": "56.4",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Mama Tried: Traditional Italian ...",
+        "Price": "14.02",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Lumberjanes, Vol. 2: Friendship ...",
+        "Price": "46.91",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Lumberjanes, Vol. 1: Beware ...",
+        "Price": "45.61",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Lumberjanes Vol. 3: A ...",
+        "Price": "19.92",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Layered: Baking, Building, and ...",
+        "Price": "40.11",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Judo: Seven Steps to ...",
+        "Price": "53.9",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Join",
+        "Price": "35.67",
+        "InStock": "True"
+    },
+    {
+        "booksName": "In the Country We ...",
+        "Price": "22.0",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Immunity: How Elie Metchnikoff ...",
+        "Price": "57.36",
+        "InStock": "True"
+    },
+    {
+        "booksName": "I Hate Fairyland, Vol. ...",
+        "Price": "29.17",
+        "InStock": "True"
+    },
+    {
+        "booksName": "I am a Hero ...",
+        "Price": "54.63",
+        "InStock": "True"
+    },
+    {
+        "booksName": "How to Be Miserable: ...",
+        "Price": "46.03",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Her Backup Boyfriend (The ...",
+        "Price": "33.97",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Giant Days, Vol. 2 ...",
+        "Price": "22.11",
+        "InStock": "True"
+    },
+    {
+        "booksName": "How to Be Miserable: ...",
+        "Price": "46.03",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Her Backup Boyfriend (The ...",
+        "Price": "33.97",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Giant Days, Vol. 2 ...",
+        "Price": "22.11",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Forever and Forever: The ...",
+        "Price": "29.69",
+        "InStock": "True"
+    },
+    {
+        "booksName": "First and First (Five ...",
+        "Price": "15.97",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Fifty Shades Darker (Fifty ...",
+        "Price": "21.96",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Everydata: The Misinformation Hidden ...",
+        "Price": "54.35",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Don't Be a Jerk: ...",
+        "Price": "37.97",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Danganronpa Volume 1",
+        "Price": "51.99",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Crown of Midnight (Throne ...",
+        "Price": "43.29",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Codename Baboushka, Volume 1: ...",
+        "Price": "36.72",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Camp Midnight",
+        "Price": "17.08",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Call the Nurse: True ...",
+        "Price": "29.14",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Burning",
+        "Price": "28.81",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Bossypants",
+        "Price": "49.46",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Bitch Planet, Vol. 1: ...",
+        "Price": "37.92",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Avatar: The Last Airbender: ...",
+        "Price": "28.09",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Algorithms to Live By: ...",
+        "Price": "30.81",
+        "InStock": "True"
+    },
+    {
+        "booksName": "A World of Flavor: ...",
+        "Price": "42.95",
+        "InStock": "True"
+    },
+    {
+        "booksName": "A Piece of Sky, ...",
+        "Price": "56.76",
+        "InStock": "True"
+    },
+    {
+        "booksName": "A Murder in Time",
+        "Price": "16.64",
+        "InStock": "True"
+    },
+    {
+        "booksName": "A Flight of Arrows ...",
+        "Price": "55.53",
+        "InStock": "True"
+    },
+    {
+        "booksName": "A Fierce and Subtle ...",
+        "Price": "28.13",
+        "InStock": "True"
+    },
+    {
+        "booksName": "A Court of Thorns ...",
+        "Price": "52.37",
+        "InStock": "True"
+    },
+    {
+        "booksName": "A Flight of Arrows ...",
+        "Price": "55.53",
+        "InStock": "T"
+    },
+    {
+        "booksName": "A Fierce and Subtle ...",
+        "Price": "28.13",
+        "InStock": "T"
+    },
+    {
+        "booksName": "A Court of Thorns ...",
+        "Price": "52.37",
+        "InStock": "T"
+    },
+    {
+        "booksName": "(Un)Qualified: How God Uses ...",
+        "Price": "54.0",
+        "InStock": "T"
+    },
+    {
+        "booksName": "You Are What You ...",
+        "Price": "21.87",
+        "InStock": "T"
+    },
+    {
+        "booksName": "William Shakespeare's Star Wars: ...",
+        "Price": "43.3",
+        "InStock": "T"
+    },
+    {
+        "booksName": "Tuesday Nights in 1980",
+        "Price": "21.04",
+        "InStock": "T"
+    },
+    {
+        "booksName": "Tracing Numbers on a ...",
+        "Price": "41.6",
+        "InStock": "T"
+    },
+    {
+        "booksName": "Throne of Glass (Throne ...",
+        "Price": "35.07",
+        "InStock": "T"
+    },
+    {
+        "booksName": "Thomas Jefferson and the ...",
+        "Price": "59.64",
+        "InStock": "T"
+    },
+    {
+        "booksName": "Thirteen Reasons Why",
+        "Price": "52.72",
+        "InStock": "T"
+    },
+    {
+        "booksName": "The White Cat and ...",
+        "Price": "58.08",
+        "InStock": "T"
+    },
+    {
+        "booksName": "The Wedding Dress",
+        "Price": "24.12",
+        "InStock": "T"
+    },
+    {
+        "booksName": "The Vacationers",
+        "Price": "42.15",
+        "InStock": "T"
+    },
+    {
+        "booksName": "The Third Wave: An ...",
+        "Price": "12.61",
+        "InStock": "T"
+    },
+    {
+        "booksName": "The Stranger",
+        "Price": "17.44",
+        "InStock": "T"
+    },
+    {
+        "booksName": "The Shadow Hero (The ...",
+        "Price": "33.14",
+        "InStock": "T"
+    },
+    {
+        "booksName": "The Secret (The Secret ...",
+        "Price": "27.37",
+        "InStock": "T"
+    },
+    {
+        "booksName": "The Regional Office Is ...",
+        "Price": "51.36",
+        "InStock": "T"
+    },
+    {
+        "booksName": "The Psychopath Test: A ...",
+        "Price": "36.0",
+        "InStock": "T"
+    },
+    {
+        "booksName": "The Project",
+        "Price": "10.65",
+        "InStock": "T"
+    },
+    {
+        "booksName": "The Power of Now: ...",
+        "Price": "43.54",
+        "InStock": "T"
+    },
+    {
+        "booksName": "The Omnivore's Dilemma: A ...",
+        "Price": "38.21",
+        "InStock": "T"
+    },
+    {
+        "booksName": "Suddenly in Love (Lake ...",
+        "Price": "55.99",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Something More Than This",
+        "Price": "16.24",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Soft Apocalypse",
+        "Price": "26.12",
+        "InStock": "True"
+    },
+    {
+        "booksName": "So You've Been Publicly ...",
+        "Price": "12.23",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Shoe Dog: A Memoir ...",
+        "Price": "23.99",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Shobu Samurai, Project Aryoku ...",
+        "Price": "29.06",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Secrets and Lace (Fatal ...",
+        "Price": "20.27",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Scarlett Epstein Hates It ...",
+        "Price": "43.55",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Romero and Juliet: A ...",
+        "Price": "36.94",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Redeeming Love",
+        "Price": "20.47",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Poses for Artists Volume ...",
+        "Price": "41.06",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Poems That Make Grown ...",
+        "Price": "14.19",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Nightingale, Sing",
+        "Price": "38.28",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Nightingale, Sky with Exit ...",
+        "Price": "41.05",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Mrs. Houdini",
+        "Price": "30.25",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Modern Romance",
+        "Price": "28.26",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Miss Peregrines Home for ...",
+        "Price": "10.76",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Louisa: The Extraordinary Life ...",
+        "Price": "16.85",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Little Red",
+        "Price": "13.47",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Library of Souls (Miss ...",
+        "Price": "48.56",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Large Print Heart of ...",
+        "Price": "19.15",
+        "InStock": "True"
+    },
+    {
+        "booksName": "I Had a Nice ...",
+        "Price": "57.36",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Hollow City (Miss Peregrines ...",
+        "Price": "42.98",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Grumbles",
+        "Price": "22.16",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Large Print Heart of ...",
+        "Price": "19.15",
+        "InStock": "True"
+    },
+    {
+        "booksName": "I Had a Nice ...",
+        "Price": "57.36",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Hollow City (Miss Peregrines ...",
+        "Price": "42.98",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Grumbles",
+        "Price": "22.16",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Full Moon over Noahs ...",
+        "Price": "49.43",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Frostbite (Vampire Academy #2)",
+        "Price": "29.99",
+        "InStock": "True"
+    },
+    {
+        "booksName": "Follow You Home",
+        "Price": "21.36",
+        "InStock": "True"
+    },
+    {
+        "booksName": "First Steps for New ...",
+        "Price": "None",
+        "InStock": "False"
+    }
+]

output.txt

@@ -0,0 +1,2595 @@
+1. __________
+
+### A Light in the ...
+
+51.77
+
+__In stock
+
+Add to basket
+
+ 2. __________
+
+### Tipping the Velvet
+
+53.74
+
+__In stock
+
+Add to basket
+
+ 3. __________
+
+### Soumission
+
+50.10
+
+__In stock
+
+Add to basket
+
+ 4. __________
+
+### Sharp Objects
+
+47.82
+
+__In stock
+
+Add to basket
+
+ 5. __________
+
+### Sapiens: A Brief History ...
+
+54.23
+
+__In stock
+
+Add to basket
+
+ 6. __________
+
+### The Requiem Red
+
+22.65
+
+__In stock
+
+Add to basket
+
+ 7. __________
+
+### The Dirty Little Secrets ...
+
+33.34
+
+__In stock
+
+Add to basket
+
+ 8. __________
+
+### The Coming Woman: A ...
+
+17.93
+
+__In stock
+
+Add to basket
+
+ 9. __________
+
+### The Boys in the ...
+
+22.60
+
+__In stock
+
+Add to basket
+
+ 10. __________
+
+### The Black Maria
+
+52.15
+
+__In stock
+
+Add to basket
+
+ 11. __________
+
+### Starving Hearts (Triangular Trade ...
+
+13.99
+
+__In stock
+
+Add to basket
+
+ 12. __________
+
+### Shakespeare's Sonnets
+
+20.66
+
+__In stock
+
+Add to basket
+
+ 13. __________
+
+### Set Me Free
+
+17.46
+
+__In stock
+
+Add to basket
+
+ 14. __________
+
+### Scott Pilgrim's Precious Little ...
+
+52.29
+
+__In stock
+
+Add to basket
+
+ 15. __________
+
+### Rip it Up and ...
+
+35.02
+
+__In stock
+
+Add to basket
+
+ 16. __________
+
+### Our Band Could Be ...
+
+57.25
+
+__In stock
+
+Add to basket
+
+ 17. __________
+
+### Olio
+
+23.88
+
+__In stock
+
+Add to basket
+
+ 18. __________
+
+### Mesaerion: The Best Science ...
+
+37.59
+
+__In stock
+
+Add to basket
+
+ 19. __________
+
+### Libertarianism for Beginners
+
+51.33
+
+__In stock
+
+Add to basket
+
+ 20. __________
+
+### It's Only the Himalayas
+
+45.17
+
+__In stock
+
+Add to basket
+
+ * Page 1 of 50 
+ * next
+None
+1. __________
+
+### The Bulletproof Diet: Lose ...
+
+49.05
+
+__In stock
+
+Add to basket
+
+ 2. __________
+
+### Eat Fat, Get Thin
+
+54.07
+
+__In stock
+
+Add to basket
+
+ 3. __________
+
+### 10-Day Green Smoothie Cleanse: ...
+
+49.71
+
+__In stock
+
+Add to basket
+
+ 4. __________
+
+### The Art and Science ...
+
+52.98
+
+__In stock
+
+Add to basket
+1. __________
+
+### Suzie Snowflake: One beautiful ...
+
+54.81
+
+__In stock
+
+Add to basket
+1. __________
+
+### Libertarianism for Beginners
+
+51.33
+
+__In stock
+
+Add to basket
+
+ 2. __________
+
+### Why the Right Went ...
+
+52.65
+
+__In stock
+
+Add to basket
+
+ 3. __________
+
+### Equal Is Unfair: America's ...
+
+56.86
+
+__In stock
+
+Add to basket
+None
+1. __________
+
+### Dark Notes
+
+19.19
+
+__In stock
+
+Add to basket
+None
+None
+1. __________
+
+### Amid the Chaos
+
+36.58
+
+__In stock
+
+Add to basket
+1. __________
+
+### The Grownup
+
+35.88
+
+__In stock
+
+Add to basket
+1. __________
+
+### The Long Shadow of ...
+
+10.97
+
+__In stock
+
+Add to basket
+1. __________
+
+### Silence in the Dark ...
+
+58.33
+
+__In stock
+
+Add to basket
+1. __________
+
+### All the Light We ...
+
+29.87
+
+__In stock
+
+Add to basket
+
+ 2. __________
+
+### The Girl You Left ...
+
+15.79
+
+__In stock
+
+Add to basket
+1. __________
+
+### Logan Kade (Fallen Crest ...
+
+13.12
+
+__In stock
+
+Add to basket
+None
+None
+1. __________
+
+### (Un)Qualified: How God Uses ...
+
+54.00
+
+__In stock
+
+Add to basket
+
+ 2. __________
+
+### Crazy Love: Overwhelmed by ...
+
+47.72
+
+__In stock
+
+Add to basket
+
+ 3. __________
+
+### Blue Like Jazz: Nonreligious ...
+
+25.77
+
+__In stock
+
+Add to basket
+None
+None
+1. __________
+
+### Online Marketing for Busy ...
+
+46.35
+
+__In stock
+
+Add to basket
+
+ 2. __________
+
+### How to Be Miserable: ...
+
+46.03
+
+__In stock
+
+Add to basket
+
+ 3. __________
+
+### Overload: How to Unplug, ...
+
+52.15
+
+__In stock
+
+Add to basket
+
+ 4. __________
+
+### You Are a Badass: ...
+
+12.08
+
+__In stock
+
+Add to basket
+
+ 5. __________
+
+### How to Stop Worrying ...
+
+46.49
+
+__In stock
+
+Add to basket
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+1. __________
+
+### In Her Wake
+
+12.84
+
+__In stock
+
+Add to basket
+
+ 2. __________
+
+### How Music Works
+
+37.32
+
+__In stock
+
+Add to basket
+
+ 3. __________
+
+### Foolproof Preserving: A Guide ...
+
+30.52
+
+__In stock
+
+Add to basket
+
+ 4. __________
+
+### Chase Me (Paris Nights ...
+
+25.27
+
+__In stock
+
+Add to basket
+
+ 5. __________
+
+### Black Dust
+
+34.53
+
+__In stock
+
+Add to basket
+
+ 6. __________
+
+### Birdsong: A Story in ...
+
+54.64
+
+__In stock
+
+Add to basket
+
+ 7. __________
+
+### America's Cradle of Quarterbacks: ...
+
+22.50
+
+__In stock
+
+Add to basket
+
+ 8. __________
+
+### Aladdin and His Wonderful ...
+
+53.13
+
+__In stock
+
+Add to basket
+
+ 9. __________
+
+### Worlds Elsewhere: Journeys Around ...
+
+40.30
+
+__In stock
+
+Add to basket
+
+ 10. __________
+
+### Wall and Piece
+
+44.18
+
+__In stock
+
+Add to basket
+
+ 11. __________
+
+### The Four Agreements: A ...
+
+17.66
+
+__In stock
+
+Add to basket
+
+ 12. __________
+
+### The Five Love Languages: ...
+
+31.05
+
+__In stock
+
+Add to basket
+
+ 13. __________
+
+### The Elephant Tree
+
+23.82
+
+__In stock
+
+Add to basket
+
+ 14. __________
+
+### The Bear and the ...
+
+36.89
+
+__In stock
+
+Add to basket
+
+ 15. __________
+
+### Sophie's World
+
+15.94
+
+__In stock
+
+Add to basket
+
+ 16. __________
+
+### Penny Maybe
+
+33.29
+
+__In stock
+
+Add to basket
+
+ 17. __________
+
+### Maude (1883-1993):She Grew Up ...
+
+18.02
+
+__In stock
+
+Add to basket
+
+ 18. __________
+
+### In a Dark, Dark ...
+
+19.63
+
+__In stock
+
+Add to basket
+
+ 19. __________
+
+### Behind Closed Doors
+
+52.22
+
+__In stock
+
+Add to basket
+
+ 20. __________
+
+### You can't bury them ...
+
+33.63
+
+__In stock
+
+Add to basket
+
+ * previous
+ * Page 2 of 50 
+ * next
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+1. __________
+
+### Slow States of Collapse: ...
+
+57.31
+
+__In stock
+
+Add to basket
+
+ 2. __________
+
+### Reasons to Stay Alive
+
+26.41
+
+__In stock
+
+Add to basket
+
+ 3. __________
+
+### Private Paris (Private #10)
+
+47.61
+
+__In stock
+
+Add to basket
+
+ 4. __________
+
+### #HigherSelfie: Wake Up Your ...
+
+23.11
+
+__In stock
+
+Add to basket
+
+ 5. __________
+
+### Without Borders (Wanderlove #1)
+
+45.07
+
+__In stock
+
+Add to basket
+
+ 6. __________
+
+### When We Collided
+
+31.77
+
+__In stock
+
+Add to basket
+
+ 7. __________
+
+### We Love You, Charlie ...
+
+50.27
+
+__In stock
+
+Add to basket
+
+ 8. __________
+
+### Untitled Collection: Sabbath Poems ...
+
+14.27
+
+__In stock
+
+Add to basket
+
+ 9. __________
+
+### Unseen City: The Majesty ...
+
+44.18
+
+__In stock
+
+Add to basket
+
+ 10. __________
+
+### Unicorn Tracks
+
+18.78
+
+__In stock
+
+Add to basket
+
+ 11. __________
+
+### Unbound: How Eight Technologies ...
+
+25.52
+
+__In stock
+
+Add to basket
+
+ 12. __________
+
+### Tsubasa: WoRLD CHRoNiCLE 2 ...
+
+16.28
+
+__In stock
+
+Add to basket
+
+ 13. __________
+
+### Throwing Rocks at the ...
+
+31.12
+
+__In stock
+
+Add to basket
+
+ 14. __________
+
+### This One Summer
+
+19.49
+
+__In stock
+
+Add to basket
+
+ 15. __________
+
+### Thirst
+
+17.27
+
+__In stock
+
+Add to basket
+
+ 16. __________
+
+### The Torch Is Passed: ...
+
+19.09
+
+__In stock
+
+Add to basket
+
+ 17. __________
+
+### The Secret of Dreadwillow ...
+
+56.13
+
+__In stock
+
+Add to basket
+
+ 18. __________
+
+### The Pioneer Woman Cooks: ...
+
+56.41
+
+__In stock
+
+Add to basket
+
+ 19. __________
+
+### The Past Never Ends
+
+56.50
+
+__In stock
+
+Add to basket
+
+ 20. __________
+
+### The Natural History of ...
+
+45.22
+
+__In stock
+
+Add to basket
+
+ * previous
+ * Page 3 of 50 
+ * next
+None
+None
+1. __________
+
+### A Light in the ...
+
+51.77
+
+__In stock
+
+Add to basket
+
+ 2. __________
+
+### Tipping the Velvet
+
+53.74
+
+__In stock
+
+Add to basket
+
+ 3. __________
+
+### Soumission
+
+50.10
+
+__In stock
+
+Add to basket
+
+ 4. __________
+
+### Sharp Objects
+
+47.82
+
+__In stock
+
+Add to basket
+
+ 5. __________
+
+### Sapiens: A Brief History ...
+
+54.23
+
+__In stock
+
+Add to basket
+
+ 6. __________
+
+### The Requiem Red
+
+22.65
+
+__In stock
+
+Add to basket
+
+ 7. __________
+
+### The Dirty Little Secrets ...
+
+33.34
+
+__In stock
+
+Add to basket
+
+ 8. __________
+
+### The Coming Woman: A ...
+
+17.93
+
+__In stock
+
+Add to basket
+
+ 9. __________
+
+### The Boys in the ...
+
+22.60
+
+__In stock
+
+Add to basket
+
+ 10. __________
+
+### The Black Maria
+
+52.15
+
+__In stock
+
+Add to basket
+
+ 11. __________
+
+### Starving Hearts (Triangular Trade ...
+
+13.99
+
+__In stock
+
+Add to basket
+
+ 12. __________
+
+### Shakespeare's Sonnets
+
+20.66
+
+__In stock
+
+Add to basket
+
+ 13. __________
+
+### Set Me Free
+
+17.46
+
+__In stock
+
+Add to basket
+
+ 14. __________
+
+### Scott Pilgrim's Precious Little ...
+
+52.29
+
+__In stock
+
+Add to basket
+
+ 15. __________
+
+### Rip it Up and ...
+
+35.02
+
+__In stock
+
+Add to basket
+
+ 16. __________
+
+### Our Band Could Be ...
+
+57.25
+
+__In stock
+
+Add to basket
+
+ 17. __________
+
+### Olio
+
+23.88
+
+__In stock
+
+Add to basket
+
+ 18. __________
+
+### Mesaerion: The Best Science ...
+
+37.59
+
+__In stock
+
+Add to basket
+
+ 19. __________
+
+### Libertarianism for Beginners
+
+51.33
+
+__In stock
+
+Add to basket
+
+ 20. __________
+
+### It's Only the Himalayas
+
+45.17
+
+__In stock
+
+Add to basket
+
+ * Page 1 of 50 
+ * next
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+1. __________
+
+### The Nameless City (The ...
+
+38.16
+
+__In stock
+
+Add to basket
+
+ 2. __________
+
+### The Murder That Never ...
+
+54.11
+
+__In stock
+
+Add to basket
+
+ 3. __________
+
+### The Most Perfect Thing: ...
+
+42.96
+
+__In stock
+
+Add to basket
+
+ 4. __________
+
+### The Mindfulness and Acceptance ...
+
+23.89
+
+__In stock
+
+Add to basket
+
+ 5. __________
+
+### The Life-Changing Magic of ...
+
+16.77
+
+__In stock
+
+Add to basket
+
+ 6. __________
+
+### The Inefficiency Assassin: Time ...
+
+20.59
+
+__In stock
+
+Add to basket
+
+ 7. __________
+
+### The Gutsy Girl: Escapades ...
+
+37.13
+
+__In stock
+
+Add to basket
+
+ 8. __________
+
+### The Electric Pencil: Drawings ...
+
+56.06
+
+__In stock
+
+Add to basket
+
+ 9. __________
+
+### The Death of Humanity: ...
+
+58.11
+
+__In stock
+
+Add to basket
+
+ 10. __________
+
+### The Bulletproof Diet: Lose ...
+
+49.05
+
+__In stock
+
+Add to basket
+
+ 11. __________
+
+### The Art Forger
+
+40.76
+
+__In stock
+
+Add to basket
+
+ 12. __________
+
+### The Age of Genius: ...
+
+19.73
+
+__In stock
+
+Add to basket
+
+ 13. __________
+
+### The Activist's Tao Te ...
+
+32.24
+
+__In stock
+
+Add to basket
+
+ 14. __________
+
+### Spark Joy: An Illustrated ...
+
+41.83
+
+__In stock
+
+Add to basket
+
+ 15. __________
+
+### Soul Reader
+
+39.58
+
+__In stock
+
+Add to basket
+
+ 16. __________
+
+### Security
+
+39.25
+
+__In stock
+
+Add to basket
+
+ 17. __________
+
+### Saga, Volume 6 (Saga ...
+
+25.02
+
+__In stock
+
+Add to basket
+
+ 18. __________
+
+### Saga, Volume 5 (Saga ...
+
+51.04
+
+__In stock
+
+Add to basket
+
+ 19. __________
+
+### Reskilling America: Learning to ...
+
+19.83
+
+__In stock
+
+Add to basket
+
+ 20. __________
+
+### Rat Queens, Vol. 3: ...
+
+50.40
+
+__In stock
+
+Add to basket
+
+ * previous
+ * Page 4 of 50 
+ * next
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+1. __________
+
+### Princess Jellyfish 2-in-1 Omnibus, ...
+
+13.61
+
+__In stock
+
+Add to basket
+
+ 2. __________
+
+### Princess Between Worlds (Wide-Awake ...
+
+13.34
+
+__In stock
+
+Add to basket
+
+ 3. __________
+
+### Pop Gun War, Volume ...
+
+18.97
+
+__In stock
+
+Add to basket
+
+ 4. __________
+
+### Political Suicide: Missteps, Peccadilloes, ...
+
+36.28
+
+__In stock
+
+Add to basket
+
+ 5. __________
+
+### Patience
+
+10.16
+
+__In stock
+
+Add to basket
+
+ 6. __________
+
+### Outcast, Vol. 1: A ...
+
+15.44
+
+__In stock
+
+Add to basket
+
+ 7. __________
+
+### orange: The Complete Collection ...
+
+48.41
+
+__In stock
+
+Add to basket
+
+ 8. __________
+
+### Online Marketing for Busy ...
+
+46.35
+
+__In stock
+
+Add to basket
+
+ 9. __________
+
+### On a Midnight Clear
+
+14.07
+
+__In stock
+
+Add to basket
+
+ 10. __________
+
+### Obsidian (Lux #1)
+
+14.86
+
+__In stock
+
+Add to basket
+
+ 11. __________
+
+### My Paris Kitchen: Recipes ...
+
+33.37
+
+__In stock
+
+Add to basket
+
+ 12. __________
+
+### Masks and Shadows
+
+56.40
+
+__In stock
+
+Add to basket
+
+ 13. __________
+
+### Mama Tried: Traditional Italian ...
+
+14.02
+
+__In stock
+
+Add to basket
+
+ 14. __________
+
+### Lumberjanes, Vol. 2: Friendship ...
+
+46.91
+
+__In stock
+
+Add to basket
+
+ 15. __________
+
+### Lumberjanes, Vol. 1: Beware ...
+
+45.61
+
+__In stock
+
+Add to basket
+
+ 16. __________
+
+### Lumberjanes Vol. 3: A ...
+
+19.92
+
+__In stock
+
+Add to basket
+
+ 17. __________
+
+### Layered: Baking, Building, and ...
+
+40.11
+
+__In stock
+
+Add to basket
+
+ 18. __________
+
+### Judo: Seven Steps to ...
+
+53.90
+
+__In stock
+
+Add to basket
+
+ 19. __________
+
+### Join
+
+35.67
+
+__In stock
+
+Add to basket
+
+ 20. __________
+
+### In the Country We ...
+
+22.00
+
+__In stock
+
+Add to basket
+
+ * previous
+ * Page 5 of 50 
+ * next
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+1. __________
+
+### Immunity: How Elie Metchnikoff ...
+
+57.36
+
+__In stock
+
+Add to basket
+
+ 2. __________
+
+### I Hate Fairyland, Vol. ...
+
+29.17
+
+__In stock
+
+Add to basket
+
+ 3. __________
+
+### I am a Hero ...
+
+54.63
+
+__In stock
+
+Add to basket
+
+ 4. __________
+
+### How to Be Miserable: ...
+
+46.03
+
+__In stock
+
+Add to basket
+
+ 5. __________
+
+### Her Backup Boyfriend (The ...
+
+33.97
+
+__In stock
+
+Add to basket
+
+ 6. __________
+
+### Giant Days, Vol. 2 ...
+
+22.11
+
+__In stock
+
+Add to basket
+
+ 7. __________
+
+### Forever and Forever: The ...
+
+29.69
+
+__In stock
+
+Add to basket
+
+ 8. __________
+
+### First and First (Five ...
+
+15.97
+
+__In stock
+
+Add to basket
+
+ 9. __________
+
+### Fifty Shades Darker (Fifty ...
+
+21.96
+
+__In stock
+
+Add to basket
+
+ 10. __________
+
+### Everydata: The Misinformation Hidden ...
+
+54.35
+
+__In stock
+
+Add to basket
+
+ 11. __________
+
+### Don't Be a Jerk: ...
+
+37.97
+
+__In stock
+
+Add to basket
+
+ 12. __________
+
+### Danganronpa Volume 1
+
+51.99
+
+__In stock
+
+Add to basket
+
+ 13. __________
+
+### Crown of Midnight (Throne ...
+
+43.29
+
+__In stock
+
+Add to basket
+
+ 14. __________
+
+### Codename Baboushka, Volume 1: ...
+
+36.72
+
+__In stock
+
+Add to basket
+
+ 15. __________
+
+### Camp Midnight
+
+17.08
+
+__In stock
+
+Add to basket
+
+ 16. __________
+
+### Call the Nurse: True ...
+
+29.14
+
+__In stock
+
+Add to basket
+
+ 17. __________
+
+### Burning
+
+28.81
+
+__In stock
+
+Add to basket
+
+ 18. __________
+
+### Bossypants
+
+49.46
+
+__In stock
+
+Add to basket
+
+ 19. __________
+
+### Bitch Planet, Vol. 1: ...
+
+37.92
+
+__In stock
+
+Add to basket
+
+ 20. __________
+
+### Avatar: The Last Airbender: ...
+
+28.09
+
+__In stock
+
+Add to basket
+
+ * previous
+ * Page 6 of 50 
+ * next
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+1. __________
+
+### Algorithms to Live By: ...
+
+30.81
+
+__In stock
+
+Add to basket
+
+ 2. __________
+
+### A World of Flavor: ...
+
+42.95
+
+__In stock
+
+Add to basket
+
+ 3. __________
+
+### A Piece of Sky, ...
+
+56.76
+
+__In stock
+
+Add to basket
+
+ 4. __________
+
+### A Murder in Time
+
+16.64
+
+__In stock
+
+Add to basket
+
+ 5. __________
+
+### A Flight of Arrows ...
+
+55.53
+
+__In stock
+
+Add to basket
+
+ 6. __________
+
+### A Fierce and Subtle ...
+
+28.13
+
+__In stock
+
+Add to basket
+
+ 7. __________
+
+### A Court of Thorns ...
+
+52.37
+
+__In stock
+
+Add to basket
+
+ 8. __________
+
+### (Un)Qualified: How God Uses ...
+
+54.00
+
+__In stock
+
+Add to basket
+
+ 9. __________
+
+### You Are What You ...
+
+21.87
+
+__In stock
+
+Add to basket
+
+ 10. __________
+
+### William Shakespeare's Star Wars: ...
+
+43.30
+
+__In stock
+
+Add to basket
+
+ 11. __________
+
+### Tuesday Nights in 1980
+
+21.04
+
+__In stock
+
+Add to basket
+
+ 12. __________
+
+### Tracing Numbers on a ...
+
+41.60
+
+__In stock
+
+Add to basket
+
+ 13. __________
+
+### Throne of Glass (Throne ...
+
+35.07
+
+__In stock
+
+Add to basket
+
+ 14. __________
+
+### Thomas Jefferson and the ...
+
+59.64
+
+__In stock
+
+Add to basket
+
+ 15. __________
+
+### Thirteen Reasons Why
+
+52.72
+
+__In stock
+
+Add to basket
+
+ 16. __________
+
+### The White Cat and ...
+
+58.08
+
+__In stock
+
+Add to basket
+
+ 17. __________
+
+### The Wedding Dress
+
+24.12
+
+__In stock
+
+Add to basket
+
+ 18. __________
+
+### The Vacationers
+
+42.15
+
+__In stock
+
+Add to basket
+
+ 19. __________
+
+### The Third Wave: An ...
+
+12.61
+
+__In stock
+
+Add to basket
+
+ 20. __________
+
+### The Stranger
+
+17.44
+
+__In stock
+
+Add to basket
+
+ * previous
+ * Page 7 of 50 
+ * next
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+1. __________
+
+### The Shadow Hero (The ...
+
+33.14
+
+__In stock
+
+Add to basket
+
+ 2. __________
+
+### The Secret (The Secret ...
+
+27.37
+
+__In stock
+
+Add to basket
+
+ 3. __________
+
+### The Regional Office Is ...
+
+51.36
+
+__In stock
+
+Add to basket
+
+ 4. __________
+
+### The Psychopath Test: A ...
+
+36.00
+
+__In stock
+
+Add to basket
+
+ 5. __________
+
+### The Project
+
+10.65
+
+__In stock
+
+Add to basket
+
+ 6. __________
+
+### The Power of Now: ...
+
+43.54
+
+__In stock
+
+Add to basket
+
+ 7. __________
+
+### The Omnivore's Dilemma: A ...
+
+38.21
+
+__In stock
+
+Add to basket
+
+ 8. __________
+
+### The Nerdy Nummies Cookbook: ...
+
+37.34
+
+__In stock
+
+Add to basket
+
+ 9. __________
+
+### The Murder of Roger ...
+
+44.10
+
+__In stock
+
+Add to basket
+
+ 10. __________
+
+### The Mistake (Off-Campus #2)
+
+43.29
+
+__In stock
+
+Add to basket
+
+ 11. __________
+
+### The Matchmaker's Playbook (Wingmen ...
+
+55.85
+
+__In stock
+
+Add to basket
+
+ 12. __________
+
+### The Love and Lemons ...
+
+37.60
+
+__In stock
+
+Add to basket
+
+ 13. __________
+
+### The Long Shadow of ...
+
+10.97
+
+__In stock
+
+Add to basket
+
+ 14. __________
+
+### The Kite Runner
+
+41.82
+
+__In stock
+
+Add to basket
+
+ 15. __________
+
+### The House by the ...
+
+36.95
+
+__In stock
+
+Add to basket
+
+ 16. __________
+
+### The Glittering Court (The ...
+
+44.28
+
+__In stock
+
+Add to basket
+
+ 17. __________
+
+### The Girl on the ...
+
+55.02
+
+__In stock
+
+Add to basket
+
+ 18. __________
+
+### The Genius of Birds
+
+17.24
+
+__In stock
+
+Add to basket
+
+ 19. __________
+
+### The Emerald Mystery
+
+23.15
+
+__In stock
+
+Add to basket
+
+ 20. __________
+
+### The Cookies & Cups ...
+
+41.25
+
+__In stock
+
+Add to basket
+
+ * previous
+ * Page 8 of 50 
+ * next
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+1. __________
+
+### The Bridge to Consciousness: ...
+
+32.00
+
+__In stock
+
+Add to basket
+
+ 2. __________
+
+### The Artist's Way: A ...
+
+38.49
+
+__In stock
+
+Add to basket
+
+ 3. __________
+
+### The Art of War
+
+33.34
+
+__In stock
+
+Add to basket
+
+ 4. __________
+
+### The Argonauts
+
+10.93
+
+__In stock
+
+Add to basket
+
+ 5. __________
+
+### The 10% Entrepreneur: Live ...
+
+27.55
+
+__In stock
+
+Add to basket
+
+ 6. __________
+
+### Suddenly in Love (Lake ...
+
+55.99
+
+__In stock
+
+Add to basket
+
+ 7. __________
+
+### Something More Than This
+
+16.24
+
+__In stock
+
+Add to basket
+
+ 8. __________
+
+### Soft Apocalypse
+
+26.12
+
+__In stock
+
+Add to basket
+
+ 9. __________
+
+### So You've Been Publicly ...
+
+12.23
+
+__In stock
+
+Add to basket
+
+ 10. __________
+
+### Shoe Dog: A Memoir ...
+
+23.99
+
+__In stock
+
+Add to basket
+
+ 11. __________
+
+### Shobu Samurai, Project Aryoku ...
+
+29.06
+
+__In stock
+
+Add to basket
+
+ 12. __________
+
+### Secrets and Lace (Fatal ...
+
+20.27
+
+__In stock
+
+Add to basket
+
+ 13. __________
+
+### Scarlett Epstein Hates It ...
+
+43.55
+
+__In stock
+
+Add to basket
+
+ 14. __________
+
+### Romero and Juliet: A ...
+
+36.94
+
+__In stock
+
+Add to basket
+
+ 15. __________
+
+### Redeeming Love
+
+20.47
+
+__In stock
+
+Add to basket
+
+ 16. __________
+
+### Poses for Artists Volume ...
+
+41.06
+
+__In stock
+
+Add to basket
+
+ 17. __________
+
+### Poems That Make Grown ...
+
+14.19
+
+__In stock
+
+Add to basket
+
+ 18. __________
+
+### Nightingale, Sing
+
+38.28
+
+__In stock
+
+Add to basket
+
+ 19. __________
+
+### Night Sky with Exit ...
+
+41.05
+
+__In stock
+
+Add to basket
+
+ 20. __________
+
+### Mrs. Houdini
+
+30.25
+
+__In stock
+
+Add to basket
+
+ * previous
+ * Page 9 of 50 
+ * next
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+None
+1. __________
+
+### Modern Romance
+
+28.26
+
+__In stock
+
+Add to basket
+
+ 2. __________
+
+### Miss Peregrines Home for ...
+
+10.76
+
+__In stock
+
+Add to basket
+
+ 3. __________
+
+### Louisa: The Extraordinary Life ...
+
+16.85
+
+__In stock
+
+Add to basket
+
+ 4. __________
+
+### Little Red
+
+13.47
+
+__In stock
+
+Add to basket
+
+ 5. __________
+
+### Library of Souls (Miss ...
+
+48.56
+
+__In stock
+
+Add to basket
+
+ 6. __________
+
+### Large Print Heart of ...
+
+19.15
+
+__In stock
+
+Add to basket
+
+ 7. __________
+
+### I Had a Nice ...
+
+57.36
+
+__In stock
+
+Add to basket
+
+ 8. __________
+
+### Hollow City (Miss Peregrines ...
+
+42.98
+
+__In stock
+
+Add to basket
+
+ 9. __________
+
+### Grumbles
+
+22.16
+
+__In stock
+
+Add to basket
+
+ 10. __________
+
+### Full Moon over Noahs ...
+
+49.43
+
+__In stock
+
+Add to basket
+
+ 11. __________
+
+### Frostbite (Vampire Academy #2)
+
+29.99
+
+__In stock
+
+Add to basket
+
+ 12. __________
+
+### Follow You Home
+
+21.36
+
+__In stock
+
+Add to basket
+
+ 13. __________
+
+### First Steps for New ...

requirements.txt

@@ -0,0 +1,7 @@
+beautifulsoup4==4.12.3
+groq==0.11.0
+html2text==2024.2.26
+itemadapter==0.9.0
+Requests==2.32.3
+Scrapy==2.11.2
+streamlit==1.37.0

webscraper/markdown_files/default.md

@@ -0,0 +1,338 @@
+Event type
+
+Everything
+
+ * Dance And Clubs
+
+ * Disco
+ * Drum And Bass
+ * Edm (Electronic Dance Music)
+ * Electro
+ * Electronica
+ * Garage
+ * Grime
+ * House
+ * Ibiza
+ * Other
+ * R&B
+ * Rap/Hip Hop
+ * Soul
+ * Techno
+ * Rock
+
+ * '60S
+ * '70S
+ * '80S
+ * Alternative/Indie
+ * Americana
+ * Blues
+ * Classic Rock
+ * Club
+ * Doom/Stoner Metal
+ * Emo/Pop Punk
+ * Folk
+ * General
+ * Glam
+ * Goth
+ * Hardcore
+ * Heritage
+ * Metal
+ * Metalcore
+ * New Wave
+ * Other Rock Music
+ * Post Punk
+ * Post Rock
+ * Prog Rock/Psych
+ * Punk/Ska
+ * Surfer Rock
+ * Tribute Band
+ * Pop
+
+ * '60S
+ * '70S
+ * '80S
+ * '90S
+ * Boy Band
+ * Club
+ * Girl Band
+ * Group
+ * Heritage
+ * Hyperpop
+ * Indie-Pop
+ * Other
+ * Solo Female
+ * Solo Male
+ * Tribute Band
+ * Other Music
+
+ * African
+ * Bluegrass
+ * Blues
+ * Classical
+ * Country
+ * Easy Listening
+ * Experimental
+ * Folk
+ * French Singers
+ * Funk
+ * Instrumental
+ * International Scene
+ * Jazz
+ * K Pop
+ * Karaoke
+ * Latin
+ * Orchestral
+ * Other
+ * Pop Rock
+ * Reggae
+ * Spoken Word
+ * World
+ * Theatre
+
+ * Ballet
+ * Dance
+ * Entertainment
+ * Family
+ * Immersive
+ * Musical
+ * Offers
+ * Opera
+ * Play
+ * Shakespeare
+ * Cinema
+
+ * Action
+ * Comedy
+ * Drama
+ * Family
+ * Film Festival
+ * Horror
+ * Other
+ * Romance
+ * Thriller
+ * Festival
+
+ * Canada
+ * Denmark
+ * Europe
+ * Food
+ * France
+ * Lifestyle
+ * Live Entertainment
+ * Music
+ * Netherlands
+ * Other
+ * Portugal
+ * Spain
+ * Switzerland
+ * United Kingdom
+ * United States
+ * Special Events
+
+ * Adult
+ * Agricultural
+ * Air Show & Military
+ * Art & Antiques
+ * Burlesque
+ * Business
+ * Childrens
+ * Christmas
+ * Drag Show
+ * Educational
+ * Fashion & Beauty
+ * Film & Tv
+ * Food & Drink
+ * Gaming
+ * Gardens
+ * Home & Lifestyle
+ * Horse Show
+ * Ice Skating
+ * In Conversation
+ * Magic
+ * Motor Events
+ * Online & Social Media
+ * Other
+ * Outdoor Theatre
+ * Podcast
+ * Shopping/Market
+ * Shows
+ * Travel & Outdoor
+ * True Crime
+ * Comedy
+
+ * Festival
+ * Improv
+ * Other
+ * Screen To Stage
+ * Stand-Up
+ * Sport
+
+ * American Football
+ * Athletics
+ * Basketball
+ * Boxing
+ * Cricket
+ * Darts
+ * Equestrian
+ * Fighting
+ * Football
+ * Golf
+ * Hockey
+ * Ice Hockey
+ * Tennis
+ * Motorsport
+ * Netball
+ * Martial Arts
+ * Other
+ * Polo
+ * Pool
+ * Rugby
+ * Snooker
+ * Winter Sports
+ * Wrestling
+ * Attractions
+
+ * Afternoon Tea In London
+ * Airport And Private Transfer
+ * Brit Movie & Tv Tour
+ * Castles
+ * Day Trips From London
+ * Family
+ * London Attractions
+ * London Bus Tour
+ * London By Night
+ * London Sightseeing Tours
+ * Monuments
+ * Museums And Exhibitions
+ * Out Of London Attractions
+ * River Cruises
+ * Royal London
+ * Seasonal (Christmas & Nye/Easter)
+ * Stonehenge Tours
+ * The London Eye
+ * Theme Parks
+ * Tours In Foreign Language
+ * Uk Theme Parks
+ * Walking And Bike Tours
+ * Warner Bro. Studio Tour London
+
+Date
+
+Anytime
+
+Today
+
+This Weekend
+
+Next Weekend
+
+Next Seven Days
+
+Specific Dates
+
+From
+
+To
+
+Location
+
+Reset
+
+Relevance Soonest Nearest Recently Added Alphabetical
+
+ * UK Parliament: Audio Tour 
+UK Parliament, London 
+Sat 14 Sep 2024 \- Sat 28 Dec 2024
+ * UK Parliament: Guided Tour 
+UK Parliament, London 
+Sat 14 Sep 2024 \- Sat 28 Dec 2024
+ * Big Ben Tour 
+UK Parliament, London 
+Sat 14 Sep 2024 \- Sat 21 Dec 2024
+ * London Zoo 
+London Zoo, Regents Park, London, NW1 4RY 
+Sat 14 Sep 2024 \- Tue 31 Dec 2024
+ * Brooklands Museum 
+Brooklands Museum, Weybridge, Surrey 
+Sat 14 Sep 2024 \- Wed 01 Jan 2025
+ * The Moonwalkers - A Journey with Tom Hanks 
+Lightroom, London 
+Sat 14 Sep 2024 \- Sun 13 Oct 2024
+ * Whipsnade Zoo 
+Whipsnade Zoo, Dunstable, LU6 2LF 
+Sat 14 Sep 2024 \- Tue 31 Dec 2024
+ * THE STATE ROOMS 
+Buckingham Palace, London 
+Sat 14 Sep 2024, 13:00 Contact venue for tickets 
+ * The Mousetrap 
+St Martin's Theatre, London 
+Sat 14 Sep 2024 \- Sat 01 Mar 2025
+ * David Hockney: Bigger & Closer 
+Lightroom, London 
+Sun 15 Sep 2024 \- Sun 06 Oct 2024
+ * THE STATE ROOMS 
+Buckingham Palace, London 
+Sun 15 Sep 2024, 13:00 Contact venue for tickets 
+ * THE STATE ROOMS 
+Buckingham Palace, London 
+Mon 16 Sep 2024, 13:00 Contact venue for tickets 
+ * THE STATE ROOMS 
+Buckingham Palace, London 
+Thu 19 Sep 2024, 13:00 Sold out 
+ * Women and Medicine in Euston and Fitzrovia Walk 
+The Euston Tap, London 
+Fri 20 Sep 2024, 12:30 
+ * THE STATE ROOMS 
+Buckingham Palace, London 
+Fri 20 Sep 2024, 13:00 Sold out 
+ * Quirky Westminster Walk 
+Fish Street Hill, Le Pain Quotidien 
+Fri 20 Sep 2024, 14:00 
+ * THE STATE ROOMS 
+Buckingham Palace, London 
+Sat 21 Sep 2024, 13:00 Sold out 
+ * Best of Blackfriars Walking Tour 
+Blackfriars Station, Central London 
+Sat 21 Sep 2024, 16:00 
+ * THE STATE ROOMS 
+Buckingham Palace, London 
+Sun 22 Sep 2024, 13:00 Sold out 
+ * THE STATE ROOMS 
+Buckingham Palace, London 
+Mon 23 Sep 2024, 13:00 Sold out 
+ * THE STATE ROOMS 
+Buckingham Palace, London 
+Thu 26 Sep 2024, 13:00 Sold out 
+ * Omar | UK's Soul Legend 
+Boisdale Of Canary Wharf, London 
+Thu 26 Sep 2024, 21:30 Artists: Omar Lye-Fook
+ * A Silk Road Oasis: Life in Ancient Dunhuang 
+British Library, London 
+Fri 27 Sep 2024 \- Sun 23 Feb 2025
+ * THE STATE ROOMS 
+Buckingham Palace, London 
+Fri 27 Sep 2024, 13:00 Sold out 
+ * THE STATE ROOMS 
+Buckingham Palace, London 
+Sat 28 Sep 2024, 13:00 Sold out 
+ * THE STATE ROOMS 
+Buckingham Palace, London 
+Sun 29 Sep 2024, 13:00 
+ * Victorian Covent Garden Walking Tour 
+The Clermont Hotel, The Clermont Hotel 
+Sat 05 Oct 2024, 12:00 
+ * Quirky Westminster Walk 
+Covent Garden tube station, London 
+Sun 06 Oct 2024, 11:00 
+ * Secrets of Spitalfields Walk 
+Liverpool Street Station, London 
+Sun 13 Oct 2024, 12:00 
+ * Halloween At Kew 
+Various venues 
+Fri 18 Oct 2024 \- Sun 03 Nov 2024
+
+1 of 2
+
+ * Next 
+
+It looks like Javascript is disabled. You will have to enable Javascript to purchase tickets.

webscraper/scrapy.cfg

@@ -0,0 +1,12 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = webscraper.settings
+shell = ipython
+
+[deploy]
+#url = http://localhost:6800/
+project = webscraper

webscraper/webscraper/items.py

@@ -0,0 +1,12 @@
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+
+class WebscraperItem(scrapy.Item):
+    # define the fields for your item here like:
+    # name = scrapy.Field()
+    url = scrapy.Field()
+    extracted_content = scrapy.Field()

webscraper/webscraper/markdown_files/default.md

@@ -0,0 +1,202 @@
+1. __________
+
+### Frankenstein
+
+38.00
+
+__In stock
+
+Add to basket
+
+ 2. __________
+
+### Forever Rockers (The Rocker ...
+
+28.80
+
+__In stock
+
+Add to basket
+
+ 3. __________
+
+### Fighting Fate (Fighting #6)
+
+39.24
+
+__In stock
+
+Add to basket
+
+ 4. __________
+
+### Emma
+
+32.93
+
+__In stock
+
+Add to basket
+
+ 5. __________
+
+### Eat, Pray, Love
+
+51.32
+
+__In stock
+
+Add to basket
+
+ 6. __________
+
+### Deep Under (Walker Security ...
+
+47.09
+
+__In stock
+
+Add to basket
+
+ 7. __________
+
+### Choosing Our Religion: The ...
+
+28.42
+
+__In stock
+
+Add to basket
+
+ 8. __________
+
+### Charlie and the Chocolate ...
+
+22.85
+
+__In stock
+
+Add to basket
+
+ 9. __________
+
+### Charity's Cross (Charles Towne ...
+
+41.24
+
+__In stock
+
+Add to basket
+
+ 10. __________
+
+### Bright Lines
+
+39.07
+
+__In stock
+
+Add to basket
+
+ 11. __________
+
+### Bridget Jones's Diary (Bridget ...
+
+29.82
+
+__In stock
+
+Add to basket
+
+ 12. __________
+
+### Bounty (Colorado Mountain #7)
+
+37.26
+
+__In stock
+
+Add to basket
+
+ 13. __________
+
+### Blood Defense (Samantha Brinkman ...
+
+20.30
+
+__In stock
+
+Add to basket
+
+ 14. __________
+
+### Bleach, Vol. 1: Strawberry ...
+
+34.65
+
+__In stock
+
+Add to basket
+
+ 15. __________
+
+### Beyond Good and Evil
+
+43.38
+
+__In stock
+
+Add to basket
+
+ 16. __________
+
+### Alice in Wonderland (Alice's ...
+
+55.53
+
+__In stock
+
+Add to basket
+
+ 17. __________
+
+### Ajin: Demi-Human, Volume 1 ...
+
+57.06
+
+__In stock
+
+Add to basket
+
+ 18. __________
+
+### A Spy's Devotion (The ...
+
+16.97
+
+__In stock
+
+Add to basket
+
+ 19. __________
+
+### 1st to Die (Women's ...
+
+53.98
+
+__In stock
+
+Add to basket
+
+ 20. __________
+
+### 1,000 Places to See ...
+
+26.08
+
+__In stock
+
+Add to basket
+
+ * previous
+ * Page 50 of 50

webscraper/webscraper/middlewares.py

@@ -0,0 +1,215 @@
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+## middlewares.py
+from scrapy import signals
+from itemadapter import is_item, ItemAdapter
+from urllib.parse import urlencode
+from random import randint
+import requests
+import base64
+
+
+class WebscraperSpiderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, or item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Request or item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info("Spider opened: %s" % spider.name)
+
+
+class WebscraperDownloaderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info("Spider opened: %s" % spider.name)
+
+
+class ScrapeOpsFakeUserAgentMiddleware:
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler.settings)
+
+    def __init__(self, settings):
+        self.scrapeops_api_key = settings.get('SCRAPEOPS_API_KEY')
+        self.scrapeops_endpoint = settings.get('SCRAPEOPS_FAKE_USER_AGENT_ENDPOINT', 'http://headers.scrapeops.io/v1/user-agents?') 
+        self.scrapeops_fake_user_agents_active = settings.get('SCRAPEOPS_FAKE_USER_AGENT_ENABLED', False)
+        self.scrapeops_num_results = settings.get('SCRAPEOPS_NUM_RESULTS')
+        self.headers_list = []
+        self._get_user_agents_list()
+        self._scrapeops_fake_user_agents_enabled()
+
+    def _get_user_agents_list(self):
+        payload = {'api_key': self.scrapeops_api_key}
+        if self.scrapeops_num_results is not None:
+            payload['num_results'] = self.scrapeops_num_results
+        response = requests.get(self.scrapeops_endpoint, params=urlencode(payload))
+        json_response = response.json()
+        self.user_agents_list = json_response.get('result', [])
+
+    def _get_random_user_agent(self):
+        random_index = randint(0, len(self.user_agents_list) - 1)
+        return self.user_agents_list[random_index]
+
+    def _scrapeops_fake_user_agents_enabled(self):
+        if self.scrapeops_api_key is None or self.scrapeops_api_key == '' or self.scrapeops_fake_user_agents_active == False:
+            self.scrapeops_fake_user_agents_active = False
+        else:
+            self.scrapeops_fake_user_agents_active = True
+    
+    def process_request(self, request, spider):        
+        random_user_agent = self._get_random_user_agent()
+        request.headers['User-Agent'] = random_user_agent
+        #print("**************************User-Agent**************************")
+        #print(request.headers['User-Agent'])
+
+
+class ScrapeOpsFakeBrowserHeaderAgentMiddleware:
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler.settings)
+
+    def __init__(self, settings):
+        self.scrapeops_api_key = settings.get('SCRAPEOPS_API_KEY')
+        self.scrapeops_endpoint = settings.get('SCRAPEOPS_FAKE_BROWSER_HEADER_ENDPOINT', 'http://headers.scrapeops.io/v1/browser-headers?') 
+        self.scrapeops_fake_browser_headers_active = settings.get('SCRAPEOPS_FAKE_BROWSER_HEADER_ENABLED', False)
+        self.scrapeops_num_results = settings.get('SCRAPEOPS_NUM_RESULTS')
+        self.headers_list = []
+        self._get_headers_list()
+        self._scrapeops_fake_browser_headers_enabled()
+
+    def _get_headers_list(self):
+        payload = {'api_key': self.scrapeops_api_key}
+        if self.scrapeops_num_results is not None:
+            payload['num_results'] = self.scrapeops_num_results
+        response = requests.get(self.scrapeops_endpoint, params=urlencode(payload))
+        json_response = response.json()
+        self.headers_list = json_response.get('result', [])
+
+    def _get_random_browser_header(self):
+        random_index = randint(0, len(self.headers_list) - 1)
+        return self.headers_list[random_index]
+
+    def _scrapeops_fake_browser_headers_enabled(self):
+        if self.scrapeops_api_key is None or self.scrapeops_api_key == '' or self.scrapeops_fake_browser_headers_active == False:
+            self.scrapeops_fake_browser_headers_active = False
+        else:
+            self.scrapeops_fake_browser_headers_active = True
+    
+    def process_request(self, request, spider):        
+        random_browser_header = self._get_random_browser_header()
+        request.headers['upgrade-insecure-requests'] = random_browser_header['upgrade-insecure-requests']
+        request.headers['user-agent'] = random_browser_header['user-agent']
+        request.headers['accept'] = random_browser_header['accept']
+        request.headers['sec-ch-ua'] = random_browser_header['sec-ch-ua']
+        request.headers['sec-ch-ua-mobile'] = random_browser_header['sec-ch-ua-mobile']
+        request.headers['sec-ch-ua-platform'] = random_browser_header['sec-ch-ua-platform']
+        request.headers['sec-fetch-site'] = random_browser_header['sec-fetch-site']
+        request.headers['sec-fetch-mod'] = random_browser_header['sec-fetch-mod']
+        request.headers['sec-fetch-user'] = random_browser_header['sec-fetch-user']
+        request.headers['accept-encoding'] = random_browser_header['accept-encoding']
+        request.headers['accept-language'] = random_browser_header['accept-language']
+        #print("**************************User-Agent**************************")
+        #print(request.headers)
+
+
+class MyProxyMiddleware(object):
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler.settings)
+
+    def __init__(self, settings):
+        self.user = settings.get('PROXY_USER')
+        self.password = settings.get('PROXY_PASSWORD')
+        self.endpoint = settings.get('PROXY_ENDPOINT')
+        self.port = settings.get('PROXY_PORT')
+
+    def process_request(self, request, spider):
+        user_credentials = '{user}:{passw}'.format(user=self.user, passw=self.password)
+        basic_authentication = 'Basic ' + base64.b64encode(user_credentials.encode()).decode()
+        host = 'http://{endpoint}:{port}'.format(endpoint=self.endpoint, port=self.port)
+        request.meta['proxy'] = host
+        request.headers['Proxy-Authorization'] = basic_authentication

webscraper/webscraper/pipelines.py

@@ -0,0 +1,109 @@
+import os
+import html2text
+from bs4 import BeautifulSoup
+import re
+
+class HtmlToMarkdownPipeline:
+    def __init__(self):
+        # Initialize the HTML to Markdown converter
+        self.converter = html2text.HTML2Text()
+        self.converter.ignore_links = True
+        self.converter.ignore_images = True
+        self.converter.body_width = 0  # Prevents line wrapping for clean markdown
+        
+        # Ensure the directory for saving Markdown files exists
+        self.output_dir = 'markdown_files'
+        if not os.path.exists(self.output_dir):
+            os.makedirs(self.output_dir)
+
+    def remove_unwanted_tags(self, html_content):
+        """
+        Remove <header>, <footer>, <script>, and <style> tags from the HTML content.
+        """
+        # Parse HTML content
+        soup = BeautifulSoup(html_content, 'html.parser')
+        
+        # Remove <header> and <footer> tags
+        for tag in soup(['header', 'footer']):
+            tag.decompose()
+        
+        # Remove <script> and <style> tags
+        for script_or_style in soup(["script", "style"]):
+            script_or_style.decompose()
+        
+        # Return the cleaned HTML as a string
+        return str(soup)
+
+    def remove_emojis(self, text):
+        """
+        Remove all emojis from the text.
+        """
+        # Regex pattern to match emojis
+        emoji_pattern = re.compile(
+            "["  
+            "\U0001F600-\U0001F64F"  # Emoticons
+            "\U0001F300-\U0001F5FF"  # Symbols & Pictographs
+            "\U0001F680-\U0001F6FF"  # Transport & Map Symbols
+            "\U0001F700-\U0001F77F"  # Alchemical Symbols
+            "\U0001F780-\U0001F7FF"  # Geometric Shapes Extended
+            "\U0001F800-\U0001F8FF"  # Supplemental Arrows Extended
+            "\U0001F900-\U0001F9FF"  # Supplemental Symbols and Pictographs
+            "\U0001FA00-\U0001FA6F"  # Chess Symbols
+            "\U0001FA70-\U0001FAFF"  # Symbols and Pictographs Extended-A
+            "\U00002702-\U000027B0"  # Dingbats
+            "\U000024C2-\U0001F251"  # Enclosed Characters
+            "]+", 
+            flags=re.UNICODE
+        )
+        return emoji_pattern.sub(r'', text)
+
+    def clean_markdown(self, markdown_content):
+        """
+        Clean the markdown content to ensure it's well-structured and readable.
+        """
+        # Remove leading/trailing spaces
+        cleaned_content = markdown_content.strip()
+        
+        # Remove excessive line breaks by replacing multiple consecutive newlines with a single newline
+        cleaned_content = re.sub(r'\n{3,}', '\n\n', cleaned_content)
+        
+        # Remove carriage return characters
+        cleaned_content = cleaned_content.replace('\r', '')
+        
+        # Remove consecutive spaces
+        cleaned_content = re.sub(r' {2,}', ' ', cleaned_content)
+        
+        # Remove non-ASCII characters
+        cleaned_content = re.sub(r'[^\x00-\x7F]+', '', cleaned_content)
+        
+        return cleaned_content
+
+    def process_item(self, item, spider):
+        # Get the HTML content from the item
+        html_content = item.get('extracted_content', '')
+        
+        # Remove unwanted tags from HTML content
+        if html_content:
+            html_content = self.remove_unwanted_tags(html_content)
+            
+            # Remove emojis from the HTML content
+            html_content = self.remove_emojis(html_content)
+            
+            # Convert HTML to Markdown
+            markdown_content = self.converter.handle(html_content)
+
+            # Clean and structure the markdown content
+            markdown_content = self.clean_markdown(markdown_content)
+
+            # Save Markdown content to file
+            title = item.get('title', 'default')
+            file_name = f"{title}.md"
+            file_path = os.path.join(self.output_dir, file_name)
+            with open(file_path, 'w', encoding='utf-8') as file:
+                file.write(markdown_content)
+            
+            # Optionally, update item to include file path or Markdown content
+            item['extracted_content'] = markdown_content
+        
+        # Return the updated item
+        return item

webscraper/webscraper/settings.py

@@ -0,0 +1,49 @@
+FEEDS ={'scraped.json' : {'format' : 'json'}}
+
+
+#For SCRAPEOPS
+SCRAPEOPS_API_KEY = '8857a1e3-3e44-428f-8809-d6028ba24f0f'
+#SCRAPEOPS_FAKE_USER_AGENT_ENDPOINT = "https://headers.scrapeops.io/v1/user-agents"
+SCRAPEOPS_FAKE_BROWSER_HEADER_ENDPOINT = "https://headers.scrapeops.io/v1/browser-headers"
+#SCRAPEOPS_FAKE_USER_AGENT_ENABLED  = True
+SCRAPEOPS_FAKE_BROWSER_HEADER_ENABLED = True
+SCRAPEOPS_NUM_RESULTS = 50
+
+
+#For PROXY
+PROXY_USER = 'your_proxy_user_here'
+PROXY_PASSWORD = 'your_proxy_password_here'
+PROXY_ENDPOINT = 'your_proxy_endpoint_here'
+PROXY_PORT = 'your_proxy_port_here'
+
+
+#User agent is one part of header
+#If we want only user agent to change use ScrapeOpsFakeUserAgentMiddleware
+#But if you want to change complete user agent  Use ScrapeOpsFakeBrowserHeaderAgentMiddleware
+#DOWNLOADER_MIDDLEWARES = {
+#    "webscraper.middlewares.MyProxyMiddleware": 300,
+#    "webscraper.middlewares.HttpProxyMiddleware": 350,
+#    "webscraper.middlewares.ScrapeOpsFakeUserAgentMiddleware": 400,
+#    "webscraper.middlewares.ScrapeOpsFakeBrowserHeaderAgentMiddleware": 400,
+#}
+
+# Obey robots.txt rules
+# True - If we need to follow the rules regarding whether the website allows us to scrape
+# False - If we do not want to follow the rules regarding whether the website allows us to scrape
+ROBOTSTXT_OBEY = True
+
+
+ITEM_PIPELINES = {
+    "webscraper.pipelines.HtmlToMarkdownPipeline": 300,
+}
+
+
+BOT_NAME = "webscraper"
+
+SPIDER_MODULES = ["webscraper.spiders"]
+NEWSPIDER_MODULE = "webscraper.spiders"
+
+# Set settings whose default value is deprecated to a future-proof value
+REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
+TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+FEED_EXPORT_ENCODING = "utf-8"

webscraper/webscraper/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

webscraper/webscraper/spiders/websiteSpider.py

@@ -0,0 +1,43 @@
+import scrapy
+from webscraper.items import WebscraperItem
+from urllib.parse import urlparse
+
+class WebsiteSpider(scrapy.Spider):
+    name = "websiteSpider"
+    start_urls = ["https://books.toscrape.com/"]
+    allowed_domains = ["books.toscrape.com"]
+    custom_xpath = "/html/body/div/div/div/div/section/div[2]"
+    
+    custom_settings = {
+        'FEEDS': {
+            'scraped.json': {'format': 'json', 'overwrite': True},
+        }
+    }
+
+    def start_requests(self):
+        # Generate requests from the start_urls
+        for url in self.start_urls:
+            yield scrapy.Request(url=url, callback=self.parse)
+
+    def parse(self, response):
+        # Create an instance of WebscraperItem
+        web_item = WebscraperItem()
+
+        # Extract content based on the user's custom XPath
+        extracted_content = response.xpath(self.custom_xpath).get()
+        
+        if(extracted_content):
+            print(extracted_content)
+
+        # Assign the content and URL to the item
+        web_item['url'] = response.url
+        web_item['extracted_content'] = extracted_content
+
+        # Yield the item to the Scrapy pipeline
+        yield web_item
+
+        # Extract and follow links to other pages on the website
+        for href in response.xpath('//a/@href').extract():
+            # Build the absolute URL
+            url = response.urljoin(href)
+            yield scrapy.Request(url, callback=self.parse)