feat: Enhance blog data processing with utility functions

- Added utility functions for loading and processing blog posts in `utils_data_loading.ipynb`.
- Implemented a new notebook `update_blog_data.ipynb` for updating blog data and vector store.
- Updated `app.py` to utilize utility functions for loading vector stores, with fallback to direct initialization.
- Improved error handling and logging during vector store loading.
- Added new dependencies in `pyproject.toml` for notebook processing and utilities.
- Documented the new utilities and usage instructions in `BLOG_DATA_UTILS.md`.

BLOG_DATA_UTILS.md

@@ -0,0 +1,72 @@
+# Blog Data Utilities
+
+This directory contains utilities for loading, processing, and maintaining blog post data for the RAG system.
+
+## Available Tools
+
+### `utils_data_loading.ipynb`
+
+This notebook contains utility functions for:
+- Loading blog posts from the data directory
+- Processing and enriching metadata (adding URLs, titles, etc.)
+- Getting statistics about the documents
+- Creating and updating vector embeddings
+- Loading existing vector stores
+
+### `update_blog_data.ipynb`
+
+This notebook demonstrates how to:
+- Use the utility functions to update the blog data
+- Process new blog posts
+- Update the vector store
+- Test the updated system with sample queries
+- Track changes over time
+
+## How to Use
+
+### Updating Blog Data
+
+When new blog posts are published, follow these steps:
+
+1. Add the markdown files to the `data/` directory
+2. Run the update notebook:
+   ```bash
+   cd /home/mafzaal/source/lets-talk
+   uv run jupyter nbconvert --to notebook --execute update_blog_data.ipynb --output executed_update_$(date +%Y%m%d).ipynb
+   ```
+   
+This will:
+- Load all blog posts (including new ones)
+- Update the vector embeddings
+- Save statistics for tracking
+
+### Customizing the Process
+
+You can customize the process by editing the `.env` file:
+
+```
+DATA_DIR=data/                             # Directory containing blog posts
+VECTOR_STORAGE_PATH=./db/vectorstore_v3    # Path to vector store
+EMBEDDING_MODEL=Snowflake/snowflake-arctic-embed-l  # Embedding model
+QDRANT_COLLECTION=thedataguy_documents     # Collection name
+BLOG_BASE_URL=https://thedataguy.pro/blog/ # Base URL for blog
+FORCE_RECREATE_EMBEDDINGS=false           # Whether to force recreation
+```
+
+### In the Chainlit App
+
+The Chainlit app (`app.py`) has been updated to use these utility functions if available. It falls back to direct initialization if they can't be loaded.
+
+## Adding Custom Processing
+
+To add custom processing for blog posts:
+
+1. Edit the `update_document_metadata` function in `utils_data_loading.ipynb`
+2. Add any additional enrichment or processing steps
+3. Update the vector store using the `update_blog_data.ipynb` notebook
+
+## Future Improvements
+
+- Add support for incremental updates (only process new posts)
+- Add webhook support to automatically update when new posts are published
+- Add tracking of embedding models and versions

app.py

@@ -1,5 +1,7 @@
 import os
 import getpass
+import sys
+import importlib.util
 from pathlib import Path
 from operator import itemgetter
 from dotenv import load_dotenv
@@ -16,23 +18,68 @@ from langchain_qdrant import QdrantVectorStore
 from qdrant_client import QdrantClient
 from qdrant_client.http.models import Distance, VectorParams
 
-# Get vector storage path from .env file with fallback
-storage_path = Path(os.environ.get("VECTOR_STORAGE_PATH", "./db/vectorstore_v3"))
-#qclient = QdrantClient(storage_path)
-
-# Load embedding model from environment variable with fallback
-embedding_model = os.environ.get("EMBEDDING_MODEL", "Snowflake/snowflake-arctic-embed-l")
-huggingface_embeddings = HuggingFaceEmbeddings(model_name=embedding_model)
-
-# Set up Qdrant vectorstore from existing collection
-collection_name = os.environ.get("QDRANT_COLLECTION", "thedataguy_documents")
+# Import utility functions from the notebook
+def import_notebook_functions(notebook_path):
+    """Import functions from a Jupyter notebook"""
+    import nbformat
+    from importlib.util import spec_from_loader, module_from_spec
+    from IPython.core.interactiveshell import InteractiveShell
+    
+    # Create a module
+    module_name = Path(notebook_path).stem
+    spec = spec_from_loader(module_name, loader=None)
+    module = module_from_spec(spec)
+    sys.modules[module_name] = module
+    
+    # Read the notebook
+    with open(notebook_path) as f:
+        nb = nbformat.read(f, as_version=4)
+    
+    # Execute code cells
+    shell = InteractiveShell.instance()
+    for cell in nb.cells:
+        if cell.cell_type == 'code':
+            # Skip example code
+            if 'if __name__ == "__main__":' in cell.source:
+                continue
+                
+            code = shell.input_transformer_manager.transform_cell(cell.source)
+            exec(code, module.__dict__)
+    
+    return module
 
-vector_store = QdrantVectorStore.from_existing_collection(
-    #client=qclient,
-    path=storage_path,
-    collection_name=collection_name,
-    embedding=huggingface_embeddings,
-)
+# Try to import utility functions if available
+try:
+    utils = import_notebook_functions('utils_data_loading.ipynb')
+    
+    # Load vector store using the utility function
+    vector_store = utils.load_vector_store(
+        storage_path=os.environ.get("VECTOR_STORAGE_PATH", "./db/vectorstore_v3"),
+        collection_name=os.environ.get("QDRANT_COLLECTION", "thedataguy_documents"),
+        embedding_model=os.environ.get("EMBEDDING_MODEL", "Snowflake/snowflake-arctic-embed-l")
+    )
+    
+    print("Successfully loaded vector store using utility functions")
+    
+except Exception as e:
+    print(f"Could not load utility functions: {e}")
+    print("Falling back to direct initialization")
+    
+    # Get vector storage path from .env file with fallback
+    storage_path = Path(os.environ.get("VECTOR_STORAGE_PATH", "./db/vectorstore_v3"))
+    
+    # Load embedding model from environment variable with fallback
+    embedding_model = os.environ.get("EMBEDDING_MODEL", "Snowflake/snowflake-arctic-embed-l")
+    huggingface_embeddings = HuggingFaceEmbeddings(model_name=embedding_model)
+    
+    # Set up Qdrant vectorstore from existing collection
+    collection_name = os.environ.get("QDRANT_COLLECTION", "thedataguy_documents")
+    
+    vector_store = QdrantVectorStore.from_existing_collection(
+        path=storage_path,
+        collection_name=collection_name,
+        embedding=huggingface_embeddings,
+    )
 
 
 # Create a retriever
@@ -97,14 +144,29 @@ async def on_message(message: cl.Message):
     # Get chain from user session
     chain = cl.user_session.get("chain")
     
-    print( message.content)
+    print(message.content)
     # Call the chain with the user message
-    response =  chain.invoke({"question": message.content})
-   
+    response = chain.invoke({"question": message.content})
+    
+    # Get the sources to display them
+    sources = []
+    for doc in response["context"]:
+        if "url" in doc.metadata:
+            # Get title from post_title metadata if available, otherwise derive from URL
+            title = doc.metadata.get("post_title", "")
+            if not title:
+                title = doc.metadata["url"].split("/")[-2].replace("-", " ").title()
+                
+            sources.append(
+                cl.Source(
+                    url=doc.metadata["url"],
+                    title=title
+                )
+            )
 
     # Send the response with sources
     await cl.Message(
         content=response["response"].content,
-
+        sources=sources
     ).send()
 

pyproject.toml

@@ -7,6 +7,7 @@ requires-python = ">=3.13"
 dependencies = [
     "chainlit>=2.5.5",
     "ipykernel>=6.29.5",
+    "ipython>=9.2.0",
     "langchain>=0.3.25",
     "langchain-community>=0.3.23",
     "langchain-core>=0.3.59",
@@ -14,6 +15,7 @@ dependencies = [
     "langchain-openai>=0.3.16",
     "langchain-qdrant>=0.2.0",
     "langchain-text-splitters>=0.3.8",
+    "nbformat>=5.10.4",
     "pandas>=2.2.3",
     "python-dotenv>=1.1.0",
     "qdrant-client>=1.14.2",

update_blog_data.ipynb

@@ -0,0 +1,256 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "b1a955e7",
+   "metadata": {},
+   "source": [
+    "# Update Blog Data\n",
+    "\n",
+    "This notebook demonstrates how to update the blog data and vector store when new blog posts are published. It uses the utility functions from `utils_data_loading.ipynb`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6ec048b4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "from dotenv import load_dotenv\n",
+    "import importlib.util\n",
+    "\n",
+    "# Load environment variables\n",
+    "load_dotenv()\n",
+    "\n",
+    "# Import utility functions from utils_data_loading.ipynb\n",
+    "# We'll do this by first converting the notebook to a Python module"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7f01d61f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Function to import the utility module\n",
+    "def import_notebook_as_module(notebook_path, module_name=\"utils_module\"):\n",
+    "    \"\"\"\n",
+    "    Import a Jupyter notebook as a Python module.\n",
+    "    \n",
+    "    Args:\n",
+    "        notebook_path: Path to the notebook\n",
+    "        module_name: Name to give the module\n",
+    "        \n",
+    "    Returns:\n",
+    "        The imported module\n",
+    "    \"\"\"\n",
+    "    import nbformat\n",
+    "    from importlib.util import spec_from_loader, module_from_spec\n",
+    "    from IPython.core.interactiveshell import InteractiveShell\n",
+    "    \n",
+    "    shell = InteractiveShell.instance()\n",
+    "    \n",
+    "    with open(notebook_path) as f:\n",
+    "        nb = nbformat.read(f, as_version=4)\n",
+    "    \n",
+    "    # Create a module\n",
+    "    spec = spec_from_loader(module_name, loader=None)\n",
+    "    module = module_from_spec(spec)\n",
+    "    sys.modules[module_name] = module\n",
+    "    \n",
+    "    # Execute only the code cells in the notebook\n",
+    "    for cell in nb.cells:\n",
+    "        if cell.cell_type == 'code':\n",
+    "            # Skip cells that start with certain keywords like \"if __name__ == \"__main__\":\"\n",
+    "            if 'if __name__ == \"__main__\":' in cell.source:\n",
+    "                continue\n",
+    "            \n",
+    "            # Execute the cell and store its content in the module\n",
+    "            code = shell.input_transformer_manager.transform_cell(cell.source)\n",
+    "            exec(code, module.__dict__)\n",
+    "    \n",
+    "    return module"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "774c1373",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Import the utility functions\n",
+    "utils = import_notebook_as_module('utils_data_loading.ipynb')\n",
+    "\n",
+    "# Now you can access all the functions from the utils module\n",
+    "print(\"Successfully imported utility functions.\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "85ae6617",
+   "metadata": {},
+   "source": [
+    "## Configuration\n",
+    "\n",
+    "Set up the configuration for data processing."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "54e9ca48",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Configuration (can be overridden from .env file)\n",
+    "DATA_DIR = os.environ.get(\"DATA_DIR\", \"data/\")\n",
+    "VECTOR_STORAGE_PATH = os.environ.get(\"VECTOR_STORAGE_PATH\", \"./db/vectorstore_v3\")\n",
+    "BLOG_BASE_URL = os.environ.get(\"BLOG_BASE_URL\", \"https://thedataguy.pro/blog/\")\n",
+    "FORCE_RECREATE_EMBEDDINGS = os.environ.get(\"FORCE_RECREATE_EMBEDDINGS\", \"false\").lower() == \"true\"\n",
+    "\n",
+    "print(f\"Data Directory: {DATA_DIR}\")\n",
+    "print(f\"Vector Storage Path: {VECTOR_STORAGE_PATH}\")\n",
+    "print(f\"Blog Base URL: {BLOG_BASE_URL}\")\n",
+    "print(f\"Force Recreate Embeddings: {FORCE_RECREATE_EMBEDDINGS}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cc19ab4c",
+   "metadata": {},
+   "source": [
+    "## Update Blog Data Process\n",
+    "\n",
+    "This process will:\n",
+    "1. Load existing blog posts\n",
+    "2. Process and update metadata\n",
+    "3. Create or update vector embeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3d56f688",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Process blog posts and create/update embeddings\n",
+    "result = utils.process_blog_posts(\n",
+    "    data_dir=DATA_DIR,\n",
+    "    create_embeddings=True,\n",
+    "    force_recreate_embeddings=FORCE_RECREATE_EMBEDDINGS\n",
+    ")\n",
+    "\n",
+    "# Access the documents and vector store\n",
+    "documents = result[\"documents\"]\n",
+    "stats = result[\"stats\"]\n",
+    "vector_store = result[\"vector_store\"]\n",
+    "\n",
+    "print(f\"\\nProcessed {len(documents)} blog posts\")\n",
+    "print(f\"Vector store created/updated at: {VECTOR_STORAGE_PATH}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ad3b2dca",
+   "metadata": {},
+   "source": [
+    "## Testing the Vector Store\n",
+    "\n",
+    "Let's test the vector store with a few queries to make sure it's working correctly."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8b552e6b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create a retriever from the vector store\n",
+    "retriever = vector_store.as_retriever(search_kwargs={\"k\": 2})\n",
+    "\n",
+    "# Test queries\n",
+    "test_queries = [\n",
+    "    \"What is RAGAS?\",\n",
+    "    \"How to build research agents?\",\n",
+    "    \"What is metric driven development?\",\n",
+    "    \"Who is TheDataGuy?\"\n",
+    "]\n",
+    "\n",
+    "for query in test_queries:\n",
+    "    print(f\"\\nQuery: {query}\")\n",
+    "    docs = retriever.invoke(query)\n",
+    "    print(f\"Retrieved {len(docs)} documents:\")\n",
+    "    for i, doc in enumerate(docs):\n",
+    "        title = doc.metadata.get(\"post_title\", \"Unknown\")\n",
+    "        url = doc.metadata.get(\"url\", \"No URL\")\n",
+    "        print(f\"{i+1}. {title} ({url})\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ddbe9282",
+   "metadata": {},
+   "source": [
+    "## Schedule This Notebook\n",
+    "\n",
+    "To keep the blog data up-to-date, you can schedule this notebook to run periodically. \n",
+    "Here are some options:\n",
+    "\n",
+    "1. Use a cron job to run this notebook with papermill\n",
+    "2. Set up a GitHub Action to run this notebook on a schedule\n",
+    "3. Use Airflow or another workflow management system\n",
+    "\n",
+    "Example of running with papermill:\n",
+    "```bash\n",
+    "papermill update_blog_data.ipynb output_$(date +%Y%m%d).ipynb\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3634e064",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Save stats to a file for tracking changes over time\n",
+    "import json\n",
+    "from datetime import datetime\n",
+    "\n",
+    "stats_dir = Path(\"stats\")\n",
+    "stats_dir.mkdir(exist_ok=True)\n",
+    "\n",
+    "# Add timestamp to stats\n",
+    "stats[\"timestamp\"] = datetime.now().isoformat()\n",
+    "\n",
+    "# Save stats\n",
+    "stats_path = stats_dir / f\"blog_stats_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json\"\n",
+    "with open(stats_path, \"w\") as f:\n",
+    "    json.dump(stats, f, indent=2)\n",
+    "\n",
+    "print(f\"Saved stats to {stats_path}\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.13.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

utils_data_loading.ipynb

@@ -0,0 +1,454 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "b31c2849",
+   "metadata": {},
+   "source": [
+    "# Utility Functions for Blog Post Loading and Processing\n",
+    "\n",
+    "This notebook contains utility functions for loading blog posts from the data directory, processing their metadata, and creating vector embeddings for use in the RAG system."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "848b0a86",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import json\n",
+    "from pathlib import Path\n",
+    "from typing import List, Dict, Any, Optional\n",
+    "\n",
+    "from langchain_community.document_loaders import DirectoryLoader\n",
+    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "from langchain.schema.document import Document\n",
+    "from langchain_huggingface import HuggingFaceEmbeddings\n",
+    "from langchain_community.vectorstores import Qdrant\n",
+    "\n",
+    "from IPython.display import Markdown, display\n",
+    "from dotenv import load_dotenv\n",
+    "\n",
+    "# Load environment variables from .env file\n",
+    "load_dotenv()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "39e32435",
+   "metadata": {},
+   "source": [
+    "## Configuration\n",
+    "\n",
+    "Load configuration from environment variables or use defaults."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5a6a5d6d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Configuration with defaults\n",
+    "DATA_DIR = os.environ.get(\"DATA_DIR\", \"data/\")\n",
+    "VECTOR_STORAGE_PATH = os.environ.get(\"VECTOR_STORAGE_PATH\", \"./db/vectorstore_v3\")\n",
+    "EMBEDDING_MODEL = os.environ.get(\"EMBEDDING_MODEL\", \"Snowflake/snowflake-arctic-embed-l\")\n",
+    "QDRANT_COLLECTION = os.environ.get(\"QDRANT_COLLECTION\", \"thedataguy_documents\")\n",
+    "BLOG_BASE_URL = os.environ.get(\"BLOG_BASE_URL\", \"https://thedataguy.pro/blog/\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "01454147",
+   "metadata": {},
+   "source": [
+    "## Utility Functions\n",
+    "\n",
+    "These functions handle the loading, processing, and storing of blog posts."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "25792cd5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def load_blog_posts(data_dir: str = DATA_DIR, \n",
+    "                   glob_pattern: str = \"*.md\", \n",
+    "                   recursive: bool = True, \n",
+    "                   show_progress: bool = True) -> List[Document]:\n",
+    "    \"\"\"\n",
+    "    Load blog posts from the specified directory.\n",
+    "    \n",
+    "    Args:\n",
+    "        data_dir: Directory containing the blog posts\n",
+    "        glob_pattern: Pattern to match files\n",
+    "        recursive: Whether to search subdirectories\n",
+    "        show_progress: Whether to show a progress bar\n",
+    "        \n",
+    "    Returns:\n",
+    "        List of Document objects containing the blog posts\n",
+    "    \"\"\"\n",
+    "    text_loader = DirectoryLoader(\n",
+    "        data_dir, \n",
+    "        glob=glob_pattern, \n",
+    "        show_progress=show_progress,\n",
+    "        recursive=recursive\n",
+    "    )\n",
+    "    \n",
+    "    documents = text_loader.load()\n",
+    "    print(f\"Loaded {len(documents)} documents from {data_dir}\")\n",
+    "    return documents"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e7ddba72",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def update_document_metadata(documents: List[Document], \n",
+    "                           data_dir_prefix: str = DATA_DIR,\n",
+    "                           blog_base_url: str = BLOG_BASE_URL,\n",
+    "                           remove_suffix: str = \"index.md\") -> List[Document]:\n",
+    "    \"\"\"\n",
+    "    Update the metadata of documents to include URL and other information.\n",
+    "    \n",
+    "    Args:\n",
+    "        documents: List of Document objects to update\n",
+    "        data_dir_prefix: Prefix to replace in source paths\n",
+    "        blog_base_url: Base URL for the blog posts\n",
+    "        remove_suffix: Suffix to remove from paths (like index.md)\n",
+    "        \n",
+    "    Returns:\n",
+    "        Updated list of Document objects\n",
+    "    \"\"\"\n",
+    "    for doc in documents:\n",
+    "        # Create URL from source path\n",
+    "        doc.metadata[\"url\"] = doc.metadata[\"source\"].replace(data_dir_prefix, blog_base_url)\n",
+    "        \n",
+    "        # Remove index.md or other suffix if present\n",
+    "        if remove_suffix and doc.metadata[\"url\"].endswith(remove_suffix):\n",
+    "            doc.metadata[\"url\"] = doc.metadata[\"url\"][:-len(remove_suffix)]\n",
+    "            \n",
+    "        # Extract post title from the directory structure\n",
+    "        path_parts = Path(doc.metadata[\"source\"]).parts\n",
+    "        if len(path_parts) > 1:\n",
+    "            # Use the directory name as post_slug\n",
+    "            doc.metadata[\"post_slug\"] = path_parts[-2]\n",
+    "            doc.metadata[\"post_title\"] = path_parts[-2].replace(\"-\", \" \").title()\n",
+    "            \n",
+    "        # Add document length as metadata\n",
+    "        doc.metadata[\"content_length\"] = len(doc.page_content)\n",
+    "    \n",
+    "    return documents"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e0dfe498",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_document_stats(documents: List[Document]) -> Dict[str, Any]:\n",
+    "    \"\"\"\n",
+    "    Get statistics about the documents.\n",
+    "    \n",
+    "    Args:\n",
+    "        documents: List of Document objects\n",
+    "        \n",
+    "    Returns:\n",
+    "        Dictionary with statistics\n",
+    "    \"\"\"\n",
+    "    stats = {\n",
+    "        \"total_documents\": len(documents),\n",
+    "        \"total_characters\": sum(len(doc.page_content) for doc in documents),\n",
+    "        \"min_length\": min(len(doc.page_content) for doc in documents),\n",
+    "        \"max_length\": max(len(doc.page_content) for doc in documents),\n",
+    "        \"avg_length\": sum(len(doc.page_content) for doc in documents) / len(documents) if documents else 0,\n",
+    "    }\n",
+    "    \n",
+    "    # Create a list of document info for analysis\n",
+    "    doc_info = []\n",
+    "    for doc in documents:\n",
+    "        doc_info.append({\n",
+    "            \"url\": doc.metadata.get(\"url\", \"\"),\n",
+    "            \"source\": doc.metadata.get(\"source\", \"\"),\n",
+    "            \"title\": doc.metadata.get(\"post_title\", \"\"),\n",
+    "            \"text_length\": doc.metadata.get(\"content_length\", 0),\n",
+    "        })\n",
+    "    \n",
+    "    stats[\"documents\"] = doc_info\n",
+    "    return stats"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0ae139c0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def display_document_stats(stats: Dict[str, Any]):\n",
+    "    \"\"\"\n",
+    "    Display document statistics in a readable format.\n",
+    "    \n",
+    "    Args:\n",
+    "        stats: Dictionary with statistics from get_document_stats\n",
+    "    \"\"\"\n",
+    "    print(f\"Total Documents: {stats['total_documents']}\")\n",
+    "    print(f\"Total Characters: {stats['total_characters']}\")\n",
+    "    print(f\"Min Length: {stats['min_length']} characters\")\n",
+    "    print(f\"Max Length: {stats['max_length']} characters\")\n",
+    "    print(f\"Average Length: {stats['avg_length']:.2f} characters\")\n",
+    "    \n",
+    "    # Display documents as a table\n",
+    "    import pandas as pd\n",
+    "    if stats[\"documents\"]:\n",
+    "        df = pd.DataFrame(stats[\"documents\"])\n",
+    "        display(df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2dcf66b4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def split_documents(documents: List[Document], \n",
+    "                   chunk_size: int = 1000, \n",
+    "                   chunk_overlap: int = 200) -> List[Document]:\n",
+    "    \"\"\"\n",
+    "    Split documents into chunks for better embedding and retrieval.\n",
+    "    \n",
+    "    Args:\n",
+    "        documents: List of Document objects to split\n",
+    "        chunk_size: Size of each chunk in characters\n",
+    "        chunk_overlap: Overlap between chunks in characters\n",
+    "        \n",
+    "    Returns:\n",
+    "        List of split Document objects\n",
+    "    \"\"\"\n",
+    "    text_splitter = RecursiveCharacterTextSplitter(\n",
+    "        chunk_size=chunk_size,\n",
+    "        chunk_overlap=chunk_overlap,\n",
+    "        length_function=len,\n",
+    "    )\n",
+    "    \n",
+    "    split_docs = text_splitter.split_documents(documents)\n",
+    "    print(f\"Split {len(documents)} documents into {len(split_docs)} chunks\")\n",
+    "    return split_docs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "527ad848",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def create_vector_store(documents: List[Document], \n",
+    "                       storage_path: str = VECTOR_STORAGE_PATH,\n",
+    "                       collection_name: str = QDRANT_COLLECTION,\n",
+    "                       embedding_model: str = EMBEDDING_MODEL,\n",
+    "                       force_recreate: bool = False) -> Qdrant:\n",
+    "    \"\"\"\n",
+    "    Create a vector store from documents.\n",
+    "    \n",
+    "    Args:\n",
+    "        documents: List of Document objects to store\n",
+    "        storage_path: Path to the vector store\n",
+    "        collection_name: Name of the collection\n",
+    "        embedding_model: Name of the embedding model\n",
+    "        force_recreate: Whether to force recreation of the vector store\n",
+    "        \n",
+    "    Returns:\n",
+    "        Qdrant vector store\n",
+    "    \"\"\"\n",
+    "    # Initialize the embedding model\n",
+    "    embeddings = HuggingFaceEmbeddings(model_name=embedding_model)\n",
+    "    \n",
+    "    # Create the directory if it doesn't exist\n",
+    "    storage_dir = Path(storage_path).parent\n",
+    "    os.makedirs(storage_dir, exist_ok=True)\n",
+    "    \n",
+    "    # Check if vector store exists\n",
+    "    vector_store_exists = Path(storage_path).exists() and not force_recreate\n",
+    "    \n",
+    "    if vector_store_exists:\n",
+    "        print(f\"Loading existing vector store from {storage_path}\")\n",
+    "        try:\n",
+    "            vector_store = Qdrant(\n",
+    "                path=storage_path,\n",
+    "                embedding_function=embeddings,\n",
+    "                collection_name=collection_name\n",
+    "            )\n",
+    "            return vector_store\n",
+    "        except Exception as e:\n",
+    "            print(f\"Error loading existing vector store: {e}\")\n",
+    "            print(\"Creating new vector store...\")\n",
+    "            force_recreate = True\n",
+    "    \n",
+    "    # Create new vector store\n",
+    "    print(f\"Creating new vector store at {storage_path}\")\n",
+    "    vector_store = Qdrant.from_documents(\n",
+    "        documents=documents,\n",
+    "        embedding=embeddings,\n",
+    "        path=storage_path,\n",
+    "        collection_name=collection_name,\n",
+    "    )\n",
+    "    \n",
+    "    return vector_store"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c78f99fc",
+   "metadata": {},
+   "source": [
+    "## Example Usage\n",
+    "\n",
+    "Here's how to use these utility functions for processing blog posts."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "132d32c6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def process_blog_posts(data_dir: str = DATA_DIR,\n",
+    "                      create_embeddings: bool = True,\n",
+    "                      force_recreate_embeddings: bool = False):\n",
+    "    \"\"\"\n",
+    "    Complete pipeline to process blog posts and optionally create vector embeddings.\n",
+    "    \n",
+    "    Args:\n",
+    "        data_dir: Directory containing the blog posts\n",
+    "        create_embeddings: Whether to create vector embeddings\n",
+    "        force_recreate_embeddings: Whether to force recreation of embeddings\n",
+    "        \n",
+    "    Returns:\n",
+    "        Dictionary with data and vector store (if created)\n",
+    "    \"\"\"\n",
+    "    # Load documents\n",
+    "    documents = load_blog_posts(data_dir)\n",
+    "    \n",
+    "    # Update metadata\n",
+    "    documents = update_document_metadata(documents)\n",
+    "    \n",
+    "    # Get and display stats\n",
+    "    stats = get_document_stats(documents)\n",
+    "    display_document_stats(stats)\n",
+    "    \n",
+    "    result = {\n",
+    "        \"documents\": documents,\n",
+    "        \"stats\": stats,\n",
+    "        \"vector_store\": None\n",
+    "    }\n",
+    "    \n",
+    "    # Create vector store if requested\n",
+    "    if create_embeddings:\n",
+    "        vector_store = create_vector_store(\n",
+    "            documents, \n",
+    "            force_recreate=force_recreate_embeddings\n",
+    "        )\n",
+    "        result[\"vector_store\"] = vector_store\n",
+    "    \n",
+    "    return result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "266d4fb3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Example usage\n",
+    "if __name__ == \"__main__\":\n",
+    "    # Process blog posts without creating embeddings\n",
+    "    result = process_blog_posts(create_embeddings=False)\n",
+    "    \n",
+    "    # Example: Access the documents\n",
+    "    print(f\"\\nDocument example: {result['documents'][0].metadata}\")\n",
+    "    \n",
+    "    # Create embeddings if needed\n",
+    "    # result = process_blog_posts(create_embeddings=True)\n",
+    "    \n",
+    "    # Retriever example\n",
+    "    # retriever = result[\"vector_store\"].as_retriever()\n",
+    "    # query = \"What is RAGAS?\"\n",
+    "    # docs = retriever.invoke(query, k=2)\n",
+    "    # print(f\"\\nRetrieved {len(docs)} documents for query: {query}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "22132649",
+   "metadata": {},
+   "source": [
+    "## Function for Loading Existing Vector Store\n",
+    "\n",
+    "This function can be used to load an existing vector store without reprocessing all blog posts."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c24e0c02",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def load_vector_store(storage_path: str = VECTOR_STORAGE_PATH,\n",
+    "                     collection_name: str = QDRANT_COLLECTION,\n",
+    "                     embedding_model: str = EMBEDDING_MODEL) -> Optional[Qdrant]:\n",
+    "    \"\"\"\n",
+    "    Load an existing vector store.\n",
+    "    \n",
+    "    Args:\n",
+    "        storage_path: Path to the vector store\n",
+    "        collection_name: Name of the collection\n",
+    "        embedding_model: Name of the embedding model\n",
+    "        \n",
+    "    Returns:\n",
+    "        Qdrant vector store or None if it doesn't exist\n",
+    "    \"\"\"\n",
+    "    # Initialize the embedding model\n",
+    "    embeddings = HuggingFaceEmbeddings(model_name=embedding_model)\n",
+    "    \n",
+    "    # Check if vector store exists\n",
+    "    if not Path(storage_path).exists():\n",
+    "        print(f\"Vector store not found at {storage_path}\")\n",
+    "        return None\n",
+    "    \n",
+    "    try:\n",
+    "        vector_store = Qdrant(\n",
+    "            path=storage_path,\n",
+    "            embedding_function=embeddings,\n",
+    "            collection_name=collection_name\n",
+    "        )\n",
+    "        print(f\"Loaded vector store from {storage_path}\")\n",
+    "        return vector_store\n",
+    "    except Exception as e:\n",
+    "        print(f\"Error loading vector store: {e}\")\n",
+    "        return None"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}