change style of chunking to combine between markdown and recursive

Readme.md

@@ -1,13 +1,12 @@
 ---
-title: RAG Chatbot Backend
-emoji: 🤖
-colorFrom: blue
-colorTo: purple
+title: FastAPI Backend ChatbotRAG
+emoji: 💻
+colorFrom: purple
+colorTo: yellow
 sdk: docker
 pinned: false
+license: mit
+short_description: This is backend of chatbotRAG project
 ---
 
-# RAG Chatbot Backend
-
-This is the FastAPI backend for my RAG chatbot.  
-It processes PDFs, stores embeddings in Pinecone, and answers queries using Groq + Gemini.
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

utils/chunking.py

@@ -1,13 +1,41 @@
-from langchain.text_splitter import MarkdownHeaderTextSplitter
+from langchain.text_splitter import MarkdownHeaderTextSplitter, RecursiveCharacterTextSplitter
 from langchain.schema import Document
+import tiktoken
 
-def split_text_by_markdown(input_md: str) -> list:
+def split_text_by_markdown(input_md: str, max_tokens: int = 2048, model: str = "cl100k_base") -> list:
+    # Step 1: Split by headers
     headers_to_split_on = [
         ("#", "Header 1"),
         ("##", "Header 2"),
         ("###", "Header 3"),
     ]
-    splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_to_split_on)
-    chunks = splitter.split_text(input_md)
-    documents = [Document(page_content=chunk.page_content, metadata=chunk.metadata) for chunk in chunks]
-    return documents
+    md_splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_to_split_on)
+    header_chunks = md_splitter.split_text(input_md)
+
+    # Step 2: Tokenizer (OpenAI/Groq style)
+    encoding = tiktoken.get_encoding(model)
+
+    # Step 3: For each header chunk, further split if it’s too long
+    final_docs = []
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=1000,    # characters per chunk (roughly ~500 tokens, safe buffer)
+        chunk_overlap=100   # overlap to preserve context
+    )
+
+    for chunk in header_chunks:
+        token_count = len(encoding.encode(chunk.page_content))
+
+        if token_count > max_tokens:
+            # Split into smaller parts
+            sub_chunks = text_splitter.split_text(chunk.page_content)
+            for sub in sub_chunks:
+                final_docs.append(
+                    Document(page_content=sub, metadata=chunk.metadata)
+                )
+        else:
+            # Keep as is
+            final_docs.append(
+                Document(page_content=chunk.page_content, metadata=chunk.metadata)
+            )
+
+    return final_docs