core/text_processors.py

from langchain.text_splitter import (
    RecursiveCharacterTextSplitter,
    MarkdownHeaderTextSplitter
)

recursive_splitter = RecursiveCharacterTextSplitter(
        chunk_size=3500,
        chunk_overlap=400,
        length_function=len,
        separators=["\n\n", "\n", ". ", " ", ""],
    )


markdown_splitter = MarkdownHeaderTextSplitter(
    headers_to_split_on=[
        ("##", "Header 2"),   
        ("###", "Header 3"),  
    ],
    strip_headers=False,
)