params.cfg

app/main.py

@@ -17,6 +17,11 @@ import PyPDF2
 from docx import Document as DocxDocument
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 
+# Local imports
+from .utils import getconfig
+
+config = getconfig("params.cfg")
+
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
@@ -86,12 +91,18 @@ def clean_and_chunk_text(text: str, doc_id: str) -> List[DocumentChunk]:
     text = re.sub(r'\s+', ' ', text)
     text = text.strip()
     
+    # Get chunking parameters from config
+    chunk_size = config.getint('chunking', 'chunk_size', fallback=700)
+    chunk_overlap = config.getint('chunking', 'chunk_overlap', fallback=50)
+    separators_str = config.get('chunking', 'separators', fallback='\n\n,\n,. ,! ,? , ,')
+    separators = [s.strip() for s in separators_str.split(',')]
+    
     # Split text into chunks
     text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=700,
-        chunk_overlap=50,
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
         length_function=len,
-        separators=["\n\n", "\n", ". ", "! ", "? ", " ", ""],
+        separators=separators,
         is_separator_regex=False
     )
     

app/utils.py

@@ -0,0 +1,18 @@
+import os
+import configparser
+import logging
+
+
+def getconfig(configfile_path: str):
+    """
+    Read the config file
+    Params
+    ----------------
+    configfile_path: file path of .cfg file
+    """
+    config = configparser.ConfigParser()
+    try:
+        config.read_file(open(configfile_path))
+        return config
+    except:
+        logging.warning("config file not found")

params.cfg

@@ -0,0 +1,7 @@
+[chunking]
+# Size of each text chunk in characters
+chunk_size = 700
+# Overlap between consecutive chunks in characters
+chunk_overlap = 50
+# Text separators for splitting, comma-separated (order of preference)
+separators = \n\n,\n,. ,! ,? , ,