Update app.py

app.py

@@ -25,19 +25,19 @@ def extract_separators_from_string(separators_str):
         Please type it in the correct format: "['separator_1', 'separator_2', etc]"
         """)
 
-def change_split_selection(text, slider_count, split_selection, separator_selection, length_unit_selection):
+def change_split_selection(text, slider_count, split_selection, separator_selection, length_unit_selection, chunk_overlap):
     return (
         gr.Textbox.update(visible=(split_selection==LABEL_RECURSIVE)),
-        chunk(text, slider_count, split_selection, separator_selection, length_unit_selection)
+        chunk(text, slider_count, split_selection, separator_selection, length_unit_selection, chunk_overlap)
     )
 
-def chunk(text, length, splitter_selection, separators_str, length_unit_selection):
+def chunk(text, length, splitter_selection, separators_str, length_unit_selection, chunk_overlap):
     separators = extract_separators_from_string(separators_str)
     length_function = (length_tokens if "token" in length_unit_selection.lower() else len)
     if splitter_selection == LABEL_TEXTSPLITTER:
         text_splitter = CharacterTextSplitter(
             chunk_size=length,
-            chunk_overlap=0,
+            chunk_overlap=chunk_overlap,
             length_function=length_function,
             strip_whitespace=False,
             is_separator_regex=False,
@@ -46,7 +46,7 @@ def chunk(text, length, splitter_selection, separators_str, length_unit_selectio
     elif splitter_selection == LABEL_RECURSIVE:
         text_splitter = RecursiveCharacterTextSplitter(
             chunk_size=length,
-            chunk_overlap=0,
+            chunk_overlap=chunk_overlap,
             length_function=length_function,
             strip_whitespace=False,
             separators=separators,
@@ -54,7 +54,9 @@ def chunk(text, length, splitter_selection, separators_str, length_unit_selectio
     splits = text_splitter.create_documents([text])
     text_splits = [split.page_content for split in splits]
 
-    output = [(split, str(i)) for i, split in enumerate(text_splits)]
+    unoverlapped_text_splits = unoverlap_list(text_splits)
+
+    output = [((split[0], 0) if split[1] else (split[0], str(i+1))) for i, split in enumerate(unoverlapped_text_splits)]
     return output
 
 
@@ -134,6 +136,9 @@ with gr.Blocks(theme=gr.themes.Soft(text_size='lg', font=["monospace"], primary_
         slider_count = gr.Slider(
             20, 500, value=200, label="Chunk length 📏", info="In the chosen unit."
         )
+        chunk_overlap = gr.Slider(
+            0, 30, value=10, label="Overlap between chunks", info="In the chosen unit."
+        )
     out = gr.HighlightedText(
         label="Output",
         show_legend=True,
@@ -141,22 +146,27 @@ with gr.Blocks(theme=gr.themes.Soft(text_size='lg', font=["monospace"], primary_
     )
     text.change(
         fn=chunk,
-        inputs=[text, slider_count, split_selection, separator_selection, length_unit_selection],
+        inputs=[text, slider_count, split_selection, separator_selection, length_unit_selection, chunk_overlap],
         outputs=out,
     )
     length_unit_selection.change(
         fn=chunk,
-        inputs=[text, slider_count, split_selection, separator_selection, length_unit_selection],
+        inputs=[text, slider_count, split_selection, separator_selection, length_unit_selection, chunk_overlap],
         outputs=out,
     )
     split_selection.change(
         fn=change_split_selection,
-        inputs=[text, slider_count, split_selection, separator_selection, length_unit_selection],
+        inputs=[text, slider_count, split_selection, separator_selection, length_unit_selection, chunk_overlap],
         outputs=[separator_selection, out],
     )
     slider_count.change(
         fn=chunk,
-        inputs=[text, slider_count, split_selection, separator_selection, length_unit_selection],
+        inputs=[text, slider_count, split_selection, separator_selection, length_unit_selection, chunk_overlap],
+        outputs=out,
+    )
+    chunk_overlap.change(
+        fn=chunk,
+        inputs=[text, slider_count, split_selection, separator_selection, length_unit_selection, chunk_overlap],
         outputs=out,
     )
 demo.launch()