Update app.py

app.py

@@ -1,6 +1,4 @@
 import gradio as gr
-from haystack import Document
-from haystack.nodes import PreProcessor
 from langchain.text_splitter import (
     CharacterTextSplitter,
     RecursiveCharacterTextSplitter,
@@ -15,18 +13,7 @@ preprocessor = PreProcessor(
 )
 
 
-def chunk(text, words, splitter_selection):
-    if "Word" in splitter_selection:
-        splits = preprocessor.split(
-            Document(text),
-            split_length=words,
-            split_by="word",
-            split_overlap=0,
-            split_respect_sentence_boundary=(
-                "respect sentence boundaries" in splitter_selection
-            ),
-        )
-        text_splits = [split.content for split in splits]
+def chunk(text, length, splitter_selection):
     elif splitter_selection == "LangChain's CharacterTextSplitter":
         text_splitter = CharacterTextSplitter(
             separator="",
@@ -39,7 +26,7 @@ def chunk(text, words, splitter_selection):
         text_splits = [split.page_content for split in splits]
     elif splitter_selection == "Langchain's RecursiveCharacterTextSplitter - vanilla":
         text_splitter = RecursiveCharacterTextSplitter(
-            chunk_size=words,
+            chunk_size=length,
             chunk_overlap=0,
             length_function=len,
             add_start_index=True,
@@ -48,7 +35,7 @@ def chunk(text, words, splitter_selection):
         text_splits = [split.page_content for split in splits]
     elif splitter_selection == "Langchain's RecursiveCharacterTextSplitter - with '.'":
         text_splitter = RecursiveCharacterTextSplitter(
-            chunk_size=words,
+            chunk_size=length,
             chunk_overlap=0,
             length_function=len,
             add_start_index=True,
@@ -107,12 +94,21 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
             "LangChain's CharacterTextSplitter",
             "Langchain's RecursiveCharacterTextSplitter - vanilla",
             "Langchain's RecursiveCharacterTextSplitter - with '.'",
-            "Haystack's PreProcessor - Word level, no sentence boundaries",
-            "Haystack's PreProcessor - Word level, respect sentence boundaries",
         ],
         value="LangChain's CharacterTextSplitter",
         label="Chunking method ",
-        info="How should we split our chunks",
+        info="How should we split our chunks?",
+    )
+    demo = gr.Interface(fn=greet, inputs="textbox", outputs="textbox")
+    separator_selection = gradio.Textbox(value=["\n\n", "\n", ".", " ", ""], label="Separators used in RecursiveCharacterTextSplitter")
+    length_unit_selection = gr.Dropdown(
+        choices=[
+            "Character count",
+            "Token count",
+        ],
+        value="Token count",
+        label="Length count",
+        info="How should we count our chunk lengths?",
     )
     slider_count = gr.Slider(
         20, 500, value=50, label="Count 🧮", info="Chunk size, in the chosen unit."
@@ -127,6 +123,11 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
         inputs=[text, slider_count, split_selection],
         outputs=out,
     )
+    length_unit_selection.change(
+        fn=chunk,
+        inputs=[text, slider_count, split_selection],
+        outputs=out,
+    )
     split_selection.change(
         fn=chunk,
         inputs=[text, slider_count, split_selection],