Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
|
@@ -53,7 +53,7 @@ def chunk(text, length, splitter_selection, separators_str, length_unit_selectio
|
|
| 53 |
if "token" in length_unit_selection.lower():
|
| 54 |
text_splitter = RecursiveCharacterTextSplitter.from_huggingface_tokenizer(
|
| 55 |
bert_tokenizer,
|
| 56 |
-
chunk_size=
|
| 57 |
chunk_overlap=0,
|
| 58 |
add_start_index=True,
|
| 59 |
strip_whitespace=False,
|
|
|
|
| 53 |
if "token" in length_unit_selection.lower():
|
| 54 |
text_splitter = RecursiveCharacterTextSplitter.from_huggingface_tokenizer(
|
| 55 |
bert_tokenizer,
|
| 56 |
+
chunk_size=length,
|
| 57 |
chunk_overlap=0,
|
| 58 |
add_start_index=True,
|
| 59 |
strip_whitespace=False,
|