Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
|
@@ -35,6 +35,7 @@ def change_split_selection(split_selection):
|
|
| 35 |
|
| 36 |
def chunk(text, length, splitter_selection, separators_str, length_unit_selection, chunk_overlap):
|
| 37 |
separators = extract_separators_from_string(separators_str)
|
|
|
|
| 38 |
length_function = (length_tokens if "token" in length_unit_selection.lower() else len)
|
| 39 |
if splitter_selection == LABEL_TEXTSPLITTER:
|
| 40 |
text_splitter = CharacterTextSplitter(
|
|
@@ -63,12 +64,14 @@ def chunk(text, length, splitter_selection, separators_str, length_unit_selectio
|
|
| 63 |
|
| 64 |
def change_preset_separators(choice):
|
| 65 |
text_splitter = RecursiveCharacterTextSplitter()
|
| 66 |
-
if choice == "Default":
|
| 67 |
return ["\n\n", "\n", " ", ""]
|
| 68 |
elif choice == "Markdown":
|
| 69 |
return text_splitter.get_separators_for_language(Language.MARKDOWN)
|
| 70 |
elif choice == "Python":
|
| 71 |
return text_splitter.get_separators_for_language(Language.PYTHON)
|
|
|
|
|
|
|
| 72 |
|
| 73 |
|
| 74 |
EXAMPLE_TEXT = """### Chapter 6
|
|
|
|
| 35 |
|
| 36 |
def chunk(text, length, splitter_selection, separators_str, length_unit_selection, chunk_overlap):
|
| 37 |
separators = extract_separators_from_string(separators_str)
|
| 38 |
+
print('Extracted separators:', ',,'.join(separators))
|
| 39 |
length_function = (length_tokens if "token" in length_unit_selection.lower() else len)
|
| 40 |
if splitter_selection == LABEL_TEXTSPLITTER:
|
| 41 |
text_splitter = CharacterTextSplitter(
|
|
|
|
| 64 |
|
| 65 |
def change_preset_separators(choice):
|
| 66 |
text_splitter = RecursiveCharacterTextSplitter()
|
| 67 |
+
if choice == "Default recursive":
|
| 68 |
return ["\n\n", "\n", " ", ""]
|
| 69 |
elif choice == "Markdown":
|
| 70 |
return text_splitter.get_separators_for_language(Language.MARKDOWN)
|
| 71 |
elif choice == "Python":
|
| 72 |
return text_splitter.get_separators_for_language(Language.PYTHON)
|
| 73 |
+
else:
|
| 74 |
+
raise gr.Error("Choice of preset not recognized.")
|
| 75 |
|
| 76 |
|
| 77 |
EXAMPLE_TEXT = """### Chapter 6
|