Spaces:
Running
Running
more decrease chunk size
Browse files
app.py
CHANGED
|
@@ -33,7 +33,7 @@ model = GroqModel('llama-3.1-70b-versatile', api_key = api_key)
|
|
| 33 |
|
| 34 |
|
| 35 |
|
| 36 |
-
def split_into_token_chunks(text: str, max_tokens: int =
|
| 37 |
"""
|
| 38 |
Splits a long string into chunks of a specified maximum number of tokens (words).
|
| 39 |
|
|
|
|
| 33 |
|
| 34 |
|
| 35 |
|
| 36 |
+
def split_into_token_chunks(text: str, max_tokens: int = 1000) -> list:
|
| 37 |
"""
|
| 38 |
Splits a long string into chunks of a specified maximum number of tokens (words).
|
| 39 |
|