add mirth/chonky_mmbert_small_multilingual_1

app.py

@@ -10,7 +10,8 @@ def run(text, model_id):
 with gr.Blocks() as demo:
     gr.Markdown("# Semantic Chunking Demo\n  **Note**: This Space runs on CPU only, so input is limited to max. 50000 characters.")
     model = gr.Dropdown(label="Choose model", value="mamei16/chonky_distilbert_base_uncased_1.1",
-                        choices=["mamei16/chonky_distilbert_base_uncased_1.1", "mirth/chonky_distilbert_base_uncased_1", "mirth/chonky_modernbert_base_1"])
+                        choices=["mamei16/chonky_distilbert_base_uncased_1.1", "mirth/chonky_distilbert_base_uncased_1", "mirth/chonky_modernbert_base_1",
+                                 "mirth/chonky_mmbert_small_multilingual_1"])
     button = gr.Button("Run", variant="primary")
     text = gr.Textbox(label='Input Text', max_length=50000)
     gr.Markdown("## Result chunks:")

chonky/__init__.py

@@ -34,7 +34,7 @@ class ParagraphSplitter:
     def __init__(self, model_id="mamei16/chonky_distilbert_base_uncased_1.1", device="cpu", model_cache_dir: str = None):
         super().__init__()
         self.device = device
-        self.is_modernbert = model_id.startswith("mirth/chonky_modernbert")
+        self.is_modernbert = model_id.startswith("mirth/chonky_modernbert") or model_id == "mirth/chonky_mmbert_small_multilingual_1"
 
         id2label = {
             0: "O",