better fix out of vocab tokens (#6)

- better fix out of vocab tokens (a0a7085477e391fd28ba330c12379c5ebe955c09)

tokenization_interns1.py

@@ -891,11 +891,13 @@ class InternS1Tokenizer(Qwen2Tokenizer):
         else:
             return self.encoder.get(token, self.encoder.get(self._unk_token))
 
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        return self.decoder.get(index, "")
+
     def convert_tokens_to_string(self, tokens):
         """Converts a sequence of tokens (string) in a single string."""
-        text = ""
-        for token in tokens:
-            text += token if token else ""
+        text = "".join(tokens)
         text = text.replace(
             "▁", "Ġ"
         )  # This discrepancy stems from differing whitespace treatment in SentencePiece versus BPE tokenization.