update tokenization decode

tokenization_ernie4_5.py

@@ -14,9 +14,8 @@
 
 import os
 from shutil import copyfile
-from typing import List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple, Union
 import sentencepiece as spm
-
 from transformers.tokenization_utils import PreTrainedTokenizer
 from transformers.utils import logging
 
@@ -84,6 +83,7 @@ class Ernie4_5_Tokenizer(PreTrainedTokenizer):
             verbose=verbose,
             **kwargs,
         )
+        self.all_spec_tok = set(self.all_special_tokens)
 
     @property
     def vocab_size(self):
@@ -149,17 +149,7 @@ class Ernie4_5_Tokenizer(PreTrainedTokenizer):
         Returns:
             str: The reconstructed string.
         """
-        current_sub_tokens = []
-        out_string = ""
-        for token in tokens:
-            # make sure that special tokens are not decoded using sentencepiece model
-            if token in self.all_special_tokens:
-                out_string += self.sp_model.decode(current_sub_tokens) + token
-                current_sub_tokens = []
-            else:
-                current_sub_tokens.append(token)
-        out_string += self.sp_model.decode(current_sub_tokens)
-        return out_string
+        return self.sp_model.decode(tokens)
 
     def prepare_for_model(self, *args, **kwargs):
         if "add_special_tokens" in kwargs:
@@ -202,13 +192,39 @@ class Ernie4_5_Tokenizer(PreTrainedTokenizer):
 
         return (out_vocab_file,)
 
-    def _decode(self, *args, **kwargs):
-        kwargs.pop("clean_up_tokenization_spaces", None)
-        kwargs.pop("spaces_between_special_tokens", None)
-        return super()._decode(
-            *args,
-            **kwargs,
-            clean_up_tokenization_spaces=False,
-            spaces_between_special_tokens=False,
-        )
+    def _decode(
+        self,
+        token_ids: Union[int, list[int]],
+        skip_special_tokens: bool = False,
+        clean_up_tokenization_spaces: Optional[bool] = False,
+        spaces_between_special_tokens: bool = False,
+        **kwargs,
+    ) -> str:
+        self._decode_use_source_tokenizer = kwargs.pop("use_source_tokenizer", False)
+
+        filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens)
+        # If given is a single id, prevents splitting the string in upcoming loop
+        if isinstance(filtered_tokens, str):
+            filtered_tokens = [filtered_tokens]
+
+        sub_texts = []
+        current_sub_text = []
+        for token in filtered_tokens:
+            if skip_special_tokens and token in self.all_spec_tok:
+                continue
+            else:
+                current_sub_text.append(token)
+        if current_sub_text:
+            sub_texts.append(self.convert_tokens_to_string(current_sub_text))
+
+        if spaces_between_special_tokens:
+            text = " ".join(sub_texts)
+        else:
+            text = "".join(sub_texts)
+
+        if clean_up_tokenization_spaces:
+            clean_text = self.clean_up_tokenization(text)
+            return clean_text
+        else:
+            return text