Upload folder using huggingface_hub

modules/SD15/SDToken.py

@@ -236,54 +236,32 @@ class SDTokenizer:
             - `pad_to_max_length` (bool, optional): Whether to pad to the maximum length. Defaults to True.
             - `min_length` (int, optional): The minimum length of the input. Defaults to None.
         """
-        # Ensure tokenizer path exists
         if tokenizer_path is None:
-            tokenizer_path = "./_internal/sd1_tokenizer/"
-        
-        # Verify path exists
-        if not os.path.exists(tokenizer_path):
-            raise ValueError(f"Tokenizer path does not exist: {tokenizer_path}")
-
-        try: 
-            if tokenizer_path is None:
-                # Use pre-bundled tokenizer
-                self.tokenizer = CLIPTokenizerFast.from_pretrained("openai/clip-vit-large-patch14")
-            else:
-                # Try local tokenizer files
-                self.tokenizer = CLIPTokenizerFast.from_pretrained(tokenizer_path)
-        except Exception as e:
-            raise RuntimeError(f"Failed to load tokenizer from {tokenizer_path}: {str(e)}")
-
+            tokenizer_path = os.path.join("_internal/sd1_tokenizer/", "")
+        self.tokenizer = tokenizer_class.from_pretrained(tokenizer_path)
         self.max_length = max_length
         self.min_length = min_length
-        
-        # Get tokens from empty string tokenization
+
         empty = self.tokenizer("")["input_ids"]
-        
         if has_start_token:
             self.tokens_start = 1
             self.start_token = empty[0]
             self.end_token = empty[1]
         else:
             self.tokens_start = 0
-            self.start_token = None 
+            self.start_token = None
             self.end_token = empty[0]
-            
         self.pad_with_end = pad_with_end
         self.pad_to_max_length = pad_to_max_length
 
-        # Create vocab lookup
         vocab = self.tokenizer.get_vocab()
         self.inv_vocab = {v: k for k, v in vocab.items()}
-        
-        # Set embedding properties
         self.embedding_directory = embedding_directory
         self.max_word_length = 8
         self.embedding_identifier = "embedding:"
         self.embedding_size = embedding_size
         self.embedding_key = embedding_key
 
-
     def _try_get_embedding(self, embedding_name: str) -> tuple:
         """#### Try to get an embedding.
 
@@ -432,7 +410,7 @@ class SDTokenizer:
 class SD1Tokenizer:
     """#### Class representing the SD1Tokenizer."""
 
-    def __init__(self, embedding_directory: str = None, clip_name: str = "l", tokenizer: type = SDTokenizer, tokenizer_data: dict = None):
+    def __init__(self, embedding_directory: str = None, clip_name: str = "l", tokenizer: type = SDTokenizer):
         """#### Initialize the SD1Tokenizer.
 
         #### Args:
@@ -441,14 +419,8 @@ class SD1Tokenizer:
             - `tokenizer` (type, optional): The tokenizer class. Defaults to SDTokenizer.
         """
         self.clip_name = clip_name
-        self.clip = f"clip_{self.clip_name}"
-        
-        # Initialize tokenizer with proper arguments
-        kwargs = {"embedding_directory": embedding_directory}
-        if tokenizer_data:
-            kwargs.update(tokenizer_data)
-            
-        setattr(self, self.clip, tokenizer(**kwargs))
+        self.clip = "clip_{}".format(self.clip_name)
+        setattr(self, self.clip, tokenizer(embedding_directory=embedding_directory))
 
     def tokenize_with_weights(self, text: str, return_word_ids: bool = False) -> dict:
         """#### Tokenize text with weights.