Upload tokenization_hy.py with huggingface_hub
Browse files- tokenization_hy.py +1 -1
    	
        tokenization_hy.py
    CHANGED
    
    | @@ -287,7 +287,7 @@ class HYTokenizer(PreTrainedTokenizer): | |
| 287 |  | 
| 288 | 
             
            # tests
         | 
| 289 | 
             
            if __name__ == "__main__":
         | 
| 290 | 
            -
                tokenizer = HYTokenizer.from_pretrained('/hy')
         | 
| 291 | 
             
                text = '你好,世界'
         | 
| 292 | 
             
                tokens = tokenizer.tokenize(text)
         | 
| 293 | 
             
                print(tokens)
         | 
|  | |
| 287 |  | 
| 288 | 
             
            # tests
         | 
| 289 | 
             
            if __name__ == "__main__":
         | 
| 290 | 
            +
                tokenizer = HYTokenizer.from_pretrained('./other_tokenizer_vocab/hy')
         | 
| 291 | 
             
                text = '你好,世界'
         | 
| 292 | 
             
                tokens = tokenizer.tokenize(text)
         | 
| 293 | 
             
                print(tokens)
         | 

