Upload processor
#2
by
sanchit-gandhi
- opened
- special_tokens_map.json +1 -0
- tokenizer_config.json +2 -1
special_tokens_map.json
CHANGED
|
@@ -122,6 +122,7 @@
|
|
| 122 |
"rstrip": false,
|
| 123 |
"single_word": false
|
| 124 |
},
|
|
|
|
| 125 |
"unk_token": {
|
| 126 |
"content": "<|endoftext|>",
|
| 127 |
"lstrip": false,
|
|
|
|
| 122 |
"rstrip": false,
|
| 123 |
"single_word": false
|
| 124 |
},
|
| 125 |
+
"pad_token": "<|endoftext|>",
|
| 126 |
"unk_token": {
|
| 127 |
"content": "<|endoftext|>",
|
| 128 |
"lstrip": false,
|
tokenizer_config.json
CHANGED
|
@@ -12988,8 +12988,9 @@
|
|
| 12988 |
"eos_token": "<|endoftext|>",
|
| 12989 |
"errors": "replace",
|
| 12990 |
"model_max_length": 1000000000000000019884624838656,
|
| 12991 |
-
"pad_token":
|
| 12992 |
"processor_class": "WhisperProcessor",
|
| 12993 |
"tokenizer_class": "WhisperTokenizer",
|
|
|
|
| 12994 |
"unk_token": "<|endoftext|>"
|
| 12995 |
}
|
|
|
|
| 12988 |
"eos_token": "<|endoftext|>",
|
| 12989 |
"errors": "replace",
|
| 12990 |
"model_max_length": 1000000000000000019884624838656,
|
| 12991 |
+
"pad_token": "<|endoftext|>",
|
| 12992 |
"processor_class": "WhisperProcessor",
|
| 12993 |
"tokenizer_class": "WhisperTokenizer",
|
| 12994 |
+
"trust_remote_code": false,
|
| 12995 |
"unk_token": "<|endoftext|>"
|
| 12996 |
}
|