feihu.hf
commited on
Commit
·
11e5ea0
1
Parent(s):
2d67d9d
update tokenizer.json
Browse files- tokenizer.json +36 -0
tokenizer.json
CHANGED
|
@@ -200,6 +200,42 @@
|
|
| 200 |
"rstrip": false,
|
| 201 |
"single_word": false,
|
| 202 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 203 |
}
|
| 204 |
],
|
| 205 |
"normalizer": {
|
|
|
|
| 200 |
"rstrip": false,
|
| 201 |
"single_word": false,
|
| 202 |
"special": false
|
| 203 |
+
},
|
| 204 |
+
{
|
| 205 |
+
"id": 151665,
|
| 206 |
+
"content": "<tool_response>",
|
| 207 |
+
"lstrip": false,
|
| 208 |
+
"normalized": false,
|
| 209 |
+
"rstrip": false,
|
| 210 |
+
"single_word": false,
|
| 211 |
+
"special": false
|
| 212 |
+
},
|
| 213 |
+
{
|
| 214 |
+
"id": 151666,
|
| 215 |
+
"content": "</tool_response>",
|
| 216 |
+
"lstrip": false,
|
| 217 |
+
"normalized": false,
|
| 218 |
+
"rstrip": false,
|
| 219 |
+
"single_word": false,
|
| 220 |
+
"special": false
|
| 221 |
+
},
|
| 222 |
+
{
|
| 223 |
+
"id": 151667,
|
| 224 |
+
"content": "<think>",
|
| 225 |
+
"lstrip": false,
|
| 226 |
+
"normalized": false,
|
| 227 |
+
"rstrip": false,
|
| 228 |
+
"single_word": false,
|
| 229 |
+
"special": false
|
| 230 |
+
},
|
| 231 |
+
{
|
| 232 |
+
"id": 151668,
|
| 233 |
+
"content": "</think>",
|
| 234 |
+
"lstrip": false,
|
| 235 |
+
"normalized": false,
|
| 236 |
+
"rstrip": false,
|
| 237 |
+
"single_word": false,
|
| 238 |
+
"special": false
|
| 239 |
}
|
| 240 |
],
|
| 241 |
"normalizer": {
|