Update tokenizer_config.json
Browse files- tokenizer_config.json +0 -6
tokenizer_config.json
CHANGED
|
@@ -9132,12 +9132,6 @@
|
|
| 9132 |
"<|CROP_ROW_SEP|>",
|
| 9133 |
"<|IMAGE_SEP|>"
|
| 9134 |
],
|
| 9135 |
-
"auto_map": {
|
| 9136 |
-
"AutoTokenizer": [
|
| 9137 |
-
"tokenization_ernie4_5.Ernie4_5_Tokenizer",
|
| 9138 |
-
null
|
| 9139 |
-
]
|
| 9140 |
-
},
|
| 9141 |
"bos_token": "<s>",
|
| 9142 |
"clean_up_tokenization_spaces": false,
|
| 9143 |
"cls_token": "<|begin_of_sentence|>",
|
|
|
|
| 9132 |
"<|CROP_ROW_SEP|>",
|
| 9133 |
"<|IMAGE_SEP|>"
|
| 9134 |
],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9135 |
"bos_token": "<s>",
|
| 9136 |
"clean_up_tokenization_spaces": false,
|
| 9137 |
"cls_token": "<|begin_of_sentence|>",
|