Update tokenizer_config.json
Browse files- tokenizer_config.json +2 -2
tokenizer_config.json
CHANGED
@@ -122,14 +122,14 @@
|
|
122 |
],
|
123 |
"bos_token": "<bos>",
|
124 |
"bos_token_id": 2,
|
125 |
-
"chat_template":
|
126 |
"clean_up_tokenization_spaces": false,
|
127 |
"eos_token": "<eos>",
|
128 |
"eos_token_id": 3,
|
129 |
"extra_special_tokens": {},
|
130 |
"mask_token": "<mask>",
|
131 |
"mask_token_id": 4,
|
132 |
-
"max_length":
|
133 |
"model_input_names": [
|
134 |
"input_ids",
|
135 |
"token_type_ids",
|
|
|
122 |
],
|
123 |
"bos_token": "<bos>",
|
124 |
"bos_token_id": 2,
|
125 |
+
"chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}<system>{{ message['content'] }}</system>\n{% elif message['role'] == 'user' %}<user>{{ message['content'] }}</user>\n{% elif message['role'] == 'assistant' %}<assistant>{{ message['content'] }}</assistant>\n{% endif %}{% endfor %}<assistant>",
|
126 |
"clean_up_tokenization_spaces": false,
|
127 |
"eos_token": "<eos>",
|
128 |
"eos_token_id": 3,
|
129 |
"extra_special_tokens": {},
|
130 |
"mask_token": "<mask>",
|
131 |
"mask_token_id": 4,
|
132 |
+
"max_length": 512,
|
133 |
"model_input_names": [
|
134 |
"input_ids",
|
135 |
"token_type_ids",
|