reaperdoesntknow commited on
Commit
96bc6a8
·
verified ·
1 Parent(s): 80788fb

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -122,14 +122,14 @@
122
  ],
123
  "bos_token": "<bos>",
124
  "bos_token_id": 2,
125
- "chat_template": null,
126
  "clean_up_tokenization_spaces": false,
127
  "eos_token": "<eos>",
128
  "eos_token_id": 3,
129
  "extra_special_tokens": {},
130
  "mask_token": "<mask>",
131
  "mask_token_id": 4,
132
- "max_length": 256,
133
  "model_input_names": [
134
  "input_ids",
135
  "token_type_ids",
 
122
  ],
123
  "bos_token": "<bos>",
124
  "bos_token_id": 2,
125
+ "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}<system>{{ message['content'] }}</system>\n{% elif message['role'] == 'user' %}<user>{{ message['content'] }}</user>\n{% elif message['role'] == 'assistant' %}<assistant>{{ message['content'] }}</assistant>\n{% endif %}{% endfor %}<assistant>",
126
  "clean_up_tokenization_spaces": false,
127
  "eos_token": "<eos>",
128
  "eos_token_id": 3,
129
  "extra_special_tokens": {},
130
  "mask_token": "<mask>",
131
  "mask_token_id": 4,
132
+ "max_length": 512,
133
  "model_input_names": [
134
  "input_ids",
135
  "token_type_ids",