S-Dreamer commited on
Commit
9fc9738
·
verified ·
1 Parent(s): c59a42f

Rename tokenizer_config.json to tokenizer.json

Browse files
tokenizer_config.json → tokenizer.json RENAMED
@@ -4,5 +4,10 @@
4
  "use_fast": true,
5
  "model_max_length": 128,
6
  "padding_side": "right",
7
- "truncation": true
 
 
 
 
 
8
  }
 
4
  "use_fast": true,
5
  "model_max_length": 128,
6
  "padding_side": "right",
7
+ "truncation": true,
8
+ "bos_token": "<s>",
9
+ "eos_token": "</s>",
10
+ "unk_token": "<unk>",
11
+ "pad_token": "<pad>",
12
+ "mask_token": "<mask>"
13
  }