Hugging Face's logo

sanjudebnath
/
MiniCoderX

Text Generation
Safetensors
English
code-generation
transformer
ast
cfg
langchain
ollama
Model card Files Community
sanjudebnath commited on
Commit
a1c81a0
·
verified ·
1 Parent(s): 6730f7a

initial test project

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +12 -0
  2. data/code_corpus.txt +3 -0
  3. data/train.jsonl +2 -0
  4. environment.yml +36 -0
  5. minicoderx-model/checkpoint-2/config.json +61 -0
  6. minicoderx-model/checkpoint-2/generation_config.json +7 -0
  7. minicoderx-model/checkpoint-2/model.safetensors +3 -0
  8. minicoderx-model/checkpoint-2/optimizer.pt +3 -0
  9. minicoderx-model/checkpoint-2/rng_state.pth +3 -0
  10. minicoderx-model/checkpoint-2/scheduler.pt +3 -0
  11. minicoderx-model/checkpoint-2/special_tokens_map.json +140 -0
  12. minicoderx-model/checkpoint-2/tokenizer.json +1263 -0
  13. minicoderx-model/checkpoint-2/tokenizer_config.json +958 -0
  14. minicoderx-model/checkpoint-2/trainer_state.json +32 -0
  15. minicoderx-model/checkpoint-2/training_args.bin +3 -0
  16. minicoderx-model/checkpoint-3/config.json +61 -0
  17. minicoderx-model/checkpoint-3/generation_config.json +7 -0
  18. minicoderx-model/checkpoint-3/model.safetensors +3 -0
  19. minicoderx-model/checkpoint-3/optimizer.pt +3 -0
  20. minicoderx-model/checkpoint-3/rng_state.pth +3 -0
  21. minicoderx-model/checkpoint-3/scheduler.pt +3 -0
  22. minicoderx-model/checkpoint-3/special_tokens_map.json +140 -0
  23. minicoderx-model/checkpoint-3/tokenizer.json +1263 -0
  24. minicoderx-model/checkpoint-3/tokenizer_config.json +958 -0
  25. minicoderx-model/checkpoint-3/trainer_state.json +32 -0
  26. minicoderx-model/checkpoint-3/training_args.bin +3 -0
  27. minicoderx-model/config.json +61 -0
  28. minicoderx-model/generation_config.json +7 -0
  29. minicoderx-model/model.safetensors +3 -0
  30. minicoderx-model/special_tokens_map.json +140 -0
  31. minicoderx-model/tokenizer.json +1263 -0
  32. minicoderx-model/tokenizer_config.json +958 -0
  33. minicoderx-model/training_args.bin +3 -0
  34. minicoderx-tokenizer/special_tokens_map.json +7 -0
  35. minicoderx-tokenizer/tokenizer.json +349 -0
  36. minicoderx-tokenizer/tokenizer_config.json +53 -0
  37. papers/ARCHCODE: Incorporating Software Requirements in Code Generation with Large Language Models.pdf +3 -0
  38. papers/AST-T5: Structure-Aware Pretraining for Code Generation and Understanding.pdf +3 -0
  39. papers/CODE4STRUCT: Code Generation for Few-Shot Event Structure Prediction.pdf +3 -0
  40. papers/CoTexT: Multi-task Learning with Code-Text Transformer.pdf +3 -0
  41. papers/CodeBERT: A Pre-Trained Model for Programming and Natural Languages.pdf +3 -0
  42. papers/CodeGeeX: A Pre-Trained Model for Code Generation with Multilingual Benchmarking on HumanEval-X.pdf +3 -0
  43. papers/CodeGemma Open Code Models Based on Gemma.pdf +3 -0
  44. papers/CodeT5+: Open Code Large Language Models for Code Understanding and Generation.pdf +3 -0
  45. papers/Evaluating the Code Quality of AI-Assisted Code Generation Tools: An Empirical Study on GitHub Copilot, Amazon CodeWhisperer, and ChatGPT.pdf +3 -0
  46. papers/GRAPH CODE BERT: PRE-TRAINING CODE REPRESEN-TATIONS WITH DATA F LOW.pdf +3 -0
  47. papers/StructCoder_ Structure-Aware Transformer for Code Generation - 2206.05239v3.pdf +3 -0
  48. papers/TreeGen: A Tree-Based Transformer Architecture for Code Generation.pdf +3 -0
  49. project.ipynb +482 -0
  50. train_t5_code.py +65 -0
.gitattributes CHANGED
@@ -33,3 +33,15 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ papers/ARCHCODE:[[:space:]]Incorporating[[:space:]]Software[[:space:]]Requirements[[:space:]]in[[:space:]]Code[[:space:]]Generation[[:space:]]with[[:space:]]Large[[:space:]]Language[[:space:]]Models.pdf filter=lfs diff=lfs merge=lfs -text
37
+ papers/AST-T5:[[:space:]]Structure-Aware[[:space:]]Pretraining[[:space:]]for[[:space:]]Code[[:space:]]Generation[[:space:]]and[[:space:]]Understanding.pdf filter=lfs diff=lfs merge=lfs -text
38
+ papers/CODE4STRUCT:[[:space:]]Code[[:space:]]Generation[[:space:]]for[[:space:]]Few-Shot[[:space:]]Event[[:space:]]Structure[[:space:]]Prediction.pdf filter=lfs diff=lfs merge=lfs -text
39
+ papers/CodeBERT:[[:space:]]A[[:space:]]Pre-Trained[[:space:]]Model[[:space:]]for[[:space:]]Programming[[:space:]]and[[:space:]]Natural[[:space:]]Languages.pdf filter=lfs diff=lfs merge=lfs -text
40
+ papers/CodeGeeX:[[:space:]]A[[:space:]]Pre-Trained[[:space:]]Model[[:space:]]for[[:space:]]Code[[:space:]]Generation[[:space:]]with[[:space:]]Multilingual[[:space:]]Benchmarking[[:space:]]on[[:space:]]HumanEval-X.pdf filter=lfs diff=lfs merge=lfs -text
41
+ papers/CodeGemma[[:space:]]Open[[:space:]]Code[[:space:]]Models[[:space:]][[:space:]]Based[[:space:]]on[[:space:]]Gemma.pdf filter=lfs diff=lfs merge=lfs -text
42
+ papers/CodeT5+:[[:space:]]Open[[:space:]]Code[[:space:]]Large[[:space:]]Language[[:space:]]Models[[:space:]]for[[:space:]]Code[[:space:]]Understanding[[:space:]]and[[:space:]]Generation.pdf filter=lfs diff=lfs merge=lfs -text
43
+ papers/CoTexT:[[:space:]]Multi-task[[:space:]]Learning[[:space:]]with[[:space:]]Code-Text[[:space:]]Transformer.pdf filter=lfs diff=lfs merge=lfs -text
44
+ papers/Evaluating[[:space:]]the[[:space:]]Code[[:space:]]Quality[[:space:]]of[[:space:]]AI-Assisted[[:space:]]Code[[:space:]]Generation[[:space:]]Tools:[[:space:]]An[[:space:]]Empirical[[:space:]]Study[[:space:]]on[[:space:]]GitHub[[:space:]]Copilot,[[:space:]]Amazon[[:space:]]CodeWhisperer,[[:space:]]and[[:space:]]ChatGPT.pdf filter=lfs diff=lfs merge=lfs -text
45
+ papers/GRAPH[[:space:]]CODE[[:space:]]BERT:[[:space:]]PRE-TRAINING[[:space:]]CODE[[:space:]]REPRESEN-TATIONS[[:space:]]WITH[[:space:]]DATA[[:space:]]F[[:space:]]LOW.pdf filter=lfs diff=lfs merge=lfs -text
46
+ papers/StructCoder_[[:space:]]Structure-Aware[[:space:]]Transformer[[:space:]]for[[:space:]]Code[[:space:]]Generation[[:space:]]-[[:space:]]2206.05239v3.pdf filter=lfs diff=lfs merge=lfs -text
47
+ papers/TreeGen:[[:space:]]A[[:space:]]Tree-Based[[:space:]]Transformer[[:space:]]Architecture[[:space:]]for[[:space:]]Code[[:space:]]Generation.pdf filter=lfs diff=lfs merge=lfs -text
data/code_corpus.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ def add(a, b): return a + b
2
+ print("Hello, world!")
3
+ class HelloWorld: pass
data/train.jsonl ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ {"input": "def add(a, b):", "output": "Returns the sum of a and b."}
2
+ {"input": "class Hello:", "output": "A simple Hello class."}
environment.yml ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: myenv
2
+ channels:
3
+ - defaults
4
+ - conda-forge
5
+ dependencies:
6
+ - python=3.10
7
+ - transformers
8
+ - datasets
9
+ - sentencepiece
10
+ - langchain
11
+ - networkx
12
+ - evaluate
13
+ - matplotlib
14
+ - seaborn
15
+ - lark
16
+ - fastapi
17
+ - uvicorn
18
+ - jupyterlab
19
+ - ipykernel
20
+ - tqdm
21
+ - scikit-learn
22
+ - pandas
23
+ - numpy
24
+ - pytorch
25
+ - pip
26
+ - pip:
27
+ - ollama
28
+ - rouge-score
29
+ - huggingface_hub
30
+ - accelerate
31
+ - peft
32
+ - bitsandbytes
33
+ - einops
34
+ - safetensors
35
+ - wandb
36
+
minicoderx-model/checkpoint-2/config.json ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "t5-small",
3
+ "architectures": [
4
+ "T5ForConditionalGeneration"
5
+ ],
6
+ "classifier_dropout": 0.0,
7
+ "d_ff": 2048,
8
+ "d_kv": 64,
9
+ "d_model": 512,
10
+ "decoder_start_token_id": 0,
11
+ "dense_act_fn": "relu",
12
+ "dropout_rate": 0.1,
13
+ "eos_token_id": 1,
14
+ "feed_forward_proj": "relu",
15
+ "initializer_factor": 1.0,
16
+ "is_encoder_decoder": true,
17
+ "is_gated_act": false,
18
+ "layer_norm_epsilon": 1e-06,
19
+ "model_type": "t5",
20
+ "n_positions": 512,
21
+ "num_decoder_layers": 6,
22
+ "num_heads": 8,
23
+ "num_layers": 6,
24
+ "output_past": true,
25
+ "pad_token_id": 0,
26
+ "relative_attention_max_distance": 128,
27
+ "relative_attention_num_buckets": 32,
28
+ "task_specific_params": {
29
+ "summarization": {
30
+ "early_stopping": true,
31
+ "length_penalty": 2.0,
32
+ "max_length": 200,
33
+ "min_length": 30,
34
+ "no_repeat_ngram_size": 3,
35
+ "num_beams": 4,
36
+ "prefix": "summarize: "
37
+ },
38
+ "translation_en_to_de": {
39
+ "early_stopping": true,
40
+ "max_length": 300,
41
+ "num_beams": 4,
42
+ "prefix": "translate English to German: "
43
+ },
44
+ "translation_en_to_fr": {
45
+ "early_stopping": true,
46
+ "max_length": 300,
47
+ "num_beams": 4,
48
+ "prefix": "translate English to French: "
49
+ },
50
+ "translation_en_to_ro": {
51
+ "early_stopping": true,
52
+ "max_length": 300,
53
+ "num_beams": 4,
54
+ "prefix": "translate English to Romanian: "
55
+ }
56
+ },
57
+ "torch_dtype": "float32",
58
+ "transformers_version": "4.49.0",
59
+ "use_cache": true,
60
+ "vocab_size": 32128
61
+ }
minicoderx-model/checkpoint-2/generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "decoder_start_token_id": 0,
4
+ "eos_token_id": 1,
5
+ "pad_token_id": 0,
6
+ "transformers_version": "4.49.0"
7
+ }
minicoderx-model/checkpoint-2/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05315572729341ea7678bacfca08ae712b079e2f0f709ea828850bb2fde1b75e
3
+ size 242041896
minicoderx-model/checkpoint-2/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:294cd1bddba987df78f346e471aeb9ee05979de08064749d4a94fb8e5bc366da
3
+ size 484159866
minicoderx-model/checkpoint-2/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d1565ec4ad382943310a6a711595e3c8d4aed2df4b2b87d014ae460d26b0d6aa
3
+ size 13990
minicoderx-model/checkpoint-2/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a27ddd0715a8a938994a234170508c4e373cb0cfabcdcd5c8c55491154f6015a
3
+ size 1064
minicoderx-model/checkpoint-2/special_tokens_map.json ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<extra_id_0>",
4
+ "<extra_id_1>",
5
+ "<extra_id_2>",
6
+ "<extra_id_3>",
7
+ "<extra_id_4>",
8
+ "<extra_id_5>",
9
+ "<extra_id_6>",
10
+ "<extra_id_7>",
11
+ "<extra_id_8>",
12
+ "<extra_id_9>",
13
+ "<extra_id_10>",
14
+ "<extra_id_11>",
15
+ "<extra_id_12>",
16
+ "<extra_id_13>",
17
+ "<extra_id_14>",
18
+ "<extra_id_15>",
19
+ "<extra_id_16>",
20
+ "<extra_id_17>",
21
+ "<extra_id_18>",
22
+ "<extra_id_19>",
23
+ "<extra_id_20>",
24
+ "<extra_id_21>",
25
+ "<extra_id_22>",
26
+ "<extra_id_23>",
27
+ "<extra_id_24>",
28
+ "<extra_id_25>",
29
+ "<extra_id_26>",
30
+ "<extra_id_27>",
31
+ "<extra_id_28>",
32
+ "<extra_id_29>",
33
+ "<extra_id_30>",
34
+ "<extra_id_31>",
35
+ "<extra_id_32>",
36
+ "<extra_id_33>",
37
+ "<extra_id_34>",
38
+ "<extra_id_35>",
39
+ "<extra_id_36>",
40
+ "<extra_id_37>",
41
+ "<extra_id_38>",
42
+ "<extra_id_39>",
43
+ "<extra_id_40>",
44
+ "<extra_id_41>",
45
+ "<extra_id_42>",
46
+ "<extra_id_43>",
47
+ "<extra_id_44>",
48
+ "<extra_id_45>",
49
+ "<extra_id_46>",
50
+ "<extra_id_47>",
51
+ "<extra_id_48>",
52
+ "<extra_id_49>",
53
+ "<extra_id_50>",
54
+ "<extra_id_51>",
55
+ "<extra_id_52>",
56
+ "<extra_id_53>",
57
+ "<extra_id_54>",
58
+ "<extra_id_55>",
59
+ "<extra_id_56>",
60
+ "<extra_id_57>",
61
+ "<extra_id_58>",
62
+ "<extra_id_59>",
63
+ "<extra_id_60>",
64
+ "<extra_id_61>",
65
+ "<extra_id_62>",
66
+ "<extra_id_63>",
67
+ "<extra_id_64>",
68
+ "<extra_id_65>",
69
+ "<extra_id_66>",
70
+ "<extra_id_67>",
71
+ "<extra_id_68>",
72
+ "<extra_id_69>",
73
+ "<extra_id_70>",
74
+ "<extra_id_71>",
75
+ "<extra_id_72>",
76
+ "<extra_id_73>",
77
+ "<extra_id_74>",
78
+ "<extra_id_75>",
79
+ "<extra_id_76>",
80
+ "<extra_id_77>",
81
+ "<extra_id_78>",
82
+ "<extra_id_79>",
83
+ "<extra_id_80>",
84
+ "<extra_id_81>",
85
+ "<extra_id_82>",
86
+ "<extra_id_83>",
87
+ "<extra_id_84>",
88
+ "<extra_id_85>",
89
+ "<extra_id_86>",
90
+ "<extra_id_87>",
91
+ "<extra_id_88>",
92
+ "<extra_id_89>",
93
+ "<extra_id_90>",
94
+ "<extra_id_91>",
95
+ "<extra_id_92>",
96
+ "<extra_id_93>",
97
+ "<extra_id_94>",
98
+ "<extra_id_95>",
99
+ "<extra_id_96>",
100
+ "<extra_id_97>",
101
+ "<extra_id_98>",
102
+ "<extra_id_99>"
103
+ ],
104
+ "cls_token": {
105
+ "content": "<s>",
106
+ "lstrip": false,
107
+ "normalized": false,
108
+ "rstrip": false,
109
+ "single_word": false
110
+ },
111
+ "eos_token": "</s>",
112
+ "mask_token": {
113
+ "content": "<mask>",
114
+ "lstrip": false,
115
+ "normalized": false,
116
+ "rstrip": false,
117
+ "single_word": false
118
+ },
119
+ "pad_token": {
120
+ "content": "<pad>",
121
+ "lstrip": false,
122
+ "normalized": false,
123
+ "rstrip": false,
124
+ "single_word": false
125
+ },
126
+ "sep_token": {
127
+ "content": "</s>",
128
+ "lstrip": false,
129
+ "normalized": false,
130
+ "rstrip": false,
131
+ "single_word": false
132
+ },
133
+ "unk_token": {
134
+ "content": "<unk>",
135
+ "lstrip": false,
136
+ "normalized": false,
137
+ "rstrip": false,
138
+ "single_word": false
139
+ }
140
+ }
minicoderx-model/checkpoint-2/tokenizer.json ADDED
@@ -0,0 +1,1263 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": {
4
+ "direction": "Right",
5
+ "max_length": 128,
6
+ "strategy": "LongestFirst",
7
+ "stride": 0
8
+ },
9
+ "padding": {
10
+ "strategy": {
11
+ "Fixed": 128
12
+ },
13
+ "direction": "Right",
14
+ "pad_to_multiple_of": null,
15
+ "pad_id": 0,
16
+ "pad_type_id": 0,
17
+ "pad_token": "<pad>"
18
+ },
19
+ "added_tokens": [
20
+ {
21
+ "id": 0,
22
+ "content": "<pad>",
23
+ "single_word": false,
24
+ "lstrip": false,
25
+ "rstrip": false,
26
+ "normalized": false,
27
+ "special": true
28
+ },
29
+ {
30
+ "id": 1,
31
+ "content": "<s>",
32
+ "single_word": false,
33
+ "lstrip": false,
34
+ "rstrip": false,
35
+ "normalized": false,
36
+ "special": true
37
+ },
38
+ {
39
+ "id": 2,
40
+ "content": "</s>",
41
+ "single_word": false,
42
+ "lstrip": false,
43
+ "rstrip": false,
44
+ "normalized": false,
45
+ "special": true
46
+ },
47
+ {
48
+ "id": 3,
49
+ "content": "<unk>",
50
+ "single_word": false,
51
+ "lstrip": false,
52
+ "rstrip": false,
53
+ "normalized": false,
54
+ "special": true
55
+ },
56
+ {
57
+ "id": 4,
58
+ "content": "<mask>",
59
+ "single_word": false,
60
+ "lstrip": false,
61
+ "rstrip": false,
62
+ "normalized": false,
63
+ "special": true
64
+ },
65
+ {
66
+ "id": 60,
67
+ "content": "<extra_id_0>",
68
+ "single_word": false,
69
+ "lstrip": false,
70
+ "rstrip": false,
71
+ "normalized": false,
72
+ "special": true
73
+ },
74
+ {
75
+ "id": 61,
76
+ "content": "<extra_id_1>",
77
+ "single_word": false,
78
+ "lstrip": false,
79
+ "rstrip": false,
80
+ "normalized": false,
81
+ "special": true
82
+ },
83
+ {
84
+ "id": 62,
85
+ "content": "<extra_id_2>",
86
+ "single_word": false,
87
+ "lstrip": false,
88
+ "rstrip": false,
89
+ "normalized": false,
90
+ "special": true
91
+ },
92
+ {
93
+ "id": 63,
94
+ "content": "<extra_id_3>",
95
+ "single_word": false,
96
+ "lstrip": false,
97
+ "rstrip": false,
98
+ "normalized": false,
99
+ "special": true
100
+ },
101
+ {
102
+ "id": 64,
103
+ "content": "<extra_id_4>",
104
+ "single_word": false,
105
+ "lstrip": false,
106
+ "rstrip": false,
107
+ "normalized": false,
108
+ "special": true
109
+ },
110
+ {
111
+ "id": 65,
112
+ "content": "<extra_id_5>",
113
+ "single_word": false,
114
+ "lstrip": false,
115
+ "rstrip": false,
116
+ "normalized": false,
117
+ "special": true
118
+ },
119
+ {
120
+ "id": 66,
121
+ "content": "<extra_id_6>",
122
+ "single_word": false,
123
+ "lstrip": false,
124
+ "rstrip": false,
125
+ "normalized": false,
126
+ "special": true
127
+ },
128
+ {
129
+ "id": 67,
130
+ "content": "<extra_id_7>",
131
+ "single_word": false,
132
+ "lstrip": false,
133
+ "rstrip": false,
134
+ "normalized": false,
135
+ "special": true
136
+ },
137
+ {
138
+ "id": 68,
139
+ "content": "<extra_id_8>",
140
+ "single_word": false,
141
+ "lstrip": false,
142
+ "rstrip": false,
143
+ "normalized": false,
144
+ "special": true
145
+ },
146
+ {
147
+ "id": 69,
148
+ "content": "<extra_id_9>",
149
+ "single_word": false,
150
+ "lstrip": false,
151
+ "rstrip": false,
152
+ "normalized": false,
153
+ "special": true
154
+ },
155
+ {
156
+ "id": 70,
157
+ "content": "<extra_id_10>",
158
+ "single_word": false,
159
+ "lstrip": false,
160
+ "rstrip": false,
161
+ "normalized": false,
162
+ "special": true
163
+ },
164
+ {
165
+ "id": 71,
166
+ "content": "<extra_id_11>",
167
+ "single_word": false,
168
+ "lstrip": false,
169
+ "rstrip": false,
170
+ "normalized": false,
171
+ "special": true
172
+ },
173
+ {
174
+ "id": 72,
175
+ "content": "<extra_id_12>",
176
+ "single_word": false,
177
+ "lstrip": false,
178
+ "rstrip": false,
179
+ "normalized": false,
180
+ "special": true
181
+ },
182
+ {
183
+ "id": 73,
184
+ "content": "<extra_id_13>",
185
+ "single_word": false,
186
+ "lstrip": false,
187
+ "rstrip": false,
188
+ "normalized": false,
189
+ "special": true
190
+ },
191
+ {
192
+ "id": 74,
193
+ "content": "<extra_id_14>",
194
+ "single_word": false,
195
+ "lstrip": false,
196
+ "rstrip": false,
197
+ "normalized": false,
198
+ "special": true
199
+ },
200
+ {
201
+ "id": 75,
202
+ "content": "<extra_id_15>",
203
+ "single_word": false,
204
+ "lstrip": false,
205
+ "rstrip": false,
206
+ "normalized": false,
207
+ "special": true
208
+ },
209
+ {
210
+ "id": 76,
211
+ "content": "<extra_id_16>",
212
+ "single_word": false,
213
+ "lstrip": false,
214
+ "rstrip": false,
215
+ "normalized": false,
216
+ "special": true
217
+ },
218
+ {
219
+ "id": 77,
220
+ "content": "<extra_id_17>",
221
+ "single_word": false,
222
+ "lstrip": false,
223
+ "rstrip": false,
224
+ "normalized": false,
225
+ "special": true
226
+ },
227
+ {
228
+ "id": 78,
229
+ "content": "<extra_id_18>",
230
+ "single_word": false,
231
+ "lstrip": false,
232
+ "rstrip": false,
233
+ "normalized": false,
234
+ "special": true
235
+ },
236
+ {
237
+ "id": 79,
238
+ "content": "<extra_id_19>",
239
+ "single_word": false,
240
+ "lstrip": false,
241
+ "rstrip": false,
242
+ "normalized": false,
243
+ "special": true
244
+ },
245
+ {
246
+ "id": 80,
247
+ "content": "<extra_id_20>",
248
+ "single_word": false,
249
+ "lstrip": false,
250
+ "rstrip": false,
251
+ "normalized": false,
252
+ "special": true
253
+ },
254
+ {
255
+ "id": 81,
256
+ "content": "<extra_id_21>",
257
+ "single_word": false,
258
+ "lstrip": false,
259
+ "rstrip": false,
260
+ "normalized": false,
261
+ "special": true
262
+ },
263
+ {
264
+ "id": 82,
265
+ "content": "<extra_id_22>",
266
+ "single_word": false,
267
+ "lstrip": false,
268
+ "rstrip": false,
269
+ "normalized": false,
270
+ "special": true
271
+ },
272
+ {
273
+ "id": 83,
274
+ "content": "<extra_id_23>",
275
+ "single_word": false,
276
+ "lstrip": false,
277
+ "rstrip": false,
278
+ "normalized": false,
279
+ "special": true
280
+ },
281
+ {
282
+ "id": 84,
283
+ "content": "<extra_id_24>",
284
+ "single_word": false,
285
+ "lstrip": false,
286
+ "rstrip": false,
287
+ "normalized": false,
288
+ "special": true
289
+ },
290
+ {
291
+ "id": 85,
292
+ "content": "<extra_id_25>",
293
+ "single_word": false,
294
+ "lstrip": false,
295
+ "rstrip": false,
296
+ "normalized": false,
297
+ "special": true
298
+ },
299
+ {
300
+ "id": 86,
301
+ "content": "<extra_id_26>",
302
+ "single_word": false,
303
+ "lstrip": false,
304
+ "rstrip": false,
305
+ "normalized": false,
306
+ "special": true
307
+ },
308
+ {
309
+ "id": 87,
310
+ "content": "<extra_id_27>",
311
+ "single_word": false,
312
+ "lstrip": false,
313
+ "rstrip": false,
314
+ "normalized": false,
315
+ "special": true
316
+ },
317
+ {
318
+ "id": 88,
319
+ "content": "<extra_id_28>",
320
+ "single_word": false,
321
+ "lstrip": false,
322
+ "rstrip": false,
323
+ "normalized": false,
324
+ "special": true
325
+ },
326
+ {
327
+ "id": 89,
328
+ "content": "<extra_id_29>",
329
+ "single_word": false,
330
+ "lstrip": false,
331
+ "rstrip": false,
332
+ "normalized": false,
333
+ "special": true
334
+ },
335
+ {
336
+ "id": 90,
337
+ "content": "<extra_id_30>",
338
+ "single_word": false,
339
+ "lstrip": false,
340
+ "rstrip": false,
341
+ "normalized": false,
342
+ "special": true
343
+ },
344
+ {
345
+ "id": 91,
346
+ "content": "<extra_id_31>",
347
+ "single_word": false,
348
+ "lstrip": false,
349
+ "rstrip": false,
350
+ "normalized": false,
351
+ "special": true
352
+ },
353
+ {
354
+ "id": 92,
355
+ "content": "<extra_id_32>",
356
+ "single_word": false,
357
+ "lstrip": false,
358
+ "rstrip": false,
359
+ "normalized": false,
360
+ "special": true
361
+ },
362
+ {
363
+ "id": 93,
364
+ "content": "<extra_id_33>",
365
+ "single_word": false,
366
+ "lstrip": false,
367
+ "rstrip": false,
368
+ "normalized": false,
369
+ "special": true
370
+ },
371
+ {
372
+ "id": 94,
373
+ "content": "<extra_id_34>",
374
+ "single_word": false,
375
+ "lstrip": false,
376
+ "rstrip": false,
377
+ "normalized": false,
378
+ "special": true
379
+ },
380
+ {
381
+ "id": 95,
382
+ "content": "<extra_id_35>",
383
+ "single_word": false,
384
+ "lstrip": false,
385
+ "rstrip": false,
386
+ "normalized": false,
387
+ "special": true
388
+ },
389
+ {
390
+ "id": 96,
391
+ "content": "<extra_id_36>",
392
+ "single_word": false,
393
+ "lstrip": false,
394
+ "rstrip": false,
395
+ "normalized": false,
396
+ "special": true
397
+ },
398
+ {
399
+ "id": 97,
400
+ "content": "<extra_id_37>",
401
+ "single_word": false,
402
+ "lstrip": false,
403
+ "rstrip": false,
404
+ "normalized": false,
405
+ "special": true
406
+ },
407
+ {
408
+ "id": 98,
409
+ "content": "<extra_id_38>",
410
+ "single_word": false,
411
+ "lstrip": false,
412
+ "rstrip": false,
413
+ "normalized": false,
414
+ "special": true
415
+ },
416
+ {
417
+ "id": 99,
418
+ "content": "<extra_id_39>",
419
+ "single_word": false,
420
+ "lstrip": false,
421
+ "rstrip": false,
422
+ "normalized": false,
423
+ "special": true
424
+ },
425
+ {
426
+ "id": 100,
427
+ "content": "<extra_id_40>",
428
+ "single_word": false,
429
+ "lstrip": false,
430
+ "rstrip": false,
431
+ "normalized": false,
432
+ "special": true
433
+ },
434
+ {
435
+ "id": 101,
436
+ "content": "<extra_id_41>",
437
+ "single_word": false,
438
+ "lstrip": false,
439
+ "rstrip": false,
440
+ "normalized": false,
441
+ "special": true
442
+ },
443
+ {
444
+ "id": 102,
445
+ "content": "<extra_id_42>",
446
+ "single_word": false,
447
+ "lstrip": false,
448
+ "rstrip": false,
449
+ "normalized": false,
450
+ "special": true
451
+ },
452
+ {
453
+ "id": 103,
454
+ "content": "<extra_id_43>",
455
+ "single_word": false,
456
+ "lstrip": false,
457
+ "rstrip": false,
458
+ "normalized": false,
459
+ "special": true
460
+ },
461
+ {
462
+ "id": 104,
463
+ "content": "<extra_id_44>",
464
+ "single_word": false,
465
+ "lstrip": false,
466
+ "rstrip": false,
467
+ "normalized": false,
468
+ "special": true
469
+ },
470
+ {
471
+ "id": 105,
472
+ "content": "<extra_id_45>",
473
+ "single_word": false,
474
+ "lstrip": false,
475
+ "rstrip": false,
476
+ "normalized": false,
477
+ "special": true
478
+ },
479
+ {
480
+ "id": 106,
481
+ "content": "<extra_id_46>",
482
+ "single_word": false,
483
+ "lstrip": false,
484
+ "rstrip": false,
485
+ "normalized": false,
486
+ "special": true
487
+ },
488
+ {
489
+ "id": 107,
490
+ "content": "<extra_id_47>",
491
+ "single_word": false,
492
+ "lstrip": false,
493
+ "rstrip": false,
494
+ "normalized": false,
495
+ "special": true
496
+ },
497
+ {
498
+ "id": 108,
499
+ "content": "<extra_id_48>",
500
+ "single_word": false,
501
+ "lstrip": false,
502
+ "rstrip": false,
503
+ "normalized": false,
504
+ "special": true
505
+ },
506
+ {
507
+ "id": 109,
508
+ "content": "<extra_id_49>",
509
+ "single_word": false,
510
+ "lstrip": false,
511
+ "rstrip": false,
512
+ "normalized": false,
513
+ "special": true
514
+ },
515
+ {
516
+ "id": 110,
517
+ "content": "<extra_id_50>",
518
+ "single_word": false,
519
+ "lstrip": false,
520
+ "rstrip": false,
521
+ "normalized": false,
522
+ "special": true
523
+ },
524
+ {
525
+ "id": 111,
526
+ "content": "<extra_id_51>",
527
+ "single_word": false,
528
+ "lstrip": false,
529
+ "rstrip": false,
530
+ "normalized": false,
531
+ "special": true
532
+ },
533
+ {
534
+ "id": 112,
535
+ "content": "<extra_id_52>",
536
+ "single_word": false,
537
+ "lstrip": false,
538
+ "rstrip": false,
539
+ "normalized": false,
540
+ "special": true
541
+ },
542
+ {
543
+ "id": 113,
544
+ "content": "<extra_id_53>",
545
+ "single_word": false,
546
+ "lstrip": false,
547
+ "rstrip": false,
548
+ "normalized": false,
549
+ "special": true
550
+ },
551
+ {
552
+ "id": 114,
553
+ "content": "<extra_id_54>",
554
+ "single_word": false,
555
+ "lstrip": false,
556
+ "rstrip": false,
557
+ "normalized": false,
558
+ "special": true
559
+ },
560
+ {
561
+ "id": 115,
562
+ "content": "<extra_id_55>",
563
+ "single_word": false,
564
+ "lstrip": false,
565
+ "rstrip": false,
566
+ "normalized": false,
567
+ "special": true
568
+ },
569
+ {
570
+ "id": 116,
571
+ "content": "<extra_id_56>",
572
+ "single_word": false,
573
+ "lstrip": false,
574
+ "rstrip": false,
575
+ "normalized": false,
576
+ "special": true
577
+ },
578
+ {
579
+ "id": 117,
580
+ "content": "<extra_id_57>",
581
+ "single_word": false,
582
+ "lstrip": false,
583
+ "rstrip": false,
584
+ "normalized": false,
585
+ "special": true
586
+ },
587
+ {
588
+ "id": 118,
589
+ "content": "<extra_id_58>",
590
+ "single_word": false,
591
+ "lstrip": false,
592
+ "rstrip": false,
593
+ "normalized": false,
594
+ "special": true
595
+ },
596
+ {
597
+ "id": 119,
598
+ "content": "<extra_id_59>",
599
+ "single_word": false,
600
+ "lstrip": false,
601
+ "rstrip": false,
602
+ "normalized": false,
603
+ "special": true
604
+ },
605
+ {
606
+ "id": 120,
607
+ "content": "<extra_id_60>",
608
+ "single_word": false,
609
+ "lstrip": false,
610
+ "rstrip": false,
611
+ "normalized": false,
612
+ "special": true
613
+ },
614
+ {
615
+ "id": 121,
616
+ "content": "<extra_id_61>",
617
+ "single_word": false,
618
+ "lstrip": false,
619
+ "rstrip": false,
620
+ "normalized": false,
621
+ "special": true
622
+ },
623
+ {
624
+ "id": 122,
625
+ "content": "<extra_id_62>",
626
+ "single_word": false,
627
+ "lstrip": false,
628
+ "rstrip": false,
629
+ "normalized": false,
630
+ "special": true
631
+ },
632
+ {
633
+ "id": 123,
634
+ "content": "<extra_id_63>",
635
+ "single_word": false,
636
+ "lstrip": false,
637
+ "rstrip": false,
638
+ "normalized": false,
639
+ "special": true
640
+ },
641
+ {
642
+ "id": 124,
643
+ "content": "<extra_id_64>",
644
+ "single_word": false,
645
+ "lstrip": false,
646
+ "rstrip": false,
647
+ "normalized": false,
648
+ "special": true
649
+ },
650
+ {
651
+ "id": 125,
652
+ "content": "<extra_id_65>",
653
+ "single_word": false,
654
+ "lstrip": false,
655
+ "rstrip": false,
656
+ "normalized": false,
657
+ "special": true
658
+ },
659
+ {
660
+ "id": 126,
661
+ "content": "<extra_id_66>",
662
+ "single_word": false,
663
+ "lstrip": false,
664
+ "rstrip": false,
665
+ "normalized": false,
666
+ "special": true
667
+ },
668
+ {
669
+ "id": 127,
670
+ "content": "<extra_id_67>",
671
+ "single_word": false,
672
+ "lstrip": false,
673
+ "rstrip": false,
674
+ "normalized": false,
675
+ "special": true
676
+ },
677
+ {
678
+ "id": 128,
679
+ "content": "<extra_id_68>",
680
+ "single_word": false,
681
+ "lstrip": false,
682
+ "rstrip": false,
683
+ "normalized": false,
684
+ "special": true
685
+ },
686
+ {
687
+ "id": 129,
688
+ "content": "<extra_id_69>",
689
+ "single_word": false,
690
+ "lstrip": false,
691
+ "rstrip": false,
692
+ "normalized": false,
693
+ "special": true
694
+ },
695
+ {
696
+ "id": 130,
697
+ "content": "<extra_id_70>",
698
+ "single_word": false,
699
+ "lstrip": false,
700
+ "rstrip": false,
701
+ "normalized": false,
702
+ "special": true
703
+ },
704
+ {
705
+ "id": 131,
706
+ "content": "<extra_id_71>",
707
+ "single_word": false,
708
+ "lstrip": false,
709
+ "rstrip": false,
710
+ "normalized": false,
711
+ "special": true
712
+ },
713
+ {
714
+ "id": 132,
715
+ "content": "<extra_id_72>",
716
+ "single_word": false,
717
+ "lstrip": false,
718
+ "rstrip": false,
719
+ "normalized": false,
720
+ "special": true
721
+ },
722
+ {
723
+ "id": 133,
724
+ "content": "<extra_id_73>",
725
+ "single_word": false,
726
+ "lstrip": false,
727
+ "rstrip": false,
728
+ "normalized": false,
729
+ "special": true
730
+ },
731
+ {
732
+ "id": 134,
733
+ "content": "<extra_id_74>",
734
+ "single_word": false,
735
+ "lstrip": false,
736
+ "rstrip": false,
737
+ "normalized": false,
738
+ "special": true
739
+ },
740
+ {
741
+ "id": 135,
742
+ "content": "<extra_id_75>",
743
+ "single_word": false,
744
+ "lstrip": false,
745
+ "rstrip": false,
746
+ "normalized": false,
747
+ "special": true
748
+ },
749
+ {
750
+ "id": 136,
751
+ "content": "<extra_id_76>",
752
+ "single_word": false,
753
+ "lstrip": false,
754
+ "rstrip": false,
755
+ "normalized": false,
756
+ "special": true
757
+ },
758
+ {
759
+ "id": 137,
760
+ "content": "<extra_id_77>",
761
+ "single_word": false,
762
+ "lstrip": false,
763
+ "rstrip": false,
764
+ "normalized": false,
765
+ "special": true
766
+ },
767
+ {
768
+ "id": 138,
769
+ "content": "<extra_id_78>",
770
+ "single_word": false,
771
+ "lstrip": false,
772
+ "rstrip": false,
773
+ "normalized": false,
774
+ "special": true
775
+ },
776
+ {
777
+ "id": 139,
778
+ "content": "<extra_id_79>",
779
+ "single_word": false,
780
+ "lstrip": false,
781
+ "rstrip": false,
782
+ "normalized": false,
783
+ "special": true
784
+ },
785
+ {
786
+ "id": 140,
787
+ "content": "<extra_id_80>",
788
+ "single_word": false,
789
+ "lstrip": false,
790
+ "rstrip": false,
791
+ "normalized": false,
792
+ "special": true
793
+ },
794
+ {
795
+ "id": 141,
796
+ "content": "<extra_id_81>",
797
+ "single_word": false,
798
+ "lstrip": false,
799
+ "rstrip": false,
800
+ "normalized": false,
801
+ "special": true
802
+ },
803
+ {
804
+ "id": 142,
805
+ "content": "<extra_id_82>",
806
+ "single_word": false,
807
+ "lstrip": false,
808
+ "rstrip": false,
809
+ "normalized": false,
810
+ "special": true
811
+ },
812
+ {
813
+ "id": 143,
814
+ "content": "<extra_id_83>",
815
+ "single_word": false,
816
+ "lstrip": false,
817
+ "rstrip": false,
818
+ "normalized": false,
819
+ "special": true
820
+ },
821
+ {
822
+ "id": 144,
823
+ "content": "<extra_id_84>",
824
+ "single_word": false,
825
+ "lstrip": false,
826
+ "rstrip": false,
827
+ "normalized": false,
828
+ "special": true
829
+ },
830
+ {
831
+ "id": 145,
832
+ "content": "<extra_id_85>",
833
+ "single_word": false,
834
+ "lstrip": false,
835
+ "rstrip": false,
836
+ "normalized": false,
837
+ "special": true
838
+ },
839
+ {
840
+ "id": 146,
841
+ "content": "<extra_id_86>",
842
+ "single_word": false,
843
+ "lstrip": false,
844
+ "rstrip": false,
845
+ "normalized": false,
846
+ "special": true
847
+ },
848
+ {
849
+ "id": 147,
850
+ "content": "<extra_id_87>",
851
+ "single_word": false,
852
+ "lstrip": false,
853
+ "rstrip": false,
854
+ "normalized": false,
855
+ "special": true
856
+ },
857
+ {
858
+ "id": 148,
859
+ "content": "<extra_id_88>",
860
+ "single_word": false,
861
+ "lstrip": false,
862
+ "rstrip": false,
863
+ "normalized": false,
864
+ "special": true
865
+ },
866
+ {
867
+ "id": 149,
868
+ "content": "<extra_id_89>",
869
+ "single_word": false,
870
+ "lstrip": false,
871
+ "rstrip": false,
872
+ "normalized": false,
873
+ "special": true
874
+ },
875
+ {
876
+ "id": 150,
877
+ "content": "<extra_id_90>",
878
+ "single_word": false,
879
+ "lstrip": false,
880
+ "rstrip": false,
881
+ "normalized": false,
882
+ "special": true
883
+ },
884
+ {
885
+ "id": 151,
886
+ "content": "<extra_id_91>",
887
+ "single_word": false,
888
+ "lstrip": false,
889
+ "rstrip": false,
890
+ "normalized": false,
891
+ "special": true
892
+ },
893
+ {
894
+ "id": 152,
895
+ "content": "<extra_id_92>",
896
+ "single_word": false,
897
+ "lstrip": false,
898
+ "rstrip": false,
899
+ "normalized": false,
900
+ "special": true
901
+ },
902
+ {
903
+ "id": 153,
904
+ "content": "<extra_id_93>",
905
+ "single_word": false,
906
+ "lstrip": false,
907
+ "rstrip": false,
908
+ "normalized": false,
909
+ "special": true
910
+ },
911
+ {
912
+ "id": 154,
913
+ "content": "<extra_id_94>",
914
+ "single_word": false,
915
+ "lstrip": false,
916
+ "rstrip": false,
917
+ "normalized": false,
918
+ "special": true
919
+ },
920
+ {
921
+ "id": 155,
922
+ "content": "<extra_id_95>",
923
+ "single_word": false,
924
+ "lstrip": false,
925
+ "rstrip": false,
926
+ "normalized": false,
927
+ "special": true
928
+ },
929
+ {
930
+ "id": 156,
931
+ "content": "<extra_id_96>",
932
+ "single_word": false,
933
+ "lstrip": false,
934
+ "rstrip": false,
935
+ "normalized": false,
936
+ "special": true
937
+ },
938
+ {
939
+ "id": 157,
940
+ "content": "<extra_id_97>",
941
+ "single_word": false,
942
+ "lstrip": false,
943
+ "rstrip": false,
944
+ "normalized": false,
945
+ "special": true
946
+ },
947
+ {
948
+ "id": 158,
949
+ "content": "<extra_id_98>",
950
+ "single_word": false,
951
+ "lstrip": false,
952
+ "rstrip": false,
953
+ "normalized": false,
954
+ "special": true
955
+ },
956
+ {
957
+ "id": 159,
958
+ "content": "<extra_id_99>",
959
+ "single_word": false,
960
+ "lstrip": false,
961
+ "rstrip": false,
962
+ "normalized": false,
963
+ "special": true
964
+ }
965
+ ],
966
+ "normalizer": {
967
+ "type": "Sequence",
968
+ "normalizers": [
969
+ {
970
+ "type": "NFD"
971
+ },
972
+ {
973
+ "type": "Lowercase"
974
+ },
975
+ {
976
+ "type": "StripAccents"
977
+ }
978
+ ]
979
+ },
980
+ "pre_tokenizer": {
981
+ "type": "Whitespace"
982
+ },
983
+ "post_processor": {
984
+ "type": "TemplateProcessing",
985
+ "single": [
986
+ {
987
+ "SpecialToken": {
988
+ "id": "<s>",
989
+ "type_id": 0
990
+ }
991
+ },
992
+ {
993
+ "Sequence": {
994
+ "id": "A",
995
+ "type_id": 0
996
+ }
997
+ },
998
+ {
999
+ "SpecialToken": {
1000
+ "id": "</s>",
1001
+ "type_id": 0
1002
+ }
1003
+ }
1004
+ ],
1005
+ "pair": [
1006
+ {
1007
+ "SpecialToken": {
1008
+ "id": "<s>",
1009
+ "type_id": 0
1010
+ }
1011
+ },
1012
+ {
1013
+ "Sequence": {
1014
+ "id": "A",
1015
+ "type_id": 0
1016
+ }
1017
+ },
1018
+ {
1019
+ "SpecialToken": {
1020
+ "id": "</s>",
1021
+ "type_id": 0
1022
+ }
1023
+ },
1024
+ {
1025
+ "SpecialToken": {
1026
+ "id": "</s>",
1027
+ "type_id": 0
1028
+ }
1029
+ },
1030
+ {
1031
+ "Sequence": {
1032
+ "id": "B",
1033
+ "type_id": 0
1034
+ }
1035
+ },
1036
+ {
1037
+ "SpecialToken": {
1038
+ "id": "</s>",
1039
+ "type_id": 0
1040
+ }
1041
+ }
1042
+ ],
1043
+ "special_tokens": {
1044
+ "</s>": {
1045
+ "id": "</s>",
1046
+ "ids": [
1047
+ 2
1048
+ ],
1049
+ "tokens": [
1050
+ "</s>"
1051
+ ]
1052
+ },
1053
+ "<s>": {
1054
+ "id": "<s>",
1055
+ "ids": [
1056
+ 1
1057
+ ],
1058
+ "tokens": [
1059
+ "<s>"
1060
+ ]
1061
+ }
1062
+ }
1063
+ },
1064
+ "decoder": null,
1065
+ "model": {
1066
+ "type": "BPE",
1067
+ "dropout": null,
1068
+ "unk_token": null,
1069
+ "continuing_subword_prefix": null,
1070
+ "end_of_word_suffix": null,
1071
+ "fuse_unk": false,
1072
+ "byte_fallback": false,
1073
+ "ignore_merges": false,
1074
+ "vocab": {
1075
+ "<pad>": 0,
1076
+ "<s>": 1,
1077
+ "</s>": 2,
1078
+ "<unk>": 3,
1079
+ "<mask>": 4,
1080
+ "!": 5,
1081
+ "\"": 6,
1082
+ "(": 7,
1083
+ ")": 8,
1084
+ "+": 9,
1085
+ ",": 10,
1086
+ ":": 11,
1087
+ "a": 12,
1088
+ "b": 13,
1089
+ "c": 14,
1090
+ "d": 15,
1091
+ "e": 16,
1092
+ "f": 17,
1093
+ "h": 18,
1094
+ "i": 19,
1095
+ "l": 20,
1096
+ "n": 21,
1097
+ "o": 22,
1098
+ "p": 23,
1099
+ "r": 24,
1100
+ "s": 25,
1101
+ "t": 26,
1102
+ "u": 27,
1103
+ "w": 28,
1104
+ "as": 29,
1105
+ "el": 30,
1106
+ "hel": 31,
1107
+ "ld": 32,
1108
+ "lo": 33,
1109
+ "or": 34,
1110
+ "wor": 35,
1111
+ "ass": 36,
1112
+ "hello": 37,
1113
+ "world": 38,
1114
+ "!\"": 39,
1115
+ "(\"": 40,
1116
+ "):": 41,
1117
+ "ad": 42,
1118
+ "cl": 43,
1119
+ "de": 44,
1120
+ "et": 45,
1121
+ "in": 46,
1122
+ "pr": 47,
1123
+ "pass": 48,
1124
+ "rn": 49,
1125
+ "ret": 50,
1126
+ "urn": 51,
1127
+ "helloworld": 52,
1128
+ "!\")": 53,
1129
+ "add": 54,
1130
+ "class": 55,
1131
+ "def": 56,
1132
+ "int": 57,
1133
+ "print": 58,
1134
+ "return": 59
1135
+ },
1136
+ "merges": [
1137
+ [
1138
+ "a",
1139
+ "s"
1140
+ ],
1141
+ [
1142
+ "e",
1143
+ "l"
1144
+ ],
1145
+ [
1146
+ "h",
1147
+ "el"
1148
+ ],
1149
+ [
1150
+ "l",
1151
+ "d"
1152
+ ],
1153
+ [
1154
+ "l",
1155
+ "o"
1156
+ ],
1157
+ [
1158
+ "o",
1159
+ "r"
1160
+ ],
1161
+ [
1162
+ "w",
1163
+ "or"
1164
+ ],
1165
+ [
1166
+ "as",
1167
+ "s"
1168
+ ],
1169
+ [
1170
+ "hel",
1171
+ "lo"
1172
+ ],
1173
+ [
1174
+ "wor",
1175
+ "ld"
1176
+ ],
1177
+ [
1178
+ "!",
1179
+ "\""
1180
+ ],
1181
+ [
1182
+ "(",
1183
+ "\""
1184
+ ],
1185
+ [
1186
+ ")",
1187
+ ":"
1188
+ ],
1189
+ [
1190
+ "a",
1191
+ "d"
1192
+ ],
1193
+ [
1194
+ "c",
1195
+ "l"
1196
+ ],
1197
+ [
1198
+ "d",
1199
+ "e"
1200
+ ],
1201
+ [
1202
+ "e",
1203
+ "t"
1204
+ ],
1205
+ [
1206
+ "i",
1207
+ "n"
1208
+ ],
1209
+ [
1210
+ "p",
1211
+ "r"
1212
+ ],
1213
+ [
1214
+ "p",
1215
+ "ass"
1216
+ ],
1217
+ [
1218
+ "r",
1219
+ "n"
1220
+ ],
1221
+ [
1222
+ "r",
1223
+ "et"
1224
+ ],
1225
+ [
1226
+ "u",
1227
+ "rn"
1228
+ ],
1229
+ [
1230
+ "hello",
1231
+ "world"
1232
+ ],
1233
+ [
1234
+ "!\"",
1235
+ ")"
1236
+ ],
1237
+ [
1238
+ "ad",
1239
+ "d"
1240
+ ],
1241
+ [
1242
+ "cl",
1243
+ "ass"
1244
+ ],
1245
+ [
1246
+ "de",
1247
+ "f"
1248
+ ],
1249
+ [
1250
+ "in",
1251
+ "t"
1252
+ ],
1253
+ [
1254
+ "pr",
1255
+ "int"
1256
+ ],
1257
+ [
1258
+ "ret",
1259
+ "urn"
1260
+ ]
1261
+ ]
1262
+ }
1263
+ }
minicoderx-model/checkpoint-2/tokenizer_config.json ADDED
@@ -0,0 +1,958 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": null,
3
+ "added_tokens_decoder": {
4
+ "0": {
5
+ "content": "<pad>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "1": {
13
+ "content": "<s>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "2": {
21
+ "content": "</s>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "3": {
29
+ "content": "<unk>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "4": {
37
+ "content": "<mask>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "60": {
45
+ "content": "<extra_id_0>",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false,
50
+ "special": true
51
+ },
52
+ "61": {
53
+ "content": "<extra_id_1>",
54
+ "lstrip": false,
55
+ "normalized": false,
56
+ "rstrip": false,
57
+ "single_word": false,
58
+ "special": true
59
+ },
60
+ "62": {
61
+ "content": "<extra_id_2>",
62
+ "lstrip": false,
63
+ "normalized": false,
64
+ "rstrip": false,
65
+ "single_word": false,
66
+ "special": true
67
+ },
68
+ "63": {
69
+ "content": "<extra_id_3>",
70
+ "lstrip": false,
71
+ "normalized": false,
72
+ "rstrip": false,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "64": {
77
+ "content": "<extra_id_4>",
78
+ "lstrip": false,
79
+ "normalized": false,
80
+ "rstrip": false,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "65": {
85
+ "content": "<extra_id_5>",
86
+ "lstrip": false,
87
+ "normalized": false,
88
+ "rstrip": false,
89
+ "single_word": false,
90
+ "special": true
91
+ },
92
+ "66": {
93
+ "content": "<extra_id_6>",
94
+ "lstrip": false,
95
+ "normalized": false,
96
+ "rstrip": false,
97
+ "single_word": false,
98
+ "special": true
99
+ },
100
+ "67": {
101
+ "content": "<extra_id_7>",
102
+ "lstrip": false,
103
+ "normalized": false,
104
+ "rstrip": false,
105
+ "single_word": false,
106
+ "special": true
107
+ },
108
+ "68": {
109
+ "content": "<extra_id_8>",
110
+ "lstrip": false,
111
+ "normalized": false,
112
+ "rstrip": false,
113
+ "single_word": false,
114
+ "special": true
115
+ },
116
+ "69": {
117
+ "content": "<extra_id_9>",
118
+ "lstrip": false,
119
+ "normalized": false,
120
+ "rstrip": false,
121
+ "single_word": false,
122
+ "special": true
123
+ },
124
+ "70": {
125
+ "content": "<extra_id_10>",
126
+ "lstrip": false,
127
+ "normalized": false,
128
+ "rstrip": false,
129
+ "single_word": false,
130
+ "special": true
131
+ },
132
+ "71": {
133
+ "content": "<extra_id_11>",
134
+ "lstrip": false,
135
+ "normalized": false,
136
+ "rstrip": false,
137
+ "single_word": false,
138
+ "special": true
139
+ },
140
+ "72": {
141
+ "content": "<extra_id_12>",
142
+ "lstrip": false,
143
+ "normalized": false,
144
+ "rstrip": false,
145
+ "single_word": false,
146
+ "special": true
147
+ },
148
+ "73": {
149
+ "content": "<extra_id_13>",
150
+ "lstrip": false,
151
+ "normalized": false,
152
+ "rstrip": false,
153
+ "single_word": false,
154
+ "special": true
155
+ },
156
+ "74": {
157
+ "content": "<extra_id_14>",
158
+ "lstrip": false,
159
+ "normalized": false,
160
+ "rstrip": false,
161
+ "single_word": false,
162
+ "special": true
163
+ },
164
+ "75": {
165
+ "content": "<extra_id_15>",
166
+ "lstrip": false,
167
+ "normalized": false,
168
+ "rstrip": false,
169
+ "single_word": false,
170
+ "special": true
171
+ },
172
+ "76": {
173
+ "content": "<extra_id_16>",
174
+ "lstrip": false,
175
+ "normalized": false,
176
+ "rstrip": false,
177
+ "single_word": false,
178
+ "special": true
179
+ },
180
+ "77": {
181
+ "content": "<extra_id_17>",
182
+ "lstrip": false,
183
+ "normalized": false,
184
+ "rstrip": false,
185
+ "single_word": false,
186
+ "special": true
187
+ },
188
+ "78": {
189
+ "content": "<extra_id_18>",
190
+ "lstrip": false,
191
+ "normalized": false,
192
+ "rstrip": false,
193
+ "single_word": false,
194
+ "special": true
195
+ },
196
+ "79": {
197
+ "content": "<extra_id_19>",
198
+ "lstrip": false,
199
+ "normalized": false,
200
+ "rstrip": false,
201
+ "single_word": false,
202
+ "special": true
203
+ },
204
+ "80": {
205
+ "content": "<extra_id_20>",
206
+ "lstrip": false,
207
+ "normalized": false,
208
+ "rstrip": false,
209
+ "single_word": false,
210
+ "special": true
211
+ },
212
+ "81": {
213
+ "content": "<extra_id_21>",
214
+ "lstrip": false,
215
+ "normalized": false,
216
+ "rstrip": false,
217
+ "single_word": false,
218
+ "special": true
219
+ },
220
+ "82": {
221
+ "content": "<extra_id_22>",
222
+ "lstrip": false,
223
+ "normalized": false,
224
+ "rstrip": false,
225
+ "single_word": false,
226
+ "special": true
227
+ },
228
+ "83": {
229
+ "content": "<extra_id_23>",
230
+ "lstrip": false,
231
+ "normalized": false,
232
+ "rstrip": false,
233
+ "single_word": false,
234
+ "special": true
235
+ },
236
+ "84": {
237
+ "content": "<extra_id_24>",
238
+ "lstrip": false,
239
+ "normalized": false,
240
+ "rstrip": false,
241
+ "single_word": false,
242
+ "special": true
243
+ },
244
+ "85": {
245
+ "content": "<extra_id_25>",
246
+ "lstrip": false,
247
+ "normalized": false,
248
+ "rstrip": false,
249
+ "single_word": false,
250
+ "special": true
251
+ },
252
+ "86": {
253
+ "content": "<extra_id_26>",
254
+ "lstrip": false,
255
+ "normalized": false,
256
+ "rstrip": false,
257
+ "single_word": false,
258
+ "special": true
259
+ },
260
+ "87": {
261
+ "content": "<extra_id_27>",
262
+ "lstrip": false,
263
+ "normalized": false,
264
+ "rstrip": false,
265
+ "single_word": false,
266
+ "special": true
267
+ },
268
+ "88": {
269
+ "content": "<extra_id_28>",
270
+ "lstrip": false,
271
+ "normalized": false,
272
+ "rstrip": false,
273
+ "single_word": false,
274
+ "special": true
275
+ },
276
+ "89": {
277
+ "content": "<extra_id_29>",
278
+ "lstrip": false,
279
+ "normalized": false,
280
+ "rstrip": false,
281
+ "single_word": false,
282
+ "special": true
283
+ },
284
+ "90": {
285
+ "content": "<extra_id_30>",
286
+ "lstrip": false,
287
+ "normalized": false,
288
+ "rstrip": false,
289
+ "single_word": false,
290
+ "special": true
291
+ },
292
+ "91": {
293
+ "content": "<extra_id_31>",
294
+ "lstrip": false,
295
+ "normalized": false,
296
+ "rstrip": false,
297
+ "single_word": false,
298
+ "special": true
299
+ },
300
+ "92": {
301
+ "content": "<extra_id_32>",
302
+ "lstrip": false,
303
+ "normalized": false,
304
+ "rstrip": false,
305
+ "single_word": false,
306
+ "special": true
307
+ },
308
+ "93": {
309
+ "content": "<extra_id_33>",
310
+ "lstrip": false,
311
+ "normalized": false,
312
+ "rstrip": false,
313
+ "single_word": false,
314
+ "special": true
315
+ },
316
+ "94": {
317
+ "content": "<extra_id_34>",
318
+ "lstrip": false,
319
+ "normalized": false,
320
+ "rstrip": false,
321
+ "single_word": false,
322
+ "special": true
323
+ },
324
+ "95": {
325
+ "content": "<extra_id_35>",
326
+ "lstrip": false,
327
+ "normalized": false,
328
+ "rstrip": false,
329
+ "single_word": false,
330
+ "special": true
331
+ },
332
+ "96": {
333
+ "content": "<extra_id_36>",
334
+ "lstrip": false,
335
+ "normalized": false,
336
+ "rstrip": false,
337
+ "single_word": false,
338
+ "special": true
339
+ },
340
+ "97": {
341
+ "content": "<extra_id_37>",
342
+ "lstrip": false,
343
+ "normalized": false,
344
+ "rstrip": false,
345
+ "single_word": false,
346
+ "special": true
347
+ },
348
+ "98": {
349
+ "content": "<extra_id_38>",
350
+ "lstrip": false,
351
+ "normalized": false,
352
+ "rstrip": false,
353
+ "single_word": false,
354
+ "special": true
355
+ },
356
+ "99": {
357
+ "content": "<extra_id_39>",
358
+ "lstrip": false,
359
+ "normalized": false,
360
+ "rstrip": false,
361
+ "single_word": false,
362
+ "special": true
363
+ },
364
+ "100": {
365
+ "content": "<extra_id_40>",
366
+ "lstrip": false,
367
+ "normalized": false,
368
+ "rstrip": false,
369
+ "single_word": false,
370
+ "special": true
371
+ },
372
+ "101": {
373
+ "content": "<extra_id_41>",
374
+ "lstrip": false,
375
+ "normalized": false,
376
+ "rstrip": false,
377
+ "single_word": false,
378
+ "special": true
379
+ },
380
+ "102": {
381
+ "content": "<extra_id_42>",
382
+ "lstrip": false,
383
+ "normalized": false,
384
+ "rstrip": false,
385
+ "single_word": false,
386
+ "special": true
387
+ },
388
+ "103": {
389
+ "content": "<extra_id_43>",
390
+ "lstrip": false,
391
+ "normalized": false,
392
+ "rstrip": false,
393
+ "single_word": false,
394
+ "special": true
395
+ },
396
+ "104": {
397
+ "content": "<extra_id_44>",
398
+ "lstrip": false,
399
+ "normalized": false,
400
+ "rstrip": false,
401
+ "single_word": false,
402
+ "special": true
403
+ },
404
+ "105": {
405
+ "content": "<extra_id_45>",
406
+ "lstrip": false,
407
+ "normalized": false,
408
+ "rstrip": false,
409
+ "single_word": false,
410
+ "special": true
411
+ },
412
+ "106": {
413
+ "content": "<extra_id_46>",
414
+ "lstrip": false,
415
+ "normalized": false,
416
+ "rstrip": false,
417
+ "single_word": false,
418
+ "special": true
419
+ },
420
+ "107": {
421
+ "content": "<extra_id_47>",
422
+ "lstrip": false,
423
+ "normalized": false,
424
+ "rstrip": false,
425
+ "single_word": false,
426
+ "special": true
427
+ },
428
+ "108": {
429
+ "content": "<extra_id_48>",
430
+ "lstrip": false,
431
+ "normalized": false,
432
+ "rstrip": false,
433
+ "single_word": false,
434
+ "special": true
435
+ },
436
+ "109": {
437
+ "content": "<extra_id_49>",
438
+ "lstrip": false,
439
+ "normalized": false,
440
+ "rstrip": false,
441
+ "single_word": false,
442
+ "special": true
443
+ },
444
+ "110": {
445
+ "content": "<extra_id_50>",
446
+ "lstrip": false,
447
+ "normalized": false,
448
+ "rstrip": false,
449
+ "single_word": false,
450
+ "special": true
451
+ },
452
+ "111": {
453
+ "content": "<extra_id_51>",
454
+ "lstrip": false,
455
+ "normalized": false,
456
+ "rstrip": false,
457
+ "single_word": false,
458
+ "special": true
459
+ },
460
+ "112": {
461
+ "content": "<extra_id_52>",
462
+ "lstrip": false,
463
+ "normalized": false,
464
+ "rstrip": false,
465
+ "single_word": false,
466
+ "special": true
467
+ },
468
+ "113": {
469
+ "content": "<extra_id_53>",
470
+ "lstrip": false,
471
+ "normalized": false,
472
+ "rstrip": false,
473
+ "single_word": false,
474
+ "special": true
475
+ },
476
+ "114": {
477
+ "content": "<extra_id_54>",
478
+ "lstrip": false,
479
+ "normalized": false,
480
+ "rstrip": false,
481
+ "single_word": false,
482
+ "special": true
483
+ },
484
+ "115": {
485
+ "content": "<extra_id_55>",
486
+ "lstrip": false,
487
+ "normalized": false,
488
+ "rstrip": false,
489
+ "single_word": false,
490
+ "special": true
491
+ },
492
+ "116": {
493
+ "content": "<extra_id_56>",
494
+ "lstrip": false,
495
+ "normalized": false,
496
+ "rstrip": false,
497
+ "single_word": false,
498
+ "special": true
499
+ },
500
+ "117": {
501
+ "content": "<extra_id_57>",
502
+ "lstrip": false,
503
+ "normalized": false,
504
+ "rstrip": false,
505
+ "single_word": false,
506
+ "special": true
507
+ },
508
+ "118": {
509
+ "content": "<extra_id_58>",
510
+ "lstrip": false,
511
+ "normalized": false,
512
+ "rstrip": false,
513
+ "single_word": false,
514
+ "special": true
515
+ },
516
+ "119": {
517
+ "content": "<extra_id_59>",
518
+ "lstrip": false,
519
+ "normalized": false,
520
+ "rstrip": false,
521
+ "single_word": false,
522
+ "special": true
523
+ },
524
+ "120": {
525
+ "content": "<extra_id_60>",
526
+ "lstrip": false,
527
+ "normalized": false,
528
+ "rstrip": false,
529
+ "single_word": false,
530
+ "special": true
531
+ },
532
+ "121": {
533
+ "content": "<extra_id_61>",
534
+ "lstrip": false,
535
+ "normalized": false,
536
+ "rstrip": false,
537
+ "single_word": false,
538
+ "special": true
539
+ },
540
+ "122": {
541
+ "content": "<extra_id_62>",
542
+ "lstrip": false,
543
+ "normalized": false,
544
+ "rstrip": false,
545
+ "single_word": false,
546
+ "special": true
547
+ },
548
+ "123": {
549
+ "content": "<extra_id_63>",
550
+ "lstrip": false,
551
+ "normalized": false,
552
+ "rstrip": false,
553
+ "single_word": false,
554
+ "special": true
555
+ },
556
+ "124": {
557
+ "content": "<extra_id_64>",
558
+ "lstrip": false,
559
+ "normalized": false,
560
+ "rstrip": false,
561
+ "single_word": false,
562
+ "special": true
563
+ },
564
+ "125": {
565
+ "content": "<extra_id_65>",
566
+ "lstrip": false,
567
+ "normalized": false,
568
+ "rstrip": false,
569
+ "single_word": false,
570
+ "special": true
571
+ },
572
+ "126": {
573
+ "content": "<extra_id_66>",
574
+ "lstrip": false,
575
+ "normalized": false,
576
+ "rstrip": false,
577
+ "single_word": false,
578
+ "special": true
579
+ },
580
+ "127": {
581
+ "content": "<extra_id_67>",
582
+ "lstrip": false,
583
+ "normalized": false,
584
+ "rstrip": false,
585
+ "single_word": false,
586
+ "special": true
587
+ },
588
+ "128": {
589
+ "content": "<extra_id_68>",
590
+ "lstrip": false,
591
+ "normalized": false,
592
+ "rstrip": false,
593
+ "single_word": false,
594
+ "special": true
595
+ },
596
+ "129": {
597
+ "content": "<extra_id_69>",
598
+ "lstrip": false,
599
+ "normalized": false,
600
+ "rstrip": false,
601
+ "single_word": false,
602
+ "special": true
603
+ },
604
+ "130": {
605
+ "content": "<extra_id_70>",
606
+ "lstrip": false,
607
+ "normalized": false,
608
+ "rstrip": false,
609
+ "single_word": false,
610
+ "special": true
611
+ },
612
+ "131": {
613
+ "content": "<extra_id_71>",
614
+ "lstrip": false,
615
+ "normalized": false,
616
+ "rstrip": false,
617
+ "single_word": false,
618
+ "special": true
619
+ },
620
+ "132": {
621
+ "content": "<extra_id_72>",
622
+ "lstrip": false,
623
+ "normalized": false,
624
+ "rstrip": false,
625
+ "single_word": false,
626
+ "special": true
627
+ },
628
+ "133": {
629
+ "content": "<extra_id_73>",
630
+ "lstrip": false,
631
+ "normalized": false,
632
+ "rstrip": false,
633
+ "single_word": false,
634
+ "special": true
635
+ },
636
+ "134": {
637
+ "content": "<extra_id_74>",
638
+ "lstrip": false,
639
+ "normalized": false,
640
+ "rstrip": false,
641
+ "single_word": false,
642
+ "special": true
643
+ },
644
+ "135": {
645
+ "content": "<extra_id_75>",
646
+ "lstrip": false,
647
+ "normalized": false,
648
+ "rstrip": false,
649
+ "single_word": false,
650
+ "special": true
651
+ },
652
+ "136": {
653
+ "content": "<extra_id_76>",
654
+ "lstrip": false,
655
+ "normalized": false,
656
+ "rstrip": false,
657
+ "single_word": false,
658
+ "special": true
659
+ },
660
+ "137": {
661
+ "content": "<extra_id_77>",
662
+ "lstrip": false,
663
+ "normalized": false,
664
+ "rstrip": false,
665
+ "single_word": false,
666
+ "special": true
667
+ },
668
+ "138": {
669
+ "content": "<extra_id_78>",
670
+ "lstrip": false,
671
+ "normalized": false,
672
+ "rstrip": false,
673
+ "single_word": false,
674
+ "special": true
675
+ },
676
+ "139": {
677
+ "content": "<extra_id_79>",
678
+ "lstrip": false,
679
+ "normalized": false,
680
+ "rstrip": false,
681
+ "single_word": false,
682
+ "special": true
683
+ },
684
+ "140": {
685
+ "content": "<extra_id_80>",
686
+ "lstrip": false,
687
+ "normalized": false,
688
+ "rstrip": false,
689
+ "single_word": false,
690
+ "special": true
691
+ },
692
+ "141": {
693
+ "content": "<extra_id_81>",
694
+ "lstrip": false,
695
+ "normalized": false,
696
+ "rstrip": false,
697
+ "single_word": false,
698
+ "special": true
699
+ },
700
+ "142": {
701
+ "content": "<extra_id_82>",
702
+ "lstrip": false,
703
+ "normalized": false,
704
+ "rstrip": false,
705
+ "single_word": false,
706
+ "special": true
707
+ },
708
+ "143": {
709
+ "content": "<extra_id_83>",
710
+ "lstrip": false,
711
+ "normalized": false,
712
+ "rstrip": false,
713
+ "single_word": false,
714
+ "special": true
715
+ },
716
+ "144": {
717
+ "content": "<extra_id_84>",
718
+ "lstrip": false,
719
+ "normalized": false,
720
+ "rstrip": false,
721
+ "single_word": false,
722
+ "special": true
723
+ },
724
+ "145": {
725
+ "content": "<extra_id_85>",
726
+ "lstrip": false,
727
+ "normalized": false,
728
+ "rstrip": false,
729
+ "single_word": false,
730
+ "special": true
731
+ },
732
+ "146": {
733
+ "content": "<extra_id_86>",
734
+ "lstrip": false,
735
+ "normalized": false,
736
+ "rstrip": false,
737
+ "single_word": false,
738
+ "special": true
739
+ },
740
+ "147": {
741
+ "content": "<extra_id_87>",
742
+ "lstrip": false,
743
+ "normalized": false,
744
+ "rstrip": false,
745
+ "single_word": false,
746
+ "special": true
747
+ },
748
+ "148": {
749
+ "content": "<extra_id_88>",
750
+ "lstrip": false,
751
+ "normalized": false,
752
+ "rstrip": false,
753
+ "single_word": false,
754
+ "special": true
755
+ },
756
+ "149": {
757
+ "content": "<extra_id_89>",
758
+ "lstrip": false,
759
+ "normalized": false,
760
+ "rstrip": false,
761
+ "single_word": false,
762
+ "special": true
763
+ },
764
+ "150": {
765
+ "content": "<extra_id_90>",
766
+ "lstrip": false,
767
+ "normalized": false,
768
+ "rstrip": false,
769
+ "single_word": false,
770
+ "special": true
771
+ },
772
+ "151": {
773
+ "content": "<extra_id_91>",
774
+ "lstrip": false,
775
+ "normalized": false,
776
+ "rstrip": false,
777
+ "single_word": false,
778
+ "special": true
779
+ },
780
+ "152": {
781
+ "content": "<extra_id_92>",
782
+ "lstrip": false,
783
+ "normalized": false,
784
+ "rstrip": false,
785
+ "single_word": false,
786
+ "special": true
787
+ },
788
+ "153": {
789
+ "content": "<extra_id_93>",
790
+ "lstrip": false,
791
+ "normalized": false,
792
+ "rstrip": false,
793
+ "single_word": false,
794
+ "special": true
795
+ },
796
+ "154": {
797
+ "content": "<extra_id_94>",
798
+ "lstrip": false,
799
+ "normalized": false,
800
+ "rstrip": false,
801
+ "single_word": false,
802
+ "special": true
803
+ },
804
+ "155": {
805
+ "content": "<extra_id_95>",
806
+ "lstrip": false,
807
+ "normalized": false,
808
+ "rstrip": false,
809
+ "single_word": false,
810
+ "special": true
811
+ },
812
+ "156": {
813
+ "content": "<extra_id_96>",
814
+ "lstrip": false,
815
+ "normalized": false,
816
+ "rstrip": false,
817
+ "single_word": false,
818
+ "special": true
819
+ },
820
+ "157": {
821
+ "content": "<extra_id_97>",
822
+ "lstrip": false,
823
+ "normalized": false,
824
+ "rstrip": false,
825
+ "single_word": false,
826
+ "special": true
827
+ },
828
+ "158": {
829
+ "content": "<extra_id_98>",
830
+ "lstrip": false,
831
+ "normalized": false,
832
+ "rstrip": false,
833
+ "single_word": false,
834
+ "special": true
835
+ },
836
+ "159": {
837
+ "content": "<extra_id_99>",
838
+ "lstrip": false,
839
+ "normalized": false,
840
+ "rstrip": false,
841
+ "single_word": false,
842
+ "special": true
843
+ }
844
+ },
845
+ "additional_special_tokens": [
846
+ "<extra_id_0>",
847
+ "<extra_id_1>",
848
+ "<extra_id_2>",
849
+ "<extra_id_3>",
850
+ "<extra_id_4>",
851
+ "<extra_id_5>",
852
+ "<extra_id_6>",
853
+ "<extra_id_7>",
854
+ "<extra_id_8>",
855
+ "<extra_id_9>",
856
+ "<extra_id_10>",
857
+ "<extra_id_11>",
858
+ "<extra_id_12>",
859
+ "<extra_id_13>",
860
+ "<extra_id_14>",
861
+ "<extra_id_15>",
862
+ "<extra_id_16>",
863
+ "<extra_id_17>",
864
+ "<extra_id_18>",
865
+ "<extra_id_19>",
866
+ "<extra_id_20>",
867
+ "<extra_id_21>",
868
+ "<extra_id_22>",
869
+ "<extra_id_23>",
870
+ "<extra_id_24>",
871
+ "<extra_id_25>",
872
+ "<extra_id_26>",
873
+ "<extra_id_27>",
874
+ "<extra_id_28>",
875
+ "<extra_id_29>",
876
+ "<extra_id_30>",
877
+ "<extra_id_31>",
878
+ "<extra_id_32>",
879
+ "<extra_id_33>",
880
+ "<extra_id_34>",
881
+ "<extra_id_35>",
882
+ "<extra_id_36>",
883
+ "<extra_id_37>",
884
+ "<extra_id_38>",
885
+ "<extra_id_39>",
886
+ "<extra_id_40>",
887
+ "<extra_id_41>",
888
+ "<extra_id_42>",
889
+ "<extra_id_43>",
890
+ "<extra_id_44>",
891
+ "<extra_id_45>",
892
+ "<extra_id_46>",
893
+ "<extra_id_47>",
894
+ "<extra_id_48>",
895
+ "<extra_id_49>",
896
+ "<extra_id_50>",
897
+ "<extra_id_51>",
898
+ "<extra_id_52>",
899
+ "<extra_id_53>",
900
+ "<extra_id_54>",
901
+ "<extra_id_55>",
902
+ "<extra_id_56>",
903
+ "<extra_id_57>",
904
+ "<extra_id_58>",
905
+ "<extra_id_59>",
906
+ "<extra_id_60>",
907
+ "<extra_id_61>",
908
+ "<extra_id_62>",
909
+ "<extra_id_63>",
910
+ "<extra_id_64>",
911
+ "<extra_id_65>",
912
+ "<extra_id_66>",
913
+ "<extra_id_67>",
914
+ "<extra_id_68>",
915
+ "<extra_id_69>",
916
+ "<extra_id_70>",
917
+ "<extra_id_71>",
918
+ "<extra_id_72>",
919
+ "<extra_id_73>",
920
+ "<extra_id_74>",
921
+ "<extra_id_75>",
922
+ "<extra_id_76>",
923
+ "<extra_id_77>",
924
+ "<extra_id_78>",
925
+ "<extra_id_79>",
926
+ "<extra_id_80>",
927
+ "<extra_id_81>",
928
+ "<extra_id_82>",
929
+ "<extra_id_83>",
930
+ "<extra_id_84>",
931
+ "<extra_id_85>",
932
+ "<extra_id_86>",
933
+ "<extra_id_87>",
934
+ "<extra_id_88>",
935
+ "<extra_id_89>",
936
+ "<extra_id_90>",
937
+ "<extra_id_91>",
938
+ "<extra_id_92>",
939
+ "<extra_id_93>",
940
+ "<extra_id_94>",
941
+ "<extra_id_95>",
942
+ "<extra_id_96>",
943
+ "<extra_id_97>",
944
+ "<extra_id_98>",
945
+ "<extra_id_99>"
946
+ ],
947
+ "clean_up_tokenization_spaces": false,
948
+ "cls_token": "<s>",
949
+ "eos_token": "</s>",
950
+ "extra_ids": 100,
951
+ "extra_special_tokens": {},
952
+ "mask_token": "<mask>",
953
+ "model_max_length": 1000000000000000019884624838656,
954
+ "pad_token": "<pad>",
955
+ "sep_token": "</s>",
956
+ "tokenizer_class": "T5TokenizerFast",
957
+ "unk_token": "<unk>"
958
+ }
minicoderx-model/checkpoint-2/trainer_state.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 2.0,
5
+ "eval_steps": 500,
6
+ "global_step": 2,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [],
11
+ "logging_steps": 10,
12
+ "max_steps": 3,
13
+ "num_input_tokens_seen": 0,
14
+ "num_train_epochs": 3,
15
+ "save_steps": 500,
16
+ "stateful_callbacks": {
17
+ "TrainerControl": {
18
+ "args": {
19
+ "should_epoch_stop": false,
20
+ "should_evaluate": false,
21
+ "should_log": false,
22
+ "should_save": true,
23
+ "should_training_stop": false
24
+ },
25
+ "attributes": {}
26
+ }
27
+ },
28
+ "total_flos": 135341801472.0,
29
+ "train_batch_size": 4,
30
+ "trial_name": null,
31
+ "trial_params": null
32
+ }
minicoderx-model/checkpoint-2/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd544df3e986cab3c0663ced7b6b3ca6903b0db2be881d69b5180d0077410a13
3
+ size 5304
minicoderx-model/checkpoint-3/config.json ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "t5-small",
3
+ "architectures": [
4
+ "T5ForConditionalGeneration"
5
+ ],
6
+ "classifier_dropout": 0.0,
7
+ "d_ff": 2048,
8
+ "d_kv": 64,
9
+ "d_model": 512,
10
+ "decoder_start_token_id": 0,
11
+ "dense_act_fn": "relu",
12
+ "dropout_rate": 0.1,
13
+ "eos_token_id": 1,
14
+ "feed_forward_proj": "relu",
15
+ "initializer_factor": 1.0,
16
+ "is_encoder_decoder": true,
17
+ "is_gated_act": false,
18
+ "layer_norm_epsilon": 1e-06,
19
+ "model_type": "t5",
20
+ "n_positions": 512,
21
+ "num_decoder_layers": 6,
22
+ "num_heads": 8,
23
+ "num_layers": 6,
24
+ "output_past": true,
25
+ "pad_token_id": 0,
26
+ "relative_attention_max_distance": 128,
27
+ "relative_attention_num_buckets": 32,
28
+ "task_specific_params": {
29
+ "summarization": {
30
+ "early_stopping": true,
31
+ "length_penalty": 2.0,
32
+ "max_length": 200,
33
+ "min_length": 30,
34
+ "no_repeat_ngram_size": 3,
35
+ "num_beams": 4,
36
+ "prefix": "summarize: "
37
+ },
38
+ "translation_en_to_de": {
39
+ "early_stopping": true,
40
+ "max_length": 300,
41
+ "num_beams": 4,
42
+ "prefix": "translate English to German: "
43
+ },
44
+ "translation_en_to_fr": {
45
+ "early_stopping": true,
46
+ "max_length": 300,
47
+ "num_beams": 4,
48
+ "prefix": "translate English to French: "
49
+ },
50
+ "translation_en_to_ro": {
51
+ "early_stopping": true,
52
+ "max_length": 300,
53
+ "num_beams": 4,
54
+ "prefix": "translate English to Romanian: "
55
+ }
56
+ },
57
+ "torch_dtype": "float32",
58
+ "transformers_version": "4.49.0",
59
+ "use_cache": true,
60
+ "vocab_size": 32128
61
+ }
minicoderx-model/checkpoint-3/generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "decoder_start_token_id": 0,
4
+ "eos_token_id": 1,
5
+ "pad_token_id": 0,
6
+ "transformers_version": "4.49.0"
7
+ }
minicoderx-model/checkpoint-3/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d58f23c7336fdae923aa51a04dcc36156d74b7fd8d1a336b6c9f62f80b899e69
3
+ size 242041896
minicoderx-model/checkpoint-3/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c2eb64c382fe83bd68b60ad8ec5356bbffb6f14c5bda4dddda53f669be44474e
3
+ size 484159866
minicoderx-model/checkpoint-3/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:337d385b937fddd772e2cc2209cc85d048e2e3ef39019b832d2ab98922f62043
3
+ size 13990
minicoderx-model/checkpoint-3/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b8a41b254d981983d12a5d0bed17f4768863ab9648e2daeb9b71c409365e3b73
3
+ size 1064
minicoderx-model/checkpoint-3/special_tokens_map.json ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<extra_id_0>",
4
+ "<extra_id_1>",
5
+ "<extra_id_2>",
6
+ "<extra_id_3>",
7
+ "<extra_id_4>",
8
+ "<extra_id_5>",
9
+ "<extra_id_6>",
10
+ "<extra_id_7>",
11
+ "<extra_id_8>",
12
+ "<extra_id_9>",
13
+ "<extra_id_10>",
14
+ "<extra_id_11>",
15
+ "<extra_id_12>",
16
+ "<extra_id_13>",
17
+ "<extra_id_14>",
18
+ "<extra_id_15>",
19
+ "<extra_id_16>",
20
+ "<extra_id_17>",
21
+ "<extra_id_18>",
22
+ "<extra_id_19>",
23
+ "<extra_id_20>",
24
+ "<extra_id_21>",
25
+ "<extra_id_22>",
26
+ "<extra_id_23>",
27
+ "<extra_id_24>",
28
+ "<extra_id_25>",
29
+ "<extra_id_26>",
30
+ "<extra_id_27>",
31
+ "<extra_id_28>",
32
+ "<extra_id_29>",
33
+ "<extra_id_30>",
34
+ "<extra_id_31>",
35
+ "<extra_id_32>",
36
+ "<extra_id_33>",
37
+ "<extra_id_34>",
38
+ "<extra_id_35>",
39
+ "<extra_id_36>",
40
+ "<extra_id_37>",
41
+ "<extra_id_38>",
42
+ "<extra_id_39>",
43
+ "<extra_id_40>",
44
+ "<extra_id_41>",
45
+ "<extra_id_42>",
46
+ "<extra_id_43>",
47
+ "<extra_id_44>",
48
+ "<extra_id_45>",
49
+ "<extra_id_46>",
50
+ "<extra_id_47>",
51
+ "<extra_id_48>",
52
+ "<extra_id_49>",
53
+ "<extra_id_50>",
54
+ "<extra_id_51>",
55
+ "<extra_id_52>",
56
+ "<extra_id_53>",
57
+ "<extra_id_54>",
58
+ "<extra_id_55>",
59
+ "<extra_id_56>",
60
+ "<extra_id_57>",
61
+ "<extra_id_58>",
62
+ "<extra_id_59>",
63
+ "<extra_id_60>",
64
+ "<extra_id_61>",
65
+ "<extra_id_62>",
66
+ "<extra_id_63>",
67
+ "<extra_id_64>",
68
+ "<extra_id_65>",
69
+ "<extra_id_66>",
70
+ "<extra_id_67>",
71
+ "<extra_id_68>",
72
+ "<extra_id_69>",
73
+ "<extra_id_70>",
74
+ "<extra_id_71>",
75
+ "<extra_id_72>",
76
+ "<extra_id_73>",
77
+ "<extra_id_74>",
78
+ "<extra_id_75>",
79
+ "<extra_id_76>",
80
+ "<extra_id_77>",
81
+ "<extra_id_78>",
82
+ "<extra_id_79>",
83
+ "<extra_id_80>",
84
+ "<extra_id_81>",
85
+ "<extra_id_82>",
86
+ "<extra_id_83>",
87
+ "<extra_id_84>",
88
+ "<extra_id_85>",
89
+ "<extra_id_86>",
90
+ "<extra_id_87>",
91
+ "<extra_id_88>",
92
+ "<extra_id_89>",
93
+ "<extra_id_90>",
94
+ "<extra_id_91>",
95
+ "<extra_id_92>",
96
+ "<extra_id_93>",
97
+ "<extra_id_94>",
98
+ "<extra_id_95>",
99
+ "<extra_id_96>",
100
+ "<extra_id_97>",
101
+ "<extra_id_98>",
102
+ "<extra_id_99>"
103
+ ],
104
+ "cls_token": {
105
+ "content": "<s>",
106
+ "lstrip": false,
107
+ "normalized": false,
108
+ "rstrip": false,
109
+ "single_word": false
110
+ },
111
+ "eos_token": "</s>",
112
+ "mask_token": {
113
+ "content": "<mask>",
114
+ "lstrip": false,
115
+ "normalized": false,
116
+ "rstrip": false,
117
+ "single_word": false
118
+ },
119
+ "pad_token": {
120
+ "content": "<pad>",
121
+ "lstrip": false,
122
+ "normalized": false,
123
+ "rstrip": false,
124
+ "single_word": false
125
+ },
126
+ "sep_token": {
127
+ "content": "</s>",
128
+ "lstrip": false,
129
+ "normalized": false,
130
+ "rstrip": false,
131
+ "single_word": false
132
+ },
133
+ "unk_token": {
134
+ "content": "<unk>",
135
+ "lstrip": false,
136
+ "normalized": false,
137
+ "rstrip": false,
138
+ "single_word": false
139
+ }
140
+ }
minicoderx-model/checkpoint-3/tokenizer.json ADDED
@@ -0,0 +1,1263 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": {
4
+ "direction": "Right",
5
+ "max_length": 128,
6
+ "strategy": "LongestFirst",
7
+ "stride": 0
8
+ },
9
+ "padding": {
10
+ "strategy": {
11
+ "Fixed": 128
12
+ },
13
+ "direction": "Right",
14
+ "pad_to_multiple_of": null,
15
+ "pad_id": 0,
16
+ "pad_type_id": 0,
17
+ "pad_token": "<pad>"
18
+ },
19
+ "added_tokens": [
20
+ {
21
+ "id": 0,
22
+ "content": "<pad>",
23
+ "single_word": false,
24
+ "lstrip": false,
25
+ "rstrip": false,
26
+ "normalized": false,
27
+ "special": true
28
+ },
29
+ {
30
+ "id": 1,
31
+ "content": "<s>",
32
+ "single_word": false,
33
+ "lstrip": false,
34
+ "rstrip": false,
35
+ "normalized": false,
36
+ "special": true
37
+ },
38
+ {
39
+ "id": 2,
40
+ "content": "</s>",
41
+ "single_word": false,
42
+ "lstrip": false,
43
+ "rstrip": false,
44
+ "normalized": false,
45
+ "special": true
46
+ },
47
+ {
48
+ "id": 3,
49
+ "content": "<unk>",
50
+ "single_word": false,
51
+ "lstrip": false,
52
+ "rstrip": false,
53
+ "normalized": false,
54
+ "special": true
55
+ },
56
+ {
57
+ "id": 4,
58
+ "content": "<mask>",
59
+ "single_word": false,
60
+ "lstrip": false,
61
+ "rstrip": false,
62
+ "normalized": false,
63
+ "special": true
64
+ },
65
+ {
66
+ "id": 60,
67
+ "content": "<extra_id_0>",
68
+ "single_word": false,
69
+ "lstrip": false,
70
+ "rstrip": false,
71
+ "normalized": false,
72
+ "special": true
73
+ },
74
+ {
75
+ "id": 61,
76
+ "content": "<extra_id_1>",
77
+ "single_word": false,
78
+ "lstrip": false,
79
+ "rstrip": false,
80
+ "normalized": false,
81
+ "special": true
82
+ },
83
+ {
84
+ "id": 62,
85
+ "content": "<extra_id_2>",
86
+ "single_word": false,
87
+ "lstrip": false,
88
+ "rstrip": false,
89
+ "normalized": false,
90
+ "special": true
91
+ },
92
+ {
93
+ "id": 63,
94
+ "content": "<extra_id_3>",
95
+ "single_word": false,
96
+ "lstrip": false,
97
+ "rstrip": false,
98
+ "normalized": false,
99
+ "special": true
100
+ },
101
+ {
102
+ "id": 64,
103
+ "content": "<extra_id_4>",
104
+ "single_word": false,
105
+ "lstrip": false,
106
+ "rstrip": false,
107
+ "normalized": false,
108
+ "special": true
109
+ },
110
+ {
111
+ "id": 65,
112
+ "content": "<extra_id_5>",
113
+ "single_word": false,
114
+ "lstrip": false,
115
+ "rstrip": false,
116
+ "normalized": false,
117
+ "special": true
118
+ },
119
+ {
120
+ "id": 66,
121
+ "content": "<extra_id_6>",
122
+ "single_word": false,
123
+ "lstrip": false,
124
+ "rstrip": false,
125
+ "normalized": false,
126
+ "special": true
127
+ },
128
+ {
129
+ "id": 67,
130
+ "content": "<extra_id_7>",
131
+ "single_word": false,
132
+ "lstrip": false,
133
+ "rstrip": false,
134
+ "normalized": false,
135
+ "special": true
136
+ },
137
+ {
138
+ "id": 68,
139
+ "content": "<extra_id_8>",
140
+ "single_word": false,
141
+ "lstrip": false,
142
+ "rstrip": false,
143
+ "normalized": false,
144
+ "special": true
145
+ },
146
+ {
147
+ "id": 69,
148
+ "content": "<extra_id_9>",
149
+ "single_word": false,
150
+ "lstrip": false,
151
+ "rstrip": false,
152
+ "normalized": false,
153
+ "special": true
154
+ },
155
+ {
156
+ "id": 70,
157
+ "content": "<extra_id_10>",
158
+ "single_word": false,
159
+ "lstrip": false,
160
+ "rstrip": false,
161
+ "normalized": false,
162
+ "special": true
163
+ },
164
+ {
165
+ "id": 71,
166
+ "content": "<extra_id_11>",
167
+ "single_word": false,
168
+ "lstrip": false,
169
+ "rstrip": false,
170
+ "normalized": false,
171
+ "special": true
172
+ },
173
+ {
174
+ "id": 72,
175
+ "content": "<extra_id_12>",
176
+ "single_word": false,
177
+ "lstrip": false,
178
+ "rstrip": false,
179
+ "normalized": false,
180
+ "special": true
181
+ },
182
+ {
183
+ "id": 73,
184
+ "content": "<extra_id_13>",
185
+ "single_word": false,
186
+ "lstrip": false,
187
+ "rstrip": false,
188
+ "normalized": false,
189
+ "special": true
190
+ },
191
+ {
192
+ "id": 74,
193
+ "content": "<extra_id_14>",
194
+ "single_word": false,
195
+ "lstrip": false,
196
+ "rstrip": false,
197
+ "normalized": false,
198
+ "special": true
199
+ },
200
+ {
201
+ "id": 75,
202
+ "content": "<extra_id_15>",
203
+ "single_word": false,
204
+ "lstrip": false,
205
+ "rstrip": false,
206
+ "normalized": false,
207
+ "special": true
208
+ },
209
+ {
210
+ "id": 76,
211
+ "content": "<extra_id_16>",
212
+ "single_word": false,
213
+ "lstrip": false,
214
+ "rstrip": false,
215
+ "normalized": false,
216
+ "special": true
217
+ },
218
+ {
219
+ "id": 77,
220
+ "content": "<extra_id_17>",
221
+ "single_word": false,
222
+ "lstrip": false,
223
+ "rstrip": false,
224
+ "normalized": false,
225
+ "special": true
226
+ },
227
+ {
228
+ "id": 78,
229
+ "content": "<extra_id_18>",
230
+ "single_word": false,
231
+ "lstrip": false,
232
+ "rstrip": false,
233
+ "normalized": false,
234
+ "special": true
235
+ },
236
+ {
237
+ "id": 79,
238
+ "content": "<extra_id_19>",
239
+ "single_word": false,
240
+ "lstrip": false,
241
+ "rstrip": false,
242
+ "normalized": false,
243
+ "special": true
244
+ },
245
+ {
246
+ "id": 80,
247
+ "content": "<extra_id_20>",
248
+ "single_word": false,
249
+ "lstrip": false,
250
+ "rstrip": false,
251
+ "normalized": false,
252
+ "special": true
253
+ },
254
+ {
255
+ "id": 81,
256
+ "content": "<extra_id_21>",
257
+ "single_word": false,
258
+ "lstrip": false,
259
+ "rstrip": false,
260
+ "normalized": false,
261
+ "special": true
262
+ },
263
+ {
264
+ "id": 82,
265
+ "content": "<extra_id_22>",
266
+ "single_word": false,
267
+ "lstrip": false,
268
+ "rstrip": false,
269
+ "normalized": false,
270
+ "special": true
271
+ },
272
+ {
273
+ "id": 83,
274
+ "content": "<extra_id_23>",
275
+ "single_word": false,
276
+ "lstrip": false,
277
+ "rstrip": false,
278
+ "normalized": false,
279
+ "special": true
280
+ },
281
+ {
282
+ "id": 84,
283
+ "content": "<extra_id_24>",
284
+ "single_word": false,
285
+ "lstrip": false,
286
+ "rstrip": false,
287
+ "normalized": false,
288
+ "special": true
289
+ },
290
+ {
291
+ "id": 85,
292
+ "content": "<extra_id_25>",
293
+ "single_word": false,
294
+ "lstrip": false,
295
+ "rstrip": false,
296
+ "normalized": false,
297
+ "special": true
298
+ },
299
+ {
300
+ "id": 86,
301
+ "content": "<extra_id_26>",
302
+ "single_word": false,
303
+ "lstrip": false,
304
+ "rstrip": false,
305
+ "normalized": false,
306
+ "special": true
307
+ },
308
+ {
309
+ "id": 87,
310
+ "content": "<extra_id_27>",
311
+ "single_word": false,
312
+ "lstrip": false,
313
+ "rstrip": false,
314
+ "normalized": false,
315
+ "special": true
316
+ },
317
+ {
318
+ "id": 88,
319
+ "content": "<extra_id_28>",
320
+ "single_word": false,
321
+ "lstrip": false,
322
+ "rstrip": false,
323
+ "normalized": false,
324
+ "special": true
325
+ },
326
+ {
327
+ "id": 89,
328
+ "content": "<extra_id_29>",
329
+ "single_word": false,
330
+ "lstrip": false,
331
+ "rstrip": false,
332
+ "normalized": false,
333
+ "special": true
334
+ },
335
+ {
336
+ "id": 90,
337
+ "content": "<extra_id_30>",
338
+ "single_word": false,
339
+ "lstrip": false,
340
+ "rstrip": false,
341
+ "normalized": false,
342
+ "special": true
343
+ },
344
+ {
345
+ "id": 91,
346
+ "content": "<extra_id_31>",
347
+ "single_word": false,
348
+ "lstrip": false,
349
+ "rstrip": false,
350
+ "normalized": false,
351
+ "special": true
352
+ },
353
+ {
354
+ "id": 92,
355
+ "content": "<extra_id_32>",
356
+ "single_word": false,
357
+ "lstrip": false,
358
+ "rstrip": false,
359
+ "normalized": false,
360
+ "special": true
361
+ },
362
+ {
363
+ "id": 93,
364
+ "content": "<extra_id_33>",
365
+ "single_word": false,
366
+ "lstrip": false,
367
+ "rstrip": false,
368
+ "normalized": false,
369
+ "special": true
370
+ },
371
+ {
372
+ "id": 94,
373
+ "content": "<extra_id_34>",
374
+ "single_word": false,
375
+ "lstrip": false,
376
+ "rstrip": false,
377
+ "normalized": false,
378
+ "special": true
379
+ },
380
+ {
381
+ "id": 95,
382
+ "content": "<extra_id_35>",
383
+ "single_word": false,
384
+ "lstrip": false,
385
+ "rstrip": false,
386
+ "normalized": false,
387
+ "special": true
388
+ },
389
+ {
390
+ "id": 96,
391
+ "content": "<extra_id_36>",
392
+ "single_word": false,
393
+ "lstrip": false,
394
+ "rstrip": false,
395
+ "normalized": false,
396
+ "special": true
397
+ },
398
+ {
399
+ "id": 97,
400
+ "content": "<extra_id_37>",
401
+ "single_word": false,
402
+ "lstrip": false,
403
+ "rstrip": false,
404
+ "normalized": false,
405
+ "special": true
406
+ },
407
+ {
408
+ "id": 98,
409
+ "content": "<extra_id_38>",
410
+ "single_word": false,
411
+ "lstrip": false,
412
+ "rstrip": false,
413
+ "normalized": false,
414
+ "special": true
415
+ },
416
+ {
417
+ "id": 99,
418
+ "content": "<extra_id_39>",
419
+ "single_word": false,
420
+ "lstrip": false,
421
+ "rstrip": false,
422
+ "normalized": false,
423
+ "special": true
424
+ },
425
+ {
426
+ "id": 100,
427
+ "content": "<extra_id_40>",
428
+ "single_word": false,
429
+ "lstrip": false,
430
+ "rstrip": false,
431
+ "normalized": false,
432
+ "special": true
433
+ },
434
+ {
435
+ "id": 101,
436
+ "content": "<extra_id_41>",
437
+ "single_word": false,
438
+ "lstrip": false,
439
+ "rstrip": false,
440
+ "normalized": false,
441
+ "special": true
442
+ },
443
+ {
444
+ "id": 102,
445
+ "content": "<extra_id_42>",
446
+ "single_word": false,
447
+ "lstrip": false,
448
+ "rstrip": false,
449
+ "normalized": false,
450
+ "special": true
451
+ },
452
+ {
453
+ "id": 103,
454
+ "content": "<extra_id_43>",
455
+ "single_word": false,
456
+ "lstrip": false,
457
+ "rstrip": false,
458
+ "normalized": false,
459
+ "special": true
460
+ },
461
+ {
462
+ "id": 104,
463
+ "content": "<extra_id_44>",
464
+ "single_word": false,
465
+ "lstrip": false,
466
+ "rstrip": false,
467
+ "normalized": false,
468
+ "special": true
469
+ },
470
+ {
471
+ "id": 105,
472
+ "content": "<extra_id_45>",
473
+ "single_word": false,
474
+ "lstrip": false,
475
+ "rstrip": false,
476
+ "normalized": false,
477
+ "special": true
478
+ },
479
+ {
480
+ "id": 106,
481
+ "content": "<extra_id_46>",
482
+ "single_word": false,
483
+ "lstrip": false,
484
+ "rstrip": false,
485
+ "normalized": false,
486
+ "special": true
487
+ },
488
+ {
489
+ "id": 107,
490
+ "content": "<extra_id_47>",
491
+ "single_word": false,
492
+ "lstrip": false,
493
+ "rstrip": false,
494
+ "normalized": false,
495
+ "special": true
496
+ },
497
+ {
498
+ "id": 108,
499
+ "content": "<extra_id_48>",
500
+ "single_word": false,
501
+ "lstrip": false,
502
+ "rstrip": false,
503
+ "normalized": false,
504
+ "special": true
505
+ },
506
+ {
507
+ "id": 109,
508
+ "content": "<extra_id_49>",
509
+ "single_word": false,
510
+ "lstrip": false,
511
+ "rstrip": false,
512
+ "normalized": false,
513
+ "special": true
514
+ },
515
+ {
516
+ "id": 110,
517
+ "content": "<extra_id_50>",
518
+ "single_word": false,
519
+ "lstrip": false,
520
+ "rstrip": false,
521
+ "normalized": false,
522
+ "special": true
523
+ },
524
+ {
525
+ "id": 111,
526
+ "content": "<extra_id_51>",
527
+ "single_word": false,
528
+ "lstrip": false,
529
+ "rstrip": false,
530
+ "normalized": false,
531
+ "special": true
532
+ },
533
+ {
534
+ "id": 112,
535
+ "content": "<extra_id_52>",
536
+ "single_word": false,
537
+ "lstrip": false,
538
+ "rstrip": false,
539
+ "normalized": false,
540
+ "special": true
541
+ },
542
+ {
543
+ "id": 113,
544
+ "content": "<extra_id_53>",
545
+ "single_word": false,
546
+ "lstrip": false,
547
+ "rstrip": false,
548
+ "normalized": false,
549
+ "special": true
550
+ },
551
+ {
552
+ "id": 114,
553
+ "content": "<extra_id_54>",
554
+ "single_word": false,
555
+ "lstrip": false,
556
+ "rstrip": false,
557
+ "normalized": false,
558
+ "special": true
559
+ },
560
+ {
561
+ "id": 115,
562
+ "content": "<extra_id_55>",
563
+ "single_word": false,
564
+ "lstrip": false,
565
+ "rstrip": false,
566
+ "normalized": false,
567
+ "special": true
568
+ },
569
+ {
570
+ "id": 116,
571
+ "content": "<extra_id_56>",
572
+ "single_word": false,
573
+ "lstrip": false,
574
+ "rstrip": false,
575
+ "normalized": false,
576
+ "special": true
577
+ },
578
+ {
579
+ "id": 117,
580
+ "content": "<extra_id_57>",
581
+ "single_word": false,
582
+ "lstrip": false,
583
+ "rstrip": false,
584
+ "normalized": false,
585
+ "special": true
586
+ },
587
+ {
588
+ "id": 118,
589
+ "content": "<extra_id_58>",
590
+ "single_word": false,
591
+ "lstrip": false,
592
+ "rstrip": false,
593
+ "normalized": false,
594
+ "special": true
595
+ },
596
+ {
597
+ "id": 119,
598
+ "content": "<extra_id_59>",
599
+ "single_word": false,
600
+ "lstrip": false,
601
+ "rstrip": false,
602
+ "normalized": false,
603
+ "special": true
604
+ },
605
+ {
606
+ "id": 120,
607
+ "content": "<extra_id_60>",
608
+ "single_word": false,
609
+ "lstrip": false,
610
+ "rstrip": false,
611
+ "normalized": false,
612
+ "special": true
613
+ },
614
+ {
615
+ "id": 121,
616
+ "content": "<extra_id_61>",
617
+ "single_word": false,
618
+ "lstrip": false,
619
+ "rstrip": false,
620
+ "normalized": false,
621
+ "special": true
622
+ },
623
+ {
624
+ "id": 122,
625
+ "content": "<extra_id_62>",
626
+ "single_word": false,
627
+ "lstrip": false,
628
+ "rstrip": false,
629
+ "normalized": false,
630
+ "special": true
631
+ },
632
+ {
633
+ "id": 123,
634
+ "content": "<extra_id_63>",
635
+ "single_word": false,
636
+ "lstrip": false,
637
+ "rstrip": false,
638
+ "normalized": false,
639
+ "special": true
640
+ },
641
+ {
642
+ "id": 124,
643
+ "content": "<extra_id_64>",
644
+ "single_word": false,
645
+ "lstrip": false,
646
+ "rstrip": false,
647
+ "normalized": false,
648
+ "special": true
649
+ },
650
+ {
651
+ "id": 125,
652
+ "content": "<extra_id_65>",
653
+ "single_word": false,
654
+ "lstrip": false,
655
+ "rstrip": false,
656
+ "normalized": false,
657
+ "special": true
658
+ },
659
+ {
660
+ "id": 126,
661
+ "content": "<extra_id_66>",
662
+ "single_word": false,
663
+ "lstrip": false,
664
+ "rstrip": false,
665
+ "normalized": false,
666
+ "special": true
667
+ },
668
+ {
669
+ "id": 127,
670
+ "content": "<extra_id_67>",
671
+ "single_word": false,
672
+ "lstrip": false,
673
+ "rstrip": false,
674
+ "normalized": false,
675
+ "special": true
676
+ },
677
+ {
678
+ "id": 128,
679
+ "content": "<extra_id_68>",
680
+ "single_word": false,
681
+ "lstrip": false,
682
+ "rstrip": false,
683
+ "normalized": false,
684
+ "special": true
685
+ },
686
+ {
687
+ "id": 129,
688
+ "content": "<extra_id_69>",
689
+ "single_word": false,
690
+ "lstrip": false,
691
+ "rstrip": false,
692
+ "normalized": false,
693
+ "special": true
694
+ },
695
+ {
696
+ "id": 130,
697
+ "content": "<extra_id_70>",
698
+ "single_word": false,
699
+ "lstrip": false,
700
+ "rstrip": false,
701
+ "normalized": false,
702
+ "special": true
703
+ },
704
+ {
705
+ "id": 131,
706
+ "content": "<extra_id_71>",
707
+ "single_word": false,
708
+ "lstrip": false,
709
+ "rstrip": false,
710
+ "normalized": false,
711
+ "special": true
712
+ },
713
+ {
714
+ "id": 132,
715
+ "content": "<extra_id_72>",
716
+ "single_word": false,
717
+ "lstrip": false,
718
+ "rstrip": false,
719
+ "normalized": false,
720
+ "special": true
721
+ },
722
+ {
723
+ "id": 133,
724
+ "content": "<extra_id_73>",
725
+ "single_word": false,
726
+ "lstrip": false,
727
+ "rstrip": false,
728
+ "normalized": false,
729
+ "special": true
730
+ },
731
+ {
732
+ "id": 134,
733
+ "content": "<extra_id_74>",
734
+ "single_word": false,
735
+ "lstrip": false,
736
+ "rstrip": false,
737
+ "normalized": false,
738
+ "special": true
739
+ },
740
+ {
741
+ "id": 135,
742
+ "content": "<extra_id_75>",
743
+ "single_word": false,
744
+ "lstrip": false,
745
+ "rstrip": false,
746
+ "normalized": false,
747
+ "special": true
748
+ },
749
+ {
750
+ "id": 136,
751
+ "content": "<extra_id_76>",
752
+ "single_word": false,
753
+ "lstrip": false,
754
+ "rstrip": false,
755
+ "normalized": false,
756
+ "special": true
757
+ },
758
+ {
759
+ "id": 137,
760
+ "content": "<extra_id_77>",
761
+ "single_word": false,
762
+ "lstrip": false,
763
+ "rstrip": false,
764
+ "normalized": false,
765
+ "special": true
766
+ },
767
+ {
768
+ "id": 138,
769
+ "content": "<extra_id_78>",
770
+ "single_word": false,
771
+ "lstrip": false,
772
+ "rstrip": false,
773
+ "normalized": false,
774
+ "special": true
775
+ },
776
+ {
777
+ "id": 139,
778
+ "content": "<extra_id_79>",
779
+ "single_word": false,
780
+ "lstrip": false,
781
+ "rstrip": false,
782
+ "normalized": false,
783
+ "special": true
784
+ },
785
+ {
786
+ "id": 140,
787
+ "content": "<extra_id_80>",
788
+ "single_word": false,
789
+ "lstrip": false,
790
+ "rstrip": false,
791
+ "normalized": false,
792
+ "special": true
793
+ },
794
+ {
795
+ "id": 141,
796
+ "content": "<extra_id_81>",
797
+ "single_word": false,
798
+ "lstrip": false,
799
+ "rstrip": false,
800
+ "normalized": false,
801
+ "special": true
802
+ },
803
+ {
804
+ "id": 142,
805
+ "content": "<extra_id_82>",
806
+ "single_word": false,
807
+ "lstrip": false,
808
+ "rstrip": false,
809
+ "normalized": false,
810
+ "special": true
811
+ },
812
+ {
813
+ "id": 143,
814
+ "content": "<extra_id_83>",
815
+ "single_word": false,
816
+ "lstrip": false,
817
+ "rstrip": false,
818
+ "normalized": false,
819
+ "special": true
820
+ },
821
+ {
822
+ "id": 144,
823
+ "content": "<extra_id_84>",
824
+ "single_word": false,
825
+ "lstrip": false,
826
+ "rstrip": false,
827
+ "normalized": false,
828
+ "special": true
829
+ },
830
+ {
831
+ "id": 145,
832
+ "content": "<extra_id_85>",
833
+ "single_word": false,
834
+ "lstrip": false,
835
+ "rstrip": false,
836
+ "normalized": false,
837
+ "special": true
838
+ },
839
+ {
840
+ "id": 146,
841
+ "content": "<extra_id_86>",
842
+ "single_word": false,
843
+ "lstrip": false,
844
+ "rstrip": false,
845
+ "normalized": false,
846
+ "special": true
847
+ },
848
+ {
849
+ "id": 147,
850
+ "content": "<extra_id_87>",
851
+ "single_word": false,
852
+ "lstrip": false,
853
+ "rstrip": false,
854
+ "normalized": false,
855
+ "special": true
856
+ },
857
+ {
858
+ "id": 148,
859
+ "content": "<extra_id_88>",
860
+ "single_word": false,
861
+ "lstrip": false,
862
+ "rstrip": false,
863
+ "normalized": false,
864
+ "special": true
865
+ },
866
+ {
867
+ "id": 149,
868
+ "content": "<extra_id_89>",
869
+ "single_word": false,
870
+ "lstrip": false,
871
+ "rstrip": false,
872
+ "normalized": false,
873
+ "special": true
874
+ },
875
+ {
876
+ "id": 150,
877
+ "content": "<extra_id_90>",
878
+ "single_word": false,
879
+ "lstrip": false,
880
+ "rstrip": false,
881
+ "normalized": false,
882
+ "special": true
883
+ },
884
+ {
885
+ "id": 151,
886
+ "content": "<extra_id_91>",
887
+ "single_word": false,
888
+ "lstrip": false,
889
+ "rstrip": false,
890
+ "normalized": false,
891
+ "special": true
892
+ },
893
+ {
894
+ "id": 152,
895
+ "content": "<extra_id_92>",
896
+ "single_word": false,
897
+ "lstrip": false,
898
+ "rstrip": false,
899
+ "normalized": false,
900
+ "special": true
901
+ },
902
+ {
903
+ "id": 153,
904
+ "content": "<extra_id_93>",
905
+ "single_word": false,
906
+ "lstrip": false,
907
+ "rstrip": false,
908
+ "normalized": false,
909
+ "special": true
910
+ },
911
+ {
912
+ "id": 154,
913
+ "content": "<extra_id_94>",
914
+ "single_word": false,
915
+ "lstrip": false,
916
+ "rstrip": false,
917
+ "normalized": false,
918
+ "special": true
919
+ },
920
+ {
921
+ "id": 155,
922
+ "content": "<extra_id_95>",
923
+ "single_word": false,
924
+ "lstrip": false,
925
+ "rstrip": false,
926
+ "normalized": false,
927
+ "special": true
928
+ },
929
+ {
930
+ "id": 156,
931
+ "content": "<extra_id_96>",
932
+ "single_word": false,
933
+ "lstrip": false,
934
+ "rstrip": false,
935
+ "normalized": false,
936
+ "special": true
937
+ },
938
+ {
939
+ "id": 157,
940
+ "content": "<extra_id_97>",
941
+ "single_word": false,
942
+ "lstrip": false,
943
+ "rstrip": false,
944
+ "normalized": false,
945
+ "special": true
946
+ },
947
+ {
948
+ "id": 158,
949
+ "content": "<extra_id_98>",
950
+ "single_word": false,
951
+ "lstrip": false,
952
+ "rstrip": false,
953
+ "normalized": false,
954
+ "special": true
955
+ },
956
+ {
957
+ "id": 159,
958
+ "content": "<extra_id_99>",
959
+ "single_word": false,
960
+ "lstrip": false,
961
+ "rstrip": false,
962
+ "normalized": false,
963
+ "special": true
964
+ }
965
+ ],
966
+ "normalizer": {
967
+ "type": "Sequence",
968
+ "normalizers": [
969
+ {
970
+ "type": "NFD"
971
+ },
972
+ {
973
+ "type": "Lowercase"
974
+ },
975
+ {
976
+ "type": "StripAccents"
977
+ }
978
+ ]
979
+ },
980
+ "pre_tokenizer": {
981
+ "type": "Whitespace"
982
+ },
983
+ "post_processor": {
984
+ "type": "TemplateProcessing",
985
+ "single": [
986
+ {
987
+ "SpecialToken": {
988
+ "id": "<s>",
989
+ "type_id": 0
990
+ }
991
+ },
992
+ {
993
+ "Sequence": {
994
+ "id": "A",
995
+ "type_id": 0
996
+ }
997
+ },
998
+ {
999
+ "SpecialToken": {
1000
+ "id": "</s>",
1001
+ "type_id": 0
1002
+ }
1003
+ }
1004
+ ],
1005
+ "pair": [
1006
+ {
1007
+ "SpecialToken": {
1008
+ "id": "<s>",
1009
+ "type_id": 0
1010
+ }
1011
+ },
1012
+ {
1013
+ "Sequence": {
1014
+ "id": "A",
1015
+ "type_id": 0
1016
+ }
1017
+ },
1018
+ {
1019
+ "SpecialToken": {
1020
+ "id": "</s>",
1021
+ "type_id": 0
1022
+ }
1023
+ },
1024
+ {
1025
+ "SpecialToken": {
1026
+ "id": "</s>",
1027
+ "type_id": 0
1028
+ }
1029
+ },
1030
+ {
1031
+ "Sequence": {
1032
+ "id": "B",
1033
+ "type_id": 0
1034
+ }
1035
+ },
1036
+ {
1037
+ "SpecialToken": {
1038
+ "id": "</s>",
1039
+ "type_id": 0
1040
+ }
1041
+ }
1042
+ ],
1043
+ "special_tokens": {
1044
+ "</s>": {
1045
+ "id": "</s>",
1046
+ "ids": [
1047
+ 2
1048
+ ],
1049
+ "tokens": [
1050
+ "</s>"
1051
+ ]
1052
+ },
1053
+ "<s>": {
1054
+ "id": "<s>",
1055
+ "ids": [
1056
+ 1
1057
+ ],
1058
+ "tokens": [
1059
+ "<s>"
1060
+ ]
1061
+ }
1062
+ }
1063
+ },
1064
+ "decoder": null,
1065
+ "model": {
1066
+ "type": "BPE",
1067
+ "dropout": null,
1068
+ "unk_token": null,
1069
+ "continuing_subword_prefix": null,
1070
+ "end_of_word_suffix": null,
1071
+ "fuse_unk": false,
1072
+ "byte_fallback": false,
1073
+ "ignore_merges": false,
1074
+ "vocab": {
1075
+ "<pad>": 0,
1076
+ "<s>": 1,
1077
+ "</s>": 2,
1078
+ "<unk>": 3,
1079
+ "<mask>": 4,
1080
+ "!": 5,
1081
+ "\"": 6,
1082
+ "(": 7,
1083
+ ")": 8,
1084
+ "+": 9,
1085
+ ",": 10,
1086
+ ":": 11,
1087
+ "a": 12,
1088
+ "b": 13,
1089
+ "c": 14,
1090
+ "d": 15,
1091
+ "e": 16,
1092
+ "f": 17,
1093
+ "h": 18,
1094
+ "i": 19,
1095
+ "l": 20,
1096
+ "n": 21,
1097
+ "o": 22,
1098
+ "p": 23,
1099
+ "r": 24,
1100
+ "s": 25,
1101
+ "t": 26,
1102
+ "u": 27,
1103
+ "w": 28,
1104
+ "as": 29,
1105
+ "el": 30,
1106
+ "hel": 31,
1107
+ "ld": 32,
1108
+ "lo": 33,
1109
+ "or": 34,
1110
+ "wor": 35,
1111
+ "ass": 36,
1112
+ "hello": 37,
1113
+ "world": 38,
1114
+ "!\"": 39,
1115
+ "(\"": 40,
1116
+ "):": 41,
1117
+ "ad": 42,
1118
+ "cl": 43,
1119
+ "de": 44,
1120
+ "et": 45,
1121
+ "in": 46,
1122
+ "pr": 47,
1123
+ "pass": 48,
1124
+ "rn": 49,
1125
+ "ret": 50,
1126
+ "urn": 51,
1127
+ "helloworld": 52,
1128
+ "!\")": 53,
1129
+ "add": 54,
1130
+ "class": 55,
1131
+ "def": 56,
1132
+ "int": 57,
1133
+ "print": 58,
1134
+ "return": 59
1135
+ },
1136
+ "merges": [
1137
+ [
1138
+ "a",
1139
+ "s"
1140
+ ],
1141
+ [
1142
+ "e",
1143
+ "l"
1144
+ ],
1145
+ [
1146
+ "h",
1147
+ "el"
1148
+ ],
1149
+ [
1150
+ "l",
1151
+ "d"
1152
+ ],
1153
+ [
1154
+ "l",
1155
+ "o"
1156
+ ],
1157
+ [
1158
+ "o",
1159
+ "r"
1160
+ ],
1161
+ [
1162
+ "w",
1163
+ "or"
1164
+ ],
1165
+ [
1166
+ "as",
1167
+ "s"
1168
+ ],
1169
+ [
1170
+ "hel",
1171
+ "lo"
1172
+ ],
1173
+ [
1174
+ "wor",
1175
+ "ld"
1176
+ ],
1177
+ [
1178
+ "!",
1179
+ "\""
1180
+ ],
1181
+ [
1182
+ "(",
1183
+ "\""
1184
+ ],
1185
+ [
1186
+ ")",
1187
+ ":"
1188
+ ],
1189
+ [
1190
+ "a",
1191
+ "d"
1192
+ ],
1193
+ [
1194
+ "c",
1195
+ "l"
1196
+ ],
1197
+ [
1198
+ "d",
1199
+ "e"
1200
+ ],
1201
+ [
1202
+ "e",
1203
+ "t"
1204
+ ],
1205
+ [
1206
+ "i",
1207
+ "n"
1208
+ ],
1209
+ [
1210
+ "p",
1211
+ "r"
1212
+ ],
1213
+ [
1214
+ "p",
1215
+ "ass"
1216
+ ],
1217
+ [
1218
+ "r",
1219
+ "n"
1220
+ ],
1221
+ [
1222
+ "r",
1223
+ "et"
1224
+ ],
1225
+ [
1226
+ "u",
1227
+ "rn"
1228
+ ],
1229
+ [
1230
+ "hello",
1231
+ "world"
1232
+ ],
1233
+ [
1234
+ "!\"",
1235
+ ")"
1236
+ ],
1237
+ [
1238
+ "ad",
1239
+ "d"
1240
+ ],
1241
+ [
1242
+ "cl",
1243
+ "ass"
1244
+ ],
1245
+ [
1246
+ "de",
1247
+ "f"
1248
+ ],
1249
+ [
1250
+ "in",
1251
+ "t"
1252
+ ],
1253
+ [
1254
+ "pr",
1255
+ "int"
1256
+ ],
1257
+ [
1258
+ "ret",
1259
+ "urn"
1260
+ ]
1261
+ ]
1262
+ }
1263
+ }
minicoderx-model/checkpoint-3/tokenizer_config.json ADDED
@@ -0,0 +1,958 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": null,
3
+ "added_tokens_decoder": {
4
+ "0": {
5
+ "content": "<pad>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "1": {
13
+ "content": "<s>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "2": {
21
+ "content": "</s>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "3": {
29
+ "content": "<unk>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "4": {
37
+ "content": "<mask>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "60": {
45
+ "content": "<extra_id_0>",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false,
50
+ "special": true
51
+ },
52
+ "61": {
53
+ "content": "<extra_id_1>",
54
+ "lstrip": false,
55
+ "normalized": false,
56
+ "rstrip": false,
57
+ "single_word": false,
58
+ "special": true
59
+ },
60
+ "62": {
61
+ "content": "<extra_id_2>",
62
+ "lstrip": false,
63
+ "normalized": false,
64
+ "rstrip": false,
65
+ "single_word": false,
66
+ "special": true
67
+ },
68
+ "63": {
69
+ "content": "<extra_id_3>",
70
+ "lstrip": false,
71
+ "normalized": false,
72
+ "rstrip": false,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "64": {
77
+ "content": "<extra_id_4>",
78
+ "lstrip": false,
79
+ "normalized": false,
80
+ "rstrip": false,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "65": {
85
+ "content": "<extra_id_5>",
86
+ "lstrip": false,
87
+ "normalized": false,
88
+ "rstrip": false,
89
+ "single_word": false,
90
+ "special": true
91
+ },
92
+ "66": {
93
+ "content": "<extra_id_6>",
94
+ "lstrip": false,
95
+ "normalized": false,
96
+ "rstrip": false,
97
+ "single_word": false,
98
+ "special": true
99
+ },
100
+ "67": {
101
+ "content": "<extra_id_7>",
102
+ "lstrip": false,
103
+ "normalized": false,
104
+ "rstrip": false,
105
+ "single_word": false,
106
+ "special": true
107
+ },
108
+ "68": {
109
+ "content": "<extra_id_8>",
110
+ "lstrip": false,
111
+ "normalized": false,
112
+ "rstrip": false,
113
+ "single_word": false,
114
+ "special": true
115
+ },
116
+ "69": {
117
+ "content": "<extra_id_9>",
118
+ "lstrip": false,
119
+ "normalized": false,
120
+ "rstrip": false,
121
+ "single_word": false,
122
+ "special": true
123
+ },
124
+ "70": {
125
+ "content": "<extra_id_10>",
126
+ "lstrip": false,
127
+ "normalized": false,
128
+ "rstrip": false,
129
+ "single_word": false,
130
+ "special": true
131
+ },
132
+ "71": {
133
+ "content": "<extra_id_11>",
134
+ "lstrip": false,
135
+ "normalized": false,
136
+ "rstrip": false,
137
+ "single_word": false,
138
+ "special": true
139
+ },
140
+ "72": {
141
+ "content": "<extra_id_12>",
142
+ "lstrip": false,
143
+ "normalized": false,
144
+ "rstrip": false,
145
+ "single_word": false,
146
+ "special": true
147
+ },
148
+ "73": {
149
+ "content": "<extra_id_13>",
150
+ "lstrip": false,
151
+ "normalized": false,
152
+ "rstrip": false,
153
+ "single_word": false,
154
+ "special": true
155
+ },
156
+ "74": {
157
+ "content": "<extra_id_14>",
158
+ "lstrip": false,
159
+ "normalized": false,
160
+ "rstrip": false,
161
+ "single_word": false,
162
+ "special": true
163
+ },
164
+ "75": {
165
+ "content": "<extra_id_15>",
166
+ "lstrip": false,
167
+ "normalized": false,
168
+ "rstrip": false,
169
+ "single_word": false,
170
+ "special": true
171
+ },
172
+ "76": {
173
+ "content": "<extra_id_16>",
174
+ "lstrip": false,
175
+ "normalized": false,
176
+ "rstrip": false,
177
+ "single_word": false,
178
+ "special": true
179
+ },
180
+ "77": {
181
+ "content": "<extra_id_17>",
182
+ "lstrip": false,
183
+ "normalized": false,
184
+ "rstrip": false,
185
+ "single_word": false,
186
+ "special": true
187
+ },
188
+ "78": {
189
+ "content": "<extra_id_18>",
190
+ "lstrip": false,
191
+ "normalized": false,
192
+ "rstrip": false,
193
+ "single_word": false,
194
+ "special": true
195
+ },
196
+ "79": {
197
+ "content": "<extra_id_19>",
198
+ "lstrip": false,
199
+ "normalized": false,
200
+ "rstrip": false,
201
+ "single_word": false,
202
+ "special": true
203
+ },
204
+ "80": {
205
+ "content": "<extra_id_20>",
206
+ "lstrip": false,
207
+ "normalized": false,
208
+ "rstrip": false,
209
+ "single_word": false,
210
+ "special": true
211
+ },
212
+ "81": {
213
+ "content": "<extra_id_21>",
214
+ "lstrip": false,
215
+ "normalized": false,
216
+ "rstrip": false,
217
+ "single_word": false,
218
+ "special": true
219
+ },
220
+ "82": {
221
+ "content": "<extra_id_22>",
222
+ "lstrip": false,
223
+ "normalized": false,
224
+ "rstrip": false,
225
+ "single_word": false,
226
+ "special": true
227
+ },
228
+ "83": {
229
+ "content": "<extra_id_23>",
230
+ "lstrip": false,
231
+ "normalized": false,
232
+ "rstrip": false,
233
+ "single_word": false,
234
+ "special": true
235
+ },
236
+ "84": {
237
+ "content": "<extra_id_24>",
238
+ "lstrip": false,
239
+ "normalized": false,
240
+ "rstrip": false,
241
+ "single_word": false,
242
+ "special": true
243
+ },
244
+ "85": {
245
+ "content": "<extra_id_25>",
246
+ "lstrip": false,
247
+ "normalized": false,
248
+ "rstrip": false,
249
+ "single_word": false,
250
+ "special": true
251
+ },
252
+ "86": {
253
+ "content": "<extra_id_26>",
254
+ "lstrip": false,
255
+ "normalized": false,
256
+ "rstrip": false,
257
+ "single_word": false,
258
+ "special": true
259
+ },
260
+ "87": {
261
+ "content": "<extra_id_27>",
262
+ "lstrip": false,
263
+ "normalized": false,
264
+ "rstrip": false,
265
+ "single_word": false,
266
+ "special": true
267
+ },
268
+ "88": {
269
+ "content": "<extra_id_28>",
270
+ "lstrip": false,
271
+ "normalized": false,
272
+ "rstrip": false,
273
+ "single_word": false,
274
+ "special": true
275
+ },
276
+ "89": {
277
+ "content": "<extra_id_29>",
278
+ "lstrip": false,
279
+ "normalized": false,
280
+ "rstrip": false,
281
+ "single_word": false,
282
+ "special": true
283
+ },
284
+ "90": {
285
+ "content": "<extra_id_30>",
286
+ "lstrip": false,
287
+ "normalized": false,
288
+ "rstrip": false,
289
+ "single_word": false,
290
+ "special": true
291
+ },
292
+ "91": {
293
+ "content": "<extra_id_31>",
294
+ "lstrip": false,
295
+ "normalized": false,
296
+ "rstrip": false,
297
+ "single_word": false,
298
+ "special": true
299
+ },
300
+ "92": {
301
+ "content": "<extra_id_32>",
302
+ "lstrip": false,
303
+ "normalized": false,
304
+ "rstrip": false,
305
+ "single_word": false,
306
+ "special": true
307
+ },
308
+ "93": {
309
+ "content": "<extra_id_33>",
310
+ "lstrip": false,
311
+ "normalized": false,
312
+ "rstrip": false,
313
+ "single_word": false,
314
+ "special": true
315
+ },
316
+ "94": {
317
+ "content": "<extra_id_34>",
318
+ "lstrip": false,
319
+ "normalized": false,
320
+ "rstrip": false,
321
+ "single_word": false,
322
+ "special": true
323
+ },
324
+ "95": {
325
+ "content": "<extra_id_35>",
326
+ "lstrip": false,
327
+ "normalized": false,
328
+ "rstrip": false,
329
+ "single_word": false,
330
+ "special": true
331
+ },
332
+ "96": {
333
+ "content": "<extra_id_36>",
334
+ "lstrip": false,
335
+ "normalized": false,
336
+ "rstrip": false,
337
+ "single_word": false,
338
+ "special": true
339
+ },
340
+ "97": {
341
+ "content": "<extra_id_37>",
342
+ "lstrip": false,
343
+ "normalized": false,
344
+ "rstrip": false,
345
+ "single_word": false,
346
+ "special": true
347
+ },
348
+ "98": {
349
+ "content": "<extra_id_38>",
350
+ "lstrip": false,
351
+ "normalized": false,
352
+ "rstrip": false,
353
+ "single_word": false,
354
+ "special": true
355
+ },
356
+ "99": {
357
+ "content": "<extra_id_39>",
358
+ "lstrip": false,
359
+ "normalized": false,
360
+ "rstrip": false,
361
+ "single_word": false,
362
+ "special": true
363
+ },
364
+ "100": {
365
+ "content": "<extra_id_40>",
366
+ "lstrip": false,
367
+ "normalized": false,
368
+ "rstrip": false,
369
+ "single_word": false,
370
+ "special": true
371
+ },
372
+ "101": {
373
+ "content": "<extra_id_41>",
374
+ "lstrip": false,
375
+ "normalized": false,
376
+ "rstrip": false,
377
+ "single_word": false,
378
+ "special": true
379
+ },
380
+ "102": {
381
+ "content": "<extra_id_42>",
382
+ "lstrip": false,
383
+ "normalized": false,
384
+ "rstrip": false,
385
+ "single_word": false,
386
+ "special": true
387
+ },
388
+ "103": {
389
+ "content": "<extra_id_43>",
390
+ "lstrip": false,
391
+ "normalized": false,
392
+ "rstrip": false,
393
+ "single_word": false,
394
+ "special": true
395
+ },
396
+ "104": {
397
+ "content": "<extra_id_44>",
398
+ "lstrip": false,
399
+ "normalized": false,
400
+ "rstrip": false,
401
+ "single_word": false,
402
+ "special": true
403
+ },
404
+ "105": {
405
+ "content": "<extra_id_45>",
406
+ "lstrip": false,
407
+ "normalized": false,
408
+ "rstrip": false,
409
+ "single_word": false,
410
+ "special": true
411
+ },
412
+ "106": {
413
+ "content": "<extra_id_46>",
414
+ "lstrip": false,
415
+ "normalized": false,
416
+ "rstrip": false,
417
+ "single_word": false,
418
+ "special": true
419
+ },
420
+ "107": {
421
+ "content": "<extra_id_47>",
422
+ "lstrip": false,
423
+ "normalized": false,
424
+ "rstrip": false,
425
+ "single_word": false,
426
+ "special": true
427
+ },
428
+ "108": {
429
+ "content": "<extra_id_48>",
430
+ "lstrip": false,
431
+ "normalized": false,
432
+ "rstrip": false,
433
+ "single_word": false,
434
+ "special": true
435
+ },
436
+ "109": {
437
+ "content": "<extra_id_49>",
438
+ "lstrip": false,
439
+ "normalized": false,
440
+ "rstrip": false,
441
+ "single_word": false,
442
+ "special": true
443
+ },
444
+ "110": {
445
+ "content": "<extra_id_50>",
446
+ "lstrip": false,
447
+ "normalized": false,
448
+ "rstrip": false,
449
+ "single_word": false,
450
+ "special": true
451
+ },
452
+ "111": {
453
+ "content": "<extra_id_51>",
454
+ "lstrip": false,
455
+ "normalized": false,
456
+ "rstrip": false,
457
+ "single_word": false,
458
+ "special": true
459
+ },
460
+ "112": {
461
+ "content": "<extra_id_52>",
462
+ "lstrip": false,
463
+ "normalized": false,
464
+ "rstrip": false,
465
+ "single_word": false,
466
+ "special": true
467
+ },
468
+ "113": {
469
+ "content": "<extra_id_53>",
470
+ "lstrip": false,
471
+ "normalized": false,
472
+ "rstrip": false,
473
+ "single_word": false,
474
+ "special": true
475
+ },
476
+ "114": {
477
+ "content": "<extra_id_54>",
478
+ "lstrip": false,
479
+ "normalized": false,
480
+ "rstrip": false,
481
+ "single_word": false,
482
+ "special": true
483
+ },
484
+ "115": {
485
+ "content": "<extra_id_55>",
486
+ "lstrip": false,
487
+ "normalized": false,
488
+ "rstrip": false,
489
+ "single_word": false,
490
+ "special": true
491
+ },
492
+ "116": {
493
+ "content": "<extra_id_56>",
494
+ "lstrip": false,
495
+ "normalized": false,
496
+ "rstrip": false,
497
+ "single_word": false,
498
+ "special": true
499
+ },
500
+ "117": {
501
+ "content": "<extra_id_57>",
502
+ "lstrip": false,
503
+ "normalized": false,
504
+ "rstrip": false,
505
+ "single_word": false,
506
+ "special": true
507
+ },
508
+ "118": {
509
+ "content": "<extra_id_58>",
510
+ "lstrip": false,
511
+ "normalized": false,
512
+ "rstrip": false,
513
+ "single_word": false,
514
+ "special": true
515
+ },
516
+ "119": {
517
+ "content": "<extra_id_59>",
518
+ "lstrip": false,
519
+ "normalized": false,
520
+ "rstrip": false,
521
+ "single_word": false,
522
+ "special": true
523
+ },
524
+ "120": {
525
+ "content": "<extra_id_60>",
526
+ "lstrip": false,
527
+ "normalized": false,
528
+ "rstrip": false,
529
+ "single_word": false,
530
+ "special": true
531
+ },
532
+ "121": {
533
+ "content": "<extra_id_61>",
534
+ "lstrip": false,
535
+ "normalized": false,
536
+ "rstrip": false,
537
+ "single_word": false,
538
+ "special": true
539
+ },
540
+ "122": {
541
+ "content": "<extra_id_62>",
542
+ "lstrip": false,
543
+ "normalized": false,
544
+ "rstrip": false,
545
+ "single_word": false,
546
+ "special": true
547
+ },
548
+ "123": {
549
+ "content": "<extra_id_63>",
550
+ "lstrip": false,
551
+ "normalized": false,
552
+ "rstrip": false,
553
+ "single_word": false,
554
+ "special": true
555
+ },
556
+ "124": {
557
+ "content": "<extra_id_64>",
558
+ "lstrip": false,
559
+ "normalized": false,
560
+ "rstrip": false,
561
+ "single_word": false,
562
+ "special": true
563
+ },
564
+ "125": {
565
+ "content": "<extra_id_65>",
566
+ "lstrip": false,
567
+ "normalized": false,
568
+ "rstrip": false,
569
+ "single_word": false,
570
+ "special": true
571
+ },
572
+ "126": {
573
+ "content": "<extra_id_66>",
574
+ "lstrip": false,
575
+ "normalized": false,
576
+ "rstrip": false,
577
+ "single_word": false,
578
+ "special": true
579
+ },
580
+ "127": {
581
+ "content": "<extra_id_67>",
582
+ "lstrip": false,
583
+ "normalized": false,
584
+ "rstrip": false,
585
+ "single_word": false,
586
+ "special": true
587
+ },
588
+ "128": {
589
+ "content": "<extra_id_68>",
590
+ "lstrip": false,
591
+ "normalized": false,
592
+ "rstrip": false,
593
+ "single_word": false,
594
+ "special": true
595
+ },
596
+ "129": {
597
+ "content": "<extra_id_69>",
598
+ "lstrip": false,
599
+ "normalized": false,
600
+ "rstrip": false,
601
+ "single_word": false,
602
+ "special": true
603
+ },
604
+ "130": {
605
+ "content": "<extra_id_70>",
606
+ "lstrip": false,
607
+ "normalized": false,
608
+ "rstrip": false,
609
+ "single_word": false,
610
+ "special": true
611
+ },
612
+ "131": {
613
+ "content": "<extra_id_71>",
614
+ "lstrip": false,
615
+ "normalized": false,
616
+ "rstrip": false,
617
+ "single_word": false,
618
+ "special": true
619
+ },
620
+ "132": {
621
+ "content": "<extra_id_72>",
622
+ "lstrip": false,
623
+ "normalized": false,
624
+ "rstrip": false,
625
+ "single_word": false,
626
+ "special": true
627
+ },
628
+ "133": {
629
+ "content": "<extra_id_73>",
630
+ "lstrip": false,
631
+ "normalized": false,
632
+ "rstrip": false,
633
+ "single_word": false,
634
+ "special": true
635
+ },
636
+ "134": {
637
+ "content": "<extra_id_74>",
638
+ "lstrip": false,
639
+ "normalized": false,
640
+ "rstrip": false,
641
+ "single_word": false,
642
+ "special": true
643
+ },
644
+ "135": {
645
+ "content": "<extra_id_75>",
646
+ "lstrip": false,
647
+ "normalized": false,
648
+ "rstrip": false,
649
+ "single_word": false,
650
+ "special": true
651
+ },
652
+ "136": {
653
+ "content": "<extra_id_76>",
654
+ "lstrip": false,
655
+ "normalized": false,
656
+ "rstrip": false,
657
+ "single_word": false,
658
+ "special": true
659
+ },
660
+ "137": {
661
+ "content": "<extra_id_77>",
662
+ "lstrip": false,
663
+ "normalized": false,
664
+ "rstrip": false,
665
+ "single_word": false,
666
+ "special": true
667
+ },
668
+ "138": {
669
+ "content": "<extra_id_78>",
670
+ "lstrip": false,
671
+ "normalized": false,
672
+ "rstrip": false,
673
+ "single_word": false,
674
+ "special": true
675
+ },
676
+ "139": {
677
+ "content": "<extra_id_79>",
678
+ "lstrip": false,
679
+ "normalized": false,
680
+ "rstrip": false,
681
+ "single_word": false,
682
+ "special": true
683
+ },
684
+ "140": {
685
+ "content": "<extra_id_80>",
686
+ "lstrip": false,
687
+ "normalized": false,
688
+ "rstrip": false,
689
+ "single_word": false,
690
+ "special": true
691
+ },
692
+ "141": {
693
+ "content": "<extra_id_81>",
694
+ "lstrip": false,
695
+ "normalized": false,
696
+ "rstrip": false,
697
+ "single_word": false,
698
+ "special": true
699
+ },
700
+ "142": {
701
+ "content": "<extra_id_82>",
702
+ "lstrip": false,
703
+ "normalized": false,
704
+ "rstrip": false,
705
+ "single_word": false,
706
+ "special": true
707
+ },
708
+ "143": {
709
+ "content": "<extra_id_83>",
710
+ "lstrip": false,
711
+ "normalized": false,
712
+ "rstrip": false,
713
+ "single_word": false,
714
+ "special": true
715
+ },
716
+ "144": {
717
+ "content": "<extra_id_84>",
718
+ "lstrip": false,
719
+ "normalized": false,
720
+ "rstrip": false,
721
+ "single_word": false,
722
+ "special": true
723
+ },
724
+ "145": {
725
+ "content": "<extra_id_85>",
726
+ "lstrip": false,
727
+ "normalized": false,
728
+ "rstrip": false,
729
+ "single_word": false,
730
+ "special": true
731
+ },
732
+ "146": {
733
+ "content": "<extra_id_86>",
734
+ "lstrip": false,
735
+ "normalized": false,
736
+ "rstrip": false,
737
+ "single_word": false,
738
+ "special": true
739
+ },
740
+ "147": {
741
+ "content": "<extra_id_87>",
742
+ "lstrip": false,
743
+ "normalized": false,
744
+ "rstrip": false,
745
+ "single_word": false,
746
+ "special": true
747
+ },
748
+ "148": {
749
+ "content": "<extra_id_88>",
750
+ "lstrip": false,
751
+ "normalized": false,
752
+ "rstrip": false,
753
+ "single_word": false,
754
+ "special": true
755
+ },
756
+ "149": {
757
+ "content": "<extra_id_89>",
758
+ "lstrip": false,
759
+ "normalized": false,
760
+ "rstrip": false,
761
+ "single_word": false,
762
+ "special": true
763
+ },
764
+ "150": {
765
+ "content": "<extra_id_90>",
766
+ "lstrip": false,
767
+ "normalized": false,
768
+ "rstrip": false,
769
+ "single_word": false,
770
+ "special": true
771
+ },
772
+ "151": {
773
+ "content": "<extra_id_91>",
774
+ "lstrip": false,
775
+ "normalized": false,
776
+ "rstrip": false,
777
+ "single_word": false,
778
+ "special": true
779
+ },
780
+ "152": {
781
+ "content": "<extra_id_92>",
782
+ "lstrip": false,
783
+ "normalized": false,
784
+ "rstrip": false,
785
+ "single_word": false,
786
+ "special": true
787
+ },
788
+ "153": {
789
+ "content": "<extra_id_93>",
790
+ "lstrip": false,
791
+ "normalized": false,
792
+ "rstrip": false,
793
+ "single_word": false,
794
+ "special": true
795
+ },
796
+ "154": {
797
+ "content": "<extra_id_94>",
798
+ "lstrip": false,
799
+ "normalized": false,
800
+ "rstrip": false,
801
+ "single_word": false,
802
+ "special": true
803
+ },
804
+ "155": {
805
+ "content": "<extra_id_95>",
806
+ "lstrip": false,
807
+ "normalized": false,
808
+ "rstrip": false,
809
+ "single_word": false,
810
+ "special": true
811
+ },
812
+ "156": {
813
+ "content": "<extra_id_96>",
814
+ "lstrip": false,
815
+ "normalized": false,
816
+ "rstrip": false,
817
+ "single_word": false,
818
+ "special": true
819
+ },
820
+ "157": {
821
+ "content": "<extra_id_97>",
822
+ "lstrip": false,
823
+ "normalized": false,
824
+ "rstrip": false,
825
+ "single_word": false,
826
+ "special": true
827
+ },
828
+ "158": {
829
+ "content": "<extra_id_98>",
830
+ "lstrip": false,
831
+ "normalized": false,
832
+ "rstrip": false,
833
+ "single_word": false,
834
+ "special": true
835
+ },
836
+ "159": {
837
+ "content": "<extra_id_99>",
838
+ "lstrip": false,
839
+ "normalized": false,
840
+ "rstrip": false,
841
+ "single_word": false,
842
+ "special": true
843
+ }
844
+ },
845
+ "additional_special_tokens": [
846
+ "<extra_id_0>",
847
+ "<extra_id_1>",
848
+ "<extra_id_2>",
849
+ "<extra_id_3>",
850
+ "<extra_id_4>",
851
+ "<extra_id_5>",
852
+ "<extra_id_6>",
853
+ "<extra_id_7>",
854
+ "<extra_id_8>",
855
+ "<extra_id_9>",
856
+ "<extra_id_10>",
857
+ "<extra_id_11>",
858
+ "<extra_id_12>",
859
+ "<extra_id_13>",
860
+ "<extra_id_14>",
861
+ "<extra_id_15>",
862
+ "<extra_id_16>",
863
+ "<extra_id_17>",
864
+ "<extra_id_18>",
865
+ "<extra_id_19>",
866
+ "<extra_id_20>",
867
+ "<extra_id_21>",
868
+ "<extra_id_22>",
869
+ "<extra_id_23>",
870
+ "<extra_id_24>",
871
+ "<extra_id_25>",
872
+ "<extra_id_26>",
873
+ "<extra_id_27>",
874
+ "<extra_id_28>",
875
+ "<extra_id_29>",
876
+ "<extra_id_30>",
877
+ "<extra_id_31>",
878
+ "<extra_id_32>",
879
+ "<extra_id_33>",
880
+ "<extra_id_34>",
881
+ "<extra_id_35>",
882
+ "<extra_id_36>",
883
+ "<extra_id_37>",
884
+ "<extra_id_38>",
885
+ "<extra_id_39>",
886
+ "<extra_id_40>",
887
+ "<extra_id_41>",
888
+ "<extra_id_42>",
889
+ "<extra_id_43>",
890
+ "<extra_id_44>",
891
+ "<extra_id_45>",
892
+ "<extra_id_46>",
893
+ "<extra_id_47>",
894
+ "<extra_id_48>",
895
+ "<extra_id_49>",
896
+ "<extra_id_50>",
897
+ "<extra_id_51>",
898
+ "<extra_id_52>",
899
+ "<extra_id_53>",
900
+ "<extra_id_54>",
901
+ "<extra_id_55>",
902
+ "<extra_id_56>",
903
+ "<extra_id_57>",
904
+ "<extra_id_58>",
905
+ "<extra_id_59>",
906
+ "<extra_id_60>",
907
+ "<extra_id_61>",
908
+ "<extra_id_62>",
909
+ "<extra_id_63>",
910
+ "<extra_id_64>",
911
+ "<extra_id_65>",
912
+ "<extra_id_66>",
913
+ "<extra_id_67>",
914
+ "<extra_id_68>",
915
+ "<extra_id_69>",
916
+ "<extra_id_70>",
917
+ "<extra_id_71>",
918
+ "<extra_id_72>",
919
+ "<extra_id_73>",
920
+ "<extra_id_74>",
921
+ "<extra_id_75>",
922
+ "<extra_id_76>",
923
+ "<extra_id_77>",
924
+ "<extra_id_78>",
925
+ "<extra_id_79>",
926
+ "<extra_id_80>",
927
+ "<extra_id_81>",
928
+ "<extra_id_82>",
929
+ "<extra_id_83>",
930
+ "<extra_id_84>",
931
+ "<extra_id_85>",
932
+ "<extra_id_86>",
933
+ "<extra_id_87>",
934
+ "<extra_id_88>",
935
+ "<extra_id_89>",
936
+ "<extra_id_90>",
937
+ "<extra_id_91>",
938
+ "<extra_id_92>",
939
+ "<extra_id_93>",
940
+ "<extra_id_94>",
941
+ "<extra_id_95>",
942
+ "<extra_id_96>",
943
+ "<extra_id_97>",
944
+ "<extra_id_98>",
945
+ "<extra_id_99>"
946
+ ],
947
+ "clean_up_tokenization_spaces": false,
948
+ "cls_token": "<s>",
949
+ "eos_token": "</s>",
950
+ "extra_ids": 100,
951
+ "extra_special_tokens": {},
952
+ "mask_token": "<mask>",
953
+ "model_max_length": 1000000000000000019884624838656,
954
+ "pad_token": "<pad>",
955
+ "sep_token": "</s>",
956
+ "tokenizer_class": "T5TokenizerFast",
957
+ "unk_token": "<unk>"
958
+ }
minicoderx-model/checkpoint-3/trainer_state.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 3.0,
5
+ "eval_steps": 500,
6
+ "global_step": 3,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [],
11
+ "logging_steps": 10,
12
+ "max_steps": 3,
13
+ "num_input_tokens_seen": 0,
14
+ "num_train_epochs": 3,
15
+ "save_steps": 500,
16
+ "stateful_callbacks": {
17
+ "TrainerControl": {
18
+ "args": {
19
+ "should_epoch_stop": false,
20
+ "should_evaluate": false,
21
+ "should_log": false,
22
+ "should_save": true,
23
+ "should_training_stop": true
24
+ },
25
+ "attributes": {}
26
+ }
27
+ },
28
+ "total_flos": 203012702208.0,
29
+ "train_batch_size": 4,
30
+ "trial_name": null,
31
+ "trial_params": null
32
+ }
minicoderx-model/checkpoint-3/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd544df3e986cab3c0663ced7b6b3ca6903b0db2be881d69b5180d0077410a13
3
+ size 5304
minicoderx-model/config.json ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "t5-small",
3
+ "architectures": [
4
+ "T5ForConditionalGeneration"
5
+ ],
6
+ "classifier_dropout": 0.0,
7
+ "d_ff": 2048,
8
+ "d_kv": 64,
9
+ "d_model": 512,
10
+ "decoder_start_token_id": 0,
11
+ "dense_act_fn": "relu",
12
+ "dropout_rate": 0.1,
13
+ "eos_token_id": 1,
14
+ "feed_forward_proj": "relu",
15
+ "initializer_factor": 1.0,
16
+ "is_encoder_decoder": true,
17
+ "is_gated_act": false,
18
+ "layer_norm_epsilon": 1e-06,
19
+ "model_type": "t5",
20
+ "n_positions": 512,
21
+ "num_decoder_layers": 6,
22
+ "num_heads": 8,
23
+ "num_layers": 6,
24
+ "output_past": true,
25
+ "pad_token_id": 0,
26
+ "relative_attention_max_distance": 128,
27
+ "relative_attention_num_buckets": 32,
28
+ "task_specific_params": {
29
+ "summarization": {
30
+ "early_stopping": true,
31
+ "length_penalty": 2.0,
32
+ "max_length": 200,
33
+ "min_length": 30,
34
+ "no_repeat_ngram_size": 3,
35
+ "num_beams": 4,
36
+ "prefix": "summarize: "
37
+ },
38
+ "translation_en_to_de": {
39
+ "early_stopping": true,
40
+ "max_length": 300,
41
+ "num_beams": 4,
42
+ "prefix": "translate English to German: "
43
+ },
44
+ "translation_en_to_fr": {
45
+ "early_stopping": true,
46
+ "max_length": 300,
47
+ "num_beams": 4,
48
+ "prefix": "translate English to French: "
49
+ },
50
+ "translation_en_to_ro": {
51
+ "early_stopping": true,
52
+ "max_length": 300,
53
+ "num_beams": 4,
54
+ "prefix": "translate English to Romanian: "
55
+ }
56
+ },
57
+ "torch_dtype": "float32",
58
+ "transformers_version": "4.49.0",
59
+ "use_cache": true,
60
+ "vocab_size": 32128
61
+ }
minicoderx-model/generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "decoder_start_token_id": 0,
4
+ "eos_token_id": 1,
5
+ "pad_token_id": 0,
6
+ "transformers_version": "4.49.0"
7
+ }
minicoderx-model/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d58f23c7336fdae923aa51a04dcc36156d74b7fd8d1a336b6c9f62f80b899e69
3
+ size 242041896
minicoderx-model/special_tokens_map.json ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<extra_id_0>",
4
+ "<extra_id_1>",
5
+ "<extra_id_2>",
6
+ "<extra_id_3>",
7
+ "<extra_id_4>",
8
+ "<extra_id_5>",
9
+ "<extra_id_6>",
10
+ "<extra_id_7>",
11
+ "<extra_id_8>",
12
+ "<extra_id_9>",
13
+ "<extra_id_10>",
14
+ "<extra_id_11>",
15
+ "<extra_id_12>",
16
+ "<extra_id_13>",
17
+ "<extra_id_14>",
18
+ "<extra_id_15>",
19
+ "<extra_id_16>",
20
+ "<extra_id_17>",
21
+ "<extra_id_18>",
22
+ "<extra_id_19>",
23
+ "<extra_id_20>",
24
+ "<extra_id_21>",
25
+ "<extra_id_22>",
26
+ "<extra_id_23>",
27
+ "<extra_id_24>",
28
+ "<extra_id_25>",
29
+ "<extra_id_26>",
30
+ "<extra_id_27>",
31
+ "<extra_id_28>",
32
+ "<extra_id_29>",
33
+ "<extra_id_30>",
34
+ "<extra_id_31>",
35
+ "<extra_id_32>",
36
+ "<extra_id_33>",
37
+ "<extra_id_34>",
38
+ "<extra_id_35>",
39
+ "<extra_id_36>",
40
+ "<extra_id_37>",
41
+ "<extra_id_38>",
42
+ "<extra_id_39>",
43
+ "<extra_id_40>",
44
+ "<extra_id_41>",
45
+ "<extra_id_42>",
46
+ "<extra_id_43>",
47
+ "<extra_id_44>",
48
+ "<extra_id_45>",
49
+ "<extra_id_46>",
50
+ "<extra_id_47>",
51
+ "<extra_id_48>",
52
+ "<extra_id_49>",
53
+ "<extra_id_50>",
54
+ "<extra_id_51>",
55
+ "<extra_id_52>",
56
+ "<extra_id_53>",
57
+ "<extra_id_54>",
58
+ "<extra_id_55>",
59
+ "<extra_id_56>",
60
+ "<extra_id_57>",
61
+ "<extra_id_58>",
62
+ "<extra_id_59>",
63
+ "<extra_id_60>",
64
+ "<extra_id_61>",
65
+ "<extra_id_62>",
66
+ "<extra_id_63>",
67
+ "<extra_id_64>",
68
+ "<extra_id_65>",
69
+ "<extra_id_66>",
70
+ "<extra_id_67>",
71
+ "<extra_id_68>",
72
+ "<extra_id_69>",
73
+ "<extra_id_70>",
74
+ "<extra_id_71>",
75
+ "<extra_id_72>",
76
+ "<extra_id_73>",
77
+ "<extra_id_74>",
78
+ "<extra_id_75>",
79
+ "<extra_id_76>",
80
+ "<extra_id_77>",
81
+ "<extra_id_78>",
82
+ "<extra_id_79>",
83
+ "<extra_id_80>",
84
+ "<extra_id_81>",
85
+ "<extra_id_82>",
86
+ "<extra_id_83>",
87
+ "<extra_id_84>",
88
+ "<extra_id_85>",
89
+ "<extra_id_86>",
90
+ "<extra_id_87>",
91
+ "<extra_id_88>",
92
+ "<extra_id_89>",
93
+ "<extra_id_90>",
94
+ "<extra_id_91>",
95
+ "<extra_id_92>",
96
+ "<extra_id_93>",
97
+ "<extra_id_94>",
98
+ "<extra_id_95>",
99
+ "<extra_id_96>",
100
+ "<extra_id_97>",
101
+ "<extra_id_98>",
102
+ "<extra_id_99>"
103
+ ],
104
+ "cls_token": {
105
+ "content": "<s>",
106
+ "lstrip": false,
107
+ "normalized": false,
108
+ "rstrip": false,
109
+ "single_word": false
110
+ },
111
+ "eos_token": "</s>",
112
+ "mask_token": {
113
+ "content": "<mask>",
114
+ "lstrip": false,
115
+ "normalized": false,
116
+ "rstrip": false,
117
+ "single_word": false
118
+ },
119
+ "pad_token": {
120
+ "content": "<pad>",
121
+ "lstrip": false,
122
+ "normalized": false,
123
+ "rstrip": false,
124
+ "single_word": false
125
+ },
126
+ "sep_token": {
127
+ "content": "</s>",
128
+ "lstrip": false,
129
+ "normalized": false,
130
+ "rstrip": false,
131
+ "single_word": false
132
+ },
133
+ "unk_token": {
134
+ "content": "<unk>",
135
+ "lstrip": false,
136
+ "normalized": false,
137
+ "rstrip": false,
138
+ "single_word": false
139
+ }
140
+ }
minicoderx-model/tokenizer.json ADDED
@@ -0,0 +1,1263 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": {
4
+ "direction": "Right",
5
+ "max_length": 128,
6
+ "strategy": "LongestFirst",
7
+ "stride": 0
8
+ },
9
+ "padding": {
10
+ "strategy": {
11
+ "Fixed": 128
12
+ },
13
+ "direction": "Right",
14
+ "pad_to_multiple_of": null,
15
+ "pad_id": 0,
16
+ "pad_type_id": 0,
17
+ "pad_token": "<pad>"
18
+ },
19
+ "added_tokens": [
20
+ {
21
+ "id": 0,
22
+ "content": "<pad>",
23
+ "single_word": false,
24
+ "lstrip": false,
25
+ "rstrip": false,
26
+ "normalized": false,
27
+ "special": true
28
+ },
29
+ {
30
+ "id": 1,
31
+ "content": "<s>",
32
+ "single_word": false,
33
+ "lstrip": false,
34
+ "rstrip": false,
35
+ "normalized": false,
36
+ "special": true
37
+ },
38
+ {
39
+ "id": 2,
40
+ "content": "</s>",
41
+ "single_word": false,
42
+ "lstrip": false,
43
+ "rstrip": false,
44
+ "normalized": false,
45
+ "special": true
46
+ },
47
+ {
48
+ "id": 3,
49
+ "content": "<unk>",
50
+ "single_word": false,
51
+ "lstrip": false,
52
+ "rstrip": false,
53
+ "normalized": false,
54
+ "special": true
55
+ },
56
+ {
57
+ "id": 4,
58
+ "content": "<mask>",
59
+ "single_word": false,
60
+ "lstrip": false,
61
+ "rstrip": false,
62
+ "normalized": false,
63
+ "special": true
64
+ },
65
+ {
66
+ "id": 60,
67
+ "content": "<extra_id_0>",
68
+ "single_word": false,
69
+ "lstrip": false,
70
+ "rstrip": false,
71
+ "normalized": false,
72
+ "special": true
73
+ },
74
+ {
75
+ "id": 61,
76
+ "content": "<extra_id_1>",
77
+ "single_word": false,
78
+ "lstrip": false,
79
+ "rstrip": false,
80
+ "normalized": false,
81
+ "special": true
82
+ },
83
+ {
84
+ "id": 62,
85
+ "content": "<extra_id_2>",
86
+ "single_word": false,
87
+ "lstrip": false,
88
+ "rstrip": false,
89
+ "normalized": false,
90
+ "special": true
91
+ },
92
+ {
93
+ "id": 63,
94
+ "content": "<extra_id_3>",
95
+ "single_word": false,
96
+ "lstrip": false,
97
+ "rstrip": false,
98
+ "normalized": false,
99
+ "special": true
100
+ },
101
+ {
102
+ "id": 64,
103
+ "content": "<extra_id_4>",
104
+ "single_word": false,
105
+ "lstrip": false,
106
+ "rstrip": false,
107
+ "normalized": false,
108
+ "special": true
109
+ },
110
+ {
111
+ "id": 65,
112
+ "content": "<extra_id_5>",
113
+ "single_word": false,
114
+ "lstrip": false,
115
+ "rstrip": false,
116
+ "normalized": false,
117
+ "special": true
118
+ },
119
+ {
120
+ "id": 66,
121
+ "content": "<extra_id_6>",
122
+ "single_word": false,
123
+ "lstrip": false,
124
+ "rstrip": false,
125
+ "normalized": false,
126
+ "special": true
127
+ },
128
+ {
129
+ "id": 67,
130
+ "content": "<extra_id_7>",
131
+ "single_word": false,
132
+ "lstrip": false,
133
+ "rstrip": false,
134
+ "normalized": false,
135
+ "special": true
136
+ },
137
+ {
138
+ "id": 68,
139
+ "content": "<extra_id_8>",
140
+ "single_word": false,
141
+ "lstrip": false,
142
+ "rstrip": false,
143
+ "normalized": false,
144
+ "special": true
145
+ },
146
+ {
147
+ "id": 69,
148
+ "content": "<extra_id_9>",
149
+ "single_word": false,
150
+ "lstrip": false,
151
+ "rstrip": false,
152
+ "normalized": false,
153
+ "special": true
154
+ },
155
+ {
156
+ "id": 70,
157
+ "content": "<extra_id_10>",
158
+ "single_word": false,
159
+ "lstrip": false,
160
+ "rstrip": false,
161
+ "normalized": false,
162
+ "special": true
163
+ },
164
+ {
165
+ "id": 71,
166
+ "content": "<extra_id_11>",
167
+ "single_word": false,
168
+ "lstrip": false,
169
+ "rstrip": false,
170
+ "normalized": false,
171
+ "special": true
172
+ },
173
+ {
174
+ "id": 72,
175
+ "content": "<extra_id_12>",
176
+ "single_word": false,
177
+ "lstrip": false,
178
+ "rstrip": false,
179
+ "normalized": false,
180
+ "special": true
181
+ },
182
+ {
183
+ "id": 73,
184
+ "content": "<extra_id_13>",
185
+ "single_word": false,
186
+ "lstrip": false,
187
+ "rstrip": false,
188
+ "normalized": false,
189
+ "special": true
190
+ },
191
+ {
192
+ "id": 74,
193
+ "content": "<extra_id_14>",
194
+ "single_word": false,
195
+ "lstrip": false,
196
+ "rstrip": false,
197
+ "normalized": false,
198
+ "special": true
199
+ },
200
+ {
201
+ "id": 75,
202
+ "content": "<extra_id_15>",
203
+ "single_word": false,
204
+ "lstrip": false,
205
+ "rstrip": false,
206
+ "normalized": false,
207
+ "special": true
208
+ },
209
+ {
210
+ "id": 76,
211
+ "content": "<extra_id_16>",
212
+ "single_word": false,
213
+ "lstrip": false,
214
+ "rstrip": false,
215
+ "normalized": false,
216
+ "special": true
217
+ },
218
+ {
219
+ "id": 77,
220
+ "content": "<extra_id_17>",
221
+ "single_word": false,
222
+ "lstrip": false,
223
+ "rstrip": false,
224
+ "normalized": false,
225
+ "special": true
226
+ },
227
+ {
228
+ "id": 78,
229
+ "content": "<extra_id_18>",
230
+ "single_word": false,
231
+ "lstrip": false,
232
+ "rstrip": false,
233
+ "normalized": false,
234
+ "special": true
235
+ },
236
+ {
237
+ "id": 79,
238
+ "content": "<extra_id_19>",
239
+ "single_word": false,
240
+ "lstrip": false,
241
+ "rstrip": false,
242
+ "normalized": false,
243
+ "special": true
244
+ },
245
+ {
246
+ "id": 80,
247
+ "content": "<extra_id_20>",
248
+ "single_word": false,
249
+ "lstrip": false,
250
+ "rstrip": false,
251
+ "normalized": false,
252
+ "special": true
253
+ },
254
+ {
255
+ "id": 81,
256
+ "content": "<extra_id_21>",
257
+ "single_word": false,
258
+ "lstrip": false,
259
+ "rstrip": false,
260
+ "normalized": false,
261
+ "special": true
262
+ },
263
+ {
264
+ "id": 82,
265
+ "content": "<extra_id_22>",
266
+ "single_word": false,
267
+ "lstrip": false,
268
+ "rstrip": false,
269
+ "normalized": false,
270
+ "special": true
271
+ },
272
+ {
273
+ "id": 83,
274
+ "content": "<extra_id_23>",
275
+ "single_word": false,
276
+ "lstrip": false,
277
+ "rstrip": false,
278
+ "normalized": false,
279
+ "special": true
280
+ },
281
+ {
282
+ "id": 84,
283
+ "content": "<extra_id_24>",
284
+ "single_word": false,
285
+ "lstrip": false,
286
+ "rstrip": false,
287
+ "normalized": false,
288
+ "special": true
289
+ },
290
+ {
291
+ "id": 85,
292
+ "content": "<extra_id_25>",
293
+ "single_word": false,
294
+ "lstrip": false,
295
+ "rstrip": false,
296
+ "normalized": false,
297
+ "special": true
298
+ },
299
+ {
300
+ "id": 86,
301
+ "content": "<extra_id_26>",
302
+ "single_word": false,
303
+ "lstrip": false,
304
+ "rstrip": false,
305
+ "normalized": false,
306
+ "special": true
307
+ },
308
+ {
309
+ "id": 87,
310
+ "content": "<extra_id_27>",
311
+ "single_word": false,
312
+ "lstrip": false,
313
+ "rstrip": false,
314
+ "normalized": false,
315
+ "special": true
316
+ },
317
+ {
318
+ "id": 88,
319
+ "content": "<extra_id_28>",
320
+ "single_word": false,
321
+ "lstrip": false,
322
+ "rstrip": false,
323
+ "normalized": false,
324
+ "special": true
325
+ },
326
+ {
327
+ "id": 89,
328
+ "content": "<extra_id_29>",
329
+ "single_word": false,
330
+ "lstrip": false,
331
+ "rstrip": false,
332
+ "normalized": false,
333
+ "special": true
334
+ },
335
+ {
336
+ "id": 90,
337
+ "content": "<extra_id_30>",
338
+ "single_word": false,
339
+ "lstrip": false,
340
+ "rstrip": false,
341
+ "normalized": false,
342
+ "special": true
343
+ },
344
+ {
345
+ "id": 91,
346
+ "content": "<extra_id_31>",
347
+ "single_word": false,
348
+ "lstrip": false,
349
+ "rstrip": false,
350
+ "normalized": false,
351
+ "special": true
352
+ },
353
+ {
354
+ "id": 92,
355
+ "content": "<extra_id_32>",
356
+ "single_word": false,
357
+ "lstrip": false,
358
+ "rstrip": false,
359
+ "normalized": false,
360
+ "special": true
361
+ },
362
+ {
363
+ "id": 93,
364
+ "content": "<extra_id_33>",
365
+ "single_word": false,
366
+ "lstrip": false,
367
+ "rstrip": false,
368
+ "normalized": false,
369
+ "special": true
370
+ },
371
+ {
372
+ "id": 94,
373
+ "content": "<extra_id_34>",
374
+ "single_word": false,
375
+ "lstrip": false,
376
+ "rstrip": false,
377
+ "normalized": false,
378
+ "special": true
379
+ },
380
+ {
381
+ "id": 95,
382
+ "content": "<extra_id_35>",
383
+ "single_word": false,
384
+ "lstrip": false,
385
+ "rstrip": false,
386
+ "normalized": false,
387
+ "special": true
388
+ },
389
+ {
390
+ "id": 96,
391
+ "content": "<extra_id_36>",
392
+ "single_word": false,
393
+ "lstrip": false,
394
+ "rstrip": false,
395
+ "normalized": false,
396
+ "special": true
397
+ },
398
+ {
399
+ "id": 97,
400
+ "content": "<extra_id_37>",
401
+ "single_word": false,
402
+ "lstrip": false,
403
+ "rstrip": false,
404
+ "normalized": false,
405
+ "special": true
406
+ },
407
+ {
408
+ "id": 98,
409
+ "content": "<extra_id_38>",
410
+ "single_word": false,
411
+ "lstrip": false,
412
+ "rstrip": false,
413
+ "normalized": false,
414
+ "special": true
415
+ },
416
+ {
417
+ "id": 99,
418
+ "content": "<extra_id_39>",
419
+ "single_word": false,
420
+ "lstrip": false,
421
+ "rstrip": false,
422
+ "normalized": false,
423
+ "special": true
424
+ },
425
+ {
426
+ "id": 100,
427
+ "content": "<extra_id_40>",
428
+ "single_word": false,
429
+ "lstrip": false,
430
+ "rstrip": false,
431
+ "normalized": false,
432
+ "special": true
433
+ },
434
+ {
435
+ "id": 101,
436
+ "content": "<extra_id_41>",
437
+ "single_word": false,
438
+ "lstrip": false,
439
+ "rstrip": false,
440
+ "normalized": false,
441
+ "special": true
442
+ },
443
+ {
444
+ "id": 102,
445
+ "content": "<extra_id_42>",
446
+ "single_word": false,
447
+ "lstrip": false,
448
+ "rstrip": false,
449
+ "normalized": false,
450
+ "special": true
451
+ },
452
+ {
453
+ "id": 103,
454
+ "content": "<extra_id_43>",
455
+ "single_word": false,
456
+ "lstrip": false,
457
+ "rstrip": false,
458
+ "normalized": false,
459
+ "special": true
460
+ },
461
+ {
462
+ "id": 104,
463
+ "content": "<extra_id_44>",
464
+ "single_word": false,
465
+ "lstrip": false,
466
+ "rstrip": false,
467
+ "normalized": false,
468
+ "special": true
469
+ },
470
+ {
471
+ "id": 105,
472
+ "content": "<extra_id_45>",
473
+ "single_word": false,
474
+ "lstrip": false,
475
+ "rstrip": false,
476
+ "normalized": false,
477
+ "special": true
478
+ },
479
+ {
480
+ "id": 106,
481
+ "content": "<extra_id_46>",
482
+ "single_word": false,
483
+ "lstrip": false,
484
+ "rstrip": false,
485
+ "normalized": false,
486
+ "special": true
487
+ },
488
+ {
489
+ "id": 107,
490
+ "content": "<extra_id_47>",
491
+ "single_word": false,
492
+ "lstrip": false,
493
+ "rstrip": false,
494
+ "normalized": false,
495
+ "special": true
496
+ },
497
+ {
498
+ "id": 108,
499
+ "content": "<extra_id_48>",
500
+ "single_word": false,
501
+ "lstrip": false,
502
+ "rstrip": false,
503
+ "normalized": false,
504
+ "special": true
505
+ },
506
+ {
507
+ "id": 109,
508
+ "content": "<extra_id_49>",
509
+ "single_word": false,
510
+ "lstrip": false,
511
+ "rstrip": false,
512
+ "normalized": false,
513
+ "special": true
514
+ },
515
+ {
516
+ "id": 110,
517
+ "content": "<extra_id_50>",
518
+ "single_word": false,
519
+ "lstrip": false,
520
+ "rstrip": false,
521
+ "normalized": false,
522
+ "special": true
523
+ },
524
+ {
525
+ "id": 111,
526
+ "content": "<extra_id_51>",
527
+ "single_word": false,
528
+ "lstrip": false,
529
+ "rstrip": false,
530
+ "normalized": false,
531
+ "special": true
532
+ },
533
+ {
534
+ "id": 112,
535
+ "content": "<extra_id_52>",
536
+ "single_word": false,
537
+ "lstrip": false,
538
+ "rstrip": false,
539
+ "normalized": false,
540
+ "special": true
541
+ },
542
+ {
543
+ "id": 113,
544
+ "content": "<extra_id_53>",
545
+ "single_word": false,
546
+ "lstrip": false,
547
+ "rstrip": false,
548
+ "normalized": false,
549
+ "special": true
550
+ },
551
+ {
552
+ "id": 114,
553
+ "content": "<extra_id_54>",
554
+ "single_word": false,
555
+ "lstrip": false,
556
+ "rstrip": false,
557
+ "normalized": false,
558
+ "special": true
559
+ },
560
+ {
561
+ "id": 115,
562
+ "content": "<extra_id_55>",
563
+ "single_word": false,
564
+ "lstrip": false,
565
+ "rstrip": false,
566
+ "normalized": false,
567
+ "special": true
568
+ },
569
+ {
570
+ "id": 116,
571
+ "content": "<extra_id_56>",
572
+ "single_word": false,
573
+ "lstrip": false,
574
+ "rstrip": false,
575
+ "normalized": false,
576
+ "special": true
577
+ },
578
+ {
579
+ "id": 117,
580
+ "content": "<extra_id_57>",
581
+ "single_word": false,
582
+ "lstrip": false,
583
+ "rstrip": false,
584
+ "normalized": false,
585
+ "special": true
586
+ },
587
+ {
588
+ "id": 118,
589
+ "content": "<extra_id_58>",
590
+ "single_word": false,
591
+ "lstrip": false,
592
+ "rstrip": false,
593
+ "normalized": false,
594
+ "special": true
595
+ },
596
+ {
597
+ "id": 119,
598
+ "content": "<extra_id_59>",
599
+ "single_word": false,
600
+ "lstrip": false,
601
+ "rstrip": false,
602
+ "normalized": false,
603
+ "special": true
604
+ },
605
+ {
606
+ "id": 120,
607
+ "content": "<extra_id_60>",
608
+ "single_word": false,
609
+ "lstrip": false,
610
+ "rstrip": false,
611
+ "normalized": false,
612
+ "special": true
613
+ },
614
+ {
615
+ "id": 121,
616
+ "content": "<extra_id_61>",
617
+ "single_word": false,
618
+ "lstrip": false,
619
+ "rstrip": false,
620
+ "normalized": false,
621
+ "special": true
622
+ },
623
+ {
624
+ "id": 122,
625
+ "content": "<extra_id_62>",
626
+ "single_word": false,
627
+ "lstrip": false,
628
+ "rstrip": false,
629
+ "normalized": false,
630
+ "special": true
631
+ },
632
+ {
633
+ "id": 123,
634
+ "content": "<extra_id_63>",
635
+ "single_word": false,
636
+ "lstrip": false,
637
+ "rstrip": false,
638
+ "normalized": false,
639
+ "special": true
640
+ },
641
+ {
642
+ "id": 124,
643
+ "content": "<extra_id_64>",
644
+ "single_word": false,
645
+ "lstrip": false,
646
+ "rstrip": false,
647
+ "normalized": false,
648
+ "special": true
649
+ },
650
+ {
651
+ "id": 125,
652
+ "content": "<extra_id_65>",
653
+ "single_word": false,
654
+ "lstrip": false,
655
+ "rstrip": false,
656
+ "normalized": false,
657
+ "special": true
658
+ },
659
+ {
660
+ "id": 126,
661
+ "content": "<extra_id_66>",
662
+ "single_word": false,
663
+ "lstrip": false,
664
+ "rstrip": false,
665
+ "normalized": false,
666
+ "special": true
667
+ },
668
+ {
669
+ "id": 127,
670
+ "content": "<extra_id_67>",
671
+ "single_word": false,
672
+ "lstrip": false,
673
+ "rstrip": false,
674
+ "normalized": false,
675
+ "special": true
676
+ },
677
+ {
678
+ "id": 128,
679
+ "content": "<extra_id_68>",
680
+ "single_word": false,
681
+ "lstrip": false,
682
+ "rstrip": false,
683
+ "normalized": false,
684
+ "special": true
685
+ },
686
+ {
687
+ "id": 129,
688
+ "content": "<extra_id_69>",
689
+ "single_word": false,
690
+ "lstrip": false,
691
+ "rstrip": false,
692
+ "normalized": false,
693
+ "special": true
694
+ },
695
+ {
696
+ "id": 130,
697
+ "content": "<extra_id_70>",
698
+ "single_word": false,
699
+ "lstrip": false,
700
+ "rstrip": false,
701
+ "normalized": false,
702
+ "special": true
703
+ },
704
+ {
705
+ "id": 131,
706
+ "content": "<extra_id_71>",
707
+ "single_word": false,
708
+ "lstrip": false,
709
+ "rstrip": false,
710
+ "normalized": false,
711
+ "special": true
712
+ },
713
+ {
714
+ "id": 132,
715
+ "content": "<extra_id_72>",
716
+ "single_word": false,
717
+ "lstrip": false,
718
+ "rstrip": false,
719
+ "normalized": false,
720
+ "special": true
721
+ },
722
+ {
723
+ "id": 133,
724
+ "content": "<extra_id_73>",
725
+ "single_word": false,
726
+ "lstrip": false,
727
+ "rstrip": false,
728
+ "normalized": false,
729
+ "special": true
730
+ },
731
+ {
732
+ "id": 134,
733
+ "content": "<extra_id_74>",
734
+ "single_word": false,
735
+ "lstrip": false,
736
+ "rstrip": false,
737
+ "normalized": false,
738
+ "special": true
739
+ },
740
+ {
741
+ "id": 135,
742
+ "content": "<extra_id_75>",
743
+ "single_word": false,
744
+ "lstrip": false,
745
+ "rstrip": false,
746
+ "normalized": false,
747
+ "special": true
748
+ },
749
+ {
750
+ "id": 136,
751
+ "content": "<extra_id_76>",
752
+ "single_word": false,
753
+ "lstrip": false,
754
+ "rstrip": false,
755
+ "normalized": false,
756
+ "special": true
757
+ },
758
+ {
759
+ "id": 137,
760
+ "content": "<extra_id_77>",
761
+ "single_word": false,
762
+ "lstrip": false,
763
+ "rstrip": false,
764
+ "normalized": false,
765
+ "special": true
766
+ },
767
+ {
768
+ "id": 138,
769
+ "content": "<extra_id_78>",
770
+ "single_word": false,
771
+ "lstrip": false,
772
+ "rstrip": false,
773
+ "normalized": false,
774
+ "special": true
775
+ },
776
+ {
777
+ "id": 139,
778
+ "content": "<extra_id_79>",
779
+ "single_word": false,
780
+ "lstrip": false,
781
+ "rstrip": false,
782
+ "normalized": false,
783
+ "special": true
784
+ },
785
+ {
786
+ "id": 140,
787
+ "content": "<extra_id_80>",
788
+ "single_word": false,
789
+ "lstrip": false,
790
+ "rstrip": false,
791
+ "normalized": false,
792
+ "special": true
793
+ },
794
+ {
795
+ "id": 141,
796
+ "content": "<extra_id_81>",
797
+ "single_word": false,
798
+ "lstrip": false,
799
+ "rstrip": false,
800
+ "normalized": false,
801
+ "special": true
802
+ },
803
+ {
804
+ "id": 142,
805
+ "content": "<extra_id_82>",
806
+ "single_word": false,
807
+ "lstrip": false,
808
+ "rstrip": false,
809
+ "normalized": false,
810
+ "special": true
811
+ },
812
+ {
813
+ "id": 143,
814
+ "content": "<extra_id_83>",
815
+ "single_word": false,
816
+ "lstrip": false,
817
+ "rstrip": false,
818
+ "normalized": false,
819
+ "special": true
820
+ },
821
+ {
822
+ "id": 144,
823
+ "content": "<extra_id_84>",
824
+ "single_word": false,
825
+ "lstrip": false,
826
+ "rstrip": false,
827
+ "normalized": false,
828
+ "special": true
829
+ },
830
+ {
831
+ "id": 145,
832
+ "content": "<extra_id_85>",
833
+ "single_word": false,
834
+ "lstrip": false,
835
+ "rstrip": false,
836
+ "normalized": false,
837
+ "special": true
838
+ },
839
+ {
840
+ "id": 146,
841
+ "content": "<extra_id_86>",
842
+ "single_word": false,
843
+ "lstrip": false,
844
+ "rstrip": false,
845
+ "normalized": false,
846
+ "special": true
847
+ },
848
+ {
849
+ "id": 147,
850
+ "content": "<extra_id_87>",
851
+ "single_word": false,
852
+ "lstrip": false,
853
+ "rstrip": false,
854
+ "normalized": false,
855
+ "special": true
856
+ },
857
+ {
858
+ "id": 148,
859
+ "content": "<extra_id_88>",
860
+ "single_word": false,
861
+ "lstrip": false,
862
+ "rstrip": false,
863
+ "normalized": false,
864
+ "special": true
865
+ },
866
+ {
867
+ "id": 149,
868
+ "content": "<extra_id_89>",
869
+ "single_word": false,
870
+ "lstrip": false,
871
+ "rstrip": false,
872
+ "normalized": false,
873
+ "special": true
874
+ },
875
+ {
876
+ "id": 150,
877
+ "content": "<extra_id_90>",
878
+ "single_word": false,
879
+ "lstrip": false,
880
+ "rstrip": false,
881
+ "normalized": false,
882
+ "special": true
883
+ },
884
+ {
885
+ "id": 151,
886
+ "content": "<extra_id_91>",
887
+ "single_word": false,
888
+ "lstrip": false,
889
+ "rstrip": false,
890
+ "normalized": false,
891
+ "special": true
892
+ },
893
+ {
894
+ "id": 152,
895
+ "content": "<extra_id_92>",
896
+ "single_word": false,
897
+ "lstrip": false,
898
+ "rstrip": false,
899
+ "normalized": false,
900
+ "special": true
901
+ },
902
+ {
903
+ "id": 153,
904
+ "content": "<extra_id_93>",
905
+ "single_word": false,
906
+ "lstrip": false,
907
+ "rstrip": false,
908
+ "normalized": false,
909
+ "special": true
910
+ },
911
+ {
912
+ "id": 154,
913
+ "content": "<extra_id_94>",
914
+ "single_word": false,
915
+ "lstrip": false,
916
+ "rstrip": false,
917
+ "normalized": false,
918
+ "special": true
919
+ },
920
+ {
921
+ "id": 155,
922
+ "content": "<extra_id_95>",
923
+ "single_word": false,
924
+ "lstrip": false,
925
+ "rstrip": false,
926
+ "normalized": false,
927
+ "special": true
928
+ },
929
+ {
930
+ "id": 156,
931
+ "content": "<extra_id_96>",
932
+ "single_word": false,
933
+ "lstrip": false,
934
+ "rstrip": false,
935
+ "normalized": false,
936
+ "special": true
937
+ },
938
+ {
939
+ "id": 157,
940
+ "content": "<extra_id_97>",
941
+ "single_word": false,
942
+ "lstrip": false,
943
+ "rstrip": false,
944
+ "normalized": false,
945
+ "special": true
946
+ },
947
+ {
948
+ "id": 158,
949
+ "content": "<extra_id_98>",
950
+ "single_word": false,
951
+ "lstrip": false,
952
+ "rstrip": false,
953
+ "normalized": false,
954
+ "special": true
955
+ },
956
+ {
957
+ "id": 159,
958
+ "content": "<extra_id_99>",
959
+ "single_word": false,
960
+ "lstrip": false,
961
+ "rstrip": false,
962
+ "normalized": false,
963
+ "special": true
964
+ }
965
+ ],
966
+ "normalizer": {
967
+ "type": "Sequence",
968
+ "normalizers": [
969
+ {
970
+ "type": "NFD"
971
+ },
972
+ {
973
+ "type": "Lowercase"
974
+ },
975
+ {
976
+ "type": "StripAccents"
977
+ }
978
+ ]
979
+ },
980
+ "pre_tokenizer": {
981
+ "type": "Whitespace"
982
+ },
983
+ "post_processor": {
984
+ "type": "TemplateProcessing",
985
+ "single": [
986
+ {
987
+ "SpecialToken": {
988
+ "id": "<s>",
989
+ "type_id": 0
990
+ }
991
+ },
992
+ {
993
+ "Sequence": {
994
+ "id": "A",
995
+ "type_id": 0
996
+ }
997
+ },
998
+ {
999
+ "SpecialToken": {
1000
+ "id": "</s>",
1001
+ "type_id": 0
1002
+ }
1003
+ }
1004
+ ],
1005
+ "pair": [
1006
+ {
1007
+ "SpecialToken": {
1008
+ "id": "<s>",
1009
+ "type_id": 0
1010
+ }
1011
+ },
1012
+ {
1013
+ "Sequence": {
1014
+ "id": "A",
1015
+ "type_id": 0
1016
+ }
1017
+ },
1018
+ {
1019
+ "SpecialToken": {
1020
+ "id": "</s>",
1021
+ "type_id": 0
1022
+ }
1023
+ },
1024
+ {
1025
+ "SpecialToken": {
1026
+ "id": "</s>",
1027
+ "type_id": 0
1028
+ }
1029
+ },
1030
+ {
1031
+ "Sequence": {
1032
+ "id": "B",
1033
+ "type_id": 0
1034
+ }
1035
+ },
1036
+ {
1037
+ "SpecialToken": {
1038
+ "id": "</s>",
1039
+ "type_id": 0
1040
+ }
1041
+ }
1042
+ ],
1043
+ "special_tokens": {
1044
+ "</s>": {
1045
+ "id": "</s>",
1046
+ "ids": [
1047
+ 2
1048
+ ],
1049
+ "tokens": [
1050
+ "</s>"
1051
+ ]
1052
+ },
1053
+ "<s>": {
1054
+ "id": "<s>",
1055
+ "ids": [
1056
+ 1
1057
+ ],
1058
+ "tokens": [
1059
+ "<s>"
1060
+ ]
1061
+ }
1062
+ }
1063
+ },
1064
+ "decoder": null,
1065
+ "model": {
1066
+ "type": "BPE",
1067
+ "dropout": null,
1068
+ "unk_token": null,
1069
+ "continuing_subword_prefix": null,
1070
+ "end_of_word_suffix": null,
1071
+ "fuse_unk": false,
1072
+ "byte_fallback": false,
1073
+ "ignore_merges": false,
1074
+ "vocab": {
1075
+ "<pad>": 0,
1076
+ "<s>": 1,
1077
+ "</s>": 2,
1078
+ "<unk>": 3,
1079
+ "<mask>": 4,
1080
+ "!": 5,
1081
+ "\"": 6,
1082
+ "(": 7,
1083
+ ")": 8,
1084
+ "+": 9,
1085
+ ",": 10,
1086
+ ":": 11,
1087
+ "a": 12,
1088
+ "b": 13,
1089
+ "c": 14,
1090
+ "d": 15,
1091
+ "e": 16,
1092
+ "f": 17,
1093
+ "h": 18,
1094
+ "i": 19,
1095
+ "l": 20,
1096
+ "n": 21,
1097
+ "o": 22,
1098
+ "p": 23,
1099
+ "r": 24,
1100
+ "s": 25,
1101
+ "t": 26,
1102
+ "u": 27,
1103
+ "w": 28,
1104
+ "as": 29,
1105
+ "el": 30,
1106
+ "hel": 31,
1107
+ "ld": 32,
1108
+ "lo": 33,
1109
+ "or": 34,
1110
+ "wor": 35,
1111
+ "ass": 36,
1112
+ "hello": 37,
1113
+ "world": 38,
1114
+ "!\"": 39,
1115
+ "(\"": 40,
1116
+ "):": 41,
1117
+ "ad": 42,
1118
+ "cl": 43,
1119
+ "de": 44,
1120
+ "et": 45,
1121
+ "in": 46,
1122
+ "pr": 47,
1123
+ "pass": 48,
1124
+ "rn": 49,
1125
+ "ret": 50,
1126
+ "urn": 51,
1127
+ "helloworld": 52,
1128
+ "!\")": 53,
1129
+ "add": 54,
1130
+ "class": 55,
1131
+ "def": 56,
1132
+ "int": 57,
1133
+ "print": 58,
1134
+ "return": 59
1135
+ },
1136
+ "merges": [
1137
+ [
1138
+ "a",
1139
+ "s"
1140
+ ],
1141
+ [
1142
+ "e",
1143
+ "l"
1144
+ ],
1145
+ [
1146
+ "h",
1147
+ "el"
1148
+ ],
1149
+ [
1150
+ "l",
1151
+ "d"
1152
+ ],
1153
+ [
1154
+ "l",
1155
+ "o"
1156
+ ],
1157
+ [
1158
+ "o",
1159
+ "r"
1160
+ ],
1161
+ [
1162
+ "w",
1163
+ "or"
1164
+ ],
1165
+ [
1166
+ "as",
1167
+ "s"
1168
+ ],
1169
+ [
1170
+ "hel",
1171
+ "lo"
1172
+ ],
1173
+ [
1174
+ "wor",
1175
+ "ld"
1176
+ ],
1177
+ [
1178
+ "!",
1179
+ "\""
1180
+ ],
1181
+ [
1182
+ "(",
1183
+ "\""
1184
+ ],
1185
+ [
1186
+ ")",
1187
+ ":"
1188
+ ],
1189
+ [
1190
+ "a",
1191
+ "d"
1192
+ ],
1193
+ [
1194
+ "c",
1195
+ "l"
1196
+ ],
1197
+ [
1198
+ "d",
1199
+ "e"
1200
+ ],
1201
+ [
1202
+ "e",
1203
+ "t"
1204
+ ],
1205
+ [
1206
+ "i",
1207
+ "n"
1208
+ ],
1209
+ [
1210
+ "p",
1211
+ "r"
1212
+ ],
1213
+ [
1214
+ "p",
1215
+ "ass"
1216
+ ],
1217
+ [
1218
+ "r",
1219
+ "n"
1220
+ ],
1221
+ [
1222
+ "r",
1223
+ "et"
1224
+ ],
1225
+ [
1226
+ "u",
1227
+ "rn"
1228
+ ],
1229
+ [
1230
+ "hello",
1231
+ "world"
1232
+ ],
1233
+ [
1234
+ "!\"",
1235
+ ")"
1236
+ ],
1237
+ [
1238
+ "ad",
1239
+ "d"
1240
+ ],
1241
+ [
1242
+ "cl",
1243
+ "ass"
1244
+ ],
1245
+ [
1246
+ "de",
1247
+ "f"
1248
+ ],
1249
+ [
1250
+ "in",
1251
+ "t"
1252
+ ],
1253
+ [
1254
+ "pr",
1255
+ "int"
1256
+ ],
1257
+ [
1258
+ "ret",
1259
+ "urn"
1260
+ ]
1261
+ ]
1262
+ }
1263
+ }
minicoderx-model/tokenizer_config.json ADDED
@@ -0,0 +1,958 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": null,
3
+ "added_tokens_decoder": {
4
+ "0": {
5
+ "content": "<pad>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "1": {
13
+ "content": "<s>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "2": {
21
+ "content": "</s>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "3": {
29
+ "content": "<unk>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "4": {
37
+ "content": "<mask>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "60": {
45
+ "content": "<extra_id_0>",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false,
50
+ "special": true
51
+ },
52
+ "61": {
53
+ "content": "<extra_id_1>",
54
+ "lstrip": false,
55
+ "normalized": false,
56
+ "rstrip": false,
57
+ "single_word": false,
58
+ "special": true
59
+ },
60
+ "62": {
61
+ "content": "<extra_id_2>",
62
+ "lstrip": false,
63
+ "normalized": false,
64
+ "rstrip": false,
65
+ "single_word": false,
66
+ "special": true
67
+ },
68
+ "63": {
69
+ "content": "<extra_id_3>",
70
+ "lstrip": false,
71
+ "normalized": false,
72
+ "rstrip": false,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "64": {
77
+ "content": "<extra_id_4>",
78
+ "lstrip": false,
79
+ "normalized": false,
80
+ "rstrip": false,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "65": {
85
+ "content": "<extra_id_5>",
86
+ "lstrip": false,
87
+ "normalized": false,
88
+ "rstrip": false,
89
+ "single_word": false,
90
+ "special": true
91
+ },
92
+ "66": {
93
+ "content": "<extra_id_6>",
94
+ "lstrip": false,
95
+ "normalized": false,
96
+ "rstrip": false,
97
+ "single_word": false,
98
+ "special": true
99
+ },
100
+ "67": {
101
+ "content": "<extra_id_7>",
102
+ "lstrip": false,
103
+ "normalized": false,
104
+ "rstrip": false,
105
+ "single_word": false,
106
+ "special": true
107
+ },
108
+ "68": {
109
+ "content": "<extra_id_8>",
110
+ "lstrip": false,
111
+ "normalized": false,
112
+ "rstrip": false,
113
+ "single_word": false,
114
+ "special": true
115
+ },
116
+ "69": {
117
+ "content": "<extra_id_9>",
118
+ "lstrip": false,
119
+ "normalized": false,
120
+ "rstrip": false,
121
+ "single_word": false,
122
+ "special": true
123
+ },
124
+ "70": {
125
+ "content": "<extra_id_10>",
126
+ "lstrip": false,
127
+ "normalized": false,
128
+ "rstrip": false,
129
+ "single_word": false,
130
+ "special": true
131
+ },
132
+ "71": {
133
+ "content": "<extra_id_11>",
134
+ "lstrip": false,
135
+ "normalized": false,
136
+ "rstrip": false,
137
+ "single_word": false,
138
+ "special": true
139
+ },
140
+ "72": {
141
+ "content": "<extra_id_12>",
142
+ "lstrip": false,
143
+ "normalized": false,
144
+ "rstrip": false,
145
+ "single_word": false,
146
+ "special": true
147
+ },
148
+ "73": {
149
+ "content": "<extra_id_13>",
150
+ "lstrip": false,
151
+ "normalized": false,
152
+ "rstrip": false,
153
+ "single_word": false,
154
+ "special": true
155
+ },
156
+ "74": {
157
+ "content": "<extra_id_14>",
158
+ "lstrip": false,
159
+ "normalized": false,
160
+ "rstrip": false,
161
+ "single_word": false,
162
+ "special": true
163
+ },
164
+ "75": {
165
+ "content": "<extra_id_15>",
166
+ "lstrip": false,
167
+ "normalized": false,
168
+ "rstrip": false,
169
+ "single_word": false,
170
+ "special": true
171
+ },
172
+ "76": {
173
+ "content": "<extra_id_16>",
174
+ "lstrip": false,
175
+ "normalized": false,
176
+ "rstrip": false,
177
+ "single_word": false,
178
+ "special": true
179
+ },
180
+ "77": {
181
+ "content": "<extra_id_17>",
182
+ "lstrip": false,
183
+ "normalized": false,
184
+ "rstrip": false,
185
+ "single_word": false,
186
+ "special": true
187
+ },
188
+ "78": {
189
+ "content": "<extra_id_18>",
190
+ "lstrip": false,
191
+ "normalized": false,
192
+ "rstrip": false,
193
+ "single_word": false,
194
+ "special": true
195
+ },
196
+ "79": {
197
+ "content": "<extra_id_19>",
198
+ "lstrip": false,
199
+ "normalized": false,
200
+ "rstrip": false,
201
+ "single_word": false,
202
+ "special": true
203
+ },
204
+ "80": {
205
+ "content": "<extra_id_20>",
206
+ "lstrip": false,
207
+ "normalized": false,
208
+ "rstrip": false,
209
+ "single_word": false,
210
+ "special": true
211
+ },
212
+ "81": {
213
+ "content": "<extra_id_21>",
214
+ "lstrip": false,
215
+ "normalized": false,
216
+ "rstrip": false,
217
+ "single_word": false,
218
+ "special": true
219
+ },
220
+ "82": {
221
+ "content": "<extra_id_22>",
222
+ "lstrip": false,
223
+ "normalized": false,
224
+ "rstrip": false,
225
+ "single_word": false,
226
+ "special": true
227
+ },
228
+ "83": {
229
+ "content": "<extra_id_23>",
230
+ "lstrip": false,
231
+ "normalized": false,
232
+ "rstrip": false,
233
+ "single_word": false,
234
+ "special": true
235
+ },
236
+ "84": {
237
+ "content": "<extra_id_24>",
238
+ "lstrip": false,
239
+ "normalized": false,
240
+ "rstrip": false,
241
+ "single_word": false,
242
+ "special": true
243
+ },
244
+ "85": {
245
+ "content": "<extra_id_25>",
246
+ "lstrip": false,
247
+ "normalized": false,
248
+ "rstrip": false,
249
+ "single_word": false,
250
+ "special": true
251
+ },
252
+ "86": {
253
+ "content": "<extra_id_26>",
254
+ "lstrip": false,
255
+ "normalized": false,
256
+ "rstrip": false,
257
+ "single_word": false,
258
+ "special": true
259
+ },
260
+ "87": {
261
+ "content": "<extra_id_27>",
262
+ "lstrip": false,
263
+ "normalized": false,
264
+ "rstrip": false,
265
+ "single_word": false,
266
+ "special": true
267
+ },
268
+ "88": {
269
+ "content": "<extra_id_28>",
270
+ "lstrip": false,
271
+ "normalized": false,
272
+ "rstrip": false,
273
+ "single_word": false,
274
+ "special": true
275
+ },
276
+ "89": {
277
+ "content": "<extra_id_29>",
278
+ "lstrip": false,
279
+ "normalized": false,
280
+ "rstrip": false,
281
+ "single_word": false,
282
+ "special": true
283
+ },
284
+ "90": {
285
+ "content": "<extra_id_30>",
286
+ "lstrip": false,
287
+ "normalized": false,
288
+ "rstrip": false,
289
+ "single_word": false,
290
+ "special": true
291
+ },
292
+ "91": {
293
+ "content": "<extra_id_31>",
294
+ "lstrip": false,
295
+ "normalized": false,
296
+ "rstrip": false,
297
+ "single_word": false,
298
+ "special": true
299
+ },
300
+ "92": {
301
+ "content": "<extra_id_32>",
302
+ "lstrip": false,
303
+ "normalized": false,
304
+ "rstrip": false,
305
+ "single_word": false,
306
+ "special": true
307
+ },
308
+ "93": {
309
+ "content": "<extra_id_33>",
310
+ "lstrip": false,
311
+ "normalized": false,
312
+ "rstrip": false,
313
+ "single_word": false,
314
+ "special": true
315
+ },
316
+ "94": {
317
+ "content": "<extra_id_34>",
318
+ "lstrip": false,
319
+ "normalized": false,
320
+ "rstrip": false,
321
+ "single_word": false,
322
+ "special": true
323
+ },
324
+ "95": {
325
+ "content": "<extra_id_35>",
326
+ "lstrip": false,
327
+ "normalized": false,
328
+ "rstrip": false,
329
+ "single_word": false,
330
+ "special": true
331
+ },
332
+ "96": {
333
+ "content": "<extra_id_36>",
334
+ "lstrip": false,
335
+ "normalized": false,
336
+ "rstrip": false,
337
+ "single_word": false,
338
+ "special": true
339
+ },
340
+ "97": {
341
+ "content": "<extra_id_37>",
342
+ "lstrip": false,
343
+ "normalized": false,
344
+ "rstrip": false,
345
+ "single_word": false,
346
+ "special": true
347
+ },
348
+ "98": {
349
+ "content": "<extra_id_38>",
350
+ "lstrip": false,
351
+ "normalized": false,
352
+ "rstrip": false,
353
+ "single_word": false,
354
+ "special": true
355
+ },
356
+ "99": {
357
+ "content": "<extra_id_39>",
358
+ "lstrip": false,
359
+ "normalized": false,
360
+ "rstrip": false,
361
+ "single_word": false,
362
+ "special": true
363
+ },
364
+ "100": {
365
+ "content": "<extra_id_40>",
366
+ "lstrip": false,
367
+ "normalized": false,
368
+ "rstrip": false,
369
+ "single_word": false,
370
+ "special": true
371
+ },
372
+ "101": {
373
+ "content": "<extra_id_41>",
374
+ "lstrip": false,
375
+ "normalized": false,
376
+ "rstrip": false,
377
+ "single_word": false,
378
+ "special": true
379
+ },
380
+ "102": {
381
+ "content": "<extra_id_42>",
382
+ "lstrip": false,
383
+ "normalized": false,
384
+ "rstrip": false,
385
+ "single_word": false,
386
+ "special": true
387
+ },
388
+ "103": {
389
+ "content": "<extra_id_43>",
390
+ "lstrip": false,
391
+ "normalized": false,
392
+ "rstrip": false,
393
+ "single_word": false,
394
+ "special": true
395
+ },
396
+ "104": {
397
+ "content": "<extra_id_44>",
398
+ "lstrip": false,
399
+ "normalized": false,
400
+ "rstrip": false,
401
+ "single_word": false,
402
+ "special": true
403
+ },
404
+ "105": {
405
+ "content": "<extra_id_45>",
406
+ "lstrip": false,
407
+ "normalized": false,
408
+ "rstrip": false,
409
+ "single_word": false,
410
+ "special": true
411
+ },
412
+ "106": {
413
+ "content": "<extra_id_46>",
414
+ "lstrip": false,
415
+ "normalized": false,
416
+ "rstrip": false,
417
+ "single_word": false,
418
+ "special": true
419
+ },
420
+ "107": {
421
+ "content": "<extra_id_47>",
422
+ "lstrip": false,
423
+ "normalized": false,
424
+ "rstrip": false,
425
+ "single_word": false,
426
+ "special": true
427
+ },
428
+ "108": {
429
+ "content": "<extra_id_48>",
430
+ "lstrip": false,
431
+ "normalized": false,
432
+ "rstrip": false,
433
+ "single_word": false,
434
+ "special": true
435
+ },
436
+ "109": {
437
+ "content": "<extra_id_49>",
438
+ "lstrip": false,
439
+ "normalized": false,
440
+ "rstrip": false,
441
+ "single_word": false,
442
+ "special": true
443
+ },
444
+ "110": {
445
+ "content": "<extra_id_50>",
446
+ "lstrip": false,
447
+ "normalized": false,
448
+ "rstrip": false,
449
+ "single_word": false,
450
+ "special": true
451
+ },
452
+ "111": {
453
+ "content": "<extra_id_51>",
454
+ "lstrip": false,
455
+ "normalized": false,
456
+ "rstrip": false,
457
+ "single_word": false,
458
+ "special": true
459
+ },
460
+ "112": {
461
+ "content": "<extra_id_52>",
462
+ "lstrip": false,
463
+ "normalized": false,
464
+ "rstrip": false,
465
+ "single_word": false,
466
+ "special": true
467
+ },
468
+ "113": {
469
+ "content": "<extra_id_53>",
470
+ "lstrip": false,
471
+ "normalized": false,
472
+ "rstrip": false,
473
+ "single_word": false,
474
+ "special": true
475
+ },
476
+ "114": {
477
+ "content": "<extra_id_54>",
478
+ "lstrip": false,
479
+ "normalized": false,
480
+ "rstrip": false,
481
+ "single_word": false,
482
+ "special": true
483
+ },
484
+ "115": {
485
+ "content": "<extra_id_55>",
486
+ "lstrip": false,
487
+ "normalized": false,
488
+ "rstrip": false,
489
+ "single_word": false,
490
+ "special": true
491
+ },
492
+ "116": {
493
+ "content": "<extra_id_56>",
494
+ "lstrip": false,
495
+ "normalized": false,
496
+ "rstrip": false,
497
+ "single_word": false,
498
+ "special": true
499
+ },
500
+ "117": {
501
+ "content": "<extra_id_57>",
502
+ "lstrip": false,
503
+ "normalized": false,
504
+ "rstrip": false,
505
+ "single_word": false,
506
+ "special": true
507
+ },
508
+ "118": {
509
+ "content": "<extra_id_58>",
510
+ "lstrip": false,
511
+ "normalized": false,
512
+ "rstrip": false,
513
+ "single_word": false,
514
+ "special": true
515
+ },
516
+ "119": {
517
+ "content": "<extra_id_59>",
518
+ "lstrip": false,
519
+ "normalized": false,
520
+ "rstrip": false,
521
+ "single_word": false,
522
+ "special": true
523
+ },
524
+ "120": {
525
+ "content": "<extra_id_60>",
526
+ "lstrip": false,
527
+ "normalized": false,
528
+ "rstrip": false,
529
+ "single_word": false,
530
+ "special": true
531
+ },
532
+ "121": {
533
+ "content": "<extra_id_61>",
534
+ "lstrip": false,
535
+ "normalized": false,
536
+ "rstrip": false,
537
+ "single_word": false,
538
+ "special": true
539
+ },
540
+ "122": {
541
+ "content": "<extra_id_62>",
542
+ "lstrip": false,
543
+ "normalized": false,
544
+ "rstrip": false,
545
+ "single_word": false,
546
+ "special": true
547
+ },
548
+ "123": {
549
+ "content": "<extra_id_63>",
550
+ "lstrip": false,
551
+ "normalized": false,
552
+ "rstrip": false,
553
+ "single_word": false,
554
+ "special": true
555
+ },
556
+ "124": {
557
+ "content": "<extra_id_64>",
558
+ "lstrip": false,
559
+ "normalized": false,
560
+ "rstrip": false,
561
+ "single_word": false,
562
+ "special": true
563
+ },
564
+ "125": {
565
+ "content": "<extra_id_65>",
566
+ "lstrip": false,
567
+ "normalized": false,
568
+ "rstrip": false,
569
+ "single_word": false,
570
+ "special": true
571
+ },
572
+ "126": {
573
+ "content": "<extra_id_66>",
574
+ "lstrip": false,
575
+ "normalized": false,
576
+ "rstrip": false,
577
+ "single_word": false,
578
+ "special": true
579
+ },
580
+ "127": {
581
+ "content": "<extra_id_67>",
582
+ "lstrip": false,
583
+ "normalized": false,
584
+ "rstrip": false,
585
+ "single_word": false,
586
+ "special": true
587
+ },
588
+ "128": {
589
+ "content": "<extra_id_68>",
590
+ "lstrip": false,
591
+ "normalized": false,
592
+ "rstrip": false,
593
+ "single_word": false,
594
+ "special": true
595
+ },
596
+ "129": {
597
+ "content": "<extra_id_69>",
598
+ "lstrip": false,
599
+ "normalized": false,
600
+ "rstrip": false,
601
+ "single_word": false,
602
+ "special": true
603
+ },
604
+ "130": {
605
+ "content": "<extra_id_70>",
606
+ "lstrip": false,
607
+ "normalized": false,
608
+ "rstrip": false,
609
+ "single_word": false,
610
+ "special": true
611
+ },
612
+ "131": {
613
+ "content": "<extra_id_71>",
614
+ "lstrip": false,
615
+ "normalized": false,
616
+ "rstrip": false,
617
+ "single_word": false,
618
+ "special": true
619
+ },
620
+ "132": {
621
+ "content": "<extra_id_72>",
622
+ "lstrip": false,
623
+ "normalized": false,
624
+ "rstrip": false,
625
+ "single_word": false,
626
+ "special": true
627
+ },
628
+ "133": {
629
+ "content": "<extra_id_73>",
630
+ "lstrip": false,
631
+ "normalized": false,
632
+ "rstrip": false,
633
+ "single_word": false,
634
+ "special": true
635
+ },
636
+ "134": {
637
+ "content": "<extra_id_74>",
638
+ "lstrip": false,
639
+ "normalized": false,
640
+ "rstrip": false,
641
+ "single_word": false,
642
+ "special": true
643
+ },
644
+ "135": {
645
+ "content": "<extra_id_75>",
646
+ "lstrip": false,
647
+ "normalized": false,
648
+ "rstrip": false,
649
+ "single_word": false,
650
+ "special": true
651
+ },
652
+ "136": {
653
+ "content": "<extra_id_76>",
654
+ "lstrip": false,
655
+ "normalized": false,
656
+ "rstrip": false,
657
+ "single_word": false,
658
+ "special": true
659
+ },
660
+ "137": {
661
+ "content": "<extra_id_77>",
662
+ "lstrip": false,
663
+ "normalized": false,
664
+ "rstrip": false,
665
+ "single_word": false,
666
+ "special": true
667
+ },
668
+ "138": {
669
+ "content": "<extra_id_78>",
670
+ "lstrip": false,
671
+ "normalized": false,
672
+ "rstrip": false,
673
+ "single_word": false,
674
+ "special": true
675
+ },
676
+ "139": {
677
+ "content": "<extra_id_79>",
678
+ "lstrip": false,
679
+ "normalized": false,
680
+ "rstrip": false,
681
+ "single_word": false,
682
+ "special": true
683
+ },
684
+ "140": {
685
+ "content": "<extra_id_80>",
686
+ "lstrip": false,
687
+ "normalized": false,
688
+ "rstrip": false,
689
+ "single_word": false,
690
+ "special": true
691
+ },
692
+ "141": {
693
+ "content": "<extra_id_81>",
694
+ "lstrip": false,
695
+ "normalized": false,
696
+ "rstrip": false,
697
+ "single_word": false,
698
+ "special": true
699
+ },
700
+ "142": {
701
+ "content": "<extra_id_82>",
702
+ "lstrip": false,
703
+ "normalized": false,
704
+ "rstrip": false,
705
+ "single_word": false,
706
+ "special": true
707
+ },
708
+ "143": {
709
+ "content": "<extra_id_83>",
710
+ "lstrip": false,
711
+ "normalized": false,
712
+ "rstrip": false,
713
+ "single_word": false,
714
+ "special": true
715
+ },
716
+ "144": {
717
+ "content": "<extra_id_84>",
718
+ "lstrip": false,
719
+ "normalized": false,
720
+ "rstrip": false,
721
+ "single_word": false,
722
+ "special": true
723
+ },
724
+ "145": {
725
+ "content": "<extra_id_85>",
726
+ "lstrip": false,
727
+ "normalized": false,
728
+ "rstrip": false,
729
+ "single_word": false,
730
+ "special": true
731
+ },
732
+ "146": {
733
+ "content": "<extra_id_86>",
734
+ "lstrip": false,
735
+ "normalized": false,
736
+ "rstrip": false,
737
+ "single_word": false,
738
+ "special": true
739
+ },
740
+ "147": {
741
+ "content": "<extra_id_87>",
742
+ "lstrip": false,
743
+ "normalized": false,
744
+ "rstrip": false,
745
+ "single_word": false,
746
+ "special": true
747
+ },
748
+ "148": {
749
+ "content": "<extra_id_88>",
750
+ "lstrip": false,
751
+ "normalized": false,
752
+ "rstrip": false,
753
+ "single_word": false,
754
+ "special": true
755
+ },
756
+ "149": {
757
+ "content": "<extra_id_89>",
758
+ "lstrip": false,
759
+ "normalized": false,
760
+ "rstrip": false,
761
+ "single_word": false,
762
+ "special": true
763
+ },
764
+ "150": {
765
+ "content": "<extra_id_90>",
766
+ "lstrip": false,
767
+ "normalized": false,
768
+ "rstrip": false,
769
+ "single_word": false,
770
+ "special": true
771
+ },
772
+ "151": {
773
+ "content": "<extra_id_91>",
774
+ "lstrip": false,
775
+ "normalized": false,
776
+ "rstrip": false,
777
+ "single_word": false,
778
+ "special": true
779
+ },
780
+ "152": {
781
+ "content": "<extra_id_92>",
782
+ "lstrip": false,
783
+ "normalized": false,
784
+ "rstrip": false,
785
+ "single_word": false,
786
+ "special": true
787
+ },
788
+ "153": {
789
+ "content": "<extra_id_93>",
790
+ "lstrip": false,
791
+ "normalized": false,
792
+ "rstrip": false,
793
+ "single_word": false,
794
+ "special": true
795
+ },
796
+ "154": {
797
+ "content": "<extra_id_94>",
798
+ "lstrip": false,
799
+ "normalized": false,
800
+ "rstrip": false,
801
+ "single_word": false,
802
+ "special": true
803
+ },
804
+ "155": {
805
+ "content": "<extra_id_95>",
806
+ "lstrip": false,
807
+ "normalized": false,
808
+ "rstrip": false,
809
+ "single_word": false,
810
+ "special": true
811
+ },
812
+ "156": {
813
+ "content": "<extra_id_96>",
814
+ "lstrip": false,
815
+ "normalized": false,
816
+ "rstrip": false,
817
+ "single_word": false,
818
+ "special": true
819
+ },
820
+ "157": {
821
+ "content": "<extra_id_97>",
822
+ "lstrip": false,
823
+ "normalized": false,
824
+ "rstrip": false,
825
+ "single_word": false,
826
+ "special": true
827
+ },
828
+ "158": {
829
+ "content": "<extra_id_98>",
830
+ "lstrip": false,
831
+ "normalized": false,
832
+ "rstrip": false,
833
+ "single_word": false,
834
+ "special": true
835
+ },
836
+ "159": {
837
+ "content": "<extra_id_99>",
838
+ "lstrip": false,
839
+ "normalized": false,
840
+ "rstrip": false,
841
+ "single_word": false,
842
+ "special": true
843
+ }
844
+ },
845
+ "additional_special_tokens": [
846
+ "<extra_id_0>",
847
+ "<extra_id_1>",
848
+ "<extra_id_2>",
849
+ "<extra_id_3>",
850
+ "<extra_id_4>",
851
+ "<extra_id_5>",
852
+ "<extra_id_6>",
853
+ "<extra_id_7>",
854
+ "<extra_id_8>",
855
+ "<extra_id_9>",
856
+ "<extra_id_10>",
857
+ "<extra_id_11>",
858
+ "<extra_id_12>",
859
+ "<extra_id_13>",
860
+ "<extra_id_14>",
861
+ "<extra_id_15>",
862
+ "<extra_id_16>",
863
+ "<extra_id_17>",
864
+ "<extra_id_18>",
865
+ "<extra_id_19>",
866
+ "<extra_id_20>",
867
+ "<extra_id_21>",
868
+ "<extra_id_22>",
869
+ "<extra_id_23>",
870
+ "<extra_id_24>",
871
+ "<extra_id_25>",
872
+ "<extra_id_26>",
873
+ "<extra_id_27>",
874
+ "<extra_id_28>",
875
+ "<extra_id_29>",
876
+ "<extra_id_30>",
877
+ "<extra_id_31>",
878
+ "<extra_id_32>",
879
+ "<extra_id_33>",
880
+ "<extra_id_34>",
881
+ "<extra_id_35>",
882
+ "<extra_id_36>",
883
+ "<extra_id_37>",
884
+ "<extra_id_38>",
885
+ "<extra_id_39>",
886
+ "<extra_id_40>",
887
+ "<extra_id_41>",
888
+ "<extra_id_42>",
889
+ "<extra_id_43>",
890
+ "<extra_id_44>",
891
+ "<extra_id_45>",
892
+ "<extra_id_46>",
893
+ "<extra_id_47>",
894
+ "<extra_id_48>",
895
+ "<extra_id_49>",
896
+ "<extra_id_50>",
897
+ "<extra_id_51>",
898
+ "<extra_id_52>",
899
+ "<extra_id_53>",
900
+ "<extra_id_54>",
901
+ "<extra_id_55>",
902
+ "<extra_id_56>",
903
+ "<extra_id_57>",
904
+ "<extra_id_58>",
905
+ "<extra_id_59>",
906
+ "<extra_id_60>",
907
+ "<extra_id_61>",
908
+ "<extra_id_62>",
909
+ "<extra_id_63>",
910
+ "<extra_id_64>",
911
+ "<extra_id_65>",
912
+ "<extra_id_66>",
913
+ "<extra_id_67>",
914
+ "<extra_id_68>",
915
+ "<extra_id_69>",
916
+ "<extra_id_70>",
917
+ "<extra_id_71>",
918
+ "<extra_id_72>",
919
+ "<extra_id_73>",
920
+ "<extra_id_74>",
921
+ "<extra_id_75>",
922
+ "<extra_id_76>",
923
+ "<extra_id_77>",
924
+ "<extra_id_78>",
925
+ "<extra_id_79>",
926
+ "<extra_id_80>",
927
+ "<extra_id_81>",
928
+ "<extra_id_82>",
929
+ "<extra_id_83>",
930
+ "<extra_id_84>",
931
+ "<extra_id_85>",
932
+ "<extra_id_86>",
933
+ "<extra_id_87>",
934
+ "<extra_id_88>",
935
+ "<extra_id_89>",
936
+ "<extra_id_90>",
937
+ "<extra_id_91>",
938
+ "<extra_id_92>",
939
+ "<extra_id_93>",
940
+ "<extra_id_94>",
941
+ "<extra_id_95>",
942
+ "<extra_id_96>",
943
+ "<extra_id_97>",
944
+ "<extra_id_98>",
945
+ "<extra_id_99>"
946
+ ],
947
+ "clean_up_tokenization_spaces": false,
948
+ "cls_token": "<s>",
949
+ "eos_token": "</s>",
950
+ "extra_ids": 100,
951
+ "extra_special_tokens": {},
952
+ "mask_token": "<mask>",
953
+ "model_max_length": 1000000000000000019884624838656,
954
+ "pad_token": "<pad>",
955
+ "sep_token": "</s>",
956
+ "tokenizer_class": "T5TokenizerFast",
957
+ "unk_token": "<unk>"
958
+ }
minicoderx-model/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd544df3e986cab3c0663ced7b6b3ca6903b0db2be881d69b5180d0077410a13
3
+ size 5304
minicoderx-tokenizer/special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": "<s>",
3
+ "mask_token": "<mask>",
4
+ "pad_token": "<pad>",
5
+ "sep_token": "</s>",
6
+ "unk_token": "<unk>"
7
+ }
minicoderx-tokenizer/tokenizer.json ADDED
@@ -0,0 +1,349 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
5
+ "added_tokens": [
6
+ {
7
+ "id": 0,
8
+ "content": "<pad>",
9
+ "single_word": false,
10
+ "lstrip": false,
11
+ "rstrip": false,
12
+ "normalized": false,
13
+ "special": true
14
+ },
15
+ {
16
+ "id": 1,
17
+ "content": "<s>",
18
+ "single_word": false,
19
+ "lstrip": false,
20
+ "rstrip": false,
21
+ "normalized": false,
22
+ "special": true
23
+ },
24
+ {
25
+ "id": 2,
26
+ "content": "</s>",
27
+ "single_word": false,
28
+ "lstrip": false,
29
+ "rstrip": false,
30
+ "normalized": false,
31
+ "special": true
32
+ },
33
+ {
34
+ "id": 3,
35
+ "content": "<unk>",
36
+ "single_word": false,
37
+ "lstrip": false,
38
+ "rstrip": false,
39
+ "normalized": false,
40
+ "special": true
41
+ },
42
+ {
43
+ "id": 4,
44
+ "content": "<mask>",
45
+ "single_word": false,
46
+ "lstrip": false,
47
+ "rstrip": false,
48
+ "normalized": false,
49
+ "special": true
50
+ }
51
+ ],
52
+ "normalizer": {
53
+ "type": "Sequence",
54
+ "normalizers": [
55
+ {
56
+ "type": "NFD"
57
+ },
58
+ {
59
+ "type": "Lowercase"
60
+ },
61
+ {
62
+ "type": "StripAccents"
63
+ }
64
+ ]
65
+ },
66
+ "pre_tokenizer": {
67
+ "type": "Whitespace"
68
+ },
69
+ "post_processor": {
70
+ "type": "TemplateProcessing",
71
+ "single": [
72
+ {
73
+ "SpecialToken": {
74
+ "id": "<s>",
75
+ "type_id": 0
76
+ }
77
+ },
78
+ {
79
+ "Sequence": {
80
+ "id": "A",
81
+ "type_id": 0
82
+ }
83
+ },
84
+ {
85
+ "SpecialToken": {
86
+ "id": "</s>",
87
+ "type_id": 0
88
+ }
89
+ }
90
+ ],
91
+ "pair": [
92
+ {
93
+ "SpecialToken": {
94
+ "id": "<s>",
95
+ "type_id": 0
96
+ }
97
+ },
98
+ {
99
+ "Sequence": {
100
+ "id": "A",
101
+ "type_id": 0
102
+ }
103
+ },
104
+ {
105
+ "SpecialToken": {
106
+ "id": "</s>",
107
+ "type_id": 0
108
+ }
109
+ },
110
+ {
111
+ "SpecialToken": {
112
+ "id": "</s>",
113
+ "type_id": 0
114
+ }
115
+ },
116
+ {
117
+ "Sequence": {
118
+ "id": "B",
119
+ "type_id": 0
120
+ }
121
+ },
122
+ {
123
+ "SpecialToken": {
124
+ "id": "</s>",
125
+ "type_id": 0
126
+ }
127
+ }
128
+ ],
129
+ "special_tokens": {
130
+ "</s>": {
131
+ "id": "</s>",
132
+ "ids": [
133
+ 2
134
+ ],
135
+ "tokens": [
136
+ "</s>"
137
+ ]
138
+ },
139
+ "<s>": {
140
+ "id": "<s>",
141
+ "ids": [
142
+ 1
143
+ ],
144
+ "tokens": [
145
+ "<s>"
146
+ ]
147
+ }
148
+ }
149
+ },
150
+ "decoder": null,
151
+ "model": {
152
+ "type": "BPE",
153
+ "dropout": null,
154
+ "unk_token": null,
155
+ "continuing_subword_prefix": null,
156
+ "end_of_word_suffix": null,
157
+ "fuse_unk": false,
158
+ "byte_fallback": false,
159
+ "ignore_merges": false,
160
+ "vocab": {
161
+ "<pad>": 0,
162
+ "<s>": 1,
163
+ "</s>": 2,
164
+ "<unk>": 3,
165
+ "<mask>": 4,
166
+ "!": 5,
167
+ "\"": 6,
168
+ "(": 7,
169
+ ")": 8,
170
+ "+": 9,
171
+ ",": 10,
172
+ ":": 11,
173
+ "a": 12,
174
+ "b": 13,
175
+ "c": 14,
176
+ "d": 15,
177
+ "e": 16,
178
+ "f": 17,
179
+ "h": 18,
180
+ "i": 19,
181
+ "l": 20,
182
+ "n": 21,
183
+ "o": 22,
184
+ "p": 23,
185
+ "r": 24,
186
+ "s": 25,
187
+ "t": 26,
188
+ "u": 27,
189
+ "w": 28,
190
+ "as": 29,
191
+ "el": 30,
192
+ "hel": 31,
193
+ "ld": 32,
194
+ "lo": 33,
195
+ "or": 34,
196
+ "wor": 35,
197
+ "ass": 36,
198
+ "hello": 37,
199
+ "world": 38,
200
+ "!\"": 39,
201
+ "(\"": 40,
202
+ "):": 41,
203
+ "ad": 42,
204
+ "cl": 43,
205
+ "de": 44,
206
+ "et": 45,
207
+ "in": 46,
208
+ "pr": 47,
209
+ "pass": 48,
210
+ "rn": 49,
211
+ "ret": 50,
212
+ "urn": 51,
213
+ "helloworld": 52,
214
+ "!\")": 53,
215
+ "add": 54,
216
+ "class": 55,
217
+ "def": 56,
218
+ "int": 57,
219
+ "print": 58,
220
+ "return": 59
221
+ },
222
+ "merges": [
223
+ [
224
+ "a",
225
+ "s"
226
+ ],
227
+ [
228
+ "e",
229
+ "l"
230
+ ],
231
+ [
232
+ "h",
233
+ "el"
234
+ ],
235
+ [
236
+ "l",
237
+ "d"
238
+ ],
239
+ [
240
+ "l",
241
+ "o"
242
+ ],
243
+ [
244
+ "o",
245
+ "r"
246
+ ],
247
+ [
248
+ "w",
249
+ "or"
250
+ ],
251
+ [
252
+ "as",
253
+ "s"
254
+ ],
255
+ [
256
+ "hel",
257
+ "lo"
258
+ ],
259
+ [
260
+ "wor",
261
+ "ld"
262
+ ],
263
+ [
264
+ "!",
265
+ "\""
266
+ ],
267
+ [
268
+ "(",
269
+ "\""
270
+ ],
271
+ [
272
+ ")",
273
+ ":"
274
+ ],
275
+ [
276
+ "a",
277
+ "d"
278
+ ],
279
+ [
280
+ "c",
281
+ "l"
282
+ ],
283
+ [
284
+ "d",
285
+ "e"
286
+ ],
287
+ [
288
+ "e",
289
+ "t"
290
+ ],
291
+ [
292
+ "i",
293
+ "n"
294
+ ],
295
+ [
296
+ "p",
297
+ "r"
298
+ ],
299
+ [
300
+ "p",
301
+ "ass"
302
+ ],
303
+ [
304
+ "r",
305
+ "n"
306
+ ],
307
+ [
308
+ "r",
309
+ "et"
310
+ ],
311
+ [
312
+ "u",
313
+ "rn"
314
+ ],
315
+ [
316
+ "hello",
317
+ "world"
318
+ ],
319
+ [
320
+ "!\"",
321
+ ")"
322
+ ],
323
+ [
324
+ "ad",
325
+ "d"
326
+ ],
327
+ [
328
+ "cl",
329
+ "ass"
330
+ ],
331
+ [
332
+ "de",
333
+ "f"
334
+ ],
335
+ [
336
+ "in",
337
+ "t"
338
+ ],
339
+ [
340
+ "pr",
341
+ "int"
342
+ ],
343
+ [
344
+ "ret",
345
+ "urn"
346
+ ]
347
+ ]
348
+ }
349
+ }
minicoderx-tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<pad>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<s>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "</s>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<unk>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "4": {
36
+ "content": "<mask>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "clean_up_tokenization_spaces": false,
45
+ "cls_token": "<s>",
46
+ "extra_special_tokens": {},
47
+ "mask_token": "<mask>",
48
+ "model_max_length": 1000000000000000019884624838656,
49
+ "pad_token": "<pad>",
50
+ "sep_token": "</s>",
51
+ "tokenizer_class": "PreTrainedTokenizer",
52
+ "unk_token": "<unk>"
53
+ }
papers/ARCHCODE: Incorporating Software Requirements in Code Generation with Large Language Models.pdf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9bdccbd0a0269d01381b46634bed78101c788a5545bcaf32e6d4f712654f908e
3
+ size 557731
papers/AST-T5: Structure-Aware Pretraining for Code Generation and Understanding.pdf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:44dceedeadc981bc793a882a8bf7b76600ea23d09c923be46295257c296b709e
3
+ size 271471
papers/CODE4STRUCT: Code Generation for Few-Shot Event Structure Prediction.pdf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:82d1cec3c94263d77c67845e21be34ae978274eae40e8991416fdac0ff7fed65
3
+ size 675766
papers/CoTexT: Multi-task Learning with Code-Text Transformer.pdf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:703e40ceeb43158a1cbaf5afb99f575bf6defd776381249f1f6bf3686206b1fc
3
+ size 163833
papers/CodeBERT: A Pre-Trained Model for Programming and Natural Languages.pdf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4208bcf8747cf7e8dfa14ed09e856b7fa0a20f45613692c11ce458791ae31b84
3
+ size 465537
papers/CodeGeeX: A Pre-Trained Model for Code Generation with Multilingual Benchmarking on HumanEval-X.pdf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5bb5681999075d5a922871da8308b5e68ccf7997cbdfa3a71f3a3291079e88d0
3
+ size 3873166
papers/CodeGemma Open Code Models Based on Gemma.pdf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:63b6582424f1feff5ca7fa03f511a4ec6b8b692a4550a50f8f62058e99de585d
3
+ size 222988
papers/CodeT5+: Open Code Large Language Models for Code Understanding and Generation.pdf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b0584d3de3784f2f8d6e49592a8fb65b0d94455cf98156adca1d28262c91d54
3
+ size 695855
papers/Evaluating the Code Quality of AI-Assisted Code Generation Tools: An Empirical Study on GitHub Copilot, Amazon CodeWhisperer, and ChatGPT.pdf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:13b0d4ea27779a59d16f6cdbd6b6b8abcc137d338cc6cadb2c60c9bca7c61d5c
3
+ size 1214009
papers/GRAPH CODE BERT: PRE-TRAINING CODE REPRESEN-TATIONS WITH DATA F LOW.pdf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:574a41f6c8d94dfec9202424319654aad7f9014eebb32d7037c2e040894b764c
3
+ size 685074
papers/StructCoder_ Structure-Aware Transformer for Code Generation - 2206.05239v3.pdf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2834776aece3516b287b4667f725d1e8aec968283abca06b27b098fc78f0a839
3
+ size 733827
papers/TreeGen: A Tree-Based Transformer Architecture for Code Generation.pdf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e18bc60048d5baf97778693146d62e5d32ff3959d5c76ec159ae126b9496e8ab
3
+ size 379513
project.ipynb ADDED
@@ -0,0 +1,482 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "id": "3e7c79bb",
6
+ "metadata": {},
7
+ "source": [
8
+ "# MiniCoderX Project - Full Pipeline Notebook"
9
+ ]
10
+ },
11
+ {
12
+ "cell_type": "markdown",
13
+ "id": "82aa402a",
14
+ "metadata": {},
15
+ "source": [
16
+ "# Step 0: Environment Setup"
17
+ ]
18
+ },
19
+ {
20
+ "cell_type": "code",
21
+ "execution_count": null,
22
+ "id": "fe661c57",
23
+ "metadata": {},
24
+ "outputs": [],
25
+ "source": [
26
+ "pip install -q tokenizers transformers datasets sentencepiece langchain_community ollama networkx evaluate rouge_score matplotlib seaborn lark fastapi uvicorn"
27
+ ]
28
+ },
29
+ {
30
+ "cell_type": "markdown",
31
+ "id": "7313bed0",
32
+ "metadata": {},
33
+ "source": [
34
+ "# Step 1: Import and Load Model"
35
+ ]
36
+ },
37
+ {
38
+ "cell_type": "code",
39
+ "execution_count": null,
40
+ "id": "2e28b42b",
41
+ "metadata": {},
42
+ "outputs": [],
43
+ "source": [
44
+ "from tokenizers import Tokenizer, models, trainers, pre_tokenizers\n",
45
+ "from tokenizers.normalizers import Sequence, Lowercase, NFD, StripAccents\n",
46
+ "from tokenizers.pre_tokenizers import Whitespace\n",
47
+ "from tokenizers.processors import TemplateProcessing\n",
48
+ "from transformers import PreTrainedTokenizerFast\n",
49
+ "import os\n",
50
+ "\n",
51
+ "tokenizer = Tokenizer(models.BPE())\n",
52
+ "tokenizer.normalizer = Sequence([NFD(), Lowercase(), StripAccents()])\n",
53
+ "tokenizer.pre_tokenizer = Whitespace()\n",
54
+ "\n",
55
+ "trainer = trainers.BpeTrainer(\n",
56
+ " vocab_size=32000,\n",
57
+ " special_tokens=[\"<pad>\", \"<s>\", \"</s>\", \"<unk>\", \"<mask>\"]\n",
58
+ ")\n",
59
+ "\n",
60
+ "data_path = \"data/code_corpus.txt\"\n",
61
+ "\n",
62
+ "if not os.path.exists(data_path):\n",
63
+ " raise FileNotFoundError(f\"Dataset not found at: {data_path}\")\n",
64
+ "else:\n",
65
+ " print(\"Dataset found:\", data_path)\n",
66
+ "\n",
67
+ "tokenizer.train([data_path], trainer)\n",
68
+ "\n",
69
+ "\n",
70
+ "tokenizer.post_processor = TemplateProcessing(\n",
71
+ " single=\"<s> $A </s>\",\n",
72
+ " pair=\"<s> $A </s> </s> $B </s>\",\n",
73
+ " special_tokens=[\n",
74
+ " (\"<s>\", tokenizer.token_to_id(\"<s>\")),\n",
75
+ " (\"</s>\", tokenizer.token_to_id(\"</s>\")),\n",
76
+ " ],\n",
77
+ ")\n",
78
+ "\n",
79
+ "tokenizer_path = \"minicoderx-tokenizer\"\n",
80
+ "os.makedirs(tokenizer_path, exist_ok=True)\n",
81
+ "tokenizer.save(f\"{tokenizer_path}/tokenizer.json\")\n",
82
+ "print(\"Tokenizer saved to:\", tokenizer_path)"
83
+ ]
84
+ },
85
+ {
86
+ "cell_type": "code",
87
+ "execution_count": null,
88
+ "id": "d1ab6421",
89
+ "metadata": {},
90
+ "outputs": [],
91
+ "source": [
92
+ "from transformers import PreTrainedTokenizerFast\n",
93
+ "\n",
94
+ "hf_tokenizer = PreTrainedTokenizerFast(\n",
95
+ " tokenizer_file=\"minicoderx-tokenizer/tokenizer.json\",\n",
96
+ " unk_token=\"<unk>\",\n",
97
+ " pad_token=\"<pad>\",\n",
98
+ " cls_token=\"<s>\",\n",
99
+ " sep_token=\"</s>\",\n",
100
+ " mask_token=\"<mask>\",\n",
101
+ ")\n",
102
+ "\n",
103
+ "hf_tokenizer.save_pretrained(\"minicoderx-tokenizer\")\n",
104
+ "print(\"HuggingFace tokenizer saved and ready.\")"
105
+ ]
106
+ },
107
+ {
108
+ "cell_type": "code",
109
+ "execution_count": null,
110
+ "id": "ba28e05f",
111
+ "metadata": {},
112
+ "outputs": [],
113
+ "source": [
114
+ "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n",
115
+ "\n",
116
+ "# Load your trained model and tokenizer\n",
117
+ "tokenizer = AutoTokenizer.from_pretrained(\"minicoderx-model\")\n",
118
+ "model = AutoModelForSeq2SeqLM.from_pretrained(\"minicoderx-model\")\n",
119
+ "\n",
120
+ "print(\"Model and tokenizer loaded.\")"
121
+ ]
122
+ },
123
+ {
124
+ "cell_type": "markdown",
125
+ "id": "852b82c3",
126
+ "metadata": {},
127
+ "source": [
128
+ "# Step 2: Inference - Code Generation"
129
+ ]
130
+ },
131
+ {
132
+ "cell_type": "code",
133
+ "execution_count": null,
134
+ "id": "6ab29f13",
135
+ "metadata": {},
136
+ "outputs": [],
137
+ "source": [
138
+ "input_text = \"Write a Python function to compute factorial\"\n",
139
+ "inputs = tokenizer(input_text, return_tensors=\"pt\")\n",
140
+ "outputs = model.generate(**inputs, max_length=128)\n",
141
+ "print(\"\\nGenerated Code:\\n\")\n",
142
+ "print(tokenizer.decode(outputs[0], skip_special_tokens=True))"
143
+ ]
144
+ },
145
+ {
146
+ "cell_type": "markdown",
147
+ "id": "e2e495b0",
148
+ "metadata": {},
149
+ "source": [
150
+ "# Step 3: Structure-Aware Encoding with AST"
151
+ ]
152
+ },
153
+ {
154
+ "cell_type": "code",
155
+ "execution_count": null,
156
+ "id": "c5337fe3",
157
+ "metadata": {},
158
+ "outputs": [],
159
+ "source": [
160
+ "import ast, networkx as nx, matplotlib.pyplot as plt, seaborn as sns\n",
161
+ "\n",
162
+ "def build_ast_graph_with_metadata(node, graph, parent=None):\n",
163
+ " node_id = str(id(node))\n",
164
+ " graph.add_node(node_id, label=type(node).__name__)\n",
165
+ " if parent:\n",
166
+ " graph.add_edge(parent, node_id)\n",
167
+ " for child in ast.iter_child_nodes(node):\n",
168
+ " build_ast_graph_with_metadata(child, graph, node_id)\n",
169
+ "\n",
170
+ "code_sample = \"\"\"\n",
171
+ "def add(a, b):\n",
172
+ " return a + b\n",
173
+ "\"\"\"\n",
174
+ "tree = ast.parse(code_sample)\n",
175
+ "G = nx.DiGraph()\n",
176
+ "build_ast_graph_with_metadata(tree, G)\n",
177
+ "pos = nx.spring_layout(G)\n",
178
+ "labels = nx.get_node_attributes(G, 'label')\n",
179
+ "nx.draw(G, pos, labels=labels, with_labels=True, node_size=1200, node_color='lightblue')\n",
180
+ "plt.title(\"AST Visualization\")\n",
181
+ "plt.show()"
182
+ ]
183
+ },
184
+ {
185
+ "cell_type": "markdown",
186
+ "id": "f732a8cf",
187
+ "metadata": {},
188
+ "source": [
189
+ "# Step 4: LangChain + Ollama Integration"
190
+ ]
191
+ },
192
+ {
193
+ "cell_type": "code",
194
+ "execution_count": null,
195
+ "id": "0b2c013c",
196
+ "metadata": {},
197
+ "outputs": [],
198
+ "source": [
199
+ "from langchain_community.llms import Ollama\n",
200
+ "from langchain.chains import LLMChain\n",
201
+ "from langchain.prompts import PromptTemplate\n",
202
+ "\n",
203
+ "llm = Ollama(model=\"minicoderx\")\n",
204
+ "prompt = PromptTemplate(input_variables=[\"instruction\"], template=\"Generate Python code for the task: {instruction}\")\n",
205
+ "chain = LLMChain(llm=llm, prompt=prompt)\n",
206
+ "print(\"\\nLangChain-Ollama Output:\")\n",
207
+ "print(chain.run(\"Create a function to reverse a string\"))"
208
+ ]
209
+ },
210
+ {
211
+ "cell_type": "markdown",
212
+ "id": "6ded4c5e",
213
+ "metadata": {},
214
+ "source": [
215
+ "# Step 5: Evaluation (MBPP)"
216
+ ]
217
+ },
218
+ {
219
+ "cell_type": "code",
220
+ "execution_count": null,
221
+ "id": "37f133a4",
222
+ "metadata": {},
223
+ "outputs": [],
224
+ "source": [
225
+ "from datasets import load_dataset\n",
226
+ "import evaluate\n",
227
+ "\n",
228
+ "dataset = load_dataset(\"mbpp\")\n",
229
+ "eval_bleu = evaluate.load(\"bleu\")\n",
230
+ "eval_rouge = evaluate.load(\"rouge\")\n",
231
+ "\n",
232
+ "sample = dataset['test'][0]\n",
233
+ "input_text = f\"Write a Python function: {sample['text']}\"\n",
234
+ "inputs = tokenizer(input_text, return_tensors=\"pt\")\n",
235
+ "output = model.generate(**inputs, max_length=128)\n",
236
+ "generated_code = tokenizer.decode(output[0], skip_special_tokens=True)\n",
237
+ "\n",
238
+ "print(\"\\nEvaluation Sample Output:\\n\", generated_code)\n",
239
+ "print(\"BLEU:\", eval_bleu.compute(predictions=[generated_code], references=[sample['code']]))\n",
240
+ "print(\"ROUGE:\", eval_rouge.compute(predictions=[generated_code], references=[sample['code']]))"
241
+ ]
242
+ },
243
+ {
244
+ "cell_type": "markdown",
245
+ "id": "2b00a47c",
246
+ "metadata": {},
247
+ "source": [
248
+ "# Step 6: Testing, Verification, and Unit Test Gen"
249
+ ]
250
+ },
251
+ {
252
+ "cell_type": "code",
253
+ "execution_count": null,
254
+ "id": "a9a8ef01",
255
+ "metadata": {},
256
+ "outputs": [],
257
+ "source": [
258
+ "import tempfile, subprocess\n",
259
+ "\n",
260
+ "def run_code(code, test_case):\n",
261
+ " with tempfile.NamedTemporaryFile(mode='w+', suffix='.py', delete=False) as tmp:\n",
262
+ " tmp.write(code + '\\n' + test_case)\n",
263
+ " tmp.flush()\n",
264
+ " result = subprocess.run(['python', tmp.name], capture_output=True, text=True)\n",
265
+ " print(\"Output:\\n\", result.stdout)\n",
266
+ " if result.stderr:\n",
267
+ " print(\"Errors:\\n\", result.stderr)\n",
268
+ "\n",
269
+ "test_case = \"print(factorial(5)) # Expected: 120\"\n",
270
+ "run_code(generated_code, test_case)\n",
271
+ "\n",
272
+ "unit_prompt = PromptTemplate(input_variables=[\"code\"], template=\"Write a unittest in Python for the following function:\\n\\n{code}\")\n",
273
+ "unit_chain = LLMChain(llm=llm, prompt=unit_prompt)\n",
274
+ "print(\"\\nGenerated Unit Test:\\n\", unit_chain.run(code=generated_code))"
275
+ ]
276
+ },
277
+ {
278
+ "cell_type": "markdown",
279
+ "id": "9b9fcc1e",
280
+ "metadata": {},
281
+ "source": [
282
+ "# Step 7: Safety and Grammar Constraints"
283
+ ]
284
+ },
285
+ {
286
+ "cell_type": "code",
287
+ "execution_count": null,
288
+ "id": "5e3dd5ee",
289
+ "metadata": {},
290
+ "outputs": [],
291
+ "source": [
292
+ "from lark import Lark, UnexpectedInput\n",
293
+ "\n",
294
+ "python_grammar = \"\"\"\n",
295
+ "start: stmt+\n",
296
+ "stmt: \"def\" NAME \"(\" [params] \")\" \":\" suite\n",
297
+ "params: NAME (\",\" NAME)*\n",
298
+ "suite: NEWLINE INDENT stmt+ DEDENT | simple_stmt\n",
299
+ "simple_stmt: NAME \"=\" expr NEWLINE\n",
300
+ "expr: atom | atom operator atom\n",
301
+ "atom: NAME | NUMBER\n",
302
+ "operator: \"+\" | \"-\" | \"*\" | \"/\"\n",
303
+ "%import common.CNAME -> NAME\n",
304
+ "%import common.NUMBER\n",
305
+ "%import common.NEWLINE\n",
306
+ "%import common.WS_INLINE\n",
307
+ "%import common.INDENT\n",
308
+ "%import common.DEDENT\n",
309
+ "%ignore WS_INLINE\n",
310
+ "\"\"\"\n",
311
+ "\n",
312
+ "parser = Lark(python_grammar, parser=\"lalr\")\n",
313
+ "\n",
314
+ "unsafe_keywords = [\"os.system\", \"subprocess\", \"eval\", \"exec\", \"open(\", \"import socket\"]\n",
315
+ "print(\"\\nSafety Check:\")\n",
316
+ "print(\"Unsafe pattern found\" if any(k in generated_code for k in unsafe_keywords) else \"Code is safe\")\n",
317
+ "\n",
318
+ "print(\"\\nGrammar Check:\")\n",
319
+ "try:\n",
320
+ " parser.parse(generated_code)\n",
321
+ " print(\"Code grammar is valid.\")\n",
322
+ "except UnexpectedInput as e:\n",
323
+ " print(\"Grammar error:\", e)"
324
+ ]
325
+ },
326
+ {
327
+ "cell_type": "markdown",
328
+ "id": "8b1e2b86",
329
+ "metadata": {},
330
+ "source": [
331
+ "# Step 8: Multi-Task Preprocessing (gen, sum, trans)"
332
+ ]
333
+ },
334
+ {
335
+ "cell_type": "code",
336
+ "execution_count": null,
337
+ "id": "09a12f1d",
338
+ "metadata": {},
339
+ "outputs": [],
340
+ "source": [
341
+ "def preprocess_multitask(example):\n",
342
+ " if example['task'] == 'gen':\n",
343
+ " input_text = f\"Write code: {example['text']}\"\n",
344
+ " output_text = example['code']\n",
345
+ " elif example['task'] == 'sum':\n",
346
+ " input_text = f\"Summarize this code: {example['code']}\"\n",
347
+ " output_text = example['text']\n",
348
+ " elif example['task'] == 'trans':\n",
349
+ " input_text = f\"Translate Java to Python: {example['java']}\"\n",
350
+ " output_text = example['python']\n",
351
+ " else:\n",
352
+ " input_text, output_text = example['text'], example['code']\n",
353
+ " model_input = tokenizer(input_text, max_length=128, truncation=True)\n",
354
+ " with tokenizer.as_target_tokenizer():\n",
355
+ " labels = tokenizer(output_text, max_length=128, truncation=True)\n",
356
+ " model_input['labels'] = labels['input_ids']\n",
357
+ " return model_input"
358
+ ]
359
+ },
360
+ {
361
+ "cell_type": "markdown",
362
+ "id": "6018db4c",
363
+ "metadata": {},
364
+ "source": [
365
+ "# Step 9: Fine-Tuning Setup"
366
+ ]
367
+ },
368
+ {
369
+ "cell_type": "code",
370
+ "execution_count": null,
371
+ "id": "c15ad38d",
372
+ "metadata": {},
373
+ "outputs": [],
374
+ "source": [
375
+ "from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments, DataCollatorForSeq2Seq\n",
376
+ "\n",
377
+ "train_dataset = dataset[\"train\"].map(preprocess_multitask, remove_columns=dataset[\"train\"].column_names)\n",
378
+ "val_dataset = dataset[\"validation\"].map(preprocess_multitask, remove_columns=dataset[\"validation\"].column_names)\n",
379
+ "\n",
380
+ "training_args = Seq2SeqTrainingArguments(\n",
381
+ " output_dir=\"./minicoderx-finetuned\",\n",
382
+ " evaluation_strategy=\"epoch\",\n",
383
+ " learning_rate=5e-5,\n",
384
+ " per_device_train_batch_size=8,\n",
385
+ " per_device_eval_batch_size=8,\n",
386
+ " weight_decay=0.01,\n",
387
+ " save_total_limit=2,\n",
388
+ " num_train_epochs=3,\n",
389
+ " predict_with_generate=True,\n",
390
+ " logging_dir=\"./logs\",\n",
391
+ " logging_steps=10,\n",
392
+ ")\n",
393
+ "\n",
394
+ "data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)\n",
395
+ "trainer = Seq2SeqTrainer(\n",
396
+ " model=model,\n",
397
+ " args=training_args,\n",
398
+ " train_dataset=train_dataset,\n",
399
+ " eval_dataset=val_dataset,\n",
400
+ " tokenizer=tokenizer,\n",
401
+ " data_collator=data_collator,\n",
402
+ ")"
403
+ ]
404
+ },
405
+ {
406
+ "cell_type": "code",
407
+ "execution_count": null,
408
+ "id": "28d3dcb1",
409
+ "metadata": {},
410
+ "outputs": [],
411
+ "source": [
412
+ "# Uncomment to run training\n",
413
+ "# trainer.train()\n",
414
+ "# trainer.save_model(\"./minicoderx-finetuned\")"
415
+ ]
416
+ },
417
+ {
418
+ "cell_type": "markdown",
419
+ "id": "8c4d0d79",
420
+ "metadata": {},
421
+ "source": [
422
+ "# Step 10: Deploy with FastAPI"
423
+ ]
424
+ },
425
+ {
426
+ "cell_type": "code",
427
+ "execution_count": null,
428
+ "id": "3f239ed4",
429
+ "metadata": {},
430
+ "outputs": [],
431
+ "source": [
432
+ "from fastapi import FastAPI\n",
433
+ "from pydantic import BaseModel\n",
434
+ "import uvicorn\n",
435
+ "\n",
436
+ "app = FastAPI()\n",
437
+ "\n",
438
+ "class CodeRequest(BaseModel):\n",
439
+ " instruction: str\n",
440
+ "\n",
441
+ "@app.post(\"/generate\")\n",
442
+ "def generate_code(req: CodeRequest):\n",
443
+ " inputs = tokenizer(req.instruction, return_tensors=\"pt\")\n",
444
+ " outputs = model.generate(**inputs, max_length=128)\n",
445
+ " code = tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
446
+ " return {\"code\": code}"
447
+ ]
448
+ },
449
+ {
450
+ "cell_type": "code",
451
+ "execution_count": null,
452
+ "id": "33ec10a2",
453
+ "metadata": {},
454
+ "outputs": [],
455
+ "source": [
456
+ "# Uncomment to run API\n",
457
+ "# uvicorn.run(app, host=\"0.0.0.0\", port=8000)"
458
+ ]
459
+ }
460
+ ],
461
+ "metadata": {
462
+ "kernelspec": {
463
+ "display_name": "myenv",
464
+ "language": "python",
465
+ "name": "python3"
466
+ },
467
+ "language_info": {
468
+ "codemirror_mode": {
469
+ "name": "ipython",
470
+ "version": 3
471
+ },
472
+ "file_extension": ".py",
473
+ "mimetype": "text/x-python",
474
+ "name": "python",
475
+ "nbconvert_exporter": "python",
476
+ "pygments_lexer": "ipython3",
477
+ "version": "3.10.16"
478
+ }
479
+ },
480
+ "nbformat": 4,
481
+ "nbformat_minor": 5
482
+ }
train_t5_code.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import T5TokenizerFast, T5ForConditionalGeneration, Trainer, TrainingArguments, DataCollatorForSeq2Seq
2
+ from datasets import load_dataset, Dataset
3
+ import os
4
+ import json
5
+
6
+ # === Load your custom tokenizer ===
7
+ tokenizer = T5TokenizerFast.from_pretrained("minicoderx-tokenizer")
8
+
9
+ # === Load or create dataset ===
10
+ def load_jsonl(path):
11
+ with open(path) as f:
12
+ data = [json.loads(line) for line in f]
13
+ return Dataset.from_dict({
14
+ "input": [x["input"] for x in data],
15
+ "output": [x["output"] for x in data]
16
+ })
17
+
18
+ dataset = load_jsonl("data/train.jsonl")
19
+
20
+ # === Tokenize dataset ===
21
+ def tokenize(batch):
22
+ return tokenizer(batch["input"], padding="max_length", truncation=True, max_length=128)
23
+
24
+ def tokenize_labels(batch):
25
+ labels = tokenizer(batch["output"], padding="max_length", truncation=True, max_length=128)
26
+ batch["labels"] = labels["input_ids"]
27
+ return batch
28
+
29
+ dataset = dataset.map(tokenize)
30
+ dataset = dataset.map(tokenize_labels)
31
+
32
+ # === Load pre-trained T5-small ===
33
+ model = T5ForConditionalGeneration.from_pretrained("t5-small")
34
+
35
+ # === Training configuration ===
36
+ training_args = TrainingArguments(
37
+ output_dir="minicoderx-model",
38
+ per_device_train_batch_size=4,
39
+ num_train_epochs=3,
40
+ logging_steps=10,
41
+ save_strategy="epoch",
42
+ evaluation_strategy="no",
43
+ save_total_limit=2,
44
+ fp16=True,
45
+ overwrite_output_dir=True,
46
+ )
47
+
48
+ data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
49
+
50
+ trainer = Trainer(
51
+ model=model,
52
+ args=training_args,
53
+ train_dataset=dataset,
54
+ data_collator=data_collator,
55
+ tokenizer=tokenizer
56
+ )
57
+
58
+ # === Train ===
59
+ trainer.train()
60
+
61
+ # === Save model ===
62
+ trainer.save_model("minicoderx-model")
63
+ tokenizer.save_pretrained("minicoderx-model")
64
+
65
+ print("Training complete and model saved.")