Upload folder using huggingface_hub

README.md

@@ -0,0 +1,60 @@
+---
+library_name: transformers
+license: other
+base_model: deepseek-ai/deepseek-coder-7b-instruct-v1.5
+tags:
+- llama-factory
+- freeze
+- generated_from_trainer
+model-index:
+- name: deepseek-nlx-330k
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# deepseek-nlx-330k
+
+This model is a fine-tuned version of [deepseek-ai/deepseek-coder-7b-instruct-v1.5](https://huggingface.co/deepseek-ai/deepseek-coder-7b-instruct-v1.5) on the codes3_query_filtered_330k_nlx dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 16
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 512
+- total_eval_batch_size: 32
+- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- num_epochs: 1.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.48.2
+- Pytorch 2.5.1+cu124
+- Datasets 3.2.0
+- Tokenizers 0.21.0

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9963898916967509,
+    "num_input_tokens_seen": 144703488,
+    "total_flos": 5.635565866281075e+18,
+    "train_loss": 0.5665888682655666,
+    "train_runtime": 10913.0245,
+    "train_samples_per_second": 3.247,
+    "train_steps_per_second": 0.006
+}

config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "deepseek-ai/deepseek-coder-7b-instruct-v1.5",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100000,
+  "eos_token_id": 100015,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "factor": 1.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 4096,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
+  "use_cache": false,
+  "vocab_size": 102400
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100000,
+  "eos_token_id": 100015,
+  "transformers_version": "4.48.2"
+}

llamaboard_config.yaml

@@ -0,0 +1,77 @@
+top.booster: liger_kernel
+top.checkpoint_path: null
+top.finetuning_type: freeze
+top.model_name: DeepSeek-Coder-7B-Instruct
+top.quantization_bit: none
+top.quantization_method: bitsandbytes
+top.rope_scaling: llama3
+top.template: deepseekcoder
+train.additional_target: ''
+train.apollo_rank: 256
+train.apollo_scale: 1
+train.apollo_target: all
+train.apollo_update_interval: 200
+train.badam_mode: layer
+train.badam_switch_interval: 50
+train.badam_switch_mode: ascending
+train.badam_update_ratio: 0.05
+train.batch_size: 16
+train.compute_type: bf16
+train.create_new_adapter: false
+train.cutoff_len: 4096
+train.dataset:
+- codes3_query_filtered_330k_nlx
+train.dataset_dir: data
+train.ds_offload: false
+train.ds_stage: none
+train.extra_args: '{}'
+train.freeze_extra_modules: ''
+train.freeze_trainable_layers: 2
+train.freeze_trainable_modules: all
+train.galore_rank: 16
+train.galore_scale: 2
+train.galore_target: all
+train.galore_update_interval: 200
+train.gradient_accumulation_steps: 8
+train.learning_rate: 5e-5
+train.logging_steps: 1
+train.lora_alpha: 16
+train.lora_dropout: 0
+train.lora_rank: 8
+train.lora_target: ''
+train.loraplus_lr_ratio: 0
+train.lr_scheduler_type: cosine
+train.mask_history: false
+train.max_grad_norm: '1.0'
+train.max_samples: '50000000'
+train.neat_packing: true
+train.neftune_alpha: 0
+train.num_train_epochs: '1'
+train.packing: true
+train.ppo_score_norm: false
+train.ppo_whiten_rewards: false
+train.pref_beta: 0.1
+train.pref_ftx: 0
+train.pref_loss: sigmoid
+train.report_to:
+- none
+train.resize_vocab: false
+train.reward_model: null
+train.save_steps: 500
+train.swanlab_api_key: ''
+train.swanlab_mode: cloud
+train.swanlab_project: llamafactory
+train.swanlab_run_name: ''
+train.swanlab_workspace: ''
+train.train_on_prompt: false
+train.training_stage: Supervised Fine-Tuning
+train.use_apollo: true
+train.use_badam: false
+train.use_dora: false
+train.use_galore: false
+train.use_llama_pro: true
+train.use_pissa: false
+train.use_rslora: false
+train.use_swanlab: false
+train.val_size: 0
+train.warmup_steps: 0

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28aba7f82f5ce656674a3e922d68c0b7bd12d2f2e6da09e47fa72eec539c53c2
+size 4987202208

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b5f404a4192d64823438ee820dd36fc5516f8e7a8357e50e200be30aef86bc0
+size 4980944400

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2289ae058e1fc527bbf37c07cafa3dcee90e54f5e4597945a586a78a4e1509b0
+size 4662149984

model.safetensors.index.json

@@ -0,0 +1,280 @@
+{
+  "metadata": {
+    "total_size": 14630264832
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

running_log.txt

@@ -0,0 +1,339 @@
+[INFO|2025-05-12 09:58:27] tokenization_utils_base.py:2034 >> loading file tokenizer.model from cache at None
+
+[INFO|2025-05-12 09:58:27] tokenization_utils_base.py:2034 >> loading file tokenizer.json from cache at /home/kiho/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-7b-instruct-v1.5/snapshots/2a050a4c59d687a85324d32e147517992117ed30/tokenizer.json
+
+[INFO|2025-05-12 09:58:27] tokenization_utils_base.py:2034 >> loading file added_tokens.json from cache at None
+
+[INFO|2025-05-12 09:58:27] tokenization_utils_base.py:2034 >> loading file special_tokens_map.json from cache at None
+
+[INFO|2025-05-12 09:58:27] tokenization_utils_base.py:2034 >> loading file tokenizer_config.json from cache at /home/kiho/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-7b-instruct-v1.5/snapshots/2a050a4c59d687a85324d32e147517992117ed30/tokenizer_config.json
+
+[INFO|2025-05-12 09:58:27] tokenization_utils_base.py:2034 >> loading file chat_template.jinja from cache at None
+
+[INFO|2025-05-12 09:58:28] tokenization_utils_base.py:2304 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+[INFO|2025-05-12 09:58:29] configuration_utils.py:696 >> loading configuration file config.json from cache at /home/kiho/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-7b-instruct-v1.5/snapshots/2a050a4c59d687a85324d32e147517992117ed30/config.json
+
+[INFO|2025-05-12 09:58:29] configuration_utils.py:768 >> Model config LlamaConfig {
+  "_name_or_path": "deepseek-ai/deepseek-coder-7b-instruct-v1.5",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100000,
+  "eos_token_id": 100015,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
+  "use_cache": true,
+  "vocab_size": 102400
+}
+
+
+[INFO|2025-05-12 09:58:29] tokenization_utils_base.py:2034 >> loading file tokenizer.model from cache at None
+
+[INFO|2025-05-12 09:58:29] tokenization_utils_base.py:2034 >> loading file tokenizer.json from cache at /home/kiho/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-7b-instruct-v1.5/snapshots/2a050a4c59d687a85324d32e147517992117ed30/tokenizer.json
+
+[INFO|2025-05-12 09:58:29] tokenization_utils_base.py:2034 >> loading file added_tokens.json from cache at None
+
+[INFO|2025-05-12 09:58:29] tokenization_utils_base.py:2034 >> loading file special_tokens_map.json from cache at None
+
+[INFO|2025-05-12 09:58:29] tokenization_utils_base.py:2034 >> loading file tokenizer_config.json from cache at /home/kiho/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-7b-instruct-v1.5/snapshots/2a050a4c59d687a85324d32e147517992117ed30/tokenizer_config.json
+
+[INFO|2025-05-12 09:58:29] tokenization_utils_base.py:2034 >> loading file chat_template.jinja from cache at None
+
+[INFO|2025-05-12 09:58:30] tokenization_utils_base.py:2304 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+[INFO|2025-05-12 09:58:30] logging.py:157 >> Loading dataset Codes3_query_filtered_330k_nlx.json...
+
+[INFO|2025-05-12 09:59:23] configuration_utils.py:696 >> loading configuration file config.json from cache at /home/kiho/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-7b-instruct-v1.5/snapshots/2a050a4c59d687a85324d32e147517992117ed30/config.json
+
+[INFO|2025-05-12 09:59:23] configuration_utils.py:768 >> Model config LlamaConfig {
+  "_name_or_path": "deepseek-ai/deepseek-coder-7b-instruct-v1.5",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100000,
+  "eos_token_id": 100015,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
+  "use_cache": true,
+  "vocab_size": 102400
+}
+
+
+[WARNING|2025-05-12 09:59:23] logging.py:162 >> Input length is smaller than max length. Consider increase input length.
+
+[INFO|2025-05-12 09:59:23] logging.py:157 >> Using llama3 scaling strategy and setting scaling factor to 1.0.
+
+[INFO|2025-05-12 09:59:23] logging.py:157 >> Using block diagonal attention for sequence packing without cross-attention.
+
+[INFO|2025-05-12 09:59:23] logging.py:157 >> Liger kernel has been applied to the model.
+
+[INFO|2025-05-12 09:59:24] modeling_utils.py:3904 >> loading weights file model.safetensors from cache at /home/kiho/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-7b-instruct-v1.5/snapshots/2a050a4c59d687a85324d32e147517992117ed30/model.safetensors.index.json
+
+[INFO|2025-05-12 10:01:40] modeling_utils.py:1582 >> Instantiating LlamaForCausalLM model under default dtype torch.bfloat16.
+
+[INFO|2025-05-12 10:01:40] configuration_utils.py:1140 >> Generate config GenerationConfig {
+  "bos_token_id": 100000,
+  "eos_token_id": 100015
+}
+
+
+[INFO|2025-05-12 10:01:45] modeling_utils.py:4888 >> All model checkpoint weights were used when initializing LlamaForCausalLM.
+
+
+[INFO|2025-05-12 10:01:45] modeling_utils.py:4896 >> All the weights of LlamaForCausalLM were initialized from the model checkpoint at deepseek-ai/deepseek-coder-7b-instruct-v1.5.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
+
+[INFO|2025-05-12 10:01:46] configuration_utils.py:1095 >> loading configuration file generation_config.json from cache at /home/kiho/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-7b-instruct-v1.5/snapshots/2a050a4c59d687a85324d32e147517992117ed30/generation_config.json
+
+[INFO|2025-05-12 10:01:46] configuration_utils.py:1140 >> Generate config GenerationConfig {
+  "bos_token_id": 100000,
+  "eos_token_id": 100015
+}
+
+
+[INFO|2025-05-12 10:01:46] logging.py:157 >> Gradient checkpointing enabled.
+
+[INFO|2025-05-12 10:01:46] logging.py:157 >> Using torch SDPA for faster training and inference.
+
+[INFO|2025-05-12 10:01:46] logging.py:157 >> Upcasting trainable params to float32.
+
+[INFO|2025-05-12 10:01:46] logging.py:157 >> Fine-tuning method: Freeze
+
+[INFO|2025-05-12 10:01:46] logging.py:157 >> Set trainable layers: .14.,.29.
+
+[INFO|2025-05-12 10:01:46] logging.py:157 >> trainable params: 404,766,720 || all params: 6,910,365,696 || trainable%: 5.8574
+
+[INFO|2025-05-12 10:01:46] trainer.py:741 >> Using auto half precision backend
+
+[INFO|2025-05-12 10:01:47] logging.py:157 >> Found linear modules: k_proj,v_proj,o_proj,down_proj,q_proj,up_proj,gate_proj
+
+[INFO|2025-05-12 10:01:47] logging.py:157 >> Using APOLLO optimizer with args: {'rank': 256, 'proj': 'random', 'proj_type': 'std', 'update_proj_gap': 200, 'scale': 1, 'scale_type': 'channel', 'scale_front': False}.
+
+[INFO|2025-05-12 10:01:47] trainer.py:2369 >> ***** Running training *****
+
+[INFO|2025-05-12 10:01:47] trainer.py:2370 >>   Num examples = 35,434
+
+[INFO|2025-05-12 10:01:47] trainer.py:2371 >>   Num Epochs = 1
+
+[INFO|2025-05-12 10:01:47] trainer.py:2372 >>   Instantaneous batch size per device = 16
+
+[INFO|2025-05-12 10:01:47] trainer.py:2375 >>   Total train batch size (w. parallel, distributed & accumulation) = 512
+
+[INFO|2025-05-12 10:01:47] trainer.py:2376 >>   Gradient Accumulation steps = 8
+
+[INFO|2025-05-12 10:01:47] trainer.py:2377 >>   Total optimization steps = 69
+
+[INFO|2025-05-12 10:01:47] trainer.py:2378 >>   Number of trainable parameters = 404,766,720
+
+[INFO|2025-05-12 10:04:36] logging.py:157 >> {'loss': 0.8402, 'learning_rate': 4.9974e-05, 'epoch': 0.01, 'throughput': 12482.16}
+
+[INFO|2025-05-12 10:07:14] logging.py:157 >> {'loss': 0.7686, 'learning_rate': 4.9896e-05, 'epoch': 0.03, 'throughput': 12852.44}
+
+[INFO|2025-05-12 10:09:52] logging.py:157 >> {'loss': 0.7563, 'learning_rate': 4.9767e-05, 'epoch': 0.04, 'throughput': 12981.50}
+
+[INFO|2025-05-12 10:12:30] logging.py:157 >> {'loss': 0.7139, 'learning_rate': 4.9587e-05, 'epoch': 0.06, 'throughput': 13054.16}
+
+[INFO|2025-05-12 10:15:08] logging.py:157 >> {'loss': 0.6793, 'learning_rate': 4.9355e-05, 'epoch': 0.07, 'throughput': 13098.17}
+
+[INFO|2025-05-12 10:17:46] logging.py:157 >> {'loss': 0.6631, 'learning_rate': 4.9073e-05, 'epoch': 0.09, 'throughput': 13124.77}
+
+[INFO|2025-05-12 10:20:25] logging.py:157 >> {'loss': 0.6400, 'learning_rate': 4.8741e-05, 'epoch': 0.10, 'throughput': 13144.27}
+
+[INFO|2025-05-12 10:23:03] logging.py:157 >> {'loss': 0.6145, 'learning_rate': 4.8360e-05, 'epoch': 0.12, 'throughput': 13158.63}
+
+[INFO|2025-05-12 10:25:41] logging.py:157 >> {'loss': 0.5997, 'learning_rate': 4.7930e-05, 'epoch': 0.13, 'throughput': 13173.26}
+
+[INFO|2025-05-12 10:28:19] logging.py:157 >> {'loss': 0.5896, 'learning_rate': 4.7453e-05, 'epoch': 0.14, 'throughput': 13182.64}
+
+[INFO|2025-05-12 10:30:56] logging.py:157 >> {'loss': 0.6025, 'learning_rate': 4.6930e-05, 'epoch': 0.16, 'throughput': 13191.88}
+
+[INFO|2025-05-12 10:33:35] logging.py:157 >> {'loss': 0.5644, 'learning_rate': 4.6360e-05, 'epoch': 0.17, 'throughput': 13197.39}
+
+[INFO|2025-05-12 10:36:12] logging.py:157 >> {'loss': 0.5558, 'learning_rate': 4.5747e-05, 'epoch': 0.19, 'throughput': 13204.31}
+
+[INFO|2025-05-12 10:38:51] logging.py:157 >> {'loss': 0.5705, 'learning_rate': 4.5091e-05, 'epoch': 0.20, 'throughput': 13208.35}
+
+[INFO|2025-05-12 10:41:29] logging.py:157 >> {'loss': 0.5694, 'learning_rate': 4.4393e-05, 'epoch': 0.22, 'throughput': 13212.19}
+
+[INFO|2025-05-12 10:44:06] logging.py:157 >> {'loss': 0.5590, 'learning_rate': 4.3655e-05, 'epoch': 0.23, 'throughput': 13217.11}
+
+[INFO|2025-05-12 10:46:44] logging.py:157 >> {'loss': 0.5628, 'learning_rate': 4.2878e-05, 'epoch': 0.25, 'throughput': 13220.46}
+
+[INFO|2025-05-12 10:49:23] logging.py:157 >> {'loss': 0.5553, 'learning_rate': 4.2064e-05, 'epoch': 0.26, 'throughput': 13223.11}
+
+[INFO|2025-05-12 10:52:01] logging.py:157 >> {'loss': 0.5401, 'learning_rate': 4.1215e-05, 'epoch': 0.27, 'throughput': 13225.48}
+
+[INFO|2025-05-12 10:54:39] logging.py:157 >> {'loss': 0.5750, 'learning_rate': 4.0332e-05, 'epoch': 0.29, 'throughput': 13226.89}
+
+[INFO|2025-05-12 10:57:17] logging.py:157 >> {'loss': 0.5398, 'learning_rate': 3.9417e-05, 'epoch': 0.30, 'throughput': 13228.58}
+
+[INFO|2025-05-12 10:59:55] logging.py:157 >> {'loss': 0.5389, 'learning_rate': 3.8472e-05, 'epoch': 0.32, 'throughput': 13231.10}
+
+[INFO|2025-05-12 11:02:32] logging.py:157 >> {'loss': 0.5451, 'learning_rate': 3.7500e-05, 'epoch': 0.33, 'throughput': 13234.31}
+
+[INFO|2025-05-12 11:05:11] logging.py:157 >> {'loss': 0.5558, 'learning_rate': 3.6502e-05, 'epoch': 0.35, 'throughput': 13235.05}
+
+[INFO|2025-05-12 11:07:48] logging.py:157 >> {'loss': 0.5403, 'learning_rate': 3.5479e-05, 'epoch': 0.36, 'throughput': 13238.00}
+
+[INFO|2025-05-12 11:10:26] logging.py:157 >> {'loss': 0.5405, 'learning_rate': 3.4435e-05, 'epoch': 0.38, 'throughput': 13240.74}
+
+[INFO|2025-05-12 11:13:04] logging.py:157 >> {'loss': 0.5577, 'learning_rate': 3.3372e-05, 'epoch': 0.39, 'throughput': 13242.13}
+
+[INFO|2025-05-12 11:15:42] logging.py:157 >> {'loss': 0.5345, 'learning_rate': 3.2291e-05, 'epoch': 0.40, 'throughput': 13243.46}
+
+[INFO|2025-05-12 11:18:20] logging.py:157 >> {'loss': 0.5484, 'learning_rate': 3.1195e-05, 'epoch': 0.42, 'throughput': 13243.76}
+
+[INFO|2025-05-12 11:20:58] logging.py:157 >> {'loss': 0.5485, 'learning_rate': 3.0086e-05, 'epoch': 0.43, 'throughput': 13245.01}
+
+[INFO|2025-05-12 11:23:36] logging.py:157 >> {'loss': 0.5299, 'learning_rate': 2.8967e-05, 'epoch': 0.45, 'throughput': 13245.72}
+
+[INFO|2025-05-12 11:26:14] logging.py:157 >> {'loss': 0.5390, 'learning_rate': 2.7840e-05, 'epoch': 0.46, 'throughput': 13246.45}
+
+[INFO|2025-05-12 11:28:52] logging.py:157 >> {'loss': 0.5248, 'learning_rate': 2.6706e-05, 'epoch': 0.48, 'throughput': 13247.54}
+
+[INFO|2025-05-12 11:31:30] logging.py:157 >> {'loss': 0.5342, 'learning_rate': 2.5569e-05, 'epoch': 0.49, 'throughput': 13248.46}
+
+[INFO|2025-05-12 11:34:08] logging.py:157 >> {'loss': 0.5431, 'learning_rate': 2.4431e-05, 'epoch': 0.51, 'throughput': 13249.53}
+
+[INFO|2025-05-12 11:36:45] logging.py:157 >> {'loss': 0.5471, 'learning_rate': 2.3294e-05, 'epoch': 0.52, 'throughput': 13250.51}
+
+[INFO|2025-05-12 11:39:23] logging.py:157 >> {'loss': 0.5420, 'learning_rate': 2.2160e-05, 'epoch': 0.53, 'throughput': 13251.85}
+
+[INFO|2025-05-12 11:42:01] logging.py:157 >> {'loss': 0.5420, 'learning_rate': 2.1033e-05, 'epoch': 0.55, 'throughput': 13253.57}
+
+[INFO|2025-05-12 11:44:38] logging.py:157 >> {'loss': 0.5420, 'learning_rate': 1.9914e-05, 'epoch': 0.56, 'throughput': 13255.10}
+
+[INFO|2025-05-12 11:47:16] logging.py:157 >> {'loss': 0.5425, 'learning_rate': 1.8805e-05, 'epoch': 0.58, 'throughput': 13256.36}
+
+[INFO|2025-05-12 11:49:54] logging.py:157 >> {'loss': 0.5482, 'learning_rate': 1.7709e-05, 'epoch': 0.59, 'throughput': 13257.04}
+
+[INFO|2025-05-12 11:52:31] logging.py:157 >> {'loss': 0.5598, 'learning_rate': 1.6628e-05, 'epoch': 0.61, 'throughput': 13258.73}
+
+[INFO|2025-05-12 11:55:08] logging.py:157 >> {'loss': 0.5402, 'learning_rate': 1.5565e-05, 'epoch': 0.62, 'throughput': 13260.29}
+
+[INFO|2025-05-12 11:57:46] logging.py:157 >> {'loss': 0.5413, 'learning_rate': 1.4521e-05, 'epoch': 0.64, 'throughput': 13261.50}
+
+[INFO|2025-05-12 12:00:24] logging.py:157 >> {'loss': 0.5560, 'learning_rate': 1.3498e-05, 'epoch': 0.65, 'throughput': 13262.39}
+
+[INFO|2025-05-12 12:03:01] logging.py:157 >> {'loss': 0.5341, 'learning_rate': 1.2500e-05, 'epoch': 0.66, 'throughput': 13263.23}
+
+[INFO|2025-05-12 12:05:39] logging.py:157 >> {'loss': 0.5436, 'learning_rate': 1.1528e-05, 'epoch': 0.68, 'throughput': 13264.02}
+
+[INFO|2025-05-12 12:08:16] logging.py:157 >> {'loss': 0.5393, 'learning_rate': 1.0583e-05, 'epoch': 0.69, 'throughput': 13264.84}
+
+[INFO|2025-05-12 12:10:54] logging.py:157 >> {'loss': 0.5598, 'learning_rate': 9.6683e-06, 'epoch': 0.71, 'throughput': 13265.65}
+
+[INFO|2025-05-12 12:13:32] logging.py:157 >> {'loss': 0.5329, 'learning_rate': 8.7854e-06, 'epoch': 0.72, 'throughput': 13265.87}
+
+[INFO|2025-05-12 12:16:10] logging.py:157 >> {'loss': 0.5384, 'learning_rate': 7.9362e-06, 'epoch': 0.74, 'throughput': 13266.73}
+
+[INFO|2025-05-12 12:18:47] logging.py:157 >> {'loss': 0.5447, 'learning_rate': 7.1223e-06, 'epoch': 0.75, 'throughput': 13267.67}
+
+[INFO|2025-05-12 12:21:25] logging.py:157 >> {'loss': 0.5291, 'learning_rate': 6.3454e-06, 'epoch': 0.77, 'throughput': 13268.20}
+
+[INFO|2025-05-12 12:24:02] logging.py:157 >> {'loss': 0.5258, 'learning_rate': 5.6072e-06, 'epoch': 0.78, 'throughput': 13269.48}
+
+[INFO|2025-05-12 12:26:40] logging.py:157 >> {'loss': 0.5508, 'learning_rate': 4.9092e-06, 'epoch': 0.79, 'throughput': 13270.12}
+
+[INFO|2025-05-12 12:29:18] logging.py:157 >> {'loss': 0.5439, 'learning_rate': 4.2529e-06, 'epoch': 0.81, 'throughput': 13270.24}
+
+[INFO|2025-05-12 12:31:56] logging.py:157 >> {'loss': 0.5261, 'learning_rate': 3.6395e-06, 'epoch': 0.82, 'throughput': 13270.03}
+
+[INFO|2025-05-12 12:34:34] logging.py:157 >> {'loss': 0.5518, 'learning_rate': 3.0704e-06, 'epoch': 0.84, 'throughput': 13270.36}
+
+[INFO|2025-05-12 12:37:11] logging.py:157 >> {'loss': 0.5444, 'learning_rate': 2.5468e-06, 'epoch': 0.85, 'throughput': 13270.70}
+
+[INFO|2025-05-12 12:39:49] logging.py:157 >> {'loss': 0.5469, 'learning_rate': 2.0697e-06, 'epoch': 0.87, 'throughput': 13270.96}
+
+[INFO|2025-05-12 12:42:27] logging.py:157 >> {'loss': 0.5335, 'learning_rate': 1.6402e-06, 'epoch': 0.88, 'throughput': 13271.26}
+
+[INFO|2025-05-12 12:45:04] logging.py:157 >> {'loss': 0.5367, 'learning_rate': 1.2590e-06, 'epoch': 0.90, 'throughput': 13272.14}
+
+[INFO|2025-05-12 12:47:42] logging.py:157 >> {'loss': 0.5510, 'learning_rate': 9.2707e-07, 'epoch': 0.91, 'throughput': 13272.46}
+
+[INFO|2025-05-12 12:50:20] logging.py:157 >> {'loss': 0.5597, 'learning_rate': 6.4502e-07, 'epoch': 0.92, 'throughput': 13272.79}
+
+[INFO|2025-05-12 12:52:57] logging.py:157 >> {'loss': 0.5369, 'learning_rate': 4.1346e-07, 'epoch': 0.94, 'throughput': 13273.81}
+
+[INFO|2025-05-12 12:55:33] logging.py:157 >> {'loss': 0.5451, 'learning_rate': 2.3285e-07, 'epoch': 0.95, 'throughput': 13276.09}
+
+[INFO|2025-05-12 12:58:09] logging.py:157 >> {'loss': 0.5237, 'learning_rate': 1.0358e-07, 'epoch': 0.97, 'throughput': 13279.42}
+
+[INFO|2025-05-12 13:00:44] logging.py:157 >> {'loss': 0.5604, 'learning_rate': 2.5908e-08, 'epoch': 0.98, 'throughput': 13283.18}
+
+[INFO|2025-05-12 13:03:19] logging.py:157 >> {'loss': 0.5340, 'learning_rate': 0.0000e+00, 'epoch': 1.00, 'throughput': 13286.80}
+
+[INFO|2025-05-12 13:03:19] trainer.py:3910 >> Saving model checkpoint to saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek-nlx-330k/checkpoint-69
+
+[INFO|2025-05-12 13:03:19] configuration_utils.py:420 >> Configuration saved in saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek-nlx-330k/checkpoint-69/config.json
+
+[INFO|2025-05-12 13:03:19] configuration_utils.py:909 >> Configuration saved in saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek-nlx-330k/checkpoint-69/generation_config.json
+
+[INFO|2025-05-12 13:03:40] modeling_utils.py:2996 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 3 checkpoint shards. You can find where each parameters has been saved in the index located at saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek-nlx-330k/checkpoint-69/model.safetensors.index.json.
+
+[INFO|2025-05-12 13:03:40] tokenization_utils_base.py:2491 >> tokenizer config file saved in saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek-nlx-330k/checkpoint-69/tokenizer_config.json
+
+[INFO|2025-05-12 13:03:40] tokenization_utils_base.py:2500 >> Special tokens file saved in saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek-nlx-330k/checkpoint-69/special_tokens_map.json
+
+[INFO|2025-05-12 13:03:40] trainer.py:2643 >> 
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
+[INFO|2025-05-12 13:03:40] trainer.py:3910 >> Saving model checkpoint to saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek-nlx-330k
+
+[INFO|2025-05-12 13:03:40] configuration_utils.py:420 >> Configuration saved in saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek-nlx-330k/config.json
+
+[INFO|2025-05-12 13:03:40] configuration_utils.py:909 >> Configuration saved in saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek-nlx-330k/generation_config.json
+
+[INFO|2025-05-12 13:04:02] modeling_utils.py:2996 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 3 checkpoint shards. You can find where each parameters has been saved in the index located at saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek-nlx-330k/model.safetensors.index.json.
+
+[INFO|2025-05-12 13:04:02] tokenization_utils_base.py:2491 >> tokenizer config file saved in saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek-nlx-330k/tokenizer_config.json
+
+[INFO|2025-05-12 13:04:02] tokenization_utils_base.py:2500 >> Special tokens file saved in saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek-nlx-330k/special_tokens_map.json
+
+[WARNING|2025-05-12 13:04:02] logging.py:162 >> No metric eval_loss to plot.
+
+[WARNING|2025-05-12 13:04:02] logging.py:162 >> No metric eval_accuracy to plot.
+
+[INFO|2025-05-12 13:04:02] modelcard.py:449 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
+

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<｜begin▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|EOT|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,149 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "100000": {
+      "content": "<｜begin▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100001": {
+      "content": "<｜end▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100002": {
+      "content": "ø",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100003": {
+      "content": "ö",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100004": {
+      "content": "ú",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100005": {
+      "content": "ÿ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100006": {
+      "content": "õ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100007": {
+      "content": "÷",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100008": {
+      "content": "û",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100009": {
+      "content": "ý",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100010": {
+      "content": "À",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100011": {
+      "content": "ù",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100012": {
+      "content": "Á",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100013": {
+      "content": "þ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100014": {
+      "content": "ü",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100015": {
+      "content": "<|EOT|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<｜begin▁of▁sentence｜>",
+  "chat_template": "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n    {%- if message['role'] == 'system' -%}\n        {%- set ns.found = true -%}\n    {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\\n'}}\n{%- endif %}\n{%- for message in messages %}\n    {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n    {%- else %}\n        {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n        {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|EOT|>",
+  "extra_special_tokens": {},
+  "legacy": true,
+  "model_max_length": 4096,
+  "pad_token": "<｜end▁of▁sentence｜>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": null,
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9963898916967509,
+    "num_input_tokens_seen": 144703488,
+    "total_flos": 5.635565866281075e+18,
+    "train_loss": 0.5665888682655666,
+    "train_runtime": 10913.0245,
+    "train_samples_per_second": 3.247,
+    "train_steps_per_second": 0.006
+}

trainer_log.jsonl

@@ -0,0 +1,70 @@
+{"current_steps": 1, "total_steps": 69, "loss": 0.8402, "lr": 4.9974091841168195e-05, "epoch": 0.01444043321299639, "percentage": 1.45, "elapsed_time": "0:02:48", "remaining_time": "3:10:24", "throughput": 12482.16, "total_tokens": 2097152}
+{"current_steps": 2, "total_steps": 69, "loss": 0.7686, "lr": 4.9896421063288286e-05, "epoch": 0.02888086642599278, "percentage": 2.9, "elapsed_time": "0:05:26", "remaining_time": "3:02:12", "throughput": 12852.44, "total_tokens": 4194304}
+{"current_steps": 3, "total_steps": 69, "loss": 0.7563, "lr": 4.976714865090827e-05, "epoch": 0.04332129963898917, "percentage": 4.35, "elapsed_time": "0:08:04", "remaining_time": "2:57:42", "throughput": 12981.5, "total_tokens": 6291456}
+{"current_steps": 4, "total_steps": 69, "loss": 0.7139, "lr": 4.958654254084355e-05, "epoch": 0.05776173285198556, "percentage": 5.8, "elapsed_time": "0:10:42", "remaining_time": "2:54:02", "throughput": 13054.16, "total_tokens": 8388608}
+{"current_steps": 5, "total_steps": 69, "loss": 0.6793, "lr": 4.9354977066836986e-05, "epoch": 0.07220216606498195, "percentage": 7.25, "elapsed_time": "0:13:20", "remaining_time": "2:50:47", "throughput": 13098.17, "total_tokens": 10485760}
+{"current_steps": 6, "total_steps": 69, "loss": 0.6631, "lr": 4.907293218369499e-05, "epoch": 0.08664259927797834, "percentage": 8.7, "elapsed_time": "0:15:58", "remaining_time": "2:47:46", "throughput": 13124.77, "total_tokens": 12582912}
+{"current_steps": 7, "total_steps": 69, "loss": 0.64, "lr": 4.874099247250798e-05, "epoch": 0.10108303249097472, "percentage": 10.14, "elapsed_time": "0:18:36", "remaining_time": "2:44:52", "throughput": 13144.27, "total_tokens": 14680064}
+{"current_steps": 8, "total_steps": 69, "loss": 0.6145, "lr": 4.835984592901678e-05, "epoch": 0.11552346570397112, "percentage": 11.59, "elapsed_time": "0:21:14", "remaining_time": "2:42:01", "throughput": 13158.63, "total_tokens": 16777216}
+{"current_steps": 9, "total_steps": 69, "loss": 0.5997, "lr": 4.793028253763633e-05, "epoch": 0.1299638989169675, "percentage": 13.04, "elapsed_time": "0:23:52", "remaining_time": "2:39:11", "throughput": 13173.26, "total_tokens": 18874368}
+{"current_steps": 10, "total_steps": 69, "loss": 0.5896, "lr": 4.74531926340924e-05, "epoch": 0.1444043321299639, "percentage": 14.49, "elapsed_time": "0:26:30", "remaining_time": "2:36:25", "throughput": 13182.64, "total_tokens": 20971520}
+{"current_steps": 11, "total_steps": 69, "loss": 0.6025, "lr": 4.6929565060064864e-05, "epoch": 0.1588447653429603, "percentage": 15.94, "elapsed_time": "0:29:08", "remaining_time": "2:33:40", "throughput": 13191.88, "total_tokens": 23068672}
+{"current_steps": 12, "total_steps": 69, "loss": 0.5644, "lr": 4.6360485113662216e-05, "epoch": 0.17328519855595667, "percentage": 17.39, "elapsed_time": "0:31:46", "remaining_time": "2:30:57", "throughput": 13197.39, "total_tokens": 25165824}
+{"current_steps": 13, "total_steps": 69, "loss": 0.5558, "lr": 4.574713229997563e-05, "epoch": 0.18772563176895307, "percentage": 18.84, "elapsed_time": "0:34:24", "remaining_time": "2:28:14", "throughput": 13204.31, "total_tokens": 27262976}
+{"current_steps": 14, "total_steps": 69, "loss": 0.5705, "lr": 4.509077788637446e-05, "epoch": 0.20216606498194944, "percentage": 20.29, "elapsed_time": "0:37:02", "remaining_time": "2:25:32", "throughput": 13208.35, "total_tokens": 29360128}
+{"current_steps": 15, "total_steps": 69, "loss": 0.5694, "lr": 4.43927822676105e-05, "epoch": 0.21660649819494585, "percentage": 21.74, "elapsed_time": "0:39:40", "remaining_time": "2:22:51", "throughput": 13212.19, "total_tokens": 31457280}
+{"current_steps": 16, "total_steps": 69, "loss": 0.559, "lr": 4.365459214619214e-05, "epoch": 0.23104693140794225, "percentage": 23.19, "elapsed_time": "0:42:18", "remaining_time": "2:20:09", "throughput": 13217.11, "total_tokens": 33554432}
+{"current_steps": 17, "total_steps": 69, "loss": 0.5628, "lr": 4.2877737533872485e-05, "epoch": 0.24548736462093862, "percentage": 24.64, "elapsed_time": "0:44:56", "remaining_time": "2:17:28", "throughput": 13220.46, "total_tokens": 35651584}
+{"current_steps": 18, "total_steps": 69, "loss": 0.5553, "lr": 4.206382858046636e-05, "epoch": 0.259927797833935, "percentage": 26.09, "elapsed_time": "0:47:34", "remaining_time": "2:14:48", "throughput": 13223.11, "total_tokens": 37748736}
+{"current_steps": 19, "total_steps": 69, "loss": 0.5401, "lr": 4.12145522365689e-05, "epoch": 0.2743682310469314, "percentage": 27.54, "elapsed_time": "0:50:12", "remaining_time": "2:12:08", "throughput": 13225.48, "total_tokens": 39845888}
+{"current_steps": 20, "total_steps": 69, "loss": 0.575, "lr": 4.033166875709291e-05, "epoch": 0.2888086642599278, "percentage": 28.99, "elapsed_time": "0:52:51", "remaining_time": "2:09:29", "throughput": 13226.89, "total_tokens": 41943040}
+{"current_steps": 21, "total_steps": 69, "loss": 0.5398, "lr": 3.941700805287168e-05, "epoch": 0.30324909747292417, "percentage": 30.43, "elapsed_time": "0:55:29", "remaining_time": "2:06:49", "throughput": 13228.58, "total_tokens": 44040192}
+{"current_steps": 22, "total_steps": 69, "loss": 0.5389, "lr": 3.8472465897889394e-05, "epoch": 0.3176895306859206, "percentage": 31.88, "elapsed_time": "0:58:07", "remaining_time": "2:04:09", "throughput": 13231.1, "total_tokens": 46137344}
+{"current_steps": 23, "total_steps": 69, "loss": 0.5451, "lr": 3.7500000000000003e-05, "epoch": 0.33212996389891697, "percentage": 33.33, "elapsed_time": "1:00:44", "remaining_time": "2:01:29", "throughput": 13234.31, "total_tokens": 48234496}
+{"current_steps": 24, "total_steps": 69, "loss": 0.5558, "lr": 3.6501625943278805e-05, "epoch": 0.34657039711191334, "percentage": 34.78, "elapsed_time": "1:03:22", "remaining_time": "1:58:50", "throughput": 13235.05, "total_tokens": 50331648}
+{"current_steps": 25, "total_steps": 69, "loss": 0.5403, "lr": 3.547941301041661e-05, "epoch": 0.36101083032490977, "percentage": 36.23, "elapsed_time": "1:06:00", "remaining_time": "1:56:10", "throughput": 13238.0, "total_tokens": 52428800}
+{"current_steps": 26, "total_steps": 69, "loss": 0.5405, "lr": 3.443547989381536e-05, "epoch": 0.37545126353790614, "percentage": 37.68, "elapsed_time": "1:08:38", "remaining_time": "1:53:30", "throughput": 13240.74, "total_tokens": 54525952}
+{"current_steps": 27, "total_steps": 69, "loss": 0.5577, "lr": 3.3371990304274656e-05, "epoch": 0.3898916967509025, "percentage": 39.13, "elapsed_time": "1:11:15", "remaining_time": "1:50:51", "throughput": 13242.13, "total_tokens": 56623104}
+{"current_steps": 28, "total_steps": 69, "loss": 0.5345, "lr": 3.2291148486370626e-05, "epoch": 0.4043321299638989, "percentage": 40.58, "elapsed_time": "1:13:53", "remaining_time": "1:48:12", "throughput": 13243.46, "total_tokens": 58720256}
+{"current_steps": 29, "total_steps": 69, "loss": 0.5484, "lr": 3.11951946498225e-05, "epoch": 0.4187725631768953, "percentage": 42.03, "elapsed_time": "1:16:32", "remaining_time": "1:45:34", "throughput": 13243.76, "total_tokens": 60817408}
+{"current_steps": 30, "total_steps": 69, "loss": 0.5485, "lr": 3.008640032631585e-05, "epoch": 0.4332129963898917, "percentage": 43.48, "elapsed_time": "1:19:10", "remaining_time": "1:42:55", "throughput": 13245.01, "total_tokens": 62914560}
+{"current_steps": 31, "total_steps": 69, "loss": 0.5299, "lr": 2.8967063661406285e-05, "epoch": 0.44765342960288806, "percentage": 44.93, "elapsed_time": "1:21:48", "remaining_time": "1:40:16", "throughput": 13245.72, "total_tokens": 65011712}
+{"current_steps": 32, "total_steps": 69, "loss": 0.539, "lr": 2.7839504651261872e-05, "epoch": 0.4620938628158845, "percentage": 46.38, "elapsed_time": "1:24:26", "remaining_time": "1:37:37", "throughput": 13246.45, "total_tokens": 67108864}
+{"current_steps": 33, "total_steps": 69, "loss": 0.5248, "lr": 2.6706060334116777e-05, "epoch": 0.47653429602888087, "percentage": 47.83, "elapsed_time": "1:27:04", "remaining_time": "1:34:58", "throughput": 13247.54, "total_tokens": 69206016}
+{"current_steps": 34, "total_steps": 69, "loss": 0.5342, "lr": 2.556907994640264e-05, "epoch": 0.49097472924187724, "percentage": 49.28, "elapsed_time": "1:29:41", "remaining_time": "1:32:20", "throughput": 13248.46, "total_tokens": 71303168}
+{"current_steps": 35, "total_steps": 69, "loss": 0.5431, "lr": 2.4430920053597356e-05, "epoch": 0.5054151624548736, "percentage": 50.72, "elapsed_time": "1:32:19", "remaining_time": "1:29:41", "throughput": 13249.53, "total_tokens": 73400320}
+{"current_steps": 36, "total_steps": 69, "loss": 0.5471, "lr": 2.329393966588323e-05, "epoch": 0.51985559566787, "percentage": 52.17, "elapsed_time": "1:34:57", "remaining_time": "1:27:02", "throughput": 13250.51, "total_tokens": 75497472}
+{"current_steps": 37, "total_steps": 69, "loss": 0.542, "lr": 2.2160495348738123e-05, "epoch": 0.5342960288808665, "percentage": 53.62, "elapsed_time": "1:37:35", "remaining_time": "1:24:24", "throughput": 13251.85, "total_tokens": 77594624}
+{"current_steps": 38, "total_steps": 69, "loss": 0.542, "lr": 2.1032936338593718e-05, "epoch": 0.5487364620938628, "percentage": 55.07, "elapsed_time": "1:40:12", "remaining_time": "1:21:45", "throughput": 13253.57, "total_tokens": 79691776}
+{"current_steps": 39, "total_steps": 69, "loss": 0.542, "lr": 1.991359967368416e-05, "epoch": 0.5631768953068592, "percentage": 56.52, "elapsed_time": "1:42:50", "remaining_time": "1:19:06", "throughput": 13255.1, "total_tokens": 81788928}
+{"current_steps": 40, "total_steps": 69, "loss": 0.5425, "lr": 1.8804805350177505e-05, "epoch": 0.5776173285198556, "percentage": 57.97, "elapsed_time": "1:45:27", "remaining_time": "1:16:27", "throughput": 13256.36, "total_tokens": 83886080}
+{"current_steps": 41, "total_steps": 69, "loss": 0.5482, "lr": 1.7708851513629377e-05, "epoch": 0.592057761732852, "percentage": 59.42, "elapsed_time": "1:48:05", "remaining_time": "1:13:49", "throughput": 13257.04, "total_tokens": 85983232}
+{"current_steps": 42, "total_steps": 69, "loss": 0.5598, "lr": 1.6628009695725346e-05, "epoch": 0.6064981949458483, "percentage": 60.87, "elapsed_time": "1:50:43", "remaining_time": "1:11:10", "throughput": 13258.73, "total_tokens": 88080384}
+{"current_steps": 43, "total_steps": 69, "loss": 0.5402, "lr": 1.5564520106184644e-05, "epoch": 0.6209386281588448, "percentage": 62.32, "elapsed_time": "1:53:20", "remaining_time": "1:08:31", "throughput": 13260.29, "total_tokens": 90177536}
+{"current_steps": 44, "total_steps": 69, "loss": 0.5413, "lr": 1.4520586989583406e-05, "epoch": 0.6353790613718412, "percentage": 63.77, "elapsed_time": "1:55:58", "remaining_time": "1:05:53", "throughput": 13261.5, "total_tokens": 92274688}
+{"current_steps": 45, "total_steps": 69, "loss": 0.556, "lr": 1.3498374056721197e-05, "epoch": 0.6498194945848376, "percentage": 65.22, "elapsed_time": "1:58:35", "remaining_time": "1:03:15", "throughput": 13262.39, "total_tokens": 94371840}
+{"current_steps": 46, "total_steps": 69, "loss": 0.5341, "lr": 1.2500000000000006e-05, "epoch": 0.6642599277978339, "percentage": 66.67, "elapsed_time": "2:01:13", "remaining_time": "1:00:36", "throughput": 13263.23, "total_tokens": 96468992}
+{"current_steps": 47, "total_steps": 69, "loss": 0.5436, "lr": 1.1527534102110612e-05, "epoch": 0.6787003610108303, "percentage": 68.12, "elapsed_time": "2:03:51", "remaining_time": "0:57:58", "throughput": 13264.02, "total_tokens": 98566144}
+{"current_steps": 48, "total_steps": 69, "loss": 0.5393, "lr": 1.0582991947128324e-05, "epoch": 0.6931407942238267, "percentage": 69.57, "elapsed_time": "2:06:28", "remaining_time": "0:55:20", "throughput": 13264.84, "total_tokens": 100663296}
+{"current_steps": 49, "total_steps": 69, "loss": 0.5598, "lr": 9.668331242907089e-06, "epoch": 0.7075812274368231, "percentage": 71.01, "elapsed_time": "2:09:06", "remaining_time": "0:52:41", "throughput": 13265.65, "total_tokens": 102760448}
+{"current_steps": 50, "total_steps": 69, "loss": 0.5329, "lr": 8.785447763431101e-06, "epoch": 0.7220216606498195, "percentage": 72.46, "elapsed_time": "2:11:44", "remaining_time": "0:50:03", "throughput": 13265.87, "total_tokens": 104857600}
+{"current_steps": 51, "total_steps": 69, "loss": 0.5384, "lr": 7.936171419533653e-06, "epoch": 0.7364620938628159, "percentage": 73.91, "elapsed_time": "2:14:21", "remaining_time": "0:47:25", "throughput": 13266.73, "total_tokens": 106954752}
+{"current_steps": 52, "total_steps": 69, "loss": 0.5447, "lr": 7.122262466127514e-06, "epoch": 0.7509025270758123, "percentage": 75.36, "elapsed_time": "2:16:59", "remaining_time": "0:44:47", "throughput": 13267.67, "total_tokens": 109051904}
+{"current_steps": 53, "total_steps": 69, "loss": 0.5291, "lr": 6.3454078538078635e-06, "epoch": 0.7653429602888087, "percentage": 76.81, "elapsed_time": "2:19:37", "remaining_time": "0:42:08", "throughput": 13268.2, "total_tokens": 111149056}
+{"current_steps": 54, "total_steps": 69, "loss": 0.5258, "lr": 5.607217732389503e-06, "epoch": 0.779783393501805, "percentage": 78.26, "elapsed_time": "2:22:14", "remaining_time": "0:39:30", "throughput": 13269.48, "total_tokens": 113246208}
+{"current_steps": 55, "total_steps": 69, "loss": 0.5508, "lr": 4.9092221136255444e-06, "epoch": 0.7942238267148014, "percentage": 79.71, "elapsed_time": "2:24:51", "remaining_time": "0:36:52", "throughput": 13270.12, "total_tokens": 115343360}
+{"current_steps": 56, "total_steps": 69, "loss": 0.5439, "lr": 4.252867700024374e-06, "epoch": 0.8086642599277978, "percentage": 81.16, "elapsed_time": "2:27:29", "remaining_time": "0:34:14", "throughput": 13270.24, "total_tokens": 117440512}
+{"current_steps": 57, "total_steps": 69, "loss": 0.5261, "lr": 3.6395148863377858e-06, "epoch": 0.8231046931407943, "percentage": 82.61, "elapsed_time": "2:30:08", "remaining_time": "0:31:36", "throughput": 13270.03, "total_tokens": 119537664}
+{"current_steps": 58, "total_steps": 69, "loss": 0.5518, "lr": 3.0704349399351435e-06, "epoch": 0.8375451263537906, "percentage": 84.06, "elapsed_time": "2:32:45", "remaining_time": "0:28:58", "throughput": 13270.36, "total_tokens": 121634816}
+{"current_steps": 59, "total_steps": 69, "loss": 0.5444, "lr": 2.5468073659076e-06, "epoch": 0.851985559566787, "percentage": 85.51, "elapsed_time": "2:35:23", "remaining_time": "0:26:20", "throughput": 13270.7, "total_tokens": 123731968}
+{"current_steps": 60, "total_steps": 69, "loss": 0.5469, "lr": 2.0697174623636794e-06, "epoch": 0.8664259927797834, "percentage": 86.96, "elapsed_time": "2:38:01", "remaining_time": "0:23:42", "throughput": 13270.96, "total_tokens": 125829120}
+{"current_steps": 61, "total_steps": 69, "loss": 0.5335, "lr": 1.6401540709832242e-06, "epoch": 0.8808664259927798, "percentage": 88.41, "elapsed_time": "2:40:39", "remaining_time": "0:21:04", "throughput": 13271.26, "total_tokens": 127926272}
+{"current_steps": 62, "total_steps": 69, "loss": 0.5367, "lr": 1.2590075274920205e-06, "epoch": 0.8953068592057761, "percentage": 89.86, "elapsed_time": "2:43:16", "remaining_time": "0:18:26", "throughput": 13272.14, "total_tokens": 130023424}
+{"current_steps": 63, "total_steps": 69, "loss": 0.551, "lr": 9.270678163050217e-07, "epoch": 0.9097472924187726, "percentage": 91.3, "elapsed_time": "2:45:54", "remaining_time": "0:15:48", "throughput": 13272.46, "total_tokens": 132120576}
+{"current_steps": 64, "total_steps": 69, "loss": 0.5597, "lr": 6.450229331630253e-07, "epoch": 0.924187725631769, "percentage": 92.75, "elapsed_time": "2:48:32", "remaining_time": "0:13:10", "throughput": 13272.79, "total_tokens": 134217728}
+{"current_steps": 65, "total_steps": 69, "loss": 0.5369, "lr": 4.134574591564494e-07, "epoch": 0.9386281588447654, "percentage": 94.2, "elapsed_time": "2:51:09", "remaining_time": "0:10:31", "throughput": 13273.81, "total_tokens": 136314880}
+{"current_steps": 66, "total_steps": 69, "loss": 0.5451, "lr": 2.3285134909173112e-07, "epoch": 0.9530685920577617, "percentage": 95.65, "elapsed_time": "2:53:45", "remaining_time": "0:07:53", "throughput": 13276.09, "total_tokens": 138412032}
+{"current_steps": 67, "total_steps": 69, "loss": 0.5237, "lr": 1.0357893671171792e-07, "epoch": 0.9675090252707581, "percentage": 97.1, "elapsed_time": "2:56:20", "remaining_time": "0:05:15", "throughput": 13279.42, "total_tokens": 140509184}
+{"current_steps": 68, "total_steps": 69, "loss": 0.5604, "lr": 2.590815883181108e-08, "epoch": 0.9819494584837545, "percentage": 98.55, "elapsed_time": "2:58:55", "remaining_time": "0:02:37", "throughput": 13283.18, "total_tokens": 142606336}
+{"current_steps": 69, "total_steps": 69, "loss": 0.534, "lr": 0.0, "epoch": 0.9963898916967509, "percentage": 100.0, "elapsed_time": "3:01:30", "remaining_time": "0:00:00", "throughput": 13286.8, "total_tokens": 144703488}
+{"current_steps": 69, "total_steps": 69, "epoch": 0.9963898916967509, "percentage": 100.0, "elapsed_time": "3:01:52", "remaining_time": "0:00:00", "throughput": 13260.92, "total_tokens": 144703488}

trainer_state.json

@@ -0,0 +1,595 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9963898916967509,
+  "eval_steps": 500,
+  "global_step": 69,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01444043321299639,
+      "grad_norm": 0.9324310421943665,
+      "learning_rate": 4.9974091841168195e-05,
+      "loss": 0.8402,
+      "num_input_tokens_seen": 2097152,
+      "step": 1
+    },
+    {
+      "epoch": 0.02888086642599278,
+      "grad_norm": 0.736538290977478,
+      "learning_rate": 4.9896421063288286e-05,
+      "loss": 0.7686,
+      "num_input_tokens_seen": 4194304,
+      "step": 2
+    },
+    {
+      "epoch": 0.04332129963898917,
+      "grad_norm": 0.6694996953010559,
+      "learning_rate": 4.976714865090827e-05,
+      "loss": 0.7563,
+      "num_input_tokens_seen": 6291456,
+      "step": 3
+    },
+    {
+      "epoch": 0.05776173285198556,
+      "grad_norm": 0.5700488090515137,
+      "learning_rate": 4.958654254084355e-05,
+      "loss": 0.7139,
+      "num_input_tokens_seen": 8388608,
+      "step": 4
+    },
+    {
+      "epoch": 0.07220216606498195,
+      "grad_norm": 0.47626349329948425,
+      "learning_rate": 4.9354977066836986e-05,
+      "loss": 0.6793,
+      "num_input_tokens_seen": 10485760,
+      "step": 5
+    },
+    {
+      "epoch": 0.08664259927797834,
+      "grad_norm": 0.4310459494590759,
+      "learning_rate": 4.907293218369499e-05,
+      "loss": 0.6631,
+      "num_input_tokens_seen": 12582912,
+      "step": 6
+    },
+    {
+      "epoch": 0.10108303249097472,
+      "grad_norm": 0.41724854707717896,
+      "learning_rate": 4.874099247250798e-05,
+      "loss": 0.64,
+      "num_input_tokens_seen": 14680064,
+      "step": 7
+    },
+    {
+      "epoch": 0.11552346570397112,
+      "grad_norm": 0.37938904762268066,
+      "learning_rate": 4.835984592901678e-05,
+      "loss": 0.6145,
+      "num_input_tokens_seen": 16777216,
+      "step": 8
+    },
+    {
+      "epoch": 0.1299638989169675,
+      "grad_norm": 0.30510303378105164,
+      "learning_rate": 4.793028253763633e-05,
+      "loss": 0.5997,
+      "num_input_tokens_seen": 18874368,
+      "step": 9
+    },
+    {
+      "epoch": 0.1444043321299639,
+      "grad_norm": 0.17869406938552856,
+      "learning_rate": 4.74531926340924e-05,
+      "loss": 0.5896,
+      "num_input_tokens_seen": 20971520,
+      "step": 10
+    },
+    {
+      "epoch": 0.1588447653429603,
+      "grad_norm": 0.1084759458899498,
+      "learning_rate": 4.6929565060064864e-05,
+      "loss": 0.6025,
+      "num_input_tokens_seen": 23068672,
+      "step": 11
+    },
+    {
+      "epoch": 0.17328519855595667,
+      "grad_norm": 0.09031977504491806,
+      "learning_rate": 4.6360485113662216e-05,
+      "loss": 0.5644,
+      "num_input_tokens_seen": 25165824,
+      "step": 12
+    },
+    {
+      "epoch": 0.18772563176895307,
+      "grad_norm": 0.08057376742362976,
+      "learning_rate": 4.574713229997563e-05,
+      "loss": 0.5558,
+      "num_input_tokens_seen": 27262976,
+      "step": 13
+    },
+    {
+      "epoch": 0.20216606498194944,
+      "grad_norm": 0.0670362040400505,
+      "learning_rate": 4.509077788637446e-05,
+      "loss": 0.5705,
+      "num_input_tokens_seen": 29360128,
+      "step": 14
+    },
+    {
+      "epoch": 0.21660649819494585,
+      "grad_norm": 0.06539376825094223,
+      "learning_rate": 4.43927822676105e-05,
+      "loss": 0.5694,
+      "num_input_tokens_seen": 31457280,
+      "step": 15
+    },
+    {
+      "epoch": 0.23104693140794225,
+      "grad_norm": 0.05930742993950844,
+      "learning_rate": 4.365459214619214e-05,
+      "loss": 0.559,
+      "num_input_tokens_seen": 33554432,
+      "step": 16
+    },
+    {
+      "epoch": 0.24548736462093862,
+      "grad_norm": 0.054464634507894516,
+      "learning_rate": 4.2877737533872485e-05,
+      "loss": 0.5628,
+      "num_input_tokens_seen": 35651584,
+      "step": 17
+    },
+    {
+      "epoch": 0.259927797833935,
+      "grad_norm": 0.053172189742326736,
+      "learning_rate": 4.206382858046636e-05,
+      "loss": 0.5553,
+      "num_input_tokens_seen": 37748736,
+      "step": 18
+    },
+    {
+      "epoch": 0.2743682310469314,
+      "grad_norm": 0.04865848645567894,
+      "learning_rate": 4.12145522365689e-05,
+      "loss": 0.5401,
+      "num_input_tokens_seen": 39845888,
+      "step": 19
+    },
+    {
+      "epoch": 0.2888086642599278,
+      "grad_norm": 0.04852156713604927,
+      "learning_rate": 4.033166875709291e-05,
+      "loss": 0.575,
+      "num_input_tokens_seen": 41943040,
+      "step": 20
+    },
+    {
+      "epoch": 0.30324909747292417,
+      "grad_norm": 0.046296387910842896,
+      "learning_rate": 3.941700805287168e-05,
+      "loss": 0.5398,
+      "num_input_tokens_seen": 44040192,
+      "step": 21
+    },
+    {
+      "epoch": 0.3176895306859206,
+      "grad_norm": 0.04640813171863556,
+      "learning_rate": 3.8472465897889394e-05,
+      "loss": 0.5389,
+      "num_input_tokens_seen": 46137344,
+      "step": 22
+    },
+    {
+      "epoch": 0.33212996389891697,
+      "grad_norm": 0.043006811290979385,
+      "learning_rate": 3.7500000000000003e-05,
+      "loss": 0.5451,
+      "num_input_tokens_seen": 48234496,
+      "step": 23
+    },
+    {
+      "epoch": 0.34657039711191334,
+      "grad_norm": 0.04406141862273216,
+      "learning_rate": 3.6501625943278805e-05,
+      "loss": 0.5558,
+      "num_input_tokens_seen": 50331648,
+      "step": 24
+    },
+    {
+      "epoch": 0.36101083032490977,
+      "grad_norm": 0.03994145616889,
+      "learning_rate": 3.547941301041661e-05,
+      "loss": 0.5403,
+      "num_input_tokens_seen": 52428800,
+      "step": 25
+    },
+    {
+      "epoch": 0.37545126353790614,
+      "grad_norm": 0.039348017424345016,
+      "learning_rate": 3.443547989381536e-05,
+      "loss": 0.5405,
+      "num_input_tokens_seen": 54525952,
+      "step": 26
+    },
+    {
+      "epoch": 0.3898916967509025,
+      "grad_norm": 0.03861572593450546,
+      "learning_rate": 3.3371990304274656e-05,
+      "loss": 0.5577,
+      "num_input_tokens_seen": 56623104,
+      "step": 27
+    },
+    {
+      "epoch": 0.4043321299638989,
+      "grad_norm": 0.03878667205572128,
+      "learning_rate": 3.2291148486370626e-05,
+      "loss": 0.5345,
+      "num_input_tokens_seen": 58720256,
+      "step": 28
+    },
+    {
+      "epoch": 0.4187725631768953,
+      "grad_norm": 0.03664080426096916,
+      "learning_rate": 3.11951946498225e-05,
+      "loss": 0.5484,
+      "num_input_tokens_seen": 60817408,
+      "step": 29
+    },
+    {
+      "epoch": 0.4332129963898917,
+      "grad_norm": 0.036824408918619156,
+      "learning_rate": 3.008640032631585e-05,
+      "loss": 0.5485,
+      "num_input_tokens_seen": 62914560,
+      "step": 30
+    },
+    {
+      "epoch": 0.44765342960288806,
+      "grad_norm": 0.037150438874959946,
+      "learning_rate": 2.8967063661406285e-05,
+      "loss": 0.5299,
+      "num_input_tokens_seen": 65011712,
+      "step": 31
+    },
+    {
+      "epoch": 0.4620938628158845,
+      "grad_norm": 0.03484778478741646,
+      "learning_rate": 2.7839504651261872e-05,
+      "loss": 0.539,
+      "num_input_tokens_seen": 67108864,
+      "step": 32
+    },
+    {
+      "epoch": 0.47653429602888087,
+      "grad_norm": 0.035446375608444214,
+      "learning_rate": 2.6706060334116777e-05,
+      "loss": 0.5248,
+      "num_input_tokens_seen": 69206016,
+      "step": 33
+    },
+    {
+      "epoch": 0.49097472924187724,
+      "grad_norm": 0.0350475013256073,
+      "learning_rate": 2.556907994640264e-05,
+      "loss": 0.5342,
+      "num_input_tokens_seen": 71303168,
+      "step": 34
+    },
+    {
+      "epoch": 0.5054151624548736,
+      "grad_norm": 0.036621786653995514,
+      "learning_rate": 2.4430920053597356e-05,
+      "loss": 0.5431,
+      "num_input_tokens_seen": 73400320,
+      "step": 35
+    },
+    {
+      "epoch": 0.51985559566787,
+      "grad_norm": 0.0347721092402935,
+      "learning_rate": 2.329393966588323e-05,
+      "loss": 0.5471,
+      "num_input_tokens_seen": 75497472,
+      "step": 36
+    },
+    {
+      "epoch": 0.5342960288808665,
+      "grad_norm": 0.03457929939031601,
+      "learning_rate": 2.2160495348738123e-05,
+      "loss": 0.542,
+      "num_input_tokens_seen": 77594624,
+      "step": 37
+    },
+    {
+      "epoch": 0.5487364620938628,
+      "grad_norm": 0.035683248192071915,
+      "learning_rate": 2.1032936338593718e-05,
+      "loss": 0.542,
+      "num_input_tokens_seen": 79691776,
+      "step": 38
+    },
+    {
+      "epoch": 0.5631768953068592,
+      "grad_norm": 0.03531257063150406,
+      "learning_rate": 1.991359967368416e-05,
+      "loss": 0.542,
+      "num_input_tokens_seen": 81788928,
+      "step": 39
+    },
+    {
+      "epoch": 0.5776173285198556,
+      "grad_norm": 0.034907545894384384,
+      "learning_rate": 1.8804805350177505e-05,
+      "loss": 0.5425,
+      "num_input_tokens_seen": 83886080,
+      "step": 40
+    },
+    {
+      "epoch": 0.592057761732852,
+      "grad_norm": 0.03420661389827728,
+      "learning_rate": 1.7708851513629377e-05,
+      "loss": 0.5482,
+      "num_input_tokens_seen": 85983232,
+      "step": 41
+    },
+    {
+      "epoch": 0.6064981949458483,
+      "grad_norm": 0.03401198983192444,
+      "learning_rate": 1.6628009695725346e-05,
+      "loss": 0.5598,
+      "num_input_tokens_seen": 88080384,
+      "step": 42
+    },
+    {
+      "epoch": 0.6209386281588448,
+      "grad_norm": 0.03434673324227333,
+      "learning_rate": 1.5564520106184644e-05,
+      "loss": 0.5402,
+      "num_input_tokens_seen": 90177536,
+      "step": 43
+    },
+    {
+      "epoch": 0.6353790613718412,
+      "grad_norm": 0.033791348338127136,
+      "learning_rate": 1.4520586989583406e-05,
+      "loss": 0.5413,
+      "num_input_tokens_seen": 92274688,
+      "step": 44
+    },
+    {
+      "epoch": 0.6498194945848376,
+      "grad_norm": 0.03330031782388687,
+      "learning_rate": 1.3498374056721197e-05,
+      "loss": 0.556,
+      "num_input_tokens_seen": 94371840,
+      "step": 45
+    },
+    {
+      "epoch": 0.6642599277978339,
+      "grad_norm": 0.03298752009868622,
+      "learning_rate": 1.2500000000000006e-05,
+      "loss": 0.5341,
+      "num_input_tokens_seen": 96468992,
+      "step": 46
+    },
+    {
+      "epoch": 0.6787003610108303,
+      "grad_norm": 0.03363870084285736,
+      "learning_rate": 1.1527534102110612e-05,
+      "loss": 0.5436,
+      "num_input_tokens_seen": 98566144,
+      "step": 47
+    },
+    {
+      "epoch": 0.6931407942238267,
+      "grad_norm": 0.032934173941612244,
+      "learning_rate": 1.0582991947128324e-05,
+      "loss": 0.5393,
+      "num_input_tokens_seen": 100663296,
+      "step": 48
+    },
+    {
+      "epoch": 0.7075812274368231,
+      "grad_norm": 0.03438662365078926,
+      "learning_rate": 9.668331242907089e-06,
+      "loss": 0.5598,
+      "num_input_tokens_seen": 102760448,
+      "step": 49
+    },
+    {
+      "epoch": 0.7220216606498195,
+      "grad_norm": 0.03351249918341637,
+      "learning_rate": 8.785447763431101e-06,
+      "loss": 0.5329,
+      "num_input_tokens_seen": 104857600,
+      "step": 50
+    },
+    {
+      "epoch": 0.7364620938628159,
+      "grad_norm": 0.0330129936337471,
+      "learning_rate": 7.936171419533653e-06,
+      "loss": 0.5384,
+      "num_input_tokens_seen": 106954752,
+      "step": 51
+    },
+    {
+      "epoch": 0.7509025270758123,
+      "grad_norm": 0.03433903306722641,
+      "learning_rate": 7.122262466127514e-06,
+      "loss": 0.5447,
+      "num_input_tokens_seen": 109051904,
+      "step": 52
+    },
+    {
+      "epoch": 0.7653429602888087,
+      "grad_norm": 0.03314002603292465,
+      "learning_rate": 6.3454078538078635e-06,
+      "loss": 0.5291,
+      "num_input_tokens_seen": 111149056,
+      "step": 53
+    },
+    {
+      "epoch": 0.779783393501805,
+      "grad_norm": 0.032993488013744354,
+      "learning_rate": 5.607217732389503e-06,
+      "loss": 0.5258,
+      "num_input_tokens_seen": 113246208,
+      "step": 54
+    },
+    {
+      "epoch": 0.7942238267148014,
+      "grad_norm": 0.032629404217004776,
+      "learning_rate": 4.9092221136255444e-06,
+      "loss": 0.5508,
+      "num_input_tokens_seen": 115343360,
+      "step": 55
+    },
+    {
+      "epoch": 0.8086642599277978,
+      "grad_norm": 0.03205695375800133,
+      "learning_rate": 4.252867700024374e-06,
+      "loss": 0.5439,
+      "num_input_tokens_seen": 117440512,
+      "step": 56
+    },
+    {
+      "epoch": 0.8231046931407943,
+      "grad_norm": 0.03285016119480133,
+      "learning_rate": 3.6395148863377858e-06,
+      "loss": 0.5261,
+      "num_input_tokens_seen": 119537664,
+      "step": 57
+    },
+    {
+      "epoch": 0.8375451263537906,
+      "grad_norm": 0.035063523799180984,
+      "learning_rate": 3.0704349399351435e-06,
+      "loss": 0.5518,
+      "num_input_tokens_seen": 121634816,
+      "step": 58
+    },
+    {
+      "epoch": 0.851985559566787,
+      "grad_norm": 0.03116844967007637,
+      "learning_rate": 2.5468073659076e-06,
+      "loss": 0.5444,
+      "num_input_tokens_seen": 123731968,
+      "step": 59
+    },
+    {
+      "epoch": 0.8664259927797834,
+      "grad_norm": 0.03255166485905647,
+      "learning_rate": 2.0697174623636794e-06,
+      "loss": 0.5469,
+      "num_input_tokens_seen": 125829120,
+      "step": 60
+    },
+    {
+      "epoch": 0.8808664259927798,
+      "grad_norm": 0.03249025344848633,
+      "learning_rate": 1.6401540709832242e-06,
+      "loss": 0.5335,
+      "num_input_tokens_seen": 127926272,
+      "step": 61
+    },
+    {
+      "epoch": 0.8953068592057761,
+      "grad_norm": 0.03506583720445633,
+      "learning_rate": 1.2590075274920205e-06,
+      "loss": 0.5367,
+      "num_input_tokens_seen": 130023424,
+      "step": 62
+    },
+    {
+      "epoch": 0.9097472924187726,
+      "grad_norm": 0.03293128311634064,
+      "learning_rate": 9.270678163050217e-07,
+      "loss": 0.551,
+      "num_input_tokens_seen": 132120576,
+      "step": 63
+    },
+    {
+      "epoch": 0.924187725631769,
+      "grad_norm": 0.03347219526767731,
+      "learning_rate": 6.450229331630253e-07,
+      "loss": 0.5597,
+      "num_input_tokens_seen": 134217728,
+      "step": 64
+    },
+    {
+      "epoch": 0.9386281588447654,
+      "grad_norm": 0.03135489672422409,
+      "learning_rate": 4.134574591564494e-07,
+      "loss": 0.5369,
+      "num_input_tokens_seen": 136314880,
+      "step": 65
+    },
+    {
+      "epoch": 0.9530685920577617,
+      "grad_norm": 0.03153960779309273,
+      "learning_rate": 2.3285134909173112e-07,
+      "loss": 0.5451,
+      "num_input_tokens_seen": 138412032,
+      "step": 66
+    },
+    {
+      "epoch": 0.9675090252707581,
+      "grad_norm": 0.03191647306084633,
+      "learning_rate": 1.0357893671171792e-07,
+      "loss": 0.5237,
+      "num_input_tokens_seen": 140509184,
+      "step": 67
+    },
+    {
+      "epoch": 0.9819494584837545,
+      "grad_norm": 0.03295959159731865,
+      "learning_rate": 2.590815883181108e-08,
+      "loss": 0.5604,
+      "num_input_tokens_seen": 142606336,
+      "step": 68
+    },
+    {
+      "epoch": 0.9963898916967509,
+      "grad_norm": 0.03191553056240082,
+      "learning_rate": 0.0,
+      "loss": 0.534,
+      "num_input_tokens_seen": 144703488,
+      "step": 69
+    },
+    {
+      "epoch": 0.9963898916967509,
+      "num_input_tokens_seen": 144703488,
+      "step": 69,
+      "total_flos": 5.635565866281075e+18,
+      "train_loss": 0.5665888682655666,
+      "train_runtime": 10913.0245,
+      "train_samples_per_second": 3.247,
+      "train_steps_per_second": 0.006
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 69,
+  "num_input_tokens_seen": 144703488,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.635565866281075e+18,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f89cedb3b4452c319d87072cbb65858846722f74c7a4b37d475becf73b8ac27
+size 5688

training_args.yaml

@@ -0,0 +1,39 @@
+apollo_rank: 256
+apollo_scale: 1
+apollo_target: all
+apollo_update_interval: 200
+bf16: true
+cutoff_len: 4096
+dataset: codes3_query_filtered_330k_nlx
+dataset_dir: data
+ddp_timeout: 180000000
+do_train: true
+enable_liger_kernel: true
+finetuning_type: freeze
+flash_attn: auto
+freeze_trainable_layers: 2
+freeze_trainable_modules: all
+gradient_accumulation_steps: 8
+include_num_input_tokens_seen: true
+learning_rate: 5.0e-05
+logging_steps: 1
+lr_scheduler_type: cosine
+max_grad_norm: 1.0
+max_samples: 50000000
+model_name_or_path: deepseek-ai/deepseek-coder-7b-instruct-v1.5
+neat_packing: true
+num_train_epochs: 1.0
+output_dir: saves/DeepSeek-Coder-7B-Instruct/freeze/deepseek-nlx-330k
+packing: true
+per_device_train_batch_size: 16
+plot_loss: true
+preprocessing_num_workers: 16
+report_to: none
+rope_scaling: llama3
+save_steps: 500
+stage: sft
+template: deepseekcoder
+trust_remote_code: true
+use_apollo: true
+use_llama_pro: true
+warmup_steps: 0