pico-decoder-small-1 trained to 50k steps

README.md

@@ -13,6 +13,8 @@ pipeline_tag: text-generation
 
 **pico-decoder-small** is a 65M parameter model in the `pico-decoder` suite — a lightweight, LLaMA-style decoder-only transformer trained from scratch using [`pico-train`](https://github.com/pico-lm/pico-train). It is designed for transparent and reproducible research into the learning dynamics of language models, and is fully compatible with the `pico-analyze` toolkit for detailed interpretability analysis.
 
+> NOTE: The `pico-decoder-small-1` branch contains the full commit history for the training run. 
+
 ## 🔧 Model Details
 
 | Field               | Value                              |

config.json

@@ -0,0 +1,22 @@
+{
+  "activation_hidden_dim": 1536,
+  "architectures": [
+    "PicoHF"
+  ],
+  "attention_n_heads": 12,
+  "attention_n_kv_heads": 4,
+  "auto_map": {
+    "AutoConfig": "pico.PicoHFConfig",
+    "AutoModelForCausalLM": "pico.PicoHF"
+  },
+  "batch_size": 1024,
+  "d_model": 384,
+  "max_seq_len": 2048,
+  "model_type": "pico",
+  "n_layers": 12,
+  "norm_eps": 1e-06,
+  "position_emb_theta": 10000.0,
+  "torch_dtype": "float32",
+  "transformers_version": "4.48.1",
+  "vocab_size": 50304
+}

eval_results/step_0.json

@@ -0,0 +1 @@
+{"paloma": 59235.520549324916}

eval_results/step_1000.json

@@ -0,0 +1 @@
+{"paloma": 792.1833685393118}

eval_results/step_10000.json

@@ -0,0 +1 @@
+{"paloma": 65.94262431332459}

eval_results/step_11000.json

@@ -0,0 +1 @@
+{"paloma": 64.34633452908918}

eval_results/step_12000.json

@@ -0,0 +1 @@
+{"paloma": 62.59192442304166}

eval_results/step_13000.json

@@ -0,0 +1 @@
+{"paloma": 61.176598006341514}

eval_results/step_14000.json

@@ -0,0 +1 @@
+{"paloma": 60.127189135634524}

eval_results/step_15000.json

@@ -0,0 +1 @@
+{"paloma": 59.05735223143773}

eval_results/step_16000.json

@@ -0,0 +1 @@
+{"paloma": 58.34294593874171}

eval_results/step_17000.json

@@ -0,0 +1 @@
+{"paloma": 57.50681012357984}

eval_results/step_18000.json

@@ -0,0 +1 @@
+{"paloma": 56.75063220922955}

eval_results/step_19000.json

@@ -0,0 +1 @@
+{"paloma": 56.22409683063055}

eval_results/step_2000.json

@@ -0,0 +1 @@
+{"paloma": 430.54463134327}

eval_results/step_20000.json

@@ -0,0 +1 @@
+{"paloma": 55.70834681912997}

eval_results/step_21000.json

@@ -0,0 +1 @@
+{"paloma": 55.09185880999947}

eval_results/step_22000.json

@@ -0,0 +1 @@
+{"paloma": 54.68077409824012}

eval_results/step_23000.json

@@ -0,0 +1 @@
+{"paloma": 54.099016925931394}

eval_results/step_24000.json

@@ -0,0 +1 @@
+{"paloma": 53.69616283332014}

eval_results/step_25000.json

@@ -0,0 +1 @@
+{"paloma": 53.325224398403634}

eval_results/step_26000.json

@@ -0,0 +1 @@
+{"paloma": 52.88207766685752}

eval_results/step_27000.json

@@ -0,0 +1 @@
+{"paloma": 52.41606502532959}

eval_results/step_28000.json

@@ -0,0 +1 @@
+{"paloma": 52.10107550388429}

eval_results/step_29000.json

@@ -0,0 +1 @@
+{"paloma": 51.815027207946116}

eval_results/step_3000.json

@@ -0,0 +1 @@
+{"paloma": 251.38611673594352}

eval_results/step_30000.json

@@ -0,0 +1 @@
+{"paloma": 51.49154515075351}

eval_results/step_31000.json

@@ -0,0 +1 @@
+{"paloma": 51.0821376223182}

eval_results/step_32000.json

@@ -0,0 +1 @@
+{"paloma": 50.92298436588526}

eval_results/step_33000.json

@@ -0,0 +1 @@
+{"paloma": 50.645718124851534}

eval_results/step_34000.json

@@ -0,0 +1 @@
+{"paloma": 50.21418930977479}

eval_results/step_35000.json

@@ -0,0 +1 @@
+{"paloma": 50.065432297311176}

eval_results/step_36000.json

@@ -0,0 +1 @@
+{"paloma": 49.722077291269336}

eval_results/step_37000.json

@@ -0,0 +1 @@
+{"paloma": 49.635000252972915}

eval_results/step_38000.json

@@ -0,0 +1 @@
+{"paloma": 49.49005253331586}

eval_results/step_39000.json

@@ -0,0 +1 @@
+{"paloma": 49.15703350899527}

eval_results/step_4000.json

@@ -0,0 +1 @@
+{"paloma": 140.36928879923937}

eval_results/step_40000.json

@@ -0,0 +1 @@
+{"paloma": 48.99870226881645}

eval_results/step_41000.json

@@ -0,0 +1 @@
+{"paloma": 48.77364599197999}

eval_results/step_42000.json

@@ -0,0 +1 @@
+{"paloma": 48.68095632488304}

eval_results/step_43000.json

@@ -0,0 +1 @@
+{"paloma": 48.350308783959846}

eval_results/step_44000.json

@@ -0,0 +1 @@
+{"paloma": 48.223110499282335}

eval_results/step_45000.json

@@ -0,0 +1 @@
+{"paloma": 48.15096046201859}

eval_results/step_46000.json

@@ -0,0 +1 @@
+{"paloma": 48.06621652671269}

eval_results/step_47000.json

@@ -0,0 +1 @@
+{"paloma": 47.83704719643144}

eval_results/step_48000.json

@@ -0,0 +1 @@
+{"paloma": 47.751127573505094}

eval_results/step_49000.json

@@ -0,0 +1 @@
+{"paloma": 47.55591929922536}

eval_results/step_5000.json

@@ -0,0 +1 @@
+{"paloma": 89.37810033688562}

eval_results/step_50000.json

@@ -0,0 +1 @@
+{"paloma": 47.414057051013984}

eval_results/step_6000.json

@@ -0,0 +1 @@
+{"paloma": 80.24826397455527}