pretrain core 4

scripts/pretrain_core_model_4.yaml

@@ -59,7 +59,8 @@ train:
 
   # Number of samples between optimizer steps across data-parallel ranks (type: int, default: 512)
   # global_batch_size: 512
-  global_batch_size: 256
+  # global_batch_size: 256
+  global_batch_size: 250
 
   # Number of samples per data-parallel rank (type: int, default: 4)
   micro_batch_size: 1