pretrain core

README.md

@@ -53,8 +53,9 @@ time python -B prepare_core_datasets.py
 ```
 
 ```
-i=0, min_len=0, max_len=1048576, block_size=4097, chunk_size=16388000, len(dataset)=1567386, len(dataset) * block_size=6421580442
-Total number of tokens in the optimized dataset '../core-data-0-0-1048576-4097-4000' is 6421580442
+i=0, min_len=0, max_len=1073741824, block_size=4097, chunk_size=16388000, len(dataset)=1287403, len(dataset) * block_size=5274490091
+Total number of tokens in the optimized dataset '../core-data-0-0-1073741824-4097-4000' is 5274490091
+
 ```
 
 ```bash

scripts/pretrain_core_model.yaml

@@ -46,7 +46,7 @@ data:
   class_path: LitData
 
   init_args:
-    data_path: "../core-data-0-0-1048576-4097-4000/"
+    data_path: "../core-data-0-0-1073741824-4097-4000/"
     num_workers: 32
 
 # Training-related arguments. See ``litgpt.args.TrainArgs`` for details
@@ -70,7 +70,7 @@ train:
   epochs:
 
   # Total number of tokens to train on (type: Optional[int], default: 3000000000000)
-  max_tokens: 6421580442
+  max_tokens: 5274490091
 
   # Limits the number of optimizer steps to run. (type: Optional[int], default: null)
   max_steps: