pretrain core 0

README.md

@@ -53,7 +53,12 @@ time python -B prepare_core_datasets.py
 ```
 
 ```
-# ...
+i=0, min_len=0, max_len=1048576, block_size=2049, chunk_size=16392000, len(dataset)=3134311, len(dataset) * block_size=6422203239
+Total number of tokens in the optimized dataset '../core-data-0-0-1048576-2049-8000' is 6422203239
+i=1, min_len=2049, max_len=8193, block_size=8193, chunk_size=16386000, len(dataset)=179944, len(dataset) * block_size=1474281192
+Total number of tokens in the optimized dataset '../core-data-1-2049-8193-8193-2000' is 1474281192
+i=2, min_len=8193, max_len=1048577, block_size=32769, chunk_size=16384500, len(dataset)=48261, len(dataset) * block_size=1581464709
+Total number of tokens in the optimized dataset '../core-data-2-8193-1048577-32769-500' is 1581464709
 ```
 
 ```bash

scripts/pretrain-core-model-0.yaml

@@ -46,7 +46,7 @@ data:
   class_path: LitData
 
   init_args:
-    data_path: "../core-data-0-8192-2000/"
+    data_path: "../core-data-0-0-1048576-2049-8000/"
     num_workers: 32
 
 # Training-related arguments. See ``litgpt.args.TrainArgs`` for details
@@ -74,7 +74,7 @@ train:
   epochs:
 
   # Total number of tokens to train on (type: Optional[int], default: 3000000000000)
-  max_tokens: ???
+  max_tokens: 6422203239
 
   # Limits the number of optimizer steps to run. (type: Optional[int], default: null)
   max_steps: