Upload config.yaml with huggingface_hub

config.yaml

@@ -0,0 +1,154 @@
+name: comma_v0p1_yolooooo
+dump_dir: /fsx/craffel/lingua_logs/comma_v0p1/
+seed: 777
+grad_acc_steps: 4
+gc_collect_freq: 1000
+probe_freq: null
+steps: 500000
+data:
+  root_dir: /scratch/craffel/lingua/data/
+  sources:
+    peS2o: 0.274065475510351
+    stackexchange: 0.134617935796937
+    stackv2_edu: 0.127770669195666
+    cccc: 0.0871992270000557
+    wikimedia: 0.0861800315862719
+    github_archive: 0.0606452345122248
+    uspto: 0.0413469377516883
+    pubmed: 0.0367902799837971
+    arxiv_papers: 0.0292395449667613
+    caselaw_access_project: 0.0193875362722656
+    wikiteam: 0.0137485410839637
+    doab: 0.0180439781895451
+    uk_hansard: 0.0144498535570883
+    pre_1929_books: 0.0115755547988338
+    ubuntu_irc: 0.00794254267719456
+    regulations: 0.00762583706405442
+    data_provenance_initiative: 0.00512264496834867
+    project_gutenberg: 0.00502100654070129
+    youtube: 0.00465917165839394
+    arxiv_abstracts: 0.00359635066160403
+    stackv2_html: 0.00225924255952781
+    usgpo: 0.00226024581728848
+    library_of_congress: 0.00222469340783564
+    biodiversity_heritage_library: 0.00221737524370278
+    pressbooks: 0.000865101033213598
+    libretexts: 0.00054149556727006
+    news: 0.000372716196818104
+    foodista: 0.000125363443065615
+    oercommons: 7.78696843693821e-05
+    python_enhancement_proposals: 1.69983991984805e-05
+    public_domain_review: 1.05448719635173e-05
+  batch_size: 2
+  seq_len: 4096
+  n_views: 2
+  seed: 42
+  add_bos: true
+  add_eos: true
+  load_async: true
+  prefetch_size: 4096
+  tokenizer:
+    name: tiktoken
+    path: /fsx/craffel/lingua/tokenizers/common-pile-tokenizer.tiktoken
+optim:
+  lr: 0.001
+  weight_decay: 0.2
+  epsilon: 1.0e-08
+  beta1: 0.9
+  beta2: 0.95
+  clip: 1.0
+  scheduler: cosine
+  warmup: 2000
+  lr_min_ratio: 1.0e-06
+  cycle_length: 1.0
+  cosine_theta: 1.0
+  annealing_step: 1000
+  decay_fraction: 0.1
+  exp_factor: 0.5
+model:
+  dim: 4096
+  n_layers: 32
+  head_dim: null
+  n_heads: 32
+  n_kv_heads: null
+  ffn_dim_multiplier: 1.0
+  multiple_of: 256
+  norm_eps: 1.0e-05
+  rope_theta: 100000.0
+  init_base_std: null
+  init_std_factor: disabled
+  max_seqlen: 4096
+  seed: 42
+  vocab_size: 64256
+  weight_tying: false
+  sliding_window: null
+distributed:
+  dp_shard: 1
+  dp_replicate: 64
+  tp_size: 1
+  selective_activation_checkpointing: false
+  compile: true
+  fsdp_type: full_shard
+  model_dtype: bf16
+  float8_recipe: null
+  float8_filter: layers\.[0-9]+\.
+  matmul_allow_tf32: false
+  detect_anomaly: false
+  compile_cache_size_limit: 8
+  spawn_method: forkserver
+env:
+  MKL_SERVICE_FORCE_INTEL: GNU
+  OMP_NUM_THREADS: '1'
+  MKL_NUM_THREADS: '1'
+  ENABLE_INTRA_NODE_COMM: '1'
+  TORCH_NCCL_AVOID_RECORD_STREAMS: '1'
+  NCCL_IB_TIMEOUT: '22'
+  NCCL_DEBUG: INFO
+  TORCH_NCCL_ASYNC_ERROR_HANDLING: '1'
+checkpoint:
+  dump:
+    every: 10000
+    keep: -1
+  eval:
+    every: 2000
+    keep: 3
+  path: /fsx/craffel/lingua_logs/comma_v0p1/checkpoints
+  init_ckpt_path: null
+  continue_training_from_init: false
+profiling:
+  run: true
+  trace_folder: profiling
+  mem_warmup: 0
+  mem_steps: 4
+  profile_warmup: 100
+  profile_steps: 4
+logging:
+  freq: 1
+  acc_freq: null
+  wandb: null
+async_eval_gpus: 8
+eval:
+  harness:
+    tasks:
+    - hellaswag
+    - task: boolq
+      dataset_kwargs:
+        trust_remote_code: true
+    - piqa
+    - task: social_iqa
+      dataset_kwargs:
+        trust_remote_code: true
+    - winogrande
+    - openbookqa
+    - arc_easy
+    - arc_challenge
+    - race
+    - commonsense_qa
+    - task: copa
+      dataset_kwargs:
+        trust_remote_code: true
+    - mmlu
+    - mmlu_pro
+  generator:
+    max_tokens: 8192
+    dtype: bf16