name: comma_v0p1_yolooooo
dump_dir: /fsx/craffel/lingua_logs/comma_v0p1/
seed: 777
grad_acc_steps: 4
gc_collect_freq: 1000
probe_freq: null
steps: 500000
data:
  root_dir: /scratch/craffel/lingua/data/
  sources:
    peS2o: 0.274065475510351
    stackexchange: 0.134617935796937
    stackv2_edu: 0.127770669195666
    cccc: 0.0871992270000557
    wikimedia: 0.0861800315862719
    github_archive: 0.0606452345122248
    uspto: 0.0413469377516883
    pubmed: 0.0367902799837971
    arxiv_papers: 0.0292395449667613
    caselaw_access_project: 0.0193875362722656
    wikiteam: 0.0137485410839637
    doab: 0.0180439781895451
    uk_hansard: 0.0144498535570883
    pre_1929_books: 0.0115755547988338
    ubuntu_irc: 0.00794254267719456
    regulations: 0.00762583706405442
    data_provenance_initiative: 0.00512264496834867
    project_gutenberg: 0.00502100654070129
    youtube: 0.00465917165839394
    arxiv_abstracts: 0.00359635066160403
    stackv2_html: 0.00225924255952781
    usgpo: 0.00226024581728848
    library_of_congress: 0.00222469340783564
    biodiversity_heritage_library: 0.00221737524370278
    pressbooks: 0.000865101033213598
    libretexts: 0.00054149556727006
    news: 0.000372716196818104
    foodista: 0.000125363443065615
    oercommons: 7.78696843693821e-05
    python_enhancement_proposals: 1.69983991984805e-05
    public_domain_review: 1.05448719635173e-05
  batch_size: 2
  seq_len: 4096
  n_views: 2
  seed: 42
  add_bos: true
  add_eos: true
  load_async: true
  prefetch_size: 4096
  tokenizer:
    name: tiktoken
    path: /fsx/craffel/lingua/tokenizers/common-pile-tokenizer.tiktoken
optim:
  lr: 0.001
  weight_decay: 0.2
  epsilon: 1.0e-08
  beta1: 0.9
  beta2: 0.95
  clip: 1.0
  scheduler: cosine
  warmup: 2000
  lr_min_ratio: 1.0e-06
  cycle_length: 1.0
  cosine_theta: 1.0
  annealing_step: 1000
  decay_fraction: 0.1
  exp_factor: 0.5
model:
  dim: 4096
  n_layers: 32
  head_dim: null
  n_heads: 32
  n_kv_heads: null
  ffn_dim_multiplier: 1.0
  multiple_of: 256
  norm_eps: 1.0e-05
  rope_theta: 100000.0
  init_base_std: null
  init_std_factor: disabled
  max_seqlen: 4096
  seed: 42
  vocab_size: 64256
  weight_tying: false
  sliding_window: null
distributed:
  dp_shard: 1
  dp_replicate: 64
  tp_size: 1
  selective_activation_checkpointing: false
  compile: true
  fsdp_type: full_shard
  model_dtype: bf16
  float8_recipe: null
  float8_filter: layers\.[0-9]+\.
  matmul_allow_tf32: false
  detect_anomaly: false
  compile_cache_size_limit: 8
  spawn_method: forkserver
env:
  MKL_SERVICE_FORCE_INTEL: GNU
  OMP_NUM_THREADS: '1'
  MKL_NUM_THREADS: '1'
  ENABLE_INTRA_NODE_COMM: '1'
  TORCH_NCCL_AVOID_RECORD_STREAMS: '1'
  NCCL_IB_TIMEOUT: '22'
  NCCL_DEBUG: INFO
  TORCH_NCCL_ASYNC_ERROR_HANDLING: '1'
checkpoint:
  dump:
    every: 10000
    keep: -1
  eval:
    every: 2000
    keep: 3
  path: /fsx/craffel/lingua_logs/comma_v0p1/checkpoints
  init_ckpt_path: null
  continue_training_from_init: false
profiling:
  run: true
  trace_folder: profiling
  mem_warmup: 0
  mem_steps: 4
  profile_warmup: 100
  profile_steps: 4
logging:
  freq: 1
  acc_freq: null
  wandb: null
async_eval_gpus: 8
eval:
  harness:
    tasks:
    - hellaswag
    - task: boolq
      dataset_kwargs:
        trust_remote_code: true
    - piqa
    - task: social_iqa
      dataset_kwargs:
        trust_remote_code: true
    - winogrande
    - openbookqa
    - arc_easy
    - arc_challenge
    - race
    - commonsense_qa
    - task: copa
      dataset_kwargs:
        trust_remote_code: true
    - mmlu
    - mmlu_pro
  generator:
    max_tokens: 8192
    dtype: bf16