| data: | |
| eval_path: ./data/babylm_dev_clean | |
| eval_samples: 256 | |
| seq_length: 128 | |
| tokenizer_path: ./models/gpt-clean-16000.json | |
| train_path: ./data/babylm_10M_clean | |
| logging: | |
| output_dir: ./models/ | |
| project: babylm-10M | |
| wandb: false | |
| model: | |
| hidden_size: 192 | |
| intermediate_size: 768 | |
| n_head: 6 | |
| n_layer: 6 | |
| name: Llama-10M-1M | |
| tie_word_embeddings: false | |
| type: Llama | |
| training: | |
| batch_size: 32 | |
| fp16: true | |
| gradient_accumulation_steps: 4 | |
| lr: 3e-4 | |
| num_epochs: 2 | |
| warmup_steps: 100 | |