| { |
| "batch_size": 18, |
| "num_epochs": 4, |
| "lr": 5e-5, |
| "seq_len": 256, |
| "d_model": 256, |
| "N": 4, |
| "h": 4, |
| "train": "dataset/openweb_fine.jsonl", |
| "test": "dataset/openweb_fine.jsonl", |
| "d_ff": 1024, |
| "dropout": 0.1, |
| "model_folder": "openweb2", |
| "model_basename": "openweb2-", |
| "preload": "03", |
| "tokenizer_file": "openweb2.tokenizer.json", |
| "experiment_name": "runs/openweb2", |
| "dataset": "dataset/dataset_general.jsonl", |
| "loss_file": "openweb2/losses.jsonl", |
| "fine_dataset": "dataset/fine_tune.jsonl", |
| "fine_epochs": 0 |
| } |
