Upload add_sub_baseline_100K

add_sub_baseline_100K/config.json

@@ -0,0 +1,38 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "intermediate_size": 2048,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 3,
+  "num_key_value_heads": 4,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151645
+}

add_sub_baseline_100K/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_baseline_100K/metrics.json

@@ -0,0 +1,385 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      200,
+      250,
+      300,
+      350,
+      400,
+      450,
+      500,
+      550,
+      600,
+      650,
+      700,
+      750,
+      800,
+      850,
+      900,
+      950,
+      1000,
+      1050,
+      1100,
+      1150,
+      1200,
+      1250,
+      1300,
+      1350,
+      1400,
+      1450,
+      1500,
+      1550,
+      1600,
+      1650,
+      1700,
+      1750,
+      1800,
+      1850,
+      1900,
+      1950,
+      2000,
+      2050,
+      2100,
+      2150,
+      2200,
+      2250,
+      2300,
+      2350,
+      2400,
+      2450,
+      2500,
+      2550,
+      2600,
+      2650,
+      2700,
+      2750,
+      2800,
+      2850,
+      2900,
+      2950,
+      3000,
+      3050,
+      3100,
+      3150,
+      3200,
+      3250,
+      3300,
+      3350,
+      3400,
+      3450,
+      3500,
+      3550,
+      3600,
+      3650,
+      3700,
+      3750,
+      3800,
+      3850,
+      3900,
+      3950,
+      4000,
+      4050,
+      4100,
+      4150,
+      4200,
+      4250,
+      4300,
+      4350,
+      4400,
+      4450,
+      4500,
+      4550,
+      4600,
+      4650
+    ],
+    "loss": [
+      10.446642875671387,
+      8.204933166503906,
+      7.287497043609619,
+      6.563387393951416,
+      5.592346668243408,
+      4.6155476570129395,
+      3.5229132175445557,
+      2.671051025390625,
+      2.1285922527313232,
+      1.856718897819519,
+      1.7758562564849854,
+      1.636636734008789,
+      1.4528383016586304,
+      0.7716706395149231,
+      0.47104448080062866,
+      0.30569639801979065,
+      0.2391534149646759,
+      0.17922067642211914,
+      0.15349432826042175,
+      0.1651923954486847,
+      0.17999541759490967,
+      0.19358131289482117,
+      0.16539938747882843,
+      0.1219974234700203,
+      0.08426837623119354,
+      0.07237092405557632,
+      0.048519425094127655,
+      0.10355325788259506,
+      0.051684457808732986,
+      0.09696381539106369,
+      0.04080639407038689,
+      0.036543793976306915,
+      0.02337537333369255,
+      0.023294461891055107,
+      0.027975469827651978,
+      0.03485460206866264,
+      0.01895260438323021,
+      0.014638577587902546,
+      0.020639639347791672,
+      0.010763270780444145,
+      0.007144362665712833,
+      0.024989277124404907,
+      0.01930844970047474,
+      0.023015184327960014,
+      0.0063614509999752045,
+      0.0028356898110359907,
+      0.002119298791512847,
+      0.020854128524661064,
+      0.0018869990017265081,
+      0.00558065390214324,
+      0.006174446549266577,
+      0.006082491483539343,
+      0.0037679767701774836,
+      0.007742547430098057,
+      0.0011493159690871835,
+      0.006048537325114012,
+      0.00800343882292509,
+      0.0007379369926638901,
+      0.0028449923265725374,
+      0.0014030687743797898,
+      0.0009590271511115134,
+      0.0025089881382882595,
+      0.0003829688939731568,
+      0.0003303142439108342,
+      0.0003975001454818994,
+      0.0034398504067212343,
+      0.00010646107693901286,
+      0.00044194384827278554,
+      0.00020321234478615224,
+      0.00010634677892085165,
+      0.00026234029792249203,
+      8.515844820067286e-05,
+      6.779689283575863e-05,
+      0.0002607057977002114,
+      0.00036781796370632946,
+      6.288080476224422e-05,
+      8.045908907661214e-05,
+      0.00011293154238956049,
+      0.0005836685886606574,
+      5.503458669409156e-05,
+      8.547406469006091e-05,
+      7.086647383403033e-05,
+      4.5273562136571854e-05,
+      4.361348692327738e-05,
+      3.827679392998107e-05,
+      4.5121512812329456e-05,
+      5.49159522051923e-05,
+      7.655313675059006e-05,
+      4.943844396620989e-05,
+      3.6672678106697276e-05,
+      4.1609022446209565e-05,
+      7.665202429052442e-05,
+      3.566138548194431e-05
+    ],
+    "base_loss": [
+      10.446642875671387,
+      8.204933166503906,
+      7.287497043609619,
+      6.563387393951416,
+      5.592346668243408,
+      4.6155476570129395,
+      3.5229132175445557,
+      2.671051025390625,
+      2.1285922527313232,
+      1.856718897819519,
+      1.7758562564849854,
+      1.636636734008789,
+      1.4528383016586304,
+      0.7716706395149231,
+      0.47104448080062866,
+      0.30569639801979065,
+      0.2391534149646759,
+      0.17922067642211914,
+      0.15349432826042175,
+      0.1651923954486847,
+      0.17999541759490967,
+      0.19358131289482117,
+      0.16539938747882843,
+      0.1219974234700203,
+      0.08426837623119354,
+      0.07237092405557632,
+      0.048519425094127655,
+      0.10355325788259506,
+      0.051684457808732986,
+      0.09696381539106369,
+      0.04080639407038689,
+      0.036543793976306915,
+      0.02337537333369255,
+      0.023294461891055107,
+      0.027975469827651978,
+      0.03485460206866264,
+      0.01895260438323021,
+      0.014638577587902546,
+      0.020639639347791672,
+      0.010763270780444145,
+      0.007144362665712833,
+      0.024989277124404907,
+      0.01930844970047474,
+      0.023015184327960014,
+      0.0063614509999752045,
+      0.0028356898110359907,
+      0.002119298791512847,
+      0.020854128524661064,
+      0.0018869990017265081,
+      0.00558065390214324,
+      0.006174446549266577,
+      0.006082491483539343,
+      0.0037679767701774836,
+      0.007742547430098057,
+      0.0011493159690871835,
+      0.006048537325114012,
+      0.00800343882292509,
+      0.0007379369926638901,
+      0.0028449923265725374,
+      0.0014030687743797898,
+      0.0009590271511115134,
+      0.0025089881382882595,
+      0.0003829688939731568,
+      0.0003303142439108342,
+      0.0003975001454818994,
+      0.0034398504067212343,
+      0.00010646107693901286,
+      0.00044194384827278554,
+      0.00020321234478615224,
+      0.00010634677892085165,
+      0.00026234029792249203,
+      8.515844820067286e-05,
+      6.779689283575863e-05,
+      0.0002607057977002114,
+      0.00036781796370632946,
+      6.288080476224422e-05,
+      8.045908907661214e-05,
+      0.00011293154238956049,
+      0.0005836685886606574,
+      5.503458669409156e-05,
+      8.547406469006091e-05,
+      7.086647383403033e-05,
+      4.5273562136571854e-05,
+      4.361348692327738e-05,
+      3.827679392998107e-05,
+      4.5121512812329456e-05,
+      5.49159522051923e-05,
+      7.655313675059006e-05,
+      4.943844396620989e-05,
+      3.6672678106697276e-05,
+      4.1609022446209565e-05,
+      7.665202429052442e-05,
+      3.566138548194431e-05
+    ],
+    "lr": [
+      4.183564567769478e-06,
+      8.452508004268944e-06,
+      1.2721451440768411e-05,
+      1.6990394877267876e-05,
+      2.1259338313767342e-05,
+      2.5528281750266808e-05,
+      2.979722518676628e-05,
+      3.406616862326574e-05,
+      3.8335112059765206e-05,
+      4.260405549626468e-05,
+      4.6872998932764145e-05,
+      5.1141942369263614e-05,
+      5.541088580576308e-05,
+      5.9679829242262546e-05,
+      6.394877267876202e-05,
+      6.821771611526148e-05,
+      7.248665955176095e-05,
+      7.675560298826042e-05,
+      7.999798087633018e-05,
+      7.994611226578842e-05,
+      7.982423928087593e-05,
+      7.963257550037617e-05,
+      7.937145680936735e-05,
+      7.904134081059389e-05,
+      7.864280602253109e-05,
+      7.817655086554887e-05,
+      7.764339243795064e-05,
+      7.704426508403295e-05,
+      7.638021875667483e-05,
+      7.565241717732657e-05,
+      7.486213579662251e-05,
+      7.401075955919167e-05,
+      7.309978047658348e-05,
+      7.21307950125619e-05,
+      7.11055012853501e-05,
+      7.002569609172871e-05,
+      6.889327175820302e-05,
+      6.771021282475686e-05,
+      6.647859256700532e-05,
+      6.520056936284096e-05,
+      6.387838290994056e-05,
+      6.251435030076157e-05,
+      6.111086196190657e-05,
+      5.967037746497144e-05,
+      5.819542121621955e-05,
+      5.668857803263459e-05,
+      5.515248861210576e-05,
+      5.358984490568317e-05,
+      5.200338540001364e-05,
+      5.0395890318224263e-05,
+      4.8770176747664026e-05,
+      4.712909370304193e-05,
+      4.547551713361344e-05,
+      4.3812344883164867e-05,
+      4.214249161162829e-05,
+      4.046888368722666e-05,
+      3.879445405810047e-05,
+      3.712213711240329e-05,
+      3.5454863535873805e-05,
+      3.3795555175896186e-05,
+      3.214711992104925e-05,
+      3.051244660511836e-05,
+      2.8894399944499974e-05,
+      2.729581551787133e-05,
+      2.5719494796923085e-05,
+      2.4168200236863477e-05,
+      2.264465043529768e-05,
+      2.1151515367966308e-05,
+      1.969141170969228e-05,
+      1.826689824873599e-05,
+      1.6880471402594932e-05,
+      1.553456084310621e-05,
+      1.4231525238518917e-05,
+      1.2973648119998136e-05,
+      1.1763133879804411e-05,
+      1.0602103908161819e-05,
+      9.492592875584728e-06,
+      8.436545167178147e-06,
+      7.43581147516069e-06,
+      6.492145555581349e-06,
+      5.607201154914275e-06,
+      4.78252911191718e-06,
+      4.019574639832518e-06,
+      3.3196747936942342e-06,
+      2.6840561271785694e-06,
+      2.113832543105203e-06,
+      1.6100033413556416e-06,
+      1.1734514676299224e-06,
+      8.049419661105129e-07,
+      5.051206387450869e-07,
+      2.7451291349783527e-07,
+      1.1352292355256922e-07,
+      2.2432799081339997e-08
+    ]
+  },
+  "final_accuracy": 1.0
+}

add_sub_baseline_100K/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1fa25258e7da94543218ca081ac521cb27604e416139b5a6a65144bb8ba97da
+size 671794850

add_sub_baseline_100K/train_config.json

@@ -0,0 +1,28 @@
+{
+  "mode": "baseline",
+  "ops": "add_sub",
+  "n_digits": 6,
+  "n_layer": 3,
+  "n_head": 4,
+  "n_embd": 512,
+  "abs_vocab": 0,
+  "K": 4,
+  "batch_size": 64,
+  "num_epochs": 3,
+  "dataset_size": 100000,
+  "lr": 8e-05,
+  "output_dir": "ckpt/r/add_sub_baseline_100K",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 167871744,
+  "run_name": "add_sub_baseline_100K",
+  "git_commit": "9e4530548a98f8c7f5c14930ac4aec4886bb4b1b",
+  "timestamp": "2026-04-07T05:30:02.482767",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "sft",
+  "final_accuracy": 1.0
+}