Hugging Face's logo

thoughtworks
/
arithmetic-sorl

Safetensors
sorl
arithmetic
interpretability
mechanistic-interpretability
qwen3
Model card Files
xet
 Community
amirali1985 commited on
Commit
e69a25b
·
verified ·
1 Parent(s): 3aab210

Upload add_baseline_50K

Browse files
Files changed (5) hide show
  1. add_baseline_50K/config.json +38 -0
  2. add_baseline_50K/generation_config.json +7 -0
  3. add_baseline_50K/metrics.json +197 -0
  4. add_baseline_50K/model.safetensors +3 -0
  5. add_baseline_50K/train_config.json +28 -0
add_baseline_50K/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "SorlModelWrapper"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": null,
8
+ "dtype": "float32",
9
+ "eos_token_id": null,
10
+ "head_dim": 128,
11
+ "hidden_act": "silu",
12
+ "hidden_size": 512,
13
+ "initializer_range": 0.02,
14
+ "intermediate_size": 2048,
15
+ "layer_types": [
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention"
19
+ ],
20
+ "max_position_embeddings": 128,
21
+ "max_window_layers": 28,
22
+ "model_type": "qwen3",
23
+ "num_attention_heads": 4,
24
+ "num_hidden_layers": 3,
25
+ "num_key_value_heads": 4,
26
+ "pad_token_id": null,
27
+ "rms_norm_eps": 1e-06,
28
+ "rope_parameters": {
29
+ "rope_theta": 10000.0,
30
+ "rope_type": "default"
31
+ },
32
+ "sliding_window": null,
33
+ "tie_word_embeddings": false,
34
+ "transformers_version": "5.5.0",
35
+ "use_cache": true,
36
+ "use_sliding_window": false,
37
+ "vocab_size": 151645
38
+ }
add_baseline_50K/generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "output_attentions": false,
4
+ "output_hidden_states": false,
5
+ "transformers_version": "5.5.0",
6
+ "use_cache": true
7
+ }
add_baseline_50K/metrics.json ADDED
@@ -0,0 +1,197 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "history": {
3
+ "step": [
4
+ 50,
5
+ 100,
6
+ 150,
7
+ 200,
8
+ 250,
9
+ 300,
10
+ 350,
11
+ 400,
12
+ 450,
13
+ 500,
14
+ 550,
15
+ 600,
16
+ 650,
17
+ 700,
18
+ 750,
19
+ 800,
20
+ 850,
21
+ 900,
22
+ 950,
23
+ 1000,
24
+ 1050,
25
+ 1100,
26
+ 1150,
27
+ 1200,
28
+ 1250,
29
+ 1300,
30
+ 1350,
31
+ 1400,
32
+ 1450,
33
+ 1500,
34
+ 1550,
35
+ 1600,
36
+ 1650,
37
+ 1700,
38
+ 1750,
39
+ 1800,
40
+ 1850,
41
+ 1900,
42
+ 1950,
43
+ 2000,
44
+ 2050,
45
+ 2100,
46
+ 2150,
47
+ 2200,
48
+ 2250,
49
+ 2300
50
+ ],
51
+ "loss": [
52
+ 9.380824089050293,
53
+ 7.289170265197754,
54
+ 6.3251800537109375,
55
+ 4.6120710372924805,
56
+ 3.3842175006866455,
57
+ 2.277524709701538,
58
+ 1.7819974422454834,
59
+ 1.07933509349823,
60
+ 0.610201358795166,
61
+ 0.46401315927505493,
62
+ 0.3064887821674347,
63
+ 0.2689448297023773,
64
+ 0.2373236119747162,
65
+ 0.17521344125270844,
66
+ 0.2070240080356598,
67
+ 0.07674235850572586,
68
+ 0.08533211052417755,
69
+ 0.08785510063171387,
70
+ 0.04789081960916519,
71
+ 0.061709098517894745,
72
+ 0.0520438477396965,
73
+ 0.05887055769562721,
74
+ 0.03880451247096062,
75
+ 0.040696412324905396,
76
+ 0.027697861194610596,
77
+ 0.03105991706252098,
78
+ 0.012362940236926079,
79
+ 0.025542084127664566,
80
+ 0.03732563182711601,
81
+ 0.012686265632510185,
82
+ 0.010935312137007713,
83
+ 0.013005566783249378,
84
+ 0.006263155955821276,
85
+ 0.006360996980220079,
86
+ 0.02447586879134178,
87
+ 0.0022957585752010345,
88
+ 0.001398123218677938,
89
+ 0.004851323552429676,
90
+ 0.0012026532785966992,
91
+ 0.0044944994151592255,
92
+ 0.0008991410140879452,
93
+ 0.005278103519231081,
94
+ 0.001702666049823165,
95
+ 0.0011946497252210975,
96
+ 0.001532713882625103,
97
+ 0.0015285462141036987
98
+ ],
99
+ "base_loss": [
100
+ 9.380824089050293,
101
+ 7.289170265197754,
102
+ 6.3251800537109375,
103
+ 4.6120710372924805,
104
+ 3.3842175006866455,
105
+ 2.277524709701538,
106
+ 1.7819974422454834,
107
+ 1.07933509349823,
108
+ 0.610201358795166,
109
+ 0.46401315927505493,
110
+ 0.3064887821674347,
111
+ 0.2689448297023773,
112
+ 0.2373236119747162,
113
+ 0.17521344125270844,
114
+ 0.2070240080356598,
115
+ 0.07674235850572586,
116
+ 0.08533211052417755,
117
+ 0.08785510063171387,
118
+ 0.04789081960916519,
119
+ 0.061709098517894745,
120
+ 0.0520438477396965,
121
+ 0.05887055769562721,
122
+ 0.03880451247096062,
123
+ 0.040696412324905396,
124
+ 0.027697861194610596,
125
+ 0.03105991706252098,
126
+ 0.012362940236926079,
127
+ 0.025542084127664566,
128
+ 0.03732563182711601,
129
+ 0.012686265632510185,
130
+ 0.010935312137007713,
131
+ 0.013005566783249378,
132
+ 0.006263155955821276,
133
+ 0.006360996980220079,
134
+ 0.02447586879134178,
135
+ 0.0022957585752010345,
136
+ 0.001398123218677938,
137
+ 0.004851323552429676,
138
+ 0.0012026532785966992,
139
+ 0.0044944994151592255,
140
+ 0.0008991410140879452,
141
+ 0.005278103519231081,
142
+ 0.001702666049823165,
143
+ 0.0011946497252210975,
144
+ 0.001532713882625103,
145
+ 0.0015285462141036987
146
+ ],
147
+ "lr": [
148
+ 8.358208955223882e-06,
149
+ 1.6886993603411513e-05,
150
+ 2.541577825159915e-05,
151
+ 3.394456289978678e-05,
152
+ 4.247334754797441e-05,
153
+ 5.100213219616205e-05,
154
+ 5.953091684434969e-05,
155
+ 6.805970149253732e-05,
156
+ 7.658848614072494e-05,
157
+ 7.994958584913144e-05,
158
+ 7.96419594562729e-05,
159
+ 7.905686510146095e-05,
160
+ 7.819839806506031e-05,
161
+ 7.707256705860444e-05,
162
+ 7.568725216771852e-05,
163
+ 7.405214969663954e-05,
164
+ 7.217870430038601e-05,
165
+ 7.008002887960812e-05,
166
+ 6.777081279880229e-05,
167
+ 6.526721907030291e-05,
168
+ 6.258677122369667e-05,
169
+ 5.9748230652499966e-05,
170
+ 5.677146529659303e-05,
171
+ 5.3677310579548395e-05,
172
+ 5.04874235742021e-05,
173
+ 4.722413141721374e-05,
174
+ 4.391027503361521e-05,
175
+ 4.056904926517458e-05,
176
+ 3.722384052157265e-05,
177
+ 3.3898063090728774e-05,
178
+ 3.061499525399694e-05,
179
+ 2.7397616353319584e-05,
180
+ 2.4268445950762915e-05,
181
+ 2.1249386206212863e-05,
182
+ 1.8361568576485003e-05,
183
+ 1.5625205908855437e-05,
184
+ 1.3059450964261466e-05,
185
+ 1.0682262360418084e-05,
186
+ 8.510278873161129e-06,
187
+ 6.558702975826241e-06,
188
+ 4.841194431812048e-06,
189
+ 3.3697746851107672e-06,
190
+ 2.1547427180102297e-06,
191
+ 1.2046029649088297e-06,
192
+ 5.260057867998169e-07,
193
+ 1.2370092306488446e-07
194
+ ]
195
+ },
196
+ "final_accuracy": 1.0
197
+ }
add_baseline_50K/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0bc05eb30d350e3cf42236141e2fb1a356a292d6bfe2900bdb77c812fd4b0656
3
+ size 671794850
add_baseline_50K/train_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "mode": "baseline",
3
+ "ops": "add",
4
+ "n_digits": 6,
5
+ "n_layer": 3,
6
+ "n_head": 4,
7
+ "n_embd": 512,
8
+ "abs_vocab": 0,
9
+ "K": 4,
10
+ "batch_size": 64,
11
+ "num_epochs": 3,
12
+ "dataset_size": 50000,
13
+ "lr": 8e-05,
14
+ "output_dir": "ckpt/r/add_baseline_50K",
15
+ "device": "cuda",
16
+ "push_to_hub": true,
17
+ "no_wandb": false,
18
+ "n_params": 167871744,
19
+ "run_name": "add_baseline_50K",
20
+ "git_commit": "9e4530548a98f8c7f5c14930ac4aec4886bb4b1b",
21
+ "timestamp": "2026-04-07T05:23:18.463596",
22
+ "tokenizer": "Qwen/Qwen3-0.6B",
23
+ "dataset_repo": "thoughtworks/arithmetic-sorl-data",
24
+ "dataset_config": "add_6digit",
25
+ "model_repo": "thoughtworks/arithmetic-sorl",
26
+ "trainer_version": "sft",
27
+ "final_accuracy": 1.0
28
+ }