| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 19.37219730941704, |
| "eval_steps": 500, |
| "global_step": 540, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.35874439461883406, |
| "grad_norm": 0.5037005543708801, |
| "learning_rate": 9.259259259259259e-07, |
| "logits/chosen": 1.851180076599121, |
| "logits/rejected": 1.8187497854232788, |
| "logps/chosen": -71.38922119140625, |
| "logps/rejected": -77.1716537475586, |
| "loss": 0.6947, |
| "rewards/accuracies": 0.36250001192092896, |
| "rewards/chosen": -0.0033944465685635805, |
| "rewards/margins": -0.004582419525831938, |
| "rewards/rejected": 0.0011879729572683573, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.7174887892376681, |
| "grad_norm": 0.4899679124355316, |
| "learning_rate": 1.8518518518518519e-06, |
| "logits/chosen": 1.804616928100586, |
| "logits/rejected": 1.8522075414657593, |
| "logps/chosen": -68.71460723876953, |
| "logps/rejected": -92.10017395019531, |
| "loss": 0.6935, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.0006334683275781572, |
| "rewards/margins": 0.002181120216846466, |
| "rewards/rejected": -0.0015476513653993607, |
| "step": 20 |
| }, |
| { |
| "epoch": 1.0762331838565022, |
| "grad_norm": 0.4610757529735565, |
| "learning_rate": 2.7777777777777783e-06, |
| "logits/chosen": 1.7368440628051758, |
| "logits/rejected": 1.747293472290039, |
| "logps/chosen": -86.10456848144531, |
| "logps/rejected": -94.18408203125, |
| "loss": 0.6923, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -0.004750942811369896, |
| "rewards/margins": -3.7298166716936976e-05, |
| "rewards/rejected": -0.0047136456705629826, |
| "step": 30 |
| }, |
| { |
| "epoch": 1.4349775784753362, |
| "grad_norm": 0.5546987056732178, |
| "learning_rate": 3.7037037037037037e-06, |
| "logits/chosen": 1.760947585105896, |
| "logits/rejected": 1.7774131298065186, |
| "logps/chosen": -72.0299301147461, |
| "logps/rejected": -75.53373718261719, |
| "loss": 0.6938, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": 0.004539580084383488, |
| "rewards/margins": 0.004153046756982803, |
| "rewards/rejected": 0.0003865335020236671, |
| "step": 40 |
| }, |
| { |
| "epoch": 1.7937219730941703, |
| "grad_norm": 0.5019991397857666, |
| "learning_rate": 4.62962962962963e-06, |
| "logits/chosen": 1.6594927310943604, |
| "logits/rejected": 1.7026996612548828, |
| "logps/chosen": -74.72074890136719, |
| "logps/rejected": -80.34657287597656, |
| "loss": 0.694, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": 0.0014741375343874097, |
| "rewards/margins": 0.00692129647359252, |
| "rewards/rejected": -0.00544715765863657, |
| "step": 50 |
| }, |
| { |
| "epoch": 2.1524663677130045, |
| "grad_norm": 0.5497386455535889, |
| "learning_rate": 4.998119881260576e-06, |
| "logits/chosen": 1.845393180847168, |
| "logits/rejected": 1.8288724422454834, |
| "logps/chosen": -85.8925552368164, |
| "logps/rejected": -79.84625244140625, |
| "loss": 0.6939, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": 0.0007343249162659049, |
| "rewards/margins": 0.0027797420043498278, |
| "rewards/rejected": -0.0020454167388379574, |
| "step": 60 |
| }, |
| { |
| "epoch": 2.5112107623318387, |
| "grad_norm": 0.53052818775177, |
| "learning_rate": 4.9866405060165044e-06, |
| "logits/chosen": 1.7219512462615967, |
| "logits/rejected": 1.739393949508667, |
| "logps/chosen": -82.69403076171875, |
| "logps/rejected": -85.24137878417969, |
| "loss": 0.6912, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": 0.004084458574652672, |
| "rewards/margins": 8.91583040356636e-05, |
| "rewards/rejected": 0.003995300270617008, |
| "step": 70 |
| }, |
| { |
| "epoch": 2.8699551569506725, |
| "grad_norm": 0.5697340965270996, |
| "learning_rate": 4.964774158361991e-06, |
| "logits/chosen": 1.688643217086792, |
| "logits/rejected": 1.7375280857086182, |
| "logps/chosen": -77.02912902832031, |
| "logps/rejected": -84.49137115478516, |
| "loss": 0.6923, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.0026642463635653257, |
| "rewards/margins": -0.0012511021923273802, |
| "rewards/rejected": -0.0014131448697298765, |
| "step": 80 |
| }, |
| { |
| "epoch": 3.2286995515695067, |
| "grad_norm": 0.6337418556213379, |
| "learning_rate": 4.93261217644956e-06, |
| "logits/chosen": 1.7215973138809204, |
| "logits/rejected": 1.6856582164764404, |
| "logps/chosen": -81.94705963134766, |
| "logps/rejected": -79.82647705078125, |
| "loss": 0.6916, |
| "rewards/accuracies": 0.4749999940395355, |
| "rewards/chosen": 0.004837460815906525, |
| "rewards/margins": 0.0014790964778512716, |
| "rewards/rejected": 0.003358363639563322, |
| "step": 90 |
| }, |
| { |
| "epoch": 3.587443946188341, |
| "grad_norm": 0.6216289401054382, |
| "learning_rate": 4.8902889044347e-06, |
| "logits/chosen": 1.8092563152313232, |
| "logits/rejected": 1.7624406814575195, |
| "logps/chosen": -91.27610778808594, |
| "logps/rejected": -76.58167266845703, |
| "loss": 0.6879, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": 0.004790940321981907, |
| "rewards/margins": 0.018742190673947334, |
| "rewards/rejected": -0.013951249420642853, |
| "step": 100 |
| }, |
| { |
| "epoch": 3.9461883408071747, |
| "grad_norm": 0.6686798930168152, |
| "learning_rate": 4.837981131305475e-06, |
| "logits/chosen": 1.8108171224594116, |
| "logits/rejected": 1.8439037799835205, |
| "logps/chosen": -68.89733123779297, |
| "logps/rejected": -90.71812438964844, |
| "loss": 0.686, |
| "rewards/accuracies": 0.5375000238418579, |
| "rewards/chosen": 0.0019698950927704573, |
| "rewards/margins": 0.011438245885074139, |
| "rewards/rejected": -0.009468351490795612, |
| "step": 110 |
| }, |
| { |
| "epoch": 4.304932735426009, |
| "grad_norm": 0.690719723701477, |
| "learning_rate": 4.775907352415367e-06, |
| "logits/chosen": 1.7892429828643799, |
| "logits/rejected": 1.7746537923812866, |
| "logps/chosen": -76.5858383178711, |
| "logps/rejected": -70.5009765625, |
| "loss": 0.6841, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.00402367627248168, |
| "rewards/margins": 0.02823176048696041, |
| "rewards/rejected": -0.024208081886172295, |
| "step": 120 |
| }, |
| { |
| "epoch": 4.663677130044843, |
| "grad_norm": 0.6445876955986023, |
| "learning_rate": 4.70432685680402e-06, |
| "logits/chosen": 1.7731869220733643, |
| "logits/rejected": 1.70940363407135, |
| "logps/chosen": -80.2589111328125, |
| "logps/rejected": -69.99195861816406, |
| "loss": 0.6807, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": 0.0028332143556326628, |
| "rewards/margins": 0.027303647249937057, |
| "rewards/rejected": -0.024470435455441475, |
| "step": 130 |
| }, |
| { |
| "epoch": 5.022421524663677, |
| "grad_norm": 0.7485262751579285, |
| "learning_rate": 4.623538644118244e-06, |
| "logits/chosen": 1.707404375076294, |
| "logits/rejected": 1.7899534702301025, |
| "logps/chosen": -85.12259674072266, |
| "logps/rejected": -96.5855712890625, |
| "loss": 0.6717, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": 0.0005208252114243805, |
| "rewards/margins": 0.046183306723833084, |
| "rewards/rejected": -0.04566247761249542, |
| "step": 140 |
| }, |
| { |
| "epoch": 5.381165919282511, |
| "grad_norm": 0.7476359009742737, |
| "learning_rate": 4.533880175657419e-06, |
| "logits/chosen": 1.8374496698379517, |
| "logits/rejected": 1.8710906505584717, |
| "logps/chosen": -91.67071533203125, |
| "logps/rejected": -89.70133209228516, |
| "loss": 0.6683, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.004734884016215801, |
| "rewards/margins": 0.06081797555088997, |
| "rewards/rejected": -0.05608309060335159, |
| "step": 150 |
| }, |
| { |
| "epoch": 5.739910313901345, |
| "grad_norm": 0.8249401450157166, |
| "learning_rate": 4.435725964760331e-06, |
| "logits/chosen": 1.7922887802124023, |
| "logits/rejected": 1.766998291015625, |
| "logps/chosen": -90.3626708984375, |
| "logps/rejected": -78.34722900390625, |
| "loss": 0.6615, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.0012385320151224732, |
| "rewards/margins": 0.06155504658818245, |
| "rewards/rejected": -0.06279357522726059, |
| "step": 160 |
| }, |
| { |
| "epoch": 6.098654708520179, |
| "grad_norm": 0.8427699208259583, |
| "learning_rate": 4.329486012421531e-06, |
| "logits/chosen": 1.8337100744247437, |
| "logits/rejected": 1.8013321161270142, |
| "logps/chosen": -88.58504486083984, |
| "logps/rejected": -82.3107681274414, |
| "loss": 0.6575, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": -0.028140008449554443, |
| "rewards/margins": 0.06505288183689117, |
| "rewards/rejected": -0.09319288283586502, |
| "step": 170 |
| }, |
| { |
| "epoch": 6.457399103139013, |
| "grad_norm": 0.8900613784790039, |
| "learning_rate": 4.215604094671835e-06, |
| "logits/chosen": 1.8928254842758179, |
| "logits/rejected": 1.8716427087783813, |
| "logps/chosen": -73.64405822753906, |
| "logps/rejected": -82.36759948730469, |
| "loss": 0.6486, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.008158551529049873, |
| "rewards/margins": 0.10160098224878311, |
| "rewards/rejected": -0.10975953191518784, |
| "step": 180 |
| }, |
| { |
| "epoch": 6.816143497757848, |
| "grad_norm": 0.8851564526557922, |
| "learning_rate": 4.094555908876765e-06, |
| "logits/chosen": 1.8064581155776978, |
| "logits/rejected": 1.7510322332382202, |
| "logps/chosen": -81.51691436767578, |
| "logps/rejected": -80.2887954711914, |
| "loss": 0.6397, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.010233854874968529, |
| "rewards/margins": 0.15614357590675354, |
| "rewards/rejected": -0.166377454996109, |
| "step": 190 |
| }, |
| { |
| "epoch": 7.174887892376682, |
| "grad_norm": 0.9349417090415955, |
| "learning_rate": 3.966847086696045e-06, |
| "logits/chosen": 1.869329810142517, |
| "logits/rejected": 1.8965225219726562, |
| "logps/chosen": -83.17039489746094, |
| "logps/rejected": -83.12516021728516, |
| "loss": 0.6421, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.06402869522571564, |
| "rewards/margins": 0.11340844631195068, |
| "rewards/rejected": -0.17743715643882751, |
| "step": 200 |
| }, |
| { |
| "epoch": 7.533632286995516, |
| "grad_norm": 0.9500117301940918, |
| "learning_rate": 3.833011082004229e-06, |
| "logits/chosen": 1.74019455909729, |
| "logits/rejected": 1.8087282180786133, |
| "logps/chosen": -69.24908447265625, |
| "logps/rejected": -78.79796600341797, |
| "loss": 0.6373, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.09633999317884445, |
| "rewards/margins": 0.10314120352268219, |
| "rewards/rejected": -0.19948120415210724, |
| "step": 210 |
| }, |
| { |
| "epoch": 7.8923766816143495, |
| "grad_norm": 0.9928329586982727, |
| "learning_rate": 3.693606942594873e-06, |
| "logits/chosen": 1.7647511959075928, |
| "logits/rejected": 1.688582420349121, |
| "logps/chosen": -85.75634002685547, |
| "logps/rejected": -84.30413818359375, |
| "loss": 0.6193, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.08094713091850281, |
| "rewards/margins": 0.20030252635478973, |
| "rewards/rejected": -0.28124964237213135, |
| "step": 220 |
| }, |
| { |
| "epoch": 8.251121076233185, |
| "grad_norm": 0.9496206045150757, |
| "learning_rate": 3.549216974976073e-06, |
| "logits/chosen": 1.6640609502792358, |
| "logits/rejected": 1.7074018716812134, |
| "logps/chosen": -78.85441589355469, |
| "logps/rejected": -101.50064849853516, |
| "loss": 0.6187, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.10971230268478394, |
| "rewards/margins": 0.16489508748054504, |
| "rewards/rejected": -0.2746073603630066, |
| "step": 230 |
| }, |
| { |
| "epoch": 8.609865470852018, |
| "grad_norm": 0.8651049137115479, |
| "learning_rate": 3.400444312011776e-06, |
| "logits/chosen": 1.8204492330551147, |
| "logits/rejected": 1.8322817087173462, |
| "logps/chosen": -80.65580749511719, |
| "logps/rejected": -85.63023376464844, |
| "loss": 0.6167, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.09459066390991211, |
| "rewards/margins": 0.2738259434700012, |
| "rewards/rejected": -0.3684166371822357, |
| "step": 240 |
| }, |
| { |
| "epoch": 8.968609865470851, |
| "grad_norm": 1.070734977722168, |
| "learning_rate": 3.2479103935691047e-06, |
| "logits/chosen": 1.6801395416259766, |
| "logits/rejected": 1.7755155563354492, |
| "logps/chosen": -79.02301788330078, |
| "logps/rejected": -99.99031066894531, |
| "loss": 0.5929, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.14009204506874084, |
| "rewards/margins": 0.26308172941207886, |
| "rewards/rejected": -0.4031738340854645, |
| "step": 250 |
| }, |
| { |
| "epoch": 9.327354260089686, |
| "grad_norm": 0.9541939496994019, |
| "learning_rate": 3.092252370695298e-06, |
| "logits/chosen": 1.662615418434143, |
| "logits/rejected": 1.7456607818603516, |
| "logps/chosen": -75.20834350585938, |
| "logps/rejected": -113.8222885131836, |
| "loss": 0.5945, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.12744423747062683, |
| "rewards/margins": 0.22806742787361145, |
| "rewards/rejected": -0.3555116653442383, |
| "step": 260 |
| }, |
| { |
| "epoch": 9.68609865470852, |
| "grad_norm": 1.0955382585525513, |
| "learning_rate": 2.9341204441673267e-06, |
| "logits/chosen": 1.7580053806304932, |
| "logits/rejected": 1.7044979333877563, |
| "logps/chosen": -74.88780212402344, |
| "logps/rejected": -77.73652648925781, |
| "loss": 0.5913, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.20694907009601593, |
| "rewards/margins": 0.28244781494140625, |
| "rewards/rejected": -0.489396870136261, |
| "step": 270 |
| }, |
| { |
| "epoch": 10.044843049327355, |
| "grad_norm": 1.0972706079483032, |
| "learning_rate": 2.7741751485313295e-06, |
| "logits/chosen": 1.6702816486358643, |
| "logits/rejected": 1.861555814743042, |
| "logps/chosen": -76.36530303955078, |
| "logps/rejected": -107.99542236328125, |
| "loss": 0.596, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.19078974425792694, |
| "rewards/margins": 0.31978002190589905, |
| "rewards/rejected": -0.5105697512626648, |
| "step": 280 |
| }, |
| { |
| "epoch": 10.403587443946188, |
| "grad_norm": 1.180747628211975, |
| "learning_rate": 2.6130845929767662e-06, |
| "logits/chosen": 1.7563337087631226, |
| "logits/rejected": 1.8194200992584229, |
| "logps/chosen": -74.66185760498047, |
| "logps/rejected": -93.19525146484375, |
| "loss": 0.5767, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.15936018526554108, |
| "rewards/margins": 0.2494836151599884, |
| "rewards/rejected": -0.4088438153266907, |
| "step": 290 |
| }, |
| { |
| "epoch": 10.762331838565022, |
| "grad_norm": 1.1262431144714355, |
| "learning_rate": 2.4515216705704396e-06, |
| "logits/chosen": 1.6524626016616821, |
| "logits/rejected": 1.6555395126342773, |
| "logps/chosen": -76.6367416381836, |
| "logps/rejected": -79.79283905029297, |
| "loss": 0.5847, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.17207466065883636, |
| "rewards/margins": 0.3527255058288574, |
| "rewards/rejected": -0.524800181388855, |
| "step": 300 |
| }, |
| { |
| "epoch": 11.121076233183857, |
| "grad_norm": 1.0337167978286743, |
| "learning_rate": 2.290161247507733e-06, |
| "logits/chosen": 1.5576703548431396, |
| "logits/rejected": 1.6868317127227783, |
| "logps/chosen": -82.24876403808594, |
| "logps/rejected": -97.26077270507812, |
| "loss": 0.5878, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.2861223816871643, |
| "rewards/margins": 0.3279086947441101, |
| "rewards/rejected": -0.6140309572219849, |
| "step": 310 |
| }, |
| { |
| "epoch": 11.47982062780269, |
| "grad_norm": 1.0992393493652344, |
| "learning_rate": 2.129677344121879e-06, |
| "logits/chosen": 1.767163872718811, |
| "logits/rejected": 1.7726752758026123, |
| "logps/chosen": -85.95258331298828, |
| "logps/rejected": -95.29878234863281, |
| "loss": 0.5674, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.25968822836875916, |
| "rewards/margins": 0.37743932008743286, |
| "rewards/rejected": -0.6371275186538696, |
| "step": 320 |
| }, |
| { |
| "epoch": 11.838565022421525, |
| "grad_norm": 1.082541823387146, |
| "learning_rate": 1.970740319426474e-06, |
| "logits/chosen": 1.6448657512664795, |
| "logits/rejected": 1.6428959369659424, |
| "logps/chosen": -103.04365539550781, |
| "logps/rejected": -82.7887954711914, |
| "loss": 0.5632, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -0.20720317959785461, |
| "rewards/margins": 0.2542288899421692, |
| "rewards/rejected": -0.4614320695400238, |
| "step": 330 |
| }, |
| { |
| "epoch": 12.197309417040358, |
| "grad_norm": 1.061893105506897, |
| "learning_rate": 1.8140140709517467e-06, |
| "logits/chosen": 1.7359678745269775, |
| "logits/rejected": 1.7358520030975342, |
| "logps/chosen": -85.93545532226562, |
| "logps/rejected": -89.08082580566406, |
| "loss": 0.5727, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.2792442739009857, |
| "rewards/margins": 0.22401955723762512, |
| "rewards/rejected": -0.5032638311386108, |
| "step": 340 |
| }, |
| { |
| "epoch": 12.556053811659194, |
| "grad_norm": 1.107885479927063, |
| "learning_rate": 1.6601532615711452e-06, |
| "logits/chosen": 1.585384726524353, |
| "logits/rejected": 1.636023759841919, |
| "logps/chosen": -93.92401123046875, |
| "logps/rejected": -90.39390563964844, |
| "loss": 0.5611, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.1979474574327469, |
| "rewards/margins": 0.33891761302948, |
| "rewards/rejected": -0.5368650555610657, |
| "step": 350 |
| }, |
| { |
| "epoch": 12.914798206278027, |
| "grad_norm": 1.0166430473327637, |
| "learning_rate": 1.509800584902108e-06, |
| "logits/chosen": 1.6831719875335693, |
| "logits/rejected": 1.624122977256775, |
| "logps/chosen": -93.83638763427734, |
| "logps/rejected": -85.26428985595703, |
| "loss": 0.5576, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.20380835235118866, |
| "rewards/margins": 0.33769887685775757, |
| "rewards/rejected": -0.5415073037147522, |
| "step": 360 |
| }, |
| { |
| "epoch": 13.27354260089686, |
| "grad_norm": 1.039067268371582, |
| "learning_rate": 1.3635840807037487e-06, |
| "logits/chosen": 1.6376430988311768, |
| "logits/rejected": 1.747945785522461, |
| "logps/chosen": -66.59056091308594, |
| "logps/rejected": -86.75489044189453, |
| "loss": 0.5502, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -0.27494093775749207, |
| "rewards/margins": 0.4173588156700134, |
| "rewards/rejected": -0.6922997832298279, |
| "step": 370 |
| }, |
| { |
| "epoch": 13.632286995515695, |
| "grad_norm": 1.0018366575241089, |
| "learning_rate": 1.2221145114853172e-06, |
| "logits/chosen": 1.6222606897354126, |
| "logits/rejected": 1.6619726419448853, |
| "logps/chosen": -87.45683288574219, |
| "logps/rejected": -83.59837341308594, |
| "loss": 0.5595, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.17884480953216553, |
| "rewards/margins": 0.2639870047569275, |
| "rewards/rejected": -0.4428318440914154, |
| "step": 380 |
| }, |
| { |
| "epoch": 13.991031390134529, |
| "grad_norm": 1.0804483890533447, |
| "learning_rate": 1.085982811283654e-06, |
| "logits/chosen": 1.63021981716156, |
| "logits/rejected": 1.6384109258651733, |
| "logps/chosen": -91.65565490722656, |
| "logps/rejected": -92.39521789550781, |
| "loss": 0.5517, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.2314184606075287, |
| "rewards/margins": 0.3744989335536957, |
| "rewards/rejected": -0.6059174537658691, |
| "step": 390 |
| }, |
| { |
| "epoch": 14.349775784753364, |
| "grad_norm": 1.126531958580017, |
| "learning_rate": 9.557576172663577e-07, |
| "logits/chosen": 1.6363227367401123, |
| "logits/rejected": 1.6808557510375977, |
| "logps/chosen": -84.4040298461914, |
| "logps/rejected": -89.61891174316406, |
| "loss": 0.5479, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.19463828206062317, |
| "rewards/margins": 0.4325459599494934, |
| "rewards/rejected": -0.627184271812439, |
| "step": 400 |
| }, |
| { |
| "epoch": 14.708520179372197, |
| "grad_norm": 1.229477882385254, |
| "learning_rate": 8.319828944714508e-07, |
| "logits/chosen": 1.7398450374603271, |
| "logits/rejected": 1.7130982875823975, |
| "logps/chosen": -79.57427978515625, |
| "logps/rejected": -91.69161224365234, |
| "loss": 0.5551, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.23391035199165344, |
| "rewards/margins": 0.42068368196487427, |
| "rewards/rejected": -0.6545940637588501, |
| "step": 410 |
| }, |
| { |
| "epoch": 15.067264573991032, |
| "grad_norm": 1.0864663124084473, |
| "learning_rate": 7.151756636052529e-07, |
| "logits/chosen": 1.6776914596557617, |
| "logits/rejected": 1.6496366262435913, |
| "logps/chosen": -80.69108581542969, |
| "logps/rejected": -81.94221496582031, |
| "loss": 0.5469, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -0.23782162368297577, |
| "rewards/margins": 0.47778135538101196, |
| "rewards/rejected": -0.7156028747558594, |
| "step": 420 |
| }, |
| { |
| "epoch": 15.426008968609866, |
| "grad_norm": 1.0339457988739014, |
| "learning_rate": 6.058238413897052e-07, |
| "logits/chosen": 1.7183252573013306, |
| "logits/rejected": 1.7133119106292725, |
| "logps/chosen": -100.38890075683594, |
| "logps/rejected": -101.6832046508789, |
| "loss": 0.551, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.34573277831077576, |
| "rewards/margins": 0.3820987939834595, |
| "rewards/rejected": -0.7278315424919128, |
| "step": 430 |
| }, |
| { |
| "epoch": 15.784753363228699, |
| "grad_norm": 1.2006279230117798, |
| "learning_rate": 5.043842024802675e-07, |
| "logits/chosen": 1.7278945446014404, |
| "logits/rejected": 1.7016990184783936, |
| "logps/chosen": -89.6147232055664, |
| "logps/rejected": -92.5606918334961, |
| "loss": 0.5419, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.24864788353443146, |
| "rewards/margins": 0.3481907546520233, |
| "rewards/rejected": -0.5968385934829712, |
| "step": 440 |
| }, |
| { |
| "epoch": 16.143497757847534, |
| "grad_norm": 1.1825172901153564, |
| "learning_rate": 4.1128047146765936e-07, |
| "logits/chosen": 1.5679899454116821, |
| "logits/rejected": 1.5998270511627197, |
| "logps/chosen": -81.74020385742188, |
| "logps/rejected": -86.86679077148438, |
| "loss": 0.5492, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.2699211537837982, |
| "rewards/margins": 0.4290841221809387, |
| "rewards/rejected": -0.6990053057670593, |
| "step": 450 |
| }, |
| { |
| "epoch": 16.50224215246637, |
| "grad_norm": 1.1816792488098145, |
| "learning_rate": 3.269015529333805e-07, |
| "logits/chosen": 1.621124029159546, |
| "logits/rejected": 1.749467134475708, |
| "logps/chosen": -73.30225372314453, |
| "logps/rejected": -92.98735046386719, |
| "loss": 0.5364, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.29340410232543945, |
| "rewards/margins": 0.39086267352104187, |
| "rewards/rejected": -0.6842667460441589, |
| "step": 460 |
| }, |
| { |
| "epoch": 16.8609865470852, |
| "grad_norm": 1.107458472251892, |
| "learning_rate": 2.515999069522676e-07, |
| "logits/chosen": 1.7169005870819092, |
| "logits/rejected": 1.6980698108673096, |
| "logps/chosen": -102.9745864868164, |
| "logps/rejected": -77.38671875, |
| "loss": 0.5485, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.22492234408855438, |
| "rewards/margins": 0.4119078516960144, |
| "rewards/rejected": -0.63683021068573, |
| "step": 470 |
| }, |
| { |
| "epoch": 17.219730941704036, |
| "grad_norm": 1.1910994052886963, |
| "learning_rate": 1.8569007682777417e-07, |
| "logits/chosen": 1.5466203689575195, |
| "logits/rejected": 1.5265737771987915, |
| "logps/chosen": -81.35516357421875, |
| "logps/rejected": -79.12394714355469, |
| "loss": 0.5546, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.2879922688007355, |
| "rewards/margins": 0.2313261479139328, |
| "rewards/rejected": -0.5193184614181519, |
| "step": 480 |
| }, |
| { |
| "epoch": 17.57847533632287, |
| "grad_norm": 1.156771183013916, |
| "learning_rate": 1.2944737520980883e-07, |
| "logits/chosen": 1.7166872024536133, |
| "logits/rejected": 1.8277479410171509, |
| "logps/chosen": -89.5985336303711, |
| "logps/rejected": -97.35225677490234, |
| "loss": 0.5312, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.27265459299087524, |
| "rewards/margins": 0.3236234784126282, |
| "rewards/rejected": -0.5962780714035034, |
| "step": 490 |
| }, |
| { |
| "epoch": 17.937219730941703, |
| "grad_norm": 1.5837684869766235, |
| "learning_rate": 8.310673408334496e-08, |
| "logits/chosen": 1.693872094154358, |
| "logits/rejected": 1.6753771305084229, |
| "logps/chosen": -84.9941177368164, |
| "logps/rejected": -78.13436126708984, |
| "loss": 0.528, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.1711592972278595, |
| "rewards/margins": 0.4582854211330414, |
| "rewards/rejected": -0.6294447183609009, |
| "step": 500 |
| }, |
| { |
| "epoch": 18.295964125560538, |
| "grad_norm": 1.1619194746017456, |
| "learning_rate": 4.6861723431538273e-08, |
| "logits/chosen": 1.6101446151733398, |
| "logits/rejected": 1.793236494064331, |
| "logps/chosen": -80.81233215332031, |
| "logps/rejected": -107.17619323730469, |
| "loss": 0.5495, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -0.2168944776058197, |
| "rewards/margins": 0.5115703344345093, |
| "rewards/rejected": -0.7284647822380066, |
| "step": 510 |
| }, |
| { |
| "epoch": 18.654708520179373, |
| "grad_norm": 1.2213138341903687, |
| "learning_rate": 2.0863742672497244e-08, |
| "logits/chosen": 1.6612660884857178, |
| "logits/rejected": 1.6667835712432861, |
| "logps/chosen": -89.79667663574219, |
| "logps/rejected": -92.6375732421875, |
| "loss": 0.5483, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -0.21362046897411346, |
| "rewards/margins": 0.5911002159118652, |
| "rewards/rejected": -0.8047205805778503, |
| "step": 520 |
| }, |
| { |
| "epoch": 19.013452914798208, |
| "grad_norm": 0.9736039042472839, |
| "learning_rate": 5.221388247169945e-09, |
| "logits/chosen": 1.5504584312438965, |
| "logits/rejected": 1.4831935167312622, |
| "logps/chosen": -100.22550201416016, |
| "logps/rejected": -77.9913558959961, |
| "loss": 0.5351, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -0.3059537410736084, |
| "rewards/margins": 0.30310431122779846, |
| "rewards/rejected": -0.6090580224990845, |
| "step": 530 |
| }, |
| { |
| "epoch": 19.37219730941704, |
| "grad_norm": 1.2035579681396484, |
| "learning_rate": 0.0, |
| "logits/chosen": 1.761300802230835, |
| "logits/rejected": 1.7381843328475952, |
| "logps/chosen": -83.55279541015625, |
| "logps/rejected": -88.41242980957031, |
| "loss": 0.5356, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -0.21133939921855927, |
| "rewards/margins": 0.5088428258895874, |
| "rewards/rejected": -0.7201822400093079, |
| "step": 540 |
| }, |
| { |
| "epoch": 19.37219730941704, |
| "step": 540, |
| "total_flos": 1.8378090112404685e+18, |
| "train_loss": 0.6078463554382324, |
| "train_runtime": 3976.7885, |
| "train_samples_per_second": 8.972, |
| "train_steps_per_second": 0.136 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 540, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 20, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 1.8378090112404685e+18, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
