diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,6329 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 2000, + "global_step": 4168, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0002399232245681382, + "grad_norm": 4.689366619482009, + "learning_rate": 1.199040767386091e-09, + "logits/chosen": -1.0011488199234009, + "logits/rejected": -0.9304202795028687, + "logps/chosen": -135.81626892089844, + "logps/rejected": -132.77923583984375, + "loss": 0.6931, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1 + }, + { + "epoch": 0.0023992322456813818, + "grad_norm": 4.862910991687305, + "learning_rate": 1.199040767386091e-08, + "logits/chosen": -0.8224125504493713, + "logits/rejected": -0.9372239708900452, + "logps/chosen": -348.92108154296875, + "logps/rejected": -284.24493408203125, + "loss": 0.6931, + "rewards/accuracies": 0.4166666567325592, + "rewards/chosen": 0.00047310083755292, + "rewards/margins": 0.0009964401833713055, + "rewards/rejected": -0.000523339316714555, + "step": 10 + }, + { + "epoch": 0.0047984644913627635, + "grad_norm": 4.872842861483512, + "learning_rate": 2.398081534772182e-08, + "logits/chosen": -0.7908538579940796, + "logits/rejected": -0.8495171666145325, + "logps/chosen": -227.61257934570312, + "logps/rejected": -197.7724609375, + "loss": 0.6933, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": 0.0003789737238548696, + "rewards/margins": 0.00038116061477921903, + "rewards/rejected": -2.1869550437259022e-06, + "step": 20 + }, + { + "epoch": 0.007197696737044146, + "grad_norm": 4.595566615642153, + "learning_rate": 3.597122302158273e-08, + "logits/chosen": -0.9029897451400757, + "logits/rejected": -0.9816384315490723, + "logps/chosen": -238.19448852539062, + "logps/rejected": -245.59716796875, + "loss": 0.693, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.00023587894975207746, + "rewards/margins": 0.0008664665510877967, + "rewards/rejected": -0.0006305875140242279, + "step": 30 + }, + { + "epoch": 0.009596928982725527, + "grad_norm": 4.745750839391565, + "learning_rate": 4.796163069544364e-08, + "logits/chosen": -0.8999044299125671, + "logits/rejected": -0.9681742787361145, + "logps/chosen": -249.49063110351562, + "logps/rejected": -239.22427368164062, + "loss": 0.6929, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": 0.0009130829130299389, + "rewards/margins": 0.00013075381866656244, + "rewards/rejected": 0.0007823290070518851, + "step": 40 + }, + { + "epoch": 0.01199616122840691, + "grad_norm": 5.057774890923517, + "learning_rate": 5.995203836930455e-08, + "logits/chosen": -0.8674479722976685, + "logits/rejected": -0.8950203061103821, + "logps/chosen": -258.783203125, + "logps/rejected": -224.3574981689453, + "loss": 0.6932, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.00040593650192022324, + "rewards/margins": -0.00040950890979729593, + "rewards/rejected": 3.572390369299683e-06, + "step": 50 + }, + { + "epoch": 0.014395393474088292, + "grad_norm": 5.092132113680646, + "learning_rate": 7.194244604316546e-08, + "logits/chosen": -0.9632419347763062, + "logits/rejected": -0.9102532267570496, + "logps/chosen": -264.07708740234375, + "logps/rejected": -246.053955078125, + "loss": 0.693, + "rewards/accuracies": 0.42500001192092896, + "rewards/chosen": -0.000676992698572576, + "rewards/margins": -0.0012114204000681639, + "rewards/rejected": 0.0005344276432879269, + "step": 60 + }, + { + "epoch": 0.016794625719769675, + "grad_norm": 4.648224429804769, + "learning_rate": 8.393285371702638e-08, + "logits/chosen": -0.7308753132820129, + "logits/rejected": -0.7530420422554016, + "logps/chosen": -268.9087829589844, + "logps/rejected": -253.7935791015625, + "loss": 0.6931, + "rewards/accuracies": 0.42500001192092896, + "rewards/chosen": -0.0011928745079785585, + "rewards/margins": -0.001063015777617693, + "rewards/rejected": -0.0001298588322242722, + "step": 70 + }, + { + "epoch": 0.019193857965451054, + "grad_norm": 4.986612393009332, + "learning_rate": 9.592326139088728e-08, + "logits/chosen": -0.8931838870048523, + "logits/rejected": -0.7523937821388245, + "logps/chosen": -179.30514526367188, + "logps/rejected": -231.7339324951172, + "loss": 0.6928, + "rewards/accuracies": 0.5, + "rewards/chosen": 5.182736640563235e-05, + "rewards/margins": 0.0005375603213906288, + "rewards/rejected": -0.0004857330641243607, + "step": 80 + }, + { + "epoch": 0.021593090211132437, + "grad_norm": 4.638674368908913, + "learning_rate": 1.0791366906474819e-07, + "logits/chosen": -0.9545013308525085, + "logits/rejected": -1.0180871486663818, + "logps/chosen": -318.1070251464844, + "logps/rejected": -274.193603515625, + "loss": 0.6929, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -6.821169517934322e-05, + "rewards/margins": -2.9464903491316363e-05, + "rewards/rejected": -3.8746675272705033e-05, + "step": 90 + }, + { + "epoch": 0.02399232245681382, + "grad_norm": 4.888424185382462, + "learning_rate": 1.199040767386091e-07, + "logits/chosen": -0.7748934030532837, + "logits/rejected": -0.6958049535751343, + "logps/chosen": -248.3775634765625, + "logps/rejected": -264.9973449707031, + "loss": 0.6929, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.001517611788585782, + "rewards/margins": -0.0018173797288909554, + "rewards/rejected": 0.00029976802761666477, + "step": 100 + }, + { + "epoch": 0.026391554702495202, + "grad_norm": 4.238219176172328, + "learning_rate": 1.3189448441247004e-07, + "logits/chosen": -0.9006500244140625, + "logits/rejected": -0.9068830609321594, + "logps/chosen": -214.5918731689453, + "logps/rejected": -218.4353790283203, + "loss": 0.6923, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.0011649501975625753, + "rewards/margins": 0.003199439961463213, + "rewards/rejected": -0.002034489531069994, + "step": 110 + }, + { + "epoch": 0.028790786948176585, + "grad_norm": 4.7854336032871485, + "learning_rate": 1.4388489208633092e-07, + "logits/chosen": -0.8462833166122437, + "logits/rejected": -0.9375897645950317, + "logps/chosen": -271.18206787109375, + "logps/rejected": -258.28997802734375, + "loss": 0.6913, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": 0.0003926992940250784, + "rewards/margins": 0.0025149863213300705, + "rewards/rejected": -0.0021222869399935007, + "step": 120 + }, + { + "epoch": 0.031190019193857964, + "grad_norm": 4.384651028319668, + "learning_rate": 1.5587529976019183e-07, + "logits/chosen": -0.9110666513442993, + "logits/rejected": -0.8872417211532593, + "logps/chosen": -196.56625366210938, + "logps/rejected": -294.2314453125, + "loss": 0.6917, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.002021104097366333, + "rewards/margins": 0.0021953617688268423, + "rewards/rejected": -0.004216466099023819, + "step": 130 + }, + { + "epoch": 0.03358925143953935, + "grad_norm": 4.619105718711405, + "learning_rate": 1.6786570743405277e-07, + "logits/chosen": -0.7167148590087891, + "logits/rejected": -0.7378164529800415, + "logps/chosen": -274.03936767578125, + "logps/rejected": -268.20806884765625, + "loss": 0.6905, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0016457755118608475, + "rewards/margins": 0.004239819012582302, + "rewards/rejected": -0.00588559452444315, + "step": 140 + }, + { + "epoch": 0.03598848368522073, + "grad_norm": 4.887855835358377, + "learning_rate": 1.7985611510791365e-07, + "logits/chosen": -0.8701818585395813, + "logits/rejected": -0.8630602955818176, + "logps/chosen": -211.195556640625, + "logps/rejected": -207.6147003173828, + "loss": 0.6904, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.002448505023494363, + "rewards/margins": 0.007906012237071991, + "rewards/rejected": -0.01035451702773571, + "step": 150 + }, + { + "epoch": 0.03838771593090211, + "grad_norm": 4.9642153171388745, + "learning_rate": 1.9184652278177456e-07, + "logits/chosen": -0.7513445615768433, + "logits/rejected": -0.800182044506073, + "logps/chosen": -288.314208984375, + "logps/rejected": -223.4950714111328, + "loss": 0.6892, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0031119356863200665, + "rewards/margins": 0.012469562701880932, + "rewards/rejected": -0.015581500716507435, + "step": 160 + }, + { + "epoch": 0.040786948176583494, + "grad_norm": 4.521207364726132, + "learning_rate": 2.038369304556355e-07, + "logits/chosen": -0.716513991355896, + "logits/rejected": -0.7601069211959839, + "logps/chosen": -318.1102294921875, + "logps/rejected": -305.749267578125, + "loss": 0.6879, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0036616616416722536, + "rewards/margins": 0.00973360612988472, + "rewards/rejected": -0.013395266607403755, + "step": 170 + }, + { + "epoch": 0.04318618042226487, + "grad_norm": 5.023702329637655, + "learning_rate": 2.1582733812949638e-07, + "logits/chosen": -0.9385740160942078, + "logits/rejected": -0.943560779094696, + "logps/chosen": -224.21755981445312, + "logps/rejected": -217.0615234375, + "loss": 0.6882, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.002689545974135399, + "rewards/margins": 0.022182326763868332, + "rewards/rejected": -0.02487187273800373, + "step": 180 + }, + { + "epoch": 0.04558541266794626, + "grad_norm": 5.705487432813769, + "learning_rate": 2.278177458033573e-07, + "logits/chosen": -0.8128509521484375, + "logits/rejected": -0.8663360476493835, + "logps/chosen": -298.9399719238281, + "logps/rejected": -244.75387573242188, + "loss": 0.6867, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.004362248349934816, + "rewards/margins": 0.006819064728915691, + "rewards/rejected": -0.011181313544511795, + "step": 190 + }, + { + "epoch": 0.04798464491362764, + "grad_norm": 4.513448783590523, + "learning_rate": 2.398081534772182e-07, + "logits/chosen": -0.8495550155639648, + "logits/rejected": -0.7926044464111328, + "logps/chosen": -300.69775390625, + "logps/rejected": -287.484375, + "loss": 0.6838, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.004859840031713247, + "rewards/margins": 0.03267320245504379, + "rewards/rejected": -0.03753304481506348, + "step": 200 + }, + { + "epoch": 0.05038387715930902, + "grad_norm": 4.771408453293688, + "learning_rate": 2.517985611510791e-07, + "logits/chosen": -0.8157240152359009, + "logits/rejected": -0.8260771632194519, + "logps/chosen": -227.16909790039062, + "logps/rejected": -251.068115234375, + "loss": 0.6835, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.007311197463423014, + "rewards/margins": 0.01891271583735943, + "rewards/rejected": -0.026223912835121155, + "step": 210 + }, + { + "epoch": 0.052783109404990404, + "grad_norm": 4.625214029931686, + "learning_rate": 2.637889688249401e-07, + "logits/chosen": -0.7995166182518005, + "logits/rejected": -0.845539927482605, + "logps/chosen": -296.12158203125, + "logps/rejected": -306.5632019042969, + "loss": 0.6835, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.013005027547478676, + "rewards/margins": 0.015108886174857616, + "rewards/rejected": -0.028113916516304016, + "step": 220 + }, + { + "epoch": 0.05518234165067178, + "grad_norm": 5.010963725044197, + "learning_rate": 2.7577937649880093e-07, + "logits/chosen": -0.7807406783103943, + "logits/rejected": -0.6935420632362366, + "logps/chosen": -223.29647827148438, + "logps/rejected": -259.60162353515625, + "loss": 0.6792, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.0160575769841671, + "rewards/margins": 0.02244911529123783, + "rewards/rejected": -0.03850669413805008, + "step": 230 + }, + { + "epoch": 0.05758157389635317, + "grad_norm": 5.3808625345693715, + "learning_rate": 2.8776978417266184e-07, + "logits/chosen": -0.8535972833633423, + "logits/rejected": -0.8847853541374207, + "logps/chosen": -278.45166015625, + "logps/rejected": -233.5392608642578, + "loss": 0.6777, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.007952993735671043, + "rewards/margins": 0.029812496155500412, + "rewards/rejected": -0.037765491753816605, + "step": 240 + }, + { + "epoch": 0.05998080614203455, + "grad_norm": 5.070039825169341, + "learning_rate": 2.997601918465228e-07, + "logits/chosen": -0.7644799947738647, + "logits/rejected": -0.7972002029418945, + "logps/chosen": -219.7716064453125, + "logps/rejected": -215.15316772460938, + "loss": 0.6762, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.017915142700076103, + "rewards/margins": 0.026944369077682495, + "rewards/rejected": -0.04485950991511345, + "step": 250 + }, + { + "epoch": 0.06238003838771593, + "grad_norm": 4.754186632320252, + "learning_rate": 3.1175059952038366e-07, + "logits/chosen": -0.9055066108703613, + "logits/rejected": -0.8144194483757019, + "logps/chosen": -254.07125854492188, + "logps/rejected": -262.3128356933594, + "loss": 0.6714, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.025571072474122047, + "rewards/margins": 0.05727959796786308, + "rewards/rejected": -0.08285067230463028, + "step": 260 + }, + { + "epoch": 0.0647792706333973, + "grad_norm": 4.987049183479494, + "learning_rate": 3.2374100719424457e-07, + "logits/chosen": -0.804566502571106, + "logits/rejected": -0.9701822996139526, + "logps/chosen": -268.6120910644531, + "logps/rejected": -215.0527801513672, + "loss": 0.671, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.025534141808748245, + "rewards/margins": 0.01729689911007881, + "rewards/rejected": -0.04283104091882706, + "step": 270 + }, + { + "epoch": 0.0671785028790787, + "grad_norm": 4.996897467037669, + "learning_rate": 3.3573141486810554e-07, + "logits/chosen": -0.9051429629325867, + "logits/rejected": -0.8783286809921265, + "logps/chosen": -281.0115966796875, + "logps/rejected": -272.17315673828125, + "loss": 0.6616, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.02867250144481659, + "rewards/margins": 0.056794364005327225, + "rewards/rejected": -0.08546686172485352, + "step": 280 + }, + { + "epoch": 0.06957773512476008, + "grad_norm": 4.661711977634519, + "learning_rate": 3.477218225419664e-07, + "logits/chosen": -0.7616173028945923, + "logits/rejected": -0.6811286807060242, + "logps/chosen": -276.2654113769531, + "logps/rejected": -253.44387817382812, + "loss": 0.6618, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.030496900901198387, + "rewards/margins": 0.05069463700056076, + "rewards/rejected": -0.0811915397644043, + "step": 290 + }, + { + "epoch": 0.07197696737044146, + "grad_norm": 5.234931356849739, + "learning_rate": 3.597122302158273e-07, + "logits/chosen": -0.8670452237129211, + "logits/rejected": -0.8828343152999878, + "logps/chosen": -253.47982788085938, + "logps/rejected": -279.6092224121094, + "loss": 0.6617, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.05864090472459793, + "rewards/margins": 0.061179112643003464, + "rewards/rejected": -0.1198200210928917, + "step": 300 + }, + { + "epoch": 0.07437619961612284, + "grad_norm": 4.795640451841223, + "learning_rate": 3.7170263788968827e-07, + "logits/chosen": -0.7577391266822815, + "logits/rejected": -0.8055315017700195, + "logps/chosen": -261.38665771484375, + "logps/rejected": -237.2653350830078, + "loss": 0.6656, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.05181436613202095, + "rewards/margins": 0.08690813928842545, + "rewards/rejected": -0.1387225091457367, + "step": 310 + }, + { + "epoch": 0.07677543186180422, + "grad_norm": 4.6531369927460835, + "learning_rate": 3.836930455635491e-07, + "logits/chosen": -0.8808850049972534, + "logits/rejected": -0.9084761738777161, + "logps/chosen": -259.6866760253906, + "logps/rejected": -239.19082641601562, + "loss": 0.6584, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06028270721435547, + "rewards/margins": 0.0768328607082367, + "rewards/rejected": -0.13711555302143097, + "step": 320 + }, + { + "epoch": 0.07917466410748561, + "grad_norm": 4.7018147060937165, + "learning_rate": 3.9568345323741003e-07, + "logits/chosen": -0.7389161586761475, + "logits/rejected": -0.6371687650680542, + "logps/chosen": -243.20669555664062, + "logps/rejected": -291.19415283203125, + "loss": 0.6533, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10904283821582794, + "rewards/margins": 0.09945854544639587, + "rewards/rejected": -0.20850138366222382, + "step": 330 + }, + { + "epoch": 0.08157389635316699, + "grad_norm": 4.83587717123604, + "learning_rate": 4.07673860911271e-07, + "logits/chosen": -0.7003086805343628, + "logits/rejected": -0.7263328433036804, + "logps/chosen": -230.62255859375, + "logps/rejected": -264.69952392578125, + "loss": 0.6503, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.0853889137506485, + "rewards/margins": 0.13564836978912354, + "rewards/rejected": -0.22103726863861084, + "step": 340 + }, + { + "epoch": 0.08397312859884837, + "grad_norm": 5.295107224792075, + "learning_rate": 4.1966426858513185e-07, + "logits/chosen": -0.8799580335617065, + "logits/rejected": -0.8661099672317505, + "logps/chosen": -296.1805725097656, + "logps/rejected": -299.8250732421875, + "loss": 0.6515, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.14768368005752563, + "rewards/margins": 0.09014255553483963, + "rewards/rejected": -0.23782619833946228, + "step": 350 + }, + { + "epoch": 0.08637236084452975, + "grad_norm": 5.350703522133975, + "learning_rate": 4.3165467625899276e-07, + "logits/chosen": -0.7589418888092041, + "logits/rejected": -0.8757201433181763, + "logps/chosen": -253.6654510498047, + "logps/rejected": -221.54281616210938, + "loss": 0.656, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.14394545555114746, + "rewards/margins": 0.0798153355717659, + "rewards/rejected": -0.22376079857349396, + "step": 360 + }, + { + "epoch": 0.08877159309021113, + "grad_norm": 5.728112222958844, + "learning_rate": 4.436450839328537e-07, + "logits/chosen": -0.7830215096473694, + "logits/rejected": -0.7601363062858582, + "logps/chosen": -245.7024688720703, + "logps/rejected": -276.0493469238281, + "loss": 0.6436, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.17416706681251526, + "rewards/margins": 0.17155224084854126, + "rewards/rejected": -0.3457193076610565, + "step": 370 + }, + { + "epoch": 0.09117082533589252, + "grad_norm": 4.9514500106693, + "learning_rate": 4.556354916067146e-07, + "logits/chosen": -0.8882322311401367, + "logits/rejected": -0.8203359842300415, + "logps/chosen": -232.29898071289062, + "logps/rejected": -257.9759826660156, + "loss": 0.6243, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.14858679473400116, + "rewards/margins": 0.20146794617176056, + "rewards/rejected": -0.3500547409057617, + "step": 380 + }, + { + "epoch": 0.0935700575815739, + "grad_norm": 5.8336231290197045, + "learning_rate": 4.676258992805755e-07, + "logits/chosen": -0.7206496596336365, + "logits/rejected": -0.7572492361068726, + "logps/chosen": -277.67913818359375, + "logps/rejected": -261.2207946777344, + "loss": 0.6287, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2389966994524002, + "rewards/margins": 0.1022852212190628, + "rewards/rejected": -0.3412818908691406, + "step": 390 + }, + { + "epoch": 0.09596928982725528, + "grad_norm": 5.45994847173407, + "learning_rate": 4.796163069544364e-07, + "logits/chosen": -0.749753475189209, + "logits/rejected": -0.8122553825378418, + "logps/chosen": -262.7334289550781, + "logps/rejected": -268.3887634277344, + "loss": 0.6307, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.2442113608121872, + "rewards/margins": 0.20316508412361145, + "rewards/rejected": -0.44737643003463745, + "step": 400 + }, + { + "epoch": 0.09836852207293666, + "grad_norm": 5.702146266766378, + "learning_rate": 4.916067146282974e-07, + "logits/chosen": -0.8613287806510925, + "logits/rejected": -0.8260523676872253, + "logps/chosen": -268.2001037597656, + "logps/rejected": -314.8636474609375, + "loss": 0.6148, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2837222218513489, + "rewards/margins": 0.21735891699790955, + "rewards/rejected": -0.5010811686515808, + "step": 410 + }, + { + "epoch": 0.10076775431861804, + "grad_norm": 5.941538001240442, + "learning_rate": 4.999992108529978e-07, + "logits/chosen": -0.7500723600387573, + "logits/rejected": -0.7804423570632935, + "logps/chosen": -338.72479248046875, + "logps/rejected": -326.6620178222656, + "loss": 0.6186, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.3654174506664276, + "rewards/margins": 0.21260254085063934, + "rewards/rejected": -0.5780200362205505, + "step": 420 + }, + { + "epoch": 0.10316698656429943, + "grad_norm": 6.009621410768706, + "learning_rate": 4.999851817115532e-07, + "logits/chosen": -0.9403728246688843, + "logits/rejected": -0.8628015518188477, + "logps/chosen": -264.67034912109375, + "logps/rejected": -304.2147216796875, + "loss": 0.6196, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.326229989528656, + "rewards/margins": 0.4290277361869812, + "rewards/rejected": -0.7552577257156372, + "step": 430 + }, + { + "epoch": 0.10556621880998081, + "grad_norm": 5.690063285652722, + "learning_rate": 4.999536171027889e-07, + "logits/chosen": -0.8182615041732788, + "logits/rejected": -0.8673620223999023, + "logps/chosen": -304.6317443847656, + "logps/rejected": -316.14019775390625, + "loss": 0.6053, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.43472570180892944, + "rewards/margins": 0.2013782262802124, + "rewards/rejected": -0.6361039280891418, + "step": 440 + }, + { + "epoch": 0.10796545105566219, + "grad_norm": 6.437625645010029, + "learning_rate": 4.999045192408369e-07, + "logits/chosen": -0.7834080457687378, + "logits/rejected": -0.8002208471298218, + "logps/chosen": -268.7138366699219, + "logps/rejected": -268.2469787597656, + "loss": 0.6128, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.46527180075645447, + "rewards/margins": 0.16177940368652344, + "rewards/rejected": -0.6270512342453003, + "step": 450 + }, + { + "epoch": 0.11036468330134357, + "grad_norm": 6.346554276129153, + "learning_rate": 4.998378915697171e-07, + "logits/chosen": -0.8159794807434082, + "logits/rejected": -0.8230727910995483, + "logps/chosen": -291.6131591796875, + "logps/rejected": -324.294189453125, + "loss": 0.584, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.402113676071167, + "rewards/margins": 0.38154715299606323, + "rewards/rejected": -0.7836608290672302, + "step": 460 + }, + { + "epoch": 0.11276391554702495, + "grad_norm": 5.770852636176762, + "learning_rate": 4.997537387630958e-07, + "logits/chosen": -0.8236961364746094, + "logits/rejected": -0.8538937568664551, + "logps/chosen": -234.36978149414062, + "logps/rejected": -280.78802490234375, + "loss": 0.5721, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.3996524512767792, + "rewards/margins": 0.3925968408584595, + "rewards/rejected": -0.7922492623329163, + "step": 470 + }, + { + "epoch": 0.11516314779270634, + "grad_norm": 6.932352951399689, + "learning_rate": 4.996520667239582e-07, + "logits/chosen": -1.0455796718597412, + "logits/rejected": -0.961613655090332, + "logps/chosen": -268.81707763671875, + "logps/rejected": -362.69891357421875, + "loss": 0.5853, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5076214075088501, + "rewards/margins": 0.4294431805610657, + "rewards/rejected": -0.9370646476745605, + "step": 480 + }, + { + "epoch": 0.11756238003838772, + "grad_norm": 7.088135104125632, + "learning_rate": 4.995328825841939e-07, + "logits/chosen": -0.8383450508117676, + "logits/rejected": -0.7808793783187866, + "logps/chosen": -248.08639526367188, + "logps/rejected": -321.13519287109375, + "loss": 0.5738, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.48594579100608826, + "rewards/margins": 0.676557719707489, + "rewards/rejected": -1.1625034809112549, + "step": 490 + }, + { + "epoch": 0.1199616122840691, + "grad_norm": 7.550410687183492, + "learning_rate": 4.993961947040967e-07, + "logits/chosen": -0.8356281518936157, + "logits/rejected": -0.8823004961013794, + "logps/chosen": -340.29791259765625, + "logps/rejected": -333.99981689453125, + "loss": 0.6047, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.7787412405014038, + "rewards/margins": 0.24195539951324463, + "rewards/rejected": -1.0206966400146484, + "step": 500 + }, + { + "epoch": 0.12236084452975048, + "grad_norm": 8.043941220353142, + "learning_rate": 4.992420126717784e-07, + "logits/chosen": -0.8867794275283813, + "logits/rejected": -0.822262167930603, + "logps/chosen": -275.9137878417969, + "logps/rejected": -352.0505065917969, + "loss": 0.5757, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.46751028299331665, + "rewards/margins": 0.7203239798545837, + "rewards/rejected": -1.1878341436386108, + "step": 510 + }, + { + "epoch": 0.12476007677543186, + "grad_norm": 7.943939511731397, + "learning_rate": 4.990703473024958e-07, + "logits/chosen": -0.790678858757019, + "logits/rejected": -0.8552435636520386, + "logps/chosen": -336.27557373046875, + "logps/rejected": -372.5177001953125, + "loss": 0.6106, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.7617267370223999, + "rewards/margins": 0.3834512233734131, + "rewards/rejected": -1.145177960395813, + "step": 520 + }, + { + "epoch": 0.12715930902111325, + "grad_norm": 7.835632823524396, + "learning_rate": 4.98881210637893e-07, + "logits/chosen": -0.9683562517166138, + "logits/rejected": -0.9080503582954407, + "logps/chosen": -253.3780975341797, + "logps/rejected": -338.4334411621094, + "loss": 0.594, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.5727646350860596, + "rewards/margins": 0.47228097915649414, + "rewards/rejected": -1.0450456142425537, + "step": 530 + }, + { + "epoch": 0.1295585412667946, + "grad_norm": 6.170675144426816, + "learning_rate": 4.986746159451553e-07, + "logits/chosen": -0.8312419056892395, + "logits/rejected": -0.8283737897872925, + "logps/chosen": -292.1555480957031, + "logps/rejected": -331.30499267578125, + "loss": 0.5932, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.5170794725418091, + "rewards/margins": 0.4533451497554779, + "rewards/rejected": -0.9704246520996094, + "step": 540 + }, + { + "epoch": 0.131957773512476, + "grad_norm": 5.877933895862304, + "learning_rate": 4.984505777160795e-07, + "logits/chosen": -0.7666348218917847, + "logits/rejected": -0.7571445107460022, + "logps/chosen": -358.6742858886719, + "logps/rejected": -395.4786071777344, + "loss": 0.6025, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.7097586393356323, + "rewards/margins": 0.32830795645713806, + "rewards/rejected": -1.0380666255950928, + "step": 550 + }, + { + "epoch": 0.1343570057581574, + "grad_norm": 6.79860706241038, + "learning_rate": 4.982091116660574e-07, + "logits/chosen": -0.9468134045600891, + "logits/rejected": -1.0048458576202393, + "logps/chosen": -250.75411987304688, + "logps/rejected": -242.416259765625, + "loss": 0.6145, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.6376280784606934, + "rewards/margins": 0.16986791789531708, + "rewards/rejected": -0.8074960708618164, + "step": 560 + }, + { + "epoch": 0.13675623800383876, + "grad_norm": 8.675977571454341, + "learning_rate": 4.979502347329732e-07, + "logits/chosen": -0.7279460430145264, + "logits/rejected": -0.702147364616394, + "logps/chosen": -355.9872131347656, + "logps/rejected": -433.03436279296875, + "loss": 0.597, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.7968459129333496, + "rewards/margins": 0.5167227983474731, + "rewards/rejected": -1.3135687112808228, + "step": 570 + }, + { + "epoch": 0.13915547024952016, + "grad_norm": 10.049920254278586, + "learning_rate": 4.976739650760151e-07, + "logits/chosen": -0.9273891448974609, + "logits/rejected": -0.9307486414909363, + "logps/chosen": -308.32110595703125, + "logps/rejected": -333.7012634277344, + "loss": 0.5745, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.6520169973373413, + "rewards/margins": 0.3839523494243622, + "rewards/rejected": -1.0359693765640259, + "step": 580 + }, + { + "epoch": 0.14155470249520152, + "grad_norm": 8.308480938577231, + "learning_rate": 4.97380322074402e-07, + "logits/chosen": -0.7263349294662476, + "logits/rejected": -0.7392244338989258, + "logps/chosen": -283.90802001953125, + "logps/rejected": -315.4358825683594, + "loss": 0.5961, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7617335319519043, + "rewards/margins": 0.3228367567062378, + "rewards/rejected": -1.084570288658142, + "step": 590 + }, + { + "epoch": 0.14395393474088292, + "grad_norm": 7.912284801575183, + "learning_rate": 4.970693263260237e-07, + "logits/chosen": -0.9102510213851929, + "logits/rejected": -0.9313921928405762, + "logps/chosen": -329.23565673828125, + "logps/rejected": -352.90655517578125, + "loss": 0.5893, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.6352382898330688, + "rewards/margins": 0.5430476665496826, + "rewards/rejected": -1.178285837173462, + "step": 600 + }, + { + "epoch": 0.1463531669865643, + "grad_norm": 7.991602124810873, + "learning_rate": 4.967409996459966e-07, + "logits/chosen": -0.8471067547798157, + "logits/rejected": -0.8511239886283875, + "logps/chosen": -330.8809814453125, + "logps/rejected": -349.6996765136719, + "loss": 0.5632, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.6577950716018677, + "rewards/margins": 0.4866298735141754, + "rewards/rejected": -1.1444250345230103, + "step": 610 + }, + { + "epoch": 0.14875239923224567, + "grad_norm": 7.858534413433715, + "learning_rate": 4.963953650651326e-07, + "logits/chosen": -0.7265298962593079, + "logits/rejected": -0.7749903202056885, + "logps/chosen": -423.787841796875, + "logps/rejected": -374.99310302734375, + "loss": 0.5782, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.9267325401306152, + "rewards/margins": 0.3885432481765747, + "rewards/rejected": -1.31527578830719, + "step": 620 + }, + { + "epoch": 0.15115163147792707, + "grad_norm": 7.914507028628532, + "learning_rate": 4.960324468283248e-07, + "logits/chosen": -1.0132184028625488, + "logits/rejected": -1.020798921585083, + "logps/chosen": -295.89031982421875, + "logps/rejected": -348.19757080078125, + "loss": 0.5395, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.8688480257987976, + "rewards/margins": 0.48463934659957886, + "rewards/rejected": -1.353487253189087, + "step": 630 + }, + { + "epoch": 0.15355086372360843, + "grad_norm": 8.168951986763473, + "learning_rate": 4.956522703928451e-07, + "logits/chosen": -0.90748131275177, + "logits/rejected": -0.8296523094177246, + "logps/chosen": -306.3932189941406, + "logps/rejected": -355.0683898925781, + "loss": 0.5378, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.8589761853218079, + "rewards/margins": 0.45014849305152893, + "rewards/rejected": -1.3091247081756592, + "step": 640 + }, + { + "epoch": 0.15595009596928983, + "grad_norm": 12.189636245889604, + "learning_rate": 4.952548624265606e-07, + "logits/chosen": -0.8050463795661926, + "logits/rejected": -0.8074838519096375, + "logps/chosen": -363.4576416015625, + "logps/rejected": -372.71533203125, + "loss": 0.5913, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.9918814897537231, + "rewards/margins": 0.31118518114089966, + "rewards/rejected": -1.3030664920806885, + "step": 650 + }, + { + "epoch": 0.15834932821497122, + "grad_norm": 8.801792548075712, + "learning_rate": 4.948402508060607e-07, + "logits/chosen": -0.9548357129096985, + "logits/rejected": -0.944226086139679, + "logps/chosen": -309.63763427734375, + "logps/rejected": -365.5370178222656, + "loss": 0.5762, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.8808960914611816, + "rewards/margins": 0.6289945244789124, + "rewards/rejected": -1.5098905563354492, + "step": 660 + }, + { + "epoch": 0.16074856046065258, + "grad_norm": 9.037871356385454, + "learning_rate": 4.944084646147038e-07, + "logits/chosen": -0.8803000450134277, + "logits/rejected": -0.9197956323623657, + "logps/chosen": -398.33673095703125, + "logps/rejected": -418.3768005371094, + "loss": 0.5874, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.9903624653816223, + "rewards/margins": 0.4092163145542145, + "rewards/rejected": -1.3995788097381592, + "step": 670 + }, + { + "epoch": 0.16314779270633398, + "grad_norm": 7.995793666629949, + "learning_rate": 4.939595341405754e-07, + "logits/chosen": -0.8957052230834961, + "logits/rejected": -0.9519813656806946, + "logps/chosen": -327.54058837890625, + "logps/rejected": -375.4893798828125, + "loss": 0.5507, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.8798785209655762, + "rewards/margins": 0.5467365384101868, + "rewards/rejected": -1.4266151189804077, + "step": 680 + }, + { + "epoch": 0.16554702495201534, + "grad_norm": 9.303423916451855, + "learning_rate": 4.93493490874365e-07, + "logits/chosen": -0.8354761004447937, + "logits/rejected": -0.8360861539840698, + "logps/chosen": -314.87786865234375, + "logps/rejected": -362.77154541015625, + "loss": 0.5205, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.8782456517219543, + "rewards/margins": 0.41027846932411194, + "rewards/rejected": -1.2885241508483887, + "step": 690 + }, + { + "epoch": 0.16794625719769674, + "grad_norm": 9.577548468423071, + "learning_rate": 4.93010367507156e-07, + "logits/chosen": -0.9991138577461243, + "logits/rejected": -0.995546817779541, + "logps/chosen": -284.5464782714844, + "logps/rejected": -341.00433349609375, + "loss": 0.537, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.8869355320930481, + "rewards/margins": 0.7838705778121948, + "rewards/rejected": -1.6708061695098877, + "step": 700 + }, + { + "epoch": 0.17034548944337813, + "grad_norm": 10.701876294903373, + "learning_rate": 4.925101979281332e-07, + "logits/chosen": -0.841864287853241, + "logits/rejected": -0.936977207660675, + "logps/chosen": -371.80328369140625, + "logps/rejected": -404.70672607421875, + "loss": 0.5508, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.8815019726753235, + "rewards/margins": 0.852567195892334, + "rewards/rejected": -1.7340691089630127, + "step": 710 + }, + { + "epoch": 0.1727447216890595, + "grad_norm": 8.673550568480893, + "learning_rate": 4.919930172222054e-07, + "logits/chosen": -0.9126158952713013, + "logits/rejected": -0.9728757739067078, + "logps/chosen": -332.90386962890625, + "logps/rejected": -394.7867736816406, + "loss": 0.5303, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.9386744499206543, + "rewards/margins": 0.6791925430297852, + "rewards/rejected": -1.61786687374115, + "step": 720 + }, + { + "epoch": 0.1751439539347409, + "grad_norm": 10.4122473453474, + "learning_rate": 4.914588616675445e-07, + "logits/chosen": -1.0145832300186157, + "logits/rejected": -1.0208103656768799, + "logps/chosen": -288.4735412597656, + "logps/rejected": -360.450927734375, + "loss": 0.576, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.78793865442276, + "rewards/margins": 0.6522720456123352, + "rewards/rejected": -1.4402107000350952, + "step": 730 + }, + { + "epoch": 0.17754318618042225, + "grad_norm": 10.505660408208904, + "learning_rate": 4.909077687330404e-07, + "logits/chosen": -0.8563149571418762, + "logits/rejected": -0.8991769552230835, + "logps/chosen": -359.00299072265625, + "logps/rejected": -381.2506408691406, + "loss": 0.5227, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.0156606435775757, + "rewards/margins": 0.48986172676086426, + "rewards/rejected": -1.50552237033844, + "step": 740 + }, + { + "epoch": 0.17994241842610365, + "grad_norm": 10.897264447866968, + "learning_rate": 4.903397770756729e-07, + "logits/chosen": -0.9542545080184937, + "logits/rejected": -0.9912341833114624, + "logps/chosen": -351.9880065917969, + "logps/rejected": -430.06072998046875, + "loss": 0.5408, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.0298378467559814, + "rewards/margins": 0.8000310063362122, + "rewards/rejected": -1.8298689126968384, + "step": 750 + }, + { + "epoch": 0.18234165067178504, + "grad_norm": 8.170911605589284, + "learning_rate": 4.897549265378004e-07, + "logits/chosen": -0.9441335797309875, + "logits/rejected": -0.9833663702011108, + "logps/chosen": -413.30047607421875, + "logps/rejected": -478.78204345703125, + "loss": 0.5212, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.0744590759277344, + "rewards/margins": 0.6504513621330261, + "rewards/rejected": -1.7249103784561157, + "step": 760 + }, + { + "epoch": 0.1847408829174664, + "grad_norm": 10.341368234668948, + "learning_rate": 4.891532581443643e-07, + "logits/chosen": -1.0848381519317627, + "logits/rejected": -1.078758955001831, + "logps/chosen": -377.7035827636719, + "logps/rejected": -493.32928466796875, + "loss": 0.5197, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.9114073514938354, + "rewards/margins": 1.1706379652023315, + "rewards/rejected": -2.082045555114746, + "step": 770 + }, + { + "epoch": 0.1871401151631478, + "grad_norm": 11.04593711106465, + "learning_rate": 4.885348141000122e-07, + "logits/chosen": -0.9228931665420532, + "logits/rejected": -0.8969389200210571, + "logps/chosen": -337.7319030761719, + "logps/rejected": -435.77752685546875, + "loss": 0.5334, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.0482293367385864, + "rewards/margins": 0.77088862657547, + "rewards/rejected": -1.8191179037094116, + "step": 780 + }, + { + "epoch": 0.18953934740882916, + "grad_norm": 9.643533943369212, + "learning_rate": 4.878996377861367e-07, + "logits/chosen": -0.9793928265571594, + "logits/rejected": -1.0138920545578003, + "logps/chosen": -314.68878173828125, + "logps/rejected": -389.3453063964844, + "loss": 0.5027, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.099780559539795, + "rewards/margins": 0.6440693140029907, + "rewards/rejected": -1.7438499927520752, + "step": 790 + }, + { + "epoch": 0.19193857965451055, + "grad_norm": 9.649660460622457, + "learning_rate": 4.872477737578327e-07, + "logits/chosen": -0.9505901336669922, + "logits/rejected": -0.8831882476806641, + "logps/chosen": -372.86322021484375, + "logps/rejected": -490.08740234375, + "loss": 0.5045, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.1724433898925781, + "rewards/margins": 1.1937037706375122, + "rewards/rejected": -2.366147041320801, + "step": 800 + }, + { + "epoch": 0.19433781190019195, + "grad_norm": 11.639238021932554, + "learning_rate": 4.865792677407718e-07, + "logits/chosen": -0.988836407661438, + "logits/rejected": -1.0147147178649902, + "logps/chosen": -381.7936706542969, + "logps/rejected": -402.01043701171875, + "loss": 0.5603, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.4327753782272339, + "rewards/margins": 0.4645722806453705, + "rewards/rejected": -1.8973478078842163, + "step": 810 + }, + { + "epoch": 0.1967370441458733, + "grad_norm": 11.366678972402584, + "learning_rate": 4.858941666279955e-07, + "logits/chosen": -0.949034571647644, + "logits/rejected": -0.978123664855957, + "logps/chosen": -379.8930358886719, + "logps/rejected": -413.527099609375, + "loss": 0.5578, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.2424204349517822, + "rewards/margins": 0.4267699122428894, + "rewards/rejected": -1.6691904067993164, + "step": 820 + }, + { + "epoch": 0.1991362763915547, + "grad_norm": 10.723571043699938, + "learning_rate": 4.851925184766247e-07, + "logits/chosen": -0.9376896023750305, + "logits/rejected": -1.007285714149475, + "logps/chosen": -336.7314453125, + "logps/rejected": -412.2381896972656, + "loss": 0.5401, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.9813965559005737, + "rewards/margins": 0.9238882064819336, + "rewards/rejected": -1.9052846431732178, + "step": 830 + }, + { + "epoch": 0.20153550863723607, + "grad_norm": 11.28027279286211, + "learning_rate": 4.844743725044897e-07, + "logits/chosen": -0.9019923210144043, + "logits/rejected": -1.030314326286316, + "logps/chosen": -322.65948486328125, + "logps/rejected": -373.43670654296875, + "loss": 0.5148, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.9081098437309265, + "rewards/margins": 0.7220152616500854, + "rewards/rejected": -1.6301252841949463, + "step": 840 + }, + { + "epoch": 0.20393474088291746, + "grad_norm": 10.272798699401713, + "learning_rate": 4.837397790866774e-07, + "logits/chosen": -1.0090559720993042, + "logits/rejected": -0.9971574544906616, + "logps/chosen": -351.38568115234375, + "logps/rejected": -452.35888671875, + "loss": 0.5287, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.7977692484855652, + "rewards/margins": 1.1507394313812256, + "rewards/rejected": -1.948508858680725, + "step": 850 + }, + { + "epoch": 0.20633397312859886, + "grad_norm": 9.299129898380613, + "learning_rate": 4.829887897519974e-07, + "logits/chosen": -1.0525013208389282, + "logits/rejected": -1.0134930610656738, + "logps/chosen": -310.9789123535156, + "logps/rejected": -408.0035400390625, + "loss": 0.5345, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.9323593378067017, + "rewards/margins": 0.7073816061019897, + "rewards/rejected": -1.6397409439086914, + "step": 860 + }, + { + "epoch": 0.20873320537428022, + "grad_norm": 8.671420413674893, + "learning_rate": 4.82221457179368e-07, + "logits/chosen": -1.0403704643249512, + "logits/rejected": -1.0115526914596558, + "logps/chosen": -338.336181640625, + "logps/rejected": -430.98468017578125, + "loss": 0.5031, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.8510007858276367, + "rewards/margins": 1.0371253490447998, + "rewards/rejected": -1.888126015663147, + "step": 870 + }, + { + "epoch": 0.21113243761996162, + "grad_norm": 9.900230839528136, + "learning_rate": 4.814378351941206e-07, + "logits/chosen": -0.9543834924697876, + "logits/rejected": -0.9598924517631531, + "logps/chosen": -324.9510803222656, + "logps/rejected": -367.14776611328125, + "loss": 0.535, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.8847082853317261, + "rewards/margins": 0.5305272936820984, + "rewards/rejected": -1.4152355194091797, + "step": 880 + }, + { + "epoch": 0.21353166986564298, + "grad_norm": 9.272434926289451, + "learning_rate": 4.806379787642241e-07, + "logits/chosen": -1.007602334022522, + "logits/rejected": -0.9781683087348938, + "logps/chosen": -325.66839599609375, + "logps/rejected": -409.8434143066406, + "loss": 0.5685, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.9686576724052429, + "rewards/margins": 0.7592862844467163, + "rewards/rejected": -1.727943778038025, + "step": 890 + }, + { + "epoch": 0.21593090211132437, + "grad_norm": 8.284902734735002, + "learning_rate": 4.798219439964293e-07, + "logits/chosen": -1.0210028886795044, + "logits/rejected": -1.084486484527588, + "logps/chosen": -330.61688232421875, + "logps/rejected": -401.58001708984375, + "loss": 0.4945, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.0252277851104736, + "rewards/margins": 0.5490860939025879, + "rewards/rejected": -1.5743141174316406, + "step": 900 + }, + { + "epoch": 0.21833013435700577, + "grad_norm": 7.769735531369734, + "learning_rate": 4.78989788132333e-07, + "logits/chosen": -1.0263112783432007, + "logits/rejected": -0.9949830770492554, + "logps/chosen": -287.7435302734375, + "logps/rejected": -406.2470703125, + "loss": 0.4639, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.8495758771896362, + "rewards/margins": 1.082848310470581, + "rewards/rejected": -1.9324243068695068, + "step": 910 + }, + { + "epoch": 0.22072936660268713, + "grad_norm": 8.170204511457829, + "learning_rate": 4.781415695443631e-07, + "logits/chosen": -0.9709676504135132, + "logits/rejected": -0.9841318130493164, + "logps/chosen": -386.8170471191406, + "logps/rejected": -453.66571044921875, + "loss": 0.5319, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.131913185119629, + "rewards/margins": 0.5731815695762634, + "rewards/rejected": -1.7050946950912476, + "step": 920 + }, + { + "epoch": 0.22312859884836853, + "grad_norm": 8.525976901789441, + "learning_rate": 4.772773477316836e-07, + "logits/chosen": -0.916145920753479, + "logits/rejected": -0.9438095092773438, + "logps/chosen": -374.3315124511719, + "logps/rejected": -448.864013671875, + "loss": 0.5149, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.0942604541778564, + "rewards/margins": 0.7264348864555359, + "rewards/rejected": -1.8206952810287476, + "step": 930 + }, + { + "epoch": 0.2255278310940499, + "grad_norm": 12.196210253104345, + "learning_rate": 4.7639718331602117e-07, + "logits/chosen": -0.8837151527404785, + "logits/rejected": -0.883104145526886, + "logps/chosen": -365.3779296875, + "logps/rejected": -482.9402770996094, + "loss": 0.513, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.106382131576538, + "rewards/margins": 1.2564634084701538, + "rewards/rejected": -2.3628456592559814, + "step": 940 + }, + { + "epoch": 0.22792706333973128, + "grad_norm": 11.24723169528466, + "learning_rate": 4.7550113803741275e-07, + "logits/chosen": -0.9752639532089233, + "logits/rejected": -1.0816590785980225, + "logps/chosen": -385.86956787109375, + "logps/rejected": -397.8951416015625, + "loss": 0.5279, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.1907027959823608, + "rewards/margins": 0.7141844630241394, + "rewards/rejected": -1.9048871994018555, + "step": 950 + }, + { + "epoch": 0.23032629558541268, + "grad_norm": 10.85809850081635, + "learning_rate": 4.7458927474987454e-07, + "logits/chosen": -0.8752844929695129, + "logits/rejected": -0.9301251173019409, + "logps/chosen": -393.0145568847656, + "logps/rejected": -391.2146301269531, + "loss": 0.4881, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.897404670715332, + "rewards/margins": 0.5560187101364136, + "rewards/rejected": -1.453423261642456, + "step": 960 + }, + { + "epoch": 0.23272552783109404, + "grad_norm": 10.402623732075206, + "learning_rate": 4.7366165741699347e-07, + "logits/chosen": -0.8860486745834351, + "logits/rejected": -0.933478057384491, + "logps/chosen": -427.3525390625, + "logps/rejected": -484.7303771972656, + "loss": 0.5073, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.2039297819137573, + "rewards/margins": 0.8201591372489929, + "rewards/rejected": -2.0240888595581055, + "step": 970 + }, + { + "epoch": 0.23512476007677544, + "grad_norm": 10.420724727543737, + "learning_rate": 4.727183511074401e-07, + "logits/chosen": -1.0526244640350342, + "logits/rejected": -1.0677237510681152, + "logps/chosen": -407.3580017089844, + "logps/rejected": -422.97711181640625, + "loss": 0.5227, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.4225349426269531, + "rewards/margins": 0.27180853486061096, + "rewards/rejected": -1.6943433284759521, + "step": 980 + }, + { + "epoch": 0.2375239923224568, + "grad_norm": 10.619438612411185, + "learning_rate": 4.717594219904043e-07, + "logits/chosen": -0.9206924438476562, + "logits/rejected": -1.0142751932144165, + "logps/chosen": -369.31671142578125, + "logps/rejected": -393.07769775390625, + "loss": 0.5147, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.1511679887771606, + "rewards/margins": 0.6668072938919067, + "rewards/rejected": -1.8179752826690674, + "step": 990 + }, + { + "epoch": 0.2399232245681382, + "grad_norm": 11.563760369851115, + "learning_rate": 4.7078493733095393e-07, + "logits/chosen": -1.0036451816558838, + "logits/rejected": -1.0272811651229858, + "logps/chosen": -328.3309020996094, + "logps/rejected": -445.1051330566406, + "loss": 0.5186, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.9604378938674927, + "rewards/margins": 1.0202256441116333, + "rewards/rejected": -1.9806636571884155, + "step": 1000 + }, + { + "epoch": 0.2423224568138196, + "grad_norm": 12.237132564608, + "learning_rate": 4.6979496548531614e-07, + "logits/chosen": -1.0756165981292725, + "logits/rejected": -1.0265034437179565, + "logps/chosen": -367.80816650390625, + "logps/rejected": -496.73583984375, + "loss": 0.5139, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.24051034450531, + "rewards/margins": 0.7898116111755371, + "rewards/rejected": -2.0303218364715576, + "step": 1010 + }, + { + "epoch": 0.24472168905950095, + "grad_norm": 10.269516186946461, + "learning_rate": 4.6878957589608293e-07, + "logits/chosen": -1.0293898582458496, + "logits/rejected": -0.9980789422988892, + "logps/chosen": -382.0529479980469, + "logps/rejected": -538.0097045898438, + "loss": 0.5223, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.3041552305221558, + "rewards/margins": 1.118542194366455, + "rewards/rejected": -2.4226975440979004, + "step": 1020 + }, + { + "epoch": 0.24712092130518235, + "grad_norm": 9.05638881736831, + "learning_rate": 4.6776883908733956e-07, + "logits/chosen": -1.1042773723602295, + "logits/rejected": -1.17487633228302, + "logps/chosen": -395.26531982421875, + "logps/rejected": -437.3297424316406, + "loss": 0.4855, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.207805871963501, + "rewards/margins": 1.0340633392333984, + "rewards/rejected": -2.2418694496154785, + "step": 1030 + }, + { + "epoch": 0.2495201535508637, + "grad_norm": 11.280624127584458, + "learning_rate": 4.667328266597178e-07, + "logits/chosen": -1.0581328868865967, + "logits/rejected": -1.0977164506912231, + "logps/chosen": -372.9267883300781, + "logps/rejected": -451.66009521484375, + "loss": 0.4696, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.2191641330718994, + "rewards/margins": 0.8757356405258179, + "rewards/rejected": -2.0948996543884277, + "step": 1040 + }, + { + "epoch": 0.2519193857965451, + "grad_norm": 9.42640563137626, + "learning_rate": 4.6568161128537354e-07, + "logits/chosen": -0.9135216474533081, + "logits/rejected": -1.0233265161514282, + "logps/chosen": -347.3189392089844, + "logps/rejected": -395.1881103515625, + "loss": 0.5045, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.0815908908843994, + "rewards/margins": 0.9593822360038757, + "rewards/rejected": -2.04097318649292, + "step": 1050 + }, + { + "epoch": 0.2543186180422265, + "grad_norm": 11.866401340802232, + "learning_rate": 4.6461526670288877e-07, + "logits/chosen": -0.9709247350692749, + "logits/rejected": -0.9719343185424805, + "logps/chosen": -392.99859619140625, + "logps/rejected": -453.5699768066406, + "loss": 0.528, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.2631556987762451, + "rewards/margins": 0.8903251886367798, + "rewards/rejected": -2.1534810066223145, + "step": 1060 + }, + { + "epoch": 0.2567178502879079, + "grad_norm": 11.302715265992923, + "learning_rate": 4.635338677120994e-07, + "logits/chosen": -1.168346643447876, + "logits/rejected": -1.1507086753845215, + "logps/chosen": -359.5755920410156, + "logps/rejected": -499.57611083984375, + "loss": 0.4546, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.1692172288894653, + "rewards/margins": 1.161725640296936, + "rewards/rejected": -2.3309428691864014, + "step": 1070 + }, + { + "epoch": 0.2591170825335892, + "grad_norm": 10.538848822959446, + "learning_rate": 4.6243749016884835e-07, + "logits/chosen": -0.9228366017341614, + "logits/rejected": -0.9742697477340698, + "logps/chosen": -393.55865478515625, + "logps/rejected": -613.6375732421875, + "loss": 0.4983, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.4483288526535034, + "rewards/margins": 1.6291669607162476, + "rewards/rejected": -3.07749605178833, + "step": 1080 + }, + { + "epoch": 0.2615163147792706, + "grad_norm": 15.598964227058538, + "learning_rate": 4.613262109796645e-07, + "logits/chosen": -1.0201536417007446, + "logits/rejected": -0.9872088432312012, + "logps/chosen": -388.32159423828125, + "logps/rejected": -569.1511840820312, + "loss": 0.5029, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.3905266523361206, + "rewards/margins": 1.4417566061019897, + "rewards/rejected": -2.8322832584381104, + "step": 1090 + }, + { + "epoch": 0.263915547024952, + "grad_norm": 10.412089533834534, + "learning_rate": 4.602001080963678e-07, + "logits/chosen": -1.0111544132232666, + "logits/rejected": -1.0401980876922607, + "logps/chosen": -415.40728759765625, + "logps/rejected": -510.05926513671875, + "loss": 0.5026, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.4814815521240234, + "rewards/margins": 1.197854995727539, + "rewards/rejected": -2.6793365478515625, + "step": 1100 + }, + { + "epoch": 0.2663147792706334, + "grad_norm": 10.947859358372769, + "learning_rate": 4.590592605106017e-07, + "logits/chosen": -1.040560007095337, + "logits/rejected": -1.0646026134490967, + "logps/chosen": -393.5159606933594, + "logps/rejected": -461.66864013671875, + "loss": 0.5327, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -1.0464909076690674, + "rewards/margins": 0.9438858032226562, + "rewards/rejected": -1.9903767108917236, + "step": 1110 + }, + { + "epoch": 0.2687140115163148, + "grad_norm": 10.271128985395928, + "learning_rate": 4.5790374824829165e-07, + "logits/chosen": -1.0148289203643799, + "logits/rejected": -1.0559489727020264, + "logps/chosen": -268.1494140625, + "logps/rejected": -381.31707763671875, + "loss": 0.4943, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.941615104675293, + "rewards/margins": 0.9399920701980591, + "rewards/rejected": -1.8816072940826416, + "step": 1120 + }, + { + "epoch": 0.27111324376199614, + "grad_norm": 14.425090791310733, + "learning_rate": 4.5673365236403216e-07, + "logits/chosen": -1.112364411354065, + "logits/rejected": -1.1866824626922607, + "logps/chosen": -286.95819091796875, + "logps/rejected": -442.0973205566406, + "loss": 0.4935, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.07503342628479, + "rewards/margins": 1.2303565740585327, + "rewards/rejected": -2.305389881134033, + "step": 1130 + }, + { + "epoch": 0.27351247600767753, + "grad_norm": 9.80302236807559, + "learning_rate": 4.5554905493540075e-07, + "logits/chosen": -1.1836650371551514, + "logits/rejected": -1.1820929050445557, + "logps/chosen": -316.4050598144531, + "logps/rejected": -470.31878662109375, + "loss": 0.4759, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.1204586029052734, + "rewards/margins": 1.4639848470687866, + "rewards/rejected": -2.5844433307647705, + "step": 1140 + }, + { + "epoch": 0.2759117082533589, + "grad_norm": 10.46266914998414, + "learning_rate": 4.5435003905720074e-07, + "logits/chosen": -1.1017481088638306, + "logits/rejected": -1.1844024658203125, + "logps/chosen": -397.47625732421875, + "logps/rejected": -495.28021240234375, + "loss": 0.4805, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.3188390731811523, + "rewards/margins": 1.2287555932998657, + "rewards/rejected": -2.5475945472717285, + "step": 1150 + }, + { + "epoch": 0.2783109404990403, + "grad_norm": 13.802897784185829, + "learning_rate": 4.531366888356324e-07, + "logits/chosen": -1.1266191005706787, + "logits/rejected": -1.1002269983291626, + "logps/chosen": -320.9498596191406, + "logps/rejected": -531.9197998046875, + "loss": 0.4492, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.4906550645828247, + "rewards/margins": 1.6824109554290771, + "rewards/rejected": -3.1730659008026123, + "step": 1160 + }, + { + "epoch": 0.2807101727447217, + "grad_norm": 18.106306987333475, + "learning_rate": 4.519090893823931e-07, + "logits/chosen": -1.197227954864502, + "logits/rejected": -1.2483270168304443, + "logps/chosen": -382.64154052734375, + "logps/rejected": -496.5115661621094, + "loss": 0.4706, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.5682321786880493, + "rewards/margins": 1.1952569484710693, + "rewards/rejected": -2.763489246368408, + "step": 1170 + }, + { + "epoch": 0.28310940499040305, + "grad_norm": 12.198433930244558, + "learning_rate": 4.5066732680870734e-07, + "logits/chosen": -1.14240562915802, + "logits/rejected": -1.2160329818725586, + "logps/chosen": -349.089599609375, + "logps/rejected": -437.95263671875, + "loss": 0.4708, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.0620535612106323, + "rewards/margins": 1.2815176248550415, + "rewards/rejected": -2.343571186065674, + "step": 1180 + }, + { + "epoch": 0.28550863723608444, + "grad_norm": 11.921423223157628, + "learning_rate": 4.494114882192862e-07, + "logits/chosen": -1.1435043811798096, + "logits/rejected": -1.1478463411331177, + "logps/chosen": -354.37200927734375, + "logps/rejected": -491.3812561035156, + "loss": 0.4856, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.1012070178985596, + "rewards/margins": 1.6031544208526611, + "rewards/rejected": -2.7043614387512207, + "step": 1190 + }, + { + "epoch": 0.28790786948176583, + "grad_norm": 12.244819859614891, + "learning_rate": 4.4814166170621735e-07, + "logits/chosen": -1.2157360315322876, + "logits/rejected": -1.2492748498916626, + "logps/chosen": -377.10089111328125, + "logps/rejected": -490.47576904296875, + "loss": 0.4863, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.4752638339996338, + "rewards/margins": 1.339400291442871, + "rewards/rejected": -2.814664363861084, + "step": 1200 + }, + { + "epoch": 0.2903071017274472, + "grad_norm": 13.831179008022444, + "learning_rate": 4.468579363427858e-07, + "logits/chosen": -1.1191552877426147, + "logits/rejected": -1.1627353429794312, + "logps/chosen": -398.43060302734375, + "logps/rejected": -515.5997314453125, + "loss": 0.4935, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.595988154411316, + "rewards/margins": 1.365837812423706, + "rewards/rejected": -2.9618258476257324, + "step": 1210 + }, + { + "epoch": 0.2927063339731286, + "grad_norm": 12.594765793696023, + "learning_rate": 4.4556040217722555e-07, + "logits/chosen": -1.248393177986145, + "logits/rejected": -1.2377097606658936, + "logps/chosen": -360.39739990234375, + "logps/rejected": -560.1267700195312, + "loss": 0.4758, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.3543295860290527, + "rewards/margins": 1.653586745262146, + "rewards/rejected": -3.007916212081909, + "step": 1220 + }, + { + "epoch": 0.29510556621880996, + "grad_norm": 14.658580352857703, + "learning_rate": 4.442491502264033e-07, + "logits/chosen": -1.1721140146255493, + "logits/rejected": -1.205334186553955, + "logps/chosen": -372.86688232421875, + "logps/rejected": -431.7432556152344, + "loss": 0.4817, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.6799089908599854, + "rewards/margins": 0.7123211622238159, + "rewards/rejected": -2.392230272293091, + "step": 1230 + }, + { + "epoch": 0.29750479846449135, + "grad_norm": 11.30132796398374, + "learning_rate": 4.429242724694338e-07, + "logits/chosen": -1.2736822366714478, + "logits/rejected": -1.2566986083984375, + "logps/chosen": -386.52886962890625, + "logps/rejected": -536.6383056640625, + "loss": 0.481, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.5427987575531006, + "rewards/margins": 1.334408164024353, + "rewards/rejected": -2.877207040786743, + "step": 1240 + }, + { + "epoch": 0.29990403071017274, + "grad_norm": 11.45540426694367, + "learning_rate": 4.4158586184122817e-07, + "logits/chosen": -1.1903879642486572, + "logits/rejected": -1.2902122735977173, + "logps/chosen": -395.0272521972656, + "logps/rejected": -514.5638427734375, + "loss": 0.4587, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.2198941707611084, + "rewards/margins": 1.4861936569213867, + "rewards/rejected": -2.706087589263916, + "step": 1250 + }, + { + "epoch": 0.30230326295585414, + "grad_norm": 11.855125244864809, + "learning_rate": 4.4023401222597443e-07, + "logits/chosen": -1.0970547199249268, + "logits/rejected": -1.2169404029846191, + "logps/chosen": -414.16546630859375, + "logps/rejected": -512.9844970703125, + "loss": 0.4475, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.4606043100357056, + "rewards/margins": 1.193366289138794, + "rewards/rejected": -2.653970241546631, + "step": 1260 + }, + { + "epoch": 0.30470249520153553, + "grad_norm": 12.674642065308364, + "learning_rate": 4.3886881845055235e-07, + "logits/chosen": -1.1634668111801147, + "logits/rejected": -1.2584116458892822, + "logps/chosen": -352.8936767578125, + "logps/rejected": -543.56396484375, + "loss": 0.425, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -1.2661120891571045, + "rewards/margins": 1.9337279796600342, + "rewards/rejected": -3.1998400688171387, + "step": 1270 + }, + { + "epoch": 0.30710172744721687, + "grad_norm": 11.264055060402159, + "learning_rate": 4.374903762778814e-07, + "logits/chosen": -1.2410491704940796, + "logits/rejected": -1.2714251279830933, + "logps/chosen": -431.96771240234375, + "logps/rejected": -563.4871826171875, + "loss": 0.4646, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.8797252178192139, + "rewards/margins": 1.5396064519882202, + "rewards/rejected": -3.4193317890167236, + "step": 1280 + }, + { + "epoch": 0.30950095969289826, + "grad_norm": 12.684189653289591, + "learning_rate": 4.3609878240020356e-07, + "logits/chosen": -1.1412585973739624, + "logits/rejected": -1.2513244152069092, + "logps/chosen": -471.1331481933594, + "logps/rejected": -574.435546875, + "loss": 0.4442, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.9110310077667236, + "rewards/margins": 1.6200447082519531, + "rewards/rejected": -3.5310757160186768, + "step": 1290 + }, + { + "epoch": 0.31190019193857965, + "grad_norm": 12.047131092826515, + "learning_rate": 4.346941344323005e-07, + "logits/chosen": -1.2705122232437134, + "logits/rejected": -1.3864275217056274, + "logps/chosen": -426.615478515625, + "logps/rejected": -498.53863525390625, + "loss": 0.4877, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.9668481349945068, + "rewards/margins": 1.2020810842514038, + "rewards/rejected": -3.1689295768737793, + "step": 1300 + }, + { + "epoch": 0.31429942418426104, + "grad_norm": 12.410299379758362, + "learning_rate": 4.332765309046467e-07, + "logits/chosen": -1.2316473722457886, + "logits/rejected": -1.24112868309021, + "logps/chosen": -399.12347412109375, + "logps/rejected": -496.7018127441406, + "loss": 0.4967, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.460193157196045, + "rewards/margins": 1.3175086975097656, + "rewards/rejected": -2.7777016162872314, + "step": 1310 + }, + { + "epoch": 0.31669865642994244, + "grad_norm": 15.436355881965927, + "learning_rate": 4.3184607125649754e-07, + "logits/chosen": -1.1702232360839844, + "logits/rejected": -1.208388328552246, + "logps/chosen": -382.8656311035156, + "logps/rejected": -577.5721435546875, + "loss": 0.4796, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.2770411968231201, + "rewards/margins": 1.7833073139190674, + "rewards/rejected": -3.0603485107421875, + "step": 1320 + }, + { + "epoch": 0.3190978886756238, + "grad_norm": 12.142589026682131, + "learning_rate": 4.304028558289141e-07, + "logits/chosen": -1.190229892730713, + "logits/rejected": -1.2123523950576782, + "logps/chosen": -417.66973876953125, + "logps/rejected": -549.3724365234375, + "loss": 0.4502, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.5125898122787476, + "rewards/margins": 1.5069903135299683, + "rewards/rejected": -3.019580125808716, + "step": 1330 + }, + { + "epoch": 0.32149712092130517, + "grad_norm": 10.693755286537069, + "learning_rate": 4.28946985857725e-07, + "logits/chosen": -1.2291617393493652, + "logits/rejected": -1.2268054485321045, + "logps/chosen": -412.4300231933594, + "logps/rejected": -607.4716186523438, + "loss": 0.4468, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.5301696062088013, + "rewards/margins": 2.023198366165161, + "rewards/rejected": -3.553368091583252, + "step": 1340 + }, + { + "epoch": 0.32389635316698656, + "grad_norm": 11.965095051615716, + "learning_rate": 4.2747856346642445e-07, + "logits/chosen": -1.1990400552749634, + "logits/rejected": -1.2139463424682617, + "logps/chosen": -364.44830322265625, + "logps/rejected": -493.9198303222656, + "loss": 0.4572, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.5109294652938843, + "rewards/margins": 1.342005729675293, + "rewards/rejected": -2.8529350757598877, + "step": 1350 + }, + { + "epoch": 0.32629558541266795, + "grad_norm": 14.030913432042324, + "learning_rate": 4.2599769165900933e-07, + "logits/chosen": -1.2126591205596924, + "logits/rejected": -1.2595001459121704, + "logps/chosen": -447.2684631347656, + "logps/rejected": -650.9678344726562, + "loss": 0.4937, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.054805278778076, + "rewards/margins": 2.237144947052002, + "rewards/rejected": -4.29194974899292, + "step": 1360 + }, + { + "epoch": 0.32869481765834935, + "grad_norm": 12.230893363228695, + "learning_rate": 4.245044743127535e-07, + "logits/chosen": -1.3876649141311646, + "logits/rejected": -1.345571517944336, + "logps/chosen": -420.2569885253906, + "logps/rejected": -553.880859375, + "loss": 0.4671, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.8658406734466553, + "rewards/margins": 1.272249460220337, + "rewards/rejected": -3.138089895248413, + "step": 1370 + }, + { + "epoch": 0.3310940499040307, + "grad_norm": 18.683125746223514, + "learning_rate": 4.229990161709214e-07, + "logits/chosen": -1.2496898174285889, + "logits/rejected": -1.1489694118499756, + "logps/chosen": -374.80059814453125, + "logps/rejected": -599.7036743164062, + "loss": 0.5007, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -1.621886968612671, + "rewards/margins": 1.9923725128173828, + "rewards/rejected": -3.614259719848633, + "step": 1380 + }, + { + "epoch": 0.3334932821497121, + "grad_norm": 11.157118917089283, + "learning_rate": 4.214814228354204e-07, + "logits/chosen": -1.2218196392059326, + "logits/rejected": -1.2584130764007568, + "logps/chosen": -449.47979736328125, + "logps/rejected": -655.5496826171875, + "loss": 0.4609, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -1.958253264427185, + "rewards/margins": 2.2051448822021484, + "rewards/rejected": -4.163398265838623, + "step": 1390 + }, + { + "epoch": 0.33589251439539347, + "grad_norm": 12.746683018380017, + "learning_rate": 4.1995180075939375e-07, + "logits/chosen": -1.3946723937988281, + "logits/rejected": -1.3772931098937988, + "logps/chosen": -442.54339599609375, + "logps/rejected": -570.6209106445312, + "loss": 0.4417, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.7088006734848022, + "rewards/margins": 1.4957315921783447, + "rewards/rejected": -3.2045321464538574, + "step": 1400 + }, + { + "epoch": 0.33829174664107486, + "grad_norm": 12.044939890412799, + "learning_rate": 4.1841025723975297e-07, + "logits/chosen": -1.188822627067566, + "logits/rejected": -1.219215989112854, + "logps/chosen": -408.2652282714844, + "logps/rejected": -595.224365234375, + "loss": 0.4384, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.3506208658218384, + "rewards/margins": 2.057248115539551, + "rewards/rejected": -3.4078686237335205, + "step": 1410 + }, + { + "epoch": 0.34069097888675626, + "grad_norm": 11.734403426656625, + "learning_rate": 4.168569004096516e-07, + "logits/chosen": -1.1842548847198486, + "logits/rejected": -1.1583616733551025, + "logps/chosen": -385.1539306640625, + "logps/rejected": -605.2198486328125, + "loss": 0.4206, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.6475471258163452, + "rewards/margins": 1.9943052530288696, + "rewards/rejected": -3.6418521404266357, + "step": 1420 + }, + { + "epoch": 0.3430902111324376, + "grad_norm": 9.504127526147306, + "learning_rate": 4.152918392308997e-07, + "logits/chosen": -1.5127391815185547, + "logits/rejected": -1.4623585939407349, + "logps/chosen": -459.7693786621094, + "logps/rejected": -617.2127685546875, + "loss": 0.4449, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.342836380004883, + "rewards/margins": 1.6365751028060913, + "rewards/rejected": -3.9794116020202637, + "step": 1430 + }, + { + "epoch": 0.345489443378119, + "grad_norm": 20.373026702217544, + "learning_rate": 4.137151834863213e-07, + "logits/chosen": -1.300527811050415, + "logits/rejected": -1.2247835397720337, + "logps/chosen": -456.451416015625, + "logps/rejected": -692.865478515625, + "loss": 0.513, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.3793184757232666, + "rewards/margins": 1.9824209213256836, + "rewards/rejected": -4.361739635467529, + "step": 1440 + }, + { + "epoch": 0.3478886756238004, + "grad_norm": 16.685897640348756, + "learning_rate": 4.121270437720526e-07, + "logits/chosen": -1.1498229503631592, + "logits/rejected": -1.1219522953033447, + "logps/chosen": -400.486328125, + "logps/rejected": -553.6771240234375, + "loss": 0.454, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.084050416946411, + "rewards/margins": 1.0678091049194336, + "rewards/rejected": -3.151859760284424, + "step": 1450 + }, + { + "epoch": 0.3502879078694818, + "grad_norm": 11.077851343051192, + "learning_rate": 4.105275314897852e-07, + "logits/chosen": -1.3562729358673096, + "logits/rejected": -1.3173191547393799, + "logps/chosen": -371.6280517578125, + "logps/rejected": -650.3944702148438, + "loss": 0.4699, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.779268503189087, + "rewards/margins": 2.3870112895965576, + "rewards/rejected": -4.1662797927856445, + "step": 1460 + }, + { + "epoch": 0.35268714011516317, + "grad_norm": 14.118121128015067, + "learning_rate": 4.089167588389508e-07, + "logits/chosen": -1.1345436573028564, + "logits/rejected": -1.2407124042510986, + "logps/chosen": -508.22467041015625, + "logps/rejected": -647.5037841796875, + "loss": 0.4706, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.88066828250885, + "rewards/margins": 1.833953619003296, + "rewards/rejected": -3.7146217823028564, + "step": 1470 + }, + { + "epoch": 0.3550863723608445, + "grad_norm": 15.18907635756053, + "learning_rate": 4.072948388088515e-07, + "logits/chosen": -1.151426911354065, + "logits/rejected": -1.1657240390777588, + "logps/chosen": -432.26025390625, + "logps/rejected": -617.5946044921875, + "loss": 0.4774, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.788640022277832, + "rewards/margins": 1.791176438331604, + "rewards/rejected": -3.5798168182373047, + "step": 1480 + }, + { + "epoch": 0.3574856046065259, + "grad_norm": 14.317431275466847, + "learning_rate": 4.056618851707334e-07, + "logits/chosen": -1.1826136112213135, + "logits/rejected": -1.2314163446426392, + "logps/chosen": -389.71746826171875, + "logps/rejected": -564.3736572265625, + "loss": 0.4339, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.384912371635437, + "rewards/margins": 1.6980838775634766, + "rewards/rejected": -3.082996368408203, + "step": 1490 + }, + { + "epoch": 0.3598848368522073, + "grad_norm": 15.887257243467523, + "learning_rate": 4.0401801246980675e-07, + "logits/chosen": -1.3013509511947632, + "logits/rejected": -1.3612343072891235, + "logps/chosen": -393.45379638671875, + "logps/rejected": -499.9305114746094, + "loss": 0.4618, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.9321129322052002, + "rewards/margins": 1.2076321840286255, + "rewards/rejected": -3.139744520187378, + "step": 1500 + }, + { + "epoch": 0.3622840690978887, + "grad_norm": 15.741703489655176, + "learning_rate": 4.0236333601721043e-07, + "logits/chosen": -1.2282559871673584, + "logits/rejected": -1.1802866458892822, + "logps/chosen": -474.82177734375, + "logps/rejected": -591.8897094726562, + "loss": 0.4637, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -1.9944359064102173, + "rewards/margins": 1.0587834119796753, + "rewards/rejected": -3.0532193183898926, + "step": 1510 + }, + { + "epoch": 0.3646833013435701, + "grad_norm": 17.204220874752522, + "learning_rate": 4.0069797188192364e-07, + "logits/chosen": -1.2478222846984863, + "logits/rejected": -1.24253511428833, + "logps/chosen": -451.41009521484375, + "logps/rejected": -622.8987426757812, + "loss": 0.4667, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8151447772979736, + "rewards/margins": 1.9790477752685547, + "rewards/rejected": -3.7941925525665283, + "step": 1520 + }, + { + "epoch": 0.3670825335892514, + "grad_norm": 14.543319199762806, + "learning_rate": 3.9902203688262417e-07, + "logits/chosen": -1.1997919082641602, + "logits/rejected": -1.2181050777435303, + "logps/chosen": -405.4238586425781, + "logps/rejected": -524.9036254882812, + "loss": 0.4355, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.5551273822784424, + "rewards/margins": 1.3034616708755493, + "rewards/rejected": -2.858588695526123, + "step": 1530 + }, + { + "epoch": 0.3694817658349328, + "grad_norm": 11.536440138005817, + "learning_rate": 3.9733564857949365e-07, + "logits/chosen": -1.27651047706604, + "logits/rejected": -1.3442704677581787, + "logps/chosen": -513.730224609375, + "logps/rejected": -606.420166015625, + "loss": 0.4516, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.227952003479004, + "rewards/margins": 1.299182653427124, + "rewards/rejected": -3.527134656906128, + "step": 1540 + }, + { + "epoch": 0.3718809980806142, + "grad_norm": 13.953076272266463, + "learning_rate": 3.9563892526597177e-07, + "logits/chosen": -1.302734136581421, + "logits/rejected": -1.257291555404663, + "logps/chosen": -375.5953369140625, + "logps/rejected": -528.0511474609375, + "loss": 0.4562, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.6560758352279663, + "rewards/margins": 1.0600662231445312, + "rewards/rejected": -2.716142177581787, + "step": 1550 + }, + { + "epoch": 0.3742802303262956, + "grad_norm": 11.662781366416757, + "learning_rate": 3.9393198596045795e-07, + "logits/chosen": -1.2684893608093262, + "logits/rejected": -1.2016541957855225, + "logps/chosen": -372.7796630859375, + "logps/rejected": -552.9669799804688, + "loss": 0.4854, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.6109774112701416, + "rewards/margins": 1.5703436136245728, + "rewards/rejected": -3.181320905685425, + "step": 1560 + }, + { + "epoch": 0.376679462571977, + "grad_norm": 11.657236582029201, + "learning_rate": 3.922149503979628e-07, + "logits/chosen": -1.2256669998168945, + "logits/rejected": -1.2542228698730469, + "logps/chosen": -448.0088806152344, + "logps/rejected": -790.6217041015625, + "loss": 0.449, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -1.899916410446167, + "rewards/margins": 3.3502612113952637, + "rewards/rejected": -5.25017786026001, + "step": 1570 + }, + { + "epoch": 0.3790786948176583, + "grad_norm": 13.35177426898264, + "learning_rate": 3.904879390217095e-07, + "logits/chosen": -1.3202365636825562, + "logits/rejected": -1.3830177783966064, + "logps/chosen": -455.22967529296875, + "logps/rejected": -581.9063720703125, + "loss": 0.4407, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.1927895545959473, + "rewards/margins": 1.4322152137756348, + "rewards/rejected": -3.625004529953003, + "step": 1580 + }, + { + "epoch": 0.3814779270633397, + "grad_norm": 13.573747186048726, + "learning_rate": 3.8875107297468463e-07, + "logits/chosen": -1.3035602569580078, + "logits/rejected": -1.245466947555542, + "logps/chosen": -416.05084228515625, + "logps/rejected": -740.8409423828125, + "loss": 0.4497, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.8670103549957275, + "rewards/margins": 2.8193886280059814, + "rewards/rejected": -4.686398983001709, + "step": 1590 + }, + { + "epoch": 0.3838771593090211, + "grad_norm": 13.749682710553607, + "learning_rate": 3.87004474091141e-07, + "logits/chosen": -1.176382303237915, + "logits/rejected": -1.177053689956665, + "logps/chosen": -393.9292907714844, + "logps/rejected": -558.60888671875, + "loss": 0.4624, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.7796415090560913, + "rewards/margins": 1.491105318069458, + "rewards/rejected": -3.2707467079162598, + "step": 1600 + }, + { + "epoch": 0.3862763915547025, + "grad_norm": 12.151112854676569, + "learning_rate": 3.8524826488805114e-07, + "logits/chosen": -1.4273042678833008, + "logits/rejected": -1.4034093618392944, + "logps/chosen": -442.8750915527344, + "logps/rejected": -616.5904541015625, + "loss": 0.5055, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -1.7405941486358643, + "rewards/margins": 2.146420955657959, + "rewards/rejected": -3.8870151042938232, + "step": 1610 + }, + { + "epoch": 0.3886756238003839, + "grad_norm": 9.870668269338232, + "learning_rate": 3.834825685565133e-07, + "logits/chosen": -1.358819842338562, + "logits/rejected": -1.4594396352767944, + "logps/chosen": -363.05926513671875, + "logps/rejected": -469.5292053222656, + "loss": 0.4181, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.3789103031158447, + "rewards/margins": 1.4029977321624756, + "rewards/rejected": -2.781907796859741, + "step": 1620 + }, + { + "epoch": 0.39107485604606523, + "grad_norm": 12.06984534361086, + "learning_rate": 3.8170750895311007e-07, + "logits/chosen": -1.233335018157959, + "logits/rejected": -1.2165110111236572, + "logps/chosen": -442.47021484375, + "logps/rejected": -587.38818359375, + "loss": 0.4164, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.698992133140564, + "rewards/margins": 1.7231460809707642, + "rewards/rejected": -3.422138214111328, + "step": 1630 + }, + { + "epoch": 0.3934740882917466, + "grad_norm": 11.992529219223993, + "learning_rate": 3.7992321059122045e-07, + "logits/chosen": -1.2353224754333496, + "logits/rejected": -1.3311399221420288, + "logps/chosen": -461.4444274902344, + "logps/rejected": -634.5919189453125, + "loss": 0.4524, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.387753963470459, + "rewards/margins": 1.8558781147003174, + "rewards/rejected": -4.2436323165893555, + "step": 1640 + }, + { + "epoch": 0.395873320537428, + "grad_norm": 13.334807101320292, + "learning_rate": 3.7812979863228576e-07, + "logits/chosen": -1.4029086828231812, + "logits/rejected": -1.4273648262023926, + "logps/chosen": -434.45782470703125, + "logps/rejected": -619.6177978515625, + "loss": 0.4505, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.4253311157226562, + "rewards/margins": 1.7180039882659912, + "rewards/rejected": -4.143334865570068, + "step": 1650 + }, + { + "epoch": 0.3982725527831094, + "grad_norm": 14.545492077815492, + "learning_rate": 3.763273988770296e-07, + "logits/chosen": -1.30009126663208, + "logits/rejected": -1.3962773084640503, + "logps/chosen": -408.5030822753906, + "logps/rejected": -618.9495239257812, + "loss": 0.4406, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -1.7208665609359741, + "rewards/margins": 2.1056978702545166, + "rewards/rejected": -3.826564311981201, + "step": 1660 + }, + { + "epoch": 0.4006717850287908, + "grad_norm": 14.42291716594792, + "learning_rate": 3.7451613775663405e-07, + "logits/chosen": -1.2345439195632935, + "logits/rejected": -1.1613664627075195, + "logps/chosen": -431.56683349609375, + "logps/rejected": -756.0855102539062, + "loss": 0.459, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.0580296516418457, + "rewards/margins": 3.100363254547119, + "rewards/rejected": -5.158392906188965, + "step": 1670 + }, + { + "epoch": 0.40307101727447214, + "grad_norm": 13.846182180264046, + "learning_rate": 3.726961423238706e-07, + "logits/chosen": -1.3649914264678955, + "logits/rejected": -1.3719325065612793, + "logps/chosen": -399.53228759765625, + "logps/rejected": -725.3392333984375, + "loss": 0.4439, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.87088143825531, + "rewards/margins": 3.0345349311828613, + "rewards/rejected": -4.905416965484619, + "step": 1680 + }, + { + "epoch": 0.40547024952015354, + "grad_norm": 13.862779001584391, + "learning_rate": 3.708675402441882e-07, + "logits/chosen": -1.1647714376449585, + "logits/rejected": -1.3258562088012695, + "logps/chosen": -452.04046630859375, + "logps/rejected": -574.329833984375, + "loss": 0.4604, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -1.89810049533844, + "rewards/margins": 1.4911009073257446, + "rewards/rejected": -3.3892014026641846, + "step": 1690 + }, + { + "epoch": 0.40786948176583493, + "grad_norm": 13.918601228648548, + "learning_rate": 3.6903045978675775e-07, + "logits/chosen": -1.1976958513259888, + "logits/rejected": -1.2260172367095947, + "logps/chosen": -395.0202941894531, + "logps/rejected": -686.57177734375, + "loss": 0.4473, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.7495826482772827, + "rewards/margins": 3.0733304023742676, + "rewards/rejected": -4.82291316986084, + "step": 1700 + }, + { + "epoch": 0.4102687140115163, + "grad_norm": 11.597661521745744, + "learning_rate": 3.6718502981547474e-07, + "logits/chosen": -1.2923458814620972, + "logits/rejected": -1.266068458557129, + "logps/chosen": -408.5785827636719, + "logps/rejected": -584.2232055664062, + "loss": 0.4637, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.6165058612823486, + "rewards/margins": 1.335237741470337, + "rewards/rejected": -2.9517433643341064, + "step": 1710 + }, + { + "epoch": 0.4126679462571977, + "grad_norm": 11.408780497698448, + "learning_rate": 3.6533137977991986e-07, + "logits/chosen": -1.2169060707092285, + "logits/rejected": -1.2535167932510376, + "logps/chosen": -421.34320068359375, + "logps/rejected": -612.0996704101562, + "loss": 0.4868, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.618795394897461, + "rewards/margins": 1.626263976097107, + "rewards/rejected": -3.2450594902038574, + "step": 1720 + }, + { + "epoch": 0.41506717850287905, + "grad_norm": 14.279826921613408, + "learning_rate": 3.6346963970627865e-07, + "logits/chosen": -1.255234718322754, + "logits/rejected": -1.146271824836731, + "logps/chosen": -421.241455078125, + "logps/rejected": -664.4832763671875, + "loss": 0.4133, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.8466880321502686, + "rewards/margins": 2.243635416030884, + "rewards/rejected": -4.090323448181152, + "step": 1730 + }, + { + "epoch": 0.41746641074856045, + "grad_norm": 13.239703723404237, + "learning_rate": 3.615999401882207e-07, + "logits/chosen": -1.5136260986328125, + "logits/rejected": -1.4776732921600342, + "logps/chosen": -430.2723693847656, + "logps/rejected": -688.0913696289062, + "loss": 0.4462, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.2629590034484863, + "rewards/margins": 2.4102649688720703, + "rewards/rejected": -4.673223495483398, + "step": 1740 + }, + { + "epoch": 0.41986564299424184, + "grad_norm": 11.59386325114309, + "learning_rate": 3.597224123777389e-07, + "logits/chosen": -1.2650483846664429, + "logits/rejected": -1.2001924514770508, + "logps/chosen": -438.1640625, + "logps/rejected": -684.7293701171875, + "loss": 0.4401, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.011852741241455, + "rewards/margins": 2.2977705001831055, + "rewards/rejected": -4.3096232414245605, + "step": 1750 + }, + { + "epoch": 0.42226487523992323, + "grad_norm": 13.837858721173825, + "learning_rate": 3.5783718797595e-07, + "logits/chosen": -1.3295023441314697, + "logits/rejected": -1.4063255786895752, + "logps/chosen": -463.84027099609375, + "logps/rejected": -608.602294921875, + "loss": 0.4515, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.9257501363754272, + "rewards/margins": 1.8007062673568726, + "rewards/rejected": -3.7264564037323, + "step": 1760 + }, + { + "epoch": 0.4246641074856046, + "grad_norm": 18.356726333155102, + "learning_rate": 3.559443992238558e-07, + "logits/chosen": -1.272377848625183, + "logits/rejected": -1.2996290922164917, + "logps/chosen": -379.01922607421875, + "logps/rejected": -670.1522827148438, + "loss": 0.478, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -1.478298306465149, + "rewards/margins": 2.6530449390411377, + "rewards/rejected": -4.131342887878418, + "step": 1770 + }, + { + "epoch": 0.42706333973128596, + "grad_norm": 11.835957489925283, + "learning_rate": 3.540441788930673e-07, + "logits/chosen": -1.210574746131897, + "logits/rejected": -1.208512306213379, + "logps/chosen": -486.7884216308594, + "logps/rejected": -726.0255126953125, + "loss": 0.4257, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.0521273612976074, + "rewards/margins": 2.712066888809204, + "rewards/rejected": -4.764194488525391, + "step": 1780 + }, + { + "epoch": 0.42946257197696736, + "grad_norm": 14.562344268328332, + "learning_rate": 3.5213666027649123e-07, + "logits/chosen": -1.318163275718689, + "logits/rejected": -1.4022140502929688, + "logps/chosen": -517.6325073242188, + "logps/rejected": -609.1475830078125, + "loss": 0.4657, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.6355819702148438, + "rewards/margins": 1.281418800354004, + "rewards/rejected": -3.9170005321502686, + "step": 1790 + }, + { + "epoch": 0.43186180422264875, + "grad_norm": 14.530291818469866, + "learning_rate": 3.5022197717898017e-07, + "logits/chosen": -1.1955803632736206, + "logits/rejected": -1.2934623956680298, + "logps/chosen": -425.0252380371094, + "logps/rejected": -642.680419921875, + "loss": 0.3978, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.185080051422119, + "rewards/margins": 2.503284215927124, + "rewards/rejected": -4.688364028930664, + "step": 1800 + }, + { + "epoch": 0.43426103646833014, + "grad_norm": 14.242954877580903, + "learning_rate": 3.4830026390794633e-07, + "logits/chosen": -1.2469186782836914, + "logits/rejected": -1.3009865283966064, + "logps/chosen": -522.1409301757812, + "logps/rejected": -738.9695434570312, + "loss": 0.3894, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.4441685676574707, + "rewards/margins": 2.662193775177002, + "rewards/rejected": -5.106362342834473, + "step": 1810 + }, + { + "epoch": 0.43666026871401153, + "grad_norm": 9.558413013643854, + "learning_rate": 3.4637165526394104e-07, + "logits/chosen": -1.3089157342910767, + "logits/rejected": -1.2989561557769775, + "logps/chosen": -416.4744567871094, + "logps/rejected": -597.2052612304688, + "loss": 0.4459, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.026801347732544, + "rewards/margins": 1.6668050289154053, + "rewards/rejected": -3.6936068534851074, + "step": 1820 + }, + { + "epoch": 0.43905950095969287, + "grad_norm": 14.440126018329256, + "learning_rate": 3.4443628653119814e-07, + "logits/chosen": -1.1547075510025024, + "logits/rejected": -1.141548752784729, + "logps/chosen": -483.82611083984375, + "logps/rejected": -792.4986572265625, + "loss": 0.4601, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.251401424407959, + "rewards/margins": 2.6689209938049316, + "rewards/rejected": -4.920322895050049, + "step": 1830 + }, + { + "epoch": 0.44145873320537427, + "grad_norm": 14.248839559891321, + "learning_rate": 3.424942934681453e-07, + "logits/chosen": -1.2738267183303833, + "logits/rejected": -1.395904302597046, + "logps/chosen": -409.0834045410156, + "logps/rejected": -667.9530639648438, + "loss": 0.442, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -1.819021463394165, + "rewards/margins": 2.630481243133545, + "rewards/rejected": -4.449502468109131, + "step": 1840 + }, + { + "epoch": 0.44385796545105566, + "grad_norm": 17.408491732936906, + "learning_rate": 3.405458122978804e-07, + "logits/chosen": -1.340673804283142, + "logits/rejected": -1.329695463180542, + "logps/chosen": -464.5760803222656, + "logps/rejected": -588.0704345703125, + "loss": 0.4376, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -1.9475841522216797, + "rewards/margins": 1.5327271223068237, + "rewards/rejected": -3.480311632156372, + "step": 1850 + }, + { + "epoch": 0.44625719769673705, + "grad_norm": 17.70224543826807, + "learning_rate": 3.3859097969861633e-07, + "logits/chosen": -1.230010986328125, + "logits/rejected": -1.2035300731658936, + "logps/chosen": -465.7073669433594, + "logps/rejected": -621.9720458984375, + "loss": 0.413, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.0379059314727783, + "rewards/margins": 1.874291181564331, + "rewards/rejected": -3.9121971130371094, + "step": 1860 + }, + { + "epoch": 0.44865642994241844, + "grad_norm": 15.872391005642767, + "learning_rate": 3.366299327940936e-07, + "logits/chosen": -1.2397372722625732, + "logits/rejected": -1.1490371227264404, + "logps/chosen": -457.95428466796875, + "logps/rejected": -709.8195190429688, + "loss": 0.4361, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.8263241052627563, + "rewards/margins": 2.2876338958740234, + "rewards/rejected": -4.113957405090332, + "step": 1870 + }, + { + "epoch": 0.4510556621880998, + "grad_norm": 13.36488995176592, + "learning_rate": 3.3466280914396117e-07, + "logits/chosen": -1.2212001085281372, + "logits/rejected": -1.2048442363739014, + "logps/chosen": -419.3158264160156, + "logps/rejected": -692.9566650390625, + "loss": 0.4364, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.9814516305923462, + "rewards/margins": 2.5255990028381348, + "rewards/rejected": -4.507050514221191, + "step": 1880 + }, + { + "epoch": 0.4534548944337812, + "grad_norm": 17.862568943622634, + "learning_rate": 3.326897467341281e-07, + "logits/chosen": -1.2388769388198853, + "logits/rejected": -1.2880442142486572, + "logps/chosen": -386.8263244628906, + "logps/rejected": -666.2364501953125, + "loss": 0.4564, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.99746835231781, + "rewards/margins": 2.632657766342163, + "rewards/rejected": -4.630125999450684, + "step": 1890 + }, + { + "epoch": 0.45585412667946257, + "grad_norm": 15.985949773009649, + "learning_rate": 3.3071088396708335e-07, + "logits/chosen": -1.3479843139648438, + "logits/rejected": -1.2633702754974365, + "logps/chosen": -371.4305114746094, + "logps/rejected": -675.2957763671875, + "loss": 0.4398, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.869970679283142, + "rewards/margins": 2.837160348892212, + "rewards/rejected": -4.7071309089660645, + "step": 1900 + }, + { + "epoch": 0.45825335892514396, + "grad_norm": 12.92794837231206, + "learning_rate": 3.2872635965218824e-07, + "logits/chosen": -1.1383055448532104, + "logits/rejected": -1.1345998048782349, + "logps/chosen": -482.2320251464844, + "logps/rejected": -699.35693359375, + "loss": 0.4548, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.414862871170044, + "rewards/margins": 1.9727367162704468, + "rewards/rejected": -4.387599468231201, + "step": 1910 + }, + { + "epoch": 0.46065259117082535, + "grad_norm": 11.420466341932578, + "learning_rate": 3.2673631299593905e-07, + "logits/chosen": -1.1113722324371338, + "logits/rejected": -1.2537376880645752, + "logps/chosen": -475.3736877441406, + "logps/rejected": -699.142822265625, + "loss": 0.429, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.2085394859313965, + "rewards/margins": 2.2637460231781006, + "rewards/rejected": -4.472285270690918, + "step": 1920 + }, + { + "epoch": 0.4630518234165067, + "grad_norm": 15.008753955364448, + "learning_rate": 3.247408835922024e-07, + "logits/chosen": -1.177137017250061, + "logits/rejected": -1.1614809036254883, + "logps/chosen": -539.2333374023438, + "logps/rejected": -795.7393188476562, + "loss": 0.4632, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.4702744483947754, + "rewards/margins": 2.489729404449463, + "rewards/rejected": -4.960003852844238, + "step": 1930 + }, + { + "epoch": 0.4654510556621881, + "grad_norm": 14.291733539194276, + "learning_rate": 3.2274021141242306e-07, + "logits/chosen": -1.1657174825668335, + "logits/rejected": -1.1828467845916748, + "logps/chosen": -456.3023376464844, + "logps/rejected": -699.6104736328125, + "loss": 0.4162, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.1304402351379395, + "rewards/margins": 2.3655619621276855, + "rewards/rejected": -4.496002197265625, + "step": 1940 + }, + { + "epoch": 0.4678502879078695, + "grad_norm": 23.848910537253687, + "learning_rate": 3.2073443679580613e-07, + "logits/chosen": -1.0966782569885254, + "logits/rejected": -1.1587860584259033, + "logps/chosen": -455.427734375, + "logps/rejected": -578.6062622070312, + "loss": 0.4329, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.9730167388916016, + "rewards/margins": 1.2179396152496338, + "rewards/rejected": -3.1909565925598145, + "step": 1950 + }, + { + "epoch": 0.47024952015355087, + "grad_norm": 11.44940099558639, + "learning_rate": 3.1872370043947194e-07, + "logits/chosen": -1.3351823091506958, + "logits/rejected": -1.3538246154785156, + "logps/chosen": -409.07293701171875, + "logps/rejected": -723.4902954101562, + "loss": 0.4044, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.5190470218658447, + "rewards/margins": 3.2088820934295654, + "rewards/rejected": -4.72792911529541, + "step": 1960 + }, + { + "epoch": 0.47264875239923226, + "grad_norm": 17.234175648711105, + "learning_rate": 3.167081433885874e-07, + "logits/chosen": -0.9567463994026184, + "logits/rejected": -1.000705599784851, + "logps/chosen": -531.3121948242188, + "logps/rejected": -729.120849609375, + "loss": 0.371, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.2870779037475586, + "rewards/margins": 1.6067097187042236, + "rewards/rejected": -3.893787384033203, + "step": 1970 + }, + { + "epoch": 0.4750479846449136, + "grad_norm": 16.389722152541037, + "learning_rate": 3.14687907026472e-07, + "logits/chosen": -1.0606454610824585, + "logits/rejected": -1.1345421075820923, + "logps/chosen": -389.0166320800781, + "logps/rejected": -602.7018432617188, + "loss": 0.4226, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -1.7767404317855835, + "rewards/margins": 1.9564529657363892, + "rewards/rejected": -3.7331931591033936, + "step": 1980 + }, + { + "epoch": 0.477447216890595, + "grad_norm": 14.662707269745031, + "learning_rate": 3.126631330646801e-07, + "logits/chosen": -1.0552016496658325, + "logits/rejected": -1.088765263557434, + "logps/chosen": -541.9423217773438, + "logps/rejected": -711.3270263671875, + "loss": 0.4593, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.5332961082458496, + "rewards/margins": 1.669792890548706, + "rewards/rejected": -4.203089714050293, + "step": 1990 + }, + { + "epoch": 0.4798464491362764, + "grad_norm": 16.149947240810953, + "learning_rate": 3.1063396353306097e-07, + "logits/chosen": -1.2002663612365723, + "logits/rejected": -1.3123780488967896, + "logps/chosen": -463.75640869140625, + "logps/rejected": -653.8046875, + "loss": 0.4244, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.132018566131592, + "rewards/margins": 2.276289224624634, + "rewards/rejected": -4.408308029174805, + "step": 2000 + }, + { + "epoch": 0.4798464491362764, + "eval_logits/chosen": -1.2865077257156372, + "eval_logits/rejected": -1.286653995513916, + "eval_logps/chosen": -464.0679626464844, + "eval_logps/rejected": -732.3422241210938, + "eval_loss": 0.4295724630355835, + "eval_rewards/accuracies": 0.824999988079071, + "eval_rewards/chosen": -2.255470037460327, + "eval_rewards/margins": 2.6071014404296875, + "eval_rewards/rejected": -4.862571716308594, + "eval_runtime": 214.5851, + "eval_samples_per_second": 20.789, + "eval_steps_per_second": 0.326, + "step": 2000 + }, + { + "epoch": 0.4822456813819578, + "grad_norm": 11.158252638659, + "learning_rate": 3.0860054076979535e-07, + "logits/chosen": -1.225662112236023, + "logits/rejected": -1.192147135734558, + "logps/chosen": -505.77191162109375, + "logps/rejected": -732.186767578125, + "loss": 0.4265, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.518340587615967, + "rewards/margins": 2.517788887023926, + "rewards/rejected": -5.036129474639893, + "step": 2010 + }, + { + "epoch": 0.4846449136276392, + "grad_norm": 15.918645486110469, + "learning_rate": 3.065630074114115e-07, + "logits/chosen": -1.2051775455474854, + "logits/rejected": -1.2806353569030762, + "logps/chosen": -469.9501953125, + "logps/rejected": -781.4722290039062, + "loss": 0.438, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.963057518005371, + "rewards/margins": 3.50740385055542, + "rewards/rejected": -5.470460891723633, + "step": 2020 + }, + { + "epoch": 0.4870441458733205, + "grad_norm": 16.15646785556536, + "learning_rate": 3.0452150638277947e-07, + "logits/chosen": -1.1673038005828857, + "logits/rejected": -1.1155571937561035, + "logps/chosen": -405.2390441894531, + "logps/rejected": -619.8092651367188, + "loss": 0.4506, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.970237374305725, + "rewards/margins": 2.0424869060516357, + "rewards/rejected": -4.01272439956665, + "step": 2030 + }, + { + "epoch": 0.4894433781190019, + "grad_norm": 11.754069249870854, + "learning_rate": 3.024761808870856e-07, + "logits/chosen": -1.3374531269073486, + "logits/rejected": -1.317359447479248, + "logps/chosen": -427.8550720214844, + "logps/rejected": -775.02880859375, + "loss": 0.3996, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.074645757675171, + "rewards/margins": 3.4690213203430176, + "rewards/rejected": -5.543666839599609, + "step": 2040 + }, + { + "epoch": 0.4918426103646833, + "grad_norm": 20.96567040506931, + "learning_rate": 3.004271743957875e-07, + "logits/chosen": -1.0144097805023193, + "logits/rejected": -0.9976661801338196, + "logps/chosen": -516.1878662109375, + "logps/rejected": -665.16259765625, + "loss": 0.4411, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.640017509460449, + "rewards/margins": 1.281677007675171, + "rewards/rejected": -3.921694278717041, + "step": 2050 + }, + { + "epoch": 0.4942418426103647, + "grad_norm": 18.281834799077405, + "learning_rate": 2.983746306385499e-07, + "logits/chosen": -1.3244729042053223, + "logits/rejected": -1.2717360258102417, + "logps/chosen": -431.5679626464844, + "logps/rejected": -677.4711303710938, + "loss": 0.4167, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.0494117736816406, + "rewards/margins": 2.2310073375701904, + "rewards/rejected": -4.28041934967041, + "step": 2060 + }, + { + "epoch": 0.4966410748560461, + "grad_norm": 17.365428628060563, + "learning_rate": 2.963186935931628e-07, + "logits/chosen": -1.2782858610153198, + "logits/rejected": -1.2341490983963013, + "logps/chosen": -478.1234436035156, + "logps/rejected": -617.0537109375, + "loss": 0.4379, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.1914291381835938, + "rewards/margins": 1.4232972860336304, + "rewards/rejected": -3.6147263050079346, + "step": 2070 + }, + { + "epoch": 0.4990403071017274, + "grad_norm": 12.528209474957825, + "learning_rate": 2.9425950747544176e-07, + "logits/chosen": -1.1168311834335327, + "logits/rejected": -1.1911078691482544, + "logps/chosen": -565.50830078125, + "logps/rejected": -848.2404174804688, + "loss": 0.4305, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.775134563446045, + "rewards/margins": 3.091364622116089, + "rewards/rejected": -5.8664984703063965, + "step": 2080 + }, + { + "epoch": 0.5014395393474088, + "grad_norm": 18.44109381423814, + "learning_rate": 2.921972167291119e-07, + "logits/chosen": -1.1824846267700195, + "logits/rejected": -1.2502390146255493, + "logps/chosen": -494.66259765625, + "logps/rejected": -717.1361694335938, + "loss": 0.4187, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.2825825214385986, + "rewards/margins": 2.0693299770355225, + "rewards/rejected": -4.351912498474121, + "step": 2090 + }, + { + "epoch": 0.5038387715930902, + "grad_norm": 15.890171516115618, + "learning_rate": 2.9013196601567567e-07, + "logits/chosen": -1.087349772453308, + "logits/rejected": -1.0854907035827637, + "logps/chosen": -417.927490234375, + "logps/rejected": -603.4684448242188, + "loss": 0.5027, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.8442258834838867, + "rewards/margins": 1.7202110290527344, + "rewards/rejected": -3.5644371509552, + "step": 2100 + }, + { + "epoch": 0.5062380038387716, + "grad_norm": 12.705040080557687, + "learning_rate": 2.8806390020426555e-07, + "logits/chosen": -1.2159066200256348, + "logits/rejected": -1.1780139207839966, + "logps/chosen": -435.548095703125, + "logps/rejected": -650.815185546875, + "loss": 0.3877, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.9098494052886963, + "rewards/margins": 2.0971081256866455, + "rewards/rejected": -4.006957054138184, + "step": 2110 + }, + { + "epoch": 0.508637236084453, + "grad_norm": 18.982869027144268, + "learning_rate": 2.8599316436148187e-07, + "logits/chosen": -1.1808390617370605, + "logits/rejected": -1.1597092151641846, + "logps/chosen": -484.2972717285156, + "logps/rejected": -646.4186401367188, + "loss": 0.4346, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.4805798530578613, + "rewards/margins": 1.6115633249282837, + "rewards/rejected": -4.0921430587768555, + "step": 2120 + }, + { + "epoch": 0.5110364683301344, + "grad_norm": 12.789123364626542, + "learning_rate": 2.8391990374121723e-07, + "logits/chosen": -1.2177742719650269, + "logits/rejected": -1.1412584781646729, + "logps/chosen": -479.531494140625, + "logps/rejected": -768.7166748046875, + "loss": 0.4518, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.5543904304504395, + "rewards/margins": 2.6575286388397217, + "rewards/rejected": -5.211918830871582, + "step": 2130 + }, + { + "epoch": 0.5134357005758158, + "grad_norm": 11.99254712258343, + "learning_rate": 2.818442637744669e-07, + "logits/chosen": -1.2230430841445923, + "logits/rejected": -1.2256507873535156, + "logps/chosen": -452.6123046875, + "logps/rejected": -675.5736083984375, + "loss": 0.4139, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.214447259902954, + "rewards/margins": 2.154179334640503, + "rewards/rejected": -4.368626594543457, + "step": 2140 + }, + { + "epoch": 0.5158349328214972, + "grad_norm": 13.21777154530317, + "learning_rate": 2.797663900591284e-07, + "logits/chosen": -1.2086105346679688, + "logits/rejected": -1.2604271173477173, + "logps/chosen": -446.7076721191406, + "logps/rejected": -617.2671508789062, + "loss": 0.398, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.9999488592147827, + "rewards/margins": 2.0054893493652344, + "rewards/rejected": -4.005438327789307, + "step": 2150 + }, + { + "epoch": 0.5182341650671785, + "grad_norm": 13.837490902323914, + "learning_rate": 2.776864283497874e-07, + "logits/chosen": -1.2421865463256836, + "logits/rejected": -1.3203766345977783, + "logps/chosen": -435.1234436035156, + "logps/rejected": -740.7435302734375, + "loss": 0.4333, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.179964542388916, + "rewards/margins": 3.039562940597534, + "rewards/rejected": -5.219527244567871, + "step": 2160 + }, + { + "epoch": 0.5206333973128598, + "grad_norm": 12.069623770694928, + "learning_rate": 2.756045245474943e-07, + "logits/chosen": -1.1063625812530518, + "logits/rejected": -1.072736144065857, + "logps/chosen": -469.37939453125, + "logps/rejected": -645.7225341796875, + "loss": 0.4177, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.081900119781494, + "rewards/margins": 1.5807701349258423, + "rewards/rejected": -3.662670135498047, + "step": 2170 + }, + { + "epoch": 0.5230326295585412, + "grad_norm": 15.586578076435153, + "learning_rate": 2.7352082468952977e-07, + "logits/chosen": -1.2151401042938232, + "logits/rejected": -1.244135856628418, + "logps/chosen": -479.72796630859375, + "logps/rejected": -837.6242065429688, + "loss": 0.4676, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.5905280113220215, + "rewards/margins": 3.3142483234405518, + "rewards/rejected": -5.904776096343994, + "step": 2180 + }, + { + "epoch": 0.5254318618042226, + "grad_norm": 17.30867538238994, + "learning_rate": 2.7143547493916e-07, + "logits/chosen": -1.2965537309646606, + "logits/rejected": -1.2317306995391846, + "logps/chosen": -434.38067626953125, + "logps/rejected": -838.6102294921875, + "loss": 0.4131, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -1.950038194656372, + "rewards/margins": 4.041052341461182, + "rewards/rejected": -5.991089820861816, + "step": 2190 + }, + { + "epoch": 0.527831094049904, + "grad_norm": 11.72682555257016, + "learning_rate": 2.693486215753853e-07, + "logits/chosen": -1.2447761297225952, + "logits/rejected": -1.2085590362548828, + "logps/chosen": -437.3580627441406, + "logps/rejected": -767.2139892578125, + "loss": 0.4264, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.104055404663086, + "rewards/margins": 3.457770586013794, + "rewards/rejected": -5.561825752258301, + "step": 2200 + }, + { + "epoch": 0.5302303262955854, + "grad_norm": 16.198064432954975, + "learning_rate": 2.6726041098267805e-07, + "logits/chosen": -1.168273687362671, + "logits/rejected": -1.2161977291107178, + "logps/chosen": -497.7745666503906, + "logps/rejected": -609.3350830078125, + "loss": 0.4851, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.3043274879455566, + "rewards/margins": 1.402414083480835, + "rewards/rejected": -3.7067413330078125, + "step": 2210 + }, + { + "epoch": 0.5326295585412668, + "grad_norm": 19.37490800094979, + "learning_rate": 2.6517098964071507e-07, + "logits/chosen": -1.182252049446106, + "logits/rejected": -1.205956220626831, + "logps/chosen": -459.35968017578125, + "logps/rejected": -567.3809814453125, + "loss": 0.477, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.175320863723755, + "rewards/margins": 0.9405016899108887, + "rewards/rejected": -3.1158225536346436, + "step": 2220 + }, + { + "epoch": 0.5350287907869482, + "grad_norm": 19.34432528811352, + "learning_rate": 2.630805041141023e-07, + "logits/chosen": -1.3205252885818481, + "logits/rejected": -1.3020694255828857, + "logps/chosen": -419.14727783203125, + "logps/rejected": -813.0269775390625, + "loss": 0.4507, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.068502426147461, + "rewards/margins": 3.749777317047119, + "rewards/rejected": -5.818280220031738, + "step": 2230 + }, + { + "epoch": 0.5374280230326296, + "grad_norm": 17.869651405457827, + "learning_rate": 2.609891010420941e-07, + "logits/chosen": -1.2761646509170532, + "logits/rejected": -1.2586033344268799, + "logps/chosen": -440.08270263671875, + "logps/rejected": -681.1771850585938, + "loss": 0.4139, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.9673511981964111, + "rewards/margins": 2.405892848968506, + "rewards/rejected": -4.373243808746338, + "step": 2240 + }, + { + "epoch": 0.539827255278311, + "grad_norm": 16.14736853278147, + "learning_rate": 2.5889692712830674e-07, + "logits/chosen": -1.1019476652145386, + "logits/rejected": -1.1302673816680908, + "logps/chosen": -390.2100524902344, + "logps/rejected": -552.5843505859375, + "loss": 0.4161, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -1.7728662490844727, + "rewards/margins": 1.6656566858291626, + "rewards/rejected": -3.438523054122925, + "step": 2250 + }, + { + "epoch": 0.5422264875239923, + "grad_norm": 16.14985626238556, + "learning_rate": 2.5680412913042843e-07, + "logits/chosen": -1.2380822896957397, + "logits/rejected": -1.2011024951934814, + "logps/chosen": -434.4576721191406, + "logps/rejected": -744.7467041015625, + "loss": 0.4166, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.052339553833008, + "rewards/margins": 3.060929298400879, + "rewards/rejected": -5.113268852233887, + "step": 2260 + }, + { + "epoch": 0.5446257197696737, + "grad_norm": 20.209606887329077, + "learning_rate": 2.5471085384992404e-07, + "logits/chosen": -1.2722258567810059, + "logits/rejected": -1.2316162586212158, + "logps/chosen": -433.7353515625, + "logps/rejected": -920.55712890625, + "loss": 0.4205, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.1150050163269043, + "rewards/margins": 4.714972019195557, + "rewards/rejected": -6.829976558685303, + "step": 2270 + }, + { + "epoch": 0.5470249520153551, + "grad_norm": 13.645177424361062, + "learning_rate": 2.526172481217381e-07, + "logits/chosen": -1.1917331218719482, + "logits/rejected": -1.1137642860412598, + "logps/chosen": -463.8427734375, + "logps/rejected": -684.7598876953125, + "loss": 0.4592, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.671189308166504, + "rewards/margins": 2.050757884979248, + "rewards/rejected": -4.72194766998291, + "step": 2280 + }, + { + "epoch": 0.5494241842610365, + "grad_norm": 19.346585502374158, + "learning_rate": 2.5052345880399456e-07, + "logits/chosen": -1.3105088472366333, + "logits/rejected": -1.3653063774108887, + "logps/chosen": -450.48150634765625, + "logps/rejected": -608.4674682617188, + "loss": 0.4074, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.3599371910095215, + "rewards/margins": 1.5191092491149902, + "rewards/rejected": -3.8790462017059326, + "step": 2290 + }, + { + "epoch": 0.5518234165067178, + "grad_norm": 11.134867800418832, + "learning_rate": 2.4842963276769555e-07, + "logits/chosen": -1.2500735521316528, + "logits/rejected": -1.1833133697509766, + "logps/chosen": -434.44805908203125, + "logps/rejected": -694.0464477539062, + "loss": 0.4129, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.3137922286987305, + "rewards/margins": 2.1529812812805176, + "rewards/rejected": -4.46677303314209, + "step": 2300 + }, + { + "epoch": 0.5542226487523992, + "grad_norm": 13.388561282642513, + "learning_rate": 2.463359168864189e-07, + "logits/chosen": -1.1017297506332397, + "logits/rejected": -1.2655115127563477, + "logps/chosen": -515.0410766601562, + "logps/rejected": -691.630126953125, + "loss": 0.4386, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.2696266174316406, + "rewards/margins": 2.1189069747924805, + "rewards/rejected": -4.388534069061279, + "step": 2310 + }, + { + "epoch": 0.5566218809980806, + "grad_norm": 21.830085417242575, + "learning_rate": 2.4424245802601555e-07, + "logits/chosen": -1.2653098106384277, + "logits/rejected": -1.2332645654678345, + "logps/chosen": -436.19073486328125, + "logps/rejected": -619.58203125, + "loss": 0.4089, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.1965491771698, + "rewards/margins": 1.2768805027008057, + "rewards/rejected": -3.4734294414520264, + "step": 2320 + }, + { + "epoch": 0.559021113243762, + "grad_norm": 14.621416023558584, + "learning_rate": 2.421494030343072e-07, + "logits/chosen": -1.167513370513916, + "logits/rejected": -1.3203797340393066, + "logps/chosen": -495.20306396484375, + "logps/rejected": -584.0275268554688, + "loss": 0.47, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.520407199859619, + "rewards/margins": 1.3572065830230713, + "rewards/rejected": -3.8776137828826904, + "step": 2330 + }, + { + "epoch": 0.5614203454894434, + "grad_norm": 13.806279084269839, + "learning_rate": 2.400568987307861e-07, + "logits/chosen": -1.2116050720214844, + "logits/rejected": -1.2625479698181152, + "logps/chosen": -463.60595703125, + "logps/rejected": -533.0452270507812, + "loss": 0.3951, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.520028591156006, + "rewards/margins": 0.7540720701217651, + "rewards/rejected": -3.2741007804870605, + "step": 2340 + }, + { + "epoch": 0.5638195777351248, + "grad_norm": 17.208724843562575, + "learning_rate": 2.379650918963156e-07, + "logits/chosen": -1.265211582183838, + "logits/rejected": -1.2508124113082886, + "logps/chosen": -419.3246154785156, + "logps/rejected": -666.0604858398438, + "loss": 0.4, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.3264572620391846, + "rewards/margins": 2.320263624191284, + "rewards/rejected": -4.646720886230469, + "step": 2350 + }, + { + "epoch": 0.5662188099808061, + "grad_norm": 19.054319387788627, + "learning_rate": 2.3587412926283438e-07, + "logits/chosen": -1.2373709678649902, + "logits/rejected": -1.1830167770385742, + "logps/chosen": -515.0059814453125, + "logps/rejected": -775.9603271484375, + "loss": 0.4394, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.3372507095336914, + "rewards/margins": 2.948981523513794, + "rewards/rejected": -5.286231994628906, + "step": 2360 + }, + { + "epoch": 0.5686180422264875, + "grad_norm": 18.046127707033907, + "learning_rate": 2.337841575030642e-07, + "logits/chosen": -1.0661755800247192, + "logits/rejected": -1.052562952041626, + "logps/chosen": -483.67071533203125, + "logps/rejected": -702.375, + "loss": 0.4293, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.1245293617248535, + "rewards/margins": 2.080724000930786, + "rewards/rejected": -4.2052531242370605, + "step": 2370 + }, + { + "epoch": 0.5710172744721689, + "grad_norm": 11.044379945267481, + "learning_rate": 2.316953232202206e-07, + "logits/chosen": -1.1942309141159058, + "logits/rejected": -1.371203899383545, + "logps/chosen": -426.2565002441406, + "logps/rejected": -538.9791870117188, + "loss": 0.3953, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.0590853691101074, + "rewards/margins": 1.724047064781189, + "rewards/rejected": -3.783132553100586, + "step": 2380 + }, + { + "epoch": 0.5734165067178503, + "grad_norm": 14.319869332524695, + "learning_rate": 2.2960777293772958e-07, + "logits/chosen": -1.1835999488830566, + "logits/rejected": -1.2751821279525757, + "logps/chosen": -406.2999572753906, + "logps/rejected": -733.0162353515625, + "loss": 0.4318, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.0234265327453613, + "rewards/margins": 3.427159070968628, + "rewards/rejected": -5.45058536529541, + "step": 2390 + }, + { + "epoch": 0.5758157389635317, + "grad_norm": 11.706540267968286, + "learning_rate": 2.2752165308894974e-07, + "logits/chosen": -1.2021677494049072, + "logits/rejected": -1.1848233938217163, + "logps/chosen": -410.93975830078125, + "logps/rejected": -670.9909057617188, + "loss": 0.4182, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.33550763130188, + "rewards/margins": 2.6671481132507324, + "rewards/rejected": -5.002655982971191, + "step": 2400 + }, + { + "epoch": 0.5782149712092131, + "grad_norm": 16.988346428741217, + "learning_rate": 2.254371100069005e-07, + "logits/chosen": -1.0889265537261963, + "logits/rejected": -0.9927159547805786, + "logps/chosen": -427.29193115234375, + "logps/rejected": -658.6199340820312, + "loss": 0.4049, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -1.9339721202850342, + "rewards/margins": 2.0747880935668945, + "rewards/rejected": -4.008760452270508, + "step": 2410 + }, + { + "epoch": 0.5806142034548945, + "grad_norm": 21.542226745337405, + "learning_rate": 2.2335428991399725e-07, + "logits/chosen": -1.1989023685455322, + "logits/rejected": -1.1998827457427979, + "logps/chosen": -413.19171142578125, + "logps/rejected": -963.6064453125, + "loss": 0.4078, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.3345561027526855, + "rewards/margins": 5.340123176574707, + "rewards/rejected": -7.674679756164551, + "step": 2420 + }, + { + "epoch": 0.5830134357005758, + "grad_norm": 15.709987012395423, + "learning_rate": 2.2127333891179458e-07, + "logits/chosen": -1.2488889694213867, + "logits/rejected": -1.2438524961471558, + "logps/chosen": -370.1678161621094, + "logps/rejected": -775.2645263671875, + "loss": 0.4262, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.8031924962997437, + "rewards/margins": 3.7851650714874268, + "rewards/rejected": -5.588356971740723, + "step": 2430 + }, + { + "epoch": 0.5854126679462572, + "grad_norm": 18.692702891715406, + "learning_rate": 2.1919440297073782e-07, + "logits/chosen": -1.1863176822662354, + "logits/rejected": -1.1739771366119385, + "logps/chosen": -411.5914001464844, + "logps/rejected": -734.830078125, + "loss": 0.4481, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.193366765975952, + "rewards/margins": 3.1098275184631348, + "rewards/rejected": -5.303194522857666, + "step": 2440 + }, + { + "epoch": 0.5878119001919386, + "grad_norm": 14.049015216558205, + "learning_rate": 2.1711762791992368e-07, + "logits/chosen": -1.1900994777679443, + "logits/rejected": -1.1979767084121704, + "logps/chosen": -491.08880615234375, + "logps/rejected": -776.205078125, + "loss": 0.4614, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.1984424591064453, + "rewards/margins": 3.0933632850646973, + "rewards/rejected": -5.291806221008301, + "step": 2450 + }, + { + "epoch": 0.5902111324376199, + "grad_norm": 15.123102008910312, + "learning_rate": 2.1504315943687114e-07, + "logits/chosen": -1.1304634809494019, + "logits/rejected": -1.0604979991912842, + "logps/chosen": -444.66839599609375, + "logps/rejected": -772.3875122070312, + "loss": 0.4161, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -2.300208330154419, + "rewards/margins": 2.822746992111206, + "rewards/rejected": -5.122956275939941, + "step": 2460 + }, + { + "epoch": 0.5926103646833013, + "grad_norm": 16.476123838994056, + "learning_rate": 2.1297114303730248e-07, + "logits/chosen": -1.0877963304519653, + "logits/rejected": -0.9808356165885925, + "logps/chosen": -430.3238220214844, + "logps/rejected": -756.1893920898438, + "loss": 0.4441, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.168168306350708, + "rewards/margins": 2.786461353302002, + "rewards/rejected": -4.954629898071289, + "step": 2470 + }, + { + "epoch": 0.5950095969289827, + "grad_norm": 16.3511804347965, + "learning_rate": 2.1090172406493616e-07, + "logits/chosen": -1.095245122909546, + "logits/rejected": -1.0183230638504028, + "logps/chosen": -397.98480224609375, + "logps/rejected": -673.2918701171875, + "loss": 0.3975, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -1.7769782543182373, + "rewards/margins": 2.612269401550293, + "rewards/rejected": -4.389247894287109, + "step": 2480 + }, + { + "epoch": 0.5974088291746641, + "grad_norm": 20.613201850030986, + "learning_rate": 2.0883504768129146e-07, + "logits/chosen": -1.2274065017700195, + "logits/rejected": -1.2041352987289429, + "logps/chosen": -488.35260009765625, + "logps/rejected": -783.0089721679688, + "loss": 0.4259, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.3304946422576904, + "rewards/margins": 2.9999914169311523, + "rewards/rejected": -5.330485820770264, + "step": 2490 + }, + { + "epoch": 0.5998080614203455, + "grad_norm": 18.3928478904682, + "learning_rate": 2.0677125885550571e-07, + "logits/chosen": -1.1274101734161377, + "logits/rejected": -1.246701955795288, + "logps/chosen": -457.32415771484375, + "logps/rejected": -619.3272094726562, + "loss": 0.4264, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.35748553276062, + "rewards/margins": 1.9687366485595703, + "rewards/rejected": -4.3262224197387695, + "step": 2500 + }, + { + "epoch": 0.6022072936660269, + "grad_norm": 18.845165093912573, + "learning_rate": 2.0471050235416587e-07, + "logits/chosen": -1.0176677703857422, + "logits/rejected": -1.1635395288467407, + "logps/chosen": -465.5758361816406, + "logps/rejected": -703.4918823242188, + "loss": 0.3765, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.2303969860076904, + "rewards/margins": 2.811887502670288, + "rewards/rejected": -5.042284965515137, + "step": 2510 + }, + { + "epoch": 0.6046065259117083, + "grad_norm": 16.99666155725327, + "learning_rate": 2.026529227311532e-07, + "logits/chosen": -1.2282030582427979, + "logits/rejected": -1.2217004299163818, + "logps/chosen": -436.674072265625, + "logps/rejected": -751.2864379882812, + "loss": 0.46, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.3597781658172607, + "rewards/margins": 3.0093655586242676, + "rewards/rejected": -5.369143962860107, + "step": 2520 + }, + { + "epoch": 0.6070057581573897, + "grad_norm": 17.94857061645596, + "learning_rate": 2.005986643175036e-07, + "logits/chosen": -1.125093698501587, + "logits/rejected": -1.0239616632461548, + "logps/chosen": -448.51837158203125, + "logps/rejected": -756.28466796875, + "loss": 0.3639, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -1.9389982223510742, + "rewards/margins": 3.0756373405456543, + "rewards/rejected": -5.014636039733887, + "step": 2530 + }, + { + "epoch": 0.6094049904030711, + "grad_norm": 14.03073566548983, + "learning_rate": 1.9854787121128328e-07, + "logits/chosen": -1.1564371585845947, + "logits/rejected": -1.2444071769714355, + "logps/chosen": -429.7975158691406, + "logps/rejected": -546.2296142578125, + "loss": 0.4717, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.235487222671509, + "rewards/margins": 1.5971519947052002, + "rewards/rejected": -3.832639217376709, + "step": 2540 + }, + { + "epoch": 0.6118042226487524, + "grad_norm": 17.056601069921566, + "learning_rate": 1.9650068726748106e-07, + "logits/chosen": -1.154388666152954, + "logits/rejected": -1.233816146850586, + "logps/chosen": -488.4398498535156, + "logps/rejected": -727.38916015625, + "loss": 0.4605, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.468658685684204, + "rewards/margins": 2.3356359004974365, + "rewards/rejected": -4.804293632507324, + "step": 2550 + }, + { + "epoch": 0.6142034548944337, + "grad_norm": 14.873549291466142, + "learning_rate": 1.9445725608791718e-07, + "logits/chosen": -1.1273863315582275, + "logits/rejected": -1.127369999885559, + "logps/chosen": -439.273681640625, + "logps/rejected": -970.4906005859375, + "loss": 0.4193, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.041533946990967, + "rewards/margins": 5.191417694091797, + "rewards/rejected": -7.2329511642456055, + "step": 2560 + }, + { + "epoch": 0.6166026871401151, + "grad_norm": 15.773654226605428, + "learning_rate": 1.924177210111705e-07, + "logits/chosen": -1.212803602218628, + "logits/rejected": -1.2397335767745972, + "logps/chosen": -417.0751037597656, + "logps/rejected": -742.2860107421875, + "loss": 0.4398, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.125763416290283, + "rewards/margins": 3.112739086151123, + "rewards/rejected": -5.238502502441406, + "step": 2570 + }, + { + "epoch": 0.6190019193857965, + "grad_norm": 13.969918658633894, + "learning_rate": 1.9038222510252364e-07, + "logits/chosen": -1.1539736986160278, + "logits/rejected": -1.1613657474517822, + "logps/chosen": -448.21173095703125, + "logps/rejected": -631.4861450195312, + "loss": 0.4127, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.106086254119873, + "rewards/margins": 1.963559865951538, + "rewards/rejected": -4.069646835327148, + "step": 2580 + }, + { + "epoch": 0.6214011516314779, + "grad_norm": 15.025163221103547, + "learning_rate": 1.883509111439277e-07, + "logits/chosen": -1.1364405155181885, + "logits/rejected": -1.1150002479553223, + "logps/chosen": -435.28558349609375, + "logps/rejected": -850.2390747070312, + "loss": 0.4349, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -2.216041326522827, + "rewards/margins": 3.508605480194092, + "rewards/rejected": -5.72464656829834, + "step": 2590 + }, + { + "epoch": 0.6238003838771593, + "grad_norm": 12.217302246897113, + "learning_rate": 1.8632392162398665e-07, + "logits/chosen": -1.1138544082641602, + "logits/rejected": -1.085440993309021, + "logps/chosen": -457.41107177734375, + "logps/rejected": -764.52490234375, + "loss": 0.4118, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.9107215404510498, + "rewards/margins": 3.08528995513916, + "rewards/rejected": -4.996011257171631, + "step": 2600 + }, + { + "epoch": 0.6261996161228407, + "grad_norm": 18.46550627636187, + "learning_rate": 1.84301398727962e-07, + "logits/chosen": -1.2148702144622803, + "logits/rejected": -1.0856149196624756, + "logps/chosen": -348.2018127441406, + "logps/rejected": -761.6448974609375, + "loss": 0.4097, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.7905614376068115, + "rewards/margins": 3.7506096363067627, + "rewards/rejected": -5.541171073913574, + "step": 2610 + }, + { + "epoch": 0.6285988483685221, + "grad_norm": 19.299201781895523, + "learning_rate": 1.8228348432779966e-07, + "logits/chosen": -1.203184962272644, + "logits/rejected": -1.1859591007232666, + "logps/chosen": -446.0292053222656, + "logps/rejected": -726.4371948242188, + "loss": 0.4633, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.329465389251709, + "rewards/margins": 2.8768537044525146, + "rewards/rejected": -5.206319332122803, + "step": 2620 + }, + { + "epoch": 0.6309980806142035, + "grad_norm": 12.068424144263599, + "learning_rate": 1.8027031997217773e-07, + "logits/chosen": -1.2793502807617188, + "logits/rejected": -1.252190351486206, + "logps/chosen": -443.053955078125, + "logps/rejected": -1062.28564453125, + "loss": 0.3593, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.4902422428131104, + "rewards/margins": 5.98113489151001, + "rewards/rejected": -8.471376419067383, + "step": 2630 + }, + { + "epoch": 0.6333973128598849, + "grad_norm": 12.134661285568798, + "learning_rate": 1.7826204687657758e-07, + "logits/chosen": -1.0787185430526733, + "logits/rejected": -1.0653938055038452, + "logps/chosen": -486.08880615234375, + "logps/rejected": -640.8261108398438, + "loss": 0.3725, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.273076295852661, + "rewards/margins": 1.9071142673492432, + "rewards/rejected": -4.180190086364746, + "step": 2640 + }, + { + "epoch": 0.6357965451055663, + "grad_norm": 17.885684332947207, + "learning_rate": 1.762588059133781e-07, + "logits/chosen": -1.0950032472610474, + "logits/rejected": -1.1858885288238525, + "logps/chosen": -497.4866638183594, + "logps/rejected": -718.3407592773438, + "loss": 0.4108, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.2074708938598633, + "rewards/margins": 2.5658068656921387, + "rewards/rejected": -4.77327823638916, + "step": 2650 + }, + { + "epoch": 0.6381957773512476, + "grad_norm": 14.528060337749242, + "learning_rate": 1.7426073760197406e-07, + "logits/chosen": -1.1224123239517212, + "logits/rejected": -1.0580319166183472, + "logps/chosen": -437.44549560546875, + "logps/rejected": -904.7830200195312, + "loss": 0.4371, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.128122091293335, + "rewards/margins": 4.340218544006348, + "rewards/rejected": -6.468340873718262, + "step": 2660 + }, + { + "epoch": 0.6405950095969289, + "grad_norm": 11.381160671599869, + "learning_rate": 1.7226798209891935e-07, + "logits/chosen": -0.9424558877944946, + "logits/rejected": -1.0885859727859497, + "logps/chosen": -470.26434326171875, + "logps/rejected": -712.3435668945312, + "loss": 0.3844, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -2.301588296890259, + "rewards/margins": 2.9525039196014404, + "rewards/rejected": -5.254092216491699, + "step": 2670 + }, + { + "epoch": 0.6429942418426103, + "grad_norm": 19.023529875568627, + "learning_rate": 1.7028067918809535e-07, + "logits/chosen": -1.064992070198059, + "logits/rejected": -1.028592824935913, + "logps/chosen": -398.50384521484375, + "logps/rejected": -970.7301025390625, + "loss": 0.4204, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.0184497833251953, + "rewards/margins": 5.186704635620117, + "rewards/rejected": -7.2051544189453125, + "step": 2680 + }, + { + "epoch": 0.6453934740882917, + "grad_norm": 16.12712460489812, + "learning_rate": 1.6829896827090584e-07, + "logits/chosen": -1.1934797763824463, + "logits/rejected": -1.231001615524292, + "logps/chosen": -464.2530822753906, + "logps/rejected": -632.6357421875, + "loss": 0.4421, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.3296985626220703, + "rewards/margins": 1.9349523782730103, + "rewards/rejected": -4.264651298522949, + "step": 2690 + }, + { + "epoch": 0.6477927063339731, + "grad_norm": 11.504903057526331, + "learning_rate": 1.6632298835649844e-07, + "logits/chosen": -1.0596282482147217, + "logits/rejected": -0.9983747601509094, + "logps/chosen": -477.0462341308594, + "logps/rejected": -902.0972900390625, + "loss": 0.3847, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.2268457412719727, + "rewards/margins": 3.9091105461120605, + "rewards/rejected": -6.135955333709717, + "step": 2700 + }, + { + "epoch": 0.6501919385796545, + "grad_norm": 27.609604865313603, + "learning_rate": 1.6435287805201364e-07, + "logits/chosen": -1.1235686540603638, + "logits/rejected": -1.0528533458709717, + "logps/chosen": -478.143798828125, + "logps/rejected": -667.5210571289062, + "loss": 0.4531, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.356539726257324, + "rewards/margins": 1.9865471124649048, + "rewards/rejected": -4.343085765838623, + "step": 2710 + }, + { + "epoch": 0.6525911708253359, + "grad_norm": 16.415350337205087, + "learning_rate": 1.6238877555286207e-07, + "logits/chosen": -1.1847976446151733, + "logits/rejected": -1.1770317554473877, + "logps/chosen": -453.5834045410156, + "logps/rejected": -752.9813232421875, + "loss": 0.3986, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -1.8817520141601562, + "rewards/margins": 2.8628299236297607, + "rewards/rejected": -4.744582176208496, + "step": 2720 + }, + { + "epoch": 0.6549904030710173, + "grad_norm": 17.669293063839785, + "learning_rate": 1.60430818633031e-07, + "logits/chosen": -1.0959994792938232, + "logits/rejected": -1.0774368047714233, + "logps/chosen": -454.6305236816406, + "logps/rejected": -751.3426513671875, + "loss": 0.398, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.12265682220459, + "rewards/margins": 2.978886604309082, + "rewards/rejected": -5.101543426513672, + "step": 2730 + }, + { + "epoch": 0.6573896353166987, + "grad_norm": 16.006219619936903, + "learning_rate": 1.5847914463541939e-07, + "logits/chosen": -1.1957377195358276, + "logits/rejected": -1.1904990673065186, + "logps/chosen": -405.86431884765625, + "logps/rejected": -793.5975952148438, + "loss": 0.3881, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.2277333736419678, + "rewards/margins": 3.6207995414733887, + "rewards/rejected": -5.8485331535339355, + "step": 2740 + }, + { + "epoch": 0.6597888675623801, + "grad_norm": 12.430236202376024, + "learning_rate": 1.5653389046220427e-07, + "logits/chosen": -1.0426452159881592, + "logits/rejected": -1.0287232398986816, + "logps/chosen": -418.88427734375, + "logps/rejected": -726.9313354492188, + "loss": 0.3936, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -1.9855321645736694, + "rewards/margins": 2.8939387798309326, + "rewards/rejected": -4.8794708251953125, + "step": 2750 + }, + { + "epoch": 0.6621880998080614, + "grad_norm": 20.11094539300265, + "learning_rate": 1.545951925652375e-07, + "logits/chosen": -1.1118113994598389, + "logits/rejected": -1.1644222736358643, + "logps/chosen": -506.046142578125, + "logps/rejected": -755.5003662109375, + "loss": 0.3998, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -2.162713050842285, + "rewards/margins": 2.99284029006958, + "rewards/rejected": -5.155553340911865, + "step": 2760 + }, + { + "epoch": 0.6645873320537428, + "grad_norm": 16.91737052802927, + "learning_rate": 1.5266318693647423e-07, + "logits/chosen": -1.131763219833374, + "logits/rejected": -1.1238648891448975, + "logps/chosen": -483.41912841796875, + "logps/rejected": -633.3712768554688, + "loss": 0.4135, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.371241569519043, + "rewards/margins": 1.6103929281234741, + "rewards/rejected": -3.9816346168518066, + "step": 2770 + }, + { + "epoch": 0.6669865642994242, + "grad_norm": 18.088895471463584, + "learning_rate": 1.5073800909843353e-07, + "logits/chosen": -1.1542402505874634, + "logits/rejected": -1.2480642795562744, + "logps/chosen": -461.5935974121094, + "logps/rejected": -689.340087890625, + "loss": 0.4061, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.0994582176208496, + "rewards/margins": 2.842416286468506, + "rewards/rejected": -4.9418745040893555, + "step": 2780 + }, + { + "epoch": 0.6693857965451055, + "grad_norm": 17.217768649723784, + "learning_rate": 1.488197940946922e-07, + "logits/chosen": -1.0978938341140747, + "logits/rejected": -1.0699193477630615, + "logps/chosen": -440.05169677734375, + "logps/rejected": -651.9158325195312, + "loss": 0.366, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.789375901222229, + "rewards/margins": 2.706338405609131, + "rewards/rejected": -4.49571418762207, + "step": 2790 + }, + { + "epoch": 0.6717850287907869, + "grad_norm": 25.169875158147637, + "learning_rate": 1.4690867648041167e-07, + "logits/chosen": -0.994094967842102, + "logits/rejected": -1.0959413051605225, + "logps/chosen": -452.80810546875, + "logps/rejected": -717.0819091796875, + "loss": 0.43, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.118849277496338, + "rewards/margins": 2.8910040855407715, + "rewards/rejected": -5.009852886199951, + "step": 2800 + }, + { + "epoch": 0.6741842610364683, + "grad_norm": 13.750415835285578, + "learning_rate": 1.4500479031289987e-07, + "logits/chosen": -0.9904336929321289, + "logits/rejected": -1.0503981113433838, + "logps/chosen": -449.04913330078125, + "logps/rejected": -674.6497802734375, + "loss": 0.454, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.8702030181884766, + "rewards/margins": 2.3403210639953613, + "rewards/rejected": -4.210524082183838, + "step": 2810 + }, + { + "epoch": 0.6765834932821497, + "grad_norm": 12.953044708545338, + "learning_rate": 1.4310826914220747e-07, + "logits/chosen": -1.0443706512451172, + "logits/rejected": -1.1001513004302979, + "logps/chosen": -534.1334838867188, + "logps/rejected": -685.5401611328125, + "loss": 0.4265, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.4353697299957275, + "rewards/margins": 1.7159233093261719, + "rewards/rejected": -4.15129280090332, + "step": 2820 + }, + { + "epoch": 0.6789827255278311, + "grad_norm": 11.770848466287676, + "learning_rate": 1.412192460017597e-07, + "logits/chosen": -1.1065876483917236, + "logits/rejected": -1.0538074970245361, + "logps/chosen": -474.156982421875, + "logps/rejected": -803.2342529296875, + "loss": 0.4204, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.451140880584717, + "rewards/margins": 3.259047031402588, + "rewards/rejected": -5.710188865661621, + "step": 2830 + }, + { + "epoch": 0.6813819577735125, + "grad_norm": 13.446807703778937, + "learning_rate": 1.3933785339902504e-07, + "logits/chosen": -1.1683580875396729, + "logits/rejected": -1.0467820167541504, + "logps/chosen": -407.0369873046875, + "logps/rejected": -731.3319091796875, + "loss": 0.4413, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.2081449031829834, + "rewards/margins": 2.7958226203918457, + "rewards/rejected": -5.003968238830566, + "step": 2840 + }, + { + "epoch": 0.6837811900191939, + "grad_norm": 12.945264354253307, + "learning_rate": 1.374642233062197e-07, + "logits/chosen": -1.0711700916290283, + "logits/rejected": -1.1465437412261963, + "logps/chosen": -492.7830505371094, + "logps/rejected": -738.3113403320312, + "loss": 0.4438, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.2333264350891113, + "rewards/margins": 2.7537784576416016, + "rewards/rejected": -4.987104892730713, + "step": 2850 + }, + { + "epoch": 0.6861804222648752, + "grad_norm": 16.03013282047824, + "learning_rate": 1.355984871510511e-07, + "logits/chosen": -1.0226466655731201, + "logits/rejected": -0.9650028347969055, + "logps/chosen": -485.961181640625, + "logps/rejected": -760.8209228515625, + "loss": 0.4099, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.129098415374756, + "rewards/margins": 2.591064691543579, + "rewards/rejected": -4.720162391662598, + "step": 2860 + }, + { + "epoch": 0.6885796545105566, + "grad_norm": 13.460390874624267, + "learning_rate": 1.3374077580749783e-07, + "logits/chosen": -1.1847305297851562, + "logits/rejected": -1.1330692768096924, + "logps/chosen": -376.7669982910156, + "logps/rejected": -649.6778564453125, + "loss": 0.412, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.9333322048187256, + "rewards/margins": 2.54672908782959, + "rewards/rejected": -4.4800615310668945, + "step": 2870 + }, + { + "epoch": 0.690978886756238, + "grad_norm": 19.972945286063492, + "learning_rate": 1.3189121958663024e-07, + "logits/chosen": -1.0317165851593018, + "logits/rejected": -1.1716341972351074, + "logps/chosen": -525.8427734375, + "logps/rejected": -662.1897583007812, + "loss": 0.4068, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.6331121921539307, + "rewards/margins": 1.6707206964492798, + "rewards/rejected": -4.303833484649658, + "step": 2880 + }, + { + "epoch": 0.6933781190019194, + "grad_norm": 20.87380867505956, + "learning_rate": 1.3004994822746895e-07, + "logits/chosen": -1.16623854637146, + "logits/rejected": -1.1874853372573853, + "logps/chosen": -418.4476623535156, + "logps/rejected": -645.4654541015625, + "loss": 0.4138, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.9430526494979858, + "rewards/margins": 2.151594877243042, + "rewards/rejected": -4.094647407531738, + "step": 2890 + }, + { + "epoch": 0.6957773512476008, + "grad_norm": 16.04665832154, + "learning_rate": 1.2821709088788434e-07, + "logits/chosen": -0.9877917170524597, + "logits/rejected": -1.0230954885482788, + "logps/chosen": -397.6720886230469, + "logps/rejected": -668.94873046875, + "loss": 0.4195, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.033013105392456, + "rewards/margins": 2.66807222366333, + "rewards/rejected": -4.701085090637207, + "step": 2900 + }, + { + "epoch": 0.6981765834932822, + "grad_norm": 18.3654519072182, + "learning_rate": 1.2639277613553736e-07, + "logits/chosen": -1.2031629085540771, + "logits/rejected": -1.1549075841903687, + "logps/chosen": -384.4791564941406, + "logps/rejected": -596.3194580078125, + "loss": 0.4244, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.0126969814300537, + "rewards/margins": 2.0921952724456787, + "rewards/rejected": -4.104892253875732, + "step": 2910 + }, + { + "epoch": 0.7005758157389635, + "grad_norm": 12.871071702081897, + "learning_rate": 1.2457713193885975e-07, + "logits/chosen": -1.0249868631362915, + "logits/rejected": -0.955339252948761, + "logps/chosen": -356.92828369140625, + "logps/rejected": -710.9237060546875, + "loss": 0.3915, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.9744545221328735, + "rewards/margins": 3.1722934246063232, + "rewards/rejected": -5.146748065948486, + "step": 2920 + }, + { + "epoch": 0.7029750479846449, + "grad_norm": 18.857927055260983, + "learning_rate": 1.2277028565807838e-07, + "logits/chosen": -1.1315641403198242, + "logits/rejected": -1.175862431526184, + "logps/chosen": -448.12420654296875, + "logps/rejected": -643.8922119140625, + "loss": 0.4123, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.075835943222046, + "rewards/margins": 2.037856340408325, + "rewards/rejected": -4.113692283630371, + "step": 2930 + }, + { + "epoch": 0.7053742802303263, + "grad_norm": 19.270697064040355, + "learning_rate": 1.209723640362815e-07, + "logits/chosen": -1.055815577507019, + "logits/rejected": -1.0176749229431152, + "logps/chosen": -472.85418701171875, + "logps/rejected": -900.6534423828125, + "loss": 0.4865, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.1895081996917725, + "rewards/margins": 4.365129470825195, + "rewards/rejected": -6.554637908935547, + "step": 2940 + }, + { + "epoch": 0.7077735124760077, + "grad_norm": 11.756470338560021, + "learning_rate": 1.191834931905277e-07, + "logits/chosen": -1.021990180015564, + "logits/rejected": -1.0114670991897583, + "logps/chosen": -526.177978515625, + "logps/rejected": -784.1201171875, + "loss": 0.397, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.455296516418457, + "rewards/margins": 2.5135977268218994, + "rewards/rejected": -4.968894004821777, + "step": 2950 + }, + { + "epoch": 0.710172744721689, + "grad_norm": 13.074744119052957, + "learning_rate": 1.1740379860299988e-07, + "logits/chosen": -1.085823655128479, + "logits/rejected": -1.0509856939315796, + "logps/chosen": -483.33160400390625, + "logps/rejected": -673.5759887695312, + "loss": 0.4438, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.2232718467712402, + "rewards/margins": 1.7320282459259033, + "rewards/rejected": -3.9553000926971436, + "step": 2960 + }, + { + "epoch": 0.7125719769673704, + "grad_norm": 13.781742295328323, + "learning_rate": 1.1563340511220254e-07, + "logits/chosen": -1.078175663948059, + "logits/rejected": -1.0989038944244385, + "logps/chosen": -510.52978515625, + "logps/rejected": -873.8924560546875, + "loss": 0.437, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.4151575565338135, + "rewards/margins": 3.7648189067840576, + "rewards/rejected": -6.179976463317871, + "step": 2970 + }, + { + "epoch": 0.7149712092130518, + "grad_norm": 12.441472662730447, + "learning_rate": 1.1387243690420556e-07, + "logits/chosen": -1.0066481828689575, + "logits/rejected": -1.0207774639129639, + "logps/chosen": -498.4524841308594, + "logps/rejected": -774.4942626953125, + "loss": 0.4093, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.1083011627197266, + "rewards/margins": 2.8360419273376465, + "rewards/rejected": -4.944342613220215, + "step": 2980 + }, + { + "epoch": 0.7173704414587332, + "grad_norm": 19.827064803673473, + "learning_rate": 1.1212101750393235e-07, + "logits/chosen": -1.1261366605758667, + "logits/rejected": -1.1484588384628296, + "logps/chosen": -450.61541748046875, + "logps/rejected": -729.1220703125, + "loss": 0.3958, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.2109882831573486, + "rewards/margins": 2.9870235919952393, + "rewards/rejected": -5.1980109214782715, + "step": 2990 + }, + { + "epoch": 0.7197696737044146, + "grad_norm": 11.684514143150514, + "learning_rate": 1.1037926976649562e-07, + "logits/chosen": -1.0930601358413696, + "logits/rejected": -1.1050859689712524, + "logps/chosen": -469.44677734375, + "logps/rejected": -770.8958129882812, + "loss": 0.4399, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.3145837783813477, + "rewards/margins": 2.7300620079040527, + "rewards/rejected": -5.0446457862854, + "step": 3000 + }, + { + "epoch": 0.722168905950096, + "grad_norm": 22.78691874055665, + "learning_rate": 1.0864731586857936e-07, + "logits/chosen": -0.9925867319107056, + "logits/rejected": -1.0378546714782715, + "logps/chosen": -467.55621337890625, + "logps/rejected": -736.3528442382812, + "loss": 0.3888, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -2.0709593296051025, + "rewards/margins": 2.980044364929199, + "rewards/rejected": -5.051003456115723, + "step": 3010 + }, + { + "epoch": 0.7245681381957774, + "grad_norm": 19.8573401803232, + "learning_rate": 1.0692527729986839e-07, + "logits/chosen": -1.0133628845214844, + "logits/rejected": -1.054368257522583, + "logps/chosen": -474.64312744140625, + "logps/rejected": -708.5240478515625, + "loss": 0.3697, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.2988035678863525, + "rewards/margins": 2.4555907249450684, + "rewards/rejected": -4.754393577575684, + "step": 3020 + }, + { + "epoch": 0.7269673704414588, + "grad_norm": 15.961064721992486, + "learning_rate": 1.0521327485452692e-07, + "logits/chosen": -1.0904114246368408, + "logits/rejected": -1.1059800386428833, + "logps/chosen": -459.8374938964844, + "logps/rejected": -742.4410400390625, + "loss": 0.4197, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.326892375946045, + "rewards/margins": 2.9531028270721436, + "rewards/rejected": -5.279995441436768, + "step": 3030 + }, + { + "epoch": 0.7293666026871402, + "grad_norm": 18.92614489476278, + "learning_rate": 1.0351142862272468e-07, + "logits/chosen": -1.0305520296096802, + "logits/rejected": -1.0847365856170654, + "logps/chosen": -404.38726806640625, + "logps/rejected": -921.81689453125, + "loss": 0.4332, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.0585224628448486, + "rewards/margins": 5.24090576171875, + "rewards/rejected": -7.299429416656494, + "step": 3040 + }, + { + "epoch": 0.7317658349328215, + "grad_norm": 17.7646623717338, + "learning_rate": 1.0181985798221343e-07, + "logits/chosen": -1.0311057567596436, + "logits/rejected": -1.0113275051116943, + "logps/chosen": -472.11846923828125, + "logps/rejected": -774.8313598632812, + "loss": 0.4757, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.2493302822113037, + "rewards/margins": 2.9238831996917725, + "rewards/rejected": -5.173213005065918, + "step": 3050 + }, + { + "epoch": 0.7341650671785028, + "grad_norm": 16.97123994535482, + "learning_rate": 1.0013868158995329e-07, + "logits/chosen": -1.055016279220581, + "logits/rejected": -1.0831139087677002, + "logps/chosen": -483.268310546875, + "logps/rejected": -701.4729614257812, + "loss": 0.4338, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.4825241565704346, + "rewards/margins": 2.385812282562256, + "rewards/rejected": -4.868335247039795, + "step": 3060 + }, + { + "epoch": 0.7365642994241842, + "grad_norm": 13.977094418250983, + "learning_rate": 9.84680173737887e-08, + "logits/chosen": -1.1702104806900024, + "logits/rejected": -1.1845287084579468, + "logps/chosen": -477.15740966796875, + "logps/rejected": -644.910400390625, + "loss": 0.4187, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.2820022106170654, + "rewards/margins": 2.027010679244995, + "rewards/rejected": -4.3090128898620605, + "step": 3070 + }, + { + "epoch": 0.7389635316698656, + "grad_norm": 14.318408739762742, + "learning_rate": 9.680798252417713e-08, + "logits/chosen": -1.2429280281066895, + "logits/rejected": -1.275286316871643, + "logps/chosen": -392.6990661621094, + "logps/rejected": -679.7120971679688, + "loss": 0.4099, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.0060112476348877, + "rewards/margins": 2.5065300464630127, + "rewards/rejected": -4.5125412940979, + "step": 3080 + }, + { + "epoch": 0.741362763915547, + "grad_norm": 17.01122593327606, + "learning_rate": 9.515869348596808e-08, + "logits/chosen": -1.0446436405181885, + "logits/rejected": -1.1268213987350464, + "logps/chosen": -495.5243225097656, + "logps/rejected": -660.196533203125, + "loss": 0.4184, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.165391206741333, + "rewards/margins": 1.9829740524291992, + "rewards/rejected": -4.148365020751953, + "step": 3090 + }, + { + "epoch": 0.7437619961612284, + "grad_norm": 13.45512588577857, + "learning_rate": 9.352026595023493e-08, + "logits/chosen": -1.1435279846191406, + "logits/rejected": -1.1522104740142822, + "logps/chosen": -485.2201232910156, + "logps/rejected": -617.6146240234375, + "loss": 0.4004, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.105456829071045, + "rewards/margins": 1.6230169534683228, + "rewards/rejected": -3.72847318649292, + "step": 3100 + }, + { + "epoch": 0.7461612284069098, + "grad_norm": 18.173194781052167, + "learning_rate": 9.189281484616004e-08, + "logits/chosen": -1.0414386987686157, + "logits/rejected": -1.0121897459030151, + "logps/chosen": -395.79931640625, + "logps/rejected": -666.1370849609375, + "loss": 0.4577, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.1656196117401123, + "rewards/margins": 2.18383526802063, + "rewards/rejected": -4.349454402923584, + "step": 3110 + }, + { + "epoch": 0.7485604606525912, + "grad_norm": 16.783716977152825, + "learning_rate": 9.027645433297249e-08, + "logits/chosen": -0.9481878280639648, + "logits/rejected": -0.9732887148857117, + "logps/chosen": -541.752685546875, + "logps/rejected": -781.5398559570312, + "loss": 0.4536, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.5279250144958496, + "rewards/margins": 2.745492458343506, + "rewards/rejected": -5.273416996002197, + "step": 3120 + }, + { + "epoch": 0.7509596928982726, + "grad_norm": 17.892434423988117, + "learning_rate": 8.867129779194066e-08, + "logits/chosen": -1.1220483779907227, + "logits/rejected": -1.1574240922927856, + "logps/chosen": -368.3999328613281, + "logps/rejected": -705.1334228515625, + "loss": 0.401, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.7526142597198486, + "rewards/margins": 3.364561080932617, + "rewards/rejected": -5.117176055908203, + "step": 3130 + }, + { + "epoch": 0.753358925143954, + "grad_norm": 16.65117790496681, + "learning_rate": 8.707745781841866e-08, + "logits/chosen": -0.945580005645752, + "logits/rejected": -0.9971720576286316, + "logps/chosen": -401.4361877441406, + "logps/rejected": -752.6023559570312, + "loss": 0.4458, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.0107078552246094, + "rewards/margins": 3.4325504302978516, + "rewards/rejected": -5.443258762359619, + "step": 3140 + }, + { + "epoch": 0.7557581573896354, + "grad_norm": 11.383207349861097, + "learning_rate": 8.549504621394831e-08, + "logits/chosen": -1.1988914012908936, + "logits/rejected": -1.152146577835083, + "logps/chosen": -380.92840576171875, + "logps/rejected": -766.2640380859375, + "loss": 0.3463, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.6404314041137695, + "rewards/margins": 3.8094921112060547, + "rewards/rejected": -5.449923515319824, + "step": 3150 + }, + { + "epoch": 0.7581573896353166, + "grad_norm": 14.817235687069312, + "learning_rate": 8.392417397841703e-08, + "logits/chosen": -1.0774648189544678, + "logits/rejected": -1.1336710453033447, + "logps/chosen": -428.7699279785156, + "logps/rejected": -665.4924926757812, + "loss": 0.433, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.9814361333847046, + "rewards/margins": 2.189484119415283, + "rewards/rejected": -4.170919895172119, + "step": 3160 + }, + { + "epoch": 0.760556621880998, + "grad_norm": 12.002932740050225, + "learning_rate": 8.236495130227083e-08, + "logits/chosen": -0.9945319890975952, + "logits/rejected": -1.0899218320846558, + "logps/chosen": -462.98907470703125, + "logps/rejected": -786.7493896484375, + "loss": 0.4364, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -1.9386112689971924, + "rewards/margins": 3.5603737831115723, + "rewards/rejected": -5.4989848136901855, + "step": 3170 + }, + { + "epoch": 0.7629558541266794, + "grad_norm": 19.054219594890455, + "learning_rate": 8.081748755878612e-08, + "logits/chosen": -1.0964199304580688, + "logits/rejected": -1.1900081634521484, + "logps/chosen": -488.88525390625, + "logps/rejected": -617.3141479492188, + "loss": 0.3881, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.3542323112487793, + "rewards/margins": 1.7926626205444336, + "rewards/rejected": -4.146894454956055, + "step": 3180 + }, + { + "epoch": 0.7653550863723608, + "grad_norm": 10.957826763580568, + "learning_rate": 7.928189129639632e-08, + "logits/chosen": -1.0295380353927612, + "logits/rejected": -1.001744031906128, + "logps/chosen": -416.1560974121094, + "logps/rejected": -653.9662475585938, + "loss": 0.3917, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -2.0308852195739746, + "rewards/margins": 2.194223165512085, + "rewards/rejected": -4.2251081466674805, + "step": 3190 + }, + { + "epoch": 0.7677543186180422, + "grad_norm": 21.757679034021272, + "learning_rate": 7.775827023107834e-08, + "logits/chosen": -1.0701841115951538, + "logits/rejected": -1.0753045082092285, + "logps/chosen": -457.8702087402344, + "logps/rejected": -725.718505859375, + "loss": 0.4356, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.3207480907440186, + "rewards/margins": 2.469742774963379, + "rewards/rejected": -4.790491104125977, + "step": 3200 + }, + { + "epoch": 0.7701535508637236, + "grad_norm": 20.167723441940918, + "learning_rate": 7.624673123879682e-08, + "logits/chosen": -0.9837926030158997, + "logits/rejected": -1.0835992097854614, + "logps/chosen": -432.2687072753906, + "logps/rejected": -644.8499755859375, + "loss": 0.4163, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.0870277881622314, + "rewards/margins": 2.2406442165374756, + "rewards/rejected": -4.327672004699707, + "step": 3210 + }, + { + "epoch": 0.772552783109405, + "grad_norm": 18.669387474954554, + "learning_rate": 7.474738034800663e-08, + "logits/chosen": -1.1607104539871216, + "logits/rejected": -1.0950881242752075, + "logps/chosen": -384.7247314453125, + "logps/rejected": -849.2708129882812, + "loss": 0.4627, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.0593299865722656, + "rewards/margins": 4.698477268218994, + "rewards/rejected": -6.75780725479126, + "step": 3220 + }, + { + "epoch": 0.7749520153550864, + "grad_norm": 12.574697681707546, + "learning_rate": 7.326032273221606e-08, + "logits/chosen": -1.1745719909667969, + "logits/rejected": -1.1211249828338623, + "logps/chosen": -492.7989196777344, + "logps/rejected": -735.6827392578125, + "loss": 0.354, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.242023468017578, + "rewards/margins": 2.6166510581970215, + "rewards/rejected": -4.8586745262146, + "step": 3230 + }, + { + "epoch": 0.7773512476007678, + "grad_norm": 23.069615047778715, + "learning_rate": 7.178566270260872e-08, + "logits/chosen": -1.189727544784546, + "logits/rejected": -1.192415475845337, + "logps/chosen": -464.24102783203125, + "logps/rejected": -783.3685302734375, + "loss": 0.4231, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.2558867931365967, + "rewards/margins": 2.903276205062866, + "rewards/rejected": -5.159163475036621, + "step": 3240 + }, + { + "epoch": 0.7797504798464492, + "grad_norm": 13.52470077231319, + "learning_rate": 7.032350370072709e-08, + "logits/chosen": -0.996944785118103, + "logits/rejected": -1.0384135246276855, + "logps/chosen": -465.0677795410156, + "logps/rejected": -785.6456298828125, + "loss": 0.3889, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.093614339828491, + "rewards/margins": 3.286773204803467, + "rewards/rejected": -5.380387306213379, + "step": 3250 + }, + { + "epoch": 0.7821497120921305, + "grad_norm": 17.442232282658164, + "learning_rate": 6.887394829121596e-08, + "logits/chosen": -1.1311534643173218, + "logits/rejected": -1.1876181364059448, + "logps/chosen": -485.97259521484375, + "logps/rejected": -1016.5963745117188, + "loss": 0.4039, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.456922769546509, + "rewards/margins": 5.4143595695495605, + "rewards/rejected": -7.871281623840332, + "step": 3260 + }, + { + "epoch": 0.7845489443378119, + "grad_norm": 14.265276149673632, + "learning_rate": 6.743709815462833e-08, + "logits/chosen": -1.149627447128296, + "logits/rejected": -1.1739815473556519, + "logps/chosen": -488.38433837890625, + "logps/rejected": -799.1815185546875, + "loss": 0.4014, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.456972360610962, + "rewards/margins": 3.4012649059295654, + "rewards/rejected": -5.858237266540527, + "step": 3270 + }, + { + "epoch": 0.7869481765834933, + "grad_norm": 14.656217972804923, + "learning_rate": 6.601305408029287e-08, + "logits/chosen": -1.1081793308258057, + "logits/rejected": -1.1610908508300781, + "logps/chosen": -469.09783935546875, + "logps/rejected": -725.4248046875, + "loss": 0.4108, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.4499924182891846, + "rewards/margins": 2.5296425819396973, + "rewards/rejected": -4.979634761810303, + "step": 3280 + }, + { + "epoch": 0.7893474088291746, + "grad_norm": 17.848039169645286, + "learning_rate": 6.460191595924366e-08, + "logits/chosen": -1.0300790071487427, + "logits/rejected": -1.0361210107803345, + "logps/chosen": -490.31488037109375, + "logps/rejected": -749.389892578125, + "loss": 0.4049, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.4620461463928223, + "rewards/margins": 2.546797513961792, + "rewards/rejected": -5.008843421936035, + "step": 3290 + }, + { + "epoch": 0.791746641074856, + "grad_norm": 13.85499152294089, + "learning_rate": 6.320378277721342e-08, + "logits/chosen": -1.1166772842407227, + "logits/rejected": -1.1094615459442139, + "logps/chosen": -472.2708435058594, + "logps/rejected": -616.1146240234375, + "loss": 0.4347, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.3808658123016357, + "rewards/margins": 1.5156285762786865, + "rewards/rejected": -3.8964946269989014, + "step": 3300 + }, + { + "epoch": 0.7941458733205374, + "grad_norm": 15.101274241746347, + "learning_rate": 6.181875260769032e-08, + "logits/chosen": -1.1072207689285278, + "logits/rejected": -1.2136280536651611, + "logps/chosen": -457.7784729003906, + "logps/rejected": -720.9715576171875, + "loss": 0.4173, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.8839908838272095, + "rewards/margins": 3.2553863525390625, + "rewards/rejected": -5.139378070831299, + "step": 3310 + }, + { + "epoch": 0.7965451055662188, + "grad_norm": 15.530326265159838, + "learning_rate": 6.044692260503797e-08, + "logits/chosen": -1.0710835456848145, + "logits/rejected": -1.1088721752166748, + "logps/chosen": -511.77593994140625, + "logps/rejected": -862.2023315429688, + "loss": 0.3341, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -2.3845109939575195, + "rewards/margins": 3.6880950927734375, + "rewards/rejected": -6.072606086730957, + "step": 3320 + }, + { + "epoch": 0.7989443378119002, + "grad_norm": 17.77444268921122, + "learning_rate": 5.9088388997680984e-08, + "logits/chosen": -1.0058419704437256, + "logits/rejected": -1.1048399209976196, + "logps/chosen": -543.4456176757812, + "logps/rejected": -763.7996826171875, + "loss": 0.4104, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.327815055847168, + "rewards/margins": 2.9269397258758545, + "rewards/rejected": -5.254754543304443, + "step": 3330 + }, + { + "epoch": 0.8013435700575816, + "grad_norm": 17.8842799505974, + "learning_rate": 5.774324708135439e-08, + "logits/chosen": -1.2106958627700806, + "logits/rejected": -1.2864594459533691, + "logps/chosen": -432.79486083984375, + "logps/rejected": -644.1822509765625, + "loss": 0.4089, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.3121211528778076, + "rewards/margins": 2.3210206031799316, + "rewards/rejected": -4.633141994476318, + "step": 3340 + }, + { + "epoch": 0.803742802303263, + "grad_norm": 11.331732688823461, + "learning_rate": 5.641159121241953e-08, + "logits/chosen": -1.2257134914398193, + "logits/rejected": -1.1192344427108765, + "logps/chosen": -437.44122314453125, + "logps/rejected": -826.15185546875, + "loss": 0.3994, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.3724796772003174, + "rewards/margins": 3.5549049377441406, + "rewards/rejected": -5.927384853363037, + "step": 3350 + }, + { + "epoch": 0.8061420345489443, + "grad_norm": 11.40432022732026, + "learning_rate": 5.5093514801245106e-08, + "logits/chosen": -1.0627083778381348, + "logits/rejected": -1.08839750289917, + "logps/chosen": -462.755126953125, + "logps/rejected": -749.5454711914062, + "loss": 0.4, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.3041393756866455, + "rewards/margins": 2.629760265350342, + "rewards/rejected": -4.933899879455566, + "step": 3360 + }, + { + "epoch": 0.8085412667946257, + "grad_norm": 16.009435064579, + "learning_rate": 5.378911030565453e-08, + "logits/chosen": -0.9565284848213196, + "logits/rejected": -0.9821860194206238, + "logps/chosen": -533.07080078125, + "logps/rejected": -830.4237060546875, + "loss": 0.4286, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.5907983779907227, + "rewards/margins": 2.66767954826355, + "rewards/rejected": -5.25847864151001, + "step": 3370 + }, + { + "epoch": 0.8109404990403071, + "grad_norm": 14.563417443153222, + "learning_rate": 5.249846922444101e-08, + "logits/chosen": -1.185606598854065, + "logits/rejected": -1.2072519063949585, + "logps/chosen": -449.8453063964844, + "logps/rejected": -1068.011962890625, + "loss": 0.3812, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.5254974365234375, + "rewards/margins": 6.294084548950195, + "rewards/rejected": -8.819581985473633, + "step": 3380 + }, + { + "epoch": 0.8133397312859885, + "grad_norm": 18.118325314041343, + "learning_rate": 5.122168209094865e-08, + "logits/chosen": -1.1093202829360962, + "logits/rejected": -1.1948482990264893, + "logps/chosen": -440.5008850097656, + "logps/rejected": -575.1368408203125, + "loss": 0.4144, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.41264009475708, + "rewards/margins": 1.3445088863372803, + "rewards/rejected": -3.7571487426757812, + "step": 3390 + }, + { + "epoch": 0.8157389635316699, + "grad_norm": 13.552322900784995, + "learning_rate": 4.995883846672222e-08, + "logits/chosen": -1.0201151371002197, + "logits/rejected": -1.129849910736084, + "logps/chosen": -577.0701293945312, + "logps/rejected": -773.9287719726562, + "loss": 0.4045, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.3463945388793945, + "rewards/margins": 2.6721103191375732, + "rewards/rejected": -5.018505573272705, + "step": 3400 + }, + { + "epoch": 0.8181381957773513, + "grad_norm": 14.631891117796933, + "learning_rate": 4.871002693522486e-08, + "logits/chosen": -1.0706162452697754, + "logits/rejected": -1.0791727304458618, + "logps/chosen": -484.5125427246094, + "logps/rejected": -756.0465087890625, + "loss": 0.4146, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.4046783447265625, + "rewards/margins": 2.9627346992492676, + "rewards/rejected": -5.367413520812988, + "step": 3410 + }, + { + "epoch": 0.8205374280230326, + "grad_norm": 11.654610414390184, + "learning_rate": 4.7475335095623956e-08, + "logits/chosen": -1.0974230766296387, + "logits/rejected": -1.0560171604156494, + "logps/chosen": -479.367919921875, + "logps/rejected": -745.7228393554688, + "loss": 0.4229, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.4068191051483154, + "rewards/margins": 2.776381731033325, + "rewards/rejected": -5.183200836181641, + "step": 3420 + }, + { + "epoch": 0.822936660268714, + "grad_norm": 21.88271199976919, + "learning_rate": 4.6254849556646714e-08, + "logits/chosen": -0.9372541308403015, + "logits/rejected": -0.9571690559387207, + "logps/chosen": -509.55133056640625, + "logps/rejected": -853.4140625, + "loss": 0.4194, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.310447931289673, + "rewards/margins": 3.700510025024414, + "rewards/rejected": -6.010957717895508, + "step": 3430 + }, + { + "epoch": 0.8253358925143954, + "grad_norm": 18.034151428931484, + "learning_rate": 4.504865593050483e-08, + "logits/chosen": -1.1065231561660767, + "logits/rejected": -1.1098486185073853, + "logps/chosen": -518.8222045898438, + "logps/rejected": -746.7293090820312, + "loss": 0.4183, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.7079575061798096, + "rewards/margins": 2.2379913330078125, + "rewards/rejected": -4.945948600769043, + "step": 3440 + }, + { + "epoch": 0.8277351247600768, + "grad_norm": 17.996732359854178, + "learning_rate": 4.385683882688895e-08, + "logits/chosen": -0.9841295480728149, + "logits/rejected": -1.083620548248291, + "logps/chosen": -518.1678466796875, + "logps/rejected": -604.3717041015625, + "loss": 0.4946, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.5432040691375732, + "rewards/margins": 1.3551008701324463, + "rewards/rejected": -3.8983047008514404, + "step": 3450 + }, + { + "epoch": 0.8301343570057581, + "grad_norm": 21.03759918631638, + "learning_rate": 4.2679481847033985e-08, + "logits/chosen": -1.0917068719863892, + "logits/rejected": -1.0973399877548218, + "logps/chosen": -475.8905334472656, + "logps/rejected": -735.8399658203125, + "loss": 0.4462, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.32417893409729, + "rewards/margins": 2.500762462615967, + "rewards/rejected": -4.824941158294678, + "step": 3460 + }, + { + "epoch": 0.8325335892514395, + "grad_norm": 15.12770624008802, + "learning_rate": 4.151666757785435e-08, + "logits/chosen": -1.122983455657959, + "logits/rejected": -1.1037867069244385, + "logps/chosen": -397.2513732910156, + "logps/rejected": -841.8653564453125, + "loss": 0.3949, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -1.7809816598892212, + "rewards/margins": 4.345281600952148, + "rewards/rejected": -6.12626314163208, + "step": 3470 + }, + { + "epoch": 0.8349328214971209, + "grad_norm": 17.000426067634862, + "learning_rate": 4.036847758615136e-08, + "logits/chosen": -0.9286991953849792, + "logits/rejected": -1.021832823753357, + "logps/chosen": -520.7754516601562, + "logps/rejected": -835.9027099609375, + "loss": 0.4339, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.9686105251312256, + "rewards/margins": 3.0616462230682373, + "rewards/rejected": -6.030257225036621, + "step": 3480 + }, + { + "epoch": 0.8373320537428023, + "grad_norm": 13.480464564958877, + "learning_rate": 3.923499241289113e-08, + "logits/chosen": -1.0932060480117798, + "logits/rejected": -1.1599218845367432, + "logps/chosen": -533.0609130859375, + "logps/rejected": -862.5074462890625, + "loss": 0.4438, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.498558759689331, + "rewards/margins": 3.8457469940185547, + "rewards/rejected": -6.344305515289307, + "step": 3490 + }, + { + "epoch": 0.8397312859884837, + "grad_norm": 18.263076008689207, + "learning_rate": 3.811629156755541e-08, + "logits/chosen": -1.043247103691101, + "logits/rejected": -1.033249020576477, + "logps/chosen": -495.8736877441406, + "logps/rejected": -736.5914306640625, + "loss": 0.4318, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.257842540740967, + "rewards/margins": 2.4572973251342773, + "rewards/rejected": -4.715139389038086, + "step": 3500 + }, + { + "epoch": 0.8421305182341651, + "grad_norm": 10.736541271527615, + "learning_rate": 3.701245352256391e-08, + "logits/chosen": -1.0726077556610107, + "logits/rejected": -1.186214566230774, + "logps/chosen": -482.1836853027344, + "logps/rejected": -610.18994140625, + "loss": 0.396, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.107025623321533, + "rewards/margins": 1.534145712852478, + "rewards/rejected": -3.6411712169647217, + "step": 3510 + }, + { + "epoch": 0.8445297504798465, + "grad_norm": 12.862620265499748, + "learning_rate": 3.592355570776984e-08, + "logits/chosen": -1.1533534526824951, + "logits/rejected": -1.1715278625488281, + "logps/chosen": -382.75006103515625, + "logps/rejected": -651.4183959960938, + "loss": 0.397, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -1.7870197296142578, + "rewards/margins": 2.5599758625030518, + "rewards/rejected": -4.3469953536987305, + "step": 3520 + }, + { + "epoch": 0.8469289827255279, + "grad_norm": 11.784757005145922, + "learning_rate": 3.484967450502904e-08, + "logits/chosen": -1.0784326791763306, + "logits/rejected": -1.1201931238174438, + "logps/chosen": -376.4716796875, + "logps/rejected": -701.4554443359375, + "loss": 0.4067, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -1.858490228652954, + "rewards/margins": 2.8213589191436768, + "rewards/rejected": -4.679849147796631, + "step": 3530 + }, + { + "epoch": 0.8493282149712092, + "grad_norm": 22.056135137773204, + "learning_rate": 3.3790885242841296e-08, + "logits/chosen": -1.0288774967193604, + "logits/rejected": -1.073750376701355, + "logps/chosen": -450.322265625, + "logps/rejected": -869.6065673828125, + "loss": 0.3731, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -2.2526326179504395, + "rewards/margins": 4.20005464553833, + "rewards/rejected": -6.4526872634887695, + "step": 3540 + }, + { + "epoch": 0.8517274472168906, + "grad_norm": 19.737949374986822, + "learning_rate": 3.274726219106677e-08, + "logits/chosen": -1.0363576412200928, + "logits/rejected": -1.0900583267211914, + "logps/chosen": -513.31298828125, + "logps/rejected": -884.9781494140625, + "loss": 0.4439, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.4154794216156006, + "rewards/margins": 3.753586530685425, + "rewards/rejected": -6.169066429138184, + "step": 3550 + }, + { + "epoch": 0.8541266794625719, + "grad_norm": 17.054064949812986, + "learning_rate": 3.171887855571642e-08, + "logits/chosen": -1.07740318775177, + "logits/rejected": -1.048232078552246, + "logps/chosen": -420.1988830566406, + "logps/rejected": -608.5756225585938, + "loss": 0.3766, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.087628126144409, + "rewards/margins": 1.9552154541015625, + "rewards/rejected": -4.042843818664551, + "step": 3560 + }, + { + "epoch": 0.8565259117082533, + "grad_norm": 17.863686096756574, + "learning_rate": 3.070580647381643e-08, + "logits/chosen": -1.0473045110702515, + "logits/rejected": -1.067647933959961, + "logps/chosen": -426.6707458496094, + "logps/rejected": -775.1964111328125, + "loss": 0.4329, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.1260108947753906, + "rewards/margins": 3.496943950653076, + "rewards/rejected": -5.622954845428467, + "step": 3570 + }, + { + "epoch": 0.8589251439539347, + "grad_norm": 12.554705710287928, + "learning_rate": 2.9708117008348576e-08, + "logits/chosen": -1.1423534154891968, + "logits/rejected": -1.2145392894744873, + "logps/chosen": -519.9121704101562, + "logps/rejected": -651.9493408203125, + "loss": 0.401, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.350532054901123, + "rewards/margins": 1.7727361917495728, + "rewards/rejected": -4.1232686042785645, + "step": 3580 + }, + { + "epoch": 0.8613243761996161, + "grad_norm": 17.33820204066364, + "learning_rate": 2.8725880143264992e-08, + "logits/chosen": -1.1288349628448486, + "logits/rejected": -1.1007946729660034, + "logps/chosen": -472.87713623046875, + "logps/rejected": -697.078125, + "loss": 0.4697, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.494798183441162, + "rewards/margins": 1.797978162765503, + "rewards/rejected": -4.292776584625244, + "step": 3590 + }, + { + "epoch": 0.8637236084452975, + "grad_norm": 19.735604660872788, + "learning_rate": 2.775916477857948e-08, + "logits/chosen": -1.0792783498764038, + "logits/rejected": -1.0856952667236328, + "logps/chosen": -425.07177734375, + "logps/rejected": -650.5399169921875, + "loss": 0.3895, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.2981741428375244, + "rewards/margins": 2.2162742614746094, + "rewards/rejected": -4.514448165893555, + "step": 3600 + }, + { + "epoch": 0.8661228406909789, + "grad_norm": 19.83106738953191, + "learning_rate": 2.680803872553408e-08, + "logits/chosen": -1.0853979587554932, + "logits/rejected": -1.0893454551696777, + "logps/chosen": -422.78076171875, + "logps/rejected": -977.7130126953125, + "loss": 0.426, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -2.013327121734619, + "rewards/margins": 5.603024959564209, + "rewards/rejected": -7.616351127624512, + "step": 3610 + }, + { + "epoch": 0.8685220729366603, + "grad_norm": 24.90411371661287, + "learning_rate": 2.5872568701842706e-08, + "logits/chosen": -1.1065361499786377, + "logits/rejected": -1.121259331703186, + "logps/chosen": -385.88641357421875, + "logps/rejected": -718.7703857421875, + "loss": 0.4678, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.989143967628479, + "rewards/margins": 3.0591297149658203, + "rewards/rejected": -5.04827356338501, + "step": 3620 + }, + { + "epoch": 0.8709213051823417, + "grad_norm": 20.14060859194896, + "learning_rate": 2.495282032701096e-08, + "logits/chosen": -1.0481445789337158, + "logits/rejected": -1.1717673540115356, + "logps/chosen": -347.03582763671875, + "logps/rejected": -661.1770629882812, + "loss": 0.3896, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -1.8835344314575195, + "rewards/margins": 3.3319649696350098, + "rewards/rejected": -5.215498924255371, + "step": 3630 + }, + { + "epoch": 0.8733205374280231, + "grad_norm": 19.14741858415139, + "learning_rate": 2.4048858117733133e-08, + "logits/chosen": -1.1484493017196655, + "logits/rejected": -1.1738967895507812, + "logps/chosen": -448.4156188964844, + "logps/rejected": -831.2892456054688, + "loss": 0.3792, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.096663475036621, + "rewards/margins": 4.216563701629639, + "rewards/rejected": -6.31322717666626, + "step": 3640 + }, + { + "epoch": 0.8757197696737045, + "grad_norm": 15.380617560449137, + "learning_rate": 2.3160745483366938e-08, + "logits/chosen": -1.0661578178405762, + "logits/rejected": -1.0423481464385986, + "logps/chosen": -451.716796875, + "logps/rejected": -690.4423217773438, + "loss": 0.4006, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.405106544494629, + "rewards/margins": 1.979060411453247, + "rewards/rejected": -4.384166717529297, + "step": 3650 + }, + { + "epoch": 0.8781190019193857, + "grad_norm": 17.884025211619726, + "learning_rate": 2.2288544721485197e-08, + "logits/chosen": -1.1487770080566406, + "logits/rejected": -1.1188671588897705, + "logps/chosen": -369.1369323730469, + "logps/rejected": -784.3793334960938, + "loss": 0.4026, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.7735888957977295, + "rewards/margins": 3.929485321044922, + "rewards/rejected": -5.7030744552612305, + "step": 3660 + }, + { + "epoch": 0.8805182341650671, + "grad_norm": 16.690714182145946, + "learning_rate": 2.1432317013506117e-08, + "logits/chosen": -1.1557639837265015, + "logits/rejected": -1.235864281654358, + "logps/chosen": -479.1221618652344, + "logps/rejected": -734.0775146484375, + "loss": 0.4415, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.424396276473999, + "rewards/margins": 2.9635913372039795, + "rewards/rejected": -5.38798713684082, + "step": 3670 + }, + { + "epoch": 0.8829174664107485, + "grad_norm": 19.313778590115387, + "learning_rate": 2.0592122420401704e-08, + "logits/chosen": -0.9829009771347046, + "logits/rejected": -1.0783188343048096, + "logps/chosen": -424.56463623046875, + "logps/rejected": -663.997802734375, + "loss": 0.4162, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.211613178253174, + "rewards/margins": 2.252444267272949, + "rewards/rejected": -4.464056968688965, + "step": 3680 + }, + { + "epoch": 0.8853166986564299, + "grad_norm": 16.700042187142635, + "learning_rate": 1.976801987848459e-08, + "logits/chosen": -1.1402798891067505, + "logits/rejected": -1.111434817314148, + "logps/chosen": -460.59893798828125, + "logps/rejected": -831.3668212890625, + "loss": 0.4262, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.2458419799804688, + "rewards/margins": 3.5192267894744873, + "rewards/rejected": -5.765069007873535, + "step": 3690 + }, + { + "epoch": 0.8877159309021113, + "grad_norm": 18.002635499528342, + "learning_rate": 1.8960067195273987e-08, + "logits/chosen": -1.149012565612793, + "logits/rejected": -1.1922236680984497, + "logps/chosen": -389.1723327636719, + "logps/rejected": -704.3775024414062, + "loss": 0.3902, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -1.9992425441741943, + "rewards/margins": 3.1408438682556152, + "rewards/rejected": -5.140086650848389, + "step": 3700 + }, + { + "epoch": 0.8901151631477927, + "grad_norm": 14.779338531754624, + "learning_rate": 1.816832104544072e-08, + "logits/chosen": -0.9585069417953491, + "logits/rejected": -0.9837729334831238, + "logps/chosen": -488.01055908203125, + "logps/rejected": -722.9725341796875, + "loss": 0.4102, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.372795581817627, + "rewards/margins": 2.566098928451538, + "rewards/rejected": -4.938894748687744, + "step": 3710 + }, + { + "epoch": 0.8925143953934741, + "grad_norm": 14.732524058414059, + "learning_rate": 1.7392836966831553e-08, + "logits/chosen": -0.8768994212150574, + "logits/rejected": -0.8977943658828735, + "logps/chosen": -459.11309814453125, + "logps/rejected": -748.9290161132812, + "loss": 0.3607, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.143648147583008, + "rewards/margins": 3.162809371948242, + "rewards/rejected": -5.30645751953125, + "step": 3720 + }, + { + "epoch": 0.8949136276391555, + "grad_norm": 17.982384045789107, + "learning_rate": 1.663366935657373e-08, + "logits/chosen": -1.1603506803512573, + "logits/rejected": -1.2332944869995117, + "logps/chosen": -403.1298828125, + "logps/rejected": -722.2110595703125, + "loss": 0.4374, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.1506400108337402, + "rewards/margins": 3.0482773780822754, + "rewards/rejected": -5.198916912078857, + "step": 3730 + }, + { + "epoch": 0.8973128598848369, + "grad_norm": 18.681352163564856, + "learning_rate": 1.5890871467258898e-08, + "logits/chosen": -0.9103895425796509, + "logits/rejected": -0.9239616394042969, + "logps/chosen": -530.7114868164062, + "logps/rejected": -747.0721435546875, + "loss": 0.4056, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.3054797649383545, + "rewards/margins": 2.478546619415283, + "rewards/rejected": -4.784027099609375, + "step": 3740 + }, + { + "epoch": 0.8997120921305183, + "grad_norm": 14.698356072509888, + "learning_rate": 1.5164495403207967e-08, + "logits/chosen": -1.153602123260498, + "logits/rejected": -1.12786066532135, + "logps/chosen": -489.3046875, + "logps/rejected": -890.0556640625, + "loss": 0.3969, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.4070847034454346, + "rewards/margins": 3.750683546066284, + "rewards/rejected": -6.157768249511719, + "step": 3750 + }, + { + "epoch": 0.9021113243761996, + "grad_norm": 15.890482276272099, + "learning_rate": 1.4454592116815962e-08, + "logits/chosen": -1.0214381217956543, + "logits/rejected": -1.004897117614746, + "logps/chosen": -439.8526916503906, + "logps/rejected": -769.1018676757812, + "loss": 0.3697, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -1.9336268901824951, + "rewards/margins": 3.114084005355835, + "rewards/rejected": -5.047710418701172, + "step": 3760 + }, + { + "epoch": 0.904510556621881, + "grad_norm": 11.251412552986052, + "learning_rate": 1.3761211404977934e-08, + "logits/chosen": -1.1649322509765625, + "logits/rejected": -1.1327035427093506, + "logps/chosen": -441.202880859375, + "logps/rejected": -933.3021240234375, + "loss": 0.3523, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.2993717193603516, + "rewards/margins": 4.983204364776611, + "rewards/rejected": -7.282576560974121, + "step": 3770 + }, + { + "epoch": 0.9069097888675623, + "grad_norm": 20.47517342425732, + "learning_rate": 1.3084401905596177e-08, + "logits/chosen": -1.0593699216842651, + "logits/rejected": -1.1538712978363037, + "logps/chosen": -489.8775939941406, + "logps/rejected": -762.0972290039062, + "loss": 0.4482, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.2437944412231445, + "rewards/margins": 3.1374268531799316, + "rewards/rejected": -5.381220817565918, + "step": 3780 + }, + { + "epoch": 0.9093090211132437, + "grad_norm": 22.249007795619104, + "learning_rate": 1.2424211094168053e-08, + "logits/chosen": -0.9747680425643921, + "logits/rejected": -1.0482494831085205, + "logps/chosen": -508.7771911621094, + "logps/rejected": -758.6298217773438, + "loss": 0.3939, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.0802645683288574, + "rewards/margins": 2.5900025367736816, + "rewards/rejected": -4.670267581939697, + "step": 3790 + }, + { + "epoch": 0.9117082533589251, + "grad_norm": 15.390160001717241, + "learning_rate": 1.1780685280456143e-08, + "logits/chosen": -1.118100881576538, + "logits/rejected": -1.0623310804367065, + "logps/chosen": -522.0684814453125, + "logps/rejected": -999.7626953125, + "loss": 0.4403, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.4960930347442627, + "rewards/margins": 4.626833438873291, + "rewards/rejected": -7.122926235198975, + "step": 3800 + }, + { + "epoch": 0.9141074856046065, + "grad_norm": 18.77137560752492, + "learning_rate": 1.1153869605239564e-08, + "logits/chosen": -1.1241267919540405, + "logits/rejected": -1.2161864042282104, + "logps/chosen": -470.51123046875, + "logps/rejected": -613.9768676757812, + "loss": 0.409, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.1745758056640625, + "rewards/margins": 1.793039083480835, + "rewards/rejected": -3.9676146507263184, + "step": 3810 + }, + { + "epoch": 0.9165067178502879, + "grad_norm": 16.685258955661418, + "learning_rate": 1.0543808037147606e-08, + "logits/chosen": -1.140171766281128, + "logits/rejected": -1.1334731578826904, + "logps/chosen": -439.6238708496094, + "logps/rejected": -944.9068603515625, + "loss": 0.4045, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.9790058135986328, + "rewards/margins": 5.0386128425598145, + "rewards/rejected": -7.017618656158447, + "step": 3820 + }, + { + "epoch": 0.9189059500959693, + "grad_norm": 11.823950534472615, + "learning_rate": 9.95054336957557e-09, + "logits/chosen": -1.1124851703643799, + "logits/rejected": -1.0793733596801758, + "logps/chosen": -450.46759033203125, + "logps/rejected": -621.4553833007812, + "loss": 0.3762, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.129909038543701, + "rewards/margins": 1.6316797733306885, + "rewards/rejected": -3.7615890502929688, + "step": 3830 + }, + { + "epoch": 0.9213051823416507, + "grad_norm": 16.192817201214602, + "learning_rate": 9.37411721768286e-09, + "logits/chosen": -1.1737887859344482, + "logits/rejected": -1.1507189273834229, + "logps/chosen": -509.4762268066406, + "logps/rejected": -871.7518310546875, + "loss": 0.4002, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.6139490604400635, + "rewards/margins": 3.223182201385498, + "rewards/rejected": -5.837131023406982, + "step": 3840 + }, + { + "epoch": 0.9237044145873321, + "grad_norm": 12.058728640051894, + "learning_rate": 8.81457001547392e-09, + "logits/chosen": -1.022780418395996, + "logits/rejected": -0.9987403154373169, + "logps/chosen": -478.4554138183594, + "logps/rejected": -652.6536865234375, + "loss": 0.4088, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.370469570159912, + "rewards/margins": 1.6161177158355713, + "rewards/rejected": -3.9865875244140625, + "step": 3850 + }, + { + "epoch": 0.9261036468330134, + "grad_norm": 13.938662102562736, + "learning_rate": 8.271941012961942e-09, + "logits/chosen": -1.0831263065338135, + "logits/rejected": -0.9888957738876343, + "logps/chosen": -427.6864318847656, + "logps/rejected": -906.8592529296875, + "loss": 0.3924, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.329150676727295, + "rewards/margins": 4.188321113586426, + "rewards/rejected": -6.5174713134765625, + "step": 3860 + }, + { + "epoch": 0.9285028790786948, + "grad_norm": 17.278174451554634, + "learning_rate": 7.746268273415568e-09, + "logits/chosen": -1.2020190954208374, + "logits/rejected": -1.1284525394439697, + "logps/chosen": -484.2650451660156, + "logps/rejected": -632.048583984375, + "loss": 0.4266, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.3161118030548096, + "rewards/margins": 1.0392600297927856, + "rewards/rejected": -3.3553714752197266, + "step": 3870 + }, + { + "epoch": 0.9309021113243762, + "grad_norm": 15.498111728144039, + "learning_rate": 7.237588670689076e-09, + "logits/chosen": -1.0772826671600342, + "logits/rejected": -1.160631537437439, + "logps/chosen": -455.0005798339844, + "logps/rejected": -892.5280151367188, + "loss": 0.3896, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.258082866668701, + "rewards/margins": 4.663710117340088, + "rewards/rejected": -6.921793460845947, + "step": 3880 + }, + { + "epoch": 0.9333013435700576, + "grad_norm": 15.18662366523353, + "learning_rate": 6.745937886635606e-09, + "logits/chosen": -1.0644091367721558, + "logits/rejected": -0.9969073534011841, + "logps/chosen": -481.2547912597656, + "logps/rejected": -849.2018432617188, + "loss": 0.3922, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.187262773513794, + "rewards/margins": 3.615166425704956, + "rewards/rejected": -5.802428722381592, + "step": 3890 + }, + { + "epoch": 0.935700575815739, + "grad_norm": 18.86870094684195, + "learning_rate": 6.271350408604409e-09, + "logits/chosen": -1.1208722591400146, + "logits/rejected": -1.0991759300231934, + "logps/chosen": -380.5445251464844, + "logps/rejected": -697.8775024414062, + "loss": 0.412, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -1.7180122137069702, + "rewards/margins": 2.9493772983551025, + "rewards/rejected": -4.667389869689941, + "step": 3900 + }, + { + "epoch": 0.9380998080614203, + "grad_norm": 13.841623997146602, + "learning_rate": 5.813859527021487e-09, + "logits/chosen": -1.1407277584075928, + "logits/rejected": -1.14281165599823, + "logps/chosen": -466.45355224609375, + "logps/rejected": -869.8348388671875, + "loss": 0.3899, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.353449821472168, + "rewards/margins": 4.2861199378967285, + "rewards/rejected": -6.6395697593688965, + "step": 3910 + }, + { + "epoch": 0.9404990403071017, + "grad_norm": 14.912969619458565, + "learning_rate": 5.373497333054616e-09, + "logits/chosen": -1.163573980331421, + "logits/rejected": -1.1741966009140015, + "logps/chosen": -495.27313232421875, + "logps/rejected": -631.5628662109375, + "loss": 0.4395, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.410792589187622, + "rewards/margins": 1.440456748008728, + "rewards/rejected": -3.8512492179870605, + "step": 3920 + }, + { + "epoch": 0.9428982725527831, + "grad_norm": 18.054972694480718, + "learning_rate": 4.950294716362213e-09, + "logits/chosen": -1.1209748983383179, + "logits/rejected": -1.1992738246917725, + "logps/chosen": -533.5642700195312, + "logps/rejected": -736.080810546875, + "loss": 0.4296, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.637540817260742, + "rewards/margins": 2.0387625694274902, + "rewards/rejected": -4.676303386688232, + "step": 3930 + }, + { + "epoch": 0.9452975047984645, + "grad_norm": 14.468498354603758, + "learning_rate": 4.544281362926422e-09, + "logits/chosen": -1.0455518960952759, + "logits/rejected": -1.061714768409729, + "logps/chosen": -488.90576171875, + "logps/rejected": -747.364013671875, + "loss": 0.4202, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.0449106693267822, + "rewards/margins": 2.6913721561431885, + "rewards/rejected": -4.736282825469971, + "step": 3940 + }, + { + "epoch": 0.9476967370441459, + "grad_norm": 15.662743345785968, + "learning_rate": 4.15548575297095e-09, + "logits/chosen": -1.0954151153564453, + "logits/rejected": -1.108139991760254, + "logps/chosen": -445.718994140625, + "logps/rejected": -760.5974731445312, + "loss": 0.4125, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.274768114089966, + "rewards/margins": 3.2226016521453857, + "rewards/rejected": -5.497369766235352, + "step": 3950 + }, + { + "epoch": 0.9500959692898272, + "grad_norm": 12.35347569811952, + "learning_rate": 3.7839351589631366e-09, + "logits/chosen": -1.0665959119796753, + "logits/rejected": -0.9220373034477234, + "logps/chosen": -430.4939880371094, + "logps/rejected": -755.52587890625, + "loss": 0.3998, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.2020294666290283, + "rewards/margins": 2.7335734367370605, + "rewards/rejected": -4.93560266494751, + "step": 3960 + }, + { + "epoch": 0.9524952015355086, + "grad_norm": 17.123505773306672, + "learning_rate": 3.4296556437010405e-09, + "logits/chosen": -1.2090275287628174, + "logits/rejected": -1.2055715322494507, + "logps/chosen": -387.20947265625, + "logps/rejected": -725.6389770507812, + "loss": 0.3924, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.0009734630584717, + "rewards/margins": 3.3572704792022705, + "rewards/rejected": -5.3582444190979, + "step": 3970 + }, + { + "epoch": 0.95489443378119, + "grad_norm": 26.53516553962381, + "learning_rate": 3.092672058485124e-09, + "logits/chosen": -1.2251334190368652, + "logits/rejected": -1.1628016233444214, + "logps/chosen": -447.06671142578125, + "logps/rejected": -905.41943359375, + "loss": 0.469, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.332759380340576, + "rewards/margins": 4.409472465515137, + "rewards/rejected": -6.742232322692871, + "step": 3980 + }, + { + "epoch": 0.9572936660268714, + "grad_norm": 15.849586481858255, + "learning_rate": 2.7730080413750356e-09, + "logits/chosen": -1.0247732400894165, + "logits/rejected": -1.0921010971069336, + "logps/chosen": -465.94317626953125, + "logps/rejected": -652.0645751953125, + "loss": 0.4045, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -2.15604829788208, + "rewards/margins": 1.82071852684021, + "rewards/rejected": -3.976767063140869, + "step": 3990 + }, + { + "epoch": 0.9596928982725528, + "grad_norm": 12.747060730063819, + "learning_rate": 2.4706860155316033e-09, + "logits/chosen": -1.0665229558944702, + "logits/rejected": -1.1173847913742065, + "logps/chosen": -532.3182983398438, + "logps/rejected": -821.4293212890625, + "loss": 0.4311, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.262028932571411, + "rewards/margins": 2.8439760208129883, + "rewards/rejected": -5.106005668640137, + "step": 4000 + }, + { + "epoch": 0.9596928982725528, + "eval_logits/chosen": -1.2033898830413818, + "eval_logits/rejected": -1.1906352043151855, + "eval_logps/chosen": -463.1195373535156, + "eval_logps/rejected": -796.0059204101562, + "eval_loss": 0.40018871426582336, + "eval_rewards/accuracies": 0.8535714149475098, + "eval_rewards/chosen": -2.245985507965088, + "eval_rewards/margins": 3.253223419189453, + "eval_rewards/rejected": -5.499208927154541, + "eval_runtime": 202.8468, + "eval_samples_per_second": 21.992, + "eval_steps_per_second": 0.345, + "step": 4000 + }, + { + "epoch": 0.9620921305182342, + "grad_norm": 21.87211176820157, + "learning_rate": 2.185727187643843e-09, + "logits/chosen": -1.122045874595642, + "logits/rejected": -1.1218305826187134, + "logps/chosen": -418.22686767578125, + "logps/rejected": -835.3477783203125, + "loss": 0.4449, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -2.2049636840820312, + "rewards/margins": 4.108201503753662, + "rewards/rejected": -6.313164710998535, + "step": 4010 + }, + { + "epoch": 0.9644913627639156, + "grad_norm": 20.178611681191477, + "learning_rate": 1.9181515464413434e-09, + "logits/chosen": -1.0017145872116089, + "logits/rejected": -0.9959618449211121, + "logps/chosen": -547.8878173828125, + "logps/rejected": -944.1638793945312, + "loss": 0.3753, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.129427433013916, + "rewards/margins": 3.9424376487731934, + "rewards/rejected": -6.071865081787109, + "step": 4020 + }, + { + "epoch": 0.966890595009597, + "grad_norm": 14.197633100109993, + "learning_rate": 1.6679778612923302e-09, + "logits/chosen": -1.0468965768814087, + "logits/rejected": -1.1212776899337769, + "logps/chosen": -515.878173828125, + "logps/rejected": -670.55224609375, + "loss": 0.3835, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.4107260704040527, + "rewards/margins": 1.5576350688934326, + "rewards/rejected": -3.9683613777160645, + "step": 4030 + }, + { + "epoch": 0.9692898272552783, + "grad_norm": 20.71404401442789, + "learning_rate": 1.43522368088686e-09, + "logits/chosen": -1.0676016807556152, + "logits/rejected": -1.0992966890335083, + "logps/chosen": -489.1631774902344, + "logps/rejected": -992.6973876953125, + "loss": 0.4542, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.4302258491516113, + "rewards/margins": 4.995226860046387, + "rewards/rejected": -7.425453186035156, + "step": 4040 + }, + { + "epoch": 0.9716890595009597, + "grad_norm": 18.762913271326, + "learning_rate": 1.2199053320059993e-09, + "logits/chosen": -1.0681426525115967, + "logits/rejected": -1.047761082649231, + "logps/chosen": -479.65350341796875, + "logps/rejected": -744.2103881835938, + "loss": 0.406, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.2822158336639404, + "rewards/margins": 2.5112454891204834, + "rewards/rejected": -4.793460845947266, + "step": 4050 + }, + { + "epoch": 0.974088291746641, + "grad_norm": 15.021979385637746, + "learning_rate": 1.0220379183764338e-09, + "logits/chosen": -1.1485240459442139, + "logits/rejected": -1.0922056436538696, + "logps/chosen": -389.207275390625, + "logps/rejected": -744.5093383789062, + "loss": 0.3763, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.987817406654358, + "rewards/margins": 3.430870771408081, + "rewards/rejected": -5.418687343597412, + "step": 4060 + }, + { + "epoch": 0.9764875239923224, + "grad_norm": 13.67162874026034, + "learning_rate": 8.416353196111503e-10, + "logits/chosen": -1.1141016483306885, + "logits/rejected": -1.0663163661956787, + "logps/chosen": -481.4126892089844, + "logps/rejected": -799.8731689453125, + "loss": 0.4452, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.6318254470825195, + "rewards/margins": 3.2658958435058594, + "rewards/rejected": -5.897720813751221, + "step": 4070 + }, + { + "epoch": 0.9788867562380038, + "grad_norm": 17.482908148705075, + "learning_rate": 6.787101902356873e-10, + "logits/chosen": -1.200620412826538, + "logits/rejected": -1.1152732372283936, + "logps/chosen": -499.2351989746094, + "logps/rejected": -834.9778442382812, + "loss": 0.3757, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.555325984954834, + "rewards/margins": 3.111412525177002, + "rewards/rejected": -5.666738510131836, + "step": 4080 + }, + { + "epoch": 0.9812859884836852, + "grad_norm": 18.707983969311634, + "learning_rate": 5.332739588005953e-10, + "logits/chosen": -1.1368467807769775, + "logits/rejected": -1.1244173049926758, + "logps/chosen": -390.2232971191406, + "logps/rejected": -816.9163818359375, + "loss": 0.4268, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.021688461303711, + "rewards/margins": 3.9690163135528564, + "rewards/rejected": -5.9907050132751465, + "step": 4090 + }, + { + "epoch": 0.9836852207293666, + "grad_norm": 21.443691190389707, + "learning_rate": 4.053368270797164e-10, + "logits/chosen": -1.0214062929153442, + "logits/rejected": -1.0243003368377686, + "logps/chosen": -468.2923889160156, + "logps/rejected": -744.1151123046875, + "loss": 0.421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.444369077682495, + "rewards/margins": 2.769819736480713, + "rewards/rejected": -5.214189052581787, + "step": 4100 + }, + { + "epoch": 0.986084452975048, + "grad_norm": 12.556013081787883, + "learning_rate": 2.949077693545354e-10, + "logits/chosen": -0.9810346364974976, + "logits/rejected": -1.0578103065490723, + "logps/chosen": -514.701416015625, + "logps/rejected": -736.8178100585938, + "loss": 0.4632, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.4852397441864014, + "rewards/margins": 1.9187476634979248, + "rewards/rejected": -4.403986930847168, + "step": 4110 + }, + { + "epoch": 0.9884836852207294, + "grad_norm": 14.55216329888536, + "learning_rate": 2.0199453178471047e-10, + "logits/chosen": -1.0134025812149048, + "logits/rejected": -1.1030604839324951, + "logps/chosen": -521.6679077148438, + "logps/rejected": -632.6597900390625, + "loss": 0.4009, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -2.2684521675109863, + "rewards/margins": 1.4823023080825806, + "rewards/rejected": -3.7507545948028564, + "step": 4120 + }, + { + "epoch": 0.9908829174664108, + "grad_norm": 15.400882538943376, + "learning_rate": 1.266036318647301e-10, + "logits/chosen": -1.0723247528076172, + "logits/rejected": -1.0602704286575317, + "logps/chosen": -508.36163330078125, + "logps/rejected": -874.0475463867188, + "loss": 0.3882, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -2.1347973346710205, + "rewards/margins": 3.8190529346466064, + "rewards/rejected": -5.9538493156433105, + "step": 4130 + }, + { + "epoch": 0.9932821497120922, + "grad_norm": 21.162328364678483, + "learning_rate": 6.874035796672339e-11, + "logits/chosen": -1.1026606559753418, + "logits/rejected": -1.1409575939178467, + "logps/chosen": -481.29962158203125, + "logps/rejected": -960.5101318359375, + "loss": 0.4091, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.1151065826416016, + "rewards/margins": 5.2843122482299805, + "rewards/rejected": -7.399418830871582, + "step": 4140 + }, + { + "epoch": 0.9956813819577736, + "grad_norm": 17.96942114686989, + "learning_rate": 2.8408768969423458e-11, + "logits/chosen": -1.1128456592559814, + "logits/rejected": -1.0932135581970215, + "logps/chosen": -469.04327392578125, + "logps/rejected": -757.17431640625, + "loss": 0.3819, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.0967726707458496, + "rewards/margins": 2.7530088424682617, + "rewards/rejected": -4.8497819900512695, + "step": 4150 + }, + { + "epoch": 0.9980806142034548, + "grad_norm": 18.413638935256603, + "learning_rate": 5.611693973617271e-12, + "logits/chosen": -1.0999760627746582, + "logits/rejected": -1.0788906812667847, + "logps/chosen": -418.7471618652344, + "logps/rejected": -718.1671142578125, + "loss": 0.4369, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.1133265495300293, + "rewards/margins": 2.8272883892059326, + "rewards/rejected": -4.940615177154541, + "step": 4160 + }, + { + "epoch": 1.0, + "step": 4168, + "total_flos": 0.0, + "train_loss": 0.4717116741438516, + "train_runtime": 20989.4384, + "train_samples_per_second": 6.354, + "train_steps_per_second": 0.199 + } + ], + "logging_steps": 10, + "max_steps": 4168, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 5000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +}