{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 702,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 503.30001945495604,
      "epoch": 0.042735042735042736,
      "grad_norm": 0.14690014719963074,
      "kl": 0.0005318611507391324,
      "learning_rate": 9.871794871794872e-05,
      "loss": -0.0402,
      "num_tokens": 33288.0,
      "reward": -1.1772359251976012,
      "reward_std": 2.312183880805969,
      "rewards/reward_function": -1.1772359311580658,
      "step": 10
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 509.85001373291016,
      "epoch": 0.08547008547008547,
      "grad_norm": 0.16410136222839355,
      "kl": 0.001959062390960753,
      "learning_rate": 9.72934472934473e-05,
      "loss": -0.008,
      "num_tokens": 67317.0,
      "reward": -1.455158567428589,
      "reward_std": 2.39455783367157,
      "rewards/reward_function": -1.4551584720611572,
      "step": 20
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 382.58334197998045,
      "epoch": 0.1282051282051282,
      "grad_norm": 0.5200854539871216,
      "kl": 0.003280931804329157,
      "learning_rate": 9.586894586894587e-05,
      "loss": 0.0011,
      "num_tokens": 93788.0,
      "reward": 2.4295308887958527,
      "reward_std": 1.948470675945282,
      "rewards/reward_function": 2.4295308709144594,
      "step": 30
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 466.25001068115233,
      "epoch": 0.17094017094017094,
      "grad_norm": 0.2215566486120224,
      "kl": 0.005862738820724189,
      "learning_rate": 9.444444444444444e-05,
      "loss": -0.0111,
      "num_tokens": 125045.0,
      "reward": 1.2611542105674745,
      "reward_std": 2.462652099132538,
      "rewards/reward_function": 1.2611541628837586,
      "step": 40
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 518.6166809082031,
      "epoch": 0.21367521367521367,
      "grad_norm": 0.15176545083522797,
      "kl": 0.008088351227343082,
      "learning_rate": 9.301994301994303e-05,
      "loss": -0.0085,
      "num_tokens": 159318.0,
      "reward": -1.0398478865623475,
      "reward_std": 1.9632926762104035,
      "rewards/reward_function": -1.039847767353058,
      "step": 50
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 436.50001125335694,
      "epoch": 0.2564102564102564,
      "grad_norm": 0.23773637413978577,
      "kl": 0.007962367637082935,
      "learning_rate": 9.15954415954416e-05,
      "loss": 0.0155,
      "num_tokens": 188622.0,
      "reward": -0.9993367567658424,
      "reward_std": 2.5938119292259216,
      "rewards/reward_function": -0.9993367329239845,
      "step": 60
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 465.0166717529297,
      "epoch": 0.29914529914529914,
      "grad_norm": 0.22686824202537537,
      "kl": 0.01036710049957037,
      "learning_rate": 9.017094017094018e-05,
      "loss": 0.0232,
      "num_tokens": 219649.0,
      "reward": -1.0854085847735404,
      "reward_std": 2.3031186699867248,
      "rewards/reward_function": -1.0854085013270378,
      "step": 70
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 594.3666900634765,
      "epoch": 0.3418803418803419,
      "grad_norm": 0.2330074906349182,
      "kl": 0.011198346642777324,
      "learning_rate": 8.874643874643875e-05,
      "loss": 0.0085,
      "num_tokens": 258521.0,
      "reward": 0.44582319259643555,
      "reward_std": 2.180183058977127,
      "rewards/reward_function": 0.44582313299179077,
      "step": 80
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 363.2166831970215,
      "epoch": 0.38461538461538464,
      "grad_norm": 1.2950036525726318,
      "kl": 0.01624974999576807,
      "learning_rate": 8.732193732193732e-05,
      "loss": -0.0279,
      "num_tokens": 283896.0,
      "reward": -0.22942656874656678,
      "reward_std": 2.698866534233093,
      "rewards/reward_function": -0.2294266164302826,
      "step": 90
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 366.3166763305664,
      "epoch": 0.42735042735042733,
      "grad_norm": 0.44916248321533203,
      "kl": 0.0371546683833003,
      "learning_rate": 8.58974358974359e-05,
      "loss": -0.034,
      "num_tokens": 309127.0,
      "reward": 1.030603051185608,
      "reward_std": 2.1768004059791566,
      "rewards/reward_function": 1.03060302734375,
      "step": 100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 627.7833419799805,
      "epoch": 0.4700854700854701,
      "grad_norm": 0.24189692735671997,
      "kl": 0.02031996361911297,
      "learning_rate": 8.447293447293447e-05,
      "loss": 0.0057,
      "num_tokens": 350124.0,
      "reward": 1.7054275810718535,
      "reward_std": 2.761507248878479,
      "rewards/reward_function": 1.7054275453090668,
      "step": 110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 466.7833419799805,
      "epoch": 0.5128205128205128,
      "grad_norm": 0.3966105282306671,
      "kl": 0.03020444568246603,
      "learning_rate": 8.304843304843305e-05,
      "loss": 0.0158,
      "num_tokens": 381323.0,
      "reward": 0.4135805606842041,
      "reward_std": 2.793270480632782,
      "rewards/reward_function": 0.41358056366443635,
      "step": 120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 418.7166778564453,
      "epoch": 0.5555555555555556,
      "grad_norm": 0.8130244016647339,
      "kl": 0.050436797365546224,
      "learning_rate": 8.162393162393163e-05,
      "loss": 0.0109,
      "num_tokens": 409632.0,
      "reward": 3.6511381447315214,
      "reward_std": 2.057768177986145,
      "rewards/reward_function": 3.651138073205948,
      "step": 130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.18334617614744,
      "epoch": 0.5982905982905983,
      "grad_norm": 0.237253338098526,
      "kl": 0.08577278926968575,
      "learning_rate": 8.01994301994302e-05,
      "loss": 0.005,
      "num_tokens": 436613.0,
      "reward": 3.9179525792598726,
      "reward_std": 2.5252403259277343,
      "rewards/reward_function": 3.9179525285959245,
      "step": 140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 534.9666793823242,
      "epoch": 0.6410256410256411,
      "grad_norm": 0.23871329426765442,
      "kl": 0.0823534980416298,
      "learning_rate": 7.877492877492878e-05,
      "loss": 0.0435,
      "num_tokens": 471891.0,
      "reward": 4.100382626056671,
      "reward_std": 2.5033695101737976,
      "rewards/reward_function": 4.100382459163666,
      "step": 150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 332.50001373291013,
      "epoch": 0.6837606837606838,
      "grad_norm": 0.8124951720237732,
      "kl": 0.15444251857697963,
      "learning_rate": 7.735042735042735e-05,
      "loss": 0.0189,
      "num_tokens": 494961.0,
      "reward": 5.122303450107575,
      "reward_std": 2.0907008171081545,
      "rewards/reward_function": 5.122303307056427,
      "step": 160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.25001373291013,
      "epoch": 0.7264957264957265,
      "grad_norm": 0.7909404635429382,
      "kl": 0.13212636522948742,
      "learning_rate": 7.592592592592593e-05,
      "loss": -0.0367,
      "num_tokens": 522258.0,
      "reward": 5.544608736038208,
      "reward_std": 1.7265446126461028,
      "rewards/reward_function": 5.544608497619629,
      "step": 170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 595.9833587646484,
      "epoch": 0.7692307692307693,
      "grad_norm": 0.15963077545166016,
      "kl": 0.09406365267932415,
      "learning_rate": 7.450142450142451e-05,
      "loss": -0.0366,
      "num_tokens": 561197.0,
      "reward": 7.151281929016113,
      "reward_std": 2.8219340562820436,
      "rewards/reward_function": 7.15128173828125,
      "step": 180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 566.9666809082031,
      "epoch": 0.811965811965812,
      "grad_norm": 0.18807636201381683,
      "kl": 0.06498432569205762,
      "learning_rate": 7.307692307692307e-05,
      "loss": -0.0543,
      "num_tokens": 598575.0,
      "reward": 1.7874940395355225,
      "reward_std": 2.3280829310417177,
      "rewards/reward_function": 1.7874938309192658,
      "step": 190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 831.9500183105469,
      "epoch": 0.8547008547008547,
      "grad_norm": 0.19820700585842133,
      "kl": 0.03369698449969292,
      "learning_rate": 7.165242165242165e-05,
      "loss": -0.0132,
      "num_tokens": 651618.0,
      "reward": 2.0692047476768494,
      "reward_std": 2.6171678781509398,
      "rewards/reward_function": 2.06920468211174,
      "step": 200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 390.78334617614746,
      "epoch": 0.8974358974358975,
      "grad_norm": 0.1917090117931366,
      "kl": 0.10797237679362297,
      "learning_rate": 7.022792022792024e-05,
      "loss": -0.011,
      "num_tokens": 677981.0,
      "reward": 5.218200743198395,
      "reward_std": 2.8841384768486025,
      "rewards/reward_function": 5.218200659751892,
      "step": 210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 485.500016784668,
      "epoch": 0.9401709401709402,
      "grad_norm": 0.18474219739437103,
      "kl": 0.11786541007459164,
      "learning_rate": 6.880341880341881e-05,
      "loss": -0.0031,
      "num_tokens": 710033.0,
      "reward": 5.224206709861756,
      "reward_std": 2.2454194366931914,
      "rewards/reward_function": 5.224206617474556,
      "step": 220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 458.06668243408205,
      "epoch": 0.9829059829059829,
      "grad_norm": 0.19587470591068268,
      "kl": 0.09677453897893429,
      "learning_rate": 6.737891737891738e-05,
      "loss": -0.0014,
      "num_tokens": 741117.0,
      "reward": 4.320850740373134,
      "reward_std": 2.3484508872032164,
      "rewards/reward_function": 4.320850642025471,
      "step": 230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 605.0666931152343,
      "epoch": 1.0256410256410255,
      "grad_norm": 0.26696279644966125,
      "kl": 0.10099472776055336,
      "learning_rate": 6.595441595441596e-05,
      "loss": -0.025,
      "num_tokens": 780847.0,
      "reward": 7.07135591506958,
      "reward_std": 2.179239821434021,
      "rewards/reward_function": 7.071355724334717,
      "step": 240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 570.1833526611329,
      "epoch": 1.0683760683760684,
      "grad_norm": 0.41289836168289185,
      "kl": 0.10571693740785122,
      "learning_rate": 6.452991452991453e-05,
      "loss": 0.0201,
      "num_tokens": 818394.0,
      "reward": 4.596634943783283,
      "reward_std": 2.0250564932823183,
      "rewards/reward_function": 4.5966349326074125,
      "step": 250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 638.4166839599609,
      "epoch": 1.1111111111111112,
      "grad_norm": 0.23291794955730438,
      "kl": 0.06395059525966644,
      "learning_rate": 6.310541310541312e-05,
      "loss": 0.0023,
      "num_tokens": 859765.0,
      "reward": 4.951379495859146,
      "reward_std": 2.139006716012955,
      "rewards/reward_function": 4.95137942135334,
      "step": 260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 425.76667404174805,
      "epoch": 1.1538461538461537,
      "grad_norm": 0.2002115249633789,
      "kl": 0.14280213601887226,
      "learning_rate": 6.168091168091168e-05,
      "loss": 0.0113,
      "num_tokens": 888389.0,
      "reward": 6.41916925907135,
      "reward_std": 2.205563151836395,
      "rewards/reward_function": 6.419169163703918,
      "step": 270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 523.6166763305664,
      "epoch": 1.1965811965811965,
      "grad_norm": 0.18461842834949493,
      "kl": 0.1309021409600973,
      "learning_rate": 6.025641025641026e-05,
      "loss": -0.0021,
      "num_tokens": 923046.0,
      "reward": 5.595602548122406,
      "reward_std": 2.1117491006851195,
      "rewards/reward_function": 5.59560244679451,
      "step": 280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 483.0666799545288,
      "epoch": 1.2393162393162394,
      "grad_norm": 3.1094970703125,
      "kl": 0.1820569921284914,
      "learning_rate": 5.883190883190883e-05,
      "loss": 0.0173,
      "num_tokens": 955420.0,
      "reward": 5.614298677444458,
      "reward_std": 2.518768322467804,
      "rewards/reward_function": 5.614298534393311,
      "step": 290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 617.6333404541016,
      "epoch": 1.282051282051282,
      "grad_norm": 0.3713136613368988,
      "kl": 0.11033637076616287,
      "learning_rate": 5.740740740740741e-05,
      "loss": 0.0062,
      "num_tokens": 995922.0,
      "reward": 4.5290528535842896,
      "reward_std": 2.494095575809479,
      "rewards/reward_function": 4.529052710533142,
      "step": 300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 442.0166839599609,
      "epoch": 1.3247863247863247,
      "grad_norm": 0.21180328726768494,
      "kl": 0.1499824345111847,
      "learning_rate": 5.5982905982905986e-05,
      "loss": -0.0013,
      "num_tokens": 1025455.0,
      "reward": 8.561688470840455,
      "reward_std": 1.9707270503044128,
      "rewards/reward_function": 8.56168818473816,
      "step": 310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 580.050008392334,
      "epoch": 1.3675213675213675,
      "grad_norm": 0.20131155848503113,
      "kl": 0.11668501645326615,
      "learning_rate": 5.4558404558404567e-05,
      "loss": -0.0146,
      "num_tokens": 1063354.0,
      "reward": 5.419061434268952,
      "reward_std": 2.1403504729270937,
      "rewards/reward_function": 5.419061243534088,
      "step": 320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 652.3166870117187,
      "epoch": 1.4102564102564101,
      "grad_norm": 0.2526203691959381,
      "kl": 0.09888706281781197,
      "learning_rate": 5.313390313390314e-05,
      "loss": 0.0084,
      "num_tokens": 1105397.0,
      "reward": 7.351496028900146,
      "reward_std": 1.773663091659546,
      "rewards/reward_function": 7.351495909690857,
      "step": 330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 718.36669921875,
      "epoch": 1.452991452991453,
      "grad_norm": 0.1585923731327057,
      "kl": 0.10820744708180427,
      "learning_rate": 5.1709401709401714e-05,
      "loss": 0.0009,
      "num_tokens": 1151751.0,
      "reward": 6.595072710514069,
      "reward_std": 1.881132996082306,
      "rewards/reward_function": 6.59507247209549,
      "step": 340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 623.8333557128906,
      "epoch": 1.4957264957264957,
      "grad_norm": 0.18694844841957092,
      "kl": 0.11230501309037208,
      "learning_rate": 5.028490028490028e-05,
      "loss": -0.0362,
      "num_tokens": 1192547.0,
      "reward": 6.264591613411904,
      "reward_std": 1.9230000615119933,
      "rewards/reward_function": 6.26459142267704,
      "step": 350
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 704.0500183105469,
      "epoch": 1.5384615384615383,
      "grad_norm": 0.20967163145542145,
      "kl": 0.08291292320936919,
      "learning_rate": 4.886039886039887e-05,
      "loss": -0.0191,
      "num_tokens": 1237892.0,
      "reward": 5.098879025876522,
      "reward_std": 2.248683771491051,
      "rewards/reward_function": 5.09887896925211,
      "step": 360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 661.6000152587891,
      "epoch": 1.5811965811965814,
      "grad_norm": 0.19468806684017181,
      "kl": 0.10559967942535878,
      "learning_rate": 4.7435897435897435e-05,
      "loss": -0.0077,
      "num_tokens": 1280828.0,
      "reward": 6.5063467741012575,
      "reward_std": 1.6363932967185975,
      "rewards/reward_function": 6.506346654891968,
      "step": 370
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 426.3000122070313,
      "epoch": 1.623931623931624,
      "grad_norm": 0.40712612867355347,
      "kl": 0.14229083359241484,
      "learning_rate": 4.6011396011396016e-05,
      "loss": -0.0067,
      "num_tokens": 1309910.0,
      "reward": 8.865666103363036,
      "reward_std": 1.6894314289093018,
      "rewards/reward_function": 8.865665578842163,
      "step": 380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 645.4000122070313,
      "epoch": 1.6666666666666665,
      "grad_norm": 0.1824493706226349,
      "kl": 0.14776081275194883,
      "learning_rate": 4.458689458689459e-05,
      "loss": 0.0223,
      "num_tokens": 1351754.0,
      "reward": 4.024516892433167,
      "reward_std": 2.152446281909943,
      "rewards/reward_function": 4.024516820907593,
      "step": 390
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 494.9333465576172,
      "epoch": 1.7094017094017095,
      "grad_norm": 0.4138166010379791,
      "kl": 0.12820503935217858,
      "learning_rate": 4.316239316239317e-05,
      "loss": -0.0496,
      "num_tokens": 1384870.0,
      "reward": 9.3906578540802,
      "reward_std": 2.297537350654602,
      "rewards/reward_function": 9.390657663345337,
      "step": 400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 761.1333557128906,
      "epoch": 1.7521367521367521,
      "grad_norm": 0.38381585478782654,
      "kl": 0.10142210572957992,
      "learning_rate": 4.1737891737891737e-05,
      "loss": 0.0004,
      "num_tokens": 1433958.0,
      "reward": 7.575464034080506,
      "reward_std": 2.367896056175232,
      "rewards/reward_function": 7.575464022159577,
      "step": 410
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 533.2500183105469,
      "epoch": 1.7948717948717947,
      "grad_norm": 0.25578761100769043,
      "kl": 0.15465135574340821,
      "learning_rate": 4.031339031339032e-05,
      "loss": 0.0144,
      "num_tokens": 1468839.0,
      "reward": 7.1474449634552,
      "reward_std": 2.076480484008789,
      "rewards/reward_function": 7.147444558143616,
      "step": 420
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 620.7833557128906,
      "epoch": 1.8376068376068377,
      "grad_norm": 0.77311772108078,
      "kl": 0.1036892857402563,
      "learning_rate": 3.888888888888889e-05,
      "loss": -0.0142,
      "num_tokens": 1509002.0,
      "reward": 5.1637330770492555,
      "reward_std": 2.2263385891914367,
      "rewards/reward_function": 5.163732886314392,
      "step": 430
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 414.7333442687988,
      "epoch": 1.8803418803418803,
      "grad_norm": 0.2505393624305725,
      "kl": 0.1861073050647974,
      "learning_rate": 3.746438746438747e-05,
      "loss": -0.0104,
      "num_tokens": 1537192.0,
      "reward": 7.412068939208984,
      "reward_std": 2.5153212428092955,
      "rewards/reward_function": 7.412068605422974,
      "step": 440
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 631.1166847229003,
      "epoch": 1.9230769230769231,
      "grad_norm": 0.1497740000486374,
      "kl": 0.13360862545669078,
      "learning_rate": 3.603988603988604e-05,
      "loss": -0.0125,
      "num_tokens": 1578569.0,
      "reward": 5.736767768859863,
      "reward_std": 2.0697311878204347,
      "rewards/reward_function": 5.736767530441284,
      "step": 450
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 552.3333518981933,
      "epoch": 1.965811965811966,
      "grad_norm": 0.24241755902767181,
      "kl": 0.11483766213059425,
      "learning_rate": 3.461538461538462e-05,
      "loss": -0.0406,
      "num_tokens": 1614949.0,
      "reward": 7.0892092227935795,
      "reward_std": 2.59124094247818,
      "rewards/reward_function": 7.089209020137787,
      "step": 460
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 612.933349609375,
      "epoch": 2.0085470085470085,
      "grad_norm": 0.18592579662799835,
      "kl": 0.1084304042160511,
      "learning_rate": 3.319088319088319e-05,
      "loss": -0.0312,
      "num_tokens": 1655031.0,
      "reward": 5.828713139891624,
      "reward_std": 2.09327831864357,
      "rewards/reward_function": 5.828712995350361,
      "step": 470
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 479.71668167114257,
      "epoch": 2.051282051282051,
      "grad_norm": 0.28115972876548767,
      "kl": 0.1521254086866975,
      "learning_rate": 3.176638176638177e-05,
      "loss": -0.0227,
      "num_tokens": 1686958.0,
      "reward": 3.6067369182594122,
      "reward_std": 2.0612903952598574,
      "rewards/reward_function": 3.6067366739269344,
      "step": 480
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 706.9000183105469,
      "epoch": 2.094017094017094,
      "grad_norm": 0.40815603733062744,
      "kl": 0.09023629724979401,
      "learning_rate": 3.034188034188034e-05,
      "loss": -0.0333,
      "num_tokens": 1732228.0,
      "reward": 8.146909832954407,
      "reward_std": 2.0977156162261963,
      "rewards/reward_function": 8.146909546852111,
      "step": 490
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 750.8833557128906,
      "epoch": 2.1367521367521367,
      "grad_norm": 0.9345102310180664,
      "kl": 0.1092647921293974,
      "learning_rate": 2.8917378917378917e-05,
      "loss": -0.0018,
      "num_tokens": 1780587.0,
      "reward": 6.364449739456177,
      "reward_std": 2.102370226383209,
      "rewards/reward_function": 6.364449632167816,
      "step": 500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 644.583358001709,
      "epoch": 2.1794871794871793,
      "grad_norm": 0.22266240417957306,
      "kl": 0.0947543952614069,
      "learning_rate": 2.7492877492877494e-05,
      "loss": -0.0332,
      "num_tokens": 1822586.0,
      "reward": 7.290940642356873,
      "reward_std": 2.075295794010162,
      "rewards/reward_function": 7.290940308570862,
      "step": 510
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 617.3166870117187,
      "epoch": 2.2222222222222223,
      "grad_norm": 0.26261353492736816,
      "kl": 0.1376684106886387,
      "learning_rate": 2.606837606837607e-05,
      "loss": -0.0418,
      "num_tokens": 1862823.0,
      "reward": 8.94625678062439,
      "reward_std": 2.00726802945137,
      "rewards/reward_function": 8.946256589889526,
      "step": 520
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 668.6000183105468,
      "epoch": 2.264957264957265,
      "grad_norm": 0.26877865195274353,
      "kl": 0.10333485826849938,
      "learning_rate": 2.4643874643874645e-05,
      "loss": 0.016,
      "num_tokens": 1905753.0,
      "reward": 7.090233588218689,
      "reward_std": 2.2102274179458616,
      "rewards/reward_function": 7.09023334980011,
      "step": 530
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 539.7333450317383,
      "epoch": 2.3076923076923075,
      "grad_norm": 0.17735101282596588,
      "kl": 0.09612916819751263,
      "learning_rate": 2.321937321937322e-05,
      "loss": -0.0373,
      "num_tokens": 1941431.0,
      "reward": 6.942164731025696,
      "reward_std": 2.261547350883484,
      "rewards/reward_function": 6.942164731025696,
      "step": 540
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 723.6166778564453,
      "epoch": 2.3504273504273505,
      "grad_norm": 0.21223032474517822,
      "kl": 0.07550707198679447,
      "learning_rate": 2.1794871794871795e-05,
      "loss": -0.0048,
      "num_tokens": 1988262.0,
      "reward": 6.751526057720184,
      "reward_std": 2.1452417016029357,
      "rewards/reward_function": 6.751525813341141,
      "step": 550
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 653.1500122070313,
      "epoch": 2.393162393162393,
      "grad_norm": 0.17820146679878235,
      "kl": 0.08245615884661675,
      "learning_rate": 2.037037037037037e-05,
      "loss": 0.007,
      "num_tokens": 2030811.0,
      "reward": 4.946912789344788,
      "reward_std": 2.218581938743591,
      "rewards/reward_function": 4.946912682056427,
      "step": 560
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 708.1666809082031,
      "epoch": 2.435897435897436,
      "grad_norm": 0.24161206185817719,
      "kl": 0.09174778610467911,
      "learning_rate": 1.8945868945868946e-05,
      "loss": -0.0261,
      "num_tokens": 2076649.0,
      "reward": 6.623177337646484,
      "reward_std": 2.7456999897956846,
      "rewards/reward_function": 6.623177027702331,
      "step": 570
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 705.9166839599609,
      "epoch": 2.4786324786324787,
      "grad_norm": 0.1828453093767166,
      "kl": 0.07772732079029084,
      "learning_rate": 1.752136752136752e-05,
      "loss": -0.0294,
      "num_tokens": 2122406.0,
      "reward": 7.905662250518799,
      "reward_std": 2.341515266895294,
      "rewards/reward_function": 7.9056620597839355,
      "step": 580
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 639.8000274658203,
      "epoch": 2.5213675213675213,
      "grad_norm": 0.14776869118213654,
      "kl": 0.11540216729044914,
      "learning_rate": 1.6096866096866097e-05,
      "loss": -0.0235,
      "num_tokens": 2163932.0,
      "reward": 6.144934403896332,
      "reward_std": 1.9559996247291564,
      "rewards/reward_function": 6.144933986663818,
      "step": 590
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 447.56668243408205,
      "epoch": 2.564102564102564,
      "grad_norm": 0.7991915941238403,
      "kl": 0.14225502200424672,
      "learning_rate": 1.4672364672364672e-05,
      "loss": -0.0103,
      "num_tokens": 2194110.0,
      "reward": 6.249829268455505,
      "reward_std": 1.8448994159698486,
      "rewards/reward_function": 6.2498291015625,
      "step": 600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 674.2833526611328,
      "epoch": 2.606837606837607,
      "grad_norm": 0.406729131937027,
      "kl": 0.08968410007655621,
      "learning_rate": 1.324786324786325e-05,
      "loss": -0.0049,
      "num_tokens": 2237801.0,
      "reward": 6.374316561222076,
      "reward_std": 1.7517470955848693,
      "rewards/reward_function": 6.374316537380219,
      "step": 610
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 460.0166763305664,
      "epoch": 2.6495726495726495,
      "grad_norm": 0.2793329954147339,
      "kl": 0.1312561396509409,
      "learning_rate": 1.1823361823361825e-05,
      "loss": -0.0294,
      "num_tokens": 2268354.0,
      "reward": 7.518070614337921,
      "reward_std": 2.5643215417861938,
      "rewards/reward_function": 7.518070566654205,
      "step": 620
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 599.0333526611328,
      "epoch": 2.6923076923076925,
      "grad_norm": 0.2505931854248047,
      "kl": 0.09553406685590744,
      "learning_rate": 1.03988603988604e-05,
      "loss": -0.0184,
      "num_tokens": 2307458.0,
      "reward": 7.650830316543579,
      "reward_std": 2.161614012718201,
      "rewards/reward_function": 7.650830030441284,
      "step": 630
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 479.4666835784912,
      "epoch": 2.735042735042735,
      "grad_norm": 0.27248629927635193,
      "kl": 0.1575175441801548,
      "learning_rate": 8.974358974358976e-06,
      "loss": -0.0378,
      "num_tokens": 2339196.0,
      "reward": 8.164103621244431,
      "reward_std": 2.156542217731476,
      "rewards/reward_function": 8.164103049039841,
      "step": 640
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 476.6000156402588,
      "epoch": 2.7777777777777777,
      "grad_norm": 0.8577276468276978,
      "kl": 0.18197493851184846,
      "learning_rate": 7.54985754985755e-06,
      "loss": -0.0162,
      "num_tokens": 2371650.0,
      "reward": 9.458398675918579,
      "reward_std": 1.9103310704231262,
      "rewards/reward_function": 9.458398389816285,
      "step": 650
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 758.2000213623047,
      "epoch": 2.8205128205128203,
      "grad_norm": 0.3212578594684601,
      "kl": 0.08359390124678612,
      "learning_rate": 6.1253561253561255e-06,
      "loss": 0.0015,
      "num_tokens": 2420274.0,
      "reward": 7.553524050116539,
      "reward_std": 1.6860621690750122,
      "rewards/reward_function": 7.553523740172386,
      "step": 660
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 625.9666748046875,
      "epoch": 2.8632478632478633,
      "grad_norm": 0.18765850365161896,
      "kl": 0.10408403836190701,
      "learning_rate": 4.700854700854701e-06,
      "loss": -0.0595,
      "num_tokens": 2460820.0,
      "reward": 5.822998178005219,
      "reward_std": 2.2071512162685396,
      "rewards/reward_function": 5.822998130321503,
      "step": 670
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 590.1833488464356,
      "epoch": 2.905982905982906,
      "grad_norm": 0.19056835770606995,
      "kl": 0.14810367673635483,
      "learning_rate": 3.2763532763532763e-06,
      "loss": 0.0101,
      "num_tokens": 2499681.0,
      "reward": 6.133286702632904,
      "reward_std": 1.5098123759031297,
      "rewards/reward_function": 6.133286654949188,
      "step": 680
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 521.3000122070313,
      "epoch": 2.948717948717949,
      "grad_norm": 0.24630853533744812,
      "kl": 0.16457773484289645,
      "learning_rate": 1.8518518518518519e-06,
      "loss": -0.0118,
      "num_tokens": 2533809.0,
      "reward": 4.986087208986282,
      "reward_std": 1.9773054122924805,
      "rewards/reward_function": 4.986086970567703,
      "step": 690
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 592.8500122070312,
      "epoch": 2.9914529914529915,
      "grad_norm": 0.3801647126674652,
      "kl": 0.09378828406333924,
      "learning_rate": 4.273504273504274e-07,
      "loss": -0.0417,
      "num_tokens": 2572590.0,
      "reward": 6.725127363204956,
      "reward_std": 2.407969039678574,
      "rewards/reward_function": 6.725127053260803,
      "step": 700
    }
  ],
  "logging_steps": 10,
  "max_steps": 702,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}