{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.7094017094017095,
  "eval_steps": 500,
  "global_step": 400,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 503.30001945495604,
      "epoch": 0.042735042735042736,
      "grad_norm": 0.14690014719963074,
      "kl": 0.0005318611507391324,
      "learning_rate": 9.871794871794872e-05,
      "loss": -0.0402,
      "num_tokens": 33288.0,
      "reward": -1.1772359251976012,
      "reward_std": 2.312183880805969,
      "rewards/reward_function": -1.1772359311580658,
      "step": 10
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 509.85001373291016,
      "epoch": 0.08547008547008547,
      "grad_norm": 0.16410136222839355,
      "kl": 0.001959062390960753,
      "learning_rate": 9.72934472934473e-05,
      "loss": -0.008,
      "num_tokens": 67317.0,
      "reward": -1.455158567428589,
      "reward_std": 2.39455783367157,
      "rewards/reward_function": -1.4551584720611572,
      "step": 20
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 382.58334197998045,
      "epoch": 0.1282051282051282,
      "grad_norm": 0.5200854539871216,
      "kl": 0.003280931804329157,
      "learning_rate": 9.586894586894587e-05,
      "loss": 0.0011,
      "num_tokens": 93788.0,
      "reward": 2.4295308887958527,
      "reward_std": 1.948470675945282,
      "rewards/reward_function": 2.4295308709144594,
      "step": 30
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 466.25001068115233,
      "epoch": 0.17094017094017094,
      "grad_norm": 0.2215566486120224,
      "kl": 0.005862738820724189,
      "learning_rate": 9.444444444444444e-05,
      "loss": -0.0111,
      "num_tokens": 125045.0,
      "reward": 1.2611542105674745,
      "reward_std": 2.462652099132538,
      "rewards/reward_function": 1.2611541628837586,
      "step": 40
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 518.6166809082031,
      "epoch": 0.21367521367521367,
      "grad_norm": 0.15176545083522797,
      "kl": 0.008088351227343082,
      "learning_rate": 9.301994301994303e-05,
      "loss": -0.0085,
      "num_tokens": 159318.0,
      "reward": -1.0398478865623475,
      "reward_std": 1.9632926762104035,
      "rewards/reward_function": -1.039847767353058,
      "step": 50
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 436.50001125335694,
      "epoch": 0.2564102564102564,
      "grad_norm": 0.23773637413978577,
      "kl": 0.007962367637082935,
      "learning_rate": 9.15954415954416e-05,
      "loss": 0.0155,
      "num_tokens": 188622.0,
      "reward": -0.9993367567658424,
      "reward_std": 2.5938119292259216,
      "rewards/reward_function": -0.9993367329239845,
      "step": 60
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 465.0166717529297,
      "epoch": 0.29914529914529914,
      "grad_norm": 0.22686824202537537,
      "kl": 0.01036710049957037,
      "learning_rate": 9.017094017094018e-05,
      "loss": 0.0232,
      "num_tokens": 219649.0,
      "reward": -1.0854085847735404,
      "reward_std": 2.3031186699867248,
      "rewards/reward_function": -1.0854085013270378,
      "step": 70
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 594.3666900634765,
      "epoch": 0.3418803418803419,
      "grad_norm": 0.2330074906349182,
      "kl": 0.011198346642777324,
      "learning_rate": 8.874643874643875e-05,
      "loss": 0.0085,
      "num_tokens": 258521.0,
      "reward": 0.44582319259643555,
      "reward_std": 2.180183058977127,
      "rewards/reward_function": 0.44582313299179077,
      "step": 80
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 363.2166831970215,
      "epoch": 0.38461538461538464,
      "grad_norm": 1.2950036525726318,
      "kl": 0.01624974999576807,
      "learning_rate": 8.732193732193732e-05,
      "loss": -0.0279,
      "num_tokens": 283896.0,
      "reward": -0.22942656874656678,
      "reward_std": 2.698866534233093,
      "rewards/reward_function": -0.2294266164302826,
      "step": 90
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 366.3166763305664,
      "epoch": 0.42735042735042733,
      "grad_norm": 0.44916248321533203,
      "kl": 0.0371546683833003,
      "learning_rate": 8.58974358974359e-05,
      "loss": -0.034,
      "num_tokens": 309127.0,
      "reward": 1.030603051185608,
      "reward_std": 2.1768004059791566,
      "rewards/reward_function": 1.03060302734375,
      "step": 100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 627.7833419799805,
      "epoch": 0.4700854700854701,
      "grad_norm": 0.24189692735671997,
      "kl": 0.02031996361911297,
      "learning_rate": 8.447293447293447e-05,
      "loss": 0.0057,
      "num_tokens": 350124.0,
      "reward": 1.7054275810718535,
      "reward_std": 2.761507248878479,
      "rewards/reward_function": 1.7054275453090668,
      "step": 110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 466.7833419799805,
      "epoch": 0.5128205128205128,
      "grad_norm": 0.3966105282306671,
      "kl": 0.03020444568246603,
      "learning_rate": 8.304843304843305e-05,
      "loss": 0.0158,
      "num_tokens": 381323.0,
      "reward": 0.4135805606842041,
      "reward_std": 2.793270480632782,
      "rewards/reward_function": 0.41358056366443635,
      "step": 120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 418.7166778564453,
      "epoch": 0.5555555555555556,
      "grad_norm": 0.8130244016647339,
      "kl": 0.050436797365546224,
      "learning_rate": 8.162393162393163e-05,
      "loss": 0.0109,
      "num_tokens": 409632.0,
      "reward": 3.6511381447315214,
      "reward_std": 2.057768177986145,
      "rewards/reward_function": 3.651138073205948,
      "step": 130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.18334617614744,
      "epoch": 0.5982905982905983,
      "grad_norm": 0.237253338098526,
      "kl": 0.08577278926968575,
      "learning_rate": 8.01994301994302e-05,
      "loss": 0.005,
      "num_tokens": 436613.0,
      "reward": 3.9179525792598726,
      "reward_std": 2.5252403259277343,
      "rewards/reward_function": 3.9179525285959245,
      "step": 140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 534.9666793823242,
      "epoch": 0.6410256410256411,
      "grad_norm": 0.23871329426765442,
      "kl": 0.0823534980416298,
      "learning_rate": 7.877492877492878e-05,
      "loss": 0.0435,
      "num_tokens": 471891.0,
      "reward": 4.100382626056671,
      "reward_std": 2.5033695101737976,
      "rewards/reward_function": 4.100382459163666,
      "step": 150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 332.50001373291013,
      "epoch": 0.6837606837606838,
      "grad_norm": 0.8124951720237732,
      "kl": 0.15444251857697963,
      "learning_rate": 7.735042735042735e-05,
      "loss": 0.0189,
      "num_tokens": 494961.0,
      "reward": 5.122303450107575,
      "reward_std": 2.0907008171081545,
      "rewards/reward_function": 5.122303307056427,
      "step": 160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.25001373291013,
      "epoch": 0.7264957264957265,
      "grad_norm": 0.7909404635429382,
      "kl": 0.13212636522948742,
      "learning_rate": 7.592592592592593e-05,
      "loss": -0.0367,
      "num_tokens": 522258.0,
      "reward": 5.544608736038208,
      "reward_std": 1.7265446126461028,
      "rewards/reward_function": 5.544608497619629,
      "step": 170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 595.9833587646484,
      "epoch": 0.7692307692307693,
      "grad_norm": 0.15963077545166016,
      "kl": 0.09406365267932415,
      "learning_rate": 7.450142450142451e-05,
      "loss": -0.0366,
      "num_tokens": 561197.0,
      "reward": 7.151281929016113,
      "reward_std": 2.8219340562820436,
      "rewards/reward_function": 7.15128173828125,
      "step": 180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 566.9666809082031,
      "epoch": 0.811965811965812,
      "grad_norm": 0.18807636201381683,
      "kl": 0.06498432569205762,
      "learning_rate": 7.307692307692307e-05,
      "loss": -0.0543,
      "num_tokens": 598575.0,
      "reward": 1.7874940395355225,
      "reward_std": 2.3280829310417177,
      "rewards/reward_function": 1.7874938309192658,
      "step": 190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 831.9500183105469,
      "epoch": 0.8547008547008547,
      "grad_norm": 0.19820700585842133,
      "kl": 0.03369698449969292,
      "learning_rate": 7.165242165242165e-05,
      "loss": -0.0132,
      "num_tokens": 651618.0,
      "reward": 2.0692047476768494,
      "reward_std": 2.6171678781509398,
      "rewards/reward_function": 2.06920468211174,
      "step": 200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 390.78334617614746,
      "epoch": 0.8974358974358975,
      "grad_norm": 0.1917090117931366,
      "kl": 0.10797237679362297,
      "learning_rate": 7.022792022792024e-05,
      "loss": -0.011,
      "num_tokens": 677981.0,
      "reward": 5.218200743198395,
      "reward_std": 2.8841384768486025,
      "rewards/reward_function": 5.218200659751892,
      "step": 210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 485.500016784668,
      "epoch": 0.9401709401709402,
      "grad_norm": 0.18474219739437103,
      "kl": 0.11786541007459164,
      "learning_rate": 6.880341880341881e-05,
      "loss": -0.0031,
      "num_tokens": 710033.0,
      "reward": 5.224206709861756,
      "reward_std": 2.2454194366931914,
      "rewards/reward_function": 5.224206617474556,
      "step": 220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 458.06668243408205,
      "epoch": 0.9829059829059829,
      "grad_norm": 0.19587470591068268,
      "kl": 0.09677453897893429,
      "learning_rate": 6.737891737891738e-05,
      "loss": -0.0014,
      "num_tokens": 741117.0,
      "reward": 4.320850740373134,
      "reward_std": 2.3484508872032164,
      "rewards/reward_function": 4.320850642025471,
      "step": 230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 605.0666931152343,
      "epoch": 1.0256410256410255,
      "grad_norm": 0.26696279644966125,
      "kl": 0.10099472776055336,
      "learning_rate": 6.595441595441596e-05,
      "loss": -0.025,
      "num_tokens": 780847.0,
      "reward": 7.07135591506958,
      "reward_std": 2.179239821434021,
      "rewards/reward_function": 7.071355724334717,
      "step": 240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 570.1833526611329,
      "epoch": 1.0683760683760684,
      "grad_norm": 0.41289836168289185,
      "kl": 0.10571693740785122,
      "learning_rate": 6.452991452991453e-05,
      "loss": 0.0201,
      "num_tokens": 818394.0,
      "reward": 4.596634943783283,
      "reward_std": 2.0250564932823183,
      "rewards/reward_function": 4.5966349326074125,
      "step": 250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 638.4166839599609,
      "epoch": 1.1111111111111112,
      "grad_norm": 0.23291794955730438,
      "kl": 0.06395059525966644,
      "learning_rate": 6.310541310541312e-05,
      "loss": 0.0023,
      "num_tokens": 859765.0,
      "reward": 4.951379495859146,
      "reward_std": 2.139006716012955,
      "rewards/reward_function": 4.95137942135334,
      "step": 260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 425.76667404174805,
      "epoch": 1.1538461538461537,
      "grad_norm": 0.2002115249633789,
      "kl": 0.14280213601887226,
      "learning_rate": 6.168091168091168e-05,
      "loss": 0.0113,
      "num_tokens": 888389.0,
      "reward": 6.41916925907135,
      "reward_std": 2.205563151836395,
      "rewards/reward_function": 6.419169163703918,
      "step": 270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 523.6166763305664,
      "epoch": 1.1965811965811965,
      "grad_norm": 0.18461842834949493,
      "kl": 0.1309021409600973,
      "learning_rate": 6.025641025641026e-05,
      "loss": -0.0021,
      "num_tokens": 923046.0,
      "reward": 5.595602548122406,
      "reward_std": 2.1117491006851195,
      "rewards/reward_function": 5.59560244679451,
      "step": 280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 483.0666799545288,
      "epoch": 1.2393162393162394,
      "grad_norm": 3.1094970703125,
      "kl": 0.1820569921284914,
      "learning_rate": 5.883190883190883e-05,
      "loss": 0.0173,
      "num_tokens": 955420.0,
      "reward": 5.614298677444458,
      "reward_std": 2.518768322467804,
      "rewards/reward_function": 5.614298534393311,
      "step": 290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 617.6333404541016,
      "epoch": 1.282051282051282,
      "grad_norm": 0.3713136613368988,
      "kl": 0.11033637076616287,
      "learning_rate": 5.740740740740741e-05,
      "loss": 0.0062,
      "num_tokens": 995922.0,
      "reward": 4.5290528535842896,
      "reward_std": 2.494095575809479,
      "rewards/reward_function": 4.529052710533142,
      "step": 300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 442.0166839599609,
      "epoch": 1.3247863247863247,
      "grad_norm": 0.21180328726768494,
      "kl": 0.1499824345111847,
      "learning_rate": 5.5982905982905986e-05,
      "loss": -0.0013,
      "num_tokens": 1025455.0,
      "reward": 8.561688470840455,
      "reward_std": 1.9707270503044128,
      "rewards/reward_function": 8.56168818473816,
      "step": 310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 580.050008392334,
      "epoch": 1.3675213675213675,
      "grad_norm": 0.20131155848503113,
      "kl": 0.11668501645326615,
      "learning_rate": 5.4558404558404567e-05,
      "loss": -0.0146,
      "num_tokens": 1063354.0,
      "reward": 5.419061434268952,
      "reward_std": 2.1403504729270937,
      "rewards/reward_function": 5.419061243534088,
      "step": 320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 652.3166870117187,
      "epoch": 1.4102564102564101,
      "grad_norm": 0.2526203691959381,
      "kl": 0.09888706281781197,
      "learning_rate": 5.313390313390314e-05,
      "loss": 0.0084,
      "num_tokens": 1105397.0,
      "reward": 7.351496028900146,
      "reward_std": 1.773663091659546,
      "rewards/reward_function": 7.351495909690857,
      "step": 330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 718.36669921875,
      "epoch": 1.452991452991453,
      "grad_norm": 0.1585923731327057,
      "kl": 0.10820744708180427,
      "learning_rate": 5.1709401709401714e-05,
      "loss": 0.0009,
      "num_tokens": 1151751.0,
      "reward": 6.595072710514069,
      "reward_std": 1.881132996082306,
      "rewards/reward_function": 6.59507247209549,
      "step": 340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 623.8333557128906,
      "epoch": 1.4957264957264957,
      "grad_norm": 0.18694844841957092,
      "kl": 0.11230501309037208,
      "learning_rate": 5.028490028490028e-05,
      "loss": -0.0362,
      "num_tokens": 1192547.0,
      "reward": 6.264591613411904,
      "reward_std": 1.9230000615119933,
      "rewards/reward_function": 6.26459142267704,
      "step": 350
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 704.0500183105469,
      "epoch": 1.5384615384615383,
      "grad_norm": 0.20967163145542145,
      "kl": 0.08291292320936919,
      "learning_rate": 4.886039886039887e-05,
      "loss": -0.0191,
      "num_tokens": 1237892.0,
      "reward": 5.098879025876522,
      "reward_std": 2.248683771491051,
      "rewards/reward_function": 5.09887896925211,
      "step": 360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 661.6000152587891,
      "epoch": 1.5811965811965814,
      "grad_norm": 0.19468806684017181,
      "kl": 0.10559967942535878,
      "learning_rate": 4.7435897435897435e-05,
      "loss": -0.0077,
      "num_tokens": 1280828.0,
      "reward": 6.5063467741012575,
      "reward_std": 1.6363932967185975,
      "rewards/reward_function": 6.506346654891968,
      "step": 370
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 426.3000122070313,
      "epoch": 1.623931623931624,
      "grad_norm": 0.40712612867355347,
      "kl": 0.14229083359241484,
      "learning_rate": 4.6011396011396016e-05,
      "loss": -0.0067,
      "num_tokens": 1309910.0,
      "reward": 8.865666103363036,
      "reward_std": 1.6894314289093018,
      "rewards/reward_function": 8.865665578842163,
      "step": 380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 645.4000122070313,
      "epoch": 1.6666666666666665,
      "grad_norm": 0.1824493706226349,
      "kl": 0.14776081275194883,
      "learning_rate": 4.458689458689459e-05,
      "loss": 0.0223,
      "num_tokens": 1351754.0,
      "reward": 4.024516892433167,
      "reward_std": 2.152446281909943,
      "rewards/reward_function": 4.024516820907593,
      "step": 390
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 494.9333465576172,
      "epoch": 1.7094017094017095,
      "grad_norm": 0.4138166010379791,
      "kl": 0.12820503935217858,
      "learning_rate": 4.316239316239317e-05,
      "loss": -0.0496,
      "num_tokens": 1384870.0,
      "reward": 9.3906578540802,
      "reward_std": 2.297537350654602,
      "rewards/reward_function": 9.390657663345337,
      "step": 400
    }
  ],
  "logging_steps": 10,
  "max_steps": 702,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}