{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 1904,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.005252100840336135,
      "grad_norm": 69.5775218105222,
      "kl": 0.15183432400226593,
      "learning_rate": 4.999659696812289e-07,
      "logps/chosen": -298.5141715116279,
      "logps/rejected": -259.0493032094595,
      "loss": 0.4994,
      "rewards/chosen": -0.04585062625796296,
      "rewards/margins": 0.01403944421908452,
      "rewards/rejected": -0.05989007047704748,
      "step": 10
    },
    {
      "epoch": 0.01050420168067227,
      "grad_norm": 55.49240989732535,
      "kl": 2.105862855911255,
      "learning_rate": 4.998638879894165e-07,
      "logps/chosen": -273.6553505345395,
      "logps/rejected": -227.15778459821428,
      "loss": 0.4776,
      "rewards/chosen": 0.49514409115439967,
      "rewards/margins": 0.21992081329039764,
      "rewards/rejected": 0.275223277864002,
      "step": 20
    },
    {
      "epoch": 0.015756302521008403,
      "grad_norm": 51.28980044595933,
      "kl": 0.28644976019859314,
      "learning_rate": 4.996937827155428e-07,
      "logps/chosen": -311.9908854166667,
      "logps/rejected": -247.46916946308724,
      "loss": 0.4486,
      "rewards/chosen": 0.30754662675467154,
      "rewards/margins": 0.5134327379181245,
      "rewards/rejected": -0.20588611116345296,
      "step": 30
    },
    {
      "epoch": 0.02100840336134454,
      "grad_norm": 53.39172095489608,
      "kl": 0.0,
      "learning_rate": 4.994557001695013e-07,
      "logps/chosen": -298.2471417682927,
      "logps/rejected": -238.9893078926282,
      "loss": 0.4172,
      "rewards/chosen": 0.01273148815806319,
      "rewards/margins": 0.8681685468269335,
      "rewards/rejected": -0.8554370586688702,
      "step": 40
    },
    {
      "epoch": 0.026260504201680673,
      "grad_norm": 48.3236105129346,
      "kl": 0.0,
      "learning_rate": 4.991497051674917e-07,
      "logps/chosen": -289.8418828616352,
      "logps/rejected": -264.62730493012424,
      "loss": 0.3922,
      "rewards/chosen": 0.3131725623172784,
      "rewards/margins": 1.3170881950509257,
      "rewards/rejected": -1.0039156327336471,
      "step": 50
    },
    {
      "epoch": 0.031512605042016806,
      "grad_norm": 51.661072159206874,
      "kl": 0.0,
      "learning_rate": 4.987758810143735e-07,
      "logps/chosen": -331.279736328125,
      "logps/rejected": -255.901513671875,
      "loss": 0.3954,
      "rewards/chosen": 0.22145886421203614,
      "rewards/margins": 1.2238109111785889,
      "rewards/rejected": -1.0023520469665528,
      "step": 60
    },
    {
      "epoch": 0.03676470588235294,
      "grad_norm": 48.98498102999305,
      "kl": 0.0,
      "learning_rate": 4.983343294809874e-07,
      "logps/chosen": -309.5669285973837,
      "logps/rejected": -255.4453125,
      "loss": 0.3732,
      "rewards/chosen": 0.5395132552745731,
      "rewards/margins": 1.6830449514910533,
      "rewards/rejected": -1.14353169621648,
      "step": 70
    },
    {
      "epoch": 0.04201680672268908,
      "grad_norm": 49.52520073364695,
      "kl": 0.0,
      "learning_rate": 4.978251707764491e-07,
      "logps/chosen": -298.83478282232704,
      "logps/rejected": -262.3364712732919,
      "loss": 0.3542,
      "rewards/chosen": 0.41468177651459315,
      "rewards/margins": 2.0195703134298686,
      "rewards/rejected": -1.6048885369152757,
      "step": 80
    },
    {
      "epoch": 0.04726890756302521,
      "grad_norm": 45.84438443510472,
      "kl": 0.0,
      "learning_rate": 4.972485435154228e-07,
      "logps/chosen": -284.00230493012424,
      "logps/rejected": -292.5305866745283,
      "loss": 0.3424,
      "rewards/chosen": 0.20849199472747235,
      "rewards/margins": 2.4833206356472837,
      "rewards/rejected": -2.2748286409198113,
      "step": 90
    },
    {
      "epoch": 0.052521008403361345,
      "grad_norm": 51.90503609381182,
      "kl": 0.0,
      "learning_rate": 4.966046046803842e-07,
      "logps/chosen": -305.3333748009554,
      "logps/rejected": -265.70988305214723,
      "loss": 0.3825,
      "rewards/chosen": 0.518976345183743,
      "rewards/margins": 2.160451905658724,
      "rewards/rejected": -1.6414755604749809,
      "step": 100
    },
    {
      "epoch": 0.05777310924369748,
      "grad_norm": 46.615982508495186,
      "kl": 0.0,
      "learning_rate": 4.958935295788841e-07,
      "logps/chosen": -346.7928911423841,
      "logps/rejected": -282.3527181952663,
      "loss": 0.3829,
      "rewards/chosen": 0.18162511358197952,
      "rewards/margins": 1.8821555777091985,
      "rewards/rejected": -1.700530464127219,
      "step": 110
    },
    {
      "epoch": 0.06302521008403361,
      "grad_norm": 43.56170699177104,
      "kl": 0.0,
      "learning_rate": 4.951155117958216e-07,
      "logps/chosen": -299.7170380015432,
      "logps/rejected": -281.8764092167722,
      "loss": 0.3787,
      "rewards/chosen": -0.607381797131197,
      "rewards/margins": 2.2764308295746973,
      "rewards/rejected": -2.883812626705894,
      "step": 120
    },
    {
      "epoch": 0.06827731092436974,
      "grad_norm": 52.35577489618577,
      "kl": 0.0,
      "learning_rate": 4.942707631407419e-07,
      "logps/chosen": -335.1082099780702,
      "logps/rejected": -281.61443477348996,
      "loss": 0.3757,
      "rewards/chosen": -0.22571238021404422,
      "rewards/margins": 2.2212163225308172,
      "rewards/rejected": -2.4469287027448616,
      "step": 130
    },
    {
      "epoch": 0.07352941176470588,
      "grad_norm": 28.41591231440036,
      "kl": 0.0,
      "learning_rate": 4.933595135901732e-07,
      "logps/chosen": -318.5261247783688,
      "logps/rejected": -269.2608676675978,
      "loss": 0.3369,
      "rewards/chosen": 0.26553474588597076,
      "rewards/margins": 2.99381714157382,
      "rewards/rejected": -2.7282823956878492,
      "step": 140
    },
    {
      "epoch": 0.07878151260504201,
      "grad_norm": 77.73773489117062,
      "kl": 0.0,
      "learning_rate": 4.923820112250169e-07,
      "logps/chosen": -301.63650901845637,
      "logps/rejected": -266.3384959795322,
      "loss": 0.3375,
      "rewards/chosen": 0.025881773673447985,
      "rewards/margins": 3.2468575937575124,
      "rewards/rejected": -3.2209758200840644,
      "step": 150
    },
    {
      "epoch": 0.08403361344537816,
      "grad_norm": 37.575572316994865,
      "kl": 0.0,
      "learning_rate": 4.913385221630096e-07,
      "logps/chosen": -250.38438482704402,
      "logps/rejected": -265.8534064440994,
      "loss": 0.3758,
      "rewards/chosen": 0.7617762343688581,
      "rewards/margins": 1.6022024720556787,
      "rewards/rejected": -0.8404262376868207,
      "step": 160
    },
    {
      "epoch": 0.08928571428571429,
      "grad_norm": 40.501974364358205,
      "kl": 0.0,
      "learning_rate": 4.902293304862749e-07,
      "logps/chosen": -270.815234375,
      "logps/rejected": -255.6860107421875,
      "loss": 0.3672,
      "rewards/chosen": 0.9947892189025879,
      "rewards/margins": 1.629128646850586,
      "rewards/rejected": -0.634339427947998,
      "step": 170
    },
    {
      "epoch": 0.09453781512605042,
      "grad_norm": 36.96138019330735,
      "kl": 0.0,
      "learning_rate": 4.890547381639833e-07,
      "logps/chosen": -299.6106373856707,
      "logps/rejected": -230.88887219551282,
      "loss": 0.3625,
      "rewards/chosen": 0.3851209733544326,
      "rewards/margins": 2.033356000961103,
      "rewards/rejected": -1.6482350276066706,
      "step": 180
    },
    {
      "epoch": 0.09978991596638656,
      "grad_norm": 44.46735174580667,
      "kl": 0.0,
      "learning_rate": 4.878150649701439e-07,
      "logps/chosen": -344.7996875,
      "logps/rejected": -265.18083639705884,
      "loss": 0.3185,
      "rewards/chosen": 0.36289586385091144,
      "rewards/margins": 2.9504150719736133,
      "rewards/rejected": -2.587519208122702,
      "step": 190
    },
    {
      "epoch": 0.10504201680672269,
      "grad_norm": 52.28178000802996,
      "kl": 0.0,
      "learning_rate": 4.865106483965486e-07,
      "logps/chosen": -292.6435797275641,
      "logps/rejected": -240.90084317835365,
      "loss": 0.3407,
      "rewards/chosen": -0.03864570764394907,
      "rewards/margins": 2.6535366936875104,
      "rewards/rejected": -2.6921824013314595,
      "step": 200
    },
    {
      "epoch": 0.11029411764705882,
      "grad_norm": 42.51233566429091,
      "kl": 0.0,
      "learning_rate": 4.851418435608919e-07,
      "logps/chosen": -300.3898746468927,
      "logps/rejected": -269.88991477272725,
      "loss": 0.3554,
      "rewards/chosen": 0.8636472023139565,
      "rewards/margins": 2.70990092417803,
      "rewards/rejected": -1.8462537218640733,
      "step": 210
    },
    {
      "epoch": 0.11554621848739496,
      "grad_norm": 45.269264573206264,
      "kl": 0.0,
      "learning_rate": 4.837090231100927e-07,
      "logps/chosen": -315.7438054733728,
      "logps/rejected": -242.41240687086093,
      "loss": 0.327,
      "rewards/chosen": 1.163497202495146,
      "rewards/margins": 2.8938327665084946,
      "rewards/rejected": -1.7303355640133484,
      "step": 220
    },
    {
      "epoch": 0.1207983193277311,
      "grad_norm": 46.30035764458271,
      "kl": 0.0,
      "learning_rate": 4.822125771188448e-07,
      "logps/chosen": -285.65898944805195,
      "logps/rejected": -271.8729762801205,
      "loss": 0.3828,
      "rewards/chosen": 0.5068141392299107,
      "rewards/margins": 1.911893440811236,
      "rewards/rejected": -1.4050793015813252,
      "step": 230
    },
    {
      "epoch": 0.12605042016806722,
      "grad_norm": 44.08666008078037,
      "kl": 0.0,
      "learning_rate": 4.806529129834207e-07,
      "logps/chosen": -280.8894210188356,
      "logps/rejected": -270.21331716954023,
      "loss": 0.329,
      "rewards/chosen": 0.5693619087950824,
      "rewards/margins": 2.748881386727015,
      "rewards/rejected": -2.1795194779319327,
      "step": 240
    },
    {
      "epoch": 0.13130252100840337,
      "grad_norm": 38.010572045662606,
      "kl": 0.0,
      "learning_rate": 4.790304553107622e-07,
      "logps/chosen": -304.2262851331361,
      "logps/rejected": -254.57090749172184,
      "loss": 0.3588,
      "rewards/chosen": 0.49599551590236685,
      "rewards/margins": 2.326870937893778,
      "rewards/rejected": -1.8308754219914114,
      "step": 250
    },
    {
      "epoch": 0.13655462184873948,
      "grad_norm": 44.14238619732289,
      "kl": 0.0,
      "learning_rate": 4.773456458028837e-07,
      "logps/chosen": -278.3834842754777,
      "logps/rejected": -268.8938362730061,
      "loss": 0.3378,
      "rewards/chosen": 0.5815278071506768,
      "rewards/margins": 2.6120607376769422,
      "rewards/rejected": -2.0305329305262654,
      "step": 260
    },
    {
      "epoch": 0.14180672268907563,
      "grad_norm": 39.61832047489729,
      "kl": 1.3655751943588257,
      "learning_rate": 4.755989431366221e-07,
      "logps/chosen": -301.3936798878205,
      "logps/rejected": -299.8501572027439,
      "loss": 0.3287,
      "rewards/chosen": 0.6345734229454627,
      "rewards/margins": 2.3345022648852494,
      "rewards/rejected": -1.6999288419397867,
      "step": 270
    },
    {
      "epoch": 0.14705882352941177,
      "grad_norm": 43.29173967453139,
      "kl": 0.0,
      "learning_rate": 4.737908228387656e-07,
      "logps/chosen": -294.7026832217262,
      "logps/rejected": -264.49424342105266,
      "loss": 0.3553,
      "rewards/chosen": 0.5891142799740746,
      "rewards/margins": 2.708873086704646,
      "rewards/rejected": -2.1197588067305717,
      "step": 280
    },
    {
      "epoch": 0.15231092436974789,
      "grad_norm": 49.48779153565427,
      "kl": 0.0,
      "learning_rate": 4.7192177715659516e-07,
      "logps/chosen": -290.1478293413174,
      "logps/rejected": -243.8566176470588,
      "loss": 0.3763,
      "rewards/chosen": 0.392075178865901,
      "rewards/margins": 2.094171158045333,
      "rewards/rejected": -1.7020959791794321,
      "step": 290
    },
    {
      "epoch": 0.15756302521008403,
      "grad_norm": 44.86612305681867,
      "kl": 0.0,
      "learning_rate": 4.699923149238736e-07,
      "logps/chosen": -283.2191996402878,
      "logps/rejected": -272.7527192679558,
      "loss": 0.3369,
      "rewards/chosen": 0.6118597675570481,
      "rewards/margins": 2.6730571222057016,
      "rewards/rejected": -2.0611973546486535,
      "step": 300
    },
    {
      "epoch": 0.16281512605042017,
      "grad_norm": 43.00617930560404,
      "kl": 0.0,
      "learning_rate": 4.680029614223198e-07,
      "logps/chosen": -272.5205015923567,
      "logps/rejected": -266.77614551380367,
      "loss": 0.3531,
      "rewards/chosen": 0.13305815921467581,
      "rewards/margins": 2.627990556226275,
      "rewards/rejected": -2.494932397011599,
      "step": 310
    },
    {
      "epoch": 0.16806722689075632,
      "grad_norm": 39.410413100307075,
      "kl": 0.0,
      "learning_rate": 4.65954258238604e-07,
      "logps/chosen": -288.9263426272455,
      "logps/rejected": -254.3640216503268,
      "loss": 0.3736,
      "rewards/chosen": 0.09242550056137724,
      "rewards/margins": 2.2175613337032276,
      "rewards/rejected": -2.1251358331418504,
      "step": 320
    },
    {
      "epoch": 0.17331932773109243,
      "grad_norm": 41.100451094010424,
      "kl": 0.0,
      "learning_rate": 4.638467631169056e-07,
      "logps/chosen": -326.0165550595238,
      "logps/rejected": -306.81566097861844,
      "loss": 0.318,
      "rewards/chosen": 0.5781731832595098,
      "rewards/margins": 3.290003924740287,
      "rewards/rejected": -2.711830741480777,
      "step": 330
    },
    {
      "epoch": 0.17857142857142858,
      "grad_norm": 59.056271025186184,
      "kl": 0.0,
      "learning_rate": 4.6168104980707103e-07,
      "logps/chosen": -285.7653391768293,
      "logps/rejected": -274.3047876602564,
      "loss": 0.347,
      "rewards/chosen": -0.003299643353718083,
      "rewards/margins": 3.289772396910705,
      "rewards/rejected": -3.293072040264423,
      "step": 340
    },
    {
      "epoch": 0.18382352941176472,
      "grad_norm": 40.5420682235125,
      "kl": 0.0,
      "learning_rate": 4.594577079084145e-07,
      "logps/chosen": -290.82744140625,
      "logps/rejected": -292.7601318359375,
      "loss": 0.3356,
      "rewards/chosen": 0.4554912567138672,
      "rewards/margins": 3.3385635375976563,
      "rewards/rejected": -2.883072280883789,
      "step": 350
    },
    {
      "epoch": 0.18907563025210083,
      "grad_norm": 73.25856842000852,
      "kl": 0.0,
      "learning_rate": 4.5717734270920466e-07,
      "logps/chosen": -281.4008819018405,
      "logps/rejected": -228.66883957006368,
      "loss": 0.3378,
      "rewards/chosen": 0.49251336407807705,
      "rewards/margins": 2.78153659942342,
      "rewards/rejected": -2.2890232353453426,
      "step": 360
    },
    {
      "epoch": 0.19432773109243698,
      "grad_norm": 36.80227868403581,
      "kl": 0.0,
      "learning_rate": 4.548405750218785e-07,
      "logps/chosen": -288.27572185672517,
      "logps/rejected": -269.6975146812081,
      "loss": 0.3552,
      "rewards/chosen": -0.02239043949640285,
      "rewards/margins": 3.2017408950516257,
      "rewards/rejected": -3.2241313345480287,
      "step": 370
    },
    {
      "epoch": 0.19957983193277312,
      "grad_norm": 52.176625363111675,
      "kl": 0.0,
      "learning_rate": 4.5244804101403025e-07,
      "logps/chosen": -262.20579637096773,
      "logps/rejected": -253.31008522727274,
      "loss": 0.3773,
      "rewards/chosen": -0.5523546772618447,
      "rewards/margins": 2.2483803339834205,
      "rewards/rejected": -2.800735011245265,
      "step": 380
    },
    {
      "epoch": 0.20483193277310924,
      "grad_norm": 40.91805541297444,
      "kl": 0.0,
      "learning_rate": 4.5000039203521976e-07,
      "logps/chosen": -305.74300986842104,
      "logps/rejected": -303.6615048363095,
      "loss": 0.3428,
      "rewards/chosen": 0.08902386615150853,
      "rewards/margins": 3.217569718086032,
      "rewards/rejected": -3.1285458519345237,
      "step": 390
    },
    {
      "epoch": 0.21008403361344538,
      "grad_norm": 48.39763770048436,
      "kl": 0.0,
      "learning_rate": 4.47498294439647e-07,
      "logps/chosen": -293.79254518072287,
      "logps/rejected": -266.05382508116884,
      "loss": 0.3273,
      "rewards/chosen": 0.20805818488798947,
      "rewards/margins": 2.784955744115871,
      "rewards/rejected": -2.5768975592278816,
      "step": 400
    },
    {
      "epoch": 0.21533613445378152,
      "grad_norm": 64.69719531344766,
      "kl": 0.0,
      "learning_rate": 4.449424294047419e-07,
      "logps/chosen": -298.2714004984663,
      "logps/rejected": -277.8911972531847,
      "loss": 0.3478,
      "rewards/chosen": 0.7857334628426955,
      "rewards/margins": 2.803303423904598,
      "rewards/rejected": -2.017569961061903,
      "step": 410
    },
    {
      "epoch": 0.22058823529411764,
      "grad_norm": 41.5548039641909,
      "kl": 0.0,
      "learning_rate": 4.4233349274571974e-07,
      "logps/chosen": -294.40205536912754,
      "logps/rejected": -262.53922240497076,
      "loss": 0.3273,
      "rewards/chosen": 0.6274464754450241,
      "rewards/margins": 3.27400759055376,
      "rewards/rejected": -2.6465611151087356,
      "step": 420
    },
    {
      "epoch": 0.22584033613445378,
      "grad_norm": 51.90314639757878,
      "kl": 0.0,
      "learning_rate": 4.396721947261496e-07,
      "logps/chosen": -290.494140625,
      "logps/rejected": -266.31001420454544,
      "loss": 0.3513,
      "rewards/chosen": 0.22074366190347328,
      "rewards/margins": 2.942957208854569,
      "rewards/rejected": -2.722213546951096,
      "step": 430
    },
    {
      "epoch": 0.23109243697478993,
      "grad_norm": 45.08774574606674,
      "kl": 0.0,
      "learning_rate": 4.3695925986459107e-07,
      "logps/chosen": -274.8046160442073,
      "logps/rejected": -247.5850110176282,
      "loss": 0.3506,
      "rewards/chosen": 0.5506789044636052,
      "rewards/margins": 3.319194662488946,
      "rewards/rejected": -2.7685157580253406,
      "step": 440
    },
    {
      "epoch": 0.23634453781512604,
      "grad_norm": 38.43146082353361,
      "kl": 0.0,
      "learning_rate": 4.341954267373494e-07,
      "logps/chosen": -293.00198951863354,
      "logps/rejected": -258.1283411949685,
      "loss": 0.3646,
      "rewards/chosen": 0.5552752477041683,
      "rewards/margins": 2.609661794480898,
      "rewards/rejected": -2.0543865467767297,
      "step": 450
    },
    {
      "epoch": 0.2415966386554622,
      "grad_norm": 42.36936446062343,
      "kl": 0.0,
      "learning_rate": 4.313814477774035e-07,
      "logps/chosen": -307.16541371855345,
      "logps/rejected": -266.2528872282609,
      "loss": 0.3188,
      "rewards/chosen": 0.6136390638051543,
      "rewards/margins": 3.2912566758975457,
      "rewards/rejected": -2.677617612092391,
      "step": 460
    },
    {
      "epoch": 0.24684873949579833,
      "grad_norm": 36.14877170239455,
      "kl": 0.0,
      "learning_rate": 4.2851808906956134e-07,
      "logps/chosen": -299.0251017011834,
      "logps/rejected": -283.08899006622516,
      "loss": 0.3248,
      "rewards/chosen": 0.8636552257650703,
      "rewards/margins": 4.432308458769002,
      "rewards/rejected": -3.568653233003932,
      "step": 470
    },
    {
      "epoch": 0.25210084033613445,
      "grad_norm": 48.52314483845558,
      "kl": 0.0,
      "learning_rate": 4.256061301418996e-07,
      "logps/chosen": -288.11904978197674,
      "logps/rejected": -273.0204022381757,
      "loss": 0.3566,
      "rewards/chosen": 0.09835364097772642,
      "rewards/margins": 3.3749882696860247,
      "rewards/rejected": -3.2766346287082984,
      "step": 480
    },
    {
      "epoch": 0.25735294117647056,
      "grad_norm": 40.328590442957,
      "kl": 0.0,
      "learning_rate": 4.2264636375354283e-07,
      "logps/chosen": -290.32015931372547,
      "logps/rejected": -256.6384496631737,
      "loss": 0.3385,
      "rewards/chosen": 0.6565687890146293,
      "rewards/margins": 2.809185004337889,
      "rewards/rejected": -2.1526162153232598,
      "step": 490
    },
    {
      "epoch": 0.26260504201680673,
      "grad_norm": 51.22796460390333,
      "kl": 0.0,
      "learning_rate": 4.1963959567884045e-07,
      "logps/chosen": -302.9179159628378,
      "logps/rejected": -269.88603742732556,
      "loss": 0.3213,
      "rewards/chosen": 1.2393865843076963,
      "rewards/margins": 3.752552790135276,
      "rewards/rejected": -2.51316620582758,
      "step": 500
    },
    {
      "epoch": 0.26785714285714285,
      "grad_norm": 40.763747675132,
      "kl": 0.0,
      "learning_rate": 4.1658664448800094e-07,
      "logps/chosen": -278.07726258116884,
      "logps/rejected": -242.03510918674698,
      "loss": 0.3613,
      "rewards/chosen": 1.3944342476981026,
      "rewards/margins": 2.644719200987824,
      "rewards/rejected": -1.2502849532897213,
      "step": 510
    },
    {
      "epoch": 0.27310924369747897,
      "grad_norm": 41.37886905858638,
      "kl": 0.0,
      "learning_rate": 4.1348834132424204e-07,
      "logps/chosen": -303.84667354559747,
      "logps/rejected": -279.02011354813664,
      "loss": 0.355,
      "rewards/chosen": 0.7681676516742826,
      "rewards/margins": 3.026760719293654,
      "rewards/rejected": -2.2585930676193713,
      "step": 520
    },
    {
      "epoch": 0.27836134453781514,
      "grad_norm": 37.134601685804,
      "kl": 0.0,
      "learning_rate": 4.103455296775181e-07,
      "logps/chosen": -311.4682323042169,
      "logps/rejected": -256.0505783279221,
      "loss": 0.3388,
      "rewards/chosen": 1.0605972473879894,
      "rewards/margins": 2.909687229061589,
      "rewards/rejected": -1.8490899816736,
      "step": 530
    },
    {
      "epoch": 0.28361344537815125,
      "grad_norm": 50.21606157716232,
      "kl": 0.0,
      "learning_rate": 4.071590651548867e-07,
      "logps/chosen": -282.52527573529414,
      "logps/rejected": -283.90199288922156,
      "loss": 0.3322,
      "rewards/chosen": 0.6519086251851001,
      "rewards/margins": 3.288977687743577,
      "rewards/rejected": -2.637069062558477,
      "step": 540
    },
    {
      "epoch": 0.28886554621848737,
      "grad_norm": 40.94800472377613,
      "kl": 0.0,
      "learning_rate": 4.039298152475754e-07,
      "logps/chosen": -296.5549363057325,
      "logps/rejected": -296.0996932515337,
      "loss": 0.3261,
      "rewards/chosen": 0.4167486786083051,
      "rewards/margins": 3.556985000742124,
      "rewards/rejected": -3.140236322133819,
      "step": 550
    },
    {
      "epoch": 0.29411764705882354,
      "grad_norm": 47.7284966576563,
      "kl": 0.0,
      "learning_rate": 4.006586590948141e-07,
      "logps/chosen": -287.6368415880503,
      "logps/rejected": -276.68611704192546,
      "loss": 0.3348,
      "rewards/chosen": -0.10697765470300831,
      "rewards/margins": 3.6730512630716423,
      "rewards/rejected": -3.7800289177746507,
      "step": 560
    },
    {
      "epoch": 0.29936974789915966,
      "grad_norm": 45.50328767344306,
      "kl": 0.0,
      "learning_rate": 3.973464872444958e-07,
      "logps/chosen": -296.46134996118013,
      "logps/rejected": -281.4715507075472,
      "loss": 0.3322,
      "rewards/chosen": 0.1290158692354001,
      "rewards/margins": 3.9525375132998652,
      "rewards/rejected": -3.8235216440644653,
      "step": 570
    },
    {
      "epoch": 0.30462184873949577,
      "grad_norm": 38.47971512593867,
      "kl": 0.0,
      "learning_rate": 3.939942014107318e-07,
      "logps/chosen": -262.15194610778445,
      "logps/rejected": -292.0669424019608,
      "loss": 0.3594,
      "rewards/chosen": 0.3589071856287425,
      "rewards/margins": 3.160983936739343,
      "rewards/rejected": -2.8020767511106004,
      "step": 580
    },
    {
      "epoch": 0.30987394957983194,
      "grad_norm": 34.207797203706725,
      "kl": 0.0,
      "learning_rate": 3.9060271422836624e-07,
      "logps/chosen": -292.61703330592104,
      "logps/rejected": -271.3793247767857,
      "loss": 0.3159,
      "rewards/chosen": 1.2860569201017682,
      "rewards/margins": 4.367648571654968,
      "rewards/rejected": -3.0815916515531994,
      "step": 590
    },
    {
      "epoch": 0.31512605042016806,
      "grad_norm": 57.39149614591673,
      "kl": 0.0,
      "learning_rate": 3.871729490045185e-07,
      "logps/chosen": -290.49703584558824,
      "logps/rejected": -252.63739583333333,
      "loss": 0.3164,
      "rewards/chosen": 0.8580891328699448,
      "rewards/margins": 3.7738939831303617,
      "rewards/rejected": -2.915804850260417,
      "step": 600
    },
    {
      "epoch": 0.32037815126050423,
      "grad_norm": 40.37544580915199,
      "kl": 0.0,
      "learning_rate": 3.837058394672196e-07,
      "logps/chosen": -273.4779296875,
      "logps/rejected": -287.0096923828125,
      "loss": 0.3289,
      "rewards/chosen": 0.3099039077758789,
      "rewards/margins": 3.173082160949707,
      "rewards/rejected": -2.863178253173828,
      "step": 610
    },
    {
      "epoch": 0.32563025210084034,
      "grad_norm": 57.45015573264363,
      "kl": 0.0,
      "learning_rate": 3.8020232951121166e-07,
      "logps/chosen": -290.50161637931035,
      "logps/rejected": -294.233125,
      "loss": 0.3188,
      "rewards/chosen": 0.1405291984821188,
      "rewards/margins": 3.6642485065178327,
      "rewards/rejected": -3.523719308035714,
      "step": 620
    },
    {
      "epoch": 0.33088235294117646,
      "grad_norm": 39.287989744489565,
      "kl": 0.0,
      "learning_rate": 3.7666337294097985e-07,
      "logps/chosen": -302.1064724392361,
      "logps/rejected": -260.4077814275568,
      "loss": 0.3435,
      "rewards/chosen": -0.013078765736685859,
      "rewards/margins": 3.530285672406958,
      "rewards/rejected": -3.5433644381436435,
      "step": 630
    },
    {
      "epoch": 0.33613445378151263,
      "grad_norm": 44.59203672342995,
      "kl": 0.0,
      "learning_rate": 3.730899332110855e-07,
      "logps/chosen": -281.5672392003676,
      "logps/rejected": -294.1050866168478,
      "loss": 0.2851,
      "rewards/chosen": 0.49381469277774587,
      "rewards/margins": 5.133490330727814,
      "rewards/rejected": -4.639675637950068,
      "step": 640
    },
    {
      "epoch": 0.34138655462184875,
      "grad_norm": 41.26338033254354,
      "kl": 0.0,
      "learning_rate": 3.694829831638738e-07,
      "logps/chosen": -272.1574797453704,
      "logps/rejected": -303.05520668512656,
      "loss": 0.3318,
      "rewards/chosen": -0.2948492309193552,
      "rewards/margins": 3.4522689825893473,
      "rewards/rejected": -3.7471182135087027,
      "step": 650
    },
    {
      "epoch": 0.34663865546218486,
      "grad_norm": 53.39190760647335,
      "kl": 0.0,
      "learning_rate": 3.658435047646238e-07,
      "logps/chosen": -281.7225378787879,
      "logps/rejected": -267.5343245967742,
      "loss": 0.3109,
      "rewards/chosen": 0.5971468838778409,
      "rewards/margins": 4.032071987834494,
      "rewards/rejected": -3.4349251039566533,
      "step": 660
    },
    {
      "epoch": 0.35189075630252103,
      "grad_norm": 50.14394406023911,
      "kl": 0.0,
      "learning_rate": 3.621724888342161e-07,
      "logps/chosen": -307.17963248239437,
      "logps/rejected": -244.93414238061797,
      "loss": 0.3456,
      "rewards/chosen": 0.5407970052369884,
      "rewards/margins": 2.9319001711099695,
      "rewards/rejected": -2.391103165872981,
      "step": 670
    },
    {
      "epoch": 0.35714285714285715,
      "grad_norm": 44.42377974125605,
      "kl": 0.0,
      "learning_rate": 3.584709347793895e-07,
      "logps/chosen": -328.5807938664596,
      "logps/rejected": -273.77439563679246,
      "loss": 0.3028,
      "rewards/chosen": 0.6922078103012179,
      "rewards/margins": 4.292771099535692,
      "rewards/rejected": -3.6005632892344734,
      "step": 680
    },
    {
      "epoch": 0.36239495798319327,
      "grad_norm": 64.33577601812173,
      "kl": 0.0,
      "learning_rate": 3.5473985032065946e-07,
      "logps/chosen": -339.54715008802816,
      "logps/rejected": -288.18354985955057,
      "loss": 0.3051,
      "rewards/chosen": 0.7634544372558594,
      "rewards/margins": 4.167792459552208,
      "rewards/rejected": -3.4043380222963484,
      "step": 690
    },
    {
      "epoch": 0.36764705882352944,
      "grad_norm": 50.09740849242821,
      "kl": 0.0,
      "learning_rate": 3.509802512179737e-07,
      "logps/chosen": -320.1032510080645,
      "logps/rejected": -247.99145359848484,
      "loss": 0.3144,
      "rewards/chosen": -0.7038151402627268,
      "rewards/margins": 4.308948235684243,
      "rewards/rejected": -5.01276337594697,
      "step": 700
    },
    {
      "epoch": 0.37289915966386555,
      "grad_norm": 48.91753923946882,
      "kl": 0.0,
      "learning_rate": 3.4719316099417983e-07,
      "logps/chosen": -319.73927600931677,
      "logps/rejected": -308.56738895440253,
      "loss": 0.3676,
      "rewards/chosen": -1.7911930439635093,
      "rewards/margins": 3.521101202302214,
      "rewards/rejected": -5.312294246265723,
      "step": 710
    },
    {
      "epoch": 0.37815126050420167,
      "grad_norm": 58.85183781985146,
      "kl": 0.0,
      "learning_rate": 3.4337961065637786e-07,
      "logps/chosen": -351.19349449685535,
      "logps/rejected": -328.6528289984472,
      "loss": 0.3281,
      "rewards/chosen": -0.32832849850444673,
      "rewards/margins": 4.381978936714885,
      "rewards/rejected": -4.710307435219332,
      "step": 720
    },
    {
      "epoch": 0.38340336134453784,
      "grad_norm": 34.12584073424692,
      "kl": 0.0,
      "learning_rate": 3.395406384152371e-07,
      "logps/chosen": -318.7544778963415,
      "logps/rejected": -281.5553635817308,
      "loss": 0.3211,
      "rewards/chosen": 0.22548980247683642,
      "rewards/margins": 3.761472765843223,
      "rewards/rejected": -3.5359829633663864,
      "step": 730
    },
    {
      "epoch": 0.38865546218487396,
      "grad_norm": 66.4865558017911,
      "kl": 0.0,
      "learning_rate": 3.356772894023505e-07,
      "logps/chosen": -274.7813126671123,
      "logps/rejected": -285.46543115601503,
      "loss": 0.376,
      "rewards/chosen": -0.13362450013186206,
      "rewards/margins": 3.3327875780371934,
      "rewards/rejected": -3.4664120781690553,
      "step": 740
    },
    {
      "epoch": 0.3939075630252101,
      "grad_norm": 23.48397277478164,
      "kl": 0.0,
      "learning_rate": 3.317906153857054e-07,
      "logps/chosen": -302.44775079617835,
      "logps/rejected": -287.7147958205521,
      "loss": 0.3351,
      "rewards/chosen": 0.14863140567852434,
      "rewards/margins": 4.358767210773425,
      "rewards/rejected": -4.2101358050949,
      "step": 750
    },
    {
      "epoch": 0.39915966386554624,
      "grad_norm": 34.1827992343062,
      "kl": 0.0,
      "learning_rate": 3.2788167448334784e-07,
      "logps/chosen": -262.08561812106916,
      "logps/rejected": -274.4614227484472,
      "loss": 0.374,
      "rewards/chosen": -1.007453294670057,
      "rewards/margins": 2.7304841113130562,
      "rewards/rejected": -3.7379374059831134,
      "step": 760
    },
    {
      "epoch": 0.40441176470588236,
      "grad_norm": 45.645905851061976,
      "kl": 0.0,
      "learning_rate": 3.2395153087531763e-07,
      "logps/chosen": -290.2492659395973,
      "logps/rejected": -247.69814510233917,
      "loss": 0.3295,
      "rewards/chosen": -0.16713363852276897,
      "rewards/margins": 3.4756729221497453,
      "rewards/rejected": -3.6428065606725144,
      "step": 770
    },
    {
      "epoch": 0.4096638655462185,
      "grad_norm": 52.083381199139474,
      "kl": 0.0,
      "learning_rate": 3.20001254513933e-07,
      "logps/chosen": -301.3512216862416,
      "logps/rejected": -320.7805875365497,
      "loss": 0.3564,
      "rewards/chosen": 0.2707460390641385,
      "rewards/margins": 3.1255901602721066,
      "rewards/rejected": -2.854844121207968,
      "step": 780
    },
    {
      "epoch": 0.41491596638655465,
      "grad_norm": 39.20450298136074,
      "kl": 0.0,
      "learning_rate": 3.160319208325044e-07,
      "logps/chosen": -320.55803206699346,
      "logps/rejected": -272.5804172904192,
      "loss": 0.3116,
      "rewards/chosen": 0.21087132871540543,
      "rewards/margins": 3.79839062605822,
      "rewards/rejected": -3.5875192973428143,
      "step": 790
    },
    {
      "epoch": 0.42016806722689076,
      "grad_norm": 49.5026281906496,
      "kl": 0.0,
      "learning_rate": 3.1204461045255597e-07,
      "logps/chosen": -305.87476748511904,
      "logps/rejected": -282.5663291529605,
      "loss": 0.3302,
      "rewards/chosen": 0.5268175942557198,
      "rewards/margins": 3.157528597609441,
      "rewards/rejected": -2.630711003353721,
      "step": 800
    },
    {
      "epoch": 0.4254201680672269,
      "grad_norm": 41.129555544878784,
      "kl": 0.0,
      "learning_rate": 3.0804040888963367e-07,
      "logps/chosen": -305.3494718309859,
      "logps/rejected": -257.2256846910112,
      "loss": 0.2981,
      "rewards/chosen": 0.24150611984897669,
      "rewards/margins": 4.293326681975907,
      "rewards/rejected": -4.051820562126931,
      "step": 810
    },
    {
      "epoch": 0.43067226890756305,
      "grad_norm": 42.46230974734401,
      "kl": 0.0,
      "learning_rate": 3.040204062577824e-07,
      "logps/chosen": -312.58470262096773,
      "logps/rejected": -261.4371448863636,
      "loss": 0.316,
      "rewards/chosen": 0.39130327778477825,
      "rewards/margins": 3.736164117628528,
      "rewards/rejected": -3.34486083984375,
      "step": 820
    },
    {
      "epoch": 0.43592436974789917,
      "grad_norm": 61.00539417491642,
      "kl": 0.0,
      "learning_rate": 2.999856969727704e-07,
      "logps/chosen": -305.4535361842105,
      "logps/rejected": -265.3578020134228,
      "loss": 0.3327,
      "rewards/chosen": 1.0649953920241686,
      "rewards/margins": 3.993097206857432,
      "rewards/rejected": -2.9281018148332634,
      "step": 830
    },
    {
      "epoch": 0.4411764705882353,
      "grad_norm": 37.0047213388179,
      "kl": 0.0,
      "learning_rate": 2.959373794541426e-07,
      "logps/chosen": -282.06252472310126,
      "logps/rejected": -256.6561053240741,
      "loss": 0.3265,
      "rewards/chosen": 0.3667142361025267,
      "rewards/margins": 3.1375505835176503,
      "rewards/rejected": -2.7708363474151234,
      "step": 840
    },
    {
      "epoch": 0.44642857142857145,
      "grad_norm": 36.25355732454002,
      "kl": 0.0,
      "learning_rate": 2.9187655582618407e-07,
      "logps/chosen": -313.9366851993865,
      "logps/rejected": -287.78662420382165,
      "loss": 0.3112,
      "rewards/chosen": 0.5094557686086081,
      "rewards/margins": 3.8714661655781546,
      "rewards/rejected": -3.3620103969695463,
      "step": 850
    },
    {
      "epoch": 0.45168067226890757,
      "grad_norm": 64.53574065510352,
      "kl": 0.0,
      "learning_rate": 2.878043316178753e-07,
      "logps/chosen": -283.32109866352204,
      "logps/rejected": -264.0886063664596,
      "loss": 0.3276,
      "rewards/chosen": 0.1551594764181653,
      "rewards/margins": 4.185146314054035,
      "rewards/rejected": -4.029986837635869,
      "step": 860
    },
    {
      "epoch": 0.4569327731092437,
      "grad_norm": 35.86505890578853,
      "kl": 0.0,
      "learning_rate": 2.837218154619193e-07,
      "logps/chosen": -312.15406436011904,
      "logps/rejected": -274.98959189967104,
      "loss": 0.3348,
      "rewards/chosen": -0.1483011245727539,
      "rewards/margins": 4.045587087932386,
      "rewards/rejected": -4.19388821250514,
      "step": 870
    },
    {
      "epoch": 0.46218487394957986,
      "grad_norm": 46.44246066253616,
      "kl": 0.0,
      "learning_rate": 2.796301187929257e-07,
      "logps/chosen": -316.67644557823127,
      "logps/rejected": -274.3472949783237,
      "loss": 0.331,
      "rewards/chosen": -0.024288294266681283,
      "rewards/margins": 3.5203153511903276,
      "rewards/rejected": -3.5446036454570087,
      "step": 880
    },
    {
      "epoch": 0.46743697478991597,
      "grad_norm": 67.03947638674309,
      "kl": 0.0,
      "learning_rate": 2.755303555448301e-07,
      "logps/chosen": -296.37250766871165,
      "logps/rejected": -287.87425358280257,
      "loss": 0.2935,
      "rewards/chosen": 0.2513600448889235,
      "rewards/margins": 4.716395453054728,
      "rewards/rejected": -4.465035408165805,
      "step": 890
    },
    {
      "epoch": 0.4726890756302521,
      "grad_norm": 44.0580333940613,
      "kl": 0.0,
      "learning_rate": 2.7142364184763424e-07,
      "logps/chosen": -275.6317777317881,
      "logps/rejected": -295.6349852071006,
      "loss": 0.3427,
      "rewards/chosen": -0.019468964330407956,
      "rewards/margins": 3.7003294679689636,
      "rewards/rejected": -3.7197984322993713,
      "step": 900
    },
    {
      "epoch": 0.47794117647058826,
      "grad_norm": 58.59520003164634,
      "kl": 0.0,
      "learning_rate": 2.673110957235479e-07,
      "logps/chosen": -305.7110357704403,
      "logps/rejected": -299.5851853649068,
      "loss": 0.2967,
      "rewards/chosen": 0.24971195436873525,
      "rewards/margins": 4.398466382260817,
      "rewards/rejected": -4.148754427892081,
      "step": 910
    },
    {
      "epoch": 0.4831932773109244,
      "grad_norm": 49.15023451078887,
      "kl": 0.0,
      "learning_rate": 2.6319383678261557e-07,
      "logps/chosen": -326.3589082154088,
      "logps/rejected": -291.8697350543478,
      "loss": 0.34,
      "rewards/chosen": -0.1070428524377211,
      "rewards/margins": 3.931509757592364,
      "rewards/rejected": -4.038552610030085,
      "step": 920
    },
    {
      "epoch": 0.4884453781512605,
      "grad_norm": 60.143948744622776,
      "kl": 0.0,
      "learning_rate": 2.5907298591791105e-07,
      "logps/chosen": -293.6828878012048,
      "logps/rejected": -316.3496601055195,
      "loss": 0.3121,
      "rewards/chosen": 0.37629242402961455,
      "rewards/margins": 4.533702884662731,
      "rewards/rejected": -4.157410460633117,
      "step": 930
    },
    {
      "epoch": 0.49369747899159666,
      "grad_norm": 41.39145780505376,
      "kl": 0.0,
      "learning_rate": 2.5494966500038264e-07,
      "logps/chosen": -298.37648703835225,
      "logps/rejected": -283.5568033854167,
      "loss": 0.3381,
      "rewards/chosen": 0.19053040851246228,
      "rewards/margins": 2.8964626018447106,
      "rewards/rejected": -2.705932193332248,
      "step": 940
    },
    {
      "epoch": 0.4989495798319328,
      "grad_norm": 44.85935706825639,
      "kl": 0.0,
      "learning_rate": 2.508249965734319e-07,
      "logps/chosen": -315.8209355828221,
      "logps/rejected": -277.33056329617835,
      "loss": 0.2815,
      "rewards/chosen": 0.5839554488293233,
      "rewards/margins": 4.811377494883264,
      "rewards/rejected": -4.227422046053941,
      "step": 950
    },
    {
      "epoch": 0.5042016806722689,
      "grad_norm": 86.97621065062417,
      "kl": 0.0,
      "learning_rate": 2.467001035473103e-07,
      "logps/chosen": -302.4603059505988,
      "logps/rejected": -287.89797794117646,
      "loss": 0.3293,
      "rewards/chosen": 0.40462347704493357,
      "rewards/margins": 3.534475173582924,
      "rewards/rejected": -3.1298516965379903,
      "step": 960
    },
    {
      "epoch": 0.509453781512605,
      "grad_norm": 32.22216772039752,
      "kl": 0.0,
      "learning_rate": 2.425761088934142e-07,
      "logps/chosen": -273.99361898291926,
      "logps/rejected": -258.7056308962264,
      "loss": 0.3326,
      "rewards/chosen": -0.09355034443162243,
      "rewards/margins": 4.204330392105327,
      "rewards/rejected": -4.29788073653695,
      "step": 970
    },
    {
      "epoch": 0.5147058823529411,
      "grad_norm": 53.215396805561575,
      "kl": 0.0,
      "learning_rate": 2.3845413533856514e-07,
      "logps/chosen": -299.4835397012579,
      "logps/rejected": -287.7150621118012,
      "loss": 0.2911,
      "rewards/chosen": 0.30934901207498033,
      "rewards/margins": 4.708151206611083,
      "rewards/rejected": -4.398802194536103,
      "step": 980
    },
    {
      "epoch": 0.5199579831932774,
      "grad_norm": 70.13460679739038,
      "kl": 0.0,
      "learning_rate": 2.343353050593553e-07,
      "logps/chosen": -318.12441770186336,
      "logps/rejected": -282.98125491352204,
      "loss": 0.3042,
      "rewards/chosen": 0.9980694314707881,
      "rewards/margins": 5.077612059345198,
      "rewards/rejected": -4.07954262787441,
      "step": 990
    },
    {
      "epoch": 0.5252100840336135,
      "grad_norm": 91.66490200361514,
      "kl": 0.0,
      "learning_rate": 2.3022073937664383e-07,
      "logps/chosen": -318.62791700487014,
      "logps/rejected": -299.1699924698795,
      "loss": 0.2987,
      "rewards/chosen": -0.4350905975738129,
      "rewards/margins": 3.7659268511272415,
      "rewards/rejected": -4.201017448701054,
      "step": 1000
    },
    {
      "epoch": 0.5304621848739496,
      "grad_norm": 47.014009528040546,
      "kl": 0.0,
      "learning_rate": 2.261115584502849e-07,
      "logps/chosen": -322.26196442610063,
      "logps/rejected": -273.58297748447205,
      "loss": 0.3504,
      "rewards/chosen": -0.4901385037404186,
      "rewards/margins": 2.980573340567617,
      "rewards/rejected": -3.4707118443080356,
      "step": 1010
    },
    {
      "epoch": 0.5357142857142857,
      "grad_norm": 65.73363572647297,
      "kl": 0.0,
      "learning_rate": 2.2200888097417302e-07,
      "logps/chosen": -265.984912109375,
      "logps/rejected": -273.97919921875,
      "loss": 0.3327,
      "rewards/chosen": 0.2538748264312744,
      "rewards/margins": 3.538307809829712,
      "rewards/rejected": -3.2844329833984376,
      "step": 1020
    },
    {
      "epoch": 0.5409663865546218,
      "grad_norm": 37.39061372036576,
      "kl": 0.0,
      "learning_rate": 2.1791382387168684e-07,
      "logps/chosen": -294.4272490530303,
      "logps/rejected": -283.10264616935484,
      "loss": 0.3218,
      "rewards/chosen": 0.5568286317767519,
      "rewards/margins": 4.048672366025744,
      "rewards/rejected": -3.491843734248992,
      "step": 1030
    },
    {
      "epoch": 0.5462184873949579,
      "grad_norm": 38.42991974376242,
      "kl": 0.0,
      "learning_rate": 2.1382750199161495e-07,
      "logps/chosen": -314.6301011029412,
      "logps/rejected": -277.451328125,
      "loss": 0.3087,
      "rewards/chosen": 0.9661560956169577,
      "rewards/margins": 3.718452645096124,
      "rewards/rejected": -2.7522965494791665,
      "step": 1040
    },
    {
      "epoch": 0.5514705882352942,
      "grad_norm": 58.32907293331433,
      "kl": 0.0,
      "learning_rate": 2.0975102780464673e-07,
      "logps/chosen": -306.8538490853659,
      "logps/rejected": -290.90675080128204,
      "loss": 0.3062,
      "rewards/chosen": 0.684349339182784,
      "rewards/margins": 3.878205602954819,
      "rewards/rejected": -3.193856263772035,
      "step": 1050
    },
    {
      "epoch": 0.5567226890756303,
      "grad_norm": 42.31783649576492,
      "kl": 0.0,
      "learning_rate": 2.0568551110051074e-07,
      "logps/chosen": -265.01874508647796,
      "logps/rejected": -256.457298136646,
      "loss": 0.3375,
      "rewards/chosen": 0.7610905485333137,
      "rewards/margins": 3.0508450962211047,
      "rewards/rejected": -2.289754547687791,
      "step": 1060
    },
    {
      "epoch": 0.5619747899159664,
      "grad_norm": 43.282189967649956,
      "kl": 0.0,
      "learning_rate": 2.016320586858422e-07,
      "logps/chosen": -270.2985276442308,
      "logps/rejected": -287.5562357088415,
      "loss": 0.3143,
      "rewards/chosen": 1.108019804343199,
      "rewards/margins": 3.3270524641064423,
      "rewards/rejected": -2.219032659763243,
      "step": 1070
    },
    {
      "epoch": 0.5672268907563025,
      "grad_norm": 59.248701734675386,
      "kl": 0.0,
      "learning_rate": 1.9759177408286337e-07,
      "logps/chosen": -312.609375,
      "logps/rejected": -267.16066028225805,
      "loss": 0.3365,
      "rewards/chosen": 1.2182994495738637,
      "rewards/margins": 3.07108659045088,
      "rewards/rejected": -1.8527871408770162,
      "step": 1080
    },
    {
      "epoch": 0.5724789915966386,
      "grad_norm": 45.19596654926437,
      "kl": 0.0,
      "learning_rate": 1.9356575722895808e-07,
      "logps/chosen": -299.373361013986,
      "logps/rejected": -271.5683042019774,
      "loss": 0.3178,
      "rewards/chosen": 1.0334772496790319,
      "rewards/margins": 3.244971479960194,
      "rewards/rejected": -2.2114942302811618,
      "step": 1090
    },
    {
      "epoch": 0.5777310924369747,
      "grad_norm": 44.21402337263303,
      "kl": 0.0,
      "learning_rate": 1.895551041772216e-07,
      "logps/chosen": -309.0861150568182,
      "logps/rejected": -255.60587565104166,
      "loss": 0.3402,
      "rewards/chosen": 0.8877213217995383,
      "rewards/margins": 3.0730268112336745,
      "rewards/rejected": -2.1853054894341364,
      "step": 1100
    },
    {
      "epoch": 0.582983193277311,
      "grad_norm": 61.69332868228022,
      "kl": 0.0,
      "learning_rate": 1.8556090679806847e-07,
      "logps/chosen": -298.82965455572287,
      "logps/rejected": -267.7830509334416,
      "loss": 0.2942,
      "rewards/chosen": 1.1525323706937123,
      "rewards/margins": 4.799233143065871,
      "rewards/rejected": -3.646700772372159,
      "step": 1110
    },
    {
      "epoch": 0.5882352941176471,
      "grad_norm": 31.77308050751697,
      "kl": 0.0,
      "learning_rate": 1.8158425248197928e-07,
      "logps/chosen": -330.5060499237805,
      "logps/rejected": -304.98880709134613,
      "loss": 0.2989,
      "rewards/chosen": 1.3816723707245617,
      "rewards/margins": 5.020397847111781,
      "rewards/rejected": -3.6387254763872194,
      "step": 1120
    },
    {
      "epoch": 0.5934873949579832,
      "grad_norm": 59.622620200950486,
      "kl": 0.0,
      "learning_rate": 1.7762622384346609e-07,
      "logps/chosen": -292.1063850308642,
      "logps/rejected": -255.3093848892405,
      "loss": 0.311,
      "rewards/chosen": 1.0882685155044367,
      "rewards/margins": 3.8796188681176687,
      "rewards/rejected": -2.791350352613232,
      "step": 1130
    },
    {
      "epoch": 0.5987394957983193,
      "grad_norm": 92.89241064630508,
      "kl": 0.0,
      "learning_rate": 1.7368789842633907e-07,
      "logps/chosen": -324.4837362421384,
      "logps/rejected": -303.8497670807453,
      "loss": 0.2984,
      "rewards/chosen": 0.8251325499336675,
      "rewards/margins": 4.589403024385337,
      "rewards/rejected": -3.7642704744516693,
      "step": 1140
    },
    {
      "epoch": 0.6039915966386554,
      "grad_norm": 42.994311331506765,
      "kl": 0.0,
      "learning_rate": 1.697703484103532e-07,
      "logps/chosen": -292.4465987042683,
      "logps/rejected": -259.40554887820514,
      "loss": 0.3131,
      "rewards/chosen": 0.9567713853789539,
      "rewards/margins": 4.075035216884363,
      "rewards/rejected": -3.1182638315054088,
      "step": 1150
    },
    {
      "epoch": 0.6092436974789915,
      "grad_norm": 63.999900745814266,
      "kl": 0.0,
      "learning_rate": 1.6587464031931526e-07,
      "logps/chosen": -303.69353428171644,
      "logps/rejected": -277.118405577957,
      "loss": 0.3398,
      "rewards/chosen": 0.867815444718546,
      "rewards/margins": 3.5977176506472253,
      "rewards/rejected": -2.7299022059286795,
      "step": 1160
    },
    {
      "epoch": 0.6144957983193278,
      "grad_norm": 33.78499266925509,
      "kl": 0.0,
      "learning_rate": 1.6200183473073048e-07,
      "logps/chosen": -299.6548768939394,
      "logps/rejected": -280.22903225806454,
      "loss": 0.2992,
      "rewards/chosen": 0.6621969512014678,
      "rewards/margins": 3.7750548928597336,
      "rewards/rejected": -3.112857941658266,
      "step": 1170
    },
    {
      "epoch": 0.6197478991596639,
      "grad_norm": 31.77803791723765,
      "kl": 0.0,
      "learning_rate": 1.5815298598706888e-07,
      "logps/chosen": -291.1797602872671,
      "logps/rejected": -286.78439465408803,
      "loss": 0.3176,
      "rewards/chosen": -0.010194837676812403,
      "rewards/margins": 4.250023752633722,
      "rewards/rejected": -4.260218590310535,
      "step": 1180
    },
    {
      "epoch": 0.625,
      "grad_norm": 37.859068866257594,
      "kl": 0.0,
      "learning_rate": 1.5432914190872756e-07,
      "logps/chosen": -283.2485907832278,
      "logps/rejected": -299.72665895061726,
      "loss": 0.3475,
      "rewards/chosen": 0.012598955178562599,
      "rewards/margins": 4.188036976614712,
      "rewards/rejected": -4.1754380214361495,
      "step": 1190
    },
    {
      "epoch": 0.6302521008403361,
      "grad_norm": 34.730160989586295,
      "kl": 0.0,
      "learning_rate": 1.505313435087698e-07,
      "logps/chosen": -300.31629922945206,
      "logps/rejected": -295.37841235632186,
      "loss": 0.3124,
      "rewards/chosen": 0.33270381248160585,
      "rewards/margins": 4.571263930005852,
      "rewards/rejected": -4.238560117524246,
      "step": 1200
    },
    {
      "epoch": 0.6355042016806722,
      "grad_norm": 73.6570576556008,
      "kl": 0.0,
      "learning_rate": 1.4676062470951705e-07,
      "logps/chosen": -316.37948140337426,
      "logps/rejected": -306.59623805732485,
      "loss": 0.318,
      "rewards/chosen": 0.23967742919921875,
      "rewards/margins": 4.2318552102252935,
      "rewards/rejected": -3.992177781026075,
      "step": 1210
    },
    {
      "epoch": 0.6407563025210085,
      "grad_norm": 86.8105694173325,
      "kl": 0.0,
      "learning_rate": 1.430180120610711e-07,
      "logps/chosen": -283.5435161226115,
      "logps/rejected": -276.21841449386505,
      "loss": 0.2916,
      "rewards/chosen": 0.35688832762894357,
      "rewards/margins": 4.4911308476394876,
      "rewards/rejected": -4.134242520010544,
      "step": 1220
    },
    {
      "epoch": 0.6460084033613446,
      "grad_norm": 65.59031614397247,
      "kl": 0.0,
      "learning_rate": 1.3930452446184385e-07,
      "logps/chosen": -324.24284699675326,
      "logps/rejected": -301.1185523343373,
      "loss": 0.3146,
      "rewards/chosen": 0.3668579745602298,
      "rewards/margins": 4.802705980843173,
      "rewards/rejected": -4.435848006282944,
      "step": 1230
    },
    {
      "epoch": 0.6512605042016807,
      "grad_norm": 82.83869882352816,
      "kl": 0.0,
      "learning_rate": 1.3562117288116923e-07,
      "logps/chosen": -315.7268702651515,
      "logps/rejected": -280.67207661290325,
      "loss": 0.3109,
      "rewards/chosen": 0.6113425514914773,
      "rewards/margins": 4.853238264067082,
      "rewards/rejected": -4.241895712575605,
      "step": 1240
    },
    {
      "epoch": 0.6565126050420168,
      "grad_norm": 53.82976674411678,
      "kl": 0.0,
      "learning_rate": 1.319689600840747e-07,
      "logps/chosen": -304.3841062898089,
      "logps/rejected": -285.0993577453988,
      "loss": 0.3324,
      "rewards/chosen": 0.08269594884981775,
      "rewards/margins": 3.7769891991853237,
      "rewards/rejected": -3.694293250335506,
      "step": 1250
    },
    {
      "epoch": 0.6617647058823529,
      "grad_norm": 34.58790215772178,
      "kl": 0.0,
      "learning_rate": 1.2834888035828596e-07,
      "logps/chosen": -301.4619855182927,
      "logps/rejected": -272.6404246794872,
      "loss": 0.2994,
      "rewards/chosen": 0.32138896569973086,
      "rewards/margins": 4.973683418073529,
      "rewards/rejected": -4.652294452373798,
      "step": 1260
    },
    {
      "epoch": 0.667016806722689,
      "grad_norm": 29.984315889657353,
      "kl": 0.0,
      "learning_rate": 1.2476191924353932e-07,
      "logps/chosen": -345.89991071428574,
      "logps/rejected": -282.63448275862066,
      "loss": 0.326,
      "rewards/chosen": 0.5500016566685267,
      "rewards/margins": 4.73898333770301,
      "rewards/rejected": -4.188981681034483,
      "step": 1270
    },
    {
      "epoch": 0.6722689075630253,
      "grad_norm": 61.179118361711794,
      "kl": 0.0,
      "learning_rate": 1.2120905326327596e-07,
      "logps/chosen": -298.07992788461536,
      "logps/rejected": -293.07983236754967,
      "loss": 0.3307,
      "rewards/chosen": 0.1393580013478296,
      "rewards/margins": 4.935787970175437,
      "rewards/rejected": -4.796429968827607,
      "step": 1280
    },
    {
      "epoch": 0.6775210084033614,
      "grad_norm": 35.426220175192356,
      "kl": 0.0,
      "learning_rate": 1.1769124965879091e-07,
      "logps/chosen": -309.7057636589404,
      "logps/rejected": -253.99468380177515,
      "loss": 0.3127,
      "rewards/chosen": 0.3561096696664166,
      "rewards/margins": 4.979999841056024,
      "rewards/rejected": -4.623890171389608,
      "step": 1290
    },
    {
      "epoch": 0.6827731092436975,
      "grad_norm": 43.346023169027426,
      "kl": 0.0,
      "learning_rate": 1.1420946612590837e-07,
      "logps/chosen": -270.4368489583333,
      "logps/rejected": -281.6362047697368,
      "loss": 0.3397,
      "rewards/chosen": 0.1588947886512393,
      "rewards/margins": 3.5548700019530486,
      "rewards/rejected": -3.395975213301809,
      "step": 1300
    },
    {
      "epoch": 0.6880252100840336,
      "grad_norm": 45.324757492893184,
      "kl": 0.0,
      "learning_rate": 1.1076465055425646e-07,
      "logps/chosen": -290.0300690406977,
      "logps/rejected": -258.5001319679054,
      "loss": 0.3082,
      "rewards/chosen": 0.7295433754144713,
      "rewards/margins": 4.570315849548762,
      "rewards/rejected": -3.8407724741342903,
      "step": 1310
    },
    {
      "epoch": 0.6932773109243697,
      "grad_norm": 263.89653772607954,
      "kl": 0.0,
      "learning_rate": 1.0735774076921128e-07,
      "logps/chosen": -287.7608642578125,
      "logps/rejected": -237.489306640625,
      "loss": 0.3446,
      "rewards/chosen": 0.27030837535858154,
      "rewards/margins": 3.555664134025574,
      "rewards/rejected": -3.2853557586669924,
      "step": 1320
    },
    {
      "epoch": 0.6985294117647058,
      "grad_norm": 61.837360014332575,
      "kl": 0.0,
      "learning_rate": 1.039896642765809e-07,
      "logps/chosen": -297.6917317708333,
      "logps/rejected": -314.11572265625,
      "loss": 0.2958,
      "rewards/chosen": 0.6960362328423394,
      "rewards/margins": 5.248802512583106,
      "rewards/rejected": -4.552766279740767,
      "step": 1330
    },
    {
      "epoch": 0.7037815126050421,
      "grad_norm": 51.294856185176336,
      "kl": 0.0,
      "learning_rate": 1.0066133801009871e-07,
      "logps/chosen": -328.30787295386904,
      "logps/rejected": -244.40604440789474,
      "loss": 0.3097,
      "rewards/chosen": 0.7595687593732562,
      "rewards/margins": 4.201100693609481,
      "rewards/rejected": -3.441531934236225,
      "step": 1340
    },
    {
      "epoch": 0.7090336134453782,
      "grad_norm": 43.77043748464739,
      "kl": 0.0,
      "learning_rate": 9.737366808179553e-08,
      "logps/chosen": -303.54956371753246,
      "logps/rejected": -250.67248682228916,
      "loss": 0.3096,
      "rewards/chosen": 0.9003205732865767,
      "rewards/margins": 4.412543414848136,
      "rewards/rejected": -3.5122228415615586,
      "step": 1350
    },
    {
      "epoch": 0.7142857142857143,
      "grad_norm": 72.17773141718298,
      "kl": 0.0,
      "learning_rate": 9.412754953531663e-08,
      "logps/chosen": -262.4382017215569,
      "logps/rejected": -292.5083997140523,
      "loss": 0.3377,
      "rewards/chosen": 0.434063574511134,
      "rewards/margins": 3.880319925358765,
      "rewards/rejected": -3.446256350847631,
      "step": 1360
    },
    {
      "epoch": 0.7195378151260504,
      "grad_norm": 61.94872379316782,
      "kl": 0.0,
      "learning_rate": 9.092386610225325e-08,
      "logps/chosen": -265.7097810444079,
      "logps/rejected": -279.9903273809524,
      "loss": 0.3299,
      "rewards/chosen": 0.8910425085770456,
      "rewards/margins": 3.7223031897293892,
      "rewards/rejected": -2.8312606811523438,
      "step": 1370
    },
    {
      "epoch": 0.7247899159663865,
      "grad_norm": 47.60678426366227,
      "kl": 0.0,
      "learning_rate": 8.776348996155317e-08,
      "logps/chosen": -290.0428740530303,
      "logps/rejected": -278.20430947580644,
      "loss": 0.3123,
      "rewards/chosen": 0.3940057927911932,
      "rewards/margins": 4.4418612930432095,
      "rewards/rejected": -4.047855500252016,
      "step": 1380
    },
    {
      "epoch": 0.7300420168067226,
      "grad_norm": 41.557754145037535,
      "kl": 0.0,
      "learning_rate": 8.464728150207636e-08,
      "logps/chosen": -295.76380699685535,
      "logps/rejected": -294.7458753881988,
      "loss": 0.3185,
      "rewards/chosen": 0.6997887953272406,
      "rewards/margins": 4.221460825000183,
      "rewards/rejected": -3.5216720296729425,
      "step": 1390
    },
    {
      "epoch": 0.7352941176470589,
      "grad_norm": 45.38199821794469,
      "kl": 0.0,
      "learning_rate": 8.15760890883607e-08,
      "logps/chosen": -275.74736265923565,
      "logps/rejected": -261.5596721625767,
      "loss": 0.3109,
      "rewards/chosen": 0.7350611109642466,
      "rewards/margins": 4.5443022829590705,
      "rewards/rejected": -3.809241171994824,
      "step": 1400
    },
    {
      "epoch": 0.740546218487395,
      "grad_norm": 22.290814745353508,
      "kl": 0.0,
      "learning_rate": 7.855074882966103e-08,
      "logps/chosen": -311.78690011160717,
      "logps/rejected": -279.9527652138158,
      "loss": 0.2985,
      "rewards/chosen": 0.7244267236618769,
      "rewards/margins": 5.052972545002337,
      "rewards/rejected": -4.328545821340461,
      "step": 1410
    },
    {
      "epoch": 0.7457983193277311,
      "grad_norm": 40.315164584155504,
      "kl": 0.0,
      "learning_rate": 7.557208435232449e-08,
      "logps/chosen": -274.712925295858,
      "logps/rejected": -278.3036268625828,
      "loss": 0.3147,
      "rewards/chosen": 1.077583267843935,
      "rewards/margins": 4.07225724310004,
      "rewards/rejected": -2.9946739752561053,
      "step": 1420
    },
    {
      "epoch": 0.7510504201680672,
      "grad_norm": 36.626625075926555,
      "kl": 0.0,
      "learning_rate": 7.264090657556443e-08,
      "logps/chosen": -292.9152413091716,
      "logps/rejected": -242.9820467715232,
      "loss": 0.3085,
      "rewards/chosen": 0.9502442489714312,
      "rewards/margins": 4.543455846136166,
      "rewards/rejected": -3.593211597164735,
      "step": 1430
    },
    {
      "epoch": 0.7563025210084033,
      "grad_norm": 43.54130112641965,
      "kl": 0.0,
      "learning_rate": 6.975801349069385e-08,
      "logps/chosen": -280.07919034090907,
      "logps/rejected": -276.84921875,
      "loss": 0.2987,
      "rewards/chosen": 0.817291444720644,
      "rewards/margins": 5.2739698721399995,
      "rewards/rejected": -4.456678427419355,
      "step": 1440
    },
    {
      "epoch": 0.7615546218487395,
      "grad_norm": 39.41222523190064,
      "kl": 0.0,
      "learning_rate": 6.692418994387799e-08,
      "logps/chosen": -288.40252246732024,
      "logps/rejected": -284.47291354790417,
      "loss": 0.2993,
      "rewards/chosen": 0.9826148538028493,
      "rewards/margins": 5.268814549138741,
      "rewards/rejected": -4.286199695335891,
      "step": 1450
    },
    {
      "epoch": 0.7668067226890757,
      "grad_norm": 42.76397567606349,
      "kl": 0.0,
      "learning_rate": 6.414020742246593e-08,
      "logps/chosen": -288.7798685213415,
      "logps/rejected": -293.82729867788464,
      "loss": 0.2617,
      "rewards/chosen": 0.9620797692275629,
      "rewards/margins": 5.808872528267026,
      "rewards/rejected": -4.846792759039463,
      "step": 1460
    },
    {
      "epoch": 0.7720588235294118,
      "grad_norm": 85.5452615105662,
      "kl": 0.0,
      "learning_rate": 6.140682384495902e-08,
      "logps/chosen": -311.87120301573424,
      "logps/rejected": -282.2307997881356,
      "loss": 0.2877,
      "rewards/chosen": 0.8217522681176246,
      "rewards/margins": 4.416604004412646,
      "rewards/rejected": -3.594851736295021,
      "step": 1470
    },
    {
      "epoch": 0.7773109243697479,
      "grad_norm": 47.14939912048749,
      "kl": 0.0,
      "learning_rate": 5.872478335467298e-08,
      "logps/chosen": -270.7433546112805,
      "logps/rejected": -274.62745392628204,
      "loss": 0.329,
      "rewards/chosen": -0.1099286428311976,
      "rewards/margins": 4.24396219187934,
      "rewards/rejected": -4.353890834710537,
      "step": 1480
    },
    {
      "epoch": 0.782563025210084,
      "grad_norm": 75.39266090840431,
      "kl": 0.0,
      "learning_rate": 5.60948161171505e-08,
      "logps/chosen": -298.3712257179054,
      "logps/rejected": -296.9724518531977,
      "loss": 0.3039,
      "rewards/chosen": 0.3439343168928817,
      "rewards/margins": 4.58285359718903,
      "rewards/rejected": -4.238919280296148,
      "step": 1490
    },
    {
      "epoch": 0.7878151260504201,
      "grad_norm": 101.47398109074602,
      "kl": 0.0,
      "learning_rate": 5.351763812137916e-08,
      "logps/chosen": -285.37012357026146,
      "logps/rejected": -305.706282747006,
      "loss": 0.3075,
      "rewards/chosen": -0.05964530371372996,
      "rewards/margins": 4.639130484553485,
      "rewards/rejected": -4.698775788267215,
      "step": 1500
    },
    {
      "epoch": 0.7930672268907563,
      "grad_norm": 51.64579106329186,
      "kl": 0.0,
      "learning_rate": 5.0993950984868836e-08,
      "logps/chosen": -273.9158336568323,
      "logps/rejected": -296.97481819968556,
      "loss": 0.3228,
      "rewards/chosen": -0.35097162471794935,
      "rewards/margins": 4.398951601500603,
      "rewards/rejected": -4.749923226218553,
      "step": 1510
    },
    {
      "epoch": 0.7983193277310925,
      "grad_norm": 164.1846468516791,
      "kl": 0.0,
      "learning_rate": 4.8524441762641284e-08,
      "logps/chosen": -323.8402122641509,
      "logps/rejected": -265.2092148680124,
      "loss": 0.3144,
      "rewards/chosen": 0.010938104593528892,
      "rewards/margins": 4.735114546994538,
      "rewards/rejected": -4.724176442401009,
      "step": 1520
    },
    {
      "epoch": 0.8035714285714286,
      "grad_norm": 57.97209020416802,
      "kl": 0.0,
      "learning_rate": 4.6109782760184956e-08,
      "logps/chosen": -283.25261976047904,
      "logps/rejected": -294.36603860294116,
      "loss": 0.3462,
      "rewards/chosen": 0.19619906305552956,
      "rewards/margins": 4.044921234215661,
      "rewards/rejected": -3.848722171160131,
      "step": 1530
    },
    {
      "epoch": 0.8088235294117647,
      "grad_norm": 41.10976267046813,
      "kl": 0.0,
      "learning_rate": 4.375063135042445e-08,
      "logps/chosen": -301.61559264053255,
      "logps/rejected": -321.35994412251654,
      "loss": 0.3079,
      "rewards/chosen": 0.2954674895698502,
      "rewards/margins": 5.432189877426845,
      "rewards/rejected": -5.136722387856995,
      "step": 1540
    },
    {
      "epoch": 0.8140756302521008,
      "grad_norm": 54.92221531592682,
      "kl": 0.0,
      "learning_rate": 4.144762979475575e-08,
      "logps/chosen": -306.94077797202794,
      "logps/rejected": -280.2052436440678,
      "loss": 0.315,
      "rewards/chosen": -0.0729095952494161,
      "rewards/margins": 4.834762834239461,
      "rewards/rejected": -4.907672429488877,
      "step": 1550
    },
    {
      "epoch": 0.819327731092437,
      "grad_norm": 48.43314600236666,
      "kl": 0.0,
      "learning_rate": 3.9201405068195385e-08,
      "logps/chosen": -312.2001696134868,
      "logps/rejected": -292.7070777529762,
      "loss": 0.3395,
      "rewards/chosen": 0.025735384539553995,
      "rewards/margins": 4.3085366947609085,
      "rewards/rejected": -4.2828013102213545,
      "step": 1560
    },
    {
      "epoch": 0.8245798319327731,
      "grad_norm": 82.79685734368942,
      "kl": 0.0,
      "learning_rate": 3.701256868869124e-08,
      "logps/chosen": -321.1130847091195,
      "logps/rejected": -254.03585986024845,
      "loss": 0.3356,
      "rewards/chosen": 0.22610101759808618,
      "rewards/margins": 4.19767455700026,
      "rewards/rejected": -3.971573539402174,
      "step": 1570
    },
    {
      "epoch": 0.8298319327731093,
      "grad_norm": 69.63465669746606,
      "kl": 0.0,
      "learning_rate": 3.488171655064107e-08,
      "logps/chosen": -289.7723396650327,
      "logps/rejected": -288.95059880239523,
      "loss": 0.3227,
      "rewards/chosen": -0.3906313079634523,
      "rewards/margins": 4.496159441592124,
      "rewards/rejected": -4.886790749555576,
      "step": 1580
    },
    {
      "epoch": 0.8350840336134454,
      "grad_norm": 66.42429414400927,
      "kl": 0.0,
      "learning_rate": 3.28094287626651e-08,
      "logps/chosen": -340.9739889705882,
      "logps/rejected": -299.48609375,
      "loss": 0.3308,
      "rewards/chosen": 0.048118759604061354,
      "rewards/margins": 4.789248316895728,
      "rewards/rejected": -4.741129557291667,
      "step": 1590
    },
    {
      "epoch": 0.8403361344537815,
      "grad_norm": 42.16586593578885,
      "kl": 0.0,
      "learning_rate": 3.079626948967534e-08,
      "logps/chosen": -312.65874787414964,
      "logps/rejected": -287.83103775289015,
      "loss": 0.3062,
      "rewards/chosen": -0.0730894049819635,
      "rewards/margins": 4.740564265312472,
      "rewards/rejected": -4.813653670294436,
      "step": 1600
    },
    {
      "epoch": 0.8455882352941176,
      "grad_norm": 42.50176028584879,
      "kl": 0.0,
      "learning_rate": 2.88427867992862e-08,
      "logps/chosen": -298.3803453947368,
      "logps/rejected": -256.55287856543623,
      "loss": 0.3216,
      "rewards/chosen": 0.292915611936335,
      "rewards/margins": 4.54555282224464,
      "rewards/rejected": -4.252637210308305,
      "step": 1610
    },
    {
      "epoch": 0.8508403361344538,
      "grad_norm": 83.97476475765178,
      "kl": 0.0,
      "learning_rate": 2.6949512512606965e-08,
      "logps/chosen": -293.518928433642,
      "logps/rejected": -296.5833910205696,
      "loss": 0.2953,
      "rewards/chosen": 0.5612210874204282,
      "rewards/margins": 5.312669320336765,
      "rewards/rejected": -4.751448232916337,
      "step": 1620
    },
    {
      "epoch": 0.8560924369747899,
      "grad_norm": 35.09955655518953,
      "kl": 0.0,
      "learning_rate": 2.5116962059457653e-08,
      "logps/chosen": -307.30205078125,
      "logps/rejected": -280.090478515625,
      "loss": 0.2881,
      "rewards/chosen": 0.4744985580444336,
      "rewards/margins": 5.155851554870606,
      "rewards/rejected": -4.681352996826172,
      "step": 1630
    },
    {
      "epoch": 0.8613445378151261,
      "grad_norm": 100.20739187851474,
      "kl": 0.0,
      "learning_rate": 2.334563433804687e-08,
      "logps/chosen": -319.6615634704969,
      "logps/rejected": -282.31458824685535,
      "loss": 0.3014,
      "rewards/chosen": 0.6534406768609278,
      "rewards/margins": 4.888445989606604,
      "rewards/rejected": -4.235005312745676,
      "step": 1640
    },
    {
      "epoch": 0.8665966386554622,
      "grad_norm": 88.81840072824282,
      "kl": 0.0,
      "learning_rate": 2.1636011579150793e-08,
      "logps/chosen": -295.85146484375,
      "logps/rejected": -290.18388671875,
      "loss": 0.2796,
      "rewards/chosen": 0.5988120079040528,
      "rewards/margins": 4.9494441032409675,
      "rewards/rejected": -4.350632095336914,
      "step": 1650
    },
    {
      "epoch": 0.8718487394957983,
      "grad_norm": 41.43708238784869,
      "kl": 0.0,
      "learning_rate": 1.998855921482906e-08,
      "logps/chosen": -276.827294921875,
      "logps/rejected": -240.9879638671875,
      "loss": 0.3411,
      "rewards/chosen": 0.4913971424102783,
      "rewards/margins": 4.203098344802856,
      "rewards/rejected": -3.711701202392578,
      "step": 1660
    },
    {
      "epoch": 0.8771008403361344,
      "grad_norm": 61.90087346557062,
      "kl": 0.0,
      "learning_rate": 1.8403725751714615e-08,
      "logps/chosen": -287.8762538580247,
      "logps/rejected": -314.7857001582278,
      "loss": 0.2888,
      "rewards/chosen": 0.6117756690508054,
      "rewards/margins": 5.216582303495031,
      "rewards/rejected": -4.604806634444225,
      "step": 1670
    },
    {
      "epoch": 0.8823529411764706,
      "grad_norm": 43.47883434595641,
      "kl": 0.0,
      "learning_rate": 1.6881942648911074e-08,
      "logps/chosen": -286.86601796407183,
      "logps/rejected": -269.488791870915,
      "loss": 0.3423,
      "rewards/chosen": 0.3007767500277765,
      "rewards/margins": 4.408972534852123,
      "rewards/rejected": -4.108195784824346,
      "step": 1680
    },
    {
      "epoch": 0.8876050420168067,
      "grad_norm": 43.77865447701488,
      "kl": 0.0,
      "learning_rate": 1.5423624200531115e-08,
      "logps/chosen": -300.1749822443182,
      "logps/rejected": -305.69072808159723,
      "loss": 0.3209,
      "rewards/chosen": 0.8302790901877664,
      "rewards/margins": 4.638904205476395,
      "rewards/rejected": -3.8086251152886286,
      "step": 1690
    },
    {
      "epoch": 0.8928571428571429,
      "grad_norm": 126.94155434263403,
      "kl": 0.0,
      "learning_rate": 1.4029167422908105e-08,
      "logps/chosen": -279.4980215097403,
      "logps/rejected": -270.3898013930723,
      "loss": 0.3016,
      "rewards/chosen": 1.0889219804243608,
      "rewards/margins": 5.310351644392734,
      "rewards/rejected": -4.221429663968373,
      "step": 1700
    },
    {
      "epoch": 0.898109243697479,
      "grad_norm": 61.01512892000863,
      "kl": 0.0,
      "learning_rate": 1.2698951946511327e-08,
      "logps/chosen": -302.3543386399371,
      "logps/rejected": -267.2125873447205,
      "loss": 0.3007,
      "rewards/chosen": 0.34754651147614485,
      "rewards/margins": 4.579919861631424,
      "rewards/rejected": -4.23237335015528,
      "step": 1710
    },
    {
      "epoch": 0.9033613445378151,
      "grad_norm": 41.136022643204946,
      "kl": 0.0,
      "learning_rate": 1.1433339912594265e-08,
      "logps/chosen": -308.92118158284023,
      "logps/rejected": -271.45762624172187,
      "loss": 0.3021,
      "rewards/chosen": 0.45501975336018397,
      "rewards/margins": 5.495058411653852,
      "rewards/rejected": -5.040038658293668,
      "step": 1720
    },
    {
      "epoch": 0.9086134453781513,
      "grad_norm": 110.83416055708793,
      "kl": 0.0,
      "learning_rate": 1.0232675874604608e-08,
      "logps/chosen": -312.7164007867133,
      "logps/rejected": -318.51147598870057,
      "loss": 0.2863,
      "rewards/chosen": 0.45250099022071677,
      "rewards/margins": 4.343536339576296,
      "rewards/rejected": -3.891035349355579,
      "step": 1730
    },
    {
      "epoch": 0.9138655462184874,
      "grad_norm": 26.122066134245365,
      "kl": 0.0,
      "learning_rate": 9.097286704381896e-09,
      "logps/chosen": -315.04382560483873,
      "logps/rejected": -272.66723484848484,
      "loss": 0.3122,
      "rewards/chosen": 0.7614468482232863,
      "rewards/margins": 4.353459750685407,
      "rewards/rejected": -3.592012902462121,
      "step": 1740
    },
    {
      "epoch": 0.9191176470588235,
      "grad_norm": 53.34556224286923,
      "kl": 0.0,
      "learning_rate": 8.02748150316937e-09,
      "logps/chosen": -283.3781151107595,
      "logps/rejected": -270.81105324074076,
      "loss": 0.3131,
      "rewards/chosen": 0.739828278746786,
      "rewards/margins": 4.5300670241653815,
      "rewards/rejected": -3.7902387454185957,
      "step": 1750
    },
    {
      "epoch": 0.9243697478991597,
      "grad_norm": 87.26559549436006,
      "kl": 0.0,
      "learning_rate": 7.023551517463089e-09,
      "logps/chosen": -271.55817018072287,
      "logps/rejected": -290.5874594155844,
      "loss": 0.3368,
      "rewards/chosen": 0.3112575117363987,
      "rewards/margins": 3.797579691753647,
      "rewards/rejected": -3.4863221800172486,
      "step": 1760
    },
    {
      "epoch": 0.9296218487394958,
      "grad_norm": 31.56919055846343,
      "kl": 0.0,
      "learning_rate": 6.085770059722634e-09,
      "logps/chosen": -272.63416371855345,
      "logps/rejected": -279.8696380046584,
      "loss": 0.2818,
      "rewards/chosen": 0.9944836118686124,
      "rewards/margins": 5.193192850999047,
      "rewards/rejected": -4.198709239130435,
      "step": 1770
    },
    {
      "epoch": 0.9348739495798319,
      "grad_norm": 37.83348753574193,
      "kl": 0.0,
      "learning_rate": 5.214392433963488e-09,
      "logps/chosen": -287.8191769622093,
      "logps/rejected": -277.8159311655405,
      "loss": 0.3277,
      "rewards/chosen": 0.5125378453454306,
      "rewards/margins": 3.781902073465152,
      "rewards/rejected": -3.2693642281197213,
      "step": 1780
    },
    {
      "epoch": 0.9401260504201681,
      "grad_norm": 45.4550208744577,
      "kl": 0.0,
      "learning_rate": 4.409655866252693e-09,
      "logps/chosen": -282.79800397398844,
      "logps/rejected": -302.68510841836735,
      "loss": 0.3304,
      "rewards/chosen": 0.5359358153591266,
      "rewards/margins": 4.274966925850198,
      "rewards/rejected": -3.7390311104910716,
      "step": 1790
    },
    {
      "epoch": 0.9453781512605042,
      "grad_norm": 47.92999315794789,
      "kl": 0.0,
      "learning_rate": 3.671779440125644e-09,
      "logps/chosen": -299.54941281847135,
      "logps/rejected": -283.9407831671779,
      "loss": 0.3024,
      "rewards/chosen": 0.9072784618207603,
      "rewards/margins": 4.687179322753467,
      "rewards/rejected": -3.779900860932707,
      "step": 1800
    },
    {
      "epoch": 0.9506302521008403,
      "grad_norm": 32.60296033727532,
      "kl": 0.0,
      "learning_rate": 3.000964036942305e-09,
      "logps/chosen": -278.7902070063694,
      "logps/rejected": -298.68158071319016,
      "loss": 0.2903,
      "rewards/chosen": 0.4591269523474821,
      "rewards/margins": 4.919044079336555,
      "rewards/rejected": -4.4599171269890725,
      "step": 1810
    },
    {
      "epoch": 0.9558823529411765,
      "grad_norm": 38.232447498754475,
      "kl": 0.0,
      "learning_rate": 2.397392281198729e-09,
      "logps/chosen": -302.90917338709676,
      "logps/rejected": -267.12088068181816,
      "loss": 0.301,
      "rewards/chosen": 0.69748047859438,
      "rewards/margins": 4.834925731908774,
      "rewards/rejected": -4.137445253314394,
      "step": 1820
    },
    {
      "epoch": 0.9611344537815126,
      "grad_norm": 22.326350506185886,
      "kl": 0.0,
      "learning_rate": 1.861228490808886e-09,
      "logps/chosen": -294.7616077769886,
      "logps/rejected": -289.75987413194446,
      "loss": 0.3229,
      "rewards/chosen": 0.6278800530867144,
      "rewards/margins": 4.9228116767575045,
      "rewards/rejected": -4.29493162367079,
      "step": 1830
    },
    {
      "epoch": 0.9663865546218487,
      "grad_norm": 40.257301749189914,
      "kl": 0.0,
      "learning_rate": 1.3926186323703903e-09,
      "logps/chosen": -274.1588462271341,
      "logps/rejected": -264.75538361378204,
      "loss": 0.3362,
      "rewards/chosen": 0.5637630369605088,
      "rewards/margins": 4.09795015822954,
      "rewards/rejected": -3.5341871212690306,
      "step": 1840
    },
    {
      "epoch": 0.9716386554621849,
      "grad_norm": 52.95763177365781,
      "kl": 0.0,
      "learning_rate": 9.916902814261774e-10,
      "logps/chosen": -288.03719758064517,
      "logps/rejected": -290.255634469697,
      "loss": 0.271,
      "rewards/chosen": 0.5408120432207661,
      "rewards/margins": 5.058847746937622,
      "rewards/rejected": -4.518035703716856,
      "step": 1850
    },
    {
      "epoch": 0.976890756302521,
      "grad_norm": 48.81020773750272,
      "kl": 0.0,
      "learning_rate": 6.585525877328968e-10,
      "logps/chosen": -290.9931640625,
      "logps/rejected": -323.75256631540697,
      "loss": 0.3325,
      "rewards/chosen": 0.10595728899981524,
      "rewards/margins": 4.301412795790331,
      "rewards/rejected": -4.195455506790516,
      "step": 1860
    },
    {
      "epoch": 0.9821428571428571,
      "grad_norm": 35.96392604697334,
      "kl": 0.0,
      "learning_rate": 3.9329624554584883e-10,
      "logps/chosen": -281.2289627259036,
      "logps/rejected": -280.72377232142856,
      "loss": 0.3147,
      "rewards/chosen": 0.46981898847832737,
      "rewards/margins": 4.369042811530275,
      "rewards/rejected": -3.899223823051948,
      "step": 1870
    },
    {
      "epoch": 0.9873949579831933,
      "grad_norm": 98.21290759884454,
      "kl": 0.0,
      "learning_rate": 1.959934689280962e-10,
      "logps/chosen": -292.91434151785717,
      "logps/rejected": -292.6137952302632,
      "loss": 0.3104,
      "rewards/chosen": 0.6014651343936012,
      "rewards/margins": 4.848261591784637,
      "rewards/rejected": -4.246796457391036,
      "step": 1880
    },
    {
      "epoch": 0.9926470588235294,
      "grad_norm": 33.92572486964675,
      "kl": 0.0,
      "learning_rate": 6.669797209069017e-11,
      "logps/chosen": -269.0030942766854,
      "logps/rejected": -273.2933263644366,
      "loss": 0.3204,
      "rewards/chosen": 0.08876853042773986,
      "rewards/margins": 4.26088122367708,
      "rewards/rejected": -4.17211269324934,
      "step": 1890
    },
    {
      "epoch": 0.9978991596638656,
      "grad_norm": 57.704679882719425,
      "kl": 0.0,
      "learning_rate": 5.444954769395771e-12,
      "logps/chosen": -258.240974378882,
      "logps/rejected": -283.904309158805,
      "loss": 0.3031,
      "rewards/chosen": 0.38151166453865004,
      "rewards/margins": 4.77392472590854,
      "rewards/rejected": -4.39241306136989,
      "step": 1900
    },
    {
      "epoch": 1.0,
      "step": 1904,
      "total_flos": 0.0,
      "train_loss": 0.3297611388589154,
      "train_runtime": 9235.4216,
      "train_samples_per_second": 6.596,
      "train_steps_per_second": 0.206
    }
  ],
  "logging_steps": 10,
  "max_steps": 1904,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}