Model save

README.md

@@ -0,0 +1,59 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- trl
+- kto
+- generated_from_trainer
+model-index:
+- name: zephyr-7b-sft-kto2
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/nlee28/huggingface/runs/m7ew3rpy)
+# zephyr-7b-sft-kto2
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.42.4
+- Pytorch 2.1.2.post303
+- Datasets 2.18.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.3297611388589154,
+    "train_runtime": 9235.4216,
+    "train_samples": 60917,
+    "train_samples_per_second": 6.596,
+    "train_steps_per_second": 0.206
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.42.4"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cec96c3b58f53bcec7434705715a4902135cab39fcaabe586add185bb936026e
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4061e674fcf909c2cf8f98ff6c01ed06116f9274dc7c4e05af3e84a632010481
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ec300c77b863f3fa30c414cd49b86c48f01da0b4a3fd3a9b66549d6ffd43776
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Jul27_17-18-46_gpu-1/events.out.tfevents.1722068762.gpu-1.2291129.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a6dccc026e56f2b950625f16df28066e290381b3cdbbeb362a4745b28336a91
-size 110616
+oid sha256:aa9cf7cac131b1da4f7ff27ea8f1a7c4c35abdd6dd7fa28ab07ac6bf25a7f989
+size 110970

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.3297611388589154,
+    "train_runtime": 9235.4216,
+    "train_samples": 60917,
+    "train_samples_per_second": 6.596,
+    "train_steps_per_second": 0.206
+}

trainer_state.json

@@ -0,0 +1,2512 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1904,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.005252100840336135,
+      "grad_norm": 69.5775218105222,
+      "kl": 0.15183432400226593,
+      "learning_rate": 4.999659696812289e-07,
+      "logps/chosen": -298.5141715116279,
+      "logps/rejected": -259.0493032094595,
+      "loss": 0.4994,
+      "rewards/chosen": -0.04585062625796296,
+      "rewards/margins": 0.01403944421908452,
+      "rewards/rejected": -0.05989007047704748,
+      "step": 10
+    },
+    {
+      "epoch": 0.01050420168067227,
+      "grad_norm": 55.49240989732535,
+      "kl": 2.105862855911255,
+      "learning_rate": 4.998638879894165e-07,
+      "logps/chosen": -273.6553505345395,
+      "logps/rejected": -227.15778459821428,
+      "loss": 0.4776,
+      "rewards/chosen": 0.49514409115439967,
+      "rewards/margins": 0.21992081329039764,
+      "rewards/rejected": 0.275223277864002,
+      "step": 20
+    },
+    {
+      "epoch": 0.015756302521008403,
+      "grad_norm": 51.28980044595933,
+      "kl": 0.28644976019859314,
+      "learning_rate": 4.996937827155428e-07,
+      "logps/chosen": -311.9908854166667,
+      "logps/rejected": -247.46916946308724,
+      "loss": 0.4486,
+      "rewards/chosen": 0.30754662675467154,
+      "rewards/margins": 0.5134327379181245,
+      "rewards/rejected": -0.20588611116345296,
+      "step": 30
+    },
+    {
+      "epoch": 0.02100840336134454,
+      "grad_norm": 53.39172095489608,
+      "kl": 0.0,
+      "learning_rate": 4.994557001695013e-07,
+      "logps/chosen": -298.2471417682927,
+      "logps/rejected": -238.9893078926282,
+      "loss": 0.4172,
+      "rewards/chosen": 0.01273148815806319,
+      "rewards/margins": 0.8681685468269335,
+      "rewards/rejected": -0.8554370586688702,
+      "step": 40
+    },
+    {
+      "epoch": 0.026260504201680673,
+      "grad_norm": 48.3236105129346,
+      "kl": 0.0,
+      "learning_rate": 4.991497051674917e-07,
+      "logps/chosen": -289.8418828616352,
+      "logps/rejected": -264.62730493012424,
+      "loss": 0.3922,
+      "rewards/chosen": 0.3131725623172784,
+      "rewards/margins": 1.3170881950509257,
+      "rewards/rejected": -1.0039156327336471,
+      "step": 50
+    },
+    {
+      "epoch": 0.031512605042016806,
+      "grad_norm": 51.661072159206874,
+      "kl": 0.0,
+      "learning_rate": 4.987758810143735e-07,
+      "logps/chosen": -331.279736328125,
+      "logps/rejected": -255.901513671875,
+      "loss": 0.3954,
+      "rewards/chosen": 0.22145886421203614,
+      "rewards/margins": 1.2238109111785889,
+      "rewards/rejected": -1.0023520469665528,
+      "step": 60
+    },
+    {
+      "epoch": 0.03676470588235294,
+      "grad_norm": 48.98498102999305,
+      "kl": 0.0,
+      "learning_rate": 4.983343294809874e-07,
+      "logps/chosen": -309.5669285973837,
+      "logps/rejected": -255.4453125,
+      "loss": 0.3732,
+      "rewards/chosen": 0.5395132552745731,
+      "rewards/margins": 1.6830449514910533,
+      "rewards/rejected": -1.14353169621648,
+      "step": 70
+    },
+    {
+      "epoch": 0.04201680672268908,
+      "grad_norm": 49.52520073364695,
+      "kl": 0.0,
+      "learning_rate": 4.978251707764491e-07,
+      "logps/chosen": -298.83478282232704,
+      "logps/rejected": -262.3364712732919,
+      "loss": 0.3542,
+      "rewards/chosen": 0.41468177651459315,
+      "rewards/margins": 2.0195703134298686,
+      "rewards/rejected": -1.6048885369152757,
+      "step": 80
+    },
+    {
+      "epoch": 0.04726890756302521,
+      "grad_norm": 45.84438443510472,
+      "kl": 0.0,
+      "learning_rate": 4.972485435154228e-07,
+      "logps/chosen": -284.00230493012424,
+      "logps/rejected": -292.5305866745283,
+      "loss": 0.3424,
+      "rewards/chosen": 0.20849199472747235,
+      "rewards/margins": 2.4833206356472837,
+      "rewards/rejected": -2.2748286409198113,
+      "step": 90
+    },
+    {
+      "epoch": 0.052521008403361345,
+      "grad_norm": 51.90503609381182,
+      "kl": 0.0,
+      "learning_rate": 4.966046046803842e-07,
+      "logps/chosen": -305.3333748009554,
+      "logps/rejected": -265.70988305214723,
+      "loss": 0.3825,
+      "rewards/chosen": 0.518976345183743,
+      "rewards/margins": 2.160451905658724,
+      "rewards/rejected": -1.6414755604749809,
+      "step": 100
+    },
+    {
+      "epoch": 0.05777310924369748,
+      "grad_norm": 46.615982508495186,
+      "kl": 0.0,
+      "learning_rate": 4.958935295788841e-07,
+      "logps/chosen": -346.7928911423841,
+      "logps/rejected": -282.3527181952663,
+      "loss": 0.3829,
+      "rewards/chosen": 0.18162511358197952,
+      "rewards/margins": 1.8821555777091985,
+      "rewards/rejected": -1.700530464127219,
+      "step": 110
+    },
+    {
+      "epoch": 0.06302521008403361,
+      "grad_norm": 43.56170699177104,
+      "kl": 0.0,
+      "learning_rate": 4.951155117958216e-07,
+      "logps/chosen": -299.7170380015432,
+      "logps/rejected": -281.8764092167722,
+      "loss": 0.3787,
+      "rewards/chosen": -0.607381797131197,
+      "rewards/margins": 2.2764308295746973,
+      "rewards/rejected": -2.883812626705894,
+      "step": 120
+    },
+    {
+      "epoch": 0.06827731092436974,
+      "grad_norm": 52.35577489618577,
+      "kl": 0.0,
+      "learning_rate": 4.942707631407419e-07,
+      "logps/chosen": -335.1082099780702,
+      "logps/rejected": -281.61443477348996,
+      "loss": 0.3757,
+      "rewards/chosen": -0.22571238021404422,
+      "rewards/margins": 2.2212163225308172,
+      "rewards/rejected": -2.4469287027448616,
+      "step": 130
+    },
+    {
+      "epoch": 0.07352941176470588,
+      "grad_norm": 28.41591231440036,
+      "kl": 0.0,
+      "learning_rate": 4.933595135901732e-07,
+      "logps/chosen": -318.5261247783688,
+      "logps/rejected": -269.2608676675978,
+      "loss": 0.3369,
+      "rewards/chosen": 0.26553474588597076,
+      "rewards/margins": 2.99381714157382,
+      "rewards/rejected": -2.7282823956878492,
+      "step": 140
+    },
+    {
+      "epoch": 0.07878151260504201,
+      "grad_norm": 77.73773489117062,
+      "kl": 0.0,
+      "learning_rate": 4.923820112250169e-07,
+      "logps/chosen": -301.63650901845637,
+      "logps/rejected": -266.3384959795322,
+      "loss": 0.3375,
+      "rewards/chosen": 0.025881773673447985,
+      "rewards/margins": 3.2468575937575124,
+      "rewards/rejected": -3.2209758200840644,
+      "step": 150
+    },
+    {
+      "epoch": 0.08403361344537816,
+      "grad_norm": 37.575572316994865,
+      "kl": 0.0,
+      "learning_rate": 4.913385221630096e-07,
+      "logps/chosen": -250.38438482704402,
+      "logps/rejected": -265.8534064440994,
+      "loss": 0.3758,
+      "rewards/chosen": 0.7617762343688581,
+      "rewards/margins": 1.6022024720556787,
+      "rewards/rejected": -0.8404262376868207,
+      "step": 160
+    },
+    {
+      "epoch": 0.08928571428571429,
+      "grad_norm": 40.501974364358205,
+      "kl": 0.0,
+      "learning_rate": 4.902293304862749e-07,
+      "logps/chosen": -270.815234375,
+      "logps/rejected": -255.6860107421875,
+      "loss": 0.3672,
+      "rewards/chosen": 0.9947892189025879,
+      "rewards/margins": 1.629128646850586,
+      "rewards/rejected": -0.634339427947998,
+      "step": 170
+    },
+    {
+      "epoch": 0.09453781512605042,
+      "grad_norm": 36.96138019330735,
+      "kl": 0.0,
+      "learning_rate": 4.890547381639833e-07,
+      "logps/chosen": -299.6106373856707,
+      "logps/rejected": -230.88887219551282,
+      "loss": 0.3625,
+      "rewards/chosen": 0.3851209733544326,
+      "rewards/margins": 2.033356000961103,
+      "rewards/rejected": -1.6482350276066706,
+      "step": 180
+    },
+    {
+      "epoch": 0.09978991596638656,
+      "grad_norm": 44.46735174580667,
+      "kl": 0.0,
+      "learning_rate": 4.878150649701439e-07,
+      "logps/chosen": -344.7996875,
+      "logps/rejected": -265.18083639705884,
+      "loss": 0.3185,
+      "rewards/chosen": 0.36289586385091144,
+      "rewards/margins": 2.9504150719736133,
+      "rewards/rejected": -2.587519208122702,
+      "step": 190
+    },
+    {
+      "epoch": 0.10504201680672269,
+      "grad_norm": 52.28178000802996,
+      "kl": 0.0,
+      "learning_rate": 4.865106483965486e-07,
+      "logps/chosen": -292.6435797275641,
+      "logps/rejected": -240.90084317835365,
+      "loss": 0.3407,
+      "rewards/chosen": -0.03864570764394907,
+      "rewards/margins": 2.6535366936875104,
+      "rewards/rejected": -2.6921824013314595,
+      "step": 200
+    },
+    {
+      "epoch": 0.11029411764705882,
+      "grad_norm": 42.51233566429091,
+      "kl": 0.0,
+      "learning_rate": 4.851418435608919e-07,
+      "logps/chosen": -300.3898746468927,
+      "logps/rejected": -269.88991477272725,
+      "loss": 0.3554,
+      "rewards/chosen": 0.8636472023139565,
+      "rewards/margins": 2.70990092417803,
+      "rewards/rejected": -1.8462537218640733,
+      "step": 210
+    },
+    {
+      "epoch": 0.11554621848739496,
+      "grad_norm": 45.269264573206264,
+      "kl": 0.0,
+      "learning_rate": 4.837090231100927e-07,
+      "logps/chosen": -315.7438054733728,
+      "logps/rejected": -242.41240687086093,
+      "loss": 0.327,
+      "rewards/chosen": 1.163497202495146,
+      "rewards/margins": 2.8938327665084946,
+      "rewards/rejected": -1.7303355640133484,
+      "step": 220
+    },
+    {
+      "epoch": 0.1207983193277311,
+      "grad_norm": 46.30035764458271,
+      "kl": 0.0,
+      "learning_rate": 4.822125771188448e-07,
+      "logps/chosen": -285.65898944805195,
+      "logps/rejected": -271.8729762801205,
+      "loss": 0.3828,
+      "rewards/chosen": 0.5068141392299107,
+      "rewards/margins": 1.911893440811236,
+      "rewards/rejected": -1.4050793015813252,
+      "step": 230
+    },
+    {
+      "epoch": 0.12605042016806722,
+      "grad_norm": 44.08666008078037,
+      "kl": 0.0,
+      "learning_rate": 4.806529129834207e-07,
+      "logps/chosen": -280.8894210188356,
+      "logps/rejected": -270.21331716954023,
+      "loss": 0.329,
+      "rewards/chosen": 0.5693619087950824,
+      "rewards/margins": 2.748881386727015,
+      "rewards/rejected": -2.1795194779319327,
+      "step": 240
+    },
+    {
+      "epoch": 0.13130252100840337,
+      "grad_norm": 38.010572045662606,
+      "kl": 0.0,
+      "learning_rate": 4.790304553107622e-07,
+      "logps/chosen": -304.2262851331361,
+      "logps/rejected": -254.57090749172184,
+      "loss": 0.3588,
+      "rewards/chosen": 0.49599551590236685,
+      "rewards/margins": 2.326870937893778,
+      "rewards/rejected": -1.8308754219914114,
+      "step": 250
+    },
+    {
+      "epoch": 0.13655462184873948,
+      "grad_norm": 44.14238619732289,
+      "kl": 0.0,
+      "learning_rate": 4.773456458028837e-07,
+      "logps/chosen": -278.3834842754777,
+      "logps/rejected": -268.8938362730061,
+      "loss": 0.3378,
+      "rewards/chosen": 0.5815278071506768,
+      "rewards/margins": 2.6120607376769422,
+      "rewards/rejected": -2.0305329305262654,
+      "step": 260
+    },
+    {
+      "epoch": 0.14180672268907563,
+      "grad_norm": 39.61832047489729,
+      "kl": 1.3655751943588257,
+      "learning_rate": 4.755989431366221e-07,
+      "logps/chosen": -301.3936798878205,
+      "logps/rejected": -299.8501572027439,
+      "loss": 0.3287,
+      "rewards/chosen": 0.6345734229454627,
+      "rewards/margins": 2.3345022648852494,
+      "rewards/rejected": -1.6999288419397867,
+      "step": 270
+    },
+    {
+      "epoch": 0.14705882352941177,
+      "grad_norm": 43.29173967453139,
+      "kl": 0.0,
+      "learning_rate": 4.737908228387656e-07,
+      "logps/chosen": -294.7026832217262,
+      "logps/rejected": -264.49424342105266,
+      "loss": 0.3553,
+      "rewards/chosen": 0.5891142799740746,
+      "rewards/margins": 2.708873086704646,
+      "rewards/rejected": -2.1197588067305717,
+      "step": 280
+    },
+    {
+      "epoch": 0.15231092436974789,
+      "grad_norm": 49.48779153565427,
+      "kl": 0.0,
+      "learning_rate": 4.7192177715659516e-07,
+      "logps/chosen": -290.1478293413174,
+      "logps/rejected": -243.8566176470588,
+      "loss": 0.3763,
+      "rewards/chosen": 0.392075178865901,
+      "rewards/margins": 2.094171158045333,
+      "rewards/rejected": -1.7020959791794321,
+      "step": 290
+    },
+    {
+      "epoch": 0.15756302521008403,
+      "grad_norm": 44.86612305681867,
+      "kl": 0.0,
+      "learning_rate": 4.699923149238736e-07,
+      "logps/chosen": -283.2191996402878,
+      "logps/rejected": -272.7527192679558,
+      "loss": 0.3369,
+      "rewards/chosen": 0.6118597675570481,
+      "rewards/margins": 2.6730571222057016,
+      "rewards/rejected": -2.0611973546486535,
+      "step": 300
+    },
+    {
+      "epoch": 0.16281512605042017,
+      "grad_norm": 43.00617930560404,
+      "kl": 0.0,
+      "learning_rate": 4.680029614223198e-07,
+      "logps/chosen": -272.5205015923567,
+      "logps/rejected": -266.77614551380367,
+      "loss": 0.3531,
+      "rewards/chosen": 0.13305815921467581,
+      "rewards/margins": 2.627990556226275,
+      "rewards/rejected": -2.494932397011599,
+      "step": 310
+    },
+    {
+      "epoch": 0.16806722689075632,
+      "grad_norm": 39.410413100307075,
+      "kl": 0.0,
+      "learning_rate": 4.65954258238604e-07,
+      "logps/chosen": -288.9263426272455,
+      "logps/rejected": -254.3640216503268,
+      "loss": 0.3736,
+      "rewards/chosen": 0.09242550056137724,
+      "rewards/margins": 2.2175613337032276,
+      "rewards/rejected": -2.1251358331418504,
+      "step": 320
+    },
+    {
+      "epoch": 0.17331932773109243,
+      "grad_norm": 41.100451094010424,
+      "kl": 0.0,
+      "learning_rate": 4.638467631169056e-07,
+      "logps/chosen": -326.0165550595238,
+      "logps/rejected": -306.81566097861844,
+      "loss": 0.318,
+      "rewards/chosen": 0.5781731832595098,
+      "rewards/margins": 3.290003924740287,
+      "rewards/rejected": -2.711830741480777,
+      "step": 330
+    },
+    {
+      "epoch": 0.17857142857142858,
+      "grad_norm": 59.056271025186184,
+      "kl": 0.0,
+      "learning_rate": 4.6168104980707103e-07,
+      "logps/chosen": -285.7653391768293,
+      "logps/rejected": -274.3047876602564,
+      "loss": 0.347,
+      "rewards/chosen": -0.003299643353718083,
+      "rewards/margins": 3.289772396910705,
+      "rewards/rejected": -3.293072040264423,
+      "step": 340
+    },
+    {
+      "epoch": 0.18382352941176472,
+      "grad_norm": 40.5420682235125,
+      "kl": 0.0,
+      "learning_rate": 4.594577079084145e-07,
+      "logps/chosen": -290.82744140625,
+      "logps/rejected": -292.7601318359375,
+      "loss": 0.3356,
+      "rewards/chosen": 0.4554912567138672,
+      "rewards/margins": 3.3385635375976563,
+      "rewards/rejected": -2.883072280883789,
+      "step": 350
+    },
+    {
+      "epoch": 0.18907563025210083,
+      "grad_norm": 73.25856842000852,
+      "kl": 0.0,
+      "learning_rate": 4.5717734270920466e-07,
+      "logps/chosen": -281.4008819018405,
+      "logps/rejected": -228.66883957006368,
+      "loss": 0.3378,
+      "rewards/chosen": 0.49251336407807705,
+      "rewards/margins": 2.78153659942342,
+      "rewards/rejected": -2.2890232353453426,
+      "step": 360
+    },
+    {
+      "epoch": 0.19432773109243698,
+      "grad_norm": 36.80227868403581,
+      "kl": 0.0,
+      "learning_rate": 4.548405750218785e-07,
+      "logps/chosen": -288.27572185672517,
+      "logps/rejected": -269.6975146812081,
+      "loss": 0.3552,
+      "rewards/chosen": -0.02239043949640285,
+      "rewards/margins": 3.2017408950516257,
+      "rewards/rejected": -3.2241313345480287,
+      "step": 370
+    },
+    {
+      "epoch": 0.19957983193277312,
+      "grad_norm": 52.176625363111675,
+      "kl": 0.0,
+      "learning_rate": 4.5244804101403025e-07,
+      "logps/chosen": -262.20579637096773,
+      "logps/rejected": -253.31008522727274,
+      "loss": 0.3773,
+      "rewards/chosen": -0.5523546772618447,
+      "rewards/margins": 2.2483803339834205,
+      "rewards/rejected": -2.800735011245265,
+      "step": 380
+    },
+    {
+      "epoch": 0.20483193277310924,
+      "grad_norm": 40.91805541297444,
+      "kl": 0.0,
+      "learning_rate": 4.5000039203521976e-07,
+      "logps/chosen": -305.74300986842104,
+      "logps/rejected": -303.6615048363095,
+      "loss": 0.3428,
+      "rewards/chosen": 0.08902386615150853,
+      "rewards/margins": 3.217569718086032,
+      "rewards/rejected": -3.1285458519345237,
+      "step": 390
+    },
+    {
+      "epoch": 0.21008403361344538,
+      "grad_norm": 48.39763770048436,
+      "kl": 0.0,
+      "learning_rate": 4.47498294439647e-07,
+      "logps/chosen": -293.79254518072287,
+      "logps/rejected": -266.05382508116884,
+      "loss": 0.3273,
+      "rewards/chosen": 0.20805818488798947,
+      "rewards/margins": 2.784955744115871,
+      "rewards/rejected": -2.5768975592278816,
+      "step": 400
+    },
+    {
+      "epoch": 0.21533613445378152,
+      "grad_norm": 64.69719531344766,
+      "kl": 0.0,
+      "learning_rate": 4.449424294047419e-07,
+      "logps/chosen": -298.2714004984663,
+      "logps/rejected": -277.8911972531847,
+      "loss": 0.3478,
+      "rewards/chosen": 0.7857334628426955,
+      "rewards/margins": 2.803303423904598,
+      "rewards/rejected": -2.017569961061903,
+      "step": 410
+    },
+    {
+      "epoch": 0.22058823529411764,
+      "grad_norm": 41.5548039641909,
+      "kl": 0.0,
+      "learning_rate": 4.4233349274571974e-07,
+      "logps/chosen": -294.40205536912754,
+      "logps/rejected": -262.53922240497076,
+      "loss": 0.3273,
+      "rewards/chosen": 0.6274464754450241,
+      "rewards/margins": 3.27400759055376,
+      "rewards/rejected": -2.6465611151087356,
+      "step": 420
+    },
+    {
+      "epoch": 0.22584033613445378,
+      "grad_norm": 51.90314639757878,
+      "kl": 0.0,
+      "learning_rate": 4.396721947261496e-07,
+      "logps/chosen": -290.494140625,
+      "logps/rejected": -266.31001420454544,
+      "loss": 0.3513,
+      "rewards/chosen": 0.22074366190347328,
+      "rewards/margins": 2.942957208854569,
+      "rewards/rejected": -2.722213546951096,
+      "step": 430
+    },
+    {
+      "epoch": 0.23109243697478993,
+      "grad_norm": 45.08774574606674,
+      "kl": 0.0,
+      "learning_rate": 4.3695925986459107e-07,
+      "logps/chosen": -274.8046160442073,
+      "logps/rejected": -247.5850110176282,
+      "loss": 0.3506,
+      "rewards/chosen": 0.5506789044636052,
+      "rewards/margins": 3.319194662488946,
+      "rewards/rejected": -2.7685157580253406,
+      "step": 440
+    },
+    {
+      "epoch": 0.23634453781512604,
+      "grad_norm": 38.43146082353361,
+      "kl": 0.0,
+      "learning_rate": 4.341954267373494e-07,
+      "logps/chosen": -293.00198951863354,
+      "logps/rejected": -258.1283411949685,
+      "loss": 0.3646,
+      "rewards/chosen": 0.5552752477041683,
+      "rewards/margins": 2.609661794480898,
+      "rewards/rejected": -2.0543865467767297,
+      "step": 450
+    },
+    {
+      "epoch": 0.2415966386554622,
+      "grad_norm": 42.36936446062343,
+      "kl": 0.0,
+      "learning_rate": 4.313814477774035e-07,
+      "logps/chosen": -307.16541371855345,
+      "logps/rejected": -266.2528872282609,
+      "loss": 0.3188,
+      "rewards/chosen": 0.6136390638051543,
+      "rewards/margins": 3.2912566758975457,
+      "rewards/rejected": -2.677617612092391,
+      "step": 460
+    },
+    {
+      "epoch": 0.24684873949579833,
+      "grad_norm": 36.14877170239455,
+      "kl": 0.0,
+      "learning_rate": 4.2851808906956134e-07,
+      "logps/chosen": -299.0251017011834,
+      "logps/rejected": -283.08899006622516,
+      "loss": 0.3248,
+      "rewards/chosen": 0.8636552257650703,
+      "rewards/margins": 4.432308458769002,
+      "rewards/rejected": -3.568653233003932,
+      "step": 470
+    },
+    {
+      "epoch": 0.25210084033613445,
+      "grad_norm": 48.52314483845558,
+      "kl": 0.0,
+      "learning_rate": 4.256061301418996e-07,
+      "logps/chosen": -288.11904978197674,
+      "logps/rejected": -273.0204022381757,
+      "loss": 0.3566,
+      "rewards/chosen": 0.09835364097772642,
+      "rewards/margins": 3.3749882696860247,
+      "rewards/rejected": -3.2766346287082984,
+      "step": 480
+    },
+    {
+      "epoch": 0.25735294117647056,
+      "grad_norm": 40.328590442957,
+      "kl": 0.0,
+      "learning_rate": 4.2264636375354283e-07,
+      "logps/chosen": -290.32015931372547,
+      "logps/rejected": -256.6384496631737,
+      "loss": 0.3385,
+      "rewards/chosen": 0.6565687890146293,
+      "rewards/margins": 2.809185004337889,
+      "rewards/rejected": -2.1526162153232598,
+      "step": 490
+    },
+    {
+      "epoch": 0.26260504201680673,
+      "grad_norm": 51.22796460390333,
+      "kl": 0.0,
+      "learning_rate": 4.1963959567884045e-07,
+      "logps/chosen": -302.9179159628378,
+      "logps/rejected": -269.88603742732556,
+      "loss": 0.3213,
+      "rewards/chosen": 1.2393865843076963,
+      "rewards/margins": 3.752552790135276,
+      "rewards/rejected": -2.51316620582758,
+      "step": 500
+    },
+    {
+      "epoch": 0.26785714285714285,
+      "grad_norm": 40.763747675132,
+      "kl": 0.0,
+      "learning_rate": 4.1658664448800094e-07,
+      "logps/chosen": -278.07726258116884,
+      "logps/rejected": -242.03510918674698,
+      "loss": 0.3613,
+      "rewards/chosen": 1.3944342476981026,
+      "rewards/margins": 2.644719200987824,
+      "rewards/rejected": -1.2502849532897213,
+      "step": 510
+    },
+    {
+      "epoch": 0.27310924369747897,
+      "grad_norm": 41.37886905858638,
+      "kl": 0.0,
+      "learning_rate": 4.1348834132424204e-07,
+      "logps/chosen": -303.84667354559747,
+      "logps/rejected": -279.02011354813664,
+      "loss": 0.355,
+      "rewards/chosen": 0.7681676516742826,
+      "rewards/margins": 3.026760719293654,
+      "rewards/rejected": -2.2585930676193713,
+      "step": 520
+    },
+    {
+      "epoch": 0.27836134453781514,
+      "grad_norm": 37.134601685804,
+      "kl": 0.0,
+      "learning_rate": 4.103455296775181e-07,
+      "logps/chosen": -311.4682323042169,
+      "logps/rejected": -256.0505783279221,
+      "loss": 0.3388,
+      "rewards/chosen": 1.0605972473879894,
+      "rewards/margins": 2.909687229061589,
+      "rewards/rejected": -1.8490899816736,
+      "step": 530
+    },
+    {
+      "epoch": 0.28361344537815125,
+      "grad_norm": 50.21606157716232,
+      "kl": 0.0,
+      "learning_rate": 4.071590651548867e-07,
+      "logps/chosen": -282.52527573529414,
+      "logps/rejected": -283.90199288922156,
+      "loss": 0.3322,
+      "rewards/chosen": 0.6519086251851001,
+      "rewards/margins": 3.288977687743577,
+      "rewards/rejected": -2.637069062558477,
+      "step": 540
+    },
+    {
+      "epoch": 0.28886554621848737,
+      "grad_norm": 40.94800472377613,
+      "kl": 0.0,
+      "learning_rate": 4.039298152475754e-07,
+      "logps/chosen": -296.5549363057325,
+      "logps/rejected": -296.0996932515337,
+      "loss": 0.3261,
+      "rewards/chosen": 0.4167486786083051,
+      "rewards/margins": 3.556985000742124,
+      "rewards/rejected": -3.140236322133819,
+      "step": 550
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 47.7284966576563,
+      "kl": 0.0,
+      "learning_rate": 4.006586590948141e-07,
+      "logps/chosen": -287.6368415880503,
+      "logps/rejected": -276.68611704192546,
+      "loss": 0.3348,
+      "rewards/chosen": -0.10697765470300831,
+      "rewards/margins": 3.6730512630716423,
+      "rewards/rejected": -3.7800289177746507,
+      "step": 560
+    },
+    {
+      "epoch": 0.29936974789915966,
+      "grad_norm": 45.50328767344306,
+      "kl": 0.0,
+      "learning_rate": 3.973464872444958e-07,
+      "logps/chosen": -296.46134996118013,
+      "logps/rejected": -281.4715507075472,
+      "loss": 0.3322,
+      "rewards/chosen": 0.1290158692354001,
+      "rewards/margins": 3.9525375132998652,
+      "rewards/rejected": -3.8235216440644653,
+      "step": 570
+    },
+    {
+      "epoch": 0.30462184873949577,
+      "grad_norm": 38.47971512593867,
+      "kl": 0.0,
+      "learning_rate": 3.939942014107318e-07,
+      "logps/chosen": -262.15194610778445,
+      "logps/rejected": -292.0669424019608,
+      "loss": 0.3594,
+      "rewards/chosen": 0.3589071856287425,
+      "rewards/margins": 3.160983936739343,
+      "rewards/rejected": -2.8020767511106004,
+      "step": 580
+    },
+    {
+      "epoch": 0.30987394957983194,
+      "grad_norm": 34.207797203706725,
+      "kl": 0.0,
+      "learning_rate": 3.9060271422836624e-07,
+      "logps/chosen": -292.61703330592104,
+      "logps/rejected": -271.3793247767857,
+      "loss": 0.3159,
+      "rewards/chosen": 1.2860569201017682,
+      "rewards/margins": 4.367648571654968,
+      "rewards/rejected": -3.0815916515531994,
+      "step": 590
+    },
+    {
+      "epoch": 0.31512605042016806,
+      "grad_norm": 57.39149614591673,
+      "kl": 0.0,
+      "learning_rate": 3.871729490045185e-07,
+      "logps/chosen": -290.49703584558824,
+      "logps/rejected": -252.63739583333333,
+      "loss": 0.3164,
+      "rewards/chosen": 0.8580891328699448,
+      "rewards/margins": 3.7738939831303617,
+      "rewards/rejected": -2.915804850260417,
+      "step": 600
+    },
+    {
+      "epoch": 0.32037815126050423,
+      "grad_norm": 40.37544580915199,
+      "kl": 0.0,
+      "learning_rate": 3.837058394672196e-07,
+      "logps/chosen": -273.4779296875,
+      "logps/rejected": -287.0096923828125,
+      "loss": 0.3289,
+      "rewards/chosen": 0.3099039077758789,
+      "rewards/margins": 3.173082160949707,
+      "rewards/rejected": -2.863178253173828,
+      "step": 610
+    },
+    {
+      "epoch": 0.32563025210084034,
+      "grad_norm": 57.45015573264363,
+      "kl": 0.0,
+      "learning_rate": 3.8020232951121166e-07,
+      "logps/chosen": -290.50161637931035,
+      "logps/rejected": -294.233125,
+      "loss": 0.3188,
+      "rewards/chosen": 0.1405291984821188,
+      "rewards/margins": 3.6642485065178327,
+      "rewards/rejected": -3.523719308035714,
+      "step": 620
+    },
+    {
+      "epoch": 0.33088235294117646,
+      "grad_norm": 39.287989744489565,
+      "kl": 0.0,
+      "learning_rate": 3.7666337294097985e-07,
+      "logps/chosen": -302.1064724392361,
+      "logps/rejected": -260.4077814275568,
+      "loss": 0.3435,
+      "rewards/chosen": -0.013078765736685859,
+      "rewards/margins": 3.530285672406958,
+      "rewards/rejected": -3.5433644381436435,
+      "step": 630
+    },
+    {
+      "epoch": 0.33613445378151263,
+      "grad_norm": 44.59203672342995,
+      "kl": 0.0,
+      "learning_rate": 3.730899332110855e-07,
+      "logps/chosen": -281.5672392003676,
+      "logps/rejected": -294.1050866168478,
+      "loss": 0.2851,
+      "rewards/chosen": 0.49381469277774587,
+      "rewards/margins": 5.133490330727814,
+      "rewards/rejected": -4.639675637950068,
+      "step": 640
+    },
+    {
+      "epoch": 0.34138655462184875,
+      "grad_norm": 41.26338033254354,
+      "kl": 0.0,
+      "learning_rate": 3.694829831638738e-07,
+      "logps/chosen": -272.1574797453704,
+      "logps/rejected": -303.05520668512656,
+      "loss": 0.3318,
+      "rewards/chosen": -0.2948492309193552,
+      "rewards/margins": 3.4522689825893473,
+      "rewards/rejected": -3.7471182135087027,
+      "step": 650
+    },
+    {
+      "epoch": 0.34663865546218486,
+      "grad_norm": 53.39190760647335,
+      "kl": 0.0,
+      "learning_rate": 3.658435047646238e-07,
+      "logps/chosen": -281.7225378787879,
+      "logps/rejected": -267.5343245967742,
+      "loss": 0.3109,
+      "rewards/chosen": 0.5971468838778409,
+      "rewards/margins": 4.032071987834494,
+      "rewards/rejected": -3.4349251039566533,
+      "step": 660
+    },
+    {
+      "epoch": 0.35189075630252103,
+      "grad_norm": 50.14394406023911,
+      "kl": 0.0,
+      "learning_rate": 3.621724888342161e-07,
+      "logps/chosen": -307.17963248239437,
+      "logps/rejected": -244.93414238061797,
+      "loss": 0.3456,
+      "rewards/chosen": 0.5407970052369884,
+      "rewards/margins": 2.9319001711099695,
+      "rewards/rejected": -2.391103165872981,
+      "step": 670
+    },
+    {
+      "epoch": 0.35714285714285715,
+      "grad_norm": 44.42377974125605,
+      "kl": 0.0,
+      "learning_rate": 3.584709347793895e-07,
+      "logps/chosen": -328.5807938664596,
+      "logps/rejected": -273.77439563679246,
+      "loss": 0.3028,
+      "rewards/chosen": 0.6922078103012179,
+      "rewards/margins": 4.292771099535692,
+      "rewards/rejected": -3.6005632892344734,
+      "step": 680
+    },
+    {
+      "epoch": 0.36239495798319327,
+      "grad_norm": 64.33577601812173,
+      "kl": 0.0,
+      "learning_rate": 3.5473985032065946e-07,
+      "logps/chosen": -339.54715008802816,
+      "logps/rejected": -288.18354985955057,
+      "loss": 0.3051,
+      "rewards/chosen": 0.7634544372558594,
+      "rewards/margins": 4.167792459552208,
+      "rewards/rejected": -3.4043380222963484,
+      "step": 690
+    },
+    {
+      "epoch": 0.36764705882352944,
+      "grad_norm": 50.09740849242821,
+      "kl": 0.0,
+      "learning_rate": 3.509802512179737e-07,
+      "logps/chosen": -320.1032510080645,
+      "logps/rejected": -247.99145359848484,
+      "loss": 0.3144,
+      "rewards/chosen": -0.7038151402627268,
+      "rewards/margins": 4.308948235684243,
+      "rewards/rejected": -5.01276337594697,
+      "step": 700
+    },
+    {
+      "epoch": 0.37289915966386555,
+      "grad_norm": 48.91753923946882,
+      "kl": 0.0,
+      "learning_rate": 3.4719316099417983e-07,
+      "logps/chosen": -319.73927600931677,
+      "logps/rejected": -308.56738895440253,
+      "loss": 0.3676,
+      "rewards/chosen": -1.7911930439635093,
+      "rewards/margins": 3.521101202302214,
+      "rewards/rejected": -5.312294246265723,
+      "step": 710
+    },
+    {
+      "epoch": 0.37815126050420167,
+      "grad_norm": 58.85183781985146,
+      "kl": 0.0,
+      "learning_rate": 3.4337961065637786e-07,
+      "logps/chosen": -351.19349449685535,
+      "logps/rejected": -328.6528289984472,
+      "loss": 0.3281,
+      "rewards/chosen": -0.32832849850444673,
+      "rewards/margins": 4.381978936714885,
+      "rewards/rejected": -4.710307435219332,
+      "step": 720
+    },
+    {
+      "epoch": 0.38340336134453784,
+      "grad_norm": 34.12584073424692,
+      "kl": 0.0,
+      "learning_rate": 3.395406384152371e-07,
+      "logps/chosen": -318.7544778963415,
+      "logps/rejected": -281.5553635817308,
+      "loss": 0.3211,
+      "rewards/chosen": 0.22548980247683642,
+      "rewards/margins": 3.761472765843223,
+      "rewards/rejected": -3.5359829633663864,
+      "step": 730
+    },
+    {
+      "epoch": 0.38865546218487396,
+      "grad_norm": 66.4865558017911,
+      "kl": 0.0,
+      "learning_rate": 3.356772894023505e-07,
+      "logps/chosen": -274.7813126671123,
+      "logps/rejected": -285.46543115601503,
+      "loss": 0.376,
+      "rewards/chosen": -0.13362450013186206,
+      "rewards/margins": 3.3327875780371934,
+      "rewards/rejected": -3.4664120781690553,
+      "step": 740
+    },
+    {
+      "epoch": 0.3939075630252101,
+      "grad_norm": 23.48397277478164,
+      "kl": 0.0,
+      "learning_rate": 3.317906153857054e-07,
+      "logps/chosen": -302.44775079617835,
+      "logps/rejected": -287.7147958205521,
+      "loss": 0.3351,
+      "rewards/chosen": 0.14863140567852434,
+      "rewards/margins": 4.358767210773425,
+      "rewards/rejected": -4.2101358050949,
+      "step": 750
+    },
+    {
+      "epoch": 0.39915966386554624,
+      "grad_norm": 34.1827992343062,
+      "kl": 0.0,
+      "learning_rate": 3.2788167448334784e-07,
+      "logps/chosen": -262.08561812106916,
+      "logps/rejected": -274.4614227484472,
+      "loss": 0.374,
+      "rewards/chosen": -1.007453294670057,
+      "rewards/margins": 2.7304841113130562,
+      "rewards/rejected": -3.7379374059831134,
+      "step": 760
+    },
+    {
+      "epoch": 0.40441176470588236,
+      "grad_norm": 45.645905851061976,
+      "kl": 0.0,
+      "learning_rate": 3.2395153087531763e-07,
+      "logps/chosen": -290.2492659395973,
+      "logps/rejected": -247.69814510233917,
+      "loss": 0.3295,
+      "rewards/chosen": -0.16713363852276897,
+      "rewards/margins": 3.4756729221497453,
+      "rewards/rejected": -3.6428065606725144,
+      "step": 770
+    },
+    {
+      "epoch": 0.4096638655462185,
+      "grad_norm": 52.083381199139474,
+      "kl": 0.0,
+      "learning_rate": 3.20001254513933e-07,
+      "logps/chosen": -301.3512216862416,
+      "logps/rejected": -320.7805875365497,
+      "loss": 0.3564,
+      "rewards/chosen": 0.2707460390641385,
+      "rewards/margins": 3.1255901602721066,
+      "rewards/rejected": -2.854844121207968,
+      "step": 780
+    },
+    {
+      "epoch": 0.41491596638655465,
+      "grad_norm": 39.20450298136074,
+      "kl": 0.0,
+      "learning_rate": 3.160319208325044e-07,
+      "logps/chosen": -320.55803206699346,
+      "logps/rejected": -272.5804172904192,
+      "loss": 0.3116,
+      "rewards/chosen": 0.21087132871540543,
+      "rewards/margins": 3.79839062605822,
+      "rewards/rejected": -3.5875192973428143,
+      "step": 790
+    },
+    {
+      "epoch": 0.42016806722689076,
+      "grad_norm": 49.5026281906496,
+      "kl": 0.0,
+      "learning_rate": 3.1204461045255597e-07,
+      "logps/chosen": -305.87476748511904,
+      "logps/rejected": -282.5663291529605,
+      "loss": 0.3302,
+      "rewards/chosen": 0.5268175942557198,
+      "rewards/margins": 3.157528597609441,
+      "rewards/rejected": -2.630711003353721,
+      "step": 800
+    },
+    {
+      "epoch": 0.4254201680672269,
+      "grad_norm": 41.129555544878784,
+      "kl": 0.0,
+      "learning_rate": 3.0804040888963367e-07,
+      "logps/chosen": -305.3494718309859,
+      "logps/rejected": -257.2256846910112,
+      "loss": 0.2981,
+      "rewards/chosen": 0.24150611984897669,
+      "rewards/margins": 4.293326681975907,
+      "rewards/rejected": -4.051820562126931,
+      "step": 810
+    },
+    {
+      "epoch": 0.43067226890756305,
+      "grad_norm": 42.46230974734401,
+      "kl": 0.0,
+      "learning_rate": 3.040204062577824e-07,
+      "logps/chosen": -312.58470262096773,
+      "logps/rejected": -261.4371448863636,
+      "loss": 0.316,
+      "rewards/chosen": 0.39130327778477825,
+      "rewards/margins": 3.736164117628528,
+      "rewards/rejected": -3.34486083984375,
+      "step": 820
+    },
+    {
+      "epoch": 0.43592436974789917,
+      "grad_norm": 61.00539417491642,
+      "kl": 0.0,
+      "learning_rate": 2.999856969727704e-07,
+      "logps/chosen": -305.4535361842105,
+      "logps/rejected": -265.3578020134228,
+      "loss": 0.3327,
+      "rewards/chosen": 1.0649953920241686,
+      "rewards/margins": 3.993097206857432,
+      "rewards/rejected": -2.9281018148332634,
+      "step": 830
+    },
+    {
+      "epoch": 0.4411764705882353,
+      "grad_norm": 37.0047213388179,
+      "kl": 0.0,
+      "learning_rate": 2.959373794541426e-07,
+      "logps/chosen": -282.06252472310126,
+      "logps/rejected": -256.6561053240741,
+      "loss": 0.3265,
+      "rewards/chosen": 0.3667142361025267,
+      "rewards/margins": 3.1375505835176503,
+      "rewards/rejected": -2.7708363474151234,
+      "step": 840
+    },
+    {
+      "epoch": 0.44642857142857145,
+      "grad_norm": 36.25355732454002,
+      "kl": 0.0,
+      "learning_rate": 2.9187655582618407e-07,
+      "logps/chosen": -313.9366851993865,
+      "logps/rejected": -287.78662420382165,
+      "loss": 0.3112,
+      "rewards/chosen": 0.5094557686086081,
+      "rewards/margins": 3.8714661655781546,
+      "rewards/rejected": -3.3620103969695463,
+      "step": 850
+    },
+    {
+      "epoch": 0.45168067226890757,
+      "grad_norm": 64.53574065510352,
+      "kl": 0.0,
+      "learning_rate": 2.878043316178753e-07,
+      "logps/chosen": -283.32109866352204,
+      "logps/rejected": -264.0886063664596,
+      "loss": 0.3276,
+      "rewards/chosen": 0.1551594764181653,
+      "rewards/margins": 4.185146314054035,
+      "rewards/rejected": -4.029986837635869,
+      "step": 860
+    },
+    {
+      "epoch": 0.4569327731092437,
+      "grad_norm": 35.86505890578853,
+      "kl": 0.0,
+      "learning_rate": 2.837218154619193e-07,
+      "logps/chosen": -312.15406436011904,
+      "logps/rejected": -274.98959189967104,
+      "loss": 0.3348,
+      "rewards/chosen": -0.1483011245727539,
+      "rewards/margins": 4.045587087932386,
+      "rewards/rejected": -4.19388821250514,
+      "step": 870
+    },
+    {
+      "epoch": 0.46218487394957986,
+      "grad_norm": 46.44246066253616,
+      "kl": 0.0,
+      "learning_rate": 2.796301187929257e-07,
+      "logps/chosen": -316.67644557823127,
+      "logps/rejected": -274.3472949783237,
+      "loss": 0.331,
+      "rewards/chosen": -0.024288294266681283,
+      "rewards/margins": 3.5203153511903276,
+      "rewards/rejected": -3.5446036454570087,
+      "step": 880
+    },
+    {
+      "epoch": 0.46743697478991597,
+      "grad_norm": 67.03947638674309,
+      "kl": 0.0,
+      "learning_rate": 2.755303555448301e-07,
+      "logps/chosen": -296.37250766871165,
+      "logps/rejected": -287.87425358280257,
+      "loss": 0.2935,
+      "rewards/chosen": 0.2513600448889235,
+      "rewards/margins": 4.716395453054728,
+      "rewards/rejected": -4.465035408165805,
+      "step": 890
+    },
+    {
+      "epoch": 0.4726890756302521,
+      "grad_norm": 44.0580333940613,
+      "kl": 0.0,
+      "learning_rate": 2.7142364184763424e-07,
+      "logps/chosen": -275.6317777317881,
+      "logps/rejected": -295.6349852071006,
+      "loss": 0.3427,
+      "rewards/chosen": -0.019468964330407956,
+      "rewards/margins": 3.7003294679689636,
+      "rewards/rejected": -3.7197984322993713,
+      "step": 900
+    },
+    {
+      "epoch": 0.47794117647058826,
+      "grad_norm": 58.59520003164634,
+      "kl": 0.0,
+      "learning_rate": 2.673110957235479e-07,
+      "logps/chosen": -305.7110357704403,
+      "logps/rejected": -299.5851853649068,
+      "loss": 0.2967,
+      "rewards/chosen": 0.24971195436873525,
+      "rewards/margins": 4.398466382260817,
+      "rewards/rejected": -4.148754427892081,
+      "step": 910
+    },
+    {
+      "epoch": 0.4831932773109244,
+      "grad_norm": 49.15023451078887,
+      "kl": 0.0,
+      "learning_rate": 2.6319383678261557e-07,
+      "logps/chosen": -326.3589082154088,
+      "logps/rejected": -291.8697350543478,
+      "loss": 0.34,
+      "rewards/chosen": -0.1070428524377211,
+      "rewards/margins": 3.931509757592364,
+      "rewards/rejected": -4.038552610030085,
+      "step": 920
+    },
+    {
+      "epoch": 0.4884453781512605,
+      "grad_norm": 60.143948744622776,
+      "kl": 0.0,
+      "learning_rate": 2.5907298591791105e-07,
+      "logps/chosen": -293.6828878012048,
+      "logps/rejected": -316.3496601055195,
+      "loss": 0.3121,
+      "rewards/chosen": 0.37629242402961455,
+      "rewards/margins": 4.533702884662731,
+      "rewards/rejected": -4.157410460633117,
+      "step": 930
+    },
+    {
+      "epoch": 0.49369747899159666,
+      "grad_norm": 41.39145780505376,
+      "kl": 0.0,
+      "learning_rate": 2.5494966500038264e-07,
+      "logps/chosen": -298.37648703835225,
+      "logps/rejected": -283.5568033854167,
+      "loss": 0.3381,
+      "rewards/chosen": 0.19053040851246228,
+      "rewards/margins": 2.8964626018447106,
+      "rewards/rejected": -2.705932193332248,
+      "step": 940
+    },
+    {
+      "epoch": 0.4989495798319328,
+      "grad_norm": 44.85935706825639,
+      "kl": 0.0,
+      "learning_rate": 2.508249965734319e-07,
+      "logps/chosen": -315.8209355828221,
+      "logps/rejected": -277.33056329617835,
+      "loss": 0.2815,
+      "rewards/chosen": 0.5839554488293233,
+      "rewards/margins": 4.811377494883264,
+      "rewards/rejected": -4.227422046053941,
+      "step": 950
+    },
+    {
+      "epoch": 0.5042016806722689,
+      "grad_norm": 86.97621065062417,
+      "kl": 0.0,
+      "learning_rate": 2.467001035473103e-07,
+      "logps/chosen": -302.4603059505988,
+      "logps/rejected": -287.89797794117646,
+      "loss": 0.3293,
+      "rewards/chosen": 0.40462347704493357,
+      "rewards/margins": 3.534475173582924,
+      "rewards/rejected": -3.1298516965379903,
+      "step": 960
+    },
+    {
+      "epoch": 0.509453781512605,
+      "grad_norm": 32.22216772039752,
+      "kl": 0.0,
+      "learning_rate": 2.425761088934142e-07,
+      "logps/chosen": -273.99361898291926,
+      "logps/rejected": -258.7056308962264,
+      "loss": 0.3326,
+      "rewards/chosen": -0.09355034443162243,
+      "rewards/margins": 4.204330392105327,
+      "rewards/rejected": -4.29788073653695,
+      "step": 970
+    },
+    {
+      "epoch": 0.5147058823529411,
+      "grad_norm": 53.215396805561575,
+      "kl": 0.0,
+      "learning_rate": 2.3845413533856514e-07,
+      "logps/chosen": -299.4835397012579,
+      "logps/rejected": -287.7150621118012,
+      "loss": 0.2911,
+      "rewards/chosen": 0.30934901207498033,
+      "rewards/margins": 4.708151206611083,
+      "rewards/rejected": -4.398802194536103,
+      "step": 980
+    },
+    {
+      "epoch": 0.5199579831932774,
+      "grad_norm": 70.13460679739038,
+      "kl": 0.0,
+      "learning_rate": 2.343353050593553e-07,
+      "logps/chosen": -318.12441770186336,
+      "logps/rejected": -282.98125491352204,
+      "loss": 0.3042,
+      "rewards/chosen": 0.9980694314707881,
+      "rewards/margins": 5.077612059345198,
+      "rewards/rejected": -4.07954262787441,
+      "step": 990
+    },
+    {
+      "epoch": 0.5252100840336135,
+      "grad_norm": 91.66490200361514,
+      "kl": 0.0,
+      "learning_rate": 2.3022073937664383e-07,
+      "logps/chosen": -318.62791700487014,
+      "logps/rejected": -299.1699924698795,
+      "loss": 0.2987,
+      "rewards/chosen": -0.4350905975738129,
+      "rewards/margins": 3.7659268511272415,
+      "rewards/rejected": -4.201017448701054,
+      "step": 1000
+    },
+    {
+      "epoch": 0.5304621848739496,
+      "grad_norm": 47.014009528040546,
+      "kl": 0.0,
+      "learning_rate": 2.261115584502849e-07,
+      "logps/chosen": -322.26196442610063,
+      "logps/rejected": -273.58297748447205,
+      "loss": 0.3504,
+      "rewards/chosen": -0.4901385037404186,
+      "rewards/margins": 2.980573340567617,
+      "rewards/rejected": -3.4707118443080356,
+      "step": 1010
+    },
+    {
+      "epoch": 0.5357142857142857,
+      "grad_norm": 65.73363572647297,
+      "kl": 0.0,
+      "learning_rate": 2.2200888097417302e-07,
+      "logps/chosen": -265.984912109375,
+      "logps/rejected": -273.97919921875,
+      "loss": 0.3327,
+      "rewards/chosen": 0.2538748264312744,
+      "rewards/margins": 3.538307809829712,
+      "rewards/rejected": -3.2844329833984376,
+      "step": 1020
+    },
+    {
+      "epoch": 0.5409663865546218,
+      "grad_norm": 37.39061372036576,
+      "kl": 0.0,
+      "learning_rate": 2.1791382387168684e-07,
+      "logps/chosen": -294.4272490530303,
+      "logps/rejected": -283.10264616935484,
+      "loss": 0.3218,
+      "rewards/chosen": 0.5568286317767519,
+      "rewards/margins": 4.048672366025744,
+      "rewards/rejected": -3.491843734248992,
+      "step": 1030
+    },
+    {
+      "epoch": 0.5462184873949579,
+      "grad_norm": 38.42991974376242,
+      "kl": 0.0,
+      "learning_rate": 2.1382750199161495e-07,
+      "logps/chosen": -314.6301011029412,
+      "logps/rejected": -277.451328125,
+      "loss": 0.3087,
+      "rewards/chosen": 0.9661560956169577,
+      "rewards/margins": 3.718452645096124,
+      "rewards/rejected": -2.7522965494791665,
+      "step": 1040
+    },
+    {
+      "epoch": 0.5514705882352942,
+      "grad_norm": 58.32907293331433,
+      "kl": 0.0,
+      "learning_rate": 2.0975102780464673e-07,
+      "logps/chosen": -306.8538490853659,
+      "logps/rejected": -290.90675080128204,
+      "loss": 0.3062,
+      "rewards/chosen": 0.684349339182784,
+      "rewards/margins": 3.878205602954819,
+      "rewards/rejected": -3.193856263772035,
+      "step": 1050
+    },
+    {
+      "epoch": 0.5567226890756303,
+      "grad_norm": 42.31783649576492,
+      "kl": 0.0,
+      "learning_rate": 2.0568551110051074e-07,
+      "logps/chosen": -265.01874508647796,
+      "logps/rejected": -256.457298136646,
+      "loss": 0.3375,
+      "rewards/chosen": 0.7610905485333137,
+      "rewards/margins": 3.0508450962211047,
+      "rewards/rejected": -2.289754547687791,
+      "step": 1060
+    },
+    {
+      "epoch": 0.5619747899159664,
+      "grad_norm": 43.282189967649956,
+      "kl": 0.0,
+      "learning_rate": 2.016320586858422e-07,
+      "logps/chosen": -270.2985276442308,
+      "logps/rejected": -287.5562357088415,
+      "loss": 0.3143,
+      "rewards/chosen": 1.108019804343199,
+      "rewards/margins": 3.3270524641064423,
+      "rewards/rejected": -2.219032659763243,
+      "step": 1070
+    },
+    {
+      "epoch": 0.5672268907563025,
+      "grad_norm": 59.248701734675386,
+      "kl": 0.0,
+      "learning_rate": 1.9759177408286337e-07,
+      "logps/chosen": -312.609375,
+      "logps/rejected": -267.16066028225805,
+      "loss": 0.3365,
+      "rewards/chosen": 1.2182994495738637,
+      "rewards/margins": 3.07108659045088,
+      "rewards/rejected": -1.8527871408770162,
+      "step": 1080
+    },
+    {
+      "epoch": 0.5724789915966386,
+      "grad_norm": 45.19596654926437,
+      "kl": 0.0,
+      "learning_rate": 1.9356575722895808e-07,
+      "logps/chosen": -299.373361013986,
+      "logps/rejected": -271.5683042019774,
+      "loss": 0.3178,
+      "rewards/chosen": 1.0334772496790319,
+      "rewards/margins": 3.244971479960194,
+      "rewards/rejected": -2.2114942302811618,
+      "step": 1090
+    },
+    {
+      "epoch": 0.5777310924369747,
+      "grad_norm": 44.21402337263303,
+      "kl": 0.0,
+      "learning_rate": 1.895551041772216e-07,
+      "logps/chosen": -309.0861150568182,
+      "logps/rejected": -255.60587565104166,
+      "loss": 0.3402,
+      "rewards/chosen": 0.8877213217995383,
+      "rewards/margins": 3.0730268112336745,
+      "rewards/rejected": -2.1853054894341364,
+      "step": 1100
+    },
+    {
+      "epoch": 0.582983193277311,
+      "grad_norm": 61.69332868228022,
+      "kl": 0.0,
+      "learning_rate": 1.8556090679806847e-07,
+      "logps/chosen": -298.82965455572287,
+      "logps/rejected": -267.7830509334416,
+      "loss": 0.2942,
+      "rewards/chosen": 1.1525323706937123,
+      "rewards/margins": 4.799233143065871,
+      "rewards/rejected": -3.646700772372159,
+      "step": 1110
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 31.77308050751697,
+      "kl": 0.0,
+      "learning_rate": 1.8158425248197928e-07,
+      "logps/chosen": -330.5060499237805,
+      "logps/rejected": -304.98880709134613,
+      "loss": 0.2989,
+      "rewards/chosen": 1.3816723707245617,
+      "rewards/margins": 5.020397847111781,
+      "rewards/rejected": -3.6387254763872194,
+      "step": 1120
+    },
+    {
+      "epoch": 0.5934873949579832,
+      "grad_norm": 59.622620200950486,
+      "kl": 0.0,
+      "learning_rate": 1.7762622384346609e-07,
+      "logps/chosen": -292.1063850308642,
+      "logps/rejected": -255.3093848892405,
+      "loss": 0.311,
+      "rewards/chosen": 1.0882685155044367,
+      "rewards/margins": 3.8796188681176687,
+      "rewards/rejected": -2.791350352613232,
+      "step": 1130
+    },
+    {
+      "epoch": 0.5987394957983193,
+      "grad_norm": 92.89241064630508,
+      "kl": 0.0,
+      "learning_rate": 1.7368789842633907e-07,
+      "logps/chosen": -324.4837362421384,
+      "logps/rejected": -303.8497670807453,
+      "loss": 0.2984,
+      "rewards/chosen": 0.8251325499336675,
+      "rewards/margins": 4.589403024385337,
+      "rewards/rejected": -3.7642704744516693,
+      "step": 1140
+    },
+    {
+      "epoch": 0.6039915966386554,
+      "grad_norm": 42.994311331506765,
+      "kl": 0.0,
+      "learning_rate": 1.697703484103532e-07,
+      "logps/chosen": -292.4465987042683,
+      "logps/rejected": -259.40554887820514,
+      "loss": 0.3131,
+      "rewards/chosen": 0.9567713853789539,
+      "rewards/margins": 4.075035216884363,
+      "rewards/rejected": -3.1182638315054088,
+      "step": 1150
+    },
+    {
+      "epoch": 0.6092436974789915,
+      "grad_norm": 63.999900745814266,
+      "kl": 0.0,
+      "learning_rate": 1.6587464031931526e-07,
+      "logps/chosen": -303.69353428171644,
+      "logps/rejected": -277.118405577957,
+      "loss": 0.3398,
+      "rewards/chosen": 0.867815444718546,
+      "rewards/margins": 3.5977176506472253,
+      "rewards/rejected": -2.7299022059286795,
+      "step": 1160
+    },
+    {
+      "epoch": 0.6144957983193278,
+      "grad_norm": 33.78499266925509,
+      "kl": 0.0,
+      "learning_rate": 1.6200183473073048e-07,
+      "logps/chosen": -299.6548768939394,
+      "logps/rejected": -280.22903225806454,
+      "loss": 0.2992,
+      "rewards/chosen": 0.6621969512014678,
+      "rewards/margins": 3.7750548928597336,
+      "rewards/rejected": -3.112857941658266,
+      "step": 1170
+    },
+    {
+      "epoch": 0.6197478991596639,
+      "grad_norm": 31.77803791723765,
+      "kl": 0.0,
+      "learning_rate": 1.5815298598706888e-07,
+      "logps/chosen": -291.1797602872671,
+      "logps/rejected": -286.78439465408803,
+      "loss": 0.3176,
+      "rewards/chosen": -0.010194837676812403,
+      "rewards/margins": 4.250023752633722,
+      "rewards/rejected": -4.260218590310535,
+      "step": 1180
+    },
+    {
+      "epoch": 0.625,
+      "grad_norm": 37.859068866257594,
+      "kl": 0.0,
+      "learning_rate": 1.5432914190872756e-07,
+      "logps/chosen": -283.2485907832278,
+      "logps/rejected": -299.72665895061726,
+      "loss": 0.3475,
+      "rewards/chosen": 0.012598955178562599,
+      "rewards/margins": 4.188036976614712,
+      "rewards/rejected": -4.1754380214361495,
+      "step": 1190
+    },
+    {
+      "epoch": 0.6302521008403361,
+      "grad_norm": 34.730160989586295,
+      "kl": 0.0,
+      "learning_rate": 1.505313435087698e-07,
+      "logps/chosen": -300.31629922945206,
+      "logps/rejected": -295.37841235632186,
+      "loss": 0.3124,
+      "rewards/chosen": 0.33270381248160585,
+      "rewards/margins": 4.571263930005852,
+      "rewards/rejected": -4.238560117524246,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6355042016806722,
+      "grad_norm": 73.6570576556008,
+      "kl": 0.0,
+      "learning_rate": 1.4676062470951705e-07,
+      "logps/chosen": -316.37948140337426,
+      "logps/rejected": -306.59623805732485,
+      "loss": 0.318,
+      "rewards/chosen": 0.23967742919921875,
+      "rewards/margins": 4.2318552102252935,
+      "rewards/rejected": -3.992177781026075,
+      "step": 1210
+    },
+    {
+      "epoch": 0.6407563025210085,
+      "grad_norm": 86.8105694173325,
+      "kl": 0.0,
+      "learning_rate": 1.430180120610711e-07,
+      "logps/chosen": -283.5435161226115,
+      "logps/rejected": -276.21841449386505,
+      "loss": 0.2916,
+      "rewards/chosen": 0.35688832762894357,
+      "rewards/margins": 4.4911308476394876,
+      "rewards/rejected": -4.134242520010544,
+      "step": 1220
+    },
+    {
+      "epoch": 0.6460084033613446,
+      "grad_norm": 65.59031614397247,
+      "kl": 0.0,
+      "learning_rate": 1.3930452446184385e-07,
+      "logps/chosen": -324.24284699675326,
+      "logps/rejected": -301.1185523343373,
+      "loss": 0.3146,
+      "rewards/chosen": 0.3668579745602298,
+      "rewards/margins": 4.802705980843173,
+      "rewards/rejected": -4.435848006282944,
+      "step": 1230
+    },
+    {
+      "epoch": 0.6512605042016807,
+      "grad_norm": 82.83869882352816,
+      "kl": 0.0,
+      "learning_rate": 1.3562117288116923e-07,
+      "logps/chosen": -315.7268702651515,
+      "logps/rejected": -280.67207661290325,
+      "loss": 0.3109,
+      "rewards/chosen": 0.6113425514914773,
+      "rewards/margins": 4.853238264067082,
+      "rewards/rejected": -4.241895712575605,
+      "step": 1240
+    },
+    {
+      "epoch": 0.6565126050420168,
+      "grad_norm": 53.82976674411678,
+      "kl": 0.0,
+      "learning_rate": 1.319689600840747e-07,
+      "logps/chosen": -304.3841062898089,
+      "logps/rejected": -285.0993577453988,
+      "loss": 0.3324,
+      "rewards/chosen": 0.08269594884981775,
+      "rewards/margins": 3.7769891991853237,
+      "rewards/rejected": -3.694293250335506,
+      "step": 1250
+    },
+    {
+      "epoch": 0.6617647058823529,
+      "grad_norm": 34.58790215772178,
+      "kl": 0.0,
+      "learning_rate": 1.2834888035828596e-07,
+      "logps/chosen": -301.4619855182927,
+      "logps/rejected": -272.6404246794872,
+      "loss": 0.2994,
+      "rewards/chosen": 0.32138896569973086,
+      "rewards/margins": 4.973683418073529,
+      "rewards/rejected": -4.652294452373798,
+      "step": 1260
+    },
+    {
+      "epoch": 0.667016806722689,
+      "grad_norm": 29.984315889657353,
+      "kl": 0.0,
+      "learning_rate": 1.2476191924353932e-07,
+      "logps/chosen": -345.89991071428574,
+      "logps/rejected": -282.63448275862066,
+      "loss": 0.326,
+      "rewards/chosen": 0.5500016566685267,
+      "rewards/margins": 4.73898333770301,
+      "rewards/rejected": -4.188981681034483,
+      "step": 1270
+    },
+    {
+      "epoch": 0.6722689075630253,
+      "grad_norm": 61.179118361711794,
+      "kl": 0.0,
+      "learning_rate": 1.2120905326327596e-07,
+      "logps/chosen": -298.07992788461536,
+      "logps/rejected": -293.07983236754967,
+      "loss": 0.3307,
+      "rewards/chosen": 0.1393580013478296,
+      "rewards/margins": 4.935787970175437,
+      "rewards/rejected": -4.796429968827607,
+      "step": 1280
+    },
+    {
+      "epoch": 0.6775210084033614,
+      "grad_norm": 35.426220175192356,
+      "kl": 0.0,
+      "learning_rate": 1.1769124965879091e-07,
+      "logps/chosen": -309.7057636589404,
+      "logps/rejected": -253.99468380177515,
+      "loss": 0.3127,
+      "rewards/chosen": 0.3561096696664166,
+      "rewards/margins": 4.979999841056024,
+      "rewards/rejected": -4.623890171389608,
+      "step": 1290
+    },
+    {
+      "epoch": 0.6827731092436975,
+      "grad_norm": 43.346023169027426,
+      "kl": 0.0,
+      "learning_rate": 1.1420946612590837e-07,
+      "logps/chosen": -270.4368489583333,
+      "logps/rejected": -281.6362047697368,
+      "loss": 0.3397,
+      "rewards/chosen": 0.1588947886512393,
+      "rewards/margins": 3.5548700019530486,
+      "rewards/rejected": -3.395975213301809,
+      "step": 1300
+    },
+    {
+      "epoch": 0.6880252100840336,
+      "grad_norm": 45.324757492893184,
+      "kl": 0.0,
+      "learning_rate": 1.1076465055425646e-07,
+      "logps/chosen": -290.0300690406977,
+      "logps/rejected": -258.5001319679054,
+      "loss": 0.3082,
+      "rewards/chosen": 0.7295433754144713,
+      "rewards/margins": 4.570315849548762,
+      "rewards/rejected": -3.8407724741342903,
+      "step": 1310
+    },
+    {
+      "epoch": 0.6932773109243697,
+      "grad_norm": 263.89653772607954,
+      "kl": 0.0,
+      "learning_rate": 1.0735774076921128e-07,
+      "logps/chosen": -287.7608642578125,
+      "logps/rejected": -237.489306640625,
+      "loss": 0.3446,
+      "rewards/chosen": 0.27030837535858154,
+      "rewards/margins": 3.555664134025574,
+      "rewards/rejected": -3.2853557586669924,
+      "step": 1320
+    },
+    {
+      "epoch": 0.6985294117647058,
+      "grad_norm": 61.837360014332575,
+      "kl": 0.0,
+      "learning_rate": 1.039896642765809e-07,
+      "logps/chosen": -297.6917317708333,
+      "logps/rejected": -314.11572265625,
+      "loss": 0.2958,
+      "rewards/chosen": 0.6960362328423394,
+      "rewards/margins": 5.248802512583106,
+      "rewards/rejected": -4.552766279740767,
+      "step": 1330
+    },
+    {
+      "epoch": 0.7037815126050421,
+      "grad_norm": 51.294856185176336,
+      "kl": 0.0,
+      "learning_rate": 1.0066133801009871e-07,
+      "logps/chosen": -328.30787295386904,
+      "logps/rejected": -244.40604440789474,
+      "loss": 0.3097,
+      "rewards/chosen": 0.7595687593732562,
+      "rewards/margins": 4.201100693609481,
+      "rewards/rejected": -3.441531934236225,
+      "step": 1340
+    },
+    {
+      "epoch": 0.7090336134453782,
+      "grad_norm": 43.77043748464739,
+      "kl": 0.0,
+      "learning_rate": 9.737366808179553e-08,
+      "logps/chosen": -303.54956371753246,
+      "logps/rejected": -250.67248682228916,
+      "loss": 0.3096,
+      "rewards/chosen": 0.9003205732865767,
+      "rewards/margins": 4.412543414848136,
+      "rewards/rejected": -3.5122228415615586,
+      "step": 1350
+    },
+    {
+      "epoch": 0.7142857142857143,
+      "grad_norm": 72.17773141718298,
+      "kl": 0.0,
+      "learning_rate": 9.412754953531663e-08,
+      "logps/chosen": -262.4382017215569,
+      "logps/rejected": -292.5083997140523,
+      "loss": 0.3377,
+      "rewards/chosen": 0.434063574511134,
+      "rewards/margins": 3.880319925358765,
+      "rewards/rejected": -3.446256350847631,
+      "step": 1360
+    },
+    {
+      "epoch": 0.7195378151260504,
+      "grad_norm": 61.94872379316782,
+      "kl": 0.0,
+      "learning_rate": 9.092386610225325e-08,
+      "logps/chosen": -265.7097810444079,
+      "logps/rejected": -279.9903273809524,
+      "loss": 0.3299,
+      "rewards/chosen": 0.8910425085770456,
+      "rewards/margins": 3.7223031897293892,
+      "rewards/rejected": -2.8312606811523438,
+      "step": 1370
+    },
+    {
+      "epoch": 0.7247899159663865,
+      "grad_norm": 47.60678426366227,
+      "kl": 0.0,
+      "learning_rate": 8.776348996155317e-08,
+      "logps/chosen": -290.0428740530303,
+      "logps/rejected": -278.20430947580644,
+      "loss": 0.3123,
+      "rewards/chosen": 0.3940057927911932,
+      "rewards/margins": 4.4418612930432095,
+      "rewards/rejected": -4.047855500252016,
+      "step": 1380
+    },
+    {
+      "epoch": 0.7300420168067226,
+      "grad_norm": 41.557754145037535,
+      "kl": 0.0,
+      "learning_rate": 8.464728150207636e-08,
+      "logps/chosen": -295.76380699685535,
+      "logps/rejected": -294.7458753881988,
+      "loss": 0.3185,
+      "rewards/chosen": 0.6997887953272406,
+      "rewards/margins": 4.221460825000183,
+      "rewards/rejected": -3.5216720296729425,
+      "step": 1390
+    },
+    {
+      "epoch": 0.7352941176470589,
+      "grad_norm": 45.38199821794469,
+      "kl": 0.0,
+      "learning_rate": 8.15760890883607e-08,
+      "logps/chosen": -275.74736265923565,
+      "logps/rejected": -261.5596721625767,
+      "loss": 0.3109,
+      "rewards/chosen": 0.7350611109642466,
+      "rewards/margins": 4.5443022829590705,
+      "rewards/rejected": -3.809241171994824,
+      "step": 1400
+    },
+    {
+      "epoch": 0.740546218487395,
+      "grad_norm": 22.290814745353508,
+      "kl": 0.0,
+      "learning_rate": 7.855074882966103e-08,
+      "logps/chosen": -311.78690011160717,
+      "logps/rejected": -279.9527652138158,
+      "loss": 0.2985,
+      "rewards/chosen": 0.7244267236618769,
+      "rewards/margins": 5.052972545002337,
+      "rewards/rejected": -4.328545821340461,
+      "step": 1410
+    },
+    {
+      "epoch": 0.7457983193277311,
+      "grad_norm": 40.315164584155504,
+      "kl": 0.0,
+      "learning_rate": 7.557208435232449e-08,
+      "logps/chosen": -274.712925295858,
+      "logps/rejected": -278.3036268625828,
+      "loss": 0.3147,
+      "rewards/chosen": 1.077583267843935,
+      "rewards/margins": 4.07225724310004,
+      "rewards/rejected": -2.9946739752561053,
+      "step": 1420
+    },
+    {
+      "epoch": 0.7510504201680672,
+      "grad_norm": 36.626625075926555,
+      "kl": 0.0,
+      "learning_rate": 7.264090657556443e-08,
+      "logps/chosen": -292.9152413091716,
+      "logps/rejected": -242.9820467715232,
+      "loss": 0.3085,
+      "rewards/chosen": 0.9502442489714312,
+      "rewards/margins": 4.543455846136166,
+      "rewards/rejected": -3.593211597164735,
+      "step": 1430
+    },
+    {
+      "epoch": 0.7563025210084033,
+      "grad_norm": 43.54130112641965,
+      "kl": 0.0,
+      "learning_rate": 6.975801349069385e-08,
+      "logps/chosen": -280.07919034090907,
+      "logps/rejected": -276.84921875,
+      "loss": 0.2987,
+      "rewards/chosen": 0.817291444720644,
+      "rewards/margins": 5.2739698721399995,
+      "rewards/rejected": -4.456678427419355,
+      "step": 1440
+    },
+    {
+      "epoch": 0.7615546218487395,
+      "grad_norm": 39.41222523190064,
+      "kl": 0.0,
+      "learning_rate": 6.692418994387799e-08,
+      "logps/chosen": -288.40252246732024,
+      "logps/rejected": -284.47291354790417,
+      "loss": 0.2993,
+      "rewards/chosen": 0.9826148538028493,
+      "rewards/margins": 5.268814549138741,
+      "rewards/rejected": -4.286199695335891,
+      "step": 1450
+    },
+    {
+      "epoch": 0.7668067226890757,
+      "grad_norm": 42.76397567606349,
+      "kl": 0.0,
+      "learning_rate": 6.414020742246593e-08,
+      "logps/chosen": -288.7798685213415,
+      "logps/rejected": -293.82729867788464,
+      "loss": 0.2617,
+      "rewards/chosen": 0.9620797692275629,
+      "rewards/margins": 5.808872528267026,
+      "rewards/rejected": -4.846792759039463,
+      "step": 1460
+    },
+    {
+      "epoch": 0.7720588235294118,
+      "grad_norm": 85.5452615105662,
+      "kl": 0.0,
+      "learning_rate": 6.140682384495902e-08,
+      "logps/chosen": -311.87120301573424,
+      "logps/rejected": -282.2307997881356,
+      "loss": 0.2877,
+      "rewards/chosen": 0.8217522681176246,
+      "rewards/margins": 4.416604004412646,
+      "rewards/rejected": -3.594851736295021,
+      "step": 1470
+    },
+    {
+      "epoch": 0.7773109243697479,
+      "grad_norm": 47.14939912048749,
+      "kl": 0.0,
+      "learning_rate": 5.872478335467298e-08,
+      "logps/chosen": -270.7433546112805,
+      "logps/rejected": -274.62745392628204,
+      "loss": 0.329,
+      "rewards/chosen": -0.1099286428311976,
+      "rewards/margins": 4.24396219187934,
+      "rewards/rejected": -4.353890834710537,
+      "step": 1480
+    },
+    {
+      "epoch": 0.782563025210084,
+      "grad_norm": 75.39266090840431,
+      "kl": 0.0,
+      "learning_rate": 5.60948161171505e-08,
+      "logps/chosen": -298.3712257179054,
+      "logps/rejected": -296.9724518531977,
+      "loss": 0.3039,
+      "rewards/chosen": 0.3439343168928817,
+      "rewards/margins": 4.58285359718903,
+      "rewards/rejected": -4.238919280296148,
+      "step": 1490
+    },
+    {
+      "epoch": 0.7878151260504201,
+      "grad_norm": 101.47398109074602,
+      "kl": 0.0,
+      "learning_rate": 5.351763812137916e-08,
+      "logps/chosen": -285.37012357026146,
+      "logps/rejected": -305.706282747006,
+      "loss": 0.3075,
+      "rewards/chosen": -0.05964530371372996,
+      "rewards/margins": 4.639130484553485,
+      "rewards/rejected": -4.698775788267215,
+      "step": 1500
+    },
+    {
+      "epoch": 0.7930672268907563,
+      "grad_norm": 51.64579106329186,
+      "kl": 0.0,
+      "learning_rate": 5.0993950984868836e-08,
+      "logps/chosen": -273.9158336568323,
+      "logps/rejected": -296.97481819968556,
+      "loss": 0.3228,
+      "rewards/chosen": -0.35097162471794935,
+      "rewards/margins": 4.398951601500603,
+      "rewards/rejected": -4.749923226218553,
+      "step": 1510
+    },
+    {
+      "epoch": 0.7983193277310925,
+      "grad_norm": 164.1846468516791,
+      "kl": 0.0,
+      "learning_rate": 4.8524441762641284e-08,
+      "logps/chosen": -323.8402122641509,
+      "logps/rejected": -265.2092148680124,
+      "loss": 0.3144,
+      "rewards/chosen": 0.010938104593528892,
+      "rewards/margins": 4.735114546994538,
+      "rewards/rejected": -4.724176442401009,
+      "step": 1520
+    },
+    {
+      "epoch": 0.8035714285714286,
+      "grad_norm": 57.97209020416802,
+      "kl": 0.0,
+      "learning_rate": 4.6109782760184956e-08,
+      "logps/chosen": -283.25261976047904,
+      "logps/rejected": -294.36603860294116,
+      "loss": 0.3462,
+      "rewards/chosen": 0.19619906305552956,
+      "rewards/margins": 4.044921234215661,
+      "rewards/rejected": -3.848722171160131,
+      "step": 1530
+    },
+    {
+      "epoch": 0.8088235294117647,
+      "grad_norm": 41.10976267046813,
+      "kl": 0.0,
+      "learning_rate": 4.375063135042445e-08,
+      "logps/chosen": -301.61559264053255,
+      "logps/rejected": -321.35994412251654,
+      "loss": 0.3079,
+      "rewards/chosen": 0.2954674895698502,
+      "rewards/margins": 5.432189877426845,
+      "rewards/rejected": -5.136722387856995,
+      "step": 1540
+    },
+    {
+      "epoch": 0.8140756302521008,
+      "grad_norm": 54.92221531592682,
+      "kl": 0.0,
+      "learning_rate": 4.144762979475575e-08,
+      "logps/chosen": -306.94077797202794,
+      "logps/rejected": -280.2052436440678,
+      "loss": 0.315,
+      "rewards/chosen": -0.0729095952494161,
+      "rewards/margins": 4.834762834239461,
+      "rewards/rejected": -4.907672429488877,
+      "step": 1550
+    },
+    {
+      "epoch": 0.819327731092437,
+      "grad_norm": 48.43314600236666,
+      "kl": 0.0,
+      "learning_rate": 3.9201405068195385e-08,
+      "logps/chosen": -312.2001696134868,
+      "logps/rejected": -292.7070777529762,
+      "loss": 0.3395,
+      "rewards/chosen": 0.025735384539553995,
+      "rewards/margins": 4.3085366947609085,
+      "rewards/rejected": -4.2828013102213545,
+      "step": 1560
+    },
+    {
+      "epoch": 0.8245798319327731,
+      "grad_norm": 82.79685734368942,
+      "kl": 0.0,
+      "learning_rate": 3.701256868869124e-08,
+      "logps/chosen": -321.1130847091195,
+      "logps/rejected": -254.03585986024845,
+      "loss": 0.3356,
+      "rewards/chosen": 0.22610101759808618,
+      "rewards/margins": 4.19767455700026,
+      "rewards/rejected": -3.971573539402174,
+      "step": 1570
+    },
+    {
+      "epoch": 0.8298319327731093,
+      "grad_norm": 69.63465669746606,
+      "kl": 0.0,
+      "learning_rate": 3.488171655064107e-08,
+      "logps/chosen": -289.7723396650327,
+      "logps/rejected": -288.95059880239523,
+      "loss": 0.3227,
+      "rewards/chosen": -0.3906313079634523,
+      "rewards/margins": 4.496159441592124,
+      "rewards/rejected": -4.886790749555576,
+      "step": 1580
+    },
+    {
+      "epoch": 0.8350840336134454,
+      "grad_norm": 66.42429414400927,
+      "kl": 0.0,
+      "learning_rate": 3.28094287626651e-08,
+      "logps/chosen": -340.9739889705882,
+      "logps/rejected": -299.48609375,
+      "loss": 0.3308,
+      "rewards/chosen": 0.048118759604061354,
+      "rewards/margins": 4.789248316895728,
+      "rewards/rejected": -4.741129557291667,
+      "step": 1590
+    },
+    {
+      "epoch": 0.8403361344537815,
+      "grad_norm": 42.16586593578885,
+      "kl": 0.0,
+      "learning_rate": 3.079626948967534e-08,
+      "logps/chosen": -312.65874787414964,
+      "logps/rejected": -287.83103775289015,
+      "loss": 0.3062,
+      "rewards/chosen": -0.0730894049819635,
+      "rewards/margins": 4.740564265312472,
+      "rewards/rejected": -4.813653670294436,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8455882352941176,
+      "grad_norm": 42.50176028584879,
+      "kl": 0.0,
+      "learning_rate": 2.88427867992862e-08,
+      "logps/chosen": -298.3803453947368,
+      "logps/rejected": -256.55287856543623,
+      "loss": 0.3216,
+      "rewards/chosen": 0.292915611936335,
+      "rewards/margins": 4.54555282224464,
+      "rewards/rejected": -4.252637210308305,
+      "step": 1610
+    },
+    {
+      "epoch": 0.8508403361344538,
+      "grad_norm": 83.97476475765178,
+      "kl": 0.0,
+      "learning_rate": 2.6949512512606965e-08,
+      "logps/chosen": -293.518928433642,
+      "logps/rejected": -296.5833910205696,
+      "loss": 0.2953,
+      "rewards/chosen": 0.5612210874204282,
+      "rewards/margins": 5.312669320336765,
+      "rewards/rejected": -4.751448232916337,
+      "step": 1620
+    },
+    {
+      "epoch": 0.8560924369747899,
+      "grad_norm": 35.09955655518953,
+      "kl": 0.0,
+      "learning_rate": 2.5116962059457653e-08,
+      "logps/chosen": -307.30205078125,
+      "logps/rejected": -280.090478515625,
+      "loss": 0.2881,
+      "rewards/chosen": 0.4744985580444336,
+      "rewards/margins": 5.155851554870606,
+      "rewards/rejected": -4.681352996826172,
+      "step": 1630
+    },
+    {
+      "epoch": 0.8613445378151261,
+      "grad_norm": 100.20739187851474,
+      "kl": 0.0,
+      "learning_rate": 2.334563433804687e-08,
+      "logps/chosen": -319.6615634704969,
+      "logps/rejected": -282.31458824685535,
+      "loss": 0.3014,
+      "rewards/chosen": 0.6534406768609278,
+      "rewards/margins": 4.888445989606604,
+      "rewards/rejected": -4.235005312745676,
+      "step": 1640
+    },
+    {
+      "epoch": 0.8665966386554622,
+      "grad_norm": 88.81840072824282,
+      "kl": 0.0,
+      "learning_rate": 2.1636011579150793e-08,
+      "logps/chosen": -295.85146484375,
+      "logps/rejected": -290.18388671875,
+      "loss": 0.2796,
+      "rewards/chosen": 0.5988120079040528,
+      "rewards/margins": 4.9494441032409675,
+      "rewards/rejected": -4.350632095336914,
+      "step": 1650
+    },
+    {
+      "epoch": 0.8718487394957983,
+      "grad_norm": 41.43708238784869,
+      "kl": 0.0,
+      "learning_rate": 1.998855921482906e-08,
+      "logps/chosen": -276.827294921875,
+      "logps/rejected": -240.9879638671875,
+      "loss": 0.3411,
+      "rewards/chosen": 0.4913971424102783,
+      "rewards/margins": 4.203098344802856,
+      "rewards/rejected": -3.711701202392578,
+      "step": 1660
+    },
+    {
+      "epoch": 0.8771008403361344,
+      "grad_norm": 61.90087346557062,
+      "kl": 0.0,
+      "learning_rate": 1.8403725751714615e-08,
+      "logps/chosen": -287.8762538580247,
+      "logps/rejected": -314.7857001582278,
+      "loss": 0.2888,
+      "rewards/chosen": 0.6117756690508054,
+      "rewards/margins": 5.216582303495031,
+      "rewards/rejected": -4.604806634444225,
+      "step": 1670
+    },
+    {
+      "epoch": 0.8823529411764706,
+      "grad_norm": 43.47883434595641,
+      "kl": 0.0,
+      "learning_rate": 1.6881942648911074e-08,
+      "logps/chosen": -286.86601796407183,
+      "logps/rejected": -269.488791870915,
+      "loss": 0.3423,
+      "rewards/chosen": 0.3007767500277765,
+      "rewards/margins": 4.408972534852123,
+      "rewards/rejected": -4.108195784824346,
+      "step": 1680
+    },
+    {
+      "epoch": 0.8876050420168067,
+      "grad_norm": 43.77865447701488,
+      "kl": 0.0,
+      "learning_rate": 1.5423624200531115e-08,
+      "logps/chosen": -300.1749822443182,
+      "logps/rejected": -305.69072808159723,
+      "loss": 0.3209,
+      "rewards/chosen": 0.8302790901877664,
+      "rewards/margins": 4.638904205476395,
+      "rewards/rejected": -3.8086251152886286,
+      "step": 1690
+    },
+    {
+      "epoch": 0.8928571428571429,
+      "grad_norm": 126.94155434263403,
+      "kl": 0.0,
+      "learning_rate": 1.4029167422908105e-08,
+      "logps/chosen": -279.4980215097403,
+      "logps/rejected": -270.3898013930723,
+      "loss": 0.3016,
+      "rewards/chosen": 1.0889219804243608,
+      "rewards/margins": 5.310351644392734,
+      "rewards/rejected": -4.221429663968373,
+      "step": 1700
+    },
+    {
+      "epoch": 0.898109243697479,
+      "grad_norm": 61.01512892000863,
+      "kl": 0.0,
+      "learning_rate": 1.2698951946511327e-08,
+      "logps/chosen": -302.3543386399371,
+      "logps/rejected": -267.2125873447205,
+      "loss": 0.3007,
+      "rewards/chosen": 0.34754651147614485,
+      "rewards/margins": 4.579919861631424,
+      "rewards/rejected": -4.23237335015528,
+      "step": 1710
+    },
+    {
+      "epoch": 0.9033613445378151,
+      "grad_norm": 41.136022643204946,
+      "kl": 0.0,
+      "learning_rate": 1.1433339912594265e-08,
+      "logps/chosen": -308.92118158284023,
+      "logps/rejected": -271.45762624172187,
+      "loss": 0.3021,
+      "rewards/chosen": 0.45501975336018397,
+      "rewards/margins": 5.495058411653852,
+      "rewards/rejected": -5.040038658293668,
+      "step": 1720
+    },
+    {
+      "epoch": 0.9086134453781513,
+      "grad_norm": 110.83416055708793,
+      "kl": 0.0,
+      "learning_rate": 1.0232675874604608e-08,
+      "logps/chosen": -312.7164007867133,
+      "logps/rejected": -318.51147598870057,
+      "loss": 0.2863,
+      "rewards/chosen": 0.45250099022071677,
+      "rewards/margins": 4.343536339576296,
+      "rewards/rejected": -3.891035349355579,
+      "step": 1730
+    },
+    {
+      "epoch": 0.9138655462184874,
+      "grad_norm": 26.122066134245365,
+      "kl": 0.0,
+      "learning_rate": 9.097286704381896e-09,
+      "logps/chosen": -315.04382560483873,
+      "logps/rejected": -272.66723484848484,
+      "loss": 0.3122,
+      "rewards/chosen": 0.7614468482232863,
+      "rewards/margins": 4.353459750685407,
+      "rewards/rejected": -3.592012902462121,
+      "step": 1740
+    },
+    {
+      "epoch": 0.9191176470588235,
+      "grad_norm": 53.34556224286923,
+      "kl": 0.0,
+      "learning_rate": 8.02748150316937e-09,
+      "logps/chosen": -283.3781151107595,
+      "logps/rejected": -270.81105324074076,
+      "loss": 0.3131,
+      "rewards/chosen": 0.739828278746786,
+      "rewards/margins": 4.5300670241653815,
+      "rewards/rejected": -3.7902387454185957,
+      "step": 1750
+    },
+    {
+      "epoch": 0.9243697478991597,
+      "grad_norm": 87.26559549436006,
+      "kl": 0.0,
+      "learning_rate": 7.023551517463089e-09,
+      "logps/chosen": -271.55817018072287,
+      "logps/rejected": -290.5874594155844,
+      "loss": 0.3368,
+      "rewards/chosen": 0.3112575117363987,
+      "rewards/margins": 3.797579691753647,
+      "rewards/rejected": -3.4863221800172486,
+      "step": 1760
+    },
+    {
+      "epoch": 0.9296218487394958,
+      "grad_norm": 31.56919055846343,
+      "kl": 0.0,
+      "learning_rate": 6.085770059722634e-09,
+      "logps/chosen": -272.63416371855345,
+      "logps/rejected": -279.8696380046584,
+      "loss": 0.2818,
+      "rewards/chosen": 0.9944836118686124,
+      "rewards/margins": 5.193192850999047,
+      "rewards/rejected": -4.198709239130435,
+      "step": 1770
+    },
+    {
+      "epoch": 0.9348739495798319,
+      "grad_norm": 37.83348753574193,
+      "kl": 0.0,
+      "learning_rate": 5.214392433963488e-09,
+      "logps/chosen": -287.8191769622093,
+      "logps/rejected": -277.8159311655405,
+      "loss": 0.3277,
+      "rewards/chosen": 0.5125378453454306,
+      "rewards/margins": 3.781902073465152,
+      "rewards/rejected": -3.2693642281197213,
+      "step": 1780
+    },
+    {
+      "epoch": 0.9401260504201681,
+      "grad_norm": 45.4550208744577,
+      "kl": 0.0,
+      "learning_rate": 4.409655866252693e-09,
+      "logps/chosen": -282.79800397398844,
+      "logps/rejected": -302.68510841836735,
+      "loss": 0.3304,
+      "rewards/chosen": 0.5359358153591266,
+      "rewards/margins": 4.274966925850198,
+      "rewards/rejected": -3.7390311104910716,
+      "step": 1790
+    },
+    {
+      "epoch": 0.9453781512605042,
+      "grad_norm": 47.92999315794789,
+      "kl": 0.0,
+      "learning_rate": 3.671779440125644e-09,
+      "logps/chosen": -299.54941281847135,
+      "logps/rejected": -283.9407831671779,
+      "loss": 0.3024,
+      "rewards/chosen": 0.9072784618207603,
+      "rewards/margins": 4.687179322753467,
+      "rewards/rejected": -3.779900860932707,
+      "step": 1800
+    },
+    {
+      "epoch": 0.9506302521008403,
+      "grad_norm": 32.60296033727532,
+      "kl": 0.0,
+      "learning_rate": 3.000964036942305e-09,
+      "logps/chosen": -278.7902070063694,
+      "logps/rejected": -298.68158071319016,
+      "loss": 0.2903,
+      "rewards/chosen": 0.4591269523474821,
+      "rewards/margins": 4.919044079336555,
+      "rewards/rejected": -4.4599171269890725,
+      "step": 1810
+    },
+    {
+      "epoch": 0.9558823529411765,
+      "grad_norm": 38.232447498754475,
+      "kl": 0.0,
+      "learning_rate": 2.397392281198729e-09,
+      "logps/chosen": -302.90917338709676,
+      "logps/rejected": -267.12088068181816,
+      "loss": 0.301,
+      "rewards/chosen": 0.69748047859438,
+      "rewards/margins": 4.834925731908774,
+      "rewards/rejected": -4.137445253314394,
+      "step": 1820
+    },
+    {
+      "epoch": 0.9611344537815126,
+      "grad_norm": 22.326350506185886,
+      "kl": 0.0,
+      "learning_rate": 1.861228490808886e-09,
+      "logps/chosen": -294.7616077769886,
+      "logps/rejected": -289.75987413194446,
+      "loss": 0.3229,
+      "rewards/chosen": 0.6278800530867144,
+      "rewards/margins": 4.9228116767575045,
+      "rewards/rejected": -4.29493162367079,
+      "step": 1830
+    },
+    {
+      "epoch": 0.9663865546218487,
+      "grad_norm": 40.257301749189914,
+      "kl": 0.0,
+      "learning_rate": 1.3926186323703903e-09,
+      "logps/chosen": -274.1588462271341,
+      "logps/rejected": -264.75538361378204,
+      "loss": 0.3362,
+      "rewards/chosen": 0.5637630369605088,
+      "rewards/margins": 4.09795015822954,
+      "rewards/rejected": -3.5341871212690306,
+      "step": 1840
+    },
+    {
+      "epoch": 0.9716386554621849,
+      "grad_norm": 52.95763177365781,
+      "kl": 0.0,
+      "learning_rate": 9.916902814261774e-10,
+      "logps/chosen": -288.03719758064517,
+      "logps/rejected": -290.255634469697,
+      "loss": 0.271,
+      "rewards/chosen": 0.5408120432207661,
+      "rewards/margins": 5.058847746937622,
+      "rewards/rejected": -4.518035703716856,
+      "step": 1850
+    },
+    {
+      "epoch": 0.976890756302521,
+      "grad_norm": 48.81020773750272,
+      "kl": 0.0,
+      "learning_rate": 6.585525877328968e-10,
+      "logps/chosen": -290.9931640625,
+      "logps/rejected": -323.75256631540697,
+      "loss": 0.3325,
+      "rewards/chosen": 0.10595728899981524,
+      "rewards/margins": 4.301412795790331,
+      "rewards/rejected": -4.195455506790516,
+      "step": 1860
+    },
+    {
+      "epoch": 0.9821428571428571,
+      "grad_norm": 35.96392604697334,
+      "kl": 0.0,
+      "learning_rate": 3.9329624554584883e-10,
+      "logps/chosen": -281.2289627259036,
+      "logps/rejected": -280.72377232142856,
+      "loss": 0.3147,
+      "rewards/chosen": 0.46981898847832737,
+      "rewards/margins": 4.369042811530275,
+      "rewards/rejected": -3.899223823051948,
+      "step": 1870
+    },
+    {
+      "epoch": 0.9873949579831933,
+      "grad_norm": 98.21290759884454,
+      "kl": 0.0,
+      "learning_rate": 1.959934689280962e-10,
+      "logps/chosen": -292.91434151785717,
+      "logps/rejected": -292.6137952302632,
+      "loss": 0.3104,
+      "rewards/chosen": 0.6014651343936012,
+      "rewards/margins": 4.848261591784637,
+      "rewards/rejected": -4.246796457391036,
+      "step": 1880
+    },
+    {
+      "epoch": 0.9926470588235294,
+      "grad_norm": 33.92572486964675,
+      "kl": 0.0,
+      "learning_rate": 6.669797209069017e-11,
+      "logps/chosen": -269.0030942766854,
+      "logps/rejected": -273.2933263644366,
+      "loss": 0.3204,
+      "rewards/chosen": 0.08876853042773986,
+      "rewards/margins": 4.26088122367708,
+      "rewards/rejected": -4.17211269324934,
+      "step": 1890
+    },
+    {
+      "epoch": 0.9978991596638656,
+      "grad_norm": 57.704679882719425,
+      "kl": 0.0,
+      "learning_rate": 5.444954769395771e-12,
+      "logps/chosen": -258.240974378882,
+      "logps/rejected": -283.904309158805,
+      "loss": 0.3031,
+      "rewards/chosen": 0.38151166453865004,
+      "rewards/margins": 4.77392472590854,
+      "rewards/rejected": -4.39241306136989,
+      "step": 1900
+    },
+    {
+      "epoch": 1.0,
+      "step": 1904,
+      "total_flos": 0.0,
+      "train_loss": 0.3297611388589154,
+      "train_runtime": 9235.4216,
+      "train_samples_per_second": 6.596,
+      "train_steps_per_second": 0.206
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1904,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}