Model save

README.md

@@ -3,31 +3,20 @@ library_name: transformers
 license: other
 base_model: trl-lib/qwen1.5-0.5b-sft
 tags:
-- alignment-handbook
 - trl
 - simpo
 - generated_from_trainer
 model-index:
-- name: trl-lib/qwen1.5-0.5b-sft
+- name: simpo-exps_qwen05b
   results: []
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-# trl-lib/qwen1.5-0.5b-sft
+# simpo-exps_qwen05b
 
 This model is a fine-tuned version of [trl-lib/qwen1.5-0.5b-sft](https://huggingface.co/trl-lib/qwen1.5-0.5b-sft) on an unknown dataset.
-It achieves the following results on the evaluation set:
-- Loss: 5.8364
-- Rewards/chosen: -16.6952
-- Rewards/rejected: -17.6561
-- Rewards/accuracies: 0.5254
-- Rewards/margins: 0.9610
-- Logps/rejected: -1.7656
-- Logps/chosen: -1.6695
-- Logits/rejected: -0.3813
-- Logits/chosen: -0.4435
 
 ## Model description
 

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 0.004531679270399638,
-    "eval_logits/chosen": -0.44351547956466675,
-    "eval_logits/rejected": -0.3813219964504242,
-    "eval_logps/chosen": -1.669515609741211,
-    "eval_logps/rejected": -1.7656141519546509,
-    "eval_loss": 5.836363315582275,
-    "eval_rewards/accuracies": 0.5254348516464233,
-    "eval_rewards/chosen": -16.69515609741211,
-    "eval_rewards/margins": 0.9609846472740173,
-    "eval_rewards/rejected": -17.65614128112793,
-    "eval_runtime": 292.82,
-    "eval_samples": 5595,
-    "eval_samples_per_second": 19.107,
-    "eval_steps_per_second": 4.778,
     "total_flos": 0.0,
-    "train_loss": 104.63558349609374,
-    "train_runtime": 57.4491,
-    "train_samples_per_second": 5.57,
-    "train_steps_per_second": 0.348
+    "train_loss": 104.54919738769532,
+    "train_runtime": 57.8361,
+    "train_samples_per_second": 5.533,
+    "train_steps_per_second": 0.346
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 0.004531679270399638,
     "total_flos": 0.0,
-    "train_loss": 104.63558349609374,
-    "train_runtime": 57.4491,
-    "train_samples_per_second": 5.57,
-    "train_steps_per_second": 0.348
+    "train_loss": 104.54919738769532,
+    "train_runtime": 57.8361,
+    "train_samples_per_second": 5.533,
+    "train_steps_per_second": 0.346
 }

trainer_state.json

@@ -10,72 +10,76 @@
   "log_history": [
     {
       "epoch": 0.0011329198175999095,
-      "grad_norm": 1640.0,
+      "grad_norm": 1648.0,
       "learning_rate": 7.464101615137755e-08,
-      "logits/chosen": -0.7980253100395203,
-      "logits/rejected": -0.6524801850318909,
-      "logps/chosen": -1.7504403591156006,
-      "logps/rejected": -1.832793951034546,
-      "loss": 106.451,
+      "logits/chosen": -0.7977954149246216,
+      "logits/rejected": -0.6530444622039795,
+      "logps/chosen": -1.7494233846664429,
+      "logps/rejected": -1.8329025506973267,
+      "loss": 106.2595,
       "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -17.504404067993164,
-      "rewards/margins": 0.8235357999801636,
-      "rewards/rejected": -18.327938079833984,
+      "rewards/chosen": -17.49423599243164,
+      "rewards/margins": 0.834790825843811,
+      "rewards/rejected": -18.329025268554688,
+      "semantic_entropy": 0.6561636328697205,
       "step": 5
     },
     {
       "epoch": 0.002265839635199819,
-      "grad_norm": 1176.0,
+      "grad_norm": 1192.0,
       "learning_rate": 4.6945927106677224e-08,
-      "logits/chosen": -0.8232762217521667,
-      "logits/rejected": -0.795711874961853,
-      "logps/chosen": -1.730991005897522,
-      "logps/rejected": -1.7889397144317627,
-      "loss": 103.937,
+      "logits/chosen": -0.8223685026168823,
+      "logits/rejected": -0.7962485551834106,
+      "logps/chosen": -1.7305948734283447,
+      "logps/rejected": -1.790734052658081,
+      "loss": 103.7675,
       "rewards/accuracies": 0.5,
-      "rewards/chosen": -17.30990982055664,
-      "rewards/margins": 0.5794881582260132,
-      "rewards/rejected": -17.8893985748291,
+      "rewards/chosen": -17.305950164794922,
+      "rewards/margins": 0.6013931035995483,
+      "rewards/rejected": -17.90734100341797,
+      "semantic_entropy": 0.6715449094772339,
       "step": 10
     },
     {
       "epoch": 0.003398759452799728,
-      "grad_norm": 2336.0,
+      "grad_norm": 2352.0,
       "learning_rate": 1.4288495612538426e-08,
-      "logits/chosen": -0.7294623851776123,
-      "logits/rejected": -0.7721316814422607,
-      "logps/chosen": -1.7250877618789673,
-      "logps/rejected": -1.7949330806732178,
-      "loss": 98.7826,
+      "logits/chosen": -0.7286165952682495,
+      "logits/rejected": -0.7718230485916138,
+      "logps/chosen": -1.7242145538330078,
+      "logps/rejected": -1.7961229085922241,
+      "loss": 98.6901,
       "rewards/accuracies": 0.5,
-      "rewards/chosen": -17.250879287719727,
-      "rewards/margins": 0.6984527707099915,
-      "rewards/rejected": -17.949331283569336,
+      "rewards/chosen": -17.242145538330078,
+      "rewards/margins": 0.7190819382667542,
+      "rewards/rejected": -17.96122932434082,
+      "semantic_entropy": 0.6638566851615906,
       "step": 15
     },
     {
       "epoch": 0.004531679270399638,
       "grad_norm": 1768.0,
       "learning_rate": 0.0,
-      "logits/chosen": -0.724795937538147,
-      "logits/rejected": -0.6882869601249695,
-      "logps/chosen": -1.827897310256958,
-      "logps/rejected": -1.8333686590194702,
-      "loss": 109.3717,
+      "logits/chosen": -0.7242414355278015,
+      "logits/rejected": -0.6874409914016724,
+      "logps/chosen": -1.828495979309082,
+      "logps/rejected": -1.8325235843658447,
+      "loss": 109.4797,
       "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -18.278972625732422,
-      "rewards/margins": 0.054715633392333984,
-      "rewards/rejected": -18.333688735961914,
+      "rewards/chosen": -18.284961700439453,
+      "rewards/margins": 0.040274858474731445,
+      "rewards/rejected": -18.32523536682129,
+      "semantic_entropy": 0.6504115462303162,
       "step": 20
     },
     {
       "epoch": 0.004531679270399638,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 104.63558349609374,
-      "train_runtime": 57.4491,
-      "train_samples_per_second": 5.57,
-      "train_steps_per_second": 0.348
+      "train_loss": 104.54919738769532,
+      "train_runtime": 57.8361,
+      "train_samples_per_second": 5.533,
+      "train_steps_per_second": 0.346
     }
   ],
   "logging_steps": 5,