Model save

README.md

@@ -3,32 +3,20 @@ library_name: transformers
 license: other
 base_model: trl-lib/qwen1.5-0.5b-sft
 tags:
-- alignment-handbook
 - trl
 - simpo
 - generated_from_trainer
 model-index:
-- name: trl-lib/qwen1.5-0.5b-sft
+- name: simpo-exps_qwen05b
   results: []
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-# trl-lib/qwen1.5-0.5b-sft
+# simpo-exps_qwen05b
 
 This model is a fine-tuned version of [trl-lib/qwen1.5-0.5b-sft](https://huggingface.co/trl-lib/qwen1.5-0.5b-sft) on an unknown dataset.
-It achieves the following results on the evaluation set:
-- Loss: 0.7799
-- Rewards/chosen: -1.7874
-- Rewards/rejected: -1.8737
-- Rewards/accuracies: 0.5215
-- Rewards/margins: 0.0863
-- Logps/rejected: -1.8737
-- Logps/chosen: -1.7874
-- Logits/rejected: -0.3295
-- Logits/chosen: -0.3597
-- Semantic Entropy: 0.6459
 
 ## Model description
 

all_results.json

@@ -1,22 +1,8 @@
 {
     "epoch": 32.0,
-    "eval_logits/chosen": -0.3597286641597748,
-    "eval_logits/rejected": -0.32949814200401306,
-    "eval_logps/chosen": -1.787438988685608,
-    "eval_logps/rejected": -1.8736913204193115,
-    "eval_loss": 0.7798751592636108,
-    "eval_rewards/accuracies": 0.5215133428573608,
-    "eval_rewards/chosen": -1.787438988685608,
-    "eval_rewards/margins": 0.08625216037034988,
-    "eval_rewards/rejected": -1.8736913204193115,
-    "eval_runtime": 60.3848,
-    "eval_samples": 1345,
-    "eval_samples_per_second": 22.274,
-    "eval_semantic_entropy": 0.6458982825279236,
-    "eval_steps_per_second": 5.581,
     "total_flos": 0.0,
-    "train_loss": 0.6878796736399333,
-    "train_runtime": 138.6013,
-    "train_samples_per_second": 6.926,
-    "train_steps_per_second": 0.433
+    "train_loss": 0.6880494674046834,
+    "train_runtime": 138.3641,
+    "train_samples_per_second": 6.938,
+    "train_steps_per_second": 0.434
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 32.0,
     "total_flos": 0.0,
-    "train_loss": 0.6878796736399333,
-    "train_runtime": 138.6013,
-    "train_samples_per_second": 6.926,
-    "train_steps_per_second": 0.433
+    "train_loss": 0.6880494674046834,
+    "train_runtime": 138.3641,
+    "train_samples_per_second": 6.938,
+    "train_steps_per_second": 0.434
 }

trainer_state.json

@@ -10,204 +10,204 @@
   "log_history": [
     {
       "epoch": 2.6666666666666665,
-      "grad_norm": 7.84375,
+      "grad_norm": 7.78125,
       "learning_rate": 6.666666666666667e-08,
-      "logits/chosen": -0.6204631328582764,
-      "logits/rejected": -0.6167532205581665,
-      "logps/chosen": -1.6595089435577393,
-      "logps/rejected": -1.862159013748169,
-      "loss": 0.7095,
+      "logits/chosen": -0.6206714510917664,
+      "logits/rejected": -0.6167551279067993,
+      "logps/chosen": -1.6594607830047607,
+      "logps/rejected": -1.8626664876937866,
+      "loss": 0.7092,
       "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.6595089435577393,
-      "rewards/margins": 0.20264975726604462,
-      "rewards/rejected": -1.862159013748169,
-      "semantic_entropy": 0.6520633697509766,
+      "rewards/chosen": -1.6594607830047607,
+      "rewards/margins": 0.2032059133052826,
+      "rewards/rejected": -1.8626664876937866,
+      "semantic_entropy": 0.6521000862121582,
       "step": 5
     },
     {
       "epoch": 5.333333333333333,
-      "grad_norm": 6.6875,
+      "grad_norm": 6.78125,
       "learning_rate": 7.892179482319296e-08,
-      "logits/chosen": -0.6442058682441711,
-      "logits/rejected": -0.5976763367652893,
-      "logps/chosen": -1.6531527042388916,
-      "logps/rejected": -2.0306272506713867,
-      "loss": 0.6574,
+      "logits/chosen": -0.644172191619873,
+      "logits/rejected": -0.5970994234085083,
+      "logps/chosen": -1.6529489755630493,
+      "logps/rejected": -2.02937912940979,
+      "loss": 0.6576,
       "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -1.6531527042388916,
-      "rewards/margins": 0.37747469544410706,
-      "rewards/rejected": -2.0306272506713867,
-      "semantic_entropy": 0.6555213928222656,
+      "rewards/chosen": -1.6529489755630493,
+      "rewards/margins": 0.3764302134513855,
+      "rewards/rejected": -2.02937912940979,
+      "semantic_entropy": 0.6556634902954102,
       "step": 10
     },
     {
       "epoch": 8.0,
-      "grad_norm": 8.5,
+      "grad_norm": 8.6875,
       "learning_rate": 7.464101615137755e-08,
-      "logits/chosen": -0.6110937595367432,
-      "logits/rejected": -0.6190561056137085,
-      "logps/chosen": -1.653534173965454,
-      "logps/rejected": -1.8783153295516968,
-      "loss": 0.6976,
+      "logits/chosen": -0.6107379794120789,
+      "logits/rejected": -0.6173809766769409,
+      "logps/chosen": -1.6537139415740967,
+      "logps/rejected": -1.878178596496582,
+      "loss": 0.6978,
       "rewards/accuracies": 0.5,
-      "rewards/chosen": -1.653534173965454,
-      "rewards/margins": 0.22478139400482178,
-      "rewards/rejected": -1.8783153295516968,
-      "semantic_entropy": 0.652945876121521,
+      "rewards/chosen": -1.6537139415740967,
+      "rewards/margins": 0.22446465492248535,
+      "rewards/rejected": -1.878178596496582,
+      "semantic_entropy": 0.6528152227401733,
       "step": 15
     },
     {
       "epoch": 10.666666666666666,
       "grad_norm": 8.625,
       "learning_rate": 6.744966551474935e-08,
-      "logits/chosen": -0.6272696852684021,
-      "logits/rejected": -0.6002117395401001,
-      "logps/chosen": -1.6610969305038452,
-      "logps/rejected": -1.9501771926879883,
-      "loss": 0.6824,
+      "logits/chosen": -0.6249920129776001,
+      "logits/rejected": -0.5978578925132751,
+      "logps/chosen": -1.661948561668396,
+      "logps/rejected": -1.9520155191421509,
+      "loss": 0.6819,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -1.6610969305038452,
-      "rewards/margins": 0.28908008337020874,
-      "rewards/rejected": -1.9501771926879883,
-      "semantic_entropy": 0.6478284597396851,
+      "rewards/chosen": -1.661948561668396,
+      "rewards/margins": 0.2900669574737549,
+      "rewards/rejected": -1.9520155191421509,
+      "semantic_entropy": 0.6477808952331543,
       "step": 20
     },
     {
       "epoch": 13.333333333333334,
       "grad_norm": 8.875,
       "learning_rate": 5.7951967208018495e-08,
-      "logits/chosen": -0.6546803712844849,
-      "logits/rejected": -0.6259561777114868,
-      "logps/chosen": -1.686605453491211,
-      "logps/rejected": -1.9335895776748657,
-      "loss": 0.7138,
+      "logits/chosen": -0.6556390523910522,
+      "logits/rejected": -0.6272687911987305,
+      "logps/chosen": -1.6880241632461548,
+      "logps/rejected": -1.9340057373046875,
+      "loss": 0.7142,
       "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.686605453491211,
-      "rewards/margins": 0.24698403477668762,
-      "rewards/rejected": -1.9335895776748657,
-      "semantic_entropy": 0.6518786549568176,
+      "rewards/chosen": -1.6880241632461548,
+      "rewards/margins": 0.24598172307014465,
+      "rewards/rejected": -1.9340057373046875,
+      "semantic_entropy": 0.6515553593635559,
       "step": 25
     },
     {
       "epoch": 16.0,
-      "grad_norm": 6.71875,
+      "grad_norm": 6.875,
       "learning_rate": 4.6945927106677224e-08,
-      "logits/chosen": -0.5952478647232056,
-      "logits/rejected": -0.60741126537323,
-      "logps/chosen": -1.6205002069473267,
-      "logps/rejected": -1.8895899057388306,
-      "loss": 0.6683,
+      "logits/chosen": -0.5940297842025757,
+      "logits/rejected": -0.6073416471481323,
+      "logps/chosen": -1.6197277307510376,
+      "logps/rejected": -1.888943076133728,
+      "loss": 0.6684,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -1.6205002069473267,
-      "rewards/margins": 0.26908960938453674,
-      "rewards/rejected": -1.8895899057388306,
-      "semantic_entropy": 0.6604139804840088,
+      "rewards/chosen": -1.6197277307510376,
+      "rewards/margins": 0.26921549439430237,
+      "rewards/rejected": -1.888943076133728,
+      "semantic_entropy": 0.6606020927429199,
       "step": 30
     },
     {
       "epoch": 18.666666666666668,
-      "grad_norm": 9.125,
+      "grad_norm": 9.1875,
       "learning_rate": 3.535628343499079e-08,
-      "logits/chosen": -0.62486732006073,
-      "logits/rejected": -0.625863254070282,
-      "logps/chosen": -1.7008514404296875,
-      "logps/rejected": -1.9429662227630615,
-      "loss": 0.7077,
+      "logits/chosen": -0.6252874732017517,
+      "logits/rejected": -0.6270566582679749,
+      "logps/chosen": -1.6999114751815796,
+      "logps/rejected": -1.9409929513931274,
+      "loss": 0.7074,
       "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -1.7008514404296875,
-      "rewards/margins": 0.24211446940898895,
-      "rewards/rejected": -1.9429662227630615,
-      "semantic_entropy": 0.640709638595581,
+      "rewards/chosen": -1.6999114751815796,
+      "rewards/margins": 0.24108140170574188,
+      "rewards/rejected": -1.9409929513931274,
+      "semantic_entropy": 0.6408571004867554,
       "step": 35
     },
     {
       "epoch": 21.333333333333332,
-      "grad_norm": 8.25,
+      "grad_norm": 8.125,
       "learning_rate": 2.4156809358433726e-08,
-      "logits/chosen": -0.615136444568634,
-      "logits/rejected": -0.5828085541725159,
-      "logps/chosen": -1.583477258682251,
-      "logps/rejected": -1.8761383295059204,
+      "logits/chosen": -0.6158267259597778,
+      "logits/rejected": -0.5839654803276062,
+      "logps/chosen": -1.583505392074585,
+      "logps/rejected": -1.8752552270889282,
       "loss": 0.6656,
       "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -1.583477258682251,
-      "rewards/margins": 0.29266101121902466,
-      "rewards/rejected": -1.8761383295059204,
-      "semantic_entropy": 0.6757807731628418,
+      "rewards/chosen": -1.583505392074585,
+      "rewards/margins": 0.29174983501434326,
+      "rewards/rejected": -1.8752552270889282,
+      "semantic_entropy": 0.6757909059524536,
       "step": 40
     },
     {
       "epoch": 24.0,
       "grad_norm": 8.75,
       "learning_rate": 1.4288495612538426e-08,
-      "logits/chosen": -0.6335859894752502,
-      "logits/rejected": -0.6234641671180725,
-      "logps/chosen": -1.683307409286499,
-      "logps/rejected": -1.9576094150543213,
-      "loss": 0.6899,
+      "logits/chosen": -0.6340750455856323,
+      "logits/rejected": -0.6243816018104553,
+      "logps/chosen": -1.68179452419281,
+      "logps/rejected": -1.9539234638214111,
+      "loss": 0.6902,
       "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -1.683307409286499,
-      "rewards/margins": 0.27430176734924316,
-      "rewards/rejected": -1.9576094150543213,
-      "semantic_entropy": 0.6437331438064575,
+      "rewards/chosen": -1.68179452419281,
+      "rewards/margins": 0.2721291184425354,
+      "rewards/rejected": -1.9539234638214111,
+      "semantic_entropy": 0.6442986130714417,
       "step": 45
     },
     {
       "epoch": 26.666666666666668,
-      "grad_norm": 8.125,
+      "grad_norm": 8.1875,
       "learning_rate": 6.58048754348255e-09,
-      "logits/chosen": -0.6407667398452759,
-      "logits/rejected": -0.6134266257286072,
-      "logps/chosen": -1.6093534231185913,
-      "logps/rejected": -1.8319820165634155,
-      "loss": 0.6942,
+      "logits/chosen": -0.6418130397796631,
+      "logits/rejected": -0.6143754720687866,
+      "logps/chosen": -1.6074516773223877,
+      "logps/rejected": -1.830583930015564,
+      "loss": 0.694,
       "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.6093534231185913,
-      "rewards/margins": 0.22262856364250183,
-      "rewards/rejected": -1.8319820165634155,
-      "semantic_entropy": 0.6697005033493042,
+      "rewards/chosen": -1.6074516773223877,
+      "rewards/margins": 0.22313210368156433,
+      "rewards/rejected": -1.830583930015564,
+      "semantic_entropy": 0.6700640916824341,
       "step": 50
     },
     {
       "epoch": 29.333333333333332,
-      "grad_norm": 7.9375,
+      "grad_norm": 8.0625,
       "learning_rate": 1.680419507380444e-09,
-      "logits/chosen": -0.6017393469810486,
-      "logits/rejected": -0.5887783765792847,
-      "logps/chosen": -1.6890275478363037,
-      "logps/rejected": -2.0059025287628174,
-      "loss": 0.6753,
+      "logits/chosen": -0.6013139486312866,
+      "logits/rejected": -0.5867229700088501,
+      "logps/chosen": -1.689805030822754,
+      "logps/rejected": -2.002596616744995,
+      "loss": 0.6768,
       "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -1.6890275478363037,
-      "rewards/margins": 0.31687501072883606,
-      "rewards/rejected": -2.0059025287628174,
-      "semantic_entropy": 0.6429846882820129,
+      "rewards/chosen": -1.689805030822754,
+      "rewards/margins": 0.3127916753292084,
+      "rewards/rejected": -2.002596616744995,
+      "semantic_entropy": 0.6430121064186096,
       "step": 55
     },
     {
       "epoch": 32.0,
       "grad_norm": 8.4375,
       "learning_rate": 0.0,
-      "logits/chosen": -0.6315954327583313,
-      "logits/rejected": -0.6313327550888062,
-      "logps/chosen": -1.6676785945892334,
-      "logps/rejected": -1.9383472204208374,
-      "loss": 0.6928,
+      "logits/chosen": -0.6301103830337524,
+      "logits/rejected": -0.6289348006248474,
+      "logps/chosen": -1.665967345237732,
+      "logps/rejected": -1.932885766029358,
+      "loss": 0.6937,
       "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.6676785945892334,
-      "rewards/margins": 0.2706685960292816,
-      "rewards/rejected": -1.9383472204208374,
-      "semantic_entropy": 0.6480283141136169,
+      "rewards/chosen": -1.665967345237732,
+      "rewards/margins": 0.2669183611869812,
+      "rewards/rejected": -1.932885766029358,
+      "semantic_entropy": 0.6483110785484314,
       "step": 60
     },
     {
       "epoch": 32.0,
       "step": 60,
       "total_flos": 0.0,
-      "train_loss": 0.6878796736399333,
-      "train_runtime": 138.6013,
-      "train_samples_per_second": 6.926,
-      "train_steps_per_second": 0.433
+      "train_loss": 0.6880494674046834,
+      "train_runtime": 138.3641,
+      "train_samples_per_second": 6.938,
+      "train_steps_per_second": 0.434
     }
   ],
   "logging_steps": 5,