Model save

README.md

@@ -1,6 +1,6 @@
 ---
 license: apache-2.0
-base_model: mistralai/Mistral-7B-v0.1
+base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
 - generated_from_trainer
 model-index:
@@ -13,19 +13,19 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-ds
 
-This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6366
-- Rewards/chosen: 0.0331
-- Rewards/rejected: -0.0356
-- Rewards/accuracies: 0.6320
-- Rewards/margins: 0.0687
-- Logps/rejected: -250.3080
-- Logps/chosen: -272.9035
-- Logits/rejected: -2.5200
-- Logits/chosen: -2.5064
-- Use Label: 9174.8564
-- Pred Label: 6857.1440
+- Loss: 0.6636
+- Rewards/chosen: 0.0135
+- Rewards/rejected: -0.0160
+- Rewards/accuracies: 0.6280
+- Rewards/margins: 0.0295
+- Logps/rejected: -259.4594
+- Logps/chosen: -284.1223
+- Logits/rejected: -2.8462
+- Logits/chosen: -2.8424
+- Use Label: 11234.4961
+- Pred Label: 4797.5039
 
 ## Model description
 
@@ -60,9 +60,9 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Use Label | Pred Label |
-|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:---------:|:----------:|
-| 0.6393        | 1.0   | 955  | 0.6366          | 0.0331         | -0.0356          | 0.6320             | 0.0687          | -250.3080      | -272.9035    | -2.5200         | -2.5064       | 8966.8564 | 6565.1440  |
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Use Label  | Pred Label |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:----------:|:----------:|
+| 0.6641        | 1.0   | 955  | 0.6636          | 0.0135         | -0.0160          | 0.6280             | 0.0295          | -259.4594      | -284.1223    | -2.8462         | -2.8424       | 10931.4961 | 4600.5039  |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:400a7361981dd0e63d715116385fd010bda670be4c1e8ed3c62d1e28e1b63a78
+oid sha256:1de4c5f8773caabb175e8e9302697663a01cfa6d9a23a5a3c3f8587fa010e133
 size 109086672

all_results.json

@@ -1,23 +1,23 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.5063796043395996,
-    "eval_logits/rejected": -2.5199685096740723,
-    "eval_logps/chosen": -272.9034729003906,
-    "eval_logps/rejected": -250.30796813964844,
-    "eval_loss": 0.6366299986839294,
-    "eval_pred_label": 6857.14404296875,
-    "eval_rewards/accuracies": 0.6320000290870667,
-    "eval_rewards/chosen": 0.03307868540287018,
-    "eval_rewards/margins": 0.06870328634977341,
-    "eval_rewards/rejected": -0.03562460467219353,
-    "eval_runtime": 469.8823,
+    "eval_logits/chosen": -2.842418670654297,
+    "eval_logits/rejected": -2.846235752105713,
+    "eval_logps/chosen": -284.122314453125,
+    "eval_logps/rejected": -259.4594421386719,
+    "eval_loss": 0.6635700464248657,
+    "eval_pred_label": 4797.50390625,
+    "eval_rewards/accuracies": 0.628000020980835,
+    "eval_rewards/chosen": 0.013506044633686543,
+    "eval_rewards/margins": 0.029479000717401505,
+    "eval_rewards/rejected": -0.015972958877682686,
+    "eval_runtime": 439.3222,
     "eval_samples": 2000,
-    "eval_samples_per_second": 4.256,
-    "eval_steps_per_second": 0.266,
-    "eval_use_label": 9174.8564453125,
-    "train_loss": 0.6567496789063458,
-    "train_runtime": 24286.6837,
+    "eval_samples_per_second": 4.552,
+    "eval_steps_per_second": 0.285,
+    "eval_use_label": 11234.49609375,
+    "train_loss": 0.6728555943953429,
+    "train_runtime": 24272.064,
     "train_samples": 61135,
-    "train_samples_per_second": 2.517,
+    "train_samples_per_second": 2.519,
     "train_steps_per_second": 0.039
 }

eval_results.json

@@ -1,18 +1,18 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.5063796043395996,
-    "eval_logits/rejected": -2.5199685096740723,
-    "eval_logps/chosen": -272.9034729003906,
-    "eval_logps/rejected": -250.30796813964844,
-    "eval_loss": 0.6366299986839294,
-    "eval_pred_label": 6857.14404296875,
-    "eval_rewards/accuracies": 0.6320000290870667,
-    "eval_rewards/chosen": 0.03307868540287018,
-    "eval_rewards/margins": 0.06870328634977341,
-    "eval_rewards/rejected": -0.03562460467219353,
-    "eval_runtime": 469.8823,
+    "eval_logits/chosen": -2.842418670654297,
+    "eval_logits/rejected": -2.846235752105713,
+    "eval_logps/chosen": -284.122314453125,
+    "eval_logps/rejected": -259.4594421386719,
+    "eval_loss": 0.6635700464248657,
+    "eval_pred_label": 4797.50390625,
+    "eval_rewards/accuracies": 0.628000020980835,
+    "eval_rewards/chosen": 0.013506044633686543,
+    "eval_rewards/margins": 0.029479000717401505,
+    "eval_rewards/rejected": -0.015972958877682686,
+    "eval_runtime": 439.3222,
     "eval_samples": 2000,
-    "eval_samples_per_second": 4.256,
-    "eval_steps_per_second": 0.266,
-    "eval_use_label": 9174.8564453125
+    "eval_samples_per_second": 4.552,
+    "eval_steps_per_second": 0.285,
+    "eval_use_label": 11234.49609375
 }

runs/Jan19_16-29-15_uclaml03.cs.ucla.edu/events.out.tfevents.1705710616.uclaml03.cs.ucla.edu.481426.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:963e934d462d48b1a6122736d4c918dbeac5b3da110a29533f949b7bab7228bc
-size 11822
+oid sha256:2af8bfe864f62deac06832b4305bb07472b81b906426793d0f60344cf9a3377d
+size 13023

runs/Jan19_16-29-15_uclaml03.cs.ucla.edu/events.out.tfevents.1705735327.uclaml03.cs.ucla.edu.481426.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a44e6c569c89adfe90d242bf101f53dc16b8c2dfea8e5ded66786852a199ed0c
+size 935

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6567496789063458,
-    "train_runtime": 24286.6837,
+    "train_loss": 0.6728555943953429,
+    "train_runtime": 24272.064,
     "train_samples": 61135,
-    "train_samples_per_second": 2.517,
+    "train_samples_per_second": 2.519,
     "train_steps_per_second": 0.039
 }

trainer_state.json

@@ -11,10 +11,10 @@
     {
       "epoch": 0.0,
       "learning_rate": 5.208333333333333e-09,
-      "logits/chosen": -2.676934003829956,
-      "logits/rejected": -2.509021043777466,
-      "logps/chosen": -304.709228515625,
-      "logps/rejected": -229.49505615234375,
+      "logits/chosen": -2.980285167694092,
+      "logits/rejected": -2.87275767326355,
+      "logps/chosen": -313.4390563964844,
+      "logps/rejected": -236.1754150390625,
       "loss": 0.6931,
       "pred_label": 0.0,
       "rewards/accuracies": 0.0,
@@ -27,172 +27,172 @@
     {
       "epoch": 0.1,
       "learning_rate": 4.976717112922002e-07,
-      "logits/chosen": -2.4758388996124268,
-      "logits/rejected": -2.4836206436157227,
-      "logps/chosen": -273.62322998046875,
-      "logps/rejected": -258.89813232421875,
-      "loss": 0.6823,
-      "pred_label": 156.05050659179688,
-      "rewards/accuracies": 0.4886363744735718,
-      "rewards/chosen": -0.002133187372237444,
-      "rewards/margins": -0.0010164172854274511,
-      "rewards/rejected": -0.0011167696211487055,
+      "logits/chosen": -2.8194870948791504,
+      "logits/rejected": -2.8288567066192627,
+      "logps/chosen": -285.2724304199219,
+      "logps/rejected": -270.956298828125,
+      "loss": 0.6838,
+      "pred_label": 150.7020263671875,
+      "rewards/accuracies": 0.5050504803657532,
+      "rewards/chosen": 0.000817809603177011,
+      "rewards/margins": 0.0014873194741085172,
+      "rewards/rejected": -0.0006695101037621498,
       "step": 100,
-      "use_label": 653.9495239257812
+      "use_label": 659.2979736328125
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.3946449359720607e-07,
-      "logits/chosen": -2.4878945350646973,
-      "logits/rejected": -2.4750781059265137,
-      "logps/chosen": -267.72540283203125,
-      "logps/rejected": -250.30291748046875,
-      "loss": 0.6801,
-      "pred_label": 512.7750244140625,
-      "rewards/accuracies": 0.5575000047683716,
-      "rewards/chosen": 0.003957623615860939,
-      "rewards/margins": 0.008218127302825451,
-      "rewards/rejected": -0.0042605032213032246,
+      "logits/chosen": -2.828075647354126,
+      "logits/rejected": -2.816530227661133,
+      "logps/chosen": -278.7549133300781,
+      "logps/rejected": -260.5694274902344,
+      "loss": 0.683,
+      "pred_label": 452.552490234375,
+      "rewards/accuracies": 0.534375011920929,
+      "rewards/chosen": 0.002707230392843485,
+      "rewards/margins": 0.005774380639195442,
+      "rewards/rejected": -0.003067150479182601,
       "step": 200,
-      "use_label": 1889.2249755859375
+      "use_label": 1949.447509765625
     },
     {
       "epoch": 0.31,
       "learning_rate": 3.812572759022118e-07,
-      "logits/chosen": -2.47322940826416,
-      "logits/rejected": -2.4768893718719482,
-      "logps/chosen": -272.0399169921875,
-      "logps/rejected": -244.90228271484375,
-      "loss": 0.6733,
-      "pred_label": 943.1199951171875,
-      "rewards/accuracies": 0.6075000166893005,
-      "rewards/chosen": 0.010850328952074051,
-      "rewards/margins": 0.01955023780465126,
-      "rewards/rejected": -0.008699909783899784,
+      "logits/chosen": -2.8141846656799316,
+      "logits/rejected": -2.8159701824188232,
+      "logps/chosen": -284.0125732421875,
+      "logps/rejected": -253.9112091064453,
+      "loss": 0.6807,
+      "pred_label": 775.85498046875,
+      "rewards/accuracies": 0.5575000047683716,
+      "rewards/chosen": 0.005504029802978039,
+      "rewards/margins": 0.009370613843202591,
+      "rewards/rejected": -0.0038665838073939085,
       "step": 300,
-      "use_label": 3058.8798828125
+      "use_label": 3226.14501953125
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.230500582072177e-07,
-      "logits/chosen": -2.486250400543213,
-      "logits/rejected": -2.466102361679077,
-      "logps/chosen": -270.7090759277344,
-      "logps/rejected": -252.9911346435547,
-      "loss": 0.6635,
-      "pred_label": 1496.31005859375,
-      "rewards/accuracies": 0.628125011920929,
-      "rewards/chosen": 0.01578013226389885,
-      "rewards/margins": 0.0331539586186409,
-      "rewards/rejected": -0.01737382635474205,
+      "logits/chosen": -2.826817512512207,
+      "logits/rejected": -2.8094358444213867,
+      "logps/chosen": -284.3566589355469,
+      "logps/rejected": -262.80731201171875,
+      "loss": 0.6769,
+      "pred_label": 1149.0574951171875,
+      "rewards/accuracies": 0.5774999856948853,
+      "rewards/chosen": 0.007384983357042074,
+      "rewards/margins": 0.015422500669956207,
+      "rewards/rejected": -0.008037514984607697,
       "step": 400,
-      "use_label": 4105.68994140625
+      "use_label": 4452.9423828125
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.648428405122235e-07,
-      "logits/chosen": -2.46098256111145,
-      "logits/rejected": -2.446549654006958,
-      "logps/chosen": -259.1031799316406,
-      "logps/rejected": -241.98345947265625,
-      "loss": 0.6543,
-      "pred_label": 2180.89990234375,
-      "rewards/accuracies": 0.6175000071525574,
-      "rewards/chosen": 0.019046209752559662,
-      "rewards/margins": 0.03530467674136162,
-      "rewards/rejected": -0.016258466988801956,
+      "logits/chosen": -2.807734966278076,
+      "logits/rejected": -2.796409845352173,
+      "logps/chosen": -269.9852600097656,
+      "logps/rejected": -252.07232666015625,
+      "loss": 0.6728,
+      "pred_label": 1592.5675048828125,
+      "rewards/accuracies": 0.5756250023841858,
+      "rewards/chosen": 0.006774631794542074,
+      "rewards/margins": 0.01554279588162899,
+      "rewards/rejected": -0.008768163621425629,
       "step": 500,
-      "use_label": 5021.10009765625
+      "use_label": 5609.4326171875
     },
     {
       "epoch": 0.63,
       "learning_rate": 2.0663562281722933e-07,
-      "logits/chosen": -2.4896275997161865,
-      "logits/rejected": -2.4663710594177246,
-      "logps/chosen": -271.8394470214844,
-      "logps/rejected": -252.83351135253906,
-      "loss": 0.646,
-      "pred_label": 2942.87255859375,
-      "rewards/accuracies": 0.6600000262260437,
-      "rewards/chosen": 0.02863229252398014,
-      "rewards/margins": 0.05473264306783676,
-      "rewards/rejected": -0.02610035613179207,
+      "logits/chosen": -2.8339650630950928,
+      "logits/rejected": -2.82075572013855,
+      "logps/chosen": -285.0927734375,
+      "logps/rejected": -265.4134826660156,
+      "loss": 0.6681,
+      "pred_label": 2111.6650390625,
+      "rewards/accuracies": 0.6206250190734863,
+      "rewards/chosen": 0.013815036043524742,
+      "rewards/margins": 0.0289370846003294,
+      "rewards/rejected": -0.015122047625482082,
       "step": 600,
-      "use_label": 5859.12744140625
+      "use_label": 6690.3349609375
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.4842840512223514e-07,
-      "logits/chosen": -2.485153913497925,
-      "logits/rejected": -2.472170352935791,
-      "logps/chosen": -269.14117431640625,
-      "logps/rejected": -238.4977569580078,
-      "loss": 0.6421,
-      "pred_label": 3756.925048828125,
-      "rewards/accuracies": 0.6418750286102295,
-      "rewards/chosen": 0.03320219740271568,
-      "rewards/margins": 0.05917687341570854,
-      "rewards/rejected": -0.02597467601299286,
+      "logits/chosen": -2.827232599258423,
+      "logits/rejected": -2.811751127243042,
+      "logps/chosen": -281.4178771972656,
+      "logps/rejected": -248.81068420410156,
+      "loss": 0.6659,
+      "pred_label": 2680.2724609375,
+      "rewards/accuracies": 0.6200000047683716,
+      "rewards/chosen": 0.01417633332312107,
+      "rewards/margins": 0.029135096818208694,
+      "rewards/rejected": -0.014958759769797325,
       "step": 700,
-      "use_label": 6645.0751953125
+      "use_label": 7721.7275390625
     },
     {
       "epoch": 0.84,
       "learning_rate": 9.022118742724097e-08,
-      "logits/chosen": -2.4880199432373047,
-      "logits/rejected": -2.5014524459838867,
-      "logps/chosen": -276.52313232421875,
-      "logps/rejected": -252.34681701660156,
-      "loss": 0.6386,
-      "pred_label": 4625.42236328125,
-      "rewards/accuracies": 0.640625,
-      "rewards/chosen": 0.03754829242825508,
-      "rewards/margins": 0.06682833284139633,
-      "rewards/rejected": -0.02928004413843155,
+      "logits/chosen": -2.8300516605377197,
+      "logits/rejected": -2.835542678833008,
+      "logps/chosen": -288.608642578125,
+      "logps/rejected": -261.5773010253906,
+      "loss": 0.6646,
+      "pred_label": 3286.232421875,
+      "rewards/accuracies": 0.6200000047683716,
+      "rewards/chosen": 0.014839441515505314,
+      "rewards/margins": 0.03136582300066948,
+      "rewards/rejected": -0.01652638241648674,
       "step": 800,
-      "use_label": 7376.57763671875
+      "use_label": 8715.767578125
     },
     {
       "epoch": 0.94,
       "learning_rate": 3.20139697322468e-08,
-      "logits/chosen": -2.4699513912200928,
-      "logits/rejected": -2.485213279724121,
-      "logps/chosen": -266.04229736328125,
-      "logps/rejected": -247.93341064453125,
-      "loss": 0.6393,
-      "pred_label": 5488.7001953125,
-      "rewards/accuracies": 0.6587499976158142,
-      "rewards/chosen": 0.03272656351327896,
-      "rewards/margins": 0.06201673671603203,
-      "rewards/rejected": -0.02929016388952732,
+      "logits/chosen": -2.8211710453033447,
+      "logits/rejected": -2.8280835151672363,
+      "logps/chosen": -277.363525390625,
+      "logps/rejected": -256.4843444824219,
+      "loss": 0.6641,
+      "pred_label": 3882.75244140625,
+      "rewards/accuracies": 0.6331250071525574,
+      "rewards/chosen": 0.01446867547929287,
+      "rewards/margins": 0.02932187356054783,
+      "rewards/rejected": -0.014853193424642086,
       "step": 900,
-      "use_label": 8113.2998046875
+      "use_label": 9719.2470703125
     },
     {
       "epoch": 1.0,
-      "eval_logits/chosen": -2.5063796043395996,
-      "eval_logits/rejected": -2.5199685096740723,
-      "eval_logps/chosen": -272.9034729003906,
-      "eval_logps/rejected": -250.30796813964844,
-      "eval_loss": 0.6366299986839294,
-      "eval_pred_label": 6565.14404296875,
-      "eval_rewards/accuracies": 0.6320000290870667,
-      "eval_rewards/chosen": 0.03307868540287018,
-      "eval_rewards/margins": 0.06870328634977341,
-      "eval_rewards/rejected": -0.03562460467219353,
-      "eval_runtime": 474.2549,
-      "eval_samples_per_second": 4.217,
-      "eval_steps_per_second": 0.264,
-      "eval_use_label": 8966.8564453125,
+      "eval_logits/chosen": -2.842418670654297,
+      "eval_logits/rejected": -2.846235752105713,
+      "eval_logps/chosen": -284.122314453125,
+      "eval_logps/rejected": -259.4594421386719,
+      "eval_loss": 0.6635700464248657,
+      "eval_pred_label": 4600.50390625,
+      "eval_rewards/accuracies": 0.628000020980835,
+      "eval_rewards/chosen": 0.013506044633686543,
+      "eval_rewards/margins": 0.029479000717401505,
+      "eval_rewards/rejected": -0.015972958877682686,
+      "eval_runtime": 438.8322,
+      "eval_samples_per_second": 4.558,
+      "eval_steps_per_second": 0.285,
+      "eval_use_label": 10931.49609375,
       "step": 955
     },
     {
       "epoch": 1.0,
       "step": 955,
       "total_flos": 0.0,
-      "train_loss": 0.6567496789063458,
-      "train_runtime": 24286.6837,
-      "train_samples_per_second": 2.517,
+      "train_loss": 0.6728555943953429,
+      "train_runtime": 24272.064,
+      "train_samples_per_second": 2.519,
       "train_steps_per_second": 0.039
     }
   ],