Update ckpt

LEGAL.md

@@ -1,7 +0,0 @@
-Legal Disclaimer
-
-Within this source code, the comments in Chinese shall be the original, governing version. Any comment in other languages are for reference only. In the event of any conflict between the Chinese language version comments and other language version comments, the Chinese language version shall prevail.
-
-法律免责声明
-
-关于代码注释部分，中文注释为官方版本，其它语言注释仅做参考。中文注释可能与其它语言注释存在不一致，当中文注释与其它语言注释存在不一致时，请以中文注释为准。

Readme.md

@@ -1,2 +0,0 @@
-# Readme
-

trainer_state.json

@@ -1,1434 +0,0 @@
-{
-  "best_global_step": null,
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 0.5235602094240838,
-  "eval_steps": 100000,
-  "global_step": 50,
-  "is_hyper_param_search": false,
-  "is_local_process_zero": true,
-  "is_world_process_zero": true,
-  "log_history": [
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 59.8125,
-      "completion_length_2": 576.0,
-      "compre": 0.7549091279506683,
-      "compre_std": 0.14603755250573158,
-      "epoch": 0.010471204188481676,
-      "finalgen": 0.7989666610956192,
-      "first_gen": 0.7695323675870895,
-      "first_gen_std": 0.09897536411881447,
-      "geneval_final": 0.8333281576633453,
-      "geneval_first": 0.833327904343605,
-      "geneval_second": 0.7552048228681087,
-      "grad_norm": 1.0952143669128418,
-      "internvl_final": 0.7692298889160156,
-      "internvl_first": 0.7423409670591354,
-      "internvl_second": 0.7017457336187363,
-      "kl_0": 0.0,
-      "kl_1": 0.0,
-      "kl_2": 0.0,
-      "learning_rate": 1.0000000000000002e-06,
-      "loss": 0.0,
-      "rewards/selfcheck": 0.640625,
-      "second_gen": 0.7350995540618896,
-      "second_gen_std": 0.16677509993314743,
-      "step": 1,
-      "tot_reward": 2.2595410495996475
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 39.84375,
-      "completion_length_2": 576.0,
-      "compre": 0.737465888261795,
-      "compre_std": 0.23401770368218422,
-      "epoch": 0.020942408376963352,
-      "finalgen": 0.7441405802965164,
-      "first_gen": 0.7343152761459351,
-      "first_gen_std": 0.17313585616648197,
-      "geneval_final": 0.6437513083219528,
-      "geneval_first": 0.653126060962677,
-      "geneval_second": 0.5812521427869797,
-      "grad_norm": 1.0801606178283691,
-      "internvl_final": 0.8145325481891632,
-      "internvl_first": 0.8075367659330368,
-      "internvl_second": 0.7767415642738342,
-      "kl_0": 0.0,
-      "kl_1": 0.0,
-      "kl_2": 0.0,
-      "learning_rate": 2.0000000000000003e-06,
-      "loss": 0.0,
-      "rewards/selfcheck": 0.66796875,
-      "second_gen": 0.6918127685785294,
-      "second_gen_std": 0.1750396341085434,
-      "step": 2,
-      "tot_reward": 2.1635939329862595
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 48.5625,
-      "completion_length_2": 576.0,
-      "compre": 0.6974785923957825,
-      "compre_std": 0.2092800009995699,
-      "epoch": 0.031413612565445025,
-      "finalgen": 0.7925812155008316,
-      "first_gen": 0.8119042217731476,
-      "first_gen_std": 0.16565837524831295,
-      "geneval_final": 0.8124982565641403,
-      "geneval_first": 0.8255188465118408,
-      "geneval_second": 0.7421867400407791,
-      "grad_norm": 1.3857570886611938,
-      "internvl_final": 0.7827608734369278,
-      "internvl_first": 0.8039457201957703,
-      "internvl_second": 0.741925835609436,
-      "kl_0": 0.004405975341796875,
-      "kl_1": 0.00030541419982910156,
-      "kl_2": 0.00640106201171875,
-      "learning_rate": 3.0000000000000005e-06,
-      "loss": 0.0001,
-      "rewards/selfcheck": 0.66015625,
-      "second_gen": 0.7375303208827972,
-      "second_gen_std": 0.244943518191576,
-      "step": 3,
-      "tot_reward": 2.2469131350517273
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 63.875,
-      "completion_length_2": 576.0,
-      "compre": 0.6304676532745361,
-      "compre_std": 0.2166498824954033,
-      "epoch": 0.041884816753926704,
-      "finalgen": 0.706999346613884,
-      "first_gen": 0.7176250070333481,
-      "first_gen_std": 0.1319098360836506,
-      "geneval_final": 0.5312501322478056,
-      "geneval_first": 0.5833326280117035,
-      "geneval_second": 0.42708506839153415,
-      "grad_norm": 0.9866518378257751,
-      "internvl_final": 0.7840945720672607,
-      "internvl_first": 0.7837744653224945,
-      "internvl_second": 0.7741784453392029,
-      "kl_0": 0.00339508056640625,
-      "kl_1": 0.00030803680419921875,
-      "kl_2": 0.005641937255859375,
-      "learning_rate": 4.000000000000001e-06,
-      "loss": 0.0001,
-      "rewards/selfcheck": 0.59765625,
-      "second_gen": 0.6680488586425781,
-      "second_gen_std": 0.13900070078670979,
-      "step": 4,
-      "tot_reward": 2.0161415189504623
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 56.875,
-      "completion_length_2": 576.0,
-      "compre": 0.7337857931852341,
-      "compre_std": 0.17267515510320663,
-      "epoch": 0.05235602094240838,
-      "finalgen": 0.8179268836975098,
-      "first_gen": 0.8474402278661728,
-      "first_gen_std": 0.10339293256402016,
-      "geneval_final": 0.4921884387731552,
-      "geneval_first": 0.5651040226221085,
-      "geneval_second": 0.4505223035812378,
-      "grad_norm": 0.9777835607528687,
-      "internvl_final": 0.88887158036232,
-      "internvl_first": 0.8897245526313782,
-      "internvl_second": 0.8588467538356781,
-      "kl_0": 0.00335693359375,
-      "kl_1": 0.0003905296325683594,
-      "kl_2": 0.006153106689453125,
-      "learning_rate": 5e-06,
-      "loss": 0.0001,
-      "rewards/selfcheck": 0.6484375,
-      "second_gen": 0.768099382519722,
-      "second_gen_std": 0.19330058619379997,
-      "step": 5,
-      "tot_reward": 2.349325403571129
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 70.28125,
-      "completion_length_2": 576.0,
-      "compre": 0.8213787376880646,
-      "compre_std": 0.17125927656888962,
-      "epoch": 0.06282722513089005,
-      "finalgen": 0.8025798797607422,
-      "first_gen": 0.792873278260231,
-      "first_gen_std": 0.1552301160991192,
-      "geneval_final": 0.7744787484407425,
-      "geneval_first": 0.7614581435918808,
-      "geneval_second": 0.638022243976593,
-      "grad_norm": 0.926077663898468,
-      "internvl_final": 0.8014426976442337,
-      "internvl_first": 0.7876186370849609,
-      "internvl_second": 0.7742805629968643,
-      "kl_0": 0.003208160400390625,
-      "kl_1": 0.004992485046386719,
-      "kl_2": 0.00876617431640625,
-      "learning_rate": 4.84e-06,
-      "loss": 0.0005,
-      "rewards/selfcheck": 0.72265625,
-      "second_gen": 0.7216387689113617,
-      "second_gen_std": 0.16806087456643581,
-      "step": 6,
-      "tot_reward": 2.3358907848596573
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 52.0,
-      "completion_length_2": 576.0,
-      "compre": 0.8553071022033691,
-      "compre_std": 0.11782849393785,
-      "epoch": 0.07329842931937172,
-      "finalgen": 0.8177923709154129,
-      "first_gen": 0.784370481967926,
-      "first_gen_std": 0.12809692788869143,
-      "geneval_final": 0.8593727946281433,
-      "geneval_first": 0.7786447405815125,
-      "geneval_second": 0.7109374031424522,
-      "grad_norm": 0.9277043342590332,
-      "internvl_final": 0.7972260862588882,
-      "internvl_first": 0.7938382625579834,
-      "internvl_second": 0.7694690227508545,
-      "kl_0": 0.004093170166015625,
-      "kl_1": 0.003676891326904297,
-      "kl_2": 0.0273284912109375,
-      "learning_rate": 4.68e-06,
-      "loss": 0.0013,
-      "rewards/selfcheck": 0.72265625,
-      "second_gen": 0.7431355491280556,
-      "second_gen_std": 0.19814559444785118,
-      "step": 7,
-      "tot_reward": 2.3828131332993507
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 62.15625,
-      "completion_length_2": 576.0,
-      "compre": 0.7571111023426056,
-      "compre_std": 0.18298335000872612,
-      "epoch": 0.08376963350785341,
-      "finalgen": 0.7291563600301743,
-      "first_gen": 0.6889523267745972,
-      "first_gen_std": 0.18111810460686684,
-      "geneval_final": 0.5364595651626587,
-      "geneval_first": 0.4895849898457527,
-      "geneval_second": 0.39583662897348404,
-      "grad_norm": 3.1168320178985596,
-      "internvl_final": 0.8005142956972122,
-      "internvl_first": 0.7751448452472687,
-      "internvl_second": 0.7655676752328873,
-      "kl_0": 0.00445556640625,
-      "kl_1": 0.3070030212402344,
-      "kl_2": 0.050140380859375,
-      "learning_rate": 4.520000000000001e-06,
-      "loss": 0.0085,
-      "rewards/selfcheck": 0.640625,
-      "second_gen": 0.6585755944252014,
-      "second_gen_std": 0.1962592490017414,
-      "step": 8,
-      "tot_reward": 2.104639023542404
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 59.03125,
-      "completion_length_2": 576.0,
-      "compre": 0.8145061582326889,
-      "compre_std": 0.20007650181651115,
-      "epoch": 0.09424083769633508,
-      "finalgen": 0.7829647809267044,
-      "first_gen": 0.7806855589151382,
-      "first_gen_std": 0.1751751657575369,
-      "geneval_final": 0.6697908341884613,
-      "geneval_first": 0.6677073389291763,
-      "geneval_second": 0.5239605121314526,
-      "grad_norm": 1.870141625404358,
-      "internvl_final": 0.8357957899570465,
-      "internvl_first": 0.842738002538681,
-      "internvl_second": 0.8030338883399963,
-      "kl_0": 0.0045166015625,
-      "kl_1": 0.00498199462890625,
-      "kl_2": 0.26806640625,
-      "learning_rate": 4.360000000000001e-06,
-      "loss": 0.005,
-      "rewards/selfcheck": 0.8671875,
-      "second_gen": 0.7249370217323303,
-      "second_gen_std": 0.2135489135980606,
-      "step": 9,
-      "tot_reward": 2.3201287388801575
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 51.4375,
-      "completion_length_2": 576.0,
-      "compre": 0.8112478852272034,
-      "compre_std": 0.11972623318433762,
-      "epoch": 0.10471204188481675,
-      "finalgen": 0.8595622777938843,
-      "first_gen": 0.8183109611272812,
-      "first_gen_std": 0.1156176682561636,
-      "geneval_final": 0.8749973326921463,
-      "geneval_first": 0.8124981969594955,
-      "geneval_second": 0.8645808100700378,
-      "grad_norm": 1.774263620376587,
-      "internvl_final": 0.8654665052890778,
-      "internvl_first": 0.83393794298172,
-      "internvl_second": 0.7860487401485443,
-      "kl_0": 0.00567626953125,
-      "kl_1": 0.03774833679199219,
-      "kl_2": 0.168212890625,
-      "learning_rate": 4.2000000000000004e-06,
-      "loss": 0.0067,
-      "rewards/selfcheck": 0.83984375,
-      "second_gen": 0.812427893280983,
-      "second_gen_std": 0.1961305681616068,
-      "step": 10,
-      "tot_reward": 2.4419867396354675
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 59.4375,
-      "completion_length_2": 576.0,
-      "compre": 0.7816697210073471,
-      "compre_std": 0.16708647273480892,
-      "epoch": 0.11518324607329843,
-      "finalgen": 0.7165499031543732,
-      "first_gen": 0.7182296440005302,
-      "first_gen_std": 0.16919826716184616,
-      "geneval_final": 0.68958330899477,
-      "geneval_first": 0.6770834550261497,
-      "geneval_second": 0.5447937361896038,
-      "grad_norm": 1.3713111877441406,
-      "internvl_final": 0.7387097775936127,
-      "internvl_first": 0.7484088242053986,
-      "internvl_second": 0.6730778962373734,
-      "kl_0": 0.00601959228515625,
-      "kl_1": 0.01692962646484375,
-      "kl_2": 0.1885986328125,
-      "learning_rate": 4.04e-06,
-      "loss": 0.0052,
-      "rewards/selfcheck": 0.70703125,
-      "second_gen": 0.6220707595348358,
-      "second_gen_std": 0.23473705351352692,
-      "step": 11,
-      "tot_reward": 2.121970124542713
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 56.96875,
-      "completion_length_2": 576.0,
-      "compre": 0.7651939988136292,
-      "compre_std": 0.16647255420684814,
-      "epoch": 0.1256544502617801,
-      "finalgen": 0.737395167350769,
-      "first_gen": 0.7266053706407547,
-      "first_gen_std": 0.16155973821878433,
-      "geneval_final": 0.5595262910433121,
-      "geneval_first": 0.6250007450580597,
-      "geneval_second": 0.48363446846678926,
-      "grad_norm": 1.548578143119812,
-      "internvl_final": 0.8127259463071823,
-      "internvl_first": 0.8052111566066742,
-      "internvl_second": 0.7006373405456543,
-      "kl_0": 0.0052490234375,
-      "kl_1": 0.105865478515625,
-      "kl_2": 0.1307373046875,
-      "learning_rate": 3.88e-06,
-      "loss": 0.0033,
-      "rewards/selfcheck": 0.7265625,
-      "second_gen": 0.6569458693265915,
-      "second_gen_std": 0.19354603812098503,
-      "step": 12,
-      "tot_reward": 2.1487452387809753
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 50.96875,
-      "completion_length_2": 576.0,
-      "compre": 0.7275254130363464,
-      "compre_std": 0.22794683650135994,
-      "epoch": 0.13612565445026178,
-      "finalgen": 0.771732971072197,
-      "first_gen": 0.7685244232416153,
-      "first_gen_std": 0.18027238175272942,
-      "geneval_final": 0.7343721091747284,
-      "geneval_first": 0.7031228244304657,
-      "geneval_second": 0.5703130066394806,
-      "grad_norm": 0.9769389629364014,
-      "internvl_final": 0.8102993816137314,
-      "internvl_first": 0.8013420253992081,
-      "internvl_second": 0.7744639068841934,
-      "kl_0": 0.005084991455078125,
-      "kl_1": 0.18712615966796875,
-      "kl_2": 0.159912109375,
-      "learning_rate": 3.7200000000000004e-06,
-      "loss": 0.0035,
-      "rewards/selfcheck": 0.66796875,
-      "second_gen": 0.7124785929918289,
-      "second_gen_std": 0.20800333097577095,
-      "step": 13,
-      "tot_reward": 2.2085284292697906
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 47.96875,
-      "completion_length_2": 576.0,
-      "compre": 0.7608457058668137,
-      "compre_std": 0.20356886088848114,
-      "epoch": 0.14659685863874344,
-      "finalgen": 0.7777686715126038,
-      "first_gen": 0.788392037153244,
-      "first_gen_std": 0.19788672029972076,
-      "geneval_final": 0.721354216337204,
-      "geneval_first": 0.7369789481163025,
-      "geneval_second": 0.611980676651001,
-      "grad_norm": 1.6281715631484985,
-      "internvl_final": 0.8137162774801254,
-      "internvl_first": 0.8182865977287292,
-      "internvl_second": 0.749033197760582,
-      "kl_0": 0.005817413330078125,
-      "kl_1": 0.04917144775390625,
-      "kl_2": 0.12237548828125,
-      "learning_rate": 3.5600000000000002e-06,
-      "loss": 0.0028,
-      "rewards/selfcheck": 0.72265625,
-      "second_gen": 0.6939488351345062,
-      "second_gen_std": 0.26941393315792084,
-      "step": 14,
-      "tot_reward": 2.243186578154564
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 55.375,
-      "completion_length_2": 576.0,
-      "compre": 0.7917969673871994,
-      "compre_std": 0.21661141142249107,
-      "epoch": 0.15706806282722513,
-      "finalgen": 0.7536004930734634,
-      "first_gen": 0.7422216385602951,
-      "first_gen_std": 0.1565408930182457,
-      "geneval_final": 0.6354171559214592,
-      "geneval_first": 0.6197923645377159,
-      "geneval_second": 0.567709818482399,
-      "grad_norm": 1.154263973236084,
-      "internvl_final": 0.8050427883863449,
-      "internvl_first": 0.7937590032815933,
-      "internvl_second": 0.7572074085474014,
-      "kl_0": 0.005199432373046875,
-      "kl_1": 0.057464599609375,
-      "kl_2": 0.116455078125,
-      "learning_rate": 3.4000000000000005e-06,
-      "loss": 0.0028,
-      "rewards/selfcheck": 0.74609375,
-      "second_gen": 0.6974593102931976,
-      "second_gen_std": 0.16282034665346146,
-      "step": 15,
-      "tot_reward": 2.231477916240692
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 51.90625,
-      "completion_length_2": 576.0,
-      "compre": 0.7714394629001617,
-      "compre_std": 0.15307269245386124,
-      "epoch": 0.16753926701570682,
-      "finalgen": 0.8282599300146103,
-      "first_gen": 0.8087948560714722,
-      "first_gen_std": 0.1717423852533102,
-      "geneval_final": 0.8437473922967911,
-      "geneval_first": 0.794269010424614,
-      "geneval_second": 0.7343740463256836,
-      "grad_norm": 0.7912507057189941,
-      "internvl_final": 0.8215305358171463,
-      "internvl_first": 0.8133489638566971,
-      "internvl_second": 0.8031008988618851,
-      "kl_0": 0.00527191162109375,
-      "kl_1": 0.067535400390625,
-      "kl_2": 0.11920166015625,
-      "learning_rate": 3.2400000000000003e-06,
-      "loss": 0.0026,
-      "rewards/selfcheck": 0.76171875,
-      "second_gen": 0.7794294506311417,
-      "second_gen_std": 0.21150891855359077,
-      "step": 16,
-      "tot_reward": 2.3596637696027756
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 50.84375,
-      "completion_length_2": 576.0,
-      "compre": 0.8622442930936813,
-      "compre_std": 0.1704554744064808,
-      "epoch": 0.17801047120418848,
-      "finalgen": 0.8196248710155487,
-      "first_gen": 0.7956789284944534,
-      "first_gen_std": 0.1943669207394123,
-      "geneval_final": 0.7427082657814026,
-      "geneval_first": 0.7041671276092529,
-      "geneval_second": 0.6416679620742798,
-      "grad_norm": 0.9625406265258789,
-      "internvl_final": 0.8516624420881271,
-      "internvl_first": 0.8354064524173737,
-      "internvl_second": 0.8153494000434875,
-      "kl_0": 0.00511932373046875,
-      "kl_1": 0.0963134765625,
-      "kl_2": 0.14788818359375,
-      "learning_rate": 3.08e-06,
-      "loss": 0.0023,
-      "rewards/selfcheck": 0.84375,
-      "second_gen": 0.7515811622142792,
-      "second_gen_std": 0.22218142822384834,
-      "step": 17,
-      "tot_reward": 2.409504383802414
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 48.40625,
-      "completion_length_2": 576.0,
-      "compre": 0.861387774348259,
-      "compre_std": 0.13461659289896488,
-      "epoch": 0.18848167539267016,
-      "finalgen": 0.8284620344638824,
-      "first_gen": 0.8211266100406647,
-      "first_gen_std": 0.17254696041345596,
-      "geneval_final": 0.7864573150873184,
-      "geneval_first": 0.7812490016222,
-      "geneval_second": 0.6484384983778,
-      "grad_norm": 0.7912412881851196,
-      "internvl_final": 0.8521965593099594,
-      "internvl_first": 0.8812606334686279,
-      "internvl_second": 0.7947878241539001,
-      "kl_0": 0.004608154296875,
-      "kl_1": 0.0240936279296875,
-      "kl_2": 0.09210205078125,
-      "learning_rate": 2.92e-06,
-      "loss": 0.0021,
-      "rewards/selfcheck": 0.78125,
-      "second_gen": 0.7385903745889664,
-      "second_gen_std": 0.20756110176444054,
-      "step": 18,
-      "tot_reward": 2.42110475897789
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 49.28125,
-      "completion_length_2": 576.0,
-      "compre": 0.7792719453573227,
-      "compre_std": 0.18045332841575146,
-      "epoch": 0.19895287958115182,
-      "finalgen": 0.7645432502031326,
-      "first_gen": 0.7664026916027069,
-      "first_gen_std": 0.1389799863100052,
-      "geneval_final": 0.7083334475755692,
-      "geneval_first": 0.6979169696569443,
-      "geneval_second": 0.674479752779007,
-      "grad_norm": 1.4261727333068848,
-      "internvl_final": 0.7876286506652832,
-      "internvl_first": 0.7962436228990555,
-      "internvl_second": 0.7371761798858643,
-      "kl_0": 0.004669189453125,
-      "kl_1": 0.01157379150390625,
-      "kl_2": 0.12786865234375,
-      "learning_rate": 2.7600000000000003e-06,
-      "loss": 0.0022,
-      "rewards/selfcheck": 0.6796875,
-      "second_gen": 0.7210484892129898,
-      "second_gen_std": 0.16337783448398113,
-      "step": 19,
-      "tot_reward": 2.2667231261730194
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 43.25,
-      "completion_length_2": 576.0,
-      "compre": 0.8075016289949417,
-      "compre_std": 0.1704008486121893,
-      "epoch": 0.2094240837696335,
-      "finalgen": 0.7825709283351898,
-      "first_gen": 0.7692948579788208,
-      "first_gen_std": 0.13403410837054253,
-      "geneval_final": 0.7145832553505898,
-      "geneval_first": 0.6958335563540459,
-      "geneval_second": 0.6552091091871262,
-      "grad_norm": 0.9123572111129761,
-      "internvl_final": 0.8629064708948135,
-      "internvl_first": 0.8535181134939194,
-      "internvl_second": 0.8180961459875107,
-      "kl_0": 0.00579833984375,
-      "kl_1": 0.050815582275390625,
-      "kl_2": 0.064666748046875,
-      "learning_rate": 2.6e-06,
-      "loss": 0.0024,
-      "rewards/selfcheck": 0.7109375,
-      "second_gen": 0.7336102575063705,
-      "second_gen_std": 0.17554061487317085,
-      "step": 20,
-      "tot_reward": 2.310406744480133
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 63.96875,
-      "completion_length_2": 576.0,
-      "compre": 0.776437446475029,
-      "compre_std": 0.18083850666880608,
-      "epoch": 0.2198952879581152,
-      "finalgen": 0.7892505973577499,
-      "first_gen": 0.7880778908729553,
-      "first_gen_std": 0.12110341340303421,
-      "geneval_final": 0.5729151666164398,
-      "geneval_first": 0.578123465180397,
-      "geneval_second": 0.5781233906745911,
-      "grad_norm": 1.147928237915039,
-      "internvl_final": 0.8042012304067612,
-      "internvl_first": 0.7973868250846863,
-      "internvl_second": 0.7700908780097961,
-      "kl_0": 0.00434112548828125,
-      "kl_1": 0.070892333984375,
-      "kl_2": 0.0592803955078125,
-      "learning_rate": 2.4400000000000004e-06,
-      "loss": 0.0019,
-      "rewards/selfcheck": 0.7890625,
-      "second_gen": 0.7677064388990402,
-      "second_gen_std": 0.14560945704579353,
-      "step": 21,
-      "tot_reward": 2.3322217762470245
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 46.78125,
-      "completion_length_2": 576.0,
-      "compre": 0.7662104368209839,
-      "compre_std": 0.18649493157863617,
-      "epoch": 0.23036649214659685,
-      "finalgen": 0.7516999542713165,
-      "first_gen": 0.7476924508810043,
-      "first_gen_std": 0.16294222325086594,
-      "geneval_final": 0.49739643931388855,
-      "geneval_first": 0.5156253203749657,
-      "geneval_second": 0.437501423060894,
-      "grad_norm": 0.7344913482666016,
-      "internvl_final": 0.8158693909645081,
-      "internvl_first": 0.8004629462957382,
-      "internvl_second": 0.7996131181716919,
-      "kl_0": 0.00626373291015625,
-      "kl_1": 0.011867523193359375,
-      "kl_2": 0.0703125,
-      "learning_rate": 2.2800000000000006e-06,
-      "loss": 0.0031,
-      "rewards/selfcheck": 0.67578125,
-      "second_gen": 0.7156740128993988,
-      "second_gen_std": 0.1773982010781765,
-      "step": 22,
-      "tot_reward": 2.229576900601387
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 38.59375,
-      "completion_length_2": 576.0,
-      "compre": 0.8990298211574554,
-      "compre_std": 0.08603513706475496,
-      "epoch": 0.24083769633507854,
-      "finalgen": 0.8516775518655777,
-      "first_gen": 0.8254719525575638,
-      "first_gen_std": 0.09153617918491364,
-      "geneval_final": 0.8333310335874557,
-      "geneval_first": 0.7708318829536438,
-      "geneval_second": 0.8229145109653473,
-      "grad_norm": 0.6684104800224304,
-      "internvl_final": 0.8759130835533142,
-      "internvl_first": 0.8701326102018356,
-      "internvl_second": 0.8153904676437378,
-      "kl_0": 0.00778961181640625,
-      "kl_1": 0.01245880126953125,
-      "kl_2": 0.0904541015625,
-      "learning_rate": 2.1200000000000005e-06,
-      "loss": 0.002,
-      "rewards/selfcheck": 0.81640625,
-      "second_gen": 0.8093397468328476,
-      "second_gen_std": 0.13964065350592136,
-      "step": 23,
-      "tot_reward": 2.533841520547867
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 50.21875,
-      "completion_length_2": 576.0,
-      "compre": 0.8330367654561996,
-      "compre_std": 0.16215411643497646,
-      "epoch": 0.2513089005235602,
-      "finalgen": 0.7843883782625198,
-      "first_gen": 0.7797036319971085,
-      "first_gen_std": 0.12760567921213806,
-      "geneval_final": 0.6000005006790161,
-      "geneval_first": 0.5979171544313431,
-      "geneval_second": 0.4541696608066559,
-      "grad_norm": 0.6334791779518127,
-      "internvl_final": 0.8782133162021637,
-      "internvl_first": 0.8767320364713669,
-      "internvl_second": 0.8230772018432617,
-      "kl_0": 0.0048828125,
-      "kl_1": 0.0131378173828125,
-      "kl_2": 0.048858642578125,
-      "learning_rate": 1.9600000000000003e-06,
-      "loss": 0.0018,
-      "rewards/selfcheck": 0.828125,
-      "second_gen": 0.7158899009227753,
-      "second_gen_std": 0.15695794858038425,
-      "step": 24,
-      "tot_reward": 2.3286302983760834
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 51.78125,
-      "completion_length_2": 576.0,
-      "compre": 0.8843115568161011,
-      "compre_std": 0.1233286838978529,
-      "epoch": 0.2617801047120419,
-      "finalgen": 0.8016263395547867,
-      "first_gen": 0.7955255508422852,
-      "first_gen_std": 0.13115915283560753,
-      "geneval_final": 0.7031246274709702,
-      "geneval_first": 0.7057287618517876,
-      "geneval_second": 0.5781264305114746,
-      "grad_norm": 1.0080742835998535,
-      "internvl_final": 0.8972594141960144,
-      "internvl_first": 0.8881365358829498,
-      "internvl_second": 0.8625568002462387,
-      "kl_0": 0.00562286376953125,
-      "kl_1": 0.161407470703125,
-      "kl_2": 0.068145751953125,
-      "learning_rate": 1.8000000000000001e-06,
-      "loss": 0.0022,
-      "rewards/selfcheck": 0.8046875,
-      "second_gen": 0.737279087305069,
-      "second_gen_std": 0.15692505612969398,
-      "step": 25,
-      "tot_reward": 2.417116194963455
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 62.65625,
-      "completion_length_2": 576.0,
-      "compre": 0.7978219985961914,
-      "compre_std": 0.17816441133618355,
-      "epoch": 0.27225130890052357,
-      "finalgen": 0.7744689732789993,
-      "first_gen": 0.7429674714803696,
-      "first_gen_std": 0.16385456919670105,
-      "geneval_final": 0.4270826429128647,
-      "geneval_first": 0.40624966472387314,
-      "geneval_second": 0.4010413736104965,
-      "grad_norm": 0.7011401057243347,
-      "internvl_final": 0.8419179618358612,
-      "internvl_first": 0.8042358160018921,
-      "internvl_second": 0.7991660982370377,
-      "kl_0": 0.0066375732421875,
-      "kl_1": 0.021697998046875,
-      "kl_2": 0.07000732421875,
-      "learning_rate": 1.64e-06,
-      "loss": 0.0022,
-      "rewards/selfcheck": 0.76953125,
-      "second_gen": 0.7295635938644409,
-      "second_gen_std": 0.17908411100506783,
-      "step": 26,
-      "tot_reward": 2.270353063941002
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 52.21875,
-      "completion_length_2": 576.0,
-      "compre": 0.8099090754985809,
-      "compre_std": 0.174162395298481,
-      "epoch": 0.28272251308900526,
-      "finalgen": 0.7575195133686066,
-      "first_gen": 0.7415773570537567,
-      "first_gen_std": 0.1676672399044037,
-      "geneval_final": 0.7812484055757523,
-      "geneval_first": 0.7718735635280609,
-      "geneval_second": 0.7218743413686752,
-      "grad_norm": 0.8027878999710083,
-      "internvl_final": 0.8015442937612534,
-      "internvl_first": 0.7773027122020721,
-      "internvl_second": 0.7680691182613373,
-      "kl_0": 0.006519317626953125,
-      "kl_1": 0.00907135009765625,
-      "kl_2": 0.057464599609375,
-      "learning_rate": 1.4800000000000002e-06,
-      "loss": 0.0028,
-      "rewards/selfcheck": 0.7578125,
-      "second_gen": 0.7214943021535873,
-      "second_gen_std": 0.19232826307415962,
-      "step": 27,
-      "tot_reward": 2.272980734705925
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 66.59375,
-      "completion_length_2": 576.0,
-      "compre": 0.806842252612114,
-      "compre_std": 0.19021093845367432,
-      "epoch": 0.2931937172774869,
-      "finalgen": 0.847057968378067,
-      "first_gen": 0.8385341465473175,
-      "first_gen_std": 0.16478701680898666,
-      "geneval_final": 0.859372615814209,
-      "geneval_first": 0.8567684590816498,
-      "geneval_second": 0.7317702323198318,
-      "grad_norm": 0.6873025298118591,
-      "internvl_final": 0.8440103977918625,
-      "internvl_first": 0.8305246829986572,
-      "internvl_second": 0.8086360841989517,
-      "kl_0": 0.00698089599609375,
-      "kl_1": 0.014835357666015625,
-      "kl_2": 0.06842041015625,
-      "learning_rate": 1.32e-06,
-      "loss": 0.0019,
-      "rewards/selfcheck": 0.859375,
-      "second_gen": 0.7720397561788559,
-      "second_gen_std": 0.1886446215212345,
-      "step": 28,
-      "tot_reward": 2.4174161553382874
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 53.53125,
-      "completion_length_2": 576.0,
-      "compre": 0.8857230991125107,
-      "compre_std": 0.10681997612118721,
-      "epoch": 0.3036649214659686,
-      "finalgen": 0.7842017859220505,
-      "first_gen": 0.7742256820201874,
-      "first_gen_std": 0.09851318411529064,
-      "geneval_final": 0.6927078068256378,
-      "geneval_first": 0.6718747466802597,
-      "geneval_second": 0.6770829856395721,
-      "grad_norm": 0.8448681235313416,
-      "internvl_final": 0.8277648985385895,
-      "internvl_first": 0.823953315615654,
-      "internvl_second": 0.7563973963260651,
-      "kl_0": 0.00862884521484375,
-      "kl_1": 0.043426513671875,
-      "kl_2": 0.080963134765625,
-      "learning_rate": 1.1600000000000001e-06,
-      "loss": 0.0026,
-      "rewards/selfcheck": 0.7890625,
-      "second_gen": 0.7272432595491409,
-      "second_gen_std": 0.1504376232624054,
-      "step": 29,
-      "tot_reward": 2.387192040681839
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 62.21875,
-      "completion_length_2": 576.0,
-      "compre": 0.8224151283502579,
-      "compre_std": 0.1981170941144228,
-      "epoch": 0.31413612565445026,
-      "finalgen": 0.8049910217523575,
-      "first_gen": 0.7610630840063095,
-      "first_gen_std": 0.21358660981059074,
-      "geneval_final": 0.7473958879709244,
-      "geneval_first": 0.7005215138196945,
-      "geneval_second": 0.6614595353603363,
-      "grad_norm": 0.8343987464904785,
-      "internvl_final": 0.8584465086460114,
-      "internvl_first": 0.8124430626630783,
-      "internvl_second": 0.825078934431076,
-      "kl_0": 0.008777618408203125,
-      "kl_1": 0.09954833984375,
-      "kl_2": 0.03533935546875,
-      "learning_rate": 1e-06,
-      "loss": 0.0016,
-      "rewards/selfcheck": 0.7421875,
-      "second_gen": 0.7491757720708847,
-      "second_gen_std": 0.1921403631567955,
-      "step": 30,
-      "tot_reward": 2.332653984427452
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 44.71875,
-      "completion_length_2": 576.0,
-      "compre": 0.8410721868276596,
-      "compre_std": 0.13702194951474667,
-      "epoch": 0.32460732984293195,
-      "finalgen": 0.7979970574378967,
-      "first_gen": 0.7568379044532776,
-      "first_gen_std": 0.15108785405755043,
-      "geneval_final": 0.747393012046814,
-      "geneval_first": 0.6536446660757065,
-      "geneval_second": 0.7421848177909851,
-      "grad_norm": 0.6864271759986877,
-      "internvl_final": 0.846704438328743,
-      "internvl_first": 0.8158496171236038,
-      "internvl_second": 0.7937302589416504,
-      "kl_0": 0.0094451904296875,
-      "kl_1": 0.01679229736328125,
-      "kl_2": 0.0709228515625,
-      "learning_rate": 9.998996832101372e-07,
-      "loss": 0.0021,
-      "rewards/selfcheck": 0.796875,
-      "second_gen": 0.7617728412151337,
-      "second_gen_std": 0.17904167994856834,
-      "step": 31,
-      "tot_reward": 2.359682932496071
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 50.53125,
-      "completion_length_2": 576.0,
-      "compre": 0.8558274954557419,
-      "compre_std": 0.17379233241081238,
-      "epoch": 0.33507853403141363,
-      "finalgen": 0.7417566031217575,
-      "first_gen": 0.6925482824444771,
-      "first_gen_std": 0.18836861476302147,
-      "geneval_final": 0.6807302311062813,
-      "geneval_first": 0.6281267628073692,
-      "geneval_second": 0.5859397873282433,
-      "grad_norm": 0.6784985661506653,
-      "internvl_final": 0.818776473402977,
-      "internvl_first": 0.7752449959516525,
-      "internvl_second": 0.8047575652599335,
-      "kl_0": 0.0068511962890625,
-      "kl_1": 0.0993804931640625,
-      "kl_2": 0.06658935546875,
-      "learning_rate": 9.997993664202746e-07,
-      "loss": 0.0015,
-      "rewards/selfcheck": 0.75,
-      "second_gen": 0.6777446419000626,
-      "second_gen_std": 0.20427290350198746,
-      "step": 32,
-      "tot_reward": 2.2261204198002815
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 59.84375,
-      "completion_length_2": 576.0,
-      "compre": 0.7981031835079193,
-      "compre_std": 0.1447254717350006,
-      "epoch": 0.34554973821989526,
-      "finalgen": 0.7495086044073105,
-      "first_gen": 0.7078296095132828,
-      "first_gen_std": 0.15199239365756512,
-      "geneval_final": 0.7072916179895401,
-      "geneval_first": 0.61770960688591,
-      "geneval_second": 0.6052097231149673,
-      "grad_norm": 1.1254621744155884,
-      "internvl_final": 0.7729150354862213,
-      "internvl_first": 0.7561691552400589,
-      "internvl_second": 0.7697331458330154,
-      "kl_0": 0.0098876953125,
-      "kl_1": 0.06658172607421875,
-      "kl_2": 0.0515594482421875,
-      "learning_rate": 9.996990496304119e-07,
-      "loss": 0.0018,
-      "rewards/selfcheck": 0.77734375,
-      "second_gen": 0.7034687846899033,
-      "second_gen_std": 0.20352383330464363,
-      "step": 33,
-      "tot_reward": 2.2094015777111053
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 44.3125,
-      "completion_length_2": 576.0,
-      "compre": 0.8364836275577545,
-      "compre_std": 0.13604356348514557,
-      "epoch": 0.35602094240837695,
-      "finalgen": 0.8379629701375961,
-      "first_gen": 0.8319551795721054,
-      "first_gen_std": 0.13666080683469772,
-      "geneval_final": 0.7968707084655762,
-      "geneval_first": 0.7812459617853165,
-      "geneval_second": 0.5781252458691597,
-      "grad_norm": 0.7418293952941895,
-      "internvl_final": 0.8514181822538376,
-      "internvl_first": 0.8490835875272751,
-      "internvl_second": 0.8219228535890579,
-      "kl_0": 0.008148193359375,
-      "kl_1": 0.0732421875,
-      "kl_2": 0.0586700439453125,
-      "learning_rate": 9.99598732840549e-07,
-      "loss": 0.0017,
-      "rewards/selfcheck": 0.92578125,
-      "second_gen": 0.7799781709909439,
-      "second_gen_std": 0.16469088196754456,
-      "step": 34,
-      "tot_reward": 2.448416978120804
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 51.0,
-      "completion_length_2": 576.0,
-      "compre": 0.8061879575252533,
-      "compre_std": 0.1319187581539154,
-      "epoch": 0.36649214659685864,
-      "finalgen": 0.8501281291246414,
-      "first_gen": 0.8229310363531113,
-      "first_gen_std": 0.11965293064713478,
-      "geneval_final": 0.7499970346689224,
-      "geneval_first": 0.7083309888839722,
-      "geneval_second": 0.6458321213722229,
-      "grad_norm": 0.6788915991783142,
-      "internvl_final": 0.8535157740116119,
-      "internvl_first": 0.8384850025177002,
-      "internvl_second": 0.8233160376548767,
-      "kl_0": 0.017181396484375,
-      "kl_1": 0.0913848876953125,
-      "kl_2": 0.07708740234375,
-      "learning_rate": 9.994984160506863e-07,
-      "loss": 0.0019,
-      "rewards/selfcheck": 0.7578125,
-      "second_gen": 0.7993690818548203,
-      "second_gen_std": 0.1662791483104229,
-      "step": 35,
-      "tot_reward": 2.428488075733185
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 48.71875,
-      "completion_length_2": 576.0,
-      "compre": 0.8119166195392609,
-      "compre_std": 0.16001782193779945,
-      "epoch": 0.3769633507853403,
-      "finalgen": 0.7916422188282013,
-      "first_gen": 0.7797348350286484,
-      "first_gen_std": 0.12765273917466402,
-      "geneval_final": 0.5937489829957485,
-      "geneval_first": 0.5677079077072449,
-      "geneval_second": 0.5937490127980709,
-      "grad_norm": 0.9368501901626587,
-      "internvl_final": 0.8316355049610138,
-      "internvl_first": 0.820400059223175,
-      "internvl_second": 0.7844327837228775,
-      "kl_0": 0.010711669921875,
-      "kl_1": 0.195526123046875,
-      "kl_2": 0.07763671875,
-      "learning_rate": 9.993980992608236e-07,
-      "loss": 0.0024,
-      "rewards/selfcheck": 0.8515625,
-      "second_gen": 0.7547754347324371,
-      "second_gen_std": 0.16838618367910385,
-      "step": 36,
-      "tot_reward": 2.3464268893003464
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 53.625,
-      "completion_length_2": 576.0,
-      "compre": 0.8024957925081253,
-      "compre_std": 0.18278253450989723,
-      "epoch": 0.387434554973822,
-      "finalgen": 0.7481552511453629,
-      "first_gen": 0.7225710302591324,
-      "first_gen_std": 0.1874546892940998,
-      "geneval_final": 0.5546881854534149,
-      "geneval_first": 0.5468755140900612,
-      "geneval_second": 0.4765642583370209,
-      "grad_norm": 0.8567041158676147,
-      "internvl_final": 0.8841516524553299,
-      "internvl_first": 0.8685643374919891,
-      "internvl_second": 0.8400794863700867,
-      "kl_0": 0.00566864013671875,
-      "kl_1": 0.02951812744140625,
-      "kl_2": 0.0916748046875,
-      "learning_rate": 9.99297782470961e-07,
-      "loss": 0.0018,
-      "rewards/selfcheck": 0.7578125,
-      "second_gen": 0.6853782534599304,
-      "second_gen_std": 0.1954638883471489,
-      "step": 37,
-      "tot_reward": 2.210445076227188
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 40.59375,
-      "completion_length_2": 576.0,
-      "compre": 0.7131433486938477,
-      "compre_std": 0.19255046173930168,
-      "epoch": 0.39790575916230364,
-      "finalgen": 0.7346409112215042,
-      "first_gen": 0.7273536771535873,
-      "first_gen_std": 0.1450207568705082,
-      "geneval_final": 0.6406233012676239,
-      "geneval_first": 0.6927057802677155,
-      "geneval_second": 0.453126210719347,
-      "grad_norm": 0.6762747764587402,
-      "internvl_final": 0.7879036962985992,
-      "internvl_first": 0.7670356035232544,
-      "internvl_second": 0.7651602327823639,
-      "kl_0": 0.01068878173828125,
-      "kl_1": 0.01580810546875,
-      "kl_2": 0.060943603515625,
-      "learning_rate": 9.991974656810983e-07,
-      "loss": 0.002,
-      "rewards/selfcheck": 0.6875,
-      "second_gen": 0.6624156683683395,
-      "second_gen_std": 0.18323800340294838,
-      "step": 38,
-      "tot_reward": 2.1029126942157745
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 51.65625,
-      "completion_length_2": 576.0,
-      "compre": 0.8225916773080826,
-      "compre_std": 0.09966968512162566,
-      "epoch": 0.4083769633507853,
-      "finalgen": 0.8234386146068573,
-      "first_gen": 0.7953141033649445,
-      "first_gen_std": 0.09945447416976094,
-      "geneval_final": 0.9166632145643234,
-      "geneval_first": 0.885413646697998,
-      "geneval_second": 0.8124980479478836,
-      "grad_norm": 0.7397149801254272,
-      "internvl_final": 0.7746958285570145,
-      "internvl_first": 0.7505313530564308,
-      "internvl_second": 0.7618338316679001,
-      "kl_0": 0.0141448974609375,
-      "kl_1": 0.054218292236328125,
-      "kl_2": 0.0968017578125,
-      "learning_rate": 9.990971488912355e-07,
-      "loss": 0.0016,
-      "rewards/selfcheck": 0.74609375,
-      "second_gen": 0.7801451981067657,
-      "second_gen_std": 0.18415237590670586,
-      "step": 39,
-      "tot_reward": 2.398050978779793
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 52.8125,
-      "completion_length_2": 576.0,
-      "compre": 0.8490500450134277,
-      "compre_std": 0.14167104102671146,
-      "epoch": 0.418848167539267,
-      "finalgen": 0.7479293346405029,
-      "first_gen": 0.6769865304231644,
-      "first_gen_std": 0.16864520870149136,
-      "geneval_final": 0.6250012814998627,
-      "geneval_first": 0.5125028006732464,
-      "geneval_second": 0.589585080742836,
-      "grad_norm": 0.9414799213409424,
-      "internvl_final": 0.8500373214483261,
-      "internvl_first": 0.8148254454135895,
-      "internvl_second": 0.7945325374603271,
-      "kl_0": 0.01337432861328125,
-      "kl_1": 0.018688201904296875,
-      "kl_2": 0.084716796875,
-      "learning_rate": 9.989968321013727e-07,
-      "loss": 0.0015,
-      "rewards/selfcheck": 0.69921875,
-      "second_gen": 0.6964385211467743,
-      "second_gen_std": 0.19724002853035927,
-      "step": 40,
-      "tot_reward": 2.2224750965833664
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 52.0625,
-      "completion_length_2": 576.0,
-      "compre": 0.7664348036050797,
-      "compre_std": 0.18268020451068878,
-      "epoch": 0.4293193717277487,
-      "finalgen": 0.7703444063663483,
-      "first_gen": 0.7305110096931458,
-      "first_gen_std": 0.1900181919336319,
-      "geneval_final": 0.6979156285524368,
-      "geneval_first": 0.6041674762964249,
-      "geneval_second": 0.47396068274974823,
-      "grad_norm": 0.8097413778305054,
-      "internvl_final": 0.8117622584104538,
-      "internvl_first": 0.77989761531353,
-      "internvl_second": 0.7899433225393295,
-      "kl_0": 0.01154327392578125,
-      "kl_1": 0.02215576171875,
-      "kl_2": 0.08843994140625,
-      "learning_rate": 9.9889651531151e-07,
-      "loss": 0.002,
-      "rewards/selfcheck": 0.6796875,
-      "second_gen": 0.6740073412656784,
-      "second_gen_std": 0.2109435349702835,
-      "step": 41,
-      "tot_reward": 2.170953154563904
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 51.65625,
-      "completion_length_2": 576.0,
-      "compre": 0.7402557134628296,
-      "compre_std": 0.1825437918305397,
-      "epoch": 0.4397905759162304,
-      "finalgen": 0.7364934235811234,
-      "first_gen": 0.716862678527832,
-      "first_gen_std": 0.159286230802536,
-      "geneval_final": 0.7604135125875473,
-      "geneval_first": 0.7322889342904091,
-      "geneval_second": 0.6989560350775719,
-      "grad_norm": 0.8425426483154297,
-      "internvl_final": 0.7959964275360107,
-      "internvl_first": 0.7877412140369415,
-      "internvl_second": 0.7823043167591095,
-      "kl_0": 0.009307861328125,
-      "kl_1": 0.07275390625,
-      "kl_2": 0.05108642578125,
-      "learning_rate": 9.987961985216472e-07,
-      "loss": 0.0017,
-      "rewards/selfcheck": 0.625,
-      "second_gen": 0.7019034326076508,
-      "second_gen_std": 0.15979842096567154,
-      "step": 42,
-      "tot_reward": 2.1590218245983124
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 54.96875,
-      "completion_length_2": 576.0,
-      "compre": 0.832016795873642,
-      "compre_std": 0.14732321724295616,
-      "epoch": 0.450261780104712,
-      "finalgen": 0.7879950702190399,
-      "first_gen": 0.766228049993515,
-      "first_gen_std": 0.14289268106222153,
-      "geneval_final": 0.8234342187643051,
-      "geneval_first": 0.7953095883131027,
-      "geneval_second": 0.7864555567502975,
-      "grad_norm": 0.708795964717865,
-      "internvl_final": 0.8033061027526855,
-      "internvl_first": 0.7949481457471848,
-      "internvl_second": 0.7645547389984131,
-      "kl_0": 0.00658416748046875,
-      "kl_1": 0.047210693359375,
-      "kl_2": 0.0685577392578125,
-      "learning_rate": 9.986958817317846e-07,
-      "loss": 0.0016,
-      "rewards/selfcheck": 0.765625,
-      "second_gen": 0.7446198761463165,
-      "second_gen_std": 0.16749507747590542,
-      "step": 43,
-      "tot_reward": 2.3428647220134735
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 35.59375,
-      "completion_length_2": 576.0,
-      "compre": 0.8220645636320114,
-      "compre_std": 0.18921106681227684,
-      "epoch": 0.4607329842931937,
-      "finalgen": 0.8380867093801498,
-      "first_gen": 0.8182298690080643,
-      "first_gen_std": 0.1516233654692769,
-      "geneval_final": 0.8619765937328339,
-      "geneval_first": 0.8333311378955841,
-      "geneval_second": 0.7604154795408249,
-      "grad_norm": 0.8835089206695557,
-      "internvl_final": 0.8661111295223236,
-      "internvl_first": 0.8650543540716171,
-      "internvl_second": 0.8211274743080139,
-      "kl_0": 0.0194091796875,
-      "kl_1": 0.0438995361328125,
-      "kl_2": 0.09698486328125,
-      "learning_rate": 9.985955649419219e-07,
-      "loss": 0.0023,
-      "rewards/selfcheck": 0.796875,
-      "second_gen": 0.7649048268795013,
-      "second_gen_std": 0.20057369023561478,
-      "step": 44,
-      "tot_reward": 2.405199259519577
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 37.4375,
-      "completion_length_2": 576.0,
-      "compre": 0.7776415646076202,
-      "compre_std": 0.16899557411670685,
-      "epoch": 0.4712041884816754,
-      "finalgen": 0.7281599640846252,
-      "first_gen": 0.7270458787679672,
-      "first_gen_std": 0.15472668409347534,
-      "geneval_final": 0.6536467671394348,
-      "geneval_first": 0.6666674166917801,
-      "geneval_second": 0.48698242753744125,
-      "grad_norm": 0.9925355911254883,
-      "internvl_final": 0.7626287937164307,
-      "internvl_first": 0.7545899301767349,
-      "internvl_second": 0.7231644093990326,
-      "kl_0": 0.0115509033203125,
-      "kl_1": 0.087677001953125,
-      "kl_2": 0.0679931640625,
-      "learning_rate": 9.984952481520591e-07,
-      "loss": 0.002,
-      "rewards/selfcheck": 0.6953125,
-      "second_gen": 0.6345587372779846,
-      "second_gen_std": 0.18290312960743904,
-      "step": 45,
-      "tot_reward": 2.139246180653572
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 64.84375,
-      "completion_length_2": 576.0,
-      "compre": 0.7952178418636322,
-      "compre_std": 0.14555131271481514,
-      "epoch": 0.4816753926701571,
-      "finalgen": 0.7756165713071823,
-      "first_gen": 0.7566163092851639,
-      "first_gen_std": 0.12462454661726952,
-      "geneval_final": 0.5625010244548321,
-      "geneval_first": 0.505210455805809,
-      "geneval_second": 0.5312514565885067,
-      "grad_norm": 0.7020373344421387,
-      "internvl_final": 0.8309992402791977,
-      "internvl_first": 0.8211416453123093,
-      "internvl_second": 0.8066484332084656,
-      "kl_0": 0.01102447509765625,
-      "kl_1": 0.05503082275390625,
-      "kl_2": 0.0787353515625,
-      "learning_rate": 9.983949313621963e-07,
-      "loss": 0.0018,
-      "rewards/selfcheck": 0.7890625,
-      "second_gen": 0.7443623840808868,
-      "second_gen_std": 0.15503870509564877,
-      "step": 46,
-      "tot_reward": 2.296196535229683
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 59.375,
-      "completion_length_2": 576.0,
-      "compre": 0.8056609481573105,
-      "compre_std": 0.15513703413307667,
-      "epoch": 0.49214659685863876,
-      "finalgen": 0.773963987827301,
-      "first_gen": 0.7415499240159988,
-      "first_gen_std": 0.16437550447881222,
-      "geneval_final": 0.7083333283662796,
-      "geneval_first": 0.6041681617498398,
-      "geneval_second": 0.5729185864329338,
-      "grad_norm": 0.9495850205421448,
-      "internvl_final": 0.8133386075496674,
-      "internvl_first": 0.8239754736423492,
-      "internvl_second": 0.7794905602931976,
-      "kl_0": 0.010894775390625,
-      "kl_1": 0.01897430419921875,
-      "kl_2": 0.09686279296875,
-      "learning_rate": 9.982946145723336e-07,
-      "loss": 0.0018,
-      "rewards/selfcheck": 0.76171875,
-      "second_gen": 0.7020281553268433,
-      "second_gen_std": 0.2232751026749611,
-      "step": 47,
-      "tot_reward": 2.2492390275001526
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 41.6875,
-      "completion_length_2": 576.0,
-      "compre": 0.7868409007787704,
-      "compre_std": 0.1367402784526348,
-      "epoch": 0.5026178010471204,
-      "finalgen": 0.7759291082620621,
-      "first_gen": 0.8014131784439087,
-      "first_gen_std": 0.1358846165239811,
-      "geneval_final": 0.7031254507601261,
-      "geneval_first": 0.7968740910291672,
-      "geneval_second": 0.645834494382143,
-      "grad_norm": 0.7847272157669067,
-      "internvl_final": 0.847522646188736,
-      "internvl_first": 0.8382976651191711,
-      "internvl_second": 0.8017169833183289,
-      "kl_0": 0.011627197265625,
-      "kl_1": 0.018829345703125,
-      "kl_2": 0.07354736328125,
-      "learning_rate": 9.98194297782471e-07,
-      "loss": 0.0018,
-      "rewards/selfcheck": 0.69921875,
-      "second_gen": 0.7293520718812943,
-      "second_gen_std": 0.1890699453651905,
-      "step": 48,
-      "tot_reward": 2.3176061511039734
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 44.8125,
-      "completion_length_2": 576.0,
-      "compre": 0.8000631183385849,
-      "compre_std": 0.1882825717329979,
-      "epoch": 0.5130890052356021,
-      "finalgen": 0.7556480169296265,
-      "first_gen": 0.7593663930892944,
-      "first_gen_std": 0.1389727033674717,
-      "geneval_final": 0.6812484711408615,
-      "geneval_first": 0.6895816922187805,
-      "geneval_second": 0.618749350309372,
-      "grad_norm": 1.2835190296173096,
-      "internvl_final": 0.8502883911132812,
-      "internvl_first": 0.8563270419836044,
-      "internvl_second": 0.8191845417022705,
-      "kl_0": 0.0147552490234375,
-      "kl_1": 0.021228790283203125,
-      "kl_2": 0.088623046875,
-      "learning_rate": 9.980939809926083e-07,
-      "loss": 0.0018,
-      "rewards/selfcheck": 0.7421875,
-      "second_gen": 0.714243933558464,
-      "second_gen_std": 0.18192566186189651,
-      "step": 49,
-      "tot_reward": 2.2736734449863434
-    },
-    {
-      "completion_length_0": 576.0,
-      "completion_length_1": 55.125,
-      "completion_length_2": 576.0,
-      "compre": 0.8787847608327866,
-      "compre_std": 0.1345262099057436,
-      "epoch": 0.5235602094240838,
-      "finalgen": 0.8423563987016678,
-      "first_gen": 0.8243885934352875,
-      "first_gen_std": 0.14151314459741116,
-      "geneval_final": 0.8437488526105881,
-      "geneval_first": 0.8124992549419403,
-      "geneval_second": 0.7552084177732468,
-      "grad_norm": 0.9143640995025635,
-      "internvl_final": 0.8766902834177017,
-      "internvl_first": 0.8708599954843521,
-      "internvl_second": 0.8306637108325958,
-      "kl_0": 0.01788330078125,
-      "kl_1": 0.01178741455078125,
-      "kl_2": 0.13427734375,
-      "learning_rate": 9.979936642027455e-07,
-      "loss": 0.0017,
-      "rewards/selfcheck": 0.8359375,
-      "second_gen": 0.7776670157909393,
-      "second_gen_std": 0.1923159956932068,
-      "step": 50,
-      "tot_reward": 2.4808403700590134
-    }
-  ],
-  "logging_steps": 1,
-  "max_steps": 9500,
-  "num_input_tokens_seen": 0,
-  "num_train_epochs": 100,
-  "save_steps": 25,
-  "stateful_callbacks": {
-    "TrainerControl": {
-      "args": {
-        "should_epoch_stop": false,
-        "should_evaluate": false,
-        "should_log": false,
-        "should_save": true,
-        "should_training_stop": false
-      },
-      "attributes": {}
-    }
-  },
-  "total_flos": 0.0,
-  "train_batch_size": 1,
-  "trial_name": null,
-  "trial_params": null
-}

training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:caeb81b431498fc553411b77c5bac0f238a7cb96a6ffc3f09db9f99a3378b860
-size 8824