Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5964d4c2cdf4261495061a0ca3e2e85e410994180ada84a14314cd2e12180337
+oid sha256:8b51b319d95e7d5cff26d790f0a354d2b3fc6bd5e71e09e60ff51392b05722c0
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e1d76ac86ce5eaffe25b1b994c3b644f05dcf334a2ee2b27f68235d493df1d2
+oid sha256:a4d3a930c929c7d5f722fb928a0f111d127dcc302787e9d613914da71a58a2a3
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d99f5d94f62c01e007ea83040143343550094de5f513fb2c5d0d57c9887dfbc7
+oid sha256:304df0a44427d4af9f16b0d31ec609fe82b63e525f36699da3cba970dfd38b44
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0490bb5615767c4824059004f015bfe06bf546ed6a20efd46a423cd127daa484
+oid sha256:f4494573154763cac14d6bd385ff79ea8076b2cd20e2e17345994b85a0ecba8c
 size 1192

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.0,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=11-learning_rate=1.9e-04-warmup_ratio=0.050-min_lr=0.000-factor=0.830/checkpoint-5000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.014670790985417587,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=12-learning_rate=7.4e-06-warmup_ratio=0.030-min_lr=0.000-factor=0.930/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.3732318580150604,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 1.0002,
+      "grad_norm": 1.1744908094406128,
+      "learning_rate": 7.392845746133611e-06,
+      "loss": 1.9933,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.35028719902038574,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.2021,
+      "grad_norm": 1.0896804332733154,
+      "learning_rate": 7.392845746133611e-06,
+      "loss": 0.7405,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.27330875396728516,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.1767,
+      "grad_norm": 0.9823786616325378,
+      "learning_rate": 7.392845746133611e-06,
+      "loss": 0.492,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.2673795223236084,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.1683,
+      "grad_norm": 0.9321771264076233,
+      "learning_rate": 7.392845746133611e-06,
+      "loss": 0.3598,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.2766689956188202,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.1635,
+      "grad_norm": 0.9124600887298584,
+      "learning_rate": 7.392845746133611e-06,
+      "loss": 0.2774,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.2843599319458008,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.1607,
+      "grad_norm": 0.7268500328063965,
+      "learning_rate": 7.392845746133611e-06,
+      "loss": 0.2236,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.25838175415992737,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.1608,
+      "grad_norm": 0.4115571081638336,
+      "learning_rate": 7.392845746133611e-06,
+      "loss": 0.185,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.2386324256658554,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.1567,
+      "grad_norm": 0.5712406039237976,
+      "learning_rate": 7.392845746133611e-06,
+      "loss": 0.1587,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.30100056529045105,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.1548,
+      "grad_norm": 0.49448713660240173,
+      "learning_rate": 7.392845746133611e-06,
+      "loss": 0.1374,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.28358349204063416,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.1545,
+      "grad_norm": 0.4770694375038147,
+      "learning_rate": 7.392845746133611e-06,
+      "loss": 0.1204,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.011400795404330534,
+      "eval_all_ligands_equal": 0.014670790985417587,
+      "eval_e3_equal": 0.4694653115333628,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": 8.529297392841361,
-      "eval_e3_heavy_atoms_difference_norm": 0.23516541198896576,
+      "eval_e3_has_attachment_point(s)": 0.9965532479010164,
+      "eval_e3_heavy_atoms_difference": -0.54909412284578,
+      "eval_e3_heavy_atoms_difference_norm": -0.03696717892690007,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": 8.966151126822801,
-      "eval_heavy_atoms_difference_norm": 0.08358236778824747,
-      "eval_linker_equal": 0.0014140521431727795,
+      "eval_e3_valid": 0.9965532479010164,
+      "eval_has_all_attachment_points": 0.9708351745470615,
+      "eval_has_three_substructures": 0.997348652231551,
+      "eval_heavy_atoms_difference": 13.469553689792312,
+      "eval_heavy_atoms_difference_norm": 0.17544293470078143,
+      "eval_linker_equal": 0.03588157313300928,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": -0.30746796288113126,
-      "eval_linker_heavy_atoms_difference_norm": -0.2548477714690266,
+      "eval_linker_has_attachment_point(s)": 0.9908086610693769,
+      "eval_linker_heavy_atoms_difference": 3.279628811312417,
+      "eval_linker_heavy_atoms_difference_norm": 0.12970442523287076,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.5128470063209534,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.0016791869200176756,
+      "eval_linker_valid": 0.9908086610693769,
+      "eval_loss": 0.3284246027469635,
+      "eval_num_fragments": 2.9978789217852406,
+      "eval_poi_equal": 0.46981882456915597,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 1.0,
-      "eval_poi_heavy_atoms_difference": 0.7443216968625718,
-      "eval_poi_heavy_atoms_difference_norm": -0.11685097660916799,
+      "eval_poi_has_attachment_point(s)": 0.8617764030048608,
+      "eval_poi_heavy_atoms_difference": 4.413610251878038,
+      "eval_poi_heavy_atoms_difference_norm": 0.1345314085241158,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 1.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1180.6897,
-      "eval_samples_per_second": 9.583,
-      "eval_steps_per_second": 0.15,
+      "eval_poi_valid": 0.8617764030048608,
+      "eval_reassembly": 0.014935925762262483,
+      "eval_reassembly_nostereo": 0.01864781263809103,
+      "eval_runtime": 1510.5239,
+      "eval_samples_per_second": 7.491,
+      "eval_steps_per_second": 0.117,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 1.0,
+      "eval_valid": 0.8566504639858595,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.26305443048477173,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.1536,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.29646655917167664,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.1535,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.2654091715812683,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.1523,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.3717709183692932,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.1509,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.2985782325267792,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.1517,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.012726469288555016,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": -0.320724701723376,
-      "eval_e3_heavy_atoms_difference_norm": -0.11057464208253638,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": -7.47635881573133,
-      "eval_heavy_atoms_difference_norm": -0.1414824807138747,
-      "eval_linker_equal": 0.001237295625276182,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": -1.982589482987185,
-      "eval_linker_heavy_atoms_difference_norm": -0.38432931960931244,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.5187253952026367,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.0014140521431727795,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 1.0,
-      "eval_poi_heavy_atoms_difference": -5.173044631020769,
-      "eval_poi_heavy_atoms_difference_norm": -0.3352618231892355,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 1.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1442.0351,
-      "eval_samples_per_second": 7.847,
-      "eval_steps_per_second": 0.123,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 1.0,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.2808986306190491,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.1501,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.34891411662101746,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.1497,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.2883577048778534,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.149,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.3312648832798004,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.1489,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.3425774872303009,
-      "learning_rate": 0.00018563759769650177,
-      "loss": 0.1485,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.006628369421122404,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": -5.345647370746796,
-      "eval_e3_heavy_atoms_difference_norm": -0.3086636210918183,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": -5.3654441007512155,
-      "eval_heavy_atoms_difference_norm": -0.11263120688338427,
-      "eval_linker_equal": 0.001590808661069377,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": 0.2595669465311533,
-      "eval_linker_heavy_atoms_difference_norm": -0.21295870050572016,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.5215916037559509,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.0014140521431727795,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 1.0,
-      "eval_poi_heavy_atoms_difference": -0.27936367653557226,
-      "eval_poi_heavy_atoms_difference_norm": -0.15289589667535677,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 1.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1534.1801,
-      "eval_samples_per_second": 7.375,
-      "eval_steps_per_second": 0.115,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 1.0,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,18 +133,18 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=11-learning_rate=1.9e-04-warmup_ratio=0.050-min_lr=0.000-factor=0.830",
+  "trial_name": "trial-number=12-learning_rate=7.4e-06-warmup_ratio=0.030-min_lr=0.000-factor=0.930",
   "trial_params": {
-    "factor": 0.8300000000000001,
-    "learning_rate": 0.00018563759769650177,
-    "min_lr": 1.6647559250833267e-10,
-    "warmup_ratio": 0.05
+    "factor": 0.93,
+    "learning_rate": 7.392845746133611e-06,
+    "min_lr": 9.986795785570395e-09,
+    "warmup_ratio": 0.03
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d81f9f3a35a2029cfdfaaab3ca8195b3c6a56cfcc2e33156b7346e5589f3645c
+oid sha256:834cacfcb957cc77693b76851141cd44b29710840556298010c7ccc4ce9bde8b
 size 7480