Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d86756848fc92b8f32c629b38001ab010f12e277f6b4bb950ef9c3a84f2c58d1
+oid sha256:5a1a533f600d03176bbc433be1da1958ca69bf5a64c3a1b518e0290ba9ec84d5
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:23ee4de09e4ef9d0b33063d422cecdfc965568d4d20f79b1b4459cec3f7b13fc
+oid sha256:6d230c26c55815659f63e169d0f9a46f10653a4cf9b847c53d32149df09fb219
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6bbd0f47bca72adc058e11195c706225cdf2c0113557c23ad307c5bc2e0b11ee
+oid sha256:919d51343ba7f79f11689f549f310632a28ac5ec6681d1245582e8580ebcf5fc
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:280e56c73026ae31058c531ae776e6904f00046b1ce3d2a28f1f305ef0adbe32
+oid sha256:9a6fdd0313393abbe7489aa1bcb1f52fb3950b20e996b10a6e64befc91a1cdcd
 size 1192

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.2837825894829872,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=24-learning_rate=1.5e-05-warmup_ratio=0.030-min_lr=0.000-factor=0.900/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.4881131241714538,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=24-learning_rate=1.5e-05-warmup_ratio=0.030-min_lr=0.000-factor=0.900/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.8980998674326116,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.3127955496311188,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.0303,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.24419963359832764,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.0256,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.2199561446905136,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.0218,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.2431618571281433,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.0188,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.18661023676395416,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.0163,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.43747238179407866,
+      "eval_e3_equal": 0.7806451612903226,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9870967741935484,
+      "eval_e3_heavy_atoms_difference": 0.3549270879363677,
+      "eval_e3_heavy_atoms_difference_norm": 0.004763396121020274,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9870967741935484,
+      "eval_has_all_attachment_points": 0.9794962439239947,
+      "eval_has_three_substructures": 0.9987627043747238,
+      "eval_heavy_atoms_difference": 7.966681396376491,
+      "eval_heavy_atoms_difference_norm": 0.10506341820964833,
+      "eval_linker_equal": 0.628104286345559,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9942554131683606,
+      "eval_linker_heavy_atoms_difference": 0.5155987627043748,
+      "eval_linker_heavy_atoms_difference_norm": 0.004264745639607716,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9942554131683606,
+      "eval_loss": 0.29560938477516174,
+      "eval_num_fragments": 3.0007070260715865,
+      "eval_poi_equal": 0.7138311975254088,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9196641626159965,
+      "eval_poi_heavy_atoms_difference": 2.46336721166593,
+      "eval_poi_heavy_atoms_difference_norm": 0.07938490626627487,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9196641626159965,
+      "eval_reassembly": 0.44410075121520104,
+      "eval_reassembly_nostereo": 0.46796288113124174,
+      "eval_runtime": 1492.3352,
+      "eval_samples_per_second": 7.582,
+      "eval_steps_per_second": 0.119,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.905346884666372,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.2039629966020584,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.0146,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.22314690053462982,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.013,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.23112636804580688,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.0115,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.24067600071430206,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.0105,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.22660063207149506,
+      "learning_rate": 1.4902432220520327e-05,
+      "loss": 0.0097,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.4881131241714538,
+      "eval_e3_equal": 0.7934600088378259,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.993813521873619,
+      "eval_e3_heavy_atoms_difference": 0.2950066283694211,
+      "eval_e3_heavy_atoms_difference_norm": 0.0028168010095192407,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.993813521873619,
+      "eval_has_all_attachment_points": 0.9835616438356164,
+      "eval_has_three_substructures": 0.9992045956694653,
+      "eval_heavy_atoms_difference": 7.484577993813522,
+      "eval_heavy_atoms_difference_norm": 0.0982968303528513,
+      "eval_linker_equal": 0.6990720282810429,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9962881131241714,
+      "eval_linker_heavy_atoms_difference": 0.5271763146266019,
+      "eval_linker_heavy_atoms_difference_norm": 0.010195950921219643,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9962881131241714,
+      "eval_loss": 0.3071575164794922,
+      "eval_num_fragments": 3.000265134776845,
+      "eval_poi_equal": 0.73194874060981,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.922757401679187,
+      "eval_poi_heavy_atoms_difference": 2.526292532037119,
+      "eval_poi_heavy_atoms_difference_norm": 0.08038544461395786,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.922757401679187,
+      "eval_reassembly": 0.49597878921785243,
+      "eval_reassembly_nostereo": 0.5214317277949624,
+      "eval_runtime": 1469.8755,
+      "eval_samples_per_second": 7.698,
+      "eval_steps_per_second": 0.12,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9151568714096332,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=24-learning_rate=1.5e-05-warmup_ratio=0.030-min_lr=0.000-factor=0.900",
   "trial_params": {