Update train_nsf_sim_cache_sid_load_pretrain.py
Browse files
train_nsf_sim_cache_sid_load_pretrain.py
CHANGED
|
@@ -289,8 +289,8 @@ def run(rank, n_gpus, hps):
|
|
| 289 |
#
|
| 290 |
Lowest_lg = f"{min(Loss_Gen_Per_Epoch):.5f}, epoch: {right_index(Loss_Gen_Per_Epoch,min(Loss_Gen_Per_Epoch))+1}"
|
| 291 |
Lowest_ld = f"{min(Loss_Disc_Per_Epoch):.5f}, epoch: {right_index(Loss_Disc_Per_Epoch,min(Loss_Disc_Per_Epoch))+1}"
|
| 292 |
-
print(f"{hps.name}_e{epoch}_s{global_step} | Loss gen total: {Loss_Gen_Per_Epoch[-1]:.8f} | Lowest loss G: {Lowest_lg}\n Loss disc: {Loss_Disc_Per_Epoch[-1]:.8f} | Lowest loss D: {Lowest_ld}")
|
| 293 |
-
print(f"Specific Value: loss gen={loss_gen:.3f}, loss fm={loss_fm:.3f}, loss mel={loss_mel:.3f}, loss kl={loss_kl:.3f}")
|
| 294 |
#
|
| 295 |
if len(Loss_Gen_Per_Epoch) > Min_for_Single_epoch and epoch % hps.save_every_epoch != 0:
|
| 296 |
if min(Loss_Gen_Per_Epoch[Min_for_Single_epoch::1]) == Loss_Gen_Per_Epoch[-1]:
|
|
@@ -322,7 +322,7 @@ def run(rank, n_gpus, hps):
|
|
| 322 |
time_left = elapsed_time_avg*(hps.total_epoch-epoch)
|
| 323 |
hour, minute, second, millisec = Calculate_format_elapsed_time(elapsed_time)
|
| 324 |
hour_left, minute_left, second_left, millisec_left = Calculate_format_elapsed_time(time_left)
|
| 325 |
-
print(f"Time Elapsed: {hour}h:{formating_time(minute)}m:{formating_time(second)}s:{millisec}ms || Time left: {hour_left}h:{formating_time(minute_left)}m:{formating_time(second_left)}s:{millisec_left}ms
|
| 326 |
#
|
| 327 |
if ((len(Loss_Gen_Per_Epoch) - right_index(Loss_Gen_Per_Epoch,min(Loss_Gen_Per_Epoch)) + 1) > overtrain and overtrain != -1):
|
| 328 |
logger.info("Over Train threshold reached. Training is done.")
|
|
|
|
| 289 |
#
|
| 290 |
Lowest_lg = f"{min(Loss_Gen_Per_Epoch):.5f}, epoch: {right_index(Loss_Gen_Per_Epoch,min(Loss_Gen_Per_Epoch))+1}"
|
| 291 |
Lowest_ld = f"{min(Loss_Disc_Per_Epoch):.5f}, epoch: {right_index(Loss_Disc_Per_Epoch,min(Loss_Disc_Per_Epoch))+1}"
|
| 292 |
+
print(f"{hps.name}_e{epoch}_s{global_step} | Loss gen total: {Loss_Gen_Per_Epoch[-1]:.8f} | Lowest loss G: {Lowest_lg}\n- Loss disc: {Loss_Disc_Per_Epoch[-1]:.8f} | Lowest loss D: {Lowest_ld}")
|
| 293 |
+
print(f"> Specific Value: loss gen={loss_gen:.3f}, loss fm={loss_fm:.3f}, loss mel={loss_mel:.3f}, loss kl={loss_kl:.3f}")
|
| 294 |
#
|
| 295 |
if len(Loss_Gen_Per_Epoch) > Min_for_Single_epoch and epoch % hps.save_every_epoch != 0:
|
| 296 |
if min(Loss_Gen_Per_Epoch[Min_for_Single_epoch::1]) == Loss_Gen_Per_Epoch[-1]:
|
|
|
|
| 322 |
time_left = elapsed_time_avg*(hps.total_epoch-epoch)
|
| 323 |
hour, minute, second, millisec = Calculate_format_elapsed_time(elapsed_time)
|
| 324 |
hour_left, minute_left, second_left, millisec_left = Calculate_format_elapsed_time(time_left)
|
| 325 |
+
print(f"-Time Elapsed: {hour}h:{formating_time(minute)}m:{formating_time(second)}s:{millisec}ms || Time left: {hour_left}h:{formating_time(minute_left)}m:{formating_time(second_left)}s:{millisec_left}ms-\n")
|
| 326 |
#
|
| 327 |
if ((len(Loss_Gen_Per_Epoch) - right_index(Loss_Gen_Per_Epoch,min(Loss_Gen_Per_Epoch)) + 1) > overtrain and overtrain != -1):
|
| 328 |
logger.info("Over Train threshold reached. Training is done.")
|