Pj12 commited on
Commit
43ff4cc
·
verified ·
1 Parent(s): 2dc975e

Update train_nsf_sim_cache_sid_load_pretrain.py

Browse files
train_nsf_sim_cache_sid_load_pretrain.py CHANGED
@@ -289,8 +289,8 @@ def run(rank, n_gpus, hps):
289
  #
290
  Lowest_lg = f"{min(Loss_Gen_Per_Epoch):.5f}, epoch: {right_index(Loss_Gen_Per_Epoch,min(Loss_Gen_Per_Epoch))+1}"
291
  Lowest_ld = f"{min(Loss_Disc_Per_Epoch):.5f}, epoch: {right_index(Loss_Disc_Per_Epoch,min(Loss_Disc_Per_Epoch))+1}"
292
- print(f"{hps.name}_e{epoch}_s{global_step} | Loss gen total: {Loss_Gen_Per_Epoch[-1]:.8f} | Lowest loss G: {Lowest_lg}\n Loss disc: {Loss_Disc_Per_Epoch[-1]:.8f} | Lowest loss D: {Lowest_ld}")
293
- print(f"Specific Value: loss gen={loss_gen:.3f}, loss fm={loss_fm:.3f}, loss mel={loss_mel:.3f}, loss kl={loss_kl:.3f}")
294
  #
295
  if len(Loss_Gen_Per_Epoch) > Min_for_Single_epoch and epoch % hps.save_every_epoch != 0:
296
  if min(Loss_Gen_Per_Epoch[Min_for_Single_epoch::1]) == Loss_Gen_Per_Epoch[-1]:
@@ -322,7 +322,7 @@ def run(rank, n_gpus, hps):
322
  time_left = elapsed_time_avg*(hps.total_epoch-epoch)
323
  hour, minute, second, millisec = Calculate_format_elapsed_time(elapsed_time)
324
  hour_left, minute_left, second_left, millisec_left = Calculate_format_elapsed_time(time_left)
325
- print(f"Time Elapsed: {hour}h:{formating_time(minute)}m:{formating_time(second)}s:{millisec}ms || Time left: {hour_left}h:{formating_time(minute_left)}m:{formating_time(second_left)}s:{millisec_left}ms\n")
326
  #
327
  if ((len(Loss_Gen_Per_Epoch) - right_index(Loss_Gen_Per_Epoch,min(Loss_Gen_Per_Epoch)) + 1) > overtrain and overtrain != -1):
328
  logger.info("Over Train threshold reached. Training is done.")
 
289
  #
290
  Lowest_lg = f"{min(Loss_Gen_Per_Epoch):.5f}, epoch: {right_index(Loss_Gen_Per_Epoch,min(Loss_Gen_Per_Epoch))+1}"
291
  Lowest_ld = f"{min(Loss_Disc_Per_Epoch):.5f}, epoch: {right_index(Loss_Disc_Per_Epoch,min(Loss_Disc_Per_Epoch))+1}"
292
+ print(f"{hps.name}_e{epoch}_s{global_step} | Loss gen total: {Loss_Gen_Per_Epoch[-1]:.8f} | Lowest loss G: {Lowest_lg}\n- Loss disc: {Loss_Disc_Per_Epoch[-1]:.8f} | Lowest loss D: {Lowest_ld}")
293
+ print(f"> Specific Value: loss gen={loss_gen:.3f}, loss fm={loss_fm:.3f}, loss mel={loss_mel:.3f}, loss kl={loss_kl:.3f}")
294
  #
295
  if len(Loss_Gen_Per_Epoch) > Min_for_Single_epoch and epoch % hps.save_every_epoch != 0:
296
  if min(Loss_Gen_Per_Epoch[Min_for_Single_epoch::1]) == Loss_Gen_Per_Epoch[-1]:
 
322
  time_left = elapsed_time_avg*(hps.total_epoch-epoch)
323
  hour, minute, second, millisec = Calculate_format_elapsed_time(elapsed_time)
324
  hour_left, minute_left, second_left, millisec_left = Calculate_format_elapsed_time(time_left)
325
+ print(f"-Time Elapsed: {hour}h:{formating_time(minute)}m:{formating_time(second)}s:{millisec}ms || Time left: {hour_left}h:{formating_time(minute_left)}m:{formating_time(second_left)}s:{millisec_left}ms-\n")
326
  #
327
  if ((len(Loss_Gen_Per_Epoch) - right_index(Loss_Gen_Per_Epoch,min(Loss_Gen_Per_Epoch)) + 1) > overtrain and overtrain != -1):
328
  logger.info("Over Train threshold reached. Training is done.")