Update modeling_super_linear.py

modeling_super_linear.py

@@ -235,8 +235,6 @@ class SparseNoisyMoE(nn.Module):
         if self.k > self.num_experts:
             print(f"Warning: k ({self.k}) is greater than the number of experts ({self.num_experts}). Setting k to {self.num_experts}.")
             self.k = self.num_experts
-       # self.ker_len = configs.ker_len
-        #self.con = configs.con
         self.d_model = configs.d_model
         self.mlp_gating = configs.mlp_gating
         self.moe_temp = configs.moe_temp
@@ -367,7 +365,7 @@ class superLinear(nn.Module):
         else:
             self.freq_experts = configs.freq_experts.split('_')
 
-        print("self.freq_experts:", self.freq_experts)
+        #print("self.freq_experts:", self.freq_experts)
 
         self.moe_loss = None
         self.top_k_experts = configs.top_k_experts
@@ -377,7 +375,7 @@ class superLinear(nn.Module):
         self.layer_type = configs.layer_type
         self.model_name = "SuperLinear"
 
-        print("self.layer_type", self.layer_type)
+        #print("self.layer_type", self.layer_type)
         self.layer_dict = {'DLinear': DLinear, 'Linear': Linear, 'NLinear': NLinear, 'RLinear': RLinear}
         path = configs.linear_checkpoints_path + configs.linear_checkpoints_dir 
         dirs = os.listdir(path)
@@ -422,20 +420,16 @@ class superLinear(nn.Module):
                 print(f"creating expert {i}")
                 self.experts[str(i)] = self.layer_dict[self.layer_type](self.seq_len, self.pred_len)
 
-        #self.manual_moe = configs.manual_moe
-
 
         if configs.misc_moe>0:
             if configs.misc_moe == 1:
-                print("Creating misc expert")
+                #print("Creating misc expert")
                 self.experts["misc"] = self.layer_dict[self.layer_type](self.seq_len, self.pred_len)
             else:
                 for i in range(configs.misc_moe):
-                    print(f"Creating misc expert {i}")
+                    #print(f"Creating misc expert {i}")
                     self.experts["misc_"+str(i)] = self.layer_dict[self.layer_type](self.seq_len, self.pred_len)
-        '''if configs.misc_moe2==1:
-                print("Creating misc expert")
-                self.experts["misc2"] = self.layer_dict[self.layer_type](self.seq_len, self.pred_len)'''
+
 
             
         self.moe = SparseNoisyMoE(configs, experts=self.experts.values())
@@ -445,7 +439,6 @@ class superLinear(nn.Module):
             print(f"Loading weights from {path}")
             path = configs.load_weights_path + "" + configs.load_weights_dir + "/" + "checkpoint.pth"
             if os.path.exists(path):
-              #  print(f"Loading weights from {path}")
                 checkpoint = torch.load(path)
                 print(len(self.experts.keys()))
                 print(self.experts.keys())