Update configuration_super_linear.py

configuration_super_linear.py

@@ -1,13 +1,6 @@
 from typing import Optional, Tuple
 
-from transformers import (
-    PretrainedConfig,
-    PreTrainedModel,
-    GenerationMixin,
-    AutoConfig,
-    AutoModelForCausalLM,
-)
-from transformers.modeling_outputs import CausalLMOutputWithCrossAttentions
+from transformers import PretrainedConfig
 
 # 1) --------------------------------------------------------------------------
 # CONFIG
@@ -28,23 +21,16 @@ class SuperLinearConfig(PretrainedConfig):
         # Model architecture parameters
         train_seq_len=512,
         train_pred_len=96,
-        seq_len=512,
-        pred_len=96,
-        inf_pred_len=96,
-        max_horizon=96,
-        auto_regressive=1,
-        
+
         # MoE parameters
-        moe_n_experts=4,
+        n_experts=4,
         top_k_experts=12,
         noisy_gating_std=0.1,
         moe_temp=1.0,
         moe_norm=False,
         layer_type='RLinear',
-        n_experts=4,
         comp_moe=12,
         freeze_experts=True,
-        moe=1,
         
         # FFT-based gating parameters
         use_fft=True,
@@ -53,46 +39,23 @@ class SuperLinearConfig(PretrainedConfig):
         # Expert configuration
         freq_experts='mean_naive_1/4_1/6_1/7_1/8_1/12_1/14_1/16_1/21_1/24_1/28_1/30_1/32_1/36_1/42_1/48_1/52_1/56_1/60_1/72_1/84_1/90_1/96_1/120_1/144_1/168_1/180_1/224_1/252_1/288_1/336_1/365_1/504_1/672_1/1008_1/1440_1/2016_1/3600',
         
-        # Model loading and saving
-        load_linear=True,
-        load_weights_full=True,
-        linear_freq_weights_path='./weights/linear_freq_weights/',
-        full_weights_path='./weights/full_weights/checkpoint.pth',
-        
         # Training parameters
         resample_long_lookback=False,
         
-        # Legacy parameters for backward compatibility
-        linear_checkpoints_path='/cs/azencot_fsas/MoE/',
-        linear_checkpoints_dir="checkpoints5",
-        manual_moe=0,
-        misc_moe=1,
-        noisy_gating_std_decay=1,
-        ker_len=50,
-        con=0,
-        d_model=512,
-        mlp_gating=1,
-        dropout=0.0,
         **kwargs,
     ):
         # Model architecture parameters
         self.train_seq_len = train_seq_len
         self.train_pred_len = train_pred_len
-        self.seq_len = seq_len
-        self.pred_len = pred_len
-        self.inf_pred_len = inf_pred_len
-        self.max_horizon = max_horizon
-        self.auto_regressive = auto_regressive
         
         # MoE parameters
         self.moe = moe
-        self.moe_n_experts = moe_n_experts
+        self.n_experts = n_experts
         self.top_k_experts = top_k_experts
         self.noisy_gating_std = noisy_gating_std
         self.moe_temp = moe_temp
         self.moe_norm = moe_norm
         self.layer_type = layer_type
-        self.n_experts = n_experts
         self.comp_moe = comp_moe
         self.freeze_experts = freeze_experts
         
@@ -103,25 +66,7 @@ class SuperLinearConfig(PretrainedConfig):
         # Expert configuration
         self.freq_experts = freq_experts
         
-        # Model loading and saving
-        self.load_linear = load_linear
-        self.load_weights_full = load_weights_full
-        self.linear_freq_weights_path = linear_freq_weights_path
-        self.full_weights_path = full_weights_path
-        
         # Training parameters
         self.resample_long_lookback = resample_long_lookback
         
-        # Legacy parameters for backward compatibility
-        self.linear_checkpoints_path = linear_checkpoints_path
-        self.linear_checkpoints_dir = linear_checkpoints_dir
-        self.manual_moe = manual_moe
-        self.misc_moe = misc_moe
-        self.noisy_gating_std_decay = noisy_gating_std_decay
-        self.ker_len = ker_len
-        self.con = con
-        self.d_model = d_model
-        self.mlp_gating = mlp_gating
-        self.dropout = dropout
-        
-        super().__init__(**kwargs)
+        super().__init__(**kwargs)