SequentialLearning
/

SuperLinear

mixture-of-experts

Model card Files Files and versions

razmars commited on Apr 23

Commit

82c7d5a

·

verified ·

1 Parent(s): c0671e8

Update configuration_super_linear.py

Files changed (1) hide show

configuration_super_linear.py +2 -2

configuration_super_linear.py CHANGED Viewed

@@ -31,12 +31,12 @@ class SuperLinearConfig(PretrainedConfig):
         inf_pred_len=96,
         max_horizon=96,
         moe_n_experts=8,
-        top_k_experts=3,
         moe =1,
         freq_experts='mean_naive_1/6_1/7_1/8_1/12_1/14_1/16_1/21_1/24_1/28_1/30_1/32_1/36_1/42_1/48_1/52_1/56_1/60_1/72_1/84_1/96_1/120_1/144_1/168_1/180_1/224_1/252_1/288_1/336_1/365_1/504_1/672_1/1008_1/1440_1/2016_1/3600',
         auto_regressive= 1,
         con= 0,
-        d_model= 512,
         dropout= 0.0,
         fft_len= 10000,
         freeze_experts= 1,

         inf_pred_len=96,
         max_horizon=96,
         moe_n_experts=8,
+        top_k_experts=5,
         moe =1,
         freq_experts='mean_naive_1/6_1/7_1/8_1/12_1/14_1/16_1/21_1/24_1/28_1/30_1/32_1/36_1/42_1/48_1/52_1/56_1/60_1/72_1/84_1/96_1/120_1/144_1/168_1/180_1/224_1/252_1/288_1/336_1/365_1/504_1/672_1/1008_1/1440_1/2016_1/3600',
         auto_regressive= 1,
         con= 0,
+        d_model= 128,
         dropout= 0.0,
         fft_len= 10000,
         freeze_experts= 1,