SequentialLearning
/

SuperLinear

mixture-of-experts

Model card Files Files and versions

razmars commited on Apr 23

Commit

c0671e8

·

verified ·

1 Parent(s): 3f3fadd

Update configuration_super_linear.py

Files changed (1) hide show

configuration_super_linear.py +1 -1

configuration_super_linear.py CHANGED Viewed

@@ -36,7 +36,7 @@ class SuperLinearConfig(PretrainedConfig):
         freq_experts='mean_naive_1/6_1/7_1/8_1/12_1/14_1/16_1/21_1/24_1/28_1/30_1/32_1/36_1/42_1/48_1/52_1/56_1/60_1/72_1/84_1/96_1/120_1/144_1/168_1/180_1/224_1/252_1/288_1/336_1/365_1/504_1/672_1/1008_1/1440_1/2016_1/3600',
         auto_regressive= 1,
         con= 0,
-        d_model= 128,
         dropout= 0.0,
         fft_len= 10000,
         freeze_experts= 1,

         freq_experts='mean_naive_1/6_1/7_1/8_1/12_1/14_1/16_1/21_1/24_1/28_1/30_1/32_1/36_1/42_1/48_1/52_1/56_1/60_1/72_1/84_1/96_1/120_1/144_1/168_1/180_1/224_1/252_1/288_1/336_1/365_1/504_1/672_1/1008_1/1440_1/2016_1/3600',
         auto_regressive= 1,
         con= 0,
+        d_model= 512,
         dropout= 0.0,
         fft_len= 10000,
         freeze_experts= 1,