tangledgroup
/

tangled-alpha-0.1-core

Text Generation

Model card Files Files and versions

mtasic85 commited on Feb 24

Commit

da80ae1

·

1 Parent(s): 1386dd6

grokadamw.GrokAdamW

Files changed (1) hide show

scripts/pretrain-core-model.yaml +1 -1

scripts/pretrain-core-model.yaml CHANGED Viewed

@@ -18,7 +18,7 @@ model_config:
   bias: False
   norm_class_name: "RMSNorm"
   mlp_class_name: "LLaMAMLP"
-  intermediate_size: 1792
   norm_eps: 1e-5
   rope_base: 500000
   rope_adjustments:

   bias: False
   norm_class_name: "RMSNorm"
   mlp_class_name: "LLaMAMLP"
+  intermediate_size: 2688 # n_embd * 5.25
   norm_eps: 1e-5
   rope_base: 500000
   rope_adjustments: