Upload folder using huggingface_hub

README.md

@@ -15,8 +15,8 @@ This tiny model is intended for debugging. It is randomly initialized using the
 
 ```python
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 import transformers
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 transformers.utils.import_utils.is_torch_fx_available = transformers.utils.import_utils.is_torch_available
 
 model_id = "tiny-random/bailing-moe-v2.5"
@@ -41,6 +41,7 @@ from pathlib import Path
 
 import accelerate
 import torch
+import transformers
 from huggingface_hub import file_exists, hf_hub_download
 from transformers import (
     AutoConfig,
@@ -49,7 +50,6 @@ from transformers import (
     GenerationConfig,
     set_seed,
 )
-import transformers
 transformers.utils.import_utils.is_torch_fx_available = transformers.utils.import_utils.is_torch_available
 source_model_id = "inclusionAI/Ring-2.5-1T"
 save_folder = "/tmp/tiny-random/bailing-moe-v25"
@@ -98,6 +98,7 @@ with torch.no_grad():
     for name, p in sorted(model.named_parameters()):
         torch.nn.init.normal_(p, 0, 0.1)
         print(name, p.shape)
+model.model.layers[1].mlp.gate.expert_bias = model.model.layers[1].mlp.gate.expert_bias.float()
 model.save_pretrained(save_folder)
 print(model)
 with open(f"{save_folder}/config.json", "r", encoding='utf-8') as f:

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4689d2bae0110253d4a2f2738e903faefce4873dcefd04e8929a08bf80f6f0bd
-size 6680608
+oid sha256:19a618643f76a64b5b7b204ca792355fa0c8c161e7586981e55d0d81caf34eb3
+size 6681112