Upload 11 files

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "chat",
+  "_name_or_path": "/mnt/petrelfs/share_data/dongxiaoyi/share_models/chat_merge/",
   "architectures": [
     "InternLMXComposerForCausalLM"
   ],
@@ -17,29 +17,9 @@
   "initializer_range": 0.02,
   "intermediate_size": 11008,
   "intern_converted_llm": true,
-  "internlm_lora": {
-    "freeze": false,
-    "learn_param": [
-      "q",
-      "v",
-      "ffn"
-    ],
-    "lora_alpha": 256,
-    "lora_dropout": 0.05,
-    "lora_r": 256
-  },
+  "internlm_lora": null,
   "kqvo_bias": true,
-  "lora_cfg": {
-    "freeze": false,
-    "learn_param": [
-      "q",
-      "v",
-      "ffn"
-    ],
-    "lora_alpha": 256,
-    "lora_dropout": 0.05,
-    "lora_r": 256
-  },
+  "lora_cfg": null,
   "max_position_embeddings": 2048,
   "model_type": "InternLMXComposer",
   "num_attention_heads": 32,
@@ -49,8 +29,8 @@
   "pad_token_id": -1,
   "rms_norm_eps": 1e-05,
   "tie_word_embeddings": false,
-  "torch_dtype": "float32",
-  "transformers_version": "4.30.2",
+  "torch_dtype": "float16",
+  "transformers_version": "4.33.1",
   "use_cache": true,
   "vocab_size": 103168
 }

modeling_InternLM.py

@@ -6,6 +6,8 @@ import torch
 import torch.utils.checkpoint
 import torch.utils.checkpoint
 from einops import rearrange
+#import rotary_emb
+#from flash_attn.layers.rotary import ApplyRotaryEmbQKV_ as LegacyApplyRotaryEmbQKV_
 from torch import nn
 from torch.nn import CrossEntropyLoss
 from transformers.activations import ACT2FN
@@ -22,6 +24,8 @@ _CONFIG_FOR_DOC = "InternLMXComposerConfig"
 
 
 def rotary_embed(x1, x2, cos, sin, conj):
+    # print(x1.shape, x2.shape, cos.shape, sin.shape)
+    #[5, 1, 32, 64] [1, 1, 64]
     x1, x2 = x1.float(), x2.float()
     if conj:
         x1, x2 = x1 * cos + x2 * sin, x1 * sin + x2 * cos

modeling_InternLM_XComposer.py

@@ -95,13 +95,15 @@ class InternLMXComposerForCausalLM(PreTrainedModel):
 
     @property
     def eoh(self):
-        return self.tokenizer.decode(torch.Tensor([103027]),
-                                     skip_special_tokens=True)
+        #return self.tokenizer.decode(torch.Tensor([103027]),
+        #                             skip_special_tokens=True)
+        return '<TOKENS_UNUSED_0>'
 
     @property
     def eoa(self):
-        return self.tokenizer.decode(torch.Tensor([103028]),
-                                     skip_special_tokens=True)
+        #return self.tokenizer.decode(torch.Tensor([103028]),
+        #                             skip_special_tokens=True)
+        return '<TOKENS_UNUSED_1>'
 
     def maybe_autocast(self, dtype=torch.float16):
         # if on cpu, don't use autocast
@@ -194,6 +196,9 @@ class InternLMXComposerForCausalLM(PreTrainedModel):
         new_kargs = copy.deepcopy(self.gen_config)
         new_kargs.update(kwargs)
         return new_kargs
+    
+    def forward(self, **kwargs):
+        return self.internlm_model(**kwargs)
 
     def generate(self, text, image=None, **kwargs):
         text_embeds = self.encode_text(text)

modeling_vit.py

@@ -368,13 +368,13 @@ class VisionTransformer(nn.Module):
                   window_size=self.patch_embed.patch_shape
                   if use_rel_pos_bias else None) for i in range(depth)
         ])
-
-        # if self.pos_embed is not None:
-        #     trunc_normal_(self.pos_embed, std=.02)
-        # trunc_normal_(self.cls_token, std=.02)
-        # self.apply(self._init_weights)
-        # self.fix_init_weight()
-
+        '''
+        if self.pos_embed is not None:
+            trunc_normal_(self.pos_embed, std=.02)
+        trunc_normal_(self.cls_token, std=.02)
+        self.apply(self._init_weights)
+        self.fix_init_weight()
+        '''
     def fix_init_weight(self):
         def rescale(param, layer_id):
             param.div_(math.sqrt(2.0 * layer_id))
@@ -518,12 +518,12 @@ def create_eva_vit_g(img_size=224,
         norm_layer=partial(nn.LayerNorm, eps=1e-6),
         use_checkpoint=use_checkpoint,
     )
-    # url = "https://storage.googleapis.com/sfr-vision-language-research/LAVIS/models/BLIP2/eva_vit_g.pth"
-    # cached_file = download_cached_file(url, check_hash=False, progress=True)
-    # state_dict = torch.load(cached_file, map_location="cpu")
-    # interpolate_pos_embed(model, state_dict)
-    #
-    # incompatible_keys = model.load_state_dict(state_dict, strict=False)
+    url = "https://storage.googleapis.com/sfr-vision-language-research/LAVIS/models/BLIP2/eva_vit_g.pth"
+    cached_file = download_cached_file(url, check_hash=False, progress=True)
+    state_dict = torch.load(cached_file, map_location="cpu")
+    interpolate_pos_embed(model, state_dict)
+
+    incompatible_keys = model.load_state_dict(state_dict, strict=False)
 
     if precision == "fp16":
         convert_weights_to_fp16(model)

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21ff673031fd4187f19721a86af4caa6a4deb1f3c2db284f763de3e53bd8f741
+size 1658715