Upload InternVideo2_Classification

config.json

@@ -1,52 +1,54 @@
 {
   "architectures": [
-    "InternVideo2_Classification_test"
+    "InternVideo2_Classification"
   ],
   "auto_map": {
-    "AutoModel": "modeling_videochat2_classification.InternVideo2_Classification_test"
+    "AutoModel": "modeling_videochat2_classification.InternVideo2_Classification"
   },
-  "bridge": {
-    "extra_num_query_token": 64,
-    "name": "qformer",
-    "num_query_token": 32,
-    "qformer_attention_probs_dropout_prob": 0.1,
-    "qformer_drop_path_rate": 0.2,
-    "qformer_hidden_dropout_prob": 0.1
+  "model_config": {
+    "bridge": {
+      "extra_num_query_token": 64,
+      "name": "qformer",
+      "num_query_token": 32,
+      "qformer_attention_probs_dropout_prob": 0.1,
+      "qformer_drop_path_rate": 0.2,
+      "qformer_hidden_dropout_prob": 0.1
+    },
+    "freeze_bridge": false,
+    "freeze_llm": false,
+    "freeze_vision_encoder": false,
+    "llm": {
+      "lora_alpha": 32,
+      "lora_dropout": 0.1,
+      "lora_r": 16,
+      "name": "mistral_7b",
+      "pretrained_llm_path": "mistralai/Mistral-7B-Instruct-v0.3",
+      "use_lora": true
+    },
+    "loss": {
+      "use_vision_regression_loss": false
+    },
+    "pretrained_paths": {},
+    "use_flash_attention": true,
+    "vision_encoder": {
+      "checkpoint_num": 48,
+      "d_model": 1408,
+      "encoder_embed_dim": 1408,
+      "img_size": 224,
+      "name": "internvideo2-1B",
+      "num_frames": 8,
+      "origin_num_frames": 4,
+      "patch_size": 14,
+      "pretrained": null,
+      "sep_image_video_pos_embed": true,
+      "tubelet_size": 1,
+      "use_checkpoint": true,
+      "vit_add_ln": true,
+      "x_vis_only": true,
+      "x_vis_return_idx": -2
+    }
   },
-  "freeze_bridge": false,
-  "freeze_llm": false,
-  "freeze_vision_encoder": false,
-  "llm": {
-    "lora_alpha": 32,
-    "lora_dropout": 0.1,
-    "lora_r": 16,
-    "name": "mistral_7b",
-    "pretrained_llm_path": "mistralai/Mistral-7B-Instruct-v0.3",
-    "use_lora": true
-  },
-  "loss": {
-    "use_vision_regression_loss": false
-  },
-  "model_type": "InternVideo2_VideoChat2_test",
-  "pretrained_paths": {},
+  "model_type": "InternVideo2_Classification_test",
   "torch_dtype": "float32",
-  "transformers_version": "4.46.1",
-  "use_flash_attention": true,
-  "vision_encoder": {
-    "checkpoint_num": 48,
-    "d_model": 1408,
-    "encoder_embed_dim": 1408,
-    "img_size": 224,
-    "name": "internvideo2-1B",
-    "num_frames": 8,
-    "origin_num_frames": 4,
-    "patch_size": 14,
-    "pretrained": null,
-    "sep_image_video_pos_embed": true,
-    "tubelet_size": 1,
-    "use_checkpoint": true,
-    "vit_add_ln": true,
-    "x_vis_only": true,
-    "x_vis_return_idx": -2
-  }
+  "transformers_version": "4.46.1"
 }

model-00007-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:06bb1178e0e08dd4363637dffed42017c083f97fe6c2023b7eed2b6dd1cf1007
+oid sha256:79c534fd5231ae42d92fde6d705d517b6721d37d87d304b7f699b6b2c850c276
 size 4109221232

modeling_videochat2_classification.py

@@ -389,15 +389,48 @@ class InternVideo2_Classification_test(PreTrainedModel):
     config_class = VideoChat2Config
     def __init__(self, config):
         super().__init__(config)
-        self.w = torch.randn(10,10, requires_grad=True)
+        self.conv1 = nn.Conv2d(1, 20, 5)
+        self.conv2 = nn.Conv2d(20, 20, 5)
+        self.model_config = config.model_config
+        self.build_bridge()
 
 
     def forward(self, x):
-        return x
+        x = self.conv1(x)
+        return self.conv2(x)
     
     def test_lol(self, x):
         return x
 
+    def build_bridge(self):
+        
+        if 'qformer' in self.model_config.bridge.name.lower():
+            from transformers import BertTokenizer
+            self.qformer_tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", truncation_side="left")
+            self.qformer_tokenizer.add_special_tokens({"bos_token": "[DEC]"})
+            self.qformer_tokenizer.padding_side = "left"
+            if self.model_config.bridge.name == 'qformer':
+                self.qformer, self.query_tokens = build_qformer(
+                        self.model_config.bridge.num_query_token, self.model_config.vision_encoder.encoder_embed_dim,
+                        qformer_hidden_dropout_prob=self.model_config.bridge.qformer_hidden_dropout_prob,
+                        qformer_attention_probs_dropout_prob=self.model_config.bridge.qformer_attention_probs_dropout_prob,
+                        qformer_drop_path_rate=self.model_config.bridge.qformer_drop_path_rate,
+                )
+            self.qformer.resize_token_embeddings(len(self.qformer_tokenizer))
+            self.qformer.cls = None
+            self.extra_num_query_token = self.model_config.bridge.extra_num_query_token
+            if self.model_config.bridge.extra_num_query_token > 0:
+                logger.info(f"Add extra {self.model_config.bridge.extra_num_query_token} tokens in QFormer")
+                self.extra_query_tokens = nn.Parameter(
+                    torch.zeros(1, self.model_config.bridge.extra_num_query_token, self.query_tokens.shape[-1])
+                )
+            
+            self.freeze_bridge = self.model_config.get("freeze_bridge", False)
+            if self.freeze_bridge:
+                logger.info("freeze bridge")
+                freeze_module(self.qformer)
+                self.query_tokens.requires_grad = False
+
 if __name__ == "__main__":
 
     tokenizer =  AutoTokenizer.from_pretrained('OpenGVLab/InternVideo2-Chat-8B',trust_remote_code=True,use_fast=False)