Delete models

models/InternVideo_next.py

@@ -1,559 +0,0 @@
-import math
-import torch
-import torch.nn.functional as F
-from timm.models.layers import DropPath, to_2tuple, trunc_normal_
-from timm.models.registry import register_model
-from torch import nn
-
-import torch.utils.checkpoint as checkpoint
-from functools import partial
-from einops import rearrange
-
-from .pos_embed import get_3d_sincos_pos_embed, get_2d_sincos_pos_embed, get_1d_sincos_pos_embed
-from .flash_attention_class import FlashAttention
-from flash_attn.modules.mlp import FusedMLP
-from flash_attn.ops.rms_norm import DropoutAddRMSNorm
-
-import einops
-
-class CrossAttention(nn.Module):
-    def __init__(
-            self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0.,
-            proj_drop=0., attn_head_dim=None, out_dim=None):
-        super().__init__()
-        if out_dim is None:
-            out_dim = dim
-        self.num_heads = num_heads
-        head_dim = dim // num_heads
-        if attn_head_dim is not None:
-            head_dim = attn_head_dim
-        all_head_dim = head_dim * self.num_heads
-        self.scale = qk_scale or head_dim ** -0.5
-        assert all_head_dim == dim
-        
-        self.q = nn.Linear(dim, all_head_dim, bias=False)
-        self.k = nn.Linear(dim, all_head_dim, bias=False)
-        self.v = nn.Linear(dim, all_head_dim, bias=False)
-        
-        if qkv_bias:
-            self.q_bias = nn.Parameter(torch.zeros(all_head_dim))
-            self.k_bias = nn.Parameter(torch.zeros(all_head_dim))
-            self.v_bias = nn.Parameter(torch.zeros(all_head_dim))
-        else:
-            self.q_bias = None
-            self.k_bias = None
-            self.v_bias = None
-        
-        self.attn_drop = nn.Dropout(attn_drop)
-        self.proj = nn.Linear(all_head_dim, out_dim)
-        self.proj_drop = nn.Dropout(proj_drop)
-    
-    def forward(self, x, k=None, v=None):
-        B, N, C = x.shape
-        N_k = k.shape[1]
-        N_v = v.shape[1]
-        
-        q_bias, k_bias, v_bias = None, None, None
-        if self.q_bias is not None:
-            q_bias = self.q_bias
-            k_bias = self.k_bias
-            v_bias = self.v_bias
-        
-        q = F.linear(input=x, weight=self.q.weight, bias=q_bias)
-        q = q.reshape(B, N, 1, self.num_heads, -1).permute(2, 0, 3, 1, 4).squeeze(0)  # (B, N_head, N_q, dim)
-        
-        k = F.linear(input=k, weight=self.k.weight, bias=k_bias)
-        k = k.reshape(B, N_k, 1, self.num_heads, -1).permute(2, 0, 3, 1, 4).squeeze(0)
-        
-        v = F.linear(input=v, weight=self.v.weight, bias=v_bias)
-        v = v.reshape(B, N_v, 1, self.num_heads, -1).permute(2, 0, 3, 1, 4).squeeze(0)
-        
-        q = q * self.scale
-        attn = (q @ k.transpose(-2, -1))  # (B, N_head, N_q, N_k)
-        
-        attn = attn.softmax(dim=-1)
-        attn = self.attn_drop(attn)
-        
-        x = (attn @ v).transpose(1, 2).reshape(B, N, -1)
-        x = self.proj(x)
-        x = self.proj_drop(x)
-        
-        return x
-
-
-class AttentiveBlock(nn.Module):
-    
-    def __init__(self, dim, num_heads, qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,
-                 drop_path=0., norm_layer=nn.LayerNorm, attn_head_dim=None, out_dim=None):
-        super().__init__()
-        
-        self.norm1_q = norm_layer(dim)
-        self.norm1_k = norm_layer(dim)
-        self.norm1_v = norm_layer(dim)
-        self.cross_attn = CrossAttention(
-            dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop,
-            proj_drop=drop, attn_head_dim=attn_head_dim, out_dim=out_dim)
-        
-        if drop_path > 0.:
-            print(f"Use DropPath in projector: {drop_path}")
-        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
-    
-    def forward(self, x_q, x_kv, pos_q, pos_k, bool_masked_pos, rel_pos_bias=None):
-        x_q = self.norm1_q(x_q + pos_q)
-        x_k = self.norm1_k(x_kv + pos_k)
-        x_v = self.norm1_v(x_kv)
-        x = self.cross_attn(x_q, k=x_k, v=x_v)
-        
-        return x
-
-
-class AttentionPoolingBlock(AttentiveBlock):
-    
-    def forward(self, x):
-        x_q = x.mean(1, keepdim=True)
-        x_kv, pos_q, pos_k = x, 0, 0
-        x = super().forward(x_q, x_kv, pos_q, pos_k, bool_masked_pos=None, rel_pos_bias=None)
-        x = x.squeeze(1)
-        return x
-
-
-class RMSNorm(nn.Module):
-    def __init__(self, hidden_size, eps=1e-6):
-        super().__init__()
-        self.weight = nn.Parameter(torch.ones(hidden_size))
-        self.variance_epsilon = eps
-    
-    def forward(self, hidden_states):
-        input_dtype = hidden_states.dtype
-        hidden_states = hidden_states.to(torch.float32)
-        variance = hidden_states.pow(2).mean(-1, keepdim=True)
-        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
-        return self.weight * hidden_states.to(input_dtype)
-
-
-class LayerScale(nn.Module):
-    def __init__(self, dim, init_values=1e-5, inplace=False, force_fp32=False):
-        super().__init__()
-        self.inplace = inplace
-        self.gamma = nn.Parameter(init_values * torch.ones(dim))
-        self.force_fp32 = force_fp32
-    
-    @torch.cuda.amp.autocast(enabled=False)
-    def forward(self, x):
-        if self.force_fp32:
-            output_type = x.dtype
-            out = x.float().mul_(self.gamma.float()) if self.inplace else x.float() * self.gamma.float()
-            return out.to(dtype=output_type)
-        else:
-            out = x.mul_(self.gamma) if self.inplace else x * self.gamma
-            return out
-
-
-class Attention(nn.Module):
-    def __init__(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0., use_flash_attn=False,
-                 causal=False, norm_layer=nn.LayerNorm, qk_normalization=False, use_fused_rmsnorm=False):
-        super().__init__()
-        assert dim % num_heads == 0, 'dim should be divisible by num_heads'
-        self.num_heads = num_heads
-        head_dim = dim // num_heads
-        self.scale = head_dim ** -0.5
-        
-        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
-        self.attn_drop = nn.Dropout(attn_drop)
-        self.proj = nn.Linear(dim, dim)
-        self.proj_drop = nn.Dropout(proj_drop)
-        
-        self.use_flash_attn = use_flash_attn
-        if use_flash_attn:
-            self.causal = causal
-            self.inner_attn = FlashAttention(attention_dropout=attn_drop)
-        
-        self.qk_normalization = qk_normalization
-        self.q_norm = norm_layer(dim) if qk_normalization else nn.Identity()
-        self.k_norm = norm_layer(dim) if qk_normalization else nn.Identity()
-        self.use_fused_rmsnorm = use_fused_rmsnorm
-    
-    def _naive_attn(self, x):
-        B, N, C = x.shape
-        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
-        q, k, v = qkv.unbind(0)  # make torchscript happy (cannot use tensor as tuple)
-        
-        if self.qk_normalization:
-            B_, H_, N_, D_ = q.shape
-            q = self.q_norm(q.transpose(1, 2).flatten(-2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
-            k = self.k_norm(k.transpose(1, 2).flatten(-2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
-        
-        attn = ((q * self.scale) @ k.transpose(-2, -1))
-        # attn = attn - attn.max(-1)[0].unsqueeze(-1)  # in case of overflow for fp16
-        attn = attn.softmax(dim=-1)
-        attn = self.attn_drop(attn)
-        
-        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
-        x = self.proj(x)
-        x = self.proj_drop(x)
-        return x
-    
-    def _flash_attn(self, x, key_padding_mask=None, need_weights=False):
-        
-        qkv = self.qkv(x)
-        qkv = rearrange(qkv, "b s (three h d) -> b s three h d", three=3, h=self.num_heads)
-        
-        if self.qk_normalization:
-            q, k, v = qkv.unbind(2)
-            if self.use_fused_rmsnorm:
-                q = self.q_norm(q.flatten(-2, -1))[0].view(q.shape)
-                k = self.k_norm(k.flatten(-2, -1))[0].view(k.shape)
-            else:
-                q = self.q_norm(q.flatten(-2, -1)).view(q.shape)
-                k = self.k_norm(k.flatten(-2, -1)).view(k.shape)
-            qkv = torch.stack([q, k, v], dim=2)
-        
-        context, _ = self.inner_attn(
-            qkv, key_padding_mask=key_padding_mask, need_weights=need_weights, causal=self.causal
-        )
-        outs = self.proj(rearrange(context, "b s h d -> b s (h d)"))
-        outs = self.proj_drop(outs)
-        return outs
-    
-    def forward(self, x):
-        x = self._naive_attn(x) if not self.use_flash_attn else self._flash_attn(x)
-        return x
-
-
-class Mlp(nn.Module):
-    """ MLP as used in Vision Transformer, MLP-Mixer and related networks
-    """
-    
-    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU,
-                 bias=True, drop=0.):
-        super().__init__()
-        out_features = out_features or in_features
-        hidden_features = hidden_features or in_features
-        bias = to_2tuple(bias)
-        drop_probs = to_2tuple(drop)
-        
-        self.fc1 = nn.Linear(in_features, hidden_features, bias=bias[0])
-        self.act = act_layer()
-        self.drop1 = nn.Dropout(drop_probs[0])
-        self.fc2 = nn.Linear(hidden_features, out_features, bias=bias[1])
-        self.drop2 = nn.Dropout(drop_probs[1])
-    
-    def forward(self, x):
-        x = self.fc1(x)
-        x = self.act(x)
-        x = self.drop1(x)
-        x = self.fc2(x)
-        x = self.drop2(x)
-        return x
-
-
-class Block(nn.Module):
-    
-    def __init__(
-            self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., init_values=None,
-            drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, use_flash_attn=False, use_fused_mlp=False,
-            fused_mlp_heuristic=1, with_cp=False, qk_normalization=False, layerscale_no_force_fp32=False,
-            use_fused_rmsnorm=False):
-        super().__init__()
-        
-        self.norm1 = norm_layer(dim)
-        self.attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop,
-                              use_flash_attn=use_flash_attn, causal=False, norm_layer=norm_layer,
-                              qk_normalization=qk_normalization,
-                              use_fused_rmsnorm=use_fused_rmsnorm)
-        self.ls1 = LayerScale(dim, init_values=init_values,
-                              force_fp32=(not layerscale_no_force_fp32)) if init_values else nn.Identity()
-        # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here
-        self.drop_path1 = DropPath(drop_path) if drop_path > 0. else nn.Identity()
-        
-        self.norm2 = norm_layer(dim)
-        mlp_hidden_dim = int(dim * mlp_ratio)
-        if use_fused_mlp:
-            self.mlp = FusedMLP(in_features=dim, hidden_features=mlp_hidden_dim, heuristic=fused_mlp_heuristic)
-        else:
-            self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
-        self.ls2 = LayerScale(dim, init_values=init_values,
-                              force_fp32=(not layerscale_no_force_fp32)) if init_values else nn.Identity()
-        self.drop_path2 = DropPath(drop_path) if drop_path > 0. else nn.Identity()
-        
-        self.with_cp = with_cp
-        self.use_fused_rmsnorm = use_fused_rmsnorm
-    
-    def forward(self, x, residual=None):
-        
-        def _inner_forward(x, residual=None):
-            if self.use_fused_rmsnorm:
-                x, residual = self.norm1(x, residual)
-                x = self.drop_path1(self.ls1(self.attn(x)))
-                x, residual = self.norm2(x, residual)
-                x = self.drop_path2(self.ls2(self.mlp(x)))
-                return x, residual
-            else:
-                assert residual is None
-                x = x + self.drop_path1(self.ls1(self.attn(self.norm1(x))))
-                x = x + self.drop_path2(self.ls2(self.mlp(self.norm2(x))))
-                return x
-        
-        if self.with_cp:
-            return checkpoint.checkpoint(_inner_forward, x, residual)
-        else:
-            return _inner_forward(x, residual=residual)
-
-class PatchEmbed(nn.Module):
-    """ 3D Image to Patch Embedding
-    """
-    
-    def __init__(
-            self, img_size=224, patch_size=16, in_chans=3, embed_dim=768, 
-            num_frames=8, tubelet_size=1, norm_layer=None
-        ):
-        super().__init__()
-        img_size = to_2tuple(img_size)
-        patch_size = to_2tuple(patch_size)
-        self.img_size = img_size
-        self.patch_size = patch_size
-        self.tubelet_size = tubelet_size
-        self.grid_size = (
-            num_frames // tubelet_size, 
-            img_size[0] // patch_size[0], 
-            img_size[1] // patch_size[1]
-        ) # (T, H, W)
-        self.num_patches = self.grid_size[0] * self.grid_size[1] * self.grid_size[2]
-        
-        self.proj = nn.Conv3d(
-            in_channels=in_chans, out_channels=embed_dim, 
-            kernel_size=(tubelet_size, patch_size[0], patch_size[1]), 
-            stride=(tubelet_size, patch_size[0], patch_size[1])
-        )
-        
-        self.norm = norm_layer(embed_dim)
-        self.norm_before = norm_layer(tubelet_size * math.prod(patch_size) * 3)
-    
-    def forward(self, x):
-        B, C, T, H, W = x.shape
-        x = x.permute(0, 2, 3, 4, 1)
-        x = einops.rearrange(x, "b (t1 t2) (ht hp) (wt wp) c -> b (t1 ht wt) (t2 hp wp c)", t2=self.tubelet_size, hp=self.patch_size[0], wp=self.patch_size[1])
-        x = self.norm_before(x) # x.shape: [B, T, HW, C]
-        x = einops.rearrange(x, "b (t1 ht wt) (t2 hp wp c) -> b (t1 t2) (ht hp) (wt wp) c", t1=T//self.tubelet_size, ht=H//self.patch_size[0], t2=self.tubelet_size, hp=self.patch_size[0], wp=self.patch_size[1])
-        x = x.permute(0, 4, 1, 2, 3)
-        x = self.proj(x)
-        x = x.flatten(3).permute(0, 2, 3, 1)
-        x = self.norm(x)
-        return x
-
-
-class InternVideo2(nn.Module):
-    def __init__(
-            self,
-            in_chans: int = 3,
-            patch_size: int = 14,
-            img_size: int = 224,
-            qkv_bias: bool = False,
-            drop_path_rate: float = 0.25,
-            embed_dim: int = 1408,
-            head_drop_path_rate: float = 0.,
-            num_heads: int = 16,
-            mlp_ratio: float = 4.3637,
-            init_values: float = 1e-5,
-            qk_normalization: bool = True,
-            depth: int = 40,
-            use_flash_attn: bool = True,
-            use_fused_rmsnorm: bool = True,
-            use_fused_mlp: bool = True,
-            fused_mlp_heuristic: int = 1,
-            attn_pool_num_heads: int = 16,
-            clip_embed_dim: int = 768,
-            layerscale_no_force_fp32: bool = False,
-            num_frames: int = 16,
-            tubelet_size: int = 1,
-            sep_pos_embed: bool = False,
-            use_checkpoint: bool = False,
-            checkpoint_num: int = 0,
-            cls_token_num: int = 4,
-        ):
-        super().__init__()
-        self.cls_token_num = cls_token_num
-        assert use_flash_attn == use_fused_rmsnorm == use_fused_mlp, print(
-            'use_flash_attn, use_fused_rmsnorm and use_fused_mlp should be consistent')
-        print(mlp_ratio)
-        
-        self.use_flash_attn = use_flash_attn
-        self.embed_dim = embed_dim
-        
-        if use_fused_rmsnorm:
-            norm_layer_for_blocks = partial(DropoutAddRMSNorm, eps=1e-6, prenorm=True)
-        else:
-            norm_layer_for_blocks = partial(RMSNorm, eps=1e-6)
-        self.norm_layer_for_blocks = norm_layer_for_blocks
-        self.patch_embed = PatchEmbed(
-            img_size, patch_size, in_chans, embed_dim,
-            num_frames=num_frames, tubelet_size=tubelet_size, norm_layer=partial(RMSNorm, eps=1e-6)
-        )
-        num_patches = self.patch_embed.num_patches
-        self.cls_token = nn.Parameter(torch.zeros(1, cls_token_num, embed_dim))
-        
-        self.sep_pos_embed = sep_pos_embed
-        if sep_pos_embed:
-            print("Use seperable position embedding")
-            grid_size = self.patch_embed.grid_size
-            self.grid_size = grid_size
-            self.pos_embed_spatial = nn.Parameter(torch.zeros(1, grid_size[1] * grid_size[2], embed_dim))
-            self.pos_embed_temporal = nn.Parameter(torch.zeros(1, grid_size[0], embed_dim))
-            self.pos_embed_cls = nn.Parameter(torch.zeros(1, 1, embed_dim))
-        else:
-            print("Use joint position embedding")
-            self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + cls_token_num, embed_dim))
-        
-        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]
-        # choose which layer to use checkpoint
-        with_cp_list = [False] * depth
-        if use_checkpoint:
-            for idx in range(depth):
-                if idx < checkpoint_num:
-                    with_cp_list[idx] = True
-        print(f"Droppath rate: {dpr}")
-        print(f"Checkpoint list: {with_cp_list}")
-        
-        self.blocks = nn.ModuleList([
-            Block(embed_dim, num_heads, mlp_ratio, qkv_bias=qkv_bias,
-                  norm_layer=norm_layer_for_blocks,
-                  drop_path=dpr[i], init_values=init_values, attn_drop=0.,
-                  use_flash_attn=use_flash_attn, use_fused_mlp=use_fused_mlp,
-                  fused_mlp_heuristic=fused_mlp_heuristic,
-                  with_cp=with_cp_list[i],
-                  qk_normalization=qk_normalization,
-                  layerscale_no_force_fp32=layerscale_no_force_fp32,
-                  use_fused_rmsnorm=use_fused_rmsnorm)
-            for i in range(depth)])
-        self.clip_projector = AttentionPoolingBlock(
-            dim=embed_dim, num_heads=attn_pool_num_heads, qkv_bias=True, qk_scale=None,
-            drop=0., attn_drop=0., drop_path=head_drop_path_rate, 
-            norm_layer=partial(nn.LayerNorm, eps=1e-5), out_dim=clip_embed_dim
-        )
-
-        self.init_pos_embed()
-        trunc_normal_(self.cls_token, std=.02)
-        self.apply(self._init_weights)
-        self.fix_init_weight()
-
-    def init_pos_embed(self):
-        print("Init pos_embed from sincos pos_embed")
-        if self.sep_pos_embed:
-            pos_embed_spatial = get_2d_sincos_pos_embed(
-                self.pos_embed_spatial.shape[-1], 
-                self.patch_embed.grid_size[1], # height & weight
-            )
-            self.pos_embed_spatial.data.copy_(torch.from_numpy(pos_embed_spatial).float().unsqueeze(0))
-            pos_embed_temporal = get_1d_sincos_pos_embed(
-                self.pos_embed_spatial.shape[-1], 
-                self.patch_embed.grid_size[0], # t_size
-            )
-            self.pos_embed_temporal.data.copy_(torch.from_numpy(pos_embed_temporal).float().unsqueeze(0))
-        else:
-            pos_embed = get_3d_sincos_pos_embed(
-                self.pos_embed.shape[-1], 
-                self.patch_embed.grid_size[1], # height & weight
-                self.patch_embed.grid_size[0], # t_size
-                cls_token=True,
-                cls_token_num=self.cls_token_num
-            )
-            self.pos_embed.data.copy_(torch.from_numpy(pos_embed).float().unsqueeze(0))
-
-    def _init_weights(self, m):
-        if isinstance(m, nn.Linear):
-            trunc_normal_(m.weight, std=.02)
-            if isinstance(m, nn.Linear) and m.bias is not None:
-                nn.init.constant_(m.bias, 0)
-        elif isinstance(m, nn.LayerNorm):
-            nn.init.constant_(m.bias, 0)
-            nn.init.constant_(m.weight, 1.0)
-
-    def fix_init_weight(self):
-        def rescale(param, layer_id):
-            param.div_(math.sqrt(2.0 * layer_id))
-
-        for layer_id, layer in enumerate(self.blocks):
-            rescale(layer.attn.proj.weight.data, layer_id + 1)
-            rescale(layer.mlp.fc2.weight.data, layer_id + 1)
-    
-    @property
-    def dtype(self):
-        return self.patch_embed.proj.weight.dtype
-
-    def get_num_layers(self):
-        return len(self.blocks)
-
-    @torch.jit.ignore
-    def no_weight_decay(self):
-        return {
-            'pos_embed', 
-            'pos_embed_spatial', 
-            'pos_embed_temporal', 
-            'pos_embed_cls',
-            'cls_token'
-        }
-    
-    def forward(self, x, projected=False):
-        x = self.patch_embed(x.type(self.dtype))
-        B, T, L, C = x.shape  # T: temporal; L: spatial
-        x = x.view([B, T * L, C])
-
-        # append cls token
-        cls_tokens = self.cls_token.expand(B, -1, -1)
-        x = torch.cat((cls_tokens, x), dim=1)
-
-        # add pos_embed
-        if self.sep_pos_embed:
-            pos_embed = self.pos_embed_spatial.repeat(
-                1, self.grid_size[0], 1
-            ) + torch.repeat_interleave(
-                self.pos_embed_temporal,
-                self.grid_size[1] * self.grid_size[2],
-                dim=1,
-            )
-            pos_embed = torch.cat(
-                [
-                    self.pos_embed_cls.expand(pos_embed.shape[0], -1, -1),
-                    pos_embed,
-                ],
-                1,
-            )
-        else:
-            pos_embed = self.pos_embed
-        x = x + pos_embed
-
-        residual = None
-        for blk in self.blocks:
-            if isinstance(x, tuple) and len(x) == 2:
-                x, residual = x
-            x = blk(x, residual=residual)
-        if isinstance(x, tuple) and len(x) == 2:
-            x, residual = x
-            if residual is not None:
-                x = x + residual
-
-        if projected:
-            return self.clip_projector(x)
-
-        return x[:, self.cls_token_num:, :]
-
-
-@register_model
-def internvideo_next_base_patch14_224(pretrained=False, **kwargs):
-    model = InternVideo2(
-        img_size=224, patch_size=14, embed_dim=768, 
-        depth=12, num_heads=12, mlp_ratio=4, 
-        attn_pool_num_heads=16, clip_embed_dim=768,
-        **kwargs
-    )
-    return model
-
-@register_model
-def internvideo_next_large_patch14_224(pretrained=False, **kwargs):
-    model = InternVideo2(
-        img_size=224, patch_size=14, embed_dim=1024, 
-        depth=24, num_heads=16, mlp_ratio=4, 
-        attn_pool_num_heads=16, clip_embed_dim=768,
-        **kwargs
-    )
-    return model

models/flash_attention_class.py

@@ -1,71 +0,0 @@
-import torch
-import torch.nn as nn
-
-from einops import rearrange
-
-from flash_attn.flash_attn_interface import flash_attn_varlen_qkvpacked_func
-from flash_attn.bert_padding import unpad_input, pad_input
-
-
-class FlashAttention(nn.Module):
-    """Implement the scaled dot product attention with softmax.
-    Arguments
-    ---------
-        softmax_scale: The temperature to use for the softmax attention.
-                      (default: 1/sqrt(d_keys) where d_keys is computed at
-                      runtime)
-        attention_dropout: The dropout rate to apply to the attention
-                           (default: 0.0)
-    """
-
-    def __init__(self, softmax_scale=None, attention_dropout=0.0, device=None, dtype=None):
-        super().__init__()
-        self.softmax_scale = softmax_scale
-        self.dropout_p = attention_dropout
-
-    def forward(self, qkv, key_padding_mask=None, causal=False, cu_seqlens=None,
-                max_s=None, need_weights=False):
-        """Implements the multihead softmax attention.
-        Arguments
-        ---------
-            qkv: The tensor containing the query, key, and value. (B, S, 3, H, D) if key_padding_mask is None
-                if unpadded: (nnz, 3, h, d)
-            key_padding_mask: a bool tensor of shape (B, S)
-        """
-        assert not need_weights
-        assert qkv.dtype in [torch.float16, torch.bfloat16]
-        assert qkv.is_cuda
-
-        if cu_seqlens is None:
-            batch_size = qkv.shape[0]
-            seqlen = qkv.shape[1]
-            if key_padding_mask is None:
-                qkv = rearrange(qkv, 'b s ... -> (b s) ...')
-                max_s = seqlen
-                cu_seqlens = torch.arange(0, (batch_size + 1) * seqlen, step=seqlen, dtype=torch.int32,
-                                          device=qkv.device)
-                output = flash_attn_varlen_qkvpacked_func(
-                    qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
-                    softmax_scale=self.softmax_scale, causal=causal
-                )
-                output = rearrange(output, '(b s) ... -> b s ...', b=batch_size)
-            else:
-                nheads = qkv.shape[-2]
-                x = rearrange(qkv, 'b s three h d -> b s (three h d)')
-                x_unpad, indices, cu_seqlens, max_s = unpad_input(x, key_padding_mask)
-                x_unpad = rearrange(x_unpad, 'nnz (three h d) -> nnz three h d', three=3, h=nheads)
-                output_unpad = flash_attn_varlen_qkvpacked_func(
-                    x_unpad, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
-                    softmax_scale=self.softmax_scale, causal=causal
-                )
-                output = rearrange(pad_input(rearrange(output_unpad, 'nnz h d -> nnz (h d)'),
-                                             indices, batch_size, seqlen),
-                                   'b s (h d) -> b s h d', h=nheads)
-        else:
-            assert max_s is not None
-            output = flash_attn_varlen_qkvpacked_func(
-                qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
-                softmax_scale=self.softmax_scale, causal=causal
-            )
-
-        return output, None

models/pos_embed.py

@@ -1,235 +0,0 @@
-import numpy as np
-import torch
-
-# --------------------------------------------------------
-# 3D sine-cosine position embedding
-# References:
-# MVD: https://github.com/ruiwang2021/mvd/blob/main/modeling_finetune.py
-# --------------------------------------------------------
-def get_3d_sincos_pos_embed(embed_dim, grid_size, t_size, cls_token=False, cls_token_num=4):
-    """
-    grid_size: int of the grid height and width
-    t_size: int of the temporal size
-    return:
-    pos_embed: [t_size*grid_size*grid_size, embed_dim] or [1+t_size*grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
-    """
-    assert embed_dim % 4 == 0
-    embed_dim_spatial = embed_dim // 4 * 3
-    embed_dim_temporal = embed_dim // 4
-
-    # spatial
-    grid_h = np.arange(grid_size, dtype=np.float32)
-    grid_w = np.arange(grid_size, dtype=np.float32)
-    grid = np.meshgrid(grid_w, grid_h)  # here w goes first
-    grid = np.stack(grid, axis=0)
-
-    grid = grid.reshape([2, 1, grid_size, grid_size])
-    pos_embed_spatial = get_2d_sincos_pos_embed_from_grid(
-        embed_dim_spatial, grid
-    )
-
-    # temporal
-    grid_t = np.arange(t_size, dtype=np.float32)
-    pos_embed_temporal = get_1d_sincos_pos_embed_from_grid(
-        embed_dim_temporal, grid_t
-    )
-
-    # concate: [T, H, W] order
-    pos_embed_temporal = pos_embed_temporal[:, np.newaxis, :]
-    pos_embed_temporal = np.repeat(
-        pos_embed_temporal, grid_size**2, axis=1
-    )  # [T, H*W, D // 4]
-    pos_embed_spatial = pos_embed_spatial[np.newaxis, :, :]
-    pos_embed_spatial = np.repeat(
-        pos_embed_spatial, t_size, axis=0
-    )  # [T, H*W, D // 4 * 3]
-
-    pos_embed = np.concatenate([pos_embed_temporal, pos_embed_spatial], axis=-1)
-    pos_embed = pos_embed.reshape([-1, embed_dim])  # [T*H*W, D]
-
-    if cls_token:
-        pos_embed = np.concatenate(
-            [np.zeros([cls_token_num, embed_dim]), pos_embed], axis=0
-        )
-    return pos_embed
-
-def get_3d_sincos_pos_embed_new(embed_dim, grid_size, t_size, cls_token=False, cls_token_num=4):
-    """
-    grid_size: tuple or list of (grid_height, grid_width)
-    t_size: int of the temporal size
-    return:
-    pos_embed: [t_size*grid_height*grid_width, embed_dim] or [1+t_size*grid_height*grid_width, embed_dim] (w/ or w/o cls_token)
-    """
-    assert embed_dim % 4 == 0
-    embed_dim_spatial = embed_dim // 4 * 3
-    embed_dim_temporal = embed_dim // 4
-    
-    # 处理 grid_size 参数，支持 int 或 tuple/list
-    if isinstance(grid_size, int):
-        grid_h = grid_size
-        grid_w = grid_size
-    else:
-        grid_h, grid_w = grid_size
-    
-    # spatial
-    grid_h_arange = np.arange(grid_h, dtype=np.float32)
-    grid_w_arange = np.arange(grid_w, dtype=np.float32)
-    grid = np.meshgrid(grid_w_arange, grid_h_arange)  # here w goes first
-    grid = np.stack(grid, axis=0)
-
-    grid = grid.reshape([2, 1, grid_h, grid_w])
-    pos_embed_spatial = get_2d_sincos_pos_embed_from_grid(
-        embed_dim_spatial, grid
-    )
-
-    # temporal
-    grid_t = np.arange(t_size, dtype=np.float32)
-    pos_embed_temporal = get_1d_sincos_pos_embed_from_grid(
-        embed_dim_temporal, grid_t
-    )
-
-    # concate: [T, H, W] order
-    pos_embed_temporal = pos_embed_temporal[:, np.newaxis, :]
-    pos_embed_temporal = np.repeat(
-        pos_embed_temporal, grid_h * grid_w, axis=1  # 修改为 grid_h * grid_w
-    )  # [T, H*W, D // 4]
-    
-    pos_embed_spatial = pos_embed_spatial[np.newaxis, :, :]
-    pos_embed_spatial = np.repeat(
-        pos_embed_spatial, t_size, axis=0
-    )  # [T, H*W, D // 4 * 3]
-
-    pos_embed = np.concatenate([pos_embed_temporal, pos_embed_spatial], axis=-1)
-    pos_embed = pos_embed.reshape([-1, embed_dim])  # [T*H*W, D]
-
-    if cls_token:
-        pos_embed = np.concatenate(
-            [np.zeros([cls_token_num, embed_dim]), pos_embed], axis=0
-        )
-    return pos_embed
-
-# --------------------------------------------------------
-# 2D sine-cosine position embedding
-# References:
-# Transformer: https://github.com/tensorflow/models/blob/master/official/nlp/transformer/model_utils.py
-# MoCo v3: https://github.com/facebookresearch/moco-v3
-# --------------------------------------------------------
-def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False):
-    """
-    grid_size: int of the grid height and width
-    return:
-    pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
-    """
-    grid_h = np.arange(grid_size, dtype=np.float32)
-    grid_w = np.arange(grid_size, dtype=np.float32)
-    grid = np.meshgrid(grid_w, grid_h)  # here w goes first
-    grid = np.stack(grid, axis=0)
-
-    grid = grid.reshape([2, 1, grid_size, grid_size])
-    pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
-    if cls_token:
-        pos_embed = np.concatenate(
-            [np.zeros([1, embed_dim]), pos_embed], axis=0
-        )
-    return pos_embed
-
-
-def get_1d_sincos_pos_embed(embed_dim, t_size, cls_token=False):
-    """
-    t_size: int of the temporal size
-    return:
-    pos_embed: [t_size, embed_dim] or [1+t_size, embed_dim] (w/ or w/o cls_token)
-    """
-    grid_t = np.arange(t_size, dtype=np.float32)
-    pos_embed = get_1d_sincos_pos_embed_from_grid(embed_dim, grid_t)
-    if cls_token:
-        pos_embed = np.concatenate(
-            [np.zeros([1, embed_dim]), pos_embed], axis=0
-        )
-    return pos_embed
-
-
-def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
-    assert embed_dim % 2 == 0
-
-    # use half of dimensions to encode grid_h
-    emb_h = get_1d_sincos_pos_embed_from_grid(
-        embed_dim // 2, grid[0]
-    )  # (H*W, D/2)
-    emb_w = get_1d_sincos_pos_embed_from_grid(
-        embed_dim // 2, grid[1]
-    )  # (H*W, D/2)
-
-    emb = np.concatenate([emb_h, emb_w], axis=1)  # (H*W, D)
-    return emb
-
-
-def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
-    """
-    embed_dim: output dimension for each position
-    pos: a list of positions to be encoded: size (M,)
-    out: (M, D)
-    """
-    assert embed_dim % 2 == 0
-    omega = np.arange(embed_dim // 2, dtype=np.float32)
-    omega /= embed_dim / 2.0
-    omega = 1.0 / 10000**omega  # (D/2,)
-
-    pos = pos.reshape(-1)  # (M,)
-    out = np.einsum("m,d->md", pos, omega)  # (M, D/2), outer product
-
-    emb_sin = np.sin(out)  # (M, D/2)
-    emb_cos = np.cos(out)  # (M, D/2)
-
-    emb = np.concatenate([emb_sin, emb_cos], axis=1)  # (M, D)
-    return emb
-
-
-def interpolate_pos_embed(checkpoint_model, model, orig_t_size=4, pos_name='vision_encoder.pos_embed'):
-    if pos_name in checkpoint_model:
-        pos_embed_checkpoint = checkpoint_model[pos_name]
-        embedding_size = pos_embed_checkpoint.shape[-1] # channel dim
-        num_patches = model.patch_embed.num_patches # 
-        num_extra_tokens = model.pos_embed.shape[-2] - num_patches # 0/1
-
-        # we use 4 frames for pretraining
-        new_t_size = model.T
-        # height (== width) for the checkpoint position embedding
-        orig_size = int(((pos_embed_checkpoint.shape[-2] - num_extra_tokens)//(orig_t_size)) ** 0.5)
-        # height (== width) for the new position embedding
-        new_size = int((num_patches // (new_t_size))** 0.5)
-        
-        # class_token and dist_token are kept unchanged
-        if orig_t_size != new_t_size:
-            print(f"Temporal interpolate from {orig_t_size} to {new_t_size} ({pos_name})")
-            extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
-            # only the position tokens are interpolated
-            pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
-            # B, L, C -> B， T, HW, C -> BHW, C, T  (B = 1)
-            pos_tokens = pos_tokens.view(1, orig_t_size, -1, embedding_size)
-            pos_tokens = pos_tokens.permute(0, 2, 3, 1).reshape(-1, embedding_size, orig_t_size)
-            pos_tokens = torch.nn.functional.interpolate(pos_tokens, size=new_t_size, mode='linear')
-            pos_tokens = pos_tokens.view(1, -1, embedding_size, new_t_size)
-            pos_tokens = pos_tokens.permute(0, 3, 1, 2).reshape(1, -1, embedding_size)
-            new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
-            checkpoint_model[pos_name] = new_pos_embed
-            pos_embed_checkpoint = new_pos_embed
-
-        # class_token and dist_token are kept unchanged
-        if orig_size != new_size:
-            print(f"Position interpolate from {orig_size}x{orig_size} to {new_size}x{new_size} ({pos_name})")
-            extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
-            # only the position tokens are interpolated
-            pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
-            # B, L, C -> BT, H, W, C -> BT, C, H, W
-            pos_tokens = pos_tokens.reshape(-1, new_t_size, orig_size, orig_size, embedding_size)
-            pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2)
-            pos_tokens = torch.nn.functional.interpolate(
-                pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False)
-            # BT, C, H, W -> BT, H, W, C ->  B, T, H, W, C
-            pos_tokens = pos_tokens.permute(0, 2, 3, 1).reshape(-1, new_t_size, new_size, new_size, embedding_size) 
-            pos_tokens = pos_tokens.flatten(1, 3) # B, L, C
-            new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
-            checkpoint_model[pos_name] = new_pos_embed
-    else:
-        raise NotImplementedError