[Init] upload model

.gitattributes

@@ -1,35 +1 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,191 @@
+{
+  "architectures": [
+    "InternVideo2_CLIP_small"
+  ],
+  "auto_map": {
+    "AutoConfig": "config.InternVideo2Config",
+    "AutoModel": "modeling_internvideo2encoder.InternVideo2_CLIP_small"
+  },
+  "auto_resume": false,
+  "batch_size": 64,
+  "batch_size_test": 4,
+  "best_key": [
+    "msrvtt_1k_test_match",
+    "t2v_r1"
+  ],
+  "compile_model": false,
+  "criterion": {
+    "clip_loss_ratio": [
+      1.0,
+      1.0
+    ],
+    "distill_final_features": true,
+    "loss_weight": {
+      "mlm": 1.0,
+      "mvm": 0.0,
+      "uta": 0.0,
+      "vtc": 1.0,
+      "vtm": 1.0
+    },
+    "mlm_masking_prob": 0.5,
+    "vtm_hard_neg": true
+  },
+  "debug": false,
+  "deep_fusion": false,
+  "deepspeed": {
+    "enable": true,
+    "stage": 1
+  },
+  "delete_ds_optim_states": true,
+  "device": "cuda",
+  "dist_url": "env://",
+  "evaluate": false,
+  "evaluation": {
+    "eval_frame_ensemble": "concat",
+    "eval_offload": true,
+    "eval_x_only": false,
+    "k_test": 128
+  },
+  "gradient_checkpointing": true,
+  "inputs": {
+    "batch_size": {
+      "image": 64,
+      "video": 64
+    },
+    "batch_size_test": {
+      "image": 4,
+      "video": 4
+    },
+    "image_res": 224,
+    "max_txt_l": {
+      "image": 32,
+      "video": 32
+    },
+    "video_input": {
+      "num_frames": 8,
+      "num_frames_test": 8,
+      "random_aug": false,
+      "sample_type": "middle",
+      "sample_type_test": "middle"
+    }
+  },
+  "jump_evaluate": false,
+  "log_freq": 100,
+  "max_txt_l": 32,
+  "mode": "pt",
+  "model": {
+    "embed_dim": 1024,
+    "find_unused_parameters": false,
+    "freeze_text": true,
+    "freeze_vision": true,
+    "load_vision_ckpt_from_internvideo2_stage2": false,
+    "model_cls": "InternVideo2_CLIP_small",
+    "multimodal": {
+      "enable": true
+    },
+    "open_text_projection": false,
+    "open_vision_clip_projector": true,
+    "temp": 0.01,
+    "temp_min": 0.01,
+    "text_encoder": {
+      "embed_dim": 512,
+      "image_cfg": {
+        "image_size": 224,
+        "model_name": "vit_b16"
+      },
+      "text_cfg": {
+        "causal_masking": true,
+        "context_length": 77,
+        "dim": 512,
+        "ffn_multiplier_per_layer": 4.0,
+        "model_name": "base",
+        "n_heads_per_layer": 8,
+        "n_transformer_layers": 12,
+        "norm_layer": "layer_norm_fp32",
+        "vocab_size": 49408
+      }
+    },
+    "vision_encoder": {
+      "align_dim": 512,
+      "attn_pool_num_heads": 16,
+      "checkpoint_num": 0,
+      "clip_embed_dim": 768,
+      "depth": 24,
+      "drop_cls_token": false,
+      "drop_path_rate": 0.0,
+      "embed_dim": 1024,
+      "fused_mlp_heuristic": 1,
+      "head_drop_path_rate": 0.0,
+      "img_size": 224,
+      "in_chans": 3,
+      "init_values": 0.1,
+      "layerscale_no_force_fp32": true,
+      "mlp_ratio": 4,
+      "name": "internvideo2_1B",
+      "num_frames": 8,
+      "num_heads": 16,
+      "patch_size": 14,
+      "qk_normalization": true,
+      "qkv_bias": false,
+      "sep_pos_embed": false,
+      "tubelet_size": 1,
+      "use_checkpoint": false,
+      "use_flash_attn": false,
+      "use_fused_mlp": false,
+      "use_fused_rmsnorm": false
+    }
+  },
+  "model_type": "internvideo2",
+  "num_frames": 8,
+  "num_frames_test": 8,
+  "num_workers": 6,
+  "optimizer": {
+    "different_lr": {
+      "enable": false,
+      "lr": 0.001,
+      "module_names": []
+    },
+    "lr": 5e-05,
+    "max_grad_norm": 3.0,
+    "opt": "adamW",
+    "opt_betas": [
+      0.9,
+      0.98
+    ],
+    "weight_decay": 0.05
+  },
+  "output_dir": null,
+  "pretrained_path": "",
+  "resume": false,
+  "save_ckpt_iter": null,
+  "save_latest": true,
+  "scheduler": {
+    "epochs": 10,
+    "min_lr_multi": 0.01,
+    "sched": "cosine",
+    "warmup_epochs": 1
+  },
+  "seed": 42,
+  "test_file": {
+    "didemo_ret_test": "available_corpus[\"didemo_ret_test\"]",
+    "msrvtt_1k_test": "available_corpus[\"msrvtt_1k_test\"]"
+  },
+  "test_types": [
+    "msrvtt_1k_test",
+    "didemo_ret_test"
+  ],
+  "text_enc": "bert_large",
+  "tokenizer": null,
+  "torch_dtype": "float16",
+  "train_file": "available_corpus[\"pretrain_example_data_1B\"]",
+  "transformers_version": "4.51.3",
+  "use_bf16": true,
+  "use_flash_sdp": false,
+  "use_half_precision": false,
+  "use_mem_efficient_sdp": false,
+  "wandb": {
+    "enable": false,
+    "entity": "opengvlab",
+    "project": "InternVideo2-Stage2"
+  }
+}

config.py

@@ -0,0 +1,240 @@
+from transformers import PretrainedConfig, PreTrainedModel, AutoModel, AutoConfig
+
+class EasyDict(dict):
+    def __init__(self, d=None, **kwargs):
+        if d is None:
+            d = {}
+        if kwargs:
+            d.update(**kwargs)
+        for k, v in d.items():
+            setattr(self, k, v)
+        # Class attributes
+        for k in self.__class__.__dict__.keys():
+            if not (k.startswith("__") and k.endswith("__")) and not k in ("update", "pop"):
+                setattr(self, k, getattr(self, k))
+
+    def __setattr__(self, name, value):
+        if isinstance(value, (list, tuple)):
+            value = [self.__class__(x) if isinstance(x, dict) else x for x in value]
+        elif isinstance(value, dict) and not isinstance(value, self.__class__):
+            value = self.__class__(value)
+        super(EasyDict, self).__setattr__(name, value)
+        super(EasyDict, self).__setitem__(name, value)
+
+    __setitem__ = __setattr__
+
+    def update(self, e=None, **f):
+        d = e or dict()
+        d.update(f)
+        for k in d:
+            setattr(self, k, d[k])
+
+    def pop(self, k, d=None):
+        if hasattr(self, k):
+            delattr(self, k)
+        return super(EasyDict, self).pop(k, d)
+
+class InternVideo2Config(PretrainedConfig):
+    model_type = "internvideo2"
+
+    def __init__(self,
+                 tokenizer=None,
+                 train_file=None,
+                 test_file=None,
+                 test_types=None,
+                 num_workers=6,
+                 best_key=None,
+                 num_frames=8,
+                 num_frames_test=8,
+                 batch_size=64,
+                 batch_size_test=4,
+                 max_txt_l=32,
+                 inputs=None,
+                 text_enc="bert_large",
+                 model=None,
+                 criterion=None,
+                 optimizer=None,
+                 scheduler=None,
+                 evaluate=False,
+                 deep_fusion=False,
+                 evaluation=None,
+                 use_half_precision=False,
+                 use_bf16=True,
+                 gradient_checkpointing=True,
+                 use_flash_sdp=False,
+                 use_mem_efficient_sdp=False,
+                 compile_model=False,
+                 wandb=None,
+                 dist_url="env://",
+                 device="cuda",
+                 mode="pt",
+                 output_dir=None,
+                 resume=False,
+                 debug=False,
+                 log_freq=100,
+                 seed=42,
+                 save_latest=True,
+                 auto_resume=False,
+                 jump_evaluate=False,
+                 pretrained_path="",
+                 save_ckpt_iter=None,
+                 delete_ds_optim_states=True,
+                 deepspeed=None,
+                 **kwargs):
+        super().__init__(**kwargs)
+
+        self.tokenizer = tokenizer
+
+        # Data configuration
+        self.train_file = train_file or "available_corpus[\"pretrain_example_data_1B\"]"
+        self.test_file = EasyDict(test_file or {
+            "msrvtt_1k_test": "available_corpus[\"msrvtt_1k_test\"]",
+            "didemo_ret_test": "available_corpus[\"didemo_ret_test\"]"
+        })
+        self.test_types = test_types or ["msrvtt_1k_test", "didemo_ret_test"]
+        self.num_workers = num_workers
+        self.best_key = best_key or ["msrvtt_1k_test_match", "t2v_r1"]
+
+        # Input configuration
+        self.num_frames = num_frames
+        self.num_frames_test = num_frames_test
+        self.batch_size = batch_size
+        self.batch_size_test = batch_size_test
+        self.max_txt_l = max_txt_l
+        self.inputs = EasyDict(inputs or {
+            "image_res": 224,
+            "video_input": EasyDict({
+                "num_frames": num_frames,
+                "sample_type": "rand",
+                "num_frames_test": num_frames_test,
+                "sample_type_test": "middle",
+                "random_aug": False
+            }),
+            "max_txt_l": EasyDict({"image": max_txt_l, "video": max_txt_l}),
+            "batch_size": EasyDict({"image": batch_size, "video": batch_size}),
+            "batch_size_test": EasyDict({"image": batch_size_test, "video": batch_size_test})
+        })
+
+        # Model configuration
+        self.text_enc = text_enc
+        self.model = EasyDict(model or {
+            "model_cls": "InternVideo2_Stage2",
+            "vision_encoder": EasyDict({
+                "name": "pretrain_internvideo2_1b_patch14_224",
+                "img_size": 224,
+                "num_frames": num_frames,
+                "tubelet_size": 1,
+                "patch_size": 14,
+                "d_model": 1408,
+                "clip_embed_dim": 768,
+                "clip_teacher_embed_dim": 3200,
+                "clip_teacher_final_dim": 768,
+                "clip_norm_type": "l2",
+                "clip_return_layer": 6,
+                "clip_student_return_interval": 1,
+                "pretrained": None,
+                "use_checkpoint": False,
+                "checkpoint_num": 40,
+                "use_flash_attn": True,
+                "use_fused_rmsnorm": True,
+                "use_fused_mlp": True,
+                "clip_teacher": None,
+                "clip_input_resolution": 224,
+                "clip_teacher_return_interval": 1,
+                "video_mask_type": "random",
+                "video_mask_ratio": 0.8,
+                "image_mask_type": "random",
+                "image_mask_ratio": 0.5,
+                "sep_image_video_pos_embed": True,
+                "keep_temporal": False,
+                "only_mask": True
+            }),
+            "text_encoder": text_enc,
+            "multimodal": EasyDict({"enable": True}),
+            "embed_dim": 512,
+            "temp": 0.07,
+            "find_unused_parameters": False
+        })
+
+        # Criterion configuration
+        self.criterion = EasyDict(criterion or {
+            "loss_weight": EasyDict({
+                "vtc": 1.0,
+                "mlm": 1.0,
+                "vtm": 1.0,
+                "mvm": 0.0,
+                "uta": 0.0
+            }),
+            "vtm_hard_neg": True,
+            "mlm_masking_prob": 0.5,
+            "distill_final_features": True,
+            "clip_loss_ratio": [1.0, 1.0]
+        })
+
+        # Optimizer configuration
+        self.optimizer = EasyDict(optimizer or {
+            "opt": "adamW",
+            "lr": 5e-5,
+            "opt_betas": [0.9, 0.98],
+            "weight_decay": 0.05,
+            "max_grad_norm": 3.0,
+            "different_lr": EasyDict({"enable": False, "module_names": [], "lr": 1e-3})
+        })
+
+        # Scheduler configuration
+        self.scheduler = EasyDict(scheduler or {
+            "sched": "cosine",
+            "epochs": 10,
+            "min_lr_multi": 0.01,
+            "warmup_epochs": 1
+        })
+
+        # Evaluation configuration
+        self.evaluate = evaluate
+        self.deep_fusion = deep_fusion
+        self.evaluation = EasyDict(evaluation or {
+            "eval_frame_ensemble": "concat",
+            "eval_x_only": False,
+            "k_test": 128,
+            "eval_offload": True
+        })
+
+        # Miscellaneous
+        self.use_half_precision = use_half_precision
+        self.use_bf16 = use_bf16
+        self.gradient_checkpointing = gradient_checkpointing
+        self.use_flash_sdp = use_flash_sdp
+        self.use_mem_efficient_sdp = use_mem_efficient_sdp
+        self.compile_model = compile_model
+
+        self.wandb = EasyDict(wandb or {
+            "enable": False,
+            "entity": "opengvlab",
+            "project": "InternVideo2-Stage2"
+        })
+
+        self.dist_url = dist_url
+        self.device = device
+        self.mode = mode
+        self.output_dir = output_dir
+        self.resume = resume
+        self.debug = debug
+        self.log_freq = log_freq
+        self.seed = seed
+
+        self.save_latest = save_latest
+        self.auto_resume = auto_resume
+        self.jump_evaluate = jump_evaluate
+        self.pretrained_path = pretrained_path
+        self.save_ckpt_iter = save_ckpt_iter
+        self.delete_ds_optim_states = delete_ds_optim_states
+
+        self.deepspeed = EasyDict(deepspeed or {
+            "enable": True,
+            "stage": 1
+        })
+    def set_num_frames(self, num_frames):
+        # print('Here ', num_frames)
+        self.num_frames = num_frames
+        self.inputs.video_input.num_frames = num_frames
+        self.model.vision_encoder.num_frames = num_frames

demo.py

@@ -0,0 +1,143 @@
+import os
+import random
+import io
+import av
+import cv2
+import decord
+import imageio
+from decord import VideoReader
+import torch
+import numpy as np
+import math
+import torch.nn.functional as F
+decord.bridge.set_bridge("torch")
+
+from transformers import AutoConfig, AutoModel
+config = AutoConfig.from_pretrained("/fs-computility/video/heyinan/iv2hf/", trust_remote_code=True)
+model = AutoModel.from_pretrained("/fs-computility/video/heyinan/iv2hf/", trust_remote_code=True).to(config.device)
+
+
+def get_frame_indices(num_frames, vlen, sample='rand', fix_start=None, input_fps=1, max_num_frames=-1, start=None, end=None):
+    start_frame, end_frame = 0, vlen
+    if start is not None:
+        start_frame = max(start_frame,int(start * input_fps))
+    if end is not None:
+        end_frame = min(end_frame,int(end * input_fps))
+
+    # Ensure start_frame is less than end_frame
+    if start_frame >= end_frame:
+        raise ValueError("Start frame index must be less than end frame index")
+
+    # Calculate the length of the clip in frames
+    clip_length = end_frame - start_frame
+
+    if sample in ["rand", "middle"]:  # uniform sampling
+        acc_samples = min(num_frames, clip_length)
+        # split the clip into `acc_samples` intervals, and sample from each interval.
+        intervals = np.linspace(start=start_frame, stop=end_frame, num=acc_samples + 1).astype(int)
+        ranges = []
+        for idx, interv in enumerate(intervals[:-1]):
+            ranges.append((interv, intervals[idx + 1] - 1))
+        if sample == 'rand':
+            try:
+                frame_indices = [random.choice(range(x[0], x[1] + 1)) for x in ranges]
+            except:
+                frame_indices = np.random.permutation(clip_length)[:acc_samples] + start_frame
+                frame_indices.sort()
+                frame_indices = list(frame_indices)
+        elif fix_start is not None:
+            frame_indices = [x[0] + fix_start for x in ranges]
+        elif sample == 'middle':
+            frame_indices = [(x[0] + x[1]) // 2 for x in ranges]
+        else:
+            raise NotImplementedError
+
+        if len(frame_indices) < num_frames:  # padded with last frame
+            padded_frame_indices = [frame_indices[-1]] * num_frames
+            padded_frame_indices[:len(frame_indices)] = frame_indices
+            frame_indices = padded_frame_indices
+    elif "fps" in sample:  # fps0.5, sequentially sample frames at 0.5 fps
+        output_fps = float(sample[3:])
+        duration = float(clip_length) / input_fps
+        delta = 1 / output_fps  # gap between frames, this is also the clip length each frame represents
+        frame_seconds = np.arange(0 + delta / 2, duration + delta / 2, delta)
+        frame_indices = np.around(frame_seconds * input_fps).astype(int) + start_frame
+        frame_indices = [e for e in frame_indices if e < end_frame]
+        if max_num_frames > 0 and len(frame_indices) > max_num_frames:
+            frame_indices = frame_indices[:max_num_frames]
+            # frame_indices = np.linspace(0 + delta / 2, duration + delta / 2, endpoint=False, num=max_num_frames)
+    else:
+        raise ValueError
+    return frame_indices
+
+def read_frames_decord(
+        video_path, num_frames, sample='middle', fix_start=None, 
+        max_num_frames=-1, client=None, trimmed30=False, start=None, end=None
+    ):
+    num_threads = 1 if video_path.endswith('.webm') else 0 # make ssv2 happy
+
+    video_reader = VideoReader(video_path, num_threads=num_threads)
+    vlen = len(video_reader)
+ 
+    fps = video_reader.get_avg_fps()
+    duration = vlen / float(fps)
+
+    frame_indices = get_frame_indices(
+        num_frames, vlen, sample=sample, fix_start=fix_start,
+        input_fps=fps, max_num_frames=max_num_frames, start=start, end=end
+    )
+
+    frames = video_reader.get_batch(frame_indices)  # (T, H, W, C), torch.uint8
+    frames = frames.permute(0, 3, 1, 2)  # (T, C, H, W), torch.uint8
+    return frames, frame_indices, duration
+
+def get_text_feature(model, texts):
+    text_input = model.tokenizer(texts).to(model.device)
+    text_features = model.encode_text(text_input)
+    return text_features
+    
+def get_similarity(video_feature, text_feature):
+    video_feature = F.normalize(video_feature, dim=-1)
+    text_feature = F.normalize(text_feature, dim=-1)
+    sim_matrix = text_feature @ video_feature.T
+    return sim_matrix
+
+def get_top_videos(model, text_features, video_features, video_paths, texts):
+    # text_features = get_text_feature(texts)
+
+    video_features = F.normalize(video_features, dim=-1)
+    text_features = F.normalize(text_features, dim=-1)
+
+    # print(text_features.shape, video_features.shape)
+    sim_matrix = text_features @ video_features.T
+    # print(sim_matrix.shape)
+
+    top_k = 5
+    sim_matrix_top_k = torch.topk(sim_matrix, top_k, dim=1)[1]
+    softmax_sim_matrix = F.softmax(sim_matrix, dim=1)
+
+    retrieval_infos = {}
+    for i in range(len(sim_matrix_top_k)):
+        print("\n",texts[i])
+        retrieval_infos[texts[i]] = []
+        for j in range(top_k):
+            print("top", j+1, ":", video_paths[sim_matrix_top_k[i][j]], "~prob:", sim_matrix[i][sim_matrix_top_k[i][j]].item())
+            retrieval_infos[texts[i]].append({"video":  video_paths[sim_matrix_top_k[i][j]], "prob": sim_matrix[i][sim_matrix_top_k[i][j]].item(), "rank": j+1})
+    return retrieval_infos
+
+if __name__=="__main__":
+    video_features = []
+    demo_videos = ["video1.mp4","video2.mp4"]
+    texts = ['a person talking', 'a logo', 'a building']
+    for video_path in demo_videos:
+        frames, frame_indices, video_duration = read_frames_decord(video_path,8)
+        frames = model.transform(frames).unsqueeze(0).to(model.device)
+        with torch.no_grad():
+            video_feature = model.encode_vision(frames, test=True)
+            video_features.append(video_feature)
+    
+    text_features = get_text_feature(model, texts)
+    video_features = torch.cat(video_features, dim=0).to(text_features.dtype).to(config.device)
+    results = get_top_videos(model, text_features, video_features, demo_videos, texts)
+
+

flash_attention_class.py

@@ -0,0 +1,74 @@
+import torch
+import torch.nn as nn
+
+from einops import rearrange
+
+from flash_attn.flash_attn_interface import flash_attn_varlen_qkvpacked_func
+from flash_attn.bert_padding import unpad_input, pad_input
+
+
+class FlashAttention(nn.Module):
+    """Implement the scaled dot product attention with softmax.
+    Arguments
+    ---------
+        softmax_scale: The temperature to use for the softmax attention.
+                      (default: 1/sqrt(d_keys) where d_keys is computed at
+                      runtime)
+        attention_dropout: The dropout rate to apply to the attention
+                           (default: 0.0)
+    """
+
+    def __init__(self, softmax_scale=None, attention_dropout=0.0, device=None, dtype=None):
+        super().__init__()
+        self.softmax_scale = softmax_scale
+        self.dropout_p = attention_dropout
+
+    def forward(self, qkv, key_padding_mask=None, causal=False, cu_seqlens=None,
+                max_s=None, need_weights=False):
+        """Implements the multihead softmax attention.
+        Arguments
+        ---------
+            qkv: The tensor containing the query, key, and value. (B, S, 3, H, D) if key_padding_mask is None
+                if unpadded: (nnz, 3, h, d)
+            key_padding_mask: a bool tensor of shape (B, S)
+        """
+        
+        # qkv = qkv.to(torch.float16)
+        
+        assert not need_weights
+        assert qkv.dtype in [torch.float16, torch.bfloat16]
+        assert qkv.is_cuda
+
+        if cu_seqlens is None:
+            batch_size = qkv.shape[0]
+            seqlen = qkv.shape[1]
+            if key_padding_mask is None:
+                qkv = rearrange(qkv, 'b s ... -> (b s) ...')
+                max_s = seqlen
+                cu_seqlens = torch.arange(0, (batch_size + 1) * seqlen, step=seqlen, dtype=torch.int32,
+                                          device=qkv.device)
+                output = flash_attn_varlen_qkvpacked_func(
+                    qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
+                    softmax_scale=self.softmax_scale, causal=causal
+                )
+                output = rearrange(output, '(b s) ... -> b s ...', b=batch_size)
+            else:
+                nheads = qkv.shape[-2]
+                x = rearrange(qkv, 'b s three h d -> b s (three h d)')
+                x_unpad, indices, cu_seqlens, max_s = unpad_input(x, key_padding_mask)
+                x_unpad = rearrange(x_unpad, 'nnz (three h d) -> nnz three h d', three=3, h=nheads)
+                output_unpad = flash_attn_varlen_qkvpacked_func(
+                    x_unpad, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
+                    softmax_scale=self.softmax_scale, causal=causal
+                )
+                output = rearrange(pad_input(rearrange(output_unpad, 'nnz h d -> nnz (h d)'),
+                                             indices, batch_size, seqlen),
+                                   'b s (h d) -> b s h d', h=nheads)
+        else:
+            assert max_s is not None
+            output = flash_attn_varlen_qkvpacked_func(
+                qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
+                softmax_scale=self.softmax_scale, causal=causal
+            )
+
+        return output, None

internvideo2.py

@@ -0,0 +1,779 @@
+import math
+import torch
+import torch.nn.functional as F
+from timm.models.layers import DropPath, to_2tuple, trunc_normal_
+from torch import nn
+
+import torch.utils.checkpoint as checkpoint
+from functools import partial
+from einops import rearrange
+
+from .pos_embed import get_3d_sincos_pos_embed, get_2d_sincos_pos_embed, get_1d_sincos_pos_embed, interpolate_pos_embed_internvideo2
+from .flash_attention_class import FlashAttention
+
+from transformers.utils import logging as error_logging
+
+# Set up logging
+error_logging.set_verbosity_error()
+
+try:
+    from flash_attn.modules.mlp import Mlp as FusedMLP
+except:
+    pass
+
+try:
+    from flash_attn.ops.rms_norm import DropoutAddRMSNorm
+except:
+    pass
+
+
+class CrossAttention(nn.Module):
+    def __init__(
+            self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0.,
+            proj_drop=0., attn_head_dim=None, out_dim=None):
+        super().__init__()
+        if out_dim is None:
+            out_dim = dim
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        if attn_head_dim is not None:
+            head_dim = attn_head_dim
+        all_head_dim = head_dim * self.num_heads
+        self.scale = qk_scale or head_dim ** -0.5
+        assert all_head_dim == dim
+        
+        self.q = nn.Linear(dim, all_head_dim, bias=False)
+        self.k = nn.Linear(dim, all_head_dim, bias=False)
+        self.v = nn.Linear(dim, all_head_dim, bias=False)
+        
+        if qkv_bias:
+            self.q_bias = nn.Parameter(torch.zeros(all_head_dim))
+            self.k_bias = nn.Parameter(torch.zeros(all_head_dim))
+            self.v_bias = nn.Parameter(torch.zeros(all_head_dim))
+        else:
+            self.q_bias = None
+            self.k_bias = None
+            self.v_bias = None
+        
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(all_head_dim, out_dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+    
+    def forward(self, x, k=None, v=None):
+        B, N, C = x.shape
+        N_k = k.shape[1]
+        N_v = v.shape[1]
+        
+        q_bias, k_bias, v_bias = None, None, None
+        if self.q_bias is not None:
+            q_bias = self.q_bias
+            k_bias = self.k_bias
+            v_bias = self.v_bias
+        
+        q = F.linear(input=x, weight=self.q.weight, bias=q_bias)
+        q = q.reshape(B, N, 1, self.num_heads, -1).permute(2, 0, 3, 1, 4).squeeze(0)  # (B, N_head, N_q, dim)
+        
+        k = F.linear(input=k, weight=self.k.weight, bias=k_bias)
+        k = k.reshape(B, N_k, 1, self.num_heads, -1).permute(2, 0, 3, 1, 4).squeeze(0)
+        
+        v = F.linear(input=v, weight=self.v.weight, bias=v_bias)
+        v = v.reshape(B, N_v, 1, self.num_heads, -1).permute(2, 0, 3, 1, 4).squeeze(0)
+        
+        q = q * self.scale
+        attn = (q @ k.transpose(-2, -1))  # (B, N_head, N_q, N_k)
+        
+        attn = attn.softmax(dim=-1)
+        attn = self.attn_drop(attn)
+        
+        x = (attn @ v).transpose(1, 2).reshape(B, N, -1)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        
+        return x
+
+
+class AttentiveBlock(nn.Module):
+    
+    def __init__(self, dim, num_heads, qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,
+                 drop_path=0., norm_layer=nn.LayerNorm, attn_head_dim=None, out_dim=None):
+        super().__init__()
+        
+        self.norm1_q = norm_layer(dim)
+        self.norm1_k = norm_layer(dim)
+        self.norm1_v = norm_layer(dim)
+        self.cross_attn = CrossAttention(
+            dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop,
+            proj_drop=drop, attn_head_dim=attn_head_dim, out_dim=out_dim)
+        
+        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+    
+    def forward(self, x_q, x_kv, pos_q, pos_k, bool_masked_pos, rel_pos_bias=None):
+        x_q = self.norm1_q(x_q + pos_q)
+        x_k = self.norm1_k(x_kv + pos_k)
+        x_v = self.norm1_v(x_kv)
+        x = self.cross_attn(x_q, k=x_k, v=x_v)
+        
+        return x
+
+
+class AttentionPoolingBlock(AttentiveBlock):
+    
+    def forward(self, x):
+        # x_q = x.mean(1, keepdim=True)
+        x_q = x
+        x_kv, pos_q, pos_k = x, 0, 0
+        x = super().forward(x_q, x_kv, pos_q, pos_k, bool_masked_pos=None, rel_pos_bias=None)
+        x = x.squeeze(1)
+        return x
+
+
+class RMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+    
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+
+class LayerScale(nn.Module):
+    def __init__(self, dim, init_values=1e-5, inplace=False, force_fp32=False):
+        super().__init__()
+        self.inplace = inplace
+        self.gamma = nn.Parameter(init_values * torch.ones(dim))
+        self.force_fp32 = force_fp32
+    
+    @torch.cuda.amp.autocast(enabled=False)
+    def forward(self, x):
+        if self.force_fp32:
+            output_type = x.dtype
+            out = x.float().mul_(self.gamma.float()) if self.inplace else x.float() * self.gamma.float()
+            return out.to(dtype=output_type)
+        else:
+            out = x.mul_(self.gamma) if self.inplace else x * self.gamma
+            return out
+
+
+class Attention(nn.Module):
+    def __init__(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0., use_flash_attn=False,
+                 causal=False, norm_layer=nn.LayerNorm, qk_normalization=False, use_fused_rmsnorm=False):
+        super().__init__()
+        assert dim % num_heads == 0, 'dim should be divisible by num_heads'
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = head_dim ** -0.5
+        
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+        
+        self.use_flash_attn = use_flash_attn
+        if use_flash_attn:
+            self.causal = causal
+            self.inner_attn = FlashAttention(attention_dropout=attn_drop)
+        
+        self.qk_normalization = qk_normalization
+        self.q_norm = norm_layer(dim) if qk_normalization else nn.Identity()
+        self.k_norm = norm_layer(dim) if qk_normalization else nn.Identity()
+        self.use_fused_rmsnorm = use_fused_rmsnorm
+    
+    def _naive_attn(self, x):
+        B, N, C = x.shape
+        # print(x.shape, torch.cuda.memory_allocated(), torch.cuda.memory_allocated())
+        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
+        q, k, v = qkv.unbind(0)  # make torchscript happy (cannot use tensor as tuple)
+        
+        if self.qk_normalization:
+            B_, H_, N_, D_ = q.shape
+            q = self.q_norm(q.transpose(1, 2).flatten(-2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
+            k = self.k_norm(k.transpose(1, 2).flatten(-2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
+        
+        attn = ((q * self.scale) @ k.transpose(-2, -1))
+        # attn = attn - attn.max(-1)[0].unsqueeze(-1)  # in case of overflow for fp16
+        attn = attn.softmax(dim=-1)
+        attn = self.attn_drop(attn)
+        # print(torch.cuda.memory_allocated(), torch.cuda.memory_allocated())
+        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+    
+    def _flash_attn(self, x, key_padding_mask=None, need_weights=False):
+        
+        qkv = self.qkv(x)
+        qkv = rearrange(qkv, "b s (three h d) -> b s three h d", three=3, h=self.num_heads)
+        
+        if self.qk_normalization:
+            q, k, v = qkv.unbind(2)
+            if self.use_fused_rmsnorm:
+                q = self.q_norm(q.flatten(-2, -1))[0].view(q.shape)
+                k = self.k_norm(k.flatten(-2, -1))[0].view(k.shape)
+            else:
+                q = self.q_norm(q.flatten(-2, -1)).view(q.shape)
+                k = self.k_norm(k.flatten(-2, -1)).view(k.shape)
+            qkv = torch.stack([q, k, v], dim=2)
+        
+        context, _ = self.inner_attn(
+            qkv, key_padding_mask=key_padding_mask, need_weights=need_weights, causal=self.causal
+        )
+        outs = self.proj(rearrange(context, "b s h d -> b s (h d)"))
+        outs = self.proj_drop(outs)
+        return outs
+    
+    def forward(self, x):
+        x = self._naive_attn(x) if not self.use_flash_attn else self._flash_attn(x)
+        return x
+
+
+class Mlp(nn.Module):
+    """ MLP as used in Vision Transformer, MLP-Mixer and related networks
+    """
+    
+    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU,
+                 bias=True, drop=0.):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        bias = to_2tuple(bias)
+        drop_probs = to_2tuple(drop)
+        
+        self.fc1 = nn.Linear(in_features, hidden_features, bias=bias[0])
+        self.act = act_layer()
+        self.drop1 = nn.Dropout(drop_probs[0])
+        self.fc2 = nn.Linear(hidden_features, out_features, bias=bias[1])
+        self.drop2 = nn.Dropout(drop_probs[1])
+    
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop1(x)
+        x = self.fc2(x)
+        x = self.drop2(x)
+        return x
+
+
+class Block(nn.Module):
+    
+    def __init__(
+            self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., init_values=None,
+            drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, use_flash_attn=False, use_fused_mlp=False,
+            fused_mlp_heuristic=1, with_cp=False, qk_normalization=False, layerscale_no_force_fp32=False,
+            use_fused_rmsnorm=False):
+        super().__init__()
+        
+        self.norm1 = norm_layer(dim)
+        self.attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop,
+                              use_flash_attn=use_flash_attn, causal=False, norm_layer=norm_layer,
+                              qk_normalization=qk_normalization,
+                              use_fused_rmsnorm=use_fused_rmsnorm)
+        self.ls1 = LayerScale(dim, init_values=init_values,
+                              force_fp32=(not layerscale_no_force_fp32)) if init_values else nn.Identity()
+        # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here
+        self.drop_path1 = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+        
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        if use_fused_mlp:
+            # self.mlp = FusedMLP(in_features=dim, hidden_features=mlp_hidden_dim, heuristic=fused_mlp_heuristic)
+            self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
+        else:
+            self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
+        self.ls2 = LayerScale(dim, init_values=init_values,
+                              force_fp32=(not layerscale_no_force_fp32)) if init_values else nn.Identity()
+        self.drop_path2 = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+        
+        self.with_cp = with_cp
+        self.use_fused_rmsnorm = use_fused_rmsnorm
+    
+    def forward(self, x, residual=None):
+        
+        def _inner_forward(x, residual=None):
+            if self.use_fused_rmsnorm:
+                x, residual = self.norm1(x, residual)
+                x = self.drop_path1(self.ls1(self.attn(x)))
+                x, residual = self.norm2(x, residual)
+                x = self.drop_path2(self.ls2(self.mlp(x)))
+                return x, residual
+            else:
+                assert residual is None
+                x = x + self.drop_path1(self.ls1(self.attn(self.norm1(x))))
+                x = x + self.drop_path2(self.ls2(self.mlp(self.norm2(x))))
+                return x
+        
+        if self.with_cp:
+            # print(f"\033[31m use_checkpoint [0m")
+            return checkpoint.checkpoint(_inner_forward, x, residual)
+        else:
+            return _inner_forward(x, residual=residual)
+
+
+class PatchEmbed(nn.Module):
+    """ 3D Image to Patch Embedding
+    """
+    
+    def __init__(
+            self, img_size=224, patch_size=16, in_chans=3, embed_dim=768, 
+            num_frames=8, tubelet_size=1, norm_layer=None
+        ):
+        super().__init__()
+        img_size = to_2tuple(img_size)
+        patch_size = to_2tuple(patch_size)
+        self.img_size = img_size
+        self.patch_size = patch_size
+        self.grid_size = (
+            num_frames // tubelet_size, 
+            img_size[0] // patch_size[0], 
+            img_size[1] // patch_size[1]
+        ) # (T, H, W)
+        self.num_patches = self.grid_size[0] * self.grid_size[1] * self.grid_size[2]
+        self.num_img_patches = self.grid_size[1] * self.grid_size[2]
+
+        self.proj = nn.Conv3d(
+            in_channels=in_chans, out_channels=embed_dim, 
+            kernel_size=(tubelet_size, patch_size[0], patch_size[1]), 
+            stride=(tubelet_size, patch_size[0], patch_size[1])
+        )
+        self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
+    
+    def forward(self, x):
+        x = self.proj(x)
+        x = x.flatten(3).permute(0, 2, 3, 1)  # B x C x T x HW => B x T x HW x C
+        x = self.norm(x)
+        return x
+
+
+class Linear_Decoder(nn.Module):
+    def __init__(self, in_channels=1408, out_channels=3200, 
+                 norm_layer=nn.LayerNorm, clip_norm_type='l2'):
+        super().__init__()
+        self.clip_norm_type = clip_norm_type
+        # logger.info(f'Normalization Type: {clip_norm_type}')
+
+        self.head = nn.Linear(in_channels, out_channels)
+        self.norm =  norm_layer(out_channels)
+
+        self.apply(self._init_weights)
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            nn.init.xavier_uniform_(m.weight)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)
+
+    def forward(self, x):
+        x = self.norm(self.head(x))
+
+        if self.clip_norm_type == 'l2':
+            x = x / x.norm(dim=-1, keepdim=True)
+        elif self.clip_norm_type == 'none':
+            pass
+        else:
+            raise NotImplementedError
+
+        return x
+
+
+class PretrainInternVideo2(nn.Module):
+    def __init__(
+            self,
+            in_chans: int = 3,
+            patch_size: int = 14,
+            img_size: int = 224,
+            qkv_bias: bool = False,
+            drop_path_rate: float = 0.25,
+            embed_dim: int = 1408,
+            num_heads: int = 16,
+            mlp_ratio: float = 48/11,
+            init_values: float = 1e-5,
+            qk_normalization: bool = True,
+            depth: int = 40,
+            use_flash_attn: bool = False,
+            use_fused_rmsnorm: bool = False,
+            use_fused_mlp: bool = False,
+            fused_mlp_heuristic: int = 1,
+            attn_pool_num_heads: int = 16,
+            clip_embed_dim: int = 768,
+            layerscale_no_force_fp32: bool = False,
+            num_frames: int = 8,
+            tubelet_size: int = 1,
+            sep_pos_embed: bool = False,
+            sep_image_video_pos_embed: bool = False,
+            use_checkpoint: bool = False,
+            checkpoint_num: int = 0,
+            # for unmasked teacher
+            clip_teacher_embed_dim: int = 3200,
+            clip_teacher_final_dim: int = 768, # if 0, not distill final features
+            clip_norm_type: str = 'l2',
+            clip_return_layer: int = 1,
+            clip_student_return_interval: int = 1,
+        ):
+        super().__init__()
+        
+        self.num_frames = num_frames
+        # print(f'num_frames: {num_frames}')
+        self.tubelet_size = tubelet_size
+        assert use_flash_attn == use_fused_rmsnorm == use_fused_mlp, 'use_flash_attn, use_fused_rmsnorm and use_fused_mlp should be consistent'
+        
+        self.use_flash_attn = use_flash_attn
+        self.embed_dim = embed_dim
+
+        self.depth = depth
+        self.clip_norm_type = clip_norm_type
+        self.return_index = []
+        for i in range(clip_return_layer):
+            self.return_index.append(depth - int(i * clip_student_return_interval) - 1)
+        # logger.info(f'Normalization Type: {clip_norm_type}')
+        # logger.info(f'Strudent Return Index: {self.return_index}')
+        
+        if use_fused_rmsnorm:
+            norm_layer_for_blocks = partial(DropoutAddRMSNorm, eps=1e-6, prenorm=True)
+        else:
+            norm_layer_for_blocks = partial(RMSNorm, eps=1e-6)
+        self.norm_layer_for_blocks = norm_layer_for_blocks
+        self.patch_embed = PatchEmbed(
+            img_size, patch_size, in_chans, embed_dim,
+            num_frames=num_frames, tubelet_size=tubelet_size,
+        )
+        num_patches = self.patch_embed.num_patches
+        num_img_patches = self.patch_embed.num_img_patches
+
+        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
+        
+        # stolen from https://github.com/facebookresearch/mae_st/blob/dc072aaaf640d06892e23a33b42223a994efe272/models_vit.py#L65-L73C17
+        self.sep_pos_embed = sep_pos_embed
+        self.sep_image_video_pos_embed = sep_image_video_pos_embed
+        if sep_pos_embed:
+            raise NotImplementedError
+        else:
+            if sep_image_video_pos_embed:
+                # logger.info("Use joint position embedding, for image and video we use different pos_embed.")
+                self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))
+                self.img_pos_embed = nn.Parameter(torch.zeros(1, num_img_patches + 1, embed_dim))
+                # for CLIP decoder
+                self.clip_pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))
+                self.clip_img_pos_embed = nn.Parameter(torch.zeros(1, num_img_patches + 1, embed_dim))
+            else:
+                # logger.info("Use joint position embedding, for image and video we use same pos_embed.")
+                self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))
+                self.clip_pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))
+        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]
+        # choose which layer to use checkpoint
+        with_cp_list = [False] * depth
+        if use_checkpoint:
+            for idx in range(depth):
+                if idx < checkpoint_num:
+                    with_cp_list[idx] = True
+        # logger.info(f"Droppath rate: {dpr}")
+        # logger.info(f"Checkpoint list: {with_cp_list}")
+        
+        self.blocks = nn.ModuleList([
+            Block(embed_dim, num_heads, mlp_ratio, qkv_bias=qkv_bias,
+                  norm_layer=norm_layer_for_blocks,
+                  drop_path=dpr[i], init_values=init_values, attn_drop=0.,
+                  use_flash_attn=use_flash_attn, use_fused_mlp=use_fused_mlp,
+                  fused_mlp_heuristic=fused_mlp_heuristic,
+                  with_cp=with_cp_list[i],
+                  qk_normalization=qk_normalization,
+                  layerscale_no_force_fp32=layerscale_no_force_fp32,
+                  use_fused_rmsnorm=use_fused_rmsnorm)
+            for i in range(depth)])
+        self.clip_projector = AttentionPoolingBlock(
+            dim=embed_dim, num_heads=attn_pool_num_heads, qkv_bias=True, qk_scale=None,
+            drop=0., attn_drop=0., norm_layer=partial(nn.LayerNorm, eps=1e-5), out_dim=clip_embed_dim)
+        
+        # CLIP decoder
+        self.clip_decoder = nn.ModuleList([
+            Linear_Decoder(
+                in_channels=embed_dim, 
+                out_channels=clip_teacher_embed_dim, 
+                norm_layer=partial(nn.LayerNorm, eps=1e-5), 
+                clip_norm_type=clip_norm_type
+            ) for _ in range(clip_return_layer)
+        ])
+        self.final_clip_decoder = nn.Identity()
+        if clip_teacher_final_dim > 0:
+            self.final_clip_decoder = Linear_Decoder(
+                in_channels=clip_embed_dim, 
+                out_channels=clip_teacher_final_dim, 
+                norm_layer=partial(nn.LayerNorm, eps=1e-5), 
+                clip_norm_type=clip_norm_type
+            )
+        
+        self.init_pos_embed()
+        trunc_normal_(self.cls_token, std=.02)
+        self.apply(self._init_weights)
+        self.fix_init_weight()
+
+    def init_pos_embed(self):
+        # logger.info("Init pos_embed from sincos pos_embed")
+        if self.sep_pos_embed:
+            raise NotImplementedError
+        else:
+            # trunc_normal_(self.pos_embed, std=.02)
+            # trunc_normal_(self.clip_pos_embed, std=.02)
+            pos_embed = get_3d_sincos_pos_embed(
+                self.pos_embed.shape[-1], 
+                self.patch_embed.grid_size[1], # height & weight
+                self.patch_embed.grid_size[0], # t_size
+                cls_token=True
+            )
+            self.pos_embed.data.copy_(torch.from_numpy(pos_embed).float().unsqueeze(0))
+            self.clip_pos_embed.data.copy_(torch.from_numpy(pos_embed).float().unsqueeze(0))
+            
+            if self.sep_image_video_pos_embed:
+                img_pos_embed = get_3d_sincos_pos_embed(
+                    self.pos_embed.shape[-1], 
+                    self.patch_embed.grid_size[1], # height & weight
+                    1,
+                    cls_token=True
+                )
+                self.img_pos_embed.data.copy_(torch.from_numpy(img_pos_embed).float().unsqueeze(0))
+                self.clip_img_pos_embed.data.copy_(torch.from_numpy(img_pos_embed).float().unsqueeze(0))
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)
+
+    def fix_init_weight(self):
+        def rescale(param, layer_id):
+            param.div_(math.sqrt(2.0 * layer_id))
+
+        for layer_id, layer in enumerate(self.blocks):
+            rescale(layer.attn.proj.weight.data, layer_id + 1)
+            rescale(layer.mlp.fc2.weight.data, layer_id + 1)
+    
+    @property
+    def dtype(self):
+        return self.patch_embed.proj.weight.dtype
+
+    def get_num_layers(self):
+        return len(self.blocks)
+
+    @torch.jit.ignore
+    def no_weight_decay(self):
+        return {
+            'pos_embed', 
+            'pos_embed_spatial', 
+            'pos_embed_temporal', 
+            'pos_embed_cls',
+            'img_pos_embed',
+            'cls_token',
+            'clip_pos_embed', 
+            'clip_pos_embed_spatial', 
+            'clip_pos_embed_temporal', 
+            'clip_pos_embed_cls',
+            'clip_img_pos_embed'
+        }
+    
+    # @torch.cuda.amp.autocast(enabled=False)
+    def forward(self, x, mask=None, use_image=False, x_vis_return_idx=-1, x_vis_only=False):
+        # print(0, x.shape)
+        x = self.patch_embed(x.type(self.dtype))
+        # print(f"x.shape: {x.shape} x.dtype: {x.dtype}, model.dtype: {self.dtype}")
+        B, T, L, C = x.shape  # T: temporal; L: spatial
+        x = x.view([B, T * L, C])   # (B, T * L, C)
+
+        # append cls token
+        cls_tokens = self.cls_token.expand(B, -1, -1)
+        x = torch.cat((cls_tokens, x), dim=1)   # (B, T * L + 1, C)
+        # print(1, x.shape)
+
+        # add pos_embed
+        if self.sep_pos_embed:
+            raise NotImplementedError
+        else:
+            if use_image:
+                # print('use image')  # No.
+                if self.sep_image_video_pos_embed:
+                    pos_embed = self.img_pos_embed
+                else:
+                    # (1, num_img_patches + 1, embed_dim)
+                    # print('origin pos_embed.shape:', self.pos_embed.shape)
+                    cls_pos_embed = self.pos_embed[:, 0:1, :]
+                    # print('cls_pos_embed.shape:', cls_pos_embed.shape)
+
+                    img_pos_embed = self.pos_embed[:, 1:, :].view(1, self.num_frames, self.patch_embed.num_patches // self.num_frames, self.embed_dim).mean(dim=1)
+                    # print('img_pos_embed.shape:', img_pos_embed.shape)
+
+                    pos_embed = torch.cat([cls_pos_embed, img_pos_embed], dim=1)
+                    # print('final img_pos_embed.shape:', pos_embed.shape)
+            else:
+                pos_embed = self.pos_embed
+        pos_embed = pos_embed[:, :x.shape[1], :]
+        x = x + pos_embed
+
+        # mask tokens, ~mask means visible
+        if mask is not None:
+            x = x[~mask].reshape(B, -1, C) 
+        else:
+            x = x.reshape(B, -1, C) 
+        residual = None
+        x_clip = []
+        for idx, blk in enumerate(self.blocks):
+            if isinstance(x, tuple) and len(x) == 2:
+                x, residual = x
+            # print(f"\033[31m这是{idx}, {x.shape}\033[0m")
+            x = blk(x, residual=residual)
+            # return intermediate features
+            if idx in self.return_index:
+                if isinstance(x, tuple) and len(x) == 2:
+                    tmp_x, tmp_residual = x
+                    if residual is not None:
+                        x_clip.append(tmp_x + tmp_residual)
+                else:
+                    x_clip.append(x)
+            if idx == (self.depth + x_vis_return_idx):
+                # print(f'idx = {idx} len(self.blocks)={len(self.blocks)}')
+                break
+        
+        if isinstance(x, tuple) and len(x) == 2:
+            x, residual = x
+            if residual is not None:
+                x = x + residual
+        
+        x_vis = x
+        # print(f'x_vis.shape:{x_vis.shape}')
+        if x_vis_only:
+            return x_vis
+        
+        x_pool_vis = self.clip_projector(x_vis) 
+        x_align = self.final_clip_decoder(x_pool_vis)
+        # print(3, x_pool_vis.shape)
+        # print(4, x_align.shape)
+
+        # align CLIP
+        x_clip = torch.stack(x_clip)
+        K, B, _, C_CLIP = x_clip.shape
+        # print(5, x_clip.shape)
+        # add pos_embed
+        if self.sep_pos_embed: 
+            raise NotImplementedError
+        else:
+            if use_image:
+                if self.sep_image_video_pos_embed:
+                    clip_pos_embed = self.clip_img_pos_embed
+                else:
+                    # (1, num_img_patches + 1, embed_dim)
+                    # print('origin pos_embed.shape:', self.pos_embed.shape)
+                    clip_cls_pos_embed = self.clip_pos_embed[:, 0:1, :]
+                    # print('cls_pos_embed.shape:', cls_pos_embed.shape)
+
+                    clip_img_pos_embed = self.clip_pos_embed[:, 1:, :].view(1, self.num_frames, self.patch_embed.num_patches // self.num_frames, self.embed_dim).mean(dim=1)
+                    # print('img_pos_embed.shape:', img_pos_embed.shape)
+
+                    clip_pos_embed = torch.cat([clip_cls_pos_embed, clip_img_pos_embed], dim=1)
+                    # print('final img_pos_embed.shape:', pos_embed.shape)
+
+            else:
+                clip_pos_embed = self.clip_pos_embed
+        
+        clip_pos_embed = clip_pos_embed.repeat(B, 1, 1)
+        if mask is not None:
+            x_clip = x_clip + clip_pos_embed[~mask].view(B, -1, C_CLIP).unsqueeze(0).repeat(K, 1, 1, 1)
+        else:
+            clip_pos_embed = clip_pos_embed.unsqueeze(0).repeat(K, 1, 1, 1)
+            clip_pos_embed = clip_pos_embed[:, :, :x_clip.shape[2], :]
+            x_clip = x_clip + clip_pos_embed
+        
+        # CLIP decoder
+        x_clip_align = []
+        for idx, clip_decoder in enumerate(self.clip_decoder):
+            x_clip_align.append(clip_decoder(x_clip[idx]))
+        x_clip_align = torch.stack(x_clip_align)
+        
+        # print(f'x_vis.shape:{x_vis.shape}, x_pool_vis.shape:{x_pool_vis.shape}')
+        return x_vis, x_pool_vis, x_clip_align, x_align
+    
+
+def pretrain_internvideo2_1b_patch14_224(config):
+    # print(config.vision_encoder.num_frames)
+    model = PretrainInternVideo2(
+        in_chans=3, img_size=224, patch_size=14,
+        embed_dim=1408, depth=40, num_heads=16, mlp_ratio=48/11,
+        clip_embed_dim=config.vision_encoder.clip_embed_dim,
+        attn_pool_num_heads=16, qkv_bias=False,
+        drop_path_rate=0.25,
+        init_values=0.00001,
+        qk_normalization=True,
+        use_flash_attn=config.vision_encoder.get('use_flash_attn', True),
+        use_fused_rmsnorm=config.vision_encoder.get('use_fused_rmsnorm', True),
+        use_fused_mlp=config.vision_encoder.get('use_fused_mlp', True),
+        fused_mlp_heuristic=1,
+        layerscale_no_force_fp32=False,
+        num_frames=config.vision_encoder.num_frames,
+        tubelet_size=config.vision_encoder.tubelet_size,
+        sep_pos_embed=False,
+        sep_image_video_pos_embed=config.vision_encoder.sep_image_video_pos_embed,
+        use_checkpoint=config.vision_encoder.use_checkpoint,
+        checkpoint_num=config.vision_encoder.checkpoint_num,
+        clip_teacher_embed_dim=config.vision_encoder.clip_teacher_embed_dim,
+        clip_teacher_final_dim=config.vision_encoder.clip_teacher_final_dim,
+        clip_norm_type=config.vision_encoder.clip_norm_type,
+        clip_return_layer=config.vision_encoder.clip_return_layer,
+        clip_student_return_interval=config.vision_encoder.clip_student_return_interval,
+    )
+
+    if config.vision_encoder.pretrained is not None:
+        # logger.info(f"Loading pretrained weights from {config.vision_encoder.pretrained}")
+        state_dict = torch.load(config.vision_encoder.pretrained, map_location='cpu')
+        interpolate_pos_embed_internvideo2(state_dict, model, orig_t_size=8)
+        message = model.load_state_dict(state_dict, strict=False)
+        # logger.info(message)
+    else:
+        pass
+        # logger.info("No pretrained weights!!!")
+    return model
+
+
+
+def pretrain_internvideo2_6b_patch14_224(config):
+    model = PretrainInternVideo2(
+        in_chans=3, img_size=224, patch_size=14,
+        embed_dim=3200, depth=48, num_heads=25, mlp_ratio=4,
+        clip_embed_dim=config.vision_encoder.clip_embed_dim,
+        attn_pool_num_heads=16, qkv_bias=False,
+        drop_path_rate=0.3,
+        init_values=0.00001,
+        qk_normalization=True,
+        use_flash_attn=config.vision_encoder.get('use_flash_attn', True),
+        use_fused_rmsnorm=config.vision_encoder.get('use_fused_rmsnorm', True),
+        use_fused_mlp=config.vision_encoder.get('use_fused_mlp', True),
+        fused_mlp_heuristic=1,
+        layerscale_no_force_fp32=False,
+        num_frames=config.vision_encoder.num_frames,
+        tubelet_size=config.vision_encoder.tubelet_size,
+        sep_pos_embed=False,
+        sep_image_video_pos_embed=config.vision_encoder.sep_image_video_pos_embed,
+        use_checkpoint=config.vision_encoder.use_checkpoint,
+        checkpoint_num=config.vision_encoder.checkpoint_num,
+        clip_teacher_embed_dim=config.vision_encoder.clip_teacher_embed_dim,
+        clip_teacher_final_dim=config.vision_encoder.clip_teacher_final_dim,
+        clip_norm_type=config.vision_encoder.clip_norm_type,
+        clip_return_layer=config.vision_encoder.clip_return_layer,
+        clip_student_return_interval=config.vision_encoder.clip_student_return_interval,
+    )
+
+    if config.vision_encoder.pretrained is not None:
+        # logger.info(f"Loading pretrained weights from {config.vision_encoder.pretrained}")
+        state_dict = torch.load(config.vision_encoder.pretrained, map_location='cpu')
+        interpolate_pos_embed_internvideo2(state_dict, model, orig_t_size=8)
+        msg = model.load_state_dict(state_dict, strict=False)
+        # logger.info(msg)
+    else:
+        pass
+        # logger.info("No pretrained weights!!!")
+    return model

internvideo2_clip_vision.py

@@ -0,0 +1,553 @@
+import logging
+import math
+import torch
+import torch.nn.functional as F
+from timm.models.layers import DropPath, to_2tuple, trunc_normal_
+from timm.models.registry import register_model
+from torch import nn
+
+import torch.utils.checkpoint as checkpoint
+from functools import partial
+from einops import rearrange
+
+from .pos_embed import get_3d_sincos_pos_embed, get_2d_sincos_pos_embed, get_1d_sincos_pos_embed
+from .flash_attention_class import FlashAttention
+from flash_attn.modules.mlp import FusedMLP
+try:
+    from flash_attn.ops.rms_norm import DropoutAddRMSNorm
+except:
+    pass
+
+from transformers.utils import logging
+import warnings
+warnings.filterwarnings("ignore")
+logger = logging.get_logger(__name__)
+
+
+class CrossAttention(nn.Module):
+    def __init__(
+            self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0.,
+            proj_drop=0., attn_head_dim=None, out_dim=None):
+        super().__init__()
+        if out_dim is None:
+            out_dim = dim
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        if attn_head_dim is not None:
+            head_dim = attn_head_dim
+        all_head_dim = head_dim * self.num_heads
+        self.scale = qk_scale or head_dim ** -0.5
+        assert all_head_dim == dim
+        
+        self.q = nn.Linear(dim, all_head_dim, bias=False)
+        self.k = nn.Linear(dim, all_head_dim, bias=False)
+        self.v = nn.Linear(dim, all_head_dim, bias=False)
+        
+        if qkv_bias:
+            self.q_bias = nn.Parameter(torch.zeros(all_head_dim))
+            self.k_bias = nn.Parameter(torch.zeros(all_head_dim))
+            self.v_bias = nn.Parameter(torch.zeros(all_head_dim))
+        else:
+            self.q_bias = None
+            self.k_bias = None
+            self.v_bias = None
+        
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(all_head_dim, out_dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+    
+    def forward(self, x, k=None, v=None):
+        B, N, C = x.shape
+        N_k = k.shape[1]
+        N_v = v.shape[1]
+        
+        q_bias, k_bias, v_bias = None, None, None
+        if self.q_bias is not None:
+            q_bias = self.q_bias
+            k_bias = self.k_bias
+            v_bias = self.v_bias
+        
+        q = F.linear(input=x, weight=self.q.weight, bias=q_bias)
+        q = q.reshape(B, N, 1, self.num_heads, -1).permute(2, 0, 3, 1, 4).squeeze(0)  # (B, N_head, N_q, dim)
+        
+        k = F.linear(input=k, weight=self.k.weight, bias=k_bias)
+        k = k.reshape(B, N_k, 1, self.num_heads, -1).permute(2, 0, 3, 1, 4).squeeze(0)
+        
+        v = F.linear(input=v, weight=self.v.weight, bias=v_bias)
+        v = v.reshape(B, N_v, 1, self.num_heads, -1).permute(2, 0, 3, 1, 4).squeeze(0)
+        
+        q = q * self.scale
+        attn = (q @ k.transpose(-2, -1))  # (B, N_head, N_q, N_k)
+        
+        attn = attn.softmax(dim=-1)
+        attn = self.attn_drop(attn)
+        
+        x = (attn @ v).transpose(1, 2).reshape(B, N, -1)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        
+        return x
+
+
+class AttentiveBlock(nn.Module):
+    
+    def __init__(self, dim, num_heads, qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,
+                 drop_path=0., norm_layer=nn.LayerNorm, attn_head_dim=None, out_dim=None):
+        super().__init__()
+        
+        self.norm1_q = norm_layer(dim)
+        self.norm1_k = norm_layer(dim)
+        self.norm1_v = norm_layer(dim)
+        self.cross_attn = CrossAttention(
+            dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop,
+            proj_drop=drop, attn_head_dim=attn_head_dim, out_dim=out_dim)
+        
+        if drop_path > 0.:
+            logger.info(f"Use DropPath in projector: {drop_path}")
+        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+    
+    def forward(self, x_q, x_kv, pos_q, pos_k, bool_masked_pos, rel_pos_bias=None):
+        x_q = self.norm1_q(x_q + pos_q)
+        x_k = self.norm1_k(x_kv + pos_k)
+        x_v = self.norm1_v(x_kv)
+        x = self.cross_attn(x_q, k=x_k, v=x_v)
+        
+        return x
+
+
+class AttentionPoolingBlock(AttentiveBlock):
+    
+    def forward(self, x):
+        x_q = x.mean(1, keepdim=True)
+        x_kv, pos_q, pos_k = x, 0, 0
+        x = super().forward(x_q, x_kv, pos_q, pos_k, bool_masked_pos=None, rel_pos_bias=None)
+        x = x.squeeze(1)
+        return x
+
+
+class RMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+    
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+
+class LayerScale(nn.Module):
+    def __init__(self, dim, init_values=1e-5, inplace=False, force_fp32=False):
+        super().__init__()
+        self.inplace = inplace
+        self.gamma = nn.Parameter(init_values * torch.ones(dim))
+        self.force_fp32 = force_fp32
+    
+    @torch.cuda.amp.autocast(enabled=False)
+    def forward(self, x):
+        if self.force_fp32:
+            output_type = x.dtype
+            out = x.float().mul_(self.gamma.float()) if self.inplace else x.float() * self.gamma.float()
+            return out.to(dtype=output_type)
+        else:
+            out = x.mul_(self.gamma) if self.inplace else x * self.gamma
+            return out
+
+
+class Attention(nn.Module):
+    def __init__(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0., use_flash_attn=False,
+                 causal=False, norm_layer=nn.LayerNorm, qk_normalization=False, use_fused_rmsnorm=False):
+        super().__init__()
+        assert dim % num_heads == 0, 'dim should be divisible by num_heads'
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = head_dim ** -0.5
+        
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+        
+        self.use_flash_attn = use_flash_attn
+        if use_flash_attn:
+            self.causal = causal
+            self.inner_attn = FlashAttention(attention_dropout=attn_drop)
+        
+        self.qk_normalization = qk_normalization
+        self.q_norm = norm_layer(dim) if qk_normalization else nn.Identity()
+        self.k_norm = norm_layer(dim) if qk_normalization else nn.Identity()
+        self.use_fused_rmsnorm = use_fused_rmsnorm
+    
+    def _naive_attn(self, x):
+        B, N, C = x.shape
+        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
+        q, k, v = qkv.unbind(0)  # make torchscript happy (cannot use tensor as tuple)
+        
+        if self.qk_normalization:
+            B_, H_, N_, D_ = q.shape
+            q = self.q_norm(q.transpose(1, 2).flatten(-2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
+            k = self.k_norm(k.transpose(1, 2).flatten(-2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
+        
+        attn = ((q * self.scale) @ k.transpose(-2, -1))
+        # attn = attn - attn.max(-1)[0].unsqueeze(-1)  # in case of overflow for fp16
+        attn = attn.softmax(dim=-1)
+        attn = self.attn_drop(attn)
+        
+        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+    
+    def _flash_attn(self, x, key_padding_mask=None, need_weights=False):
+        
+        qkv = self.qkv(x)
+        qkv = rearrange(qkv, "b s (three h d) -> b s three h d", three=3, h=self.num_heads)
+        
+        if self.qk_normalization:
+            q, k, v = qkv.unbind(2)
+            if self.use_fused_rmsnorm:
+                q = self.q_norm(q.flatten(-2, -1))[0].view(q.shape)
+                k = self.k_norm(k.flatten(-2, -1))[0].view(k.shape)
+            else:
+                q = self.q_norm(q.flatten(-2, -1)).view(q.shape)
+                k = self.k_norm(k.flatten(-2, -1)).view(k.shape)
+            qkv = torch.stack([q, k, v], dim=2)
+        
+        context, _ = self.inner_attn(
+            qkv, key_padding_mask=key_padding_mask, need_weights=need_weights, causal=self.causal
+        )
+        outs = self.proj(rearrange(context, "b s h d -> b s (h d)"))
+        outs = self.proj_drop(outs)
+        return outs
+    
+    def forward(self, x):
+        x = self._naive_attn(x) if not self.use_flash_attn else self._flash_attn(x)
+        return x
+
+
+class Mlp(nn.Module):
+    """ MLP as used in Vision Transformer, MLP-Mixer and related networks
+    """
+    
+    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU,
+                 bias=True, drop=0.):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        bias = to_2tuple(bias)
+        drop_probs = to_2tuple(drop)
+        
+        self.fc1 = nn.Linear(in_features, hidden_features, bias=bias[0])
+        self.act = act_layer()
+        self.drop1 = nn.Dropout(drop_probs[0])
+        self.fc2 = nn.Linear(hidden_features, out_features, bias=bias[1])
+        self.drop2 = nn.Dropout(drop_probs[1])
+    
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop1(x)
+        x = self.fc2(x)
+        x = self.drop2(x)
+        return x
+
+
+class Block(nn.Module):
+    
+    def __init__(
+            self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., init_values=None,
+            drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, use_flash_attn=False, use_fused_mlp=False,
+            fused_mlp_heuristic=1, with_cp=False, qk_normalization=False, layerscale_no_force_fp32=False,
+            use_fused_rmsnorm=False):
+        super().__init__()
+        
+        self.norm1 = norm_layer(dim)
+        self.attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop,
+                              use_flash_attn=use_flash_attn, causal=False, norm_layer=norm_layer,
+                              qk_normalization=qk_normalization,
+                              use_fused_rmsnorm=use_fused_rmsnorm)
+        self.ls1 = LayerScale(dim, init_values=init_values,
+                              force_fp32=(not layerscale_no_force_fp32)) if init_values else nn.Identity()
+        # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here
+        self.drop_path1 = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+        
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        if use_fused_mlp:
+            self.mlp = FusedMLP(in_features=dim, hidden_features=mlp_hidden_dim, heuristic=fused_mlp_heuristic)
+        else:
+            self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
+        self.ls2 = LayerScale(dim, init_values=init_values,
+                              force_fp32=(not layerscale_no_force_fp32)) if init_values else nn.Identity()
+        self.drop_path2 = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+        
+        self.with_cp = with_cp
+        self.use_fused_rmsnorm = use_fused_rmsnorm
+    
+    def forward(self, x, residual=None):
+        
+        def _inner_forward(x, residual=None):
+            if self.use_fused_rmsnorm:
+                x, residual = self.norm1(x, residual)
+                x = self.drop_path1(self.ls1(self.attn(x)))
+                x, residual = self.norm2(x, residual)
+                x = self.drop_path2(self.ls2(self.mlp(x)))
+                return x, residual
+            else:
+                assert residual is None
+                x = x + self.drop_path1(self.ls1(self.attn(self.norm1(x))))
+                x = x + self.drop_path2(self.ls2(self.mlp(self.norm2(x))))
+                return x
+        
+        if self.with_cp:
+            return checkpoint.checkpoint(_inner_forward, x, residual)
+        else:
+            return _inner_forward(x, residual=residual)
+
+
+class PatchEmbed(nn.Module):
+    """ 3D Image to Patch Embedding
+    """
+    
+    def __init__(
+            self, img_size=224, patch_size=16, in_chans=3, embed_dim=768, 
+            num_frames=8, tubelet_size=1, norm_layer=None
+        ):
+        super().__init__()
+        img_size = to_2tuple(img_size)
+        patch_size = to_2tuple(patch_size)
+        self.img_size = img_size
+        self.patch_size = patch_size
+        self.tubelet_size = tubelet_size
+        self.grid_size = (
+            num_frames // tubelet_size, 
+            img_size[0] // patch_size[0], 
+            img_size[1] // patch_size[1]
+        ) # (T, H, W)
+        self.num_patches = self.grid_size[0] * self.grid_size[1] * self.grid_size[2]
+        
+        self.proj = nn.Conv3d(
+            in_channels=in_chans, out_channels=embed_dim, 
+            kernel_size=(tubelet_size, patch_size[0], patch_size[1]), 
+            stride=(tubelet_size, patch_size[0], patch_size[1])
+        )
+        self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
+    
+    def forward(self, x):
+        x = self.proj(x)
+        x = x.flatten(3).permute(0, 2, 3, 1)  # B x C x T x HW => B x T x HW x C
+        x = self.norm(x)
+        return x
+
+
+class InternVideo2(nn.Module):
+    def __init__(
+            self,
+            in_chans: int = 3,
+            patch_size: int = 14,
+            img_size: int = 224,
+            qkv_bias: bool = False,
+            drop_path_rate: float = 0.25, # may need ablation
+            head_drop_path_rate: float = 0.,
+            embed_dim: int = 1408,
+            num_heads: int = 16,
+            mlp_ratio: float = 48/11,
+            init_values: float = 1e-5, # may need ablation
+            qk_normalization: bool = True,
+            depth: int = 40,
+            use_flash_attn: bool = True,
+            use_fused_rmsnorm: bool = True,
+            use_fused_mlp: bool = True,
+            fused_mlp_heuristic: int = 1,
+            attn_pool_num_heads: int = 16,
+            clip_embed_dim: int = 768,
+            layerscale_no_force_fp32: bool = False, # when True for training?
+            num_frames: int = 8,
+            tubelet_size: int = 1,
+            sep_pos_embed: bool = False,
+            use_checkpoint: bool = False,
+            checkpoint_num: int = 0,
+        ):
+        super().__init__()
+        
+        assert use_flash_attn == use_fused_rmsnorm == use_fused_mlp, logger.info(
+            'use_flash_attn, use_fused_rmsnorm and use_fused_mlp should be consistent')
+        
+        self.use_flash_attn = use_flash_attn
+        self.embed_dim = embed_dim
+        self.T = num_frames // tubelet_size
+        
+        if use_fused_rmsnorm:
+            norm_layer_for_blocks = partial(DropoutAddRMSNorm, eps=1e-6, prenorm=True)
+        else:
+            norm_layer_for_blocks = partial(RMSNorm, eps=1e-6)
+        self.norm_layer_for_blocks = norm_layer_for_blocks
+        self.patch_embed = PatchEmbed(
+            img_size, patch_size, in_chans, embed_dim,
+            num_frames=num_frames, tubelet_size=tubelet_size,
+        )
+        num_patches = self.patch_embed.num_patches
+        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
+        
+        # stolen from https://github.com/facebookresearch/mae_st/blob/dc072aaaf640d06892e23a33b42223a994efe272/models_vit.py#L65-L73C17
+        self.sep_pos_embed = sep_pos_embed
+        if sep_pos_embed:
+            logger.info("Use seperable position embedding")
+            grid_size = self.patch_embed.grid_size
+            self.grid_size = grid_size
+            self.pos_embed_spatial = nn.Parameter(torch.zeros(1, grid_size[1] * grid_size[2], embed_dim))
+            self.pos_embed_temporal = nn.Parameter(torch.zeros(1, grid_size[0], embed_dim))
+            self.pos_embed_cls = nn.Parameter(torch.zeros(1, 1, embed_dim))
+        else:
+            logger.info("Use joint position embedding")
+            self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))
+        
+        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]
+        # choose which layer to use checkpoint
+        with_cp_list = [False] * depth
+        if use_checkpoint:
+            for idx in range(depth):
+                if idx < checkpoint_num:
+                    with_cp_list[idx] = True
+        logger.info(f"Droppath rate: {dpr}")
+        logger.info(f"Checkpoint list: {with_cp_list}")
+        
+        self.blocks = nn.ModuleList([
+            Block(embed_dim, num_heads, mlp_ratio, qkv_bias=qkv_bias,
+                  norm_layer=norm_layer_for_blocks,
+                  drop_path=dpr[i], init_values=init_values, attn_drop=0.,
+                  use_flash_attn=use_flash_attn, use_fused_mlp=use_fused_mlp,
+                  fused_mlp_heuristic=fused_mlp_heuristic,
+                  with_cp=with_cp_list[i],
+                  qk_normalization=qk_normalization,
+                  layerscale_no_force_fp32=layerscale_no_force_fp32,
+                  use_fused_rmsnorm=use_fused_rmsnorm)
+            for i in range(depth)])
+        self.clip_projector = AttentionPoolingBlock(
+            dim=embed_dim, num_heads=attn_pool_num_heads, qkv_bias=True, qk_scale=None,
+            drop=0., attn_drop=0., drop_path=head_drop_path_rate, 
+            norm_layer=partial(nn.LayerNorm, eps=1e-5), out_dim=clip_embed_dim
+        )
+        
+        self.fc_norm = nn.Identity()
+
+        self.init_pos_embed()
+        trunc_normal_(self.cls_token, std=.02)
+        self.apply(self._init_weights)
+        self.fix_init_weight()
+
+    def init_pos_embed(self):
+        logger.info("Init pos_embed from sincos pos_embed")
+        if self.sep_pos_embed:
+            # trunc_normal_(self.pos_embed_spatial, std=.02)
+            # trunc_normal_(self.pos_embed_temporal, std=.02)
+            # trunc_normal_(self.pos_embed_cls, std=.02)
+            pos_embed_spatial = get_2d_sincos_pos_embed(
+                self.pos_embed_spatial.shape[-1], 
+                self.patch_embed.grid_size[1], # height & weight
+            )
+            self.pos_embed_spatial.data.copy_(torch.from_numpy(pos_embed_spatial).float().unsqueeze(0))
+            pos_embed_temporal = get_1d_sincos_pos_embed(
+                self.pos_embed_spatial.shape[-1], 
+                self.patch_embed.grid_size[0], # t_size
+            )
+            self.pos_embed_temporal.data.copy_(torch.from_numpy(pos_embed_temporal).float().unsqueeze(0))
+        else:
+            # trunc_normal_(self.pos_embed, std=.02)
+            pos_embed = get_3d_sincos_pos_embed(
+                self.pos_embed.shape[-1], 
+                self.patch_embed.grid_size[1], # height & weight
+                self.patch_embed.grid_size[0], # t_size
+                cls_token=True
+            )
+            self.pos_embed.data.copy_(torch.from_numpy(pos_embed).float().unsqueeze(0))
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)
+
+    def fix_init_weight(self):
+        def rescale(param, layer_id):
+            param.div_(math.sqrt(2.0 * layer_id))
+
+        for layer_id, layer in enumerate(self.blocks):
+            rescale(layer.attn.proj.weight.data, layer_id + 1)
+            rescale(layer.mlp.fc2.weight.data, layer_id + 1)
+    
+    @property
+    def dtype(self):
+        return self.patch_embed.proj.weight.dtype
+
+    def get_num_layers(self):
+        return len(self.blocks)
+
+    @torch.jit.ignore
+    def no_weight_decay(self):
+        return {
+            'pos_embed', 
+            'pos_embed_spatial', 
+            'pos_embed_temporal', 
+            'pos_embed_cls',
+            'cls_token'
+        }
+    
+    def forward(self, x, use_image=False):
+        x = self.patch_embed(x.type(self.dtype))
+        B, T, L, C = x.shape  # T: temporal; L: spatial
+        x = x.view([B, T * L, C])
+
+        # append cls token
+        cls_tokens = self.cls_token.expand(B, -1, -1)
+        x = torch.cat((cls_tokens, x), dim=1)
+
+        # add pos_embed
+        if self.sep_pos_embed:
+            if use_image:
+                pos_embed = self.pos_embed_spatial
+            else:
+                pos_embed = self.pos_embed_spatial.repeat(
+                    1, self.grid_size[0], 1
+                ) + torch.repeat_interleave(
+                    self.pos_embed_temporal,
+                    self.grid_size[1] * self.grid_size[2],
+                    dim=1,
+                )
+            pos_embed = torch.cat(
+                [
+                    self.pos_embed_cls.expand(pos_embed.shape[0], -1, -1),
+                    pos_embed,
+                ],
+                1,
+            )
+        else:
+            if use_image:
+                cls_pos_embed = self.pos_embed[:, :1, :]
+                img_pos_embed = self.pos_embed[:, 1:, :].view(1, self.T, L, C).mean(dim=1)
+                pos_embed = torch.cat([cls_pos_embed, img_pos_embed], dim=1)
+            else:
+                pos_embed = self.pos_embed
+
+        x = x + pos_embed
+
+        residual = None
+        for blk in self.blocks:
+            if isinstance(x, tuple) and len(x) == 2:
+                x, residual = x
+            x = blk(x, residual=residual)
+        if isinstance(x, tuple) and len(x) == 2:
+            x, residual = x
+            if residual is not None:
+                x = x + residual
+        
+        x = self.clip_projector(x)
+
+        x = self.fc_norm(x)
+        return x

mobile_clip.py

@@ -0,0 +1,264 @@
+#
+# For licensing see accompanying LICENSE file.
+# Copyright (C) 2024 Apple Inc. All Rights Reserved.
+#
+import math
+from typing import Optional, Sequence
+
+import torch
+from torch import Tensor, nn
+
+from typing import Dict
+import open_clip
+
+from .mobile_clip_transformer import (
+    PositionalEmbedding,
+    TransformerEncoder,
+    get_normalization_layer,
+)
+
+
+class TextTransformer(nn.Module):
+    def __init__(self, cfg: dict, projection_dim: int, *args, **kwargs) -> None:
+        super().__init__()
+
+        model_dim = cfg["dim"]
+        no_scale_embedding = cfg.get("no_scale_embedding", False)
+        no_pos_embedding = cfg.get("no_pos_embedding", False)
+        embed_dropout = cfg.get("embed_dropout", 0.0)
+        norm_layer = cfg["norm_layer"]
+        variant = cfg["model_name"]
+        self.vocab_size = cfg["vocab_size"]
+        self.projection_dim = projection_dim
+
+        # Token embedding layer
+        self.embedding_layer = nn.Embedding(
+            embedding_dim=model_dim, num_embeddings=self.vocab_size
+        )
+        self.embed_scale = 1.0 if no_scale_embedding else model_dim**-0.5
+
+        # Context length
+        context_length = cfg["context_length"]
+        assert (
+            context_length is not None
+        ), "Context length can't be None. Please set value accordingly."
+
+        self.positional_embedding = (
+            None
+            if no_pos_embedding
+            else PositionalEmbedding(
+                num_embeddings=context_length, embedding_dim=model_dim
+            )
+        )
+
+        self.embedding_dropout = nn.Dropout(p=embed_dropout)
+
+        # Transformer layer
+        n_transformer_layers = cfg["n_transformer_layers"]
+
+        # FFN multipliers for transformer layer
+        ffn_multipliers = cfg["ffn_multiplier_per_layer"]
+        if isinstance(ffn_multipliers, (float, int)):
+            ffn_multipliers = [ffn_multipliers] * n_transformer_layers
+
+        if not isinstance(ffn_multipliers, Sequence):
+            Warning(
+                "{} expects FFN multipliers as a list, whose length is the same as"
+                " number of transformer layers. Got: {}".format(
+                    self.__class__.__name__, type(ffn_multipliers)
+                )
+            )
+        elif (
+            isinstance(ffn_multipliers, Sequence)
+            and len(ffn_multipliers) != n_transformer_layers
+        ):
+            Warning(
+                "We need FFN multiplier for each transformer layer. Got {} ffn"
+                " multipliers while number of transformer layers = {}".format(
+                    len(ffn_multipliers), n_transformer_layers
+                )
+            )
+        ffn_dims = [
+            int(math.ceil(model_dim * ffn_mult / 16.0) * 16.0)
+            for ffn_mult in ffn_multipliers
+        ]
+
+        # Heads for transformer layers
+        mha_heads = cfg["n_heads_per_layer"]
+        if isinstance(mha_heads, int):
+            mha_heads = [mha_heads] * n_transformer_layers
+
+        if not isinstance(mha_heads, Sequence):
+            Warning(
+                "{} expects MHA heads as a list, whose length is the same as number of "
+                "transformer layers. Got: {}".format(
+                    self.__class__.__name__, type(mha_heads)
+                )
+            )
+        elif isinstance(mha_heads, Sequence) and len(mha_heads) != n_transformer_layers:
+            Warning(
+                "{} needs MHA heads for each transformer layer. Got {} mha heads while"
+                " number of transformer layers = {}".format(
+                    self.__class__.__name__, len(mha_heads), n_transformer_layers
+                )
+            )
+
+        if variant == "base":
+            self.transformer = nn.ModuleList(
+                [
+                    TransformerEncoder(
+                        embed_dim=model_dim,
+                        num_heads=mha_heads[layer_idx],
+                        ffn_latent_dim=ffn_dims[layer_idx],
+                        transformer_norm_layer=norm_layer,
+                    )
+                    for layer_idx in range(n_transformer_layers)
+                ]
+            )
+        elif variant == "mct":
+            raise NotImplementedError
+        else:
+            raise ValueError("Unrecognized text encoder variant {}".format(variant))
+
+        self.final_layer_norm = get_normalization_layer(
+            num_features=model_dim, norm_type=norm_layer
+        )
+
+        self.projection_layer = nn.Parameter(
+            torch.empty(model_dim, self.projection_dim)
+        )
+        self.model_dim = model_dim
+        self.causal_masking = cfg["causal_masking"]
+
+    def forward_embedding(self, text_tokens: Tensor) -> Tensor:
+        """Return text embedding for all tokens.
+
+        Args:
+            text_tokens: a tensor of token indices. Shape: [batch_size, context_length]
+
+        Returns:
+            A tensor of [batch_size, context_length, hidden_dim].
+        """
+        # [batch_size, context_length] --> [batch_size, context_length, hidden_dim]
+        token_emb = self.embedding_layer(text_tokens)
+        seq_len = token_emb.shape[1]
+        if self.positional_embedding is not None:
+            token_emb = token_emb + self.positional_embedding(seq_len).to(
+                token_emb.dtype
+            )
+        token_emb = self.embedding_dropout(token_emb)
+        return token_emb
+
+    def build_attention_mask(self, context_length: int, batch_size: int) -> Tensor:
+        """Build causal attention mask [batch_size, context_length, context_length]."""
+        # Build mask with full attention between the tokens
+        # pytorch uses additive attention mask; fill with -inf
+        mask = torch.empty(context_length, context_length)
+        mask.fill_(float("-inf"))
+        mask.triu_(1)  # zero out the lower diagonal
+        mask = mask.unsqueeze(0)  # add dummy batch dimension
+        mask = mask.expand(batch_size, -1, -1)
+        return mask
+
+    def encode_text(
+        self,
+        text_tokens: Tensor,
+        key_padding_mask: Optional[Tensor] = None,
+        return_all_tokens: bool = False,
+        *args,
+        **kwargs
+    ) -> Tensor:
+        """Return text token embeddings.
+
+        Args:
+            text_tokens: a tensor of token indices. Shape: [batch_size, context_length]
+            key_padding_mask: a tensor of boolean values as the padding mask.
+                Shape: [batch_size, context_length]
+            return_all_tokens: a boolean flag to return all tokens, defaults to False
+                to return only EOT token embedding.
+        Returns:
+            A tensor of [batch_size, context_length, hidden_dim] if return_all_tokens is
+            True, otherwise a tensor of [batch_size, hidden_dim].
+        """
+        # Discrete tokens to continuous embeddings
+        # [batch_size, context_length] --> [batch_size, context_length, hidden_dim]
+        token_emb = self.forward_embedding(text_tokens)
+
+        # [1, context_length, context_length]
+        attn_mask = None
+        if self.causal_masking:
+            attn_mask = self.build_attention_mask(
+                context_length=text_tokens.shape[1], batch_size=text_tokens.shape[0]
+            )
+            attn_mask = attn_mask.to(device=token_emb.device, dtype=token_emb.dtype)
+            key_padding_mask = None
+
+        for layer in self.transformer:
+            token_emb = layer(
+                token_emb,
+                key_padding_mask=key_padding_mask,
+                attn_mask=attn_mask,
+            )
+
+        # Apply layer norm
+        token_emb = self.final_layer_norm(token_emb)
+
+        if return_all_tokens:
+            return token_emb
+
+        # Take features from the eot embedding (eot_token is the highest number in each sequence)
+        token_emb = token_emb[
+            torch.arange(text_tokens.shape[0]), text_tokens.argmax(dim=-1)
+        ]
+
+        token_emb = token_emb @ self.projection_layer
+        return token_emb
+
+    def forward(
+        self,
+        text_tokens: Tensor,
+        key_padding_mask: Optional[Tensor] = None,
+        return_all_tokens: bool = False,
+        *args,
+        **kwargs
+    ) -> Tensor:
+        # Image-text pair data with single caption
+        # [B, CL] --> [B, d]
+        text_tokens = self.encode_text(
+            text_tokens=text_tokens,
+            key_padding_mask=key_padding_mask,
+            return_all_tokens=return_all_tokens,
+            *args,
+            **kwargs
+        )
+        return text_tokens
+
+
+class ClipTokenizer(nn.Module):
+    def __init__(self, cfg, *args, **kwargs):
+        super().__init__()
+        self.context_length = cfg["text_cfg"]["context_length"]
+        model_name = getattr(cfg["text_cfg"], "open_clip_tokenizer", "ViT-B-16")
+        self.tokenizer = open_clip.get_tokenizer(model_name)
+
+    def get_vocab_size(self) -> int:
+        return len(self.tokenizer.encoder)
+
+    def get_encodings(self) -> Dict[str, int]:
+        return self.tokenizer.encoder
+
+    def get_eot_token(self) -> int:
+        # Tokenizing an empty string returns a list [sot_id, eot_id]
+        return self.tokenizer("")[1]
+
+    def get_sot_token(self) -> int:
+        # Tokenizing an empty string returns a list [sot_id, eot_id]
+        return self.tokenizer("")[0]
+
+    def forward(self, input_sentence: str, *args, **kwargs) -> Tensor:
+        # tokenizer returns indices as a string
+        tokenized_sentence = self.tokenizer(input_sentence, self.context_length)
+        assert (
+            tokenized_sentence.shape[-1] == self.context_length
+        ), "Tokenized tensor should be exactly `context_length` long."
+        return tokenized_sentence

mobile_clip_transformer.py

@@ -0,0 +1,449 @@
+#
+# For licensing see accompanying LICENSE file.
+# Copyright (C) 2024 Apple Inc. All Rights Reserved.
+#
+"""
+Implementation of the following modules is borrowed from ml-cvnets repo:
+https://github.com/apple/ml-cvnets/blob/main/cvnets/layers/multi_head_attention.py
+https://github.com/apple/ml-cvnets/blob/main/cvnets/text_encoders/transformer.py
+
+Please see ACKNOWLEDGEMENTS for license details.
+"""
+
+from typing import List, Optional, Union
+
+import torch
+from torch import Size, Tensor, nn
+from torch.nn import functional as F
+from torchvision.ops import StochasticDepth
+
+
+class LayerNormFP32(nn.LayerNorm):
+    """
+    Applies `Layer Normalization <https://arxiv.org/abs/1607.06450>`_ over a input tensor with FP32 precision
+    """
+
+    def __init__(
+        self,
+        normalized_shape: Union[int, List[int], Size],
+        eps: Optional[float] = 1e-5,
+        elementwise_affine: Optional[bool] = True,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(
+            normalized_shape=normalized_shape,
+            eps=eps,
+            elementwise_affine=elementwise_affine,
+            *args,
+            **kwargs,
+        )
+
+    def forward(self, x: Tensor) -> Tensor:
+        # Convert input from dtype X to FP32 and perform normalization operation.
+        # This may help with underflow/overflow issues that we typically see with normalization layers
+        inp_dtype = x.dtype
+        return super().forward(x.to(torch.float32)).to(inp_dtype)
+
+
+def get_normalization_layer(norm_type, num_features):
+    if norm_type == "layer_norm":
+        return nn.LayerNorm(num_features)
+    elif norm_type == "layer_norm_fp32":
+        return LayerNormFP32(num_features)
+    else:
+        raise NotImplementedError(f"Option: {norm_type} not supported.")
+
+
+class PositionalEmbedding(nn.Module):
+    def __init__(
+        self,
+        num_embeddings: int,
+        embedding_dim: int,
+        padding_idx: Optional[int] = None,
+        is_learnable: Optional[bool] = False,
+        interpolation_mode: Optional[str] = "bilinear",
+        *args,
+        **kwargs,
+    ):
+        super().__init__()
+        # Add other pos embedding here and logic to choose between them
+        module = LearnablePositionalEmbedding
+
+        self.pos_embed = module(
+            num_embeddings=num_embeddings,
+            embedding_dim=embedding_dim,
+            padding_idx=padding_idx,
+            interpolation_mode=interpolation_mode,
+            *args,
+            **kwargs,
+        )
+
+    def forward(self, seq_len: int, *args, **kwargs) -> Tensor:
+        return self.pos_embed(seq_len, *args, **kwargs)
+
+    def __repr__(self):
+        return self.pos_embed.__repr__()
+
+
+class LearnablePositionalEmbedding(nn.Module):
+    """Learnable Positional embedding"""
+
+    def __init__(
+        self,
+        num_embeddings: int,
+        embedding_dim: int,
+        padding_idx: Optional[int] = None,
+        interpolation_mode: Optional[str] = "bilinear",
+        *args,
+        **kwargs,
+    ):
+        super().__init__()
+        self.pos_embed = nn.Parameter(torch.empty(1, 1, num_embeddings, embedding_dim))
+        self.embedding_dim = embedding_dim
+        self.num_embeddings = num_embeddings
+        self.padding_idx = padding_idx
+        self.interpolation_mode = interpolation_mode
+
+        self.reset_parameters()
+
+    def reset_parameters(self) -> None:
+        nn.init.trunc_normal_(self.pos_embed, mean=0, std=self.embedding_dim**-0.5)
+        if self.padding_idx is not None:
+            with torch.no_grad():
+                self.pos_embed[:, :, self.padding_idx, ...] = 0.0
+
+    def forward(self, seq_len: int, *args, **kwargs) -> Tensor:
+        # scale pos embedding
+        pos_embed = self.pos_embed
+        if self.padding_idx is not None:
+            with torch.no_grad():
+                pos_embed[:, :, self.padding_idx, ...] = 0.0
+
+        if seq_len != self.num_embeddings:
+            pos_embed = F.interpolate(
+                pos_embed,
+                size=(seq_len, self.embedding_dim),
+                mode=self.interpolation_mode,
+            )
+
+        # Input is of the form [Batch, Seq_len, Embedding_dim]
+        return pos_embed.reshape(1, seq_len, self.embedding_dim)
+
+    def __repr__(self):
+        return "{}(num_embeddings={}, embedding_dim={}, padding_idx={})".format(
+            self.__class__.__name__,
+            self.num_embeddings,
+            self.embedding_dim,
+            self.padding_idx,
+        )
+
+
+class MultiHeadAttention(nn.Module):
+    """
+    This layer applies a multi-head self- or cross-attention as described in
+    `Attention is all you need <https://arxiv.org/abs/1706.03762>`_ paper
+
+    Args:
+        embed_dim (int): :math:`C_{in}` from an expected input of size :math:`(N, S, C_{in})`
+        num_heads (int): Number of heads in multi-head attention
+        attn_dropout (Optional[float]): Attention dropout. Default: 0.0
+        bias (Optional[bool]): Use bias or not. Default: ``True``
+
+    Shape:
+        - Input:
+           - Query tensor (x_q) :math:`(N, S, C_{in})` where :math:`N` is batch size, :math:`S` is number of source tokens,
+        and :math:`C_{in}` is input embedding dim
+           - Optional Key-Value tensor (x_kv) :math:`(N, T, C_{in})` where :math:`T` is number of target tokens
+        - Output: same shape as the input
+
+    """
+
+    def __init__(
+        self,
+        embed_dim: int,
+        num_heads: int,
+        attn_dropout: Optional[float] = 0.0,
+        bias: Optional[bool] = True,
+        output_dim: Optional[int] = None,
+        *args,
+        **kwargs,
+    ) -> None:
+        if output_dim is None:
+            output_dim = embed_dim
+        super().__init__()
+        if embed_dim % num_heads != 0:
+            Warning(
+                "Embedding dim must be divisible by number of heads in {}. Got: embed_dim={} and num_heads={}".format(
+                    self.__class__.__name__, embed_dim, num_heads
+                )
+            )
+
+        self.qkv_proj = nn.Linear(
+            in_features=embed_dim, out_features=3 * embed_dim, bias=bias
+        )
+
+        self.attn_dropout = nn.Dropout(p=attn_dropout)
+        self.out_proj = nn.Linear(
+            in_features=embed_dim, out_features=output_dim, bias=bias
+        )
+
+        self.head_dim = embed_dim // num_heads
+        self.scaling = self.head_dim**-0.5
+        self.softmax = nn.Softmax(dim=-1)
+        self.num_heads = num_heads
+        self.embed_dim = embed_dim
+        self.use_separate_proj_weight = embed_dim != output_dim
+
+    def __repr__(self):
+        return "{}(head_dim={}, num_heads={}, attn_dropout={})".format(
+            self.__class__.__name__, self.head_dim, self.num_heads, self.attn_dropout.p
+        )
+
+    def _forward_impl(
+        self,
+        x_q: Tensor,
+        x_kv: Optional[Tensor] = None,
+        key_padding_mask: Optional[Tensor] = None,
+        attn_mask: Optional[Tensor] = None,
+    ) -> Tensor:
+        # [N, S, C]
+        b_sz, S_len, in_channels = x_q.shape
+
+        if x_kv is None:
+            # self-attention
+            # [N, S, C] --> [N, S, 3C] --> [N, S, 3, h, c] where C = hc
+            qkv = self.qkv_proj(x_q).reshape(b_sz, S_len, 3, self.num_heads, -1)
+            # [N, S, 3, h, c] --> [N, h, 3, S, C]
+            qkv = qkv.transpose(1, 3).contiguous()
+
+            # [N, h, 3, S, C] --> [N, h, S, C] x 3
+            query, key, value = qkv[:, :, 0], qkv[:, :, 1], qkv[:, :, 2]
+        else:
+            T_len = x_kv.shape[1]
+
+            # cross-attention
+            # [N, S, C]
+            query = F.linear(
+                x_q,
+                weight=self.qkv_proj.weight[: self.embed_dim, ...],
+                bias=self.qkv_proj.bias[: self.embed_dim]
+                if self.qkv_proj.bias is not None
+                else None,
+            )
+            # [N, S, C] --> [N, S, h, c] --> [N, h, S, c]
+            query = (
+                query.reshape(b_sz, S_len, self.num_heads, self.head_dim)
+                .transpose(1, 2)
+                .contiguous()
+            )
+
+            # [N, T, C] --> [N, T, 2C]
+            kv = F.linear(
+                x_kv,
+                weight=self.qkv_proj.weight[self.embed_dim :, ...],
+                bias=self.qkv_proj.bias[self.embed_dim :]
+                if self.qkv_proj.bias is not None
+                else None,
+            )
+            # [N, T, 2C] --> [N, T, 2, h, c]
+            kv = kv.reshape(b_sz, T_len, 2, self.num_heads, self.head_dim)
+            # [N, T, 2, h, c] --> [N, h, 2, T, c]
+            kv = kv.transpose(1, 3).contiguous()
+            key, value = kv[:, :, 0], kv[:, :, 1]
+
+        query = query * self.scaling
+
+        # [N h, T, c] --> [N, h, c, T]
+        key = key.transpose(-1, -2)
+
+        # QK^T
+        # [N, h, S, c] x [N, h, c, T] --> [N, h, S, T]
+        attn = torch.matmul(query, key)
+
+        batch_size, num_heads, num_src_tokens, num_tgt_tokens = attn.shape
+        if attn_mask is not None:
+            # attn_mask shape should be the same as attn
+            assert list(attn_mask.shape) == [
+                batch_size,
+                num_src_tokens,
+                num_tgt_tokens,
+            ], "Shape of attention mask should be [{}, {}, {}]. Got: {}".format(
+                batch_size, num_src_tokens, num_tgt_tokens, attn_mask.shape
+            )
+            # [N, S, T] --> [N, 1, S, T]
+            attn_mask = attn_mask.unsqueeze(1)
+            attn = attn + attn_mask
+
+        if key_padding_mask is not None:
+            # Do not attend to padding positions
+            # key padding mask size is [N, T]
+            assert key_padding_mask.dim() == 2 and list(key_padding_mask.shape) == [
+                batch_size,
+                num_tgt_tokens,
+            ], "Key_padding_mask should be 2-dimension with shape [{}, {}]. Got: {}".format(
+                batch_size, num_tgt_tokens, key_padding_mask.shape
+            )
+            attn = attn.masked_fill(
+                key_padding_mask.unsqueeze(1)
+                .unsqueeze(2)
+                .to(torch.bool),  # [N, T] --> [N, 1, 1, T]
+                float("-inf"),
+            )
+
+        attn_dtype = attn.dtype
+        attn_as_float = self.softmax(attn.float())
+        attn = attn_as_float.to(attn_dtype)
+        attn = self.attn_dropout(attn)
+
+        # weighted sum
+        # [N, h, S, T] x [N, h, T, c] --> [N, h, S, c]
+        out = torch.matmul(attn, value)
+
+        # [N, h, S, c] --> [N, S, h, c] --> [N, S, C]
+        out = out.transpose(1, 2).reshape(b_sz, S_len, -1)
+        out = self.out_proj(out)
+
+        return out
+
+    def forward(
+        self,
+        x_q: Tensor,
+        x_kv: Optional[Tensor] = None,
+        key_padding_mask: Optional[Tensor] = None,
+        attn_mask: Optional[Tensor] = None,
+        *args,
+        **kwargs,
+    ) -> Tensor:
+        # [Batch , Sequence, Hidden_dim]
+        return self._forward_impl(
+            x_q=x_q,
+            x_kv=x_kv,
+            key_padding_mask=key_padding_mask,
+            attn_mask=attn_mask,
+        )
+
+
+class TransformerEncoder(nn.Module):
+    """
+    This class defines the pre-norm `Transformer encoder <https://arxiv.org/abs/1706.03762>`_
+    Args:
+        embed_dim: :math:`C_{in}` from an expected input of size :math:`(N, P, C_{in})`.
+        ffn_latent_dim: Inner dimension of the FFN.
+        num_heads: Number of heads in multi-head attention. Default: 8.
+        attn_dropout: Dropout rate for attention in multi-head attention. Default: 0.0
+        dropout: Dropout rate. Default: 0.0.
+        ffn_dropout: Dropout between FFN layers. Default: 0.0.
+        transformer_norm_layer: Normalization layer. Default: layer_norm.
+        stochastic_dropout: Stochastic dropout setting. Default: 0.0.
+
+    Shape:
+        - Input: :math:`(N, P, C_{in})` where :math:`N` is batch size, :math:`P` is number of patches,
+        and :math:`C_{in}` is input embedding dim
+        - Output: same shape as the input
+    """
+
+    def __init__(
+        self,
+        embed_dim: int,
+        ffn_latent_dim: int,
+        num_heads: Optional[int] = 8,
+        attn_dropout: Optional[float] = 0.0,
+        dropout: Optional[float] = 0.0,
+        ffn_dropout: Optional[float] = 0.0,
+        transformer_norm_layer: Optional[str] = "layer_norm",
+        stochastic_dropout: Optional[float] = 0.0,
+        *args,
+        **kwargs,
+    ) -> None:
+
+        super().__init__()
+
+        # Build attention layer
+        attn_unit = MultiHeadAttention(
+            embed_dim,
+            num_heads,
+            attn_dropout=attn_dropout,
+            bias=True,
+        )
+
+        self.pre_norm_mha = nn.Sequential(
+            get_normalization_layer(
+                norm_type=transformer_norm_layer, num_features=embed_dim
+            ),
+            attn_unit,
+            nn.Dropout(p=dropout),
+        )
+
+        act_name = nn.GELU()
+        self.pre_norm_ffn = nn.Sequential(
+            get_normalization_layer(
+                norm_type=transformer_norm_layer, num_features=embed_dim
+            ),
+            nn.Linear(in_features=embed_dim, out_features=ffn_latent_dim, bias=True),
+            act_name,
+            nn.Dropout(p=ffn_dropout),
+            nn.Linear(in_features=ffn_latent_dim, out_features=embed_dim, bias=True),
+            nn.Dropout(p=dropout),
+        )
+
+        self.drop_path = nn.Identity()
+        if stochastic_dropout > 0.0:
+            if dropout > 0.0:
+                Warning(
+                    "Stochastic dropout and dropout are mutually exclusive. "
+                    "Use either of them, but not both."
+                    "Got: {} and {}".format(stochastic_dropout, dropout)
+                )
+            self.drop_path = StochasticDepth(p=stochastic_dropout, mode="row")
+
+        self.embed_dim = embed_dim
+        self.ffn_dim = ffn_latent_dim
+        self.ffn_dropout = ffn_dropout
+        self.stochastic_dropout = stochastic_dropout
+        self.std_dropout = dropout
+        self.attn_fn_name = attn_unit.__class__.__name__
+        self.act_fn_name = act_name.__class__.__name__
+        self.norm_type = transformer_norm_layer
+
+    def __repr__(self) -> str:
+        return "{}(embed_dim={}, ffn_dim={}, dropout={}, ffn_dropout={}, stochastic_dropout={}, attn_fn={}, act_fn={}, norm_fn={})".format(
+            self.__class__.__name__,
+            self.embed_dim,
+            self.ffn_dim,
+            self.std_dropout,
+            self.ffn_dropout,
+            self.stochastic_dropout,
+            self.attn_fn_name,
+            self.act_fn_name,
+            self.norm_type,
+        )
+
+    def forward(
+        self,
+        x: Tensor,
+        x_prev: Optional[Tensor] = None,
+        key_padding_mask: Optional[Tensor] = None,
+        attn_mask: Optional[Tensor] = None,
+        *args,
+        **kwargs,
+    ) -> Tensor:
+
+        # Multi-head attention
+        res = x
+        x = self.pre_norm_mha[0](x)  # norm
+        x = self.pre_norm_mha[1](
+            x_q=x,
+            x_kv=x_prev,
+            key_padding_mask=key_padding_mask,
+            attn_mask=attn_mask,
+            *args,
+            **kwargs,
+        )  # mha
+
+        x = self.drop_path(self.pre_norm_mha[2](x))  # applying stochastic depth
+        x = x + res
+
+        # Feed forward network
+        x = x + self.drop_path(self.pre_norm_ffn(x))
+        return x

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:136b42a078a8ec440e38b56d91d570fb0969643a641795e06e171162ab176b4e
+size 745562274

modeling_internvideo2encoder.py

@@ -0,0 +1,152 @@
+# from .internvideo2_stage2 import InternVideo2_Stage2 as IV2S2
+from transformers import PretrainedConfig, PreTrainedModel, AutoModel, AutoConfig
+from .config import InternVideo2Config as config
+import warnings
+import torch
+from torch import nn
+import torchvision.transforms as transforms
+from torchvision.transforms import InterpolationMode
+from transformers.utils import logging
+warnings.filterwarnings("ignore")
+from .internvideo2_clip_vision import InternVideo2
+from .mobile_clip import TextTransformer, ClipTokenizer
+logger = logging.get_logger(__name__)
+
+class InternVideo2_CLIP_small(PreTrainedModel):
+    config_class = config
+
+    def __init__(self, config,  tokenizer=None, is_pretrain=True):
+        super().__init__(config)
+        self.config = config
+        self.tokenizer = tokenizer
+        self.is_pretrain = is_pretrain
+        print(config)
+        if tokenizer is None:
+            self.tokenizer = ClipTokenizer(self.config.model.text_encoder)
+        # self.model = IV2S2(self.config).to('cpu').to(torch.float16)
+        self.vision_encoder = self.build_vision_encoder()
+
+        self.vision_align = nn.Sequential(
+            nn.LayerNorm(self.config.model.vision_encoder.clip_embed_dim),
+            nn.Linear(
+                self.config.model.vision_encoder.clip_embed_dim, 
+                self.config.model.vision_encoder.align_dim
+            ),
+        )
+        self.text_encoder = self.build_text_encoder(cfg=self.config.model.text_encoder['text_cfg'], projection_dim=self.config.model.text_encoder["embed_dim"])
+        # adopt 1 / 100. as in ViCLIP
+        self.temp = nn.parameter.Parameter(torch.ones([]) * config.model.temp)
+        self.temp_min = config.model.temp_min
+
+        if self.config.model.freeze_vision:
+            for name, p in self.vision_encoder.named_parameters():
+                if self.config.model.open_vision_clip_projector and name.startswith('clip_projector'):
+                    logger.info(f"Unfreeze {name}")
+                else:
+                    logger.info(f"Freeze {name}")
+                    p.requires_grad = False
+        if self.config.model.freeze_text:
+            for name, p in self.text_encoder.named_parameters():
+                if self.config.model.open_text_projection and name.startswith('projection_layer'):
+                    logger.info(f"Unfreeze {name}")
+                else:
+                    logger.info(f"Freeze {name}")
+                    p.requires_grad = False
+        img_size = self.config.model.vision_encoder.img_size
+        self.transform = transforms.Compose(
+            [
+                transforms.Resize(
+                    (img_size, img_size),
+                    interpolation=InterpolationMode.BICUBIC,
+                ),
+                transforms.Lambda(lambda x: x.float().div(255.0)),
+                transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
+            ]
+        )
+
+
+    @torch.no_grad()
+    def clip_contrastive_temperature(self):
+        """Seems only used during pre-training"""
+        self.temp.clamp_(min=self.temp_min)
+
+    def encode_vision(self, image, test=False):
+        """encode image / videos as features.
+
+        Args:
+            image (torch.Tensor): The input images.
+            test (bool): Whether testing.
+
+        Returns: tuple.
+            - vision_embeds (torch.Tensor): The features of all patches. Shape: [B,C].
+
+        """
+        T = image.shape[1]
+        use_image = True if T == 1 else False
+        image = image.permute(0, 2, 1, 3, 4) # [B,T,C,H,W] -> [B,C,T,H,W]
+
+        vision_embeds = self.vision_encoder(image, use_image=use_image)
+        vision_embeds = self.vision_align(vision_embeds)
+        return vision_embeds
+
+    def encode_text(self, text):
+        """encode text.
+        Args:
+            text (dict): The output of huggingface's `PreTrainedTokenizer`. contains keys:
+                - input_ids (torch.Tensor): Token ids to be fed to a model. Shape: [B,L].
+                - attention_mask (torch.Tensor): The mask indicate padded tokens. Shape: [B,L]. 0 is padded token.
+                - other keys refer to "https://huggingface.co/docs/transformers/v4.21.2/en/main_classes/tokenizer#transformers.PreTrainedTokenizer.__call__".
+        Returns: tuple.
+            - text_embeds (torch.Tensor): The features of all tokens. Shape: [B,C].
+
+        """
+        text_embeds = self.text_encoder(text)
+        return text_embeds
+
+    def build_vision_encoder(self):
+        """build vision encoder
+        Returns: (vision_encoder, vision_layernorm). Each is a `nn.Module`.
+
+        """
+        vision_encoder = InternVideo2(
+            in_chans=self.config.model.vision_encoder.in_chans,
+            patch_size=self.config.model.vision_encoder.patch_size,
+            img_size=self.config.model.vision_encoder.img_size,
+            qkv_bias=self.config.model.vision_encoder.qkv_bias,
+            drop_path_rate=self.config.model.vision_encoder.drop_path_rate,
+            head_drop_path_rate=self.config.model.vision_encoder.head_drop_path_rate,
+            embed_dim=self.config.model.vision_encoder.embed_dim,
+            num_heads=self.config.model.vision_encoder.num_heads,
+            mlp_ratio=self.config.model.vision_encoder.mlp_ratio,
+            init_values=self.config.model.vision_encoder.init_values,
+            qk_normalization=self.config.model.vision_encoder.qk_normalization,
+            depth=self.config.model.vision_encoder.depth,
+            use_flash_attn=self.config.model.vision_encoder.use_flash_attn,
+            use_fused_rmsnorm=self.config.model.vision_encoder.use_fused_rmsnorm,
+            use_fused_mlp=self.config.model.vision_encoder.use_fused_mlp,
+            fused_mlp_heuristic=self.config.model.vision_encoder.fused_mlp_heuristic,
+            attn_pool_num_heads=self.config.model.vision_encoder.attn_pool_num_heads,
+            clip_embed_dim=self.config.model.vision_encoder.clip_embed_dim,
+            layerscale_no_force_fp32=self.config.model.vision_encoder.layerscale_no_force_fp32,
+            num_frames=self.config.model.vision_encoder.num_frames,
+            tubelet_size=self.config.model.vision_encoder.tubelet_size,
+            sep_pos_embed=self.config.model.vision_encoder.sep_pos_embed,
+            use_checkpoint=self.config.model.vision_encoder.use_checkpoint,
+            checkpoint_num=self.config.model.vision_encoder.checkpoint_num,
+        )
+        return vision_encoder
+
+    def build_text_encoder(self, cfg, projection_dim):
+        """build text_encoder and possiblly video-to-text multimodal fusion encoder.
+        Returns: nn.Module. The text encoder
+
+        """
+        text_encoder = TextTransformer(cfg, projection_dim)
+
+        return text_encoder
+    
+if __name__ == "__main__":
+    model_config = config()
+    model = InternVideo2Stage2VideoEncoder(model_config)
+    x = torch.randn(2, 3, 8, 224, 224, dtype=torch.float16).to(model_config.device)
+    output = model(x)

pos_embed.py

@@ -0,0 +1,299 @@
+import numpy as np
+import torch
+import logging
+
+logger = logging.getLogger(__name__)
+
+# --------------------------------------------------------
+# 3D sine-cosine position embedding
+# References:
+# MVD: https://github.com/ruiwang2021/mvd/blob/main/modeling_finetune.py
+# --------------------------------------------------------
+def get_3d_sincos_pos_embed(embed_dim, grid_size, t_size, cls_token=False):
+    """
+    grid_size: int of the grid height and width
+    t_size: int of the temporal size
+    return:
+    pos_embed: [t_size*grid_size*grid_size, embed_dim] or [1+t_size*grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
+    """
+    assert embed_dim % 4 == 0
+    embed_dim_spatial = embed_dim // 4 * 3
+    embed_dim_temporal = embed_dim // 4
+
+    # spatial
+    grid_h = np.arange(grid_size, dtype=np.float32)
+    grid_w = np.arange(grid_size, dtype=np.float32)
+    grid = np.meshgrid(grid_w, grid_h)  # here w goes first
+    grid = np.stack(grid, axis=0)
+
+    grid = grid.reshape([2, 1, grid_size, grid_size])
+    pos_embed_spatial = get_2d_sincos_pos_embed_from_grid(
+        embed_dim_spatial, grid
+    )
+
+    # temporal
+    grid_t = np.arange(t_size, dtype=np.float32)
+    pos_embed_temporal = get_1d_sincos_pos_embed_from_grid(
+        embed_dim_temporal, grid_t
+    )
+
+    # concate: [T, H, W] order
+    pos_embed_temporal = pos_embed_temporal[:, np.newaxis, :]
+    pos_embed_temporal = np.repeat(
+        pos_embed_temporal, grid_size**2, axis=1
+    )  # [T, H*W, D // 4]
+    pos_embed_spatial = pos_embed_spatial[np.newaxis, :, :]
+    pos_embed_spatial = np.repeat(
+        pos_embed_spatial, t_size, axis=0
+    )  # [T, H*W, D // 4 * 3]
+
+    pos_embed = np.concatenate([pos_embed_temporal, pos_embed_spatial], axis=-1)
+    pos_embed = pos_embed.reshape([-1, embed_dim])  # [T*H*W, D]
+
+    if cls_token:
+        pos_embed = np.concatenate(
+            [np.zeros([1, embed_dim]), pos_embed], axis=0
+        )
+    return pos_embed
+
+
+# --------------------------------------------------------
+# 2D sine-cosine position embedding
+# References:
+# Transformer: https://github.com/tensorflow/models/blob/master/official/nlp/transformer/model_utils.py
+# MoCo v3: https://github.com/facebookresearch/moco-v3
+# --------------------------------------------------------
+def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False):
+    """
+    grid_size: int of the grid height and width
+    return:
+    pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
+    """
+    grid_h = np.arange(grid_size, dtype=np.float32)
+    grid_w = np.arange(grid_size, dtype=np.float32)
+    grid = np.meshgrid(grid_w, grid_h)  # here w goes first
+    grid = np.stack(grid, axis=0)
+
+    grid = grid.reshape([2, 1, grid_size, grid_size])
+    pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
+    if cls_token:
+        pos_embed = np.concatenate(
+            [np.zeros([1, embed_dim]), pos_embed], axis=0
+        )
+    return pos_embed
+
+
+def get_1d_sincos_pos_embed(embed_dim, t_size, cls_token=False):
+    """
+    t_size: int of the temporal size
+    return:
+    pos_embed: [t_size, embed_dim] or [1+t_size, embed_dim] (w/ or w/o cls_token)
+    """
+    grid_t = np.arange(t_size, dtype=np.float32)
+    pos_embed = get_1d_sincos_pos_embed_from_grid(embed_dim, grid_t)
+    if cls_token:
+        pos_embed = np.concatenate(
+            [np.zeros([1, embed_dim]), pos_embed], axis=0
+        )
+    return pos_embed
+
+
+def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
+    assert embed_dim % 2 == 0
+
+    # use half of dimensions to encode grid_h
+    emb_h = get_1d_sincos_pos_embed_from_grid(
+        embed_dim // 2, grid[0]
+    )  # (H*W, D/2)
+    emb_w = get_1d_sincos_pos_embed_from_grid(
+        embed_dim // 2, grid[1]
+    )  # (H*W, D/2)
+
+    emb = np.concatenate([emb_h, emb_w], axis=1)  # (H*W, D)
+    return emb
+
+
+def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
+    """
+    embed_dim: output dimension for each position
+    pos: a list of positions to be encoded: size (M,)
+    out: (M, D)
+    """
+    assert embed_dim % 2 == 0
+    omega = np.arange(embed_dim // 2, dtype=np.float32)
+    omega /= embed_dim / 2.0
+    omega = 1.0 / 10000**omega  # (D/2,)
+
+    pos = pos.reshape(-1)  # (M,)
+    out = np.einsum("m,d->md", pos, omega)  # (M, D/2), outer product
+
+    emb_sin = np.sin(out)  # (M, D/2)
+    emb_cos = np.cos(out)  # (M, D/2)
+
+    emb = np.concatenate([emb_sin, emb_cos], axis=1)  # (M, D)
+    return emb
+
+
+def interpolate_pos_embed(checkpoint_model, model, orig_t_size=4, pos_name='vision_encoder.pos_embed'):
+    if pos_name in checkpoint_model:
+        pos_embed_checkpoint = checkpoint_model[pos_name]
+        embedding_size = pos_embed_checkpoint.shape[-1] # channel dim
+        num_patches = model.patch_embed.num_patches # 
+        num_extra_tokens = model.pos_embed.shape[-2] - num_patches # 0/1
+
+        # we use 4 frames for pretraining
+        new_t_size = model.T
+        # height (== width) for the checkpoint position embedding
+        orig_size = int(((pos_embed_checkpoint.shape[-2] - num_extra_tokens)//(orig_t_size)) ** 0.5)
+        # height (== width) for the new position embedding
+        new_size = int((num_patches // (new_t_size))** 0.5)
+        
+        # class_token and dist_token are kept unchanged
+        if orig_t_size != new_t_size:
+            logger.info(f"Temporal interpolate from {orig_t_size} to {new_t_size} ({pos_name})")
+            extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
+            # only the position tokens are interpolated
+            pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
+            # B, L, C -> B， T, HW, C -> BHW, C, T  (B = 1)
+            pos_tokens = pos_tokens.view(1, orig_t_size, -1, embedding_size)
+            pos_tokens = pos_tokens.permute(0, 2, 3, 1).reshape(-1, embedding_size, orig_t_size)
+            pos_tokens = torch.nn.functional.interpolate(pos_tokens, size=new_t_size, mode='linear')
+            pos_tokens = pos_tokens.view(1, -1, embedding_size, new_t_size)
+            pos_tokens = pos_tokens.permute(0, 3, 1, 2).reshape(1, -1, embedding_size)
+            new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
+            checkpoint_model[pos_name] = new_pos_embed
+            pos_embed_checkpoint = new_pos_embed
+
+        # class_token and dist_token are kept unchanged
+        if orig_size != new_size:
+            logger.info(f"Position interpolate from {orig_size}x{orig_size} to {new_size}x{new_size} ({pos_name})")
+            extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
+            # only the position tokens are interpolated
+            pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
+            # B, L, C -> BT, H, W, C -> BT, C, H, W
+            pos_tokens = pos_tokens.reshape(-1, new_t_size, orig_size, orig_size, embedding_size)
+            pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2)
+            pos_tokens = torch.nn.functional.interpolate(
+                pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False)
+            # BT, C, H, W -> BT, H, W, C ->  B, T, H, W, C
+            pos_tokens = pos_tokens.permute(0, 2, 3, 1).reshape(-1, new_t_size, new_size, new_size, embedding_size) 
+            pos_tokens = pos_tokens.flatten(1, 3) # B, L, C
+            new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
+            checkpoint_model[pos_name] = new_pos_embed
+
+
+def interpolate_pos_embed_internvideo2(checkpoint_model, model, orig_t_size = 8):
+    # interpolate position embedding
+    for pos_name in ['pos_embed', 'clip_pos_embed']:
+        if pos_name in checkpoint_model:
+            pos_embed_checkpoint = checkpoint_model[pos_name]
+            embedding_size = pos_embed_checkpoint.shape[-1] # channel dim
+            num_patches = model.patch_embed.num_patches # 
+            num_extra_tokens = model.pos_embed.shape[-2] - num_patches # 0/1
+
+            # we use 8 frames for pretraining
+            # new_t_size = args.num_frames * args.num_segments // model.patch_embed.tubelet_size
+            new_t_size = model.num_frames // model.tubelet_size
+            # height (== width) for the checkpoint position embedding
+            orig_size = int(((pos_embed_checkpoint.shape[-2] - num_extra_tokens)//(orig_t_size)) ** 0.5)
+            # height (== width) for the new position embedding
+            new_size = int((num_patches // (new_t_size))** 0.5)
+            
+            # class_token and dist_token are kept unchanged
+            if orig_t_size != new_t_size:
+                logger.info(f"Temporal interpolate from {orig_t_size} to {new_t_size} ({pos_name})")
+                extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
+                # only the position tokens are interpolated
+                pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
+                # B, L, C -> B， T, HW, C -> BHW, C, T  (B = 1)
+                pos_tokens = pos_tokens.view(1, orig_t_size, -1, embedding_size)
+                pos_tokens = pos_tokens.permute(0, 2, 3, 1).reshape(-1, embedding_size, orig_t_size)
+                pos_tokens = torch.nn.functional.interpolate(pos_tokens, size=new_t_size, mode='linear')
+                pos_tokens = pos_tokens.view(1, -1, embedding_size, new_t_size)
+                pos_tokens = pos_tokens.permute(0, 3, 1, 2).reshape(1, -1, embedding_size)
+                new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
+                checkpoint_model[pos_name] = new_pos_embed
+                pos_embed_checkpoint = new_pos_embed
+
+            # class_token and dist_token are kept unchanged
+            if orig_size != new_size:
+                logger.info(f"Position interpolate from {orig_size}x{orig_size} to {new_size}x{new_size} ({pos_name})")
+                extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
+                # only the position tokens are interpolated
+                pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
+                # B, L, C -> BT, H, W, C -> BT, C, H, W
+                pos_tokens = pos_tokens.reshape(-1, new_t_size, orig_size, orig_size, embedding_size)
+                pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2)
+                pos_tokens = torch.nn.functional.interpolate(
+                    pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False)
+                # BT, C, H, W -> BT, H, W, C ->  B, T, H, W, C
+                pos_tokens = pos_tokens.permute(0, 2, 3, 1).reshape(-1, new_t_size, new_size, new_size, embedding_size) 
+                pos_tokens = pos_tokens.flatten(1, 3) # B, L, C
+                new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
+                checkpoint_model[pos_name] = new_pos_embed
+    
+    if 'pos_embed_spatial' in checkpoint_model or 'pos_embed_temporal' in checkpoint_model:
+        raise NotImplementedError
+
+
+def interpolate_pos_embed_internvideo2_new(checkpoint_model, model, orig_t_size = 8):
+    pos_names = []
+    for k in checkpoint_model.keys():
+        if ('pos_embed' in k or 'clip_pos_embed' in k) and 'img_pos_embed' not in k:
+            pos_names.append(k)
+    
+    logger.info(f"pos names list for interpolating: {pos_names}")
+
+    assert len(pos_names) > 0, checkpoint_model.keys()
+
+    if 'pos_embed_spatial' in checkpoint_model.keys() or 'pos_embed_temporal' in checkpoint_model.keys():
+        raise NotImplementedError
+    
+    # interpolate position embedding
+    for pos_name in pos_names:
+
+        pos_embed_checkpoint = checkpoint_model[pos_name]
+        embedding_size = pos_embed_checkpoint.shape[-1] # channel dim
+        num_patches = model.patch_embed.num_patches # 
+        num_extra_tokens = model.pos_embed.shape[-2] - num_patches # 0/1
+
+        # we use 8 frames for pretraining
+        # new_t_size = args.num_frames * args.num_segments // model.patch_embed.tubelet_size
+        new_t_size = model.num_frames // model.tubelet_size
+        # height (== width) for the checkpoint position embedding
+        orig_size = int(((pos_embed_checkpoint.shape[-2] - num_extra_tokens)//(orig_t_size)) ** 0.5)
+        # height (== width) for the new position embedding
+        new_size = int((num_patches // (new_t_size))** 0.5)
+        
+        # class_token and dist_token are kept unchanged
+        if orig_t_size != new_t_size:
+            logger.info(f"Temporal interpolate from {orig_t_size} to {new_t_size} ({pos_name})")
+            extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
+            # only the position tokens are interpolated
+            pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
+            # B, L, C -> B， T, HW, C -> BHW, C, T  (B = 1)
+            pos_tokens = pos_tokens.view(1, orig_t_size, -1, embedding_size)
+            pos_tokens = pos_tokens.permute(0, 2, 3, 1).reshape(-1, embedding_size, orig_t_size)
+            pos_tokens = torch.nn.functional.interpolate(pos_tokens, size=new_t_size, mode='linear')
+            pos_tokens = pos_tokens.view(1, -1, embedding_size, new_t_size)
+            pos_tokens = pos_tokens.permute(0, 3, 1, 2).reshape(1, -1, embedding_size)
+            new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
+            checkpoint_model[pos_name] = new_pos_embed
+            pos_embed_checkpoint = new_pos_embed
+
+        # class_token and dist_token are kept unchanged
+        if orig_size != new_size:
+            logger.info(f"Position interpolate from {orig_size}x{orig_size} to {new_size}x{new_size} ({pos_name})")
+            extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
+            # only the position tokens are interpolated
+            pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
+            # B, L, C -> BT, H, W, C -> BT, C, H, W
+            pos_tokens = pos_tokens.reshape(-1, new_t_size, orig_size, orig_size, embedding_size)
+            pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2)
+            pos_tokens = torch.nn.functional.interpolate(
+                pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False)
+            # BT, C, H, W -> BT, H, W, C ->  B, T, H, W, C
+            pos_tokens = pos_tokens.permute(0, 2, 3, 1).reshape(-1, new_t_size, new_size, new_size, embedding_size) 
+            pos_tokens = pos_tokens.flatten(1, 3) # B, L, C
+            new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
+            checkpoint_model[pos_name] = new_pos_embed

test.ipynb

@@ -0,0 +1,424 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "metadata": {}
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/root/miniconda3/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "InternVideo2Config {\n",
+      "  \"_attn_implementation_autoset\": true,\n",
+      "  \"architectures\": [\n",
+      "    \"InternVideo2_CLIP_small\"\n",
+      "  ],\n",
+      "  \"auto_map\": {\n",
+      "    \"AutoConfig\": \"config.InternVideo2Config\",\n",
+      "    \"AutoModel\": \"modeling_internvideo2encoder.InternVideo2_CLIP_small\"\n",
+      "  },\n",
+      "  \"auto_resume\": false,\n",
+      "  \"batch_size\": 64,\n",
+      "  \"batch_size_test\": 4,\n",
+      "  \"best_key\": [\n",
+      "    \"msrvtt_1k_test_match\",\n",
+      "    \"t2v_r1\"\n",
+      "  ],\n",
+      "  \"compile_model\": false,\n",
+      "  \"criterion\": {\n",
+      "    \"clip_loss_ratio\": [\n",
+      "      1.0,\n",
+      "      1.0\n",
+      "    ],\n",
+      "    \"distill_final_features\": true,\n",
+      "    \"loss_weight\": {\n",
+      "      \"mlm\": 1.0,\n",
+      "      \"mvm\": 0.0,\n",
+      "      \"uta\": 0.0,\n",
+      "      \"vtc\": 1.0,\n",
+      "      \"vtm\": 1.0\n",
+      "    },\n",
+      "    \"mlm_masking_prob\": 0.5,\n",
+      "    \"vtm_hard_neg\": true\n",
+      "  },\n",
+      "  \"debug\": false,\n",
+      "  \"deep_fusion\": false,\n",
+      "  \"deepspeed\": {\n",
+      "    \"enable\": true,\n",
+      "    \"stage\": 1\n",
+      "  },\n",
+      "  \"delete_ds_optim_states\": true,\n",
+      "  \"device\": \"cuda\",\n",
+      "  \"dist_url\": \"env://\",\n",
+      "  \"evaluate\": false,\n",
+      "  \"evaluation\": {\n",
+      "    \"eval_frame_ensemble\": \"concat\",\n",
+      "    \"eval_offload\": true,\n",
+      "    \"eval_x_only\": false,\n",
+      "    \"k_test\": 128\n",
+      "  },\n",
+      "  \"gradient_checkpointing\": true,\n",
+      "  \"inputs\": {\n",
+      "    \"batch_size\": {\n",
+      "      \"image\": 64,\n",
+      "      \"video\": 64\n",
+      "    },\n",
+      "    \"batch_size_test\": {\n",
+      "      \"image\": 4,\n",
+      "      \"video\": 4\n",
+      "    },\n",
+      "    \"image_res\": 224,\n",
+      "    \"max_txt_l\": {\n",
+      "      \"image\": 32,\n",
+      "      \"video\": 32\n",
+      "    },\n",
+      "    \"video_input\": {\n",
+      "      \"num_frames\": 8,\n",
+      "      \"num_frames_test\": 8,\n",
+      "      \"random_aug\": false,\n",
+      "      \"sample_type\": \"middle\",\n",
+      "      \"sample_type_test\": \"middle\"\n",
+      "    }\n",
+      "  },\n",
+      "  \"jump_evaluate\": false,\n",
+      "  \"log_freq\": 100,\n",
+      "  \"max_txt_l\": 32,\n",
+      "  \"mode\": \"pt\",\n",
+      "  \"model\": {\n",
+      "    \"embed_dim\": 1024,\n",
+      "    \"find_unused_parameters\": false,\n",
+      "    \"freeze_text\": true,\n",
+      "    \"freeze_vision\": true,\n",
+      "    \"load_vision_ckpt_from_internvideo2_stage2\": false,\n",
+      "    \"model_cls\": \"InternVideo2_CLIP_small\",\n",
+      "    \"multimodal\": {\n",
+      "      \"enable\": true\n",
+      "    },\n",
+      "    \"open_text_projection\": false,\n",
+      "    \"open_vision_clip_projector\": true,\n",
+      "    \"temp\": 0.01,\n",
+      "    \"temp_min\": 0.01,\n",
+      "    \"text_encoder\": {\n",
+      "      \"embed_dim\": 512,\n",
+      "      \"image_cfg\": {\n",
+      "        \"image_size\": 224,\n",
+      "        \"model_name\": \"vit_b16\"\n",
+      "      },\n",
+      "      \"text_cfg\": {\n",
+      "        \"causal_masking\": true,\n",
+      "        \"context_length\": 77,\n",
+      "        \"dim\": 512,\n",
+      "        \"ffn_multiplier_per_layer\": 4.0,\n",
+      "        \"model_name\": \"base\",\n",
+      "        \"n_heads_per_layer\": 8,\n",
+      "        \"n_transformer_layers\": 12,\n",
+      "        \"norm_layer\": \"layer_norm_fp32\",\n",
+      "        \"vocab_size\": 49408\n",
+      "      }\n",
+      "    },\n",
+      "    \"vision_encoder\": {\n",
+      "      \"align_dim\": 512,\n",
+      "      \"attn_pool_num_heads\": 16,\n",
+      "      \"checkpoint_num\": 0,\n",
+      "      \"clip_embed_dim\": 768,\n",
+      "      \"depth\": 24,\n",
+      "      \"drop_cls_token\": false,\n",
+      "      \"drop_path_rate\": 0.0,\n",
+      "      \"embed_dim\": 1024,\n",
+      "      \"fused_mlp_heuristic\": 1,\n",
+      "      \"head_drop_path_rate\": 0.0,\n",
+      "      \"img_size\": 224,\n",
+      "      \"in_chans\": 3,\n",
+      "      \"init_values\": 0.1,\n",
+      "      \"layerscale_no_force_fp32\": true,\n",
+      "      \"mlp_ratio\": 4,\n",
+      "      \"name\": \"internvideo2_1B\",\n",
+      "      \"num_frames\": 8,\n",
+      "      \"num_heads\": 16,\n",
+      "      \"patch_size\": 14,\n",
+      "      \"qk_normalization\": true,\n",
+      "      \"qkv_bias\": false,\n",
+      "      \"sep_pos_embed\": false,\n",
+      "      \"tubelet_size\": 1,\n",
+      "      \"use_checkpoint\": false,\n",
+      "      \"use_flash_attn\": false,\n",
+      "      \"use_fused_mlp\": false,\n",
+      "      \"use_fused_rmsnorm\": false\n",
+      "    }\n",
+      "  },\n",
+      "  \"model_type\": \"internvideo2\",\n",
+      "  \"num_frames\": 8,\n",
+      "  \"num_frames_test\": 8,\n",
+      "  \"num_workers\": 6,\n",
+      "  \"optimizer\": {\n",
+      "    \"different_lr\": {\n",
+      "      \"enable\": false,\n",
+      "      \"lr\": 0.001,\n",
+      "      \"module_names\": []\n",
+      "    },\n",
+      "    \"lr\": 5e-05,\n",
+      "    \"max_grad_norm\": 3.0,\n",
+      "    \"opt\": \"adamW\",\n",
+      "    \"opt_betas\": [\n",
+      "      0.9,\n",
+      "      0.98\n",
+      "    ],\n",
+      "    \"weight_decay\": 0.05\n",
+      "  },\n",
+      "  \"output_dir\": null,\n",
+      "  \"pretrained_path\": \"\",\n",
+      "  \"resume\": false,\n",
+      "  \"save_ckpt_iter\": null,\n",
+      "  \"save_latest\": true,\n",
+      "  \"scheduler\": {\n",
+      "    \"epochs\": 10,\n",
+      "    \"min_lr_multi\": 0.01,\n",
+      "    \"sched\": \"cosine\",\n",
+      "    \"warmup_epochs\": 1\n",
+      "  },\n",
+      "  \"seed\": 42,\n",
+      "  \"test_file\": {\n",
+      "    \"didemo_ret_test\": \"available_corpus[\\\"didemo_ret_test\\\"]\",\n",
+      "    \"msrvtt_1k_test\": \"available_corpus[\\\"msrvtt_1k_test\\\"]\"\n",
+      "  },\n",
+      "  \"test_types\": [\n",
+      "    \"msrvtt_1k_test\",\n",
+      "    \"didemo_ret_test\"\n",
+      "  ],\n",
+      "  \"text_enc\": \"bert_large\",\n",
+      "  \"tokenizer\": null,\n",
+      "  \"torch_dtype\": \"float32\",\n",
+      "  \"train_file\": \"available_corpus[\\\"pretrain_example_data_1B\\\"]\",\n",
+      "  \"transformers_version\": \"4.51.3\",\n",
+      "  \"use_bf16\": true,\n",
+      "  \"use_flash_sdp\": false,\n",
+      "  \"use_half_precision\": false,\n",
+      "  \"use_mem_efficient_sdp\": false,\n",
+      "  \"wandb\": {\n",
+      "    \"enable\": false,\n",
+      "    \"entity\": \"opengvlab\",\n",
+      "    \"project\": \"InternVideo2-Stage2\"\n",
+      "  }\n",
+      "}\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import AutoConfig, AutoModel\n",
+    "config = AutoConfig.from_pretrained(\"/fs-computility/video/heyinan/iv2hf/\", trust_remote_code=True)\n",
+    "model = AutoModel.from_pretrained(\"/fs-computility/video/heyinan/iv2hf/\", trust_remote_code=True).to(config.device)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "metadata": {}
+   },
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import random\n",
+    "import io\n",
+    "import av\n",
+    "import cv2\n",
+    "import decord\n",
+    "import imageio\n",
+    "from decord import VideoReader\n",
+    "import torch\n",
+    "import numpy as np\n",
+    "import math\n",
+    "import torch.nn.functional as F\n",
+    "decord.bridge.set_bridge(\"torch\")\n",
+    "\n",
+    "\n",
+    "def get_frame_indices(num_frames, vlen, sample='rand', fix_start=None, input_fps=1, max_num_frames=-1, start=None, end=None):\n",
+    "    start_frame, end_frame = 0, vlen\n",
+    "    if start is not None:\n",
+    "        start_frame = max(start_frame,int(start * input_fps))\n",
+    "    if end is not None:\n",
+    "        end_frame = min(end_frame,int(end * input_fps))\n",
+    "\n",
+    "    # Ensure start_frame is less than end_frame\n",
+    "    if start_frame >= end_frame:\n",
+    "        raise ValueError(\"Start frame index must be less than end frame index\")\n",
+    "\n",
+    "    # Calculate the length of the clip in frames\n",
+    "    clip_length = end_frame - start_frame\n",
+    "\n",
+    "    if sample in [\"rand\", \"middle\"]:  # uniform sampling\n",
+    "        acc_samples = min(num_frames, clip_length)\n",
+    "        # split the clip into `acc_samples` intervals, and sample from each interval.\n",
+    "        intervals = np.linspace(start=start_frame, stop=end_frame, num=acc_samples + 1).astype(int)\n",
+    "        ranges = []\n",
+    "        for idx, interv in enumerate(intervals[:-1]):\n",
+    "            ranges.append((interv, intervals[idx + 1] - 1))\n",
+    "        if sample == 'rand':\n",
+    "            try:\n",
+    "                frame_indices = [random.choice(range(x[0], x[1] + 1)) for x in ranges]\n",
+    "            except:\n",
+    "                frame_indices = np.random.permutation(clip_length)[:acc_samples] + start_frame\n",
+    "                frame_indices.sort()\n",
+    "                frame_indices = list(frame_indices)\n",
+    "        elif fix_start is not None:\n",
+    "            frame_indices = [x[0] + fix_start for x in ranges]\n",
+    "        elif sample == 'middle':\n",
+    "            frame_indices = [(x[0] + x[1]) // 2 for x in ranges]\n",
+    "        else:\n",
+    "            raise NotImplementedError\n",
+    "\n",
+    "        if len(frame_indices) < num_frames:  # padded with last frame\n",
+    "            padded_frame_indices = [frame_indices[-1]] * num_frames\n",
+    "            padded_frame_indices[:len(frame_indices)] = frame_indices\n",
+    "            frame_indices = padded_frame_indices\n",
+    "    elif \"fps\" in sample:  # fps0.5, sequentially sample frames at 0.5 fps\n",
+    "        output_fps = float(sample[3:])\n",
+    "        duration = float(clip_length) / input_fps\n",
+    "        delta = 1 / output_fps  # gap between frames, this is also the clip length each frame represents\n",
+    "        frame_seconds = np.arange(0 + delta / 2, duration + delta / 2, delta)\n",
+    "        frame_indices = np.around(frame_seconds * input_fps).astype(int) + start_frame\n",
+    "        frame_indices = [e for e in frame_indices if e < end_frame]\n",
+    "        if max_num_frames > 0 and len(frame_indices) > max_num_frames:\n",
+    "            frame_indices = frame_indices[:max_num_frames]\n",
+    "            # frame_indices = np.linspace(0 + delta / 2, duration + delta / 2, endpoint=False, num=max_num_frames)\n",
+    "    else:\n",
+    "        raise ValueError\n",
+    "    return frame_indices\n",
+    "\n",
+    "def read_frames_decord(\n",
+    "        video_path, num_frames, sample='middle', fix_start=None, \n",
+    "        max_num_frames=-1, client=None, trimmed30=False, start=None, end=None\n",
+    "    ):\n",
+    "    num_threads = 1 if video_path.endswith('.webm') else 0 # make ssv2 happy\n",
+    "\n",
+    "    video_reader = VideoReader(video_path, num_threads=num_threads)\n",
+    "    vlen = len(video_reader)\n",
+    " \n",
+    "    fps = video_reader.get_avg_fps()\n",
+    "    duration = vlen / float(fps)\n",
+    "\n",
+    "    frame_indices = get_frame_indices(\n",
+    "        num_frames, vlen, sample=sample, fix_start=fix_start,\n",
+    "        input_fps=fps, max_num_frames=max_num_frames, start=start, end=end\n",
+    "    )\n",
+    "\n",
+    "    frames = video_reader.get_batch(frame_indices)  # (T, H, W, C), torch.uint8\n",
+    "    frames = frames.permute(0, 3, 1, 2)  # (T, C, H, W), torch.uint8\n",
+    "    return frames, frame_indices, duration"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "metadata": {}
+   },
+   "outputs": [],
+   "source": [
+    "def get_text_feature(model, texts):\n",
+    "    text_input = model.tokenizer(texts).to(model.device)\n",
+    "    text_features = model.encode_text(text_input)\n",
+    "    return text_features\n",
+    "    \n",
+    "def get_similarity(video_feature, text_feature):\n",
+    "    video_feature = F.normalize(video_feature, dim=-1)\n",
+    "    text_feature = F.normalize(text_feature, dim=-1)\n",
+    "    sim_matrix = text_feature @ video_feature.T\n",
+    "    return sim_matrix"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {
+    "metadata": {}
+   },
+   "outputs": [],
+   "source": [
+    "def get_top_videos(model, text_features, video_features, video_paths, texts):\n",
+    "    # text_features = get_text_feature(texts)\n",
+    "\n",
+    "    video_features = F.normalize(video_features, dim=-1)\n",
+    "    text_features = F.normalize(text_features, dim=-1)\n",
+    "\n",
+    "    # print(text_features.shape, video_features.shape)\n",
+    "    sim_matrix = text_features @ video_features.T\n",
+    "    # print(sim_matrix.shape)\n",
+    "\n",
+    "    top_k = 5\n",
+    "    sim_matrix_top_k = torch.topk(sim_matrix, top_k, dim=1)[1]\n",
+    "    softmax_sim_matrix = F.softmax(sim_matrix, dim=1)\n",
+    "\n",
+    "    retrieval_infos = {}\n",
+    "    for i in range(len(sim_matrix_top_k)):\n",
+    "        print(\"\\n\",texts[i])\n",
+    "        retrieval_infos[texts[i]] = []\n",
+    "        for j in range(top_k):\n",
+    "            print(\"top\", j+1, \":\", video_paths[sim_matrix_top_k[i][j]], \"~prob:\", sim_matrix[i][sim_matrix_top_k[i][j]].item())\n",
+    "            retrieval_infos[texts[i]].append({\"video\":  video_paths[sim_matrix_top_k[i][j]], \"prob\": sim_matrix[i][sim_matrix_top_k[i][j]].item(), \"rank\": j+1})\n",
+    "    return retrieval_infos"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "metadata": {}
+   },
+   "outputs": [],
+   "source": [
+    "if __name__==\"__main__\":\n",
+    "    video_features = []\n",
+    "    demo_videos = [\"video-scene-00030.mp4\",\"video-scene-00031.mp4\",\"xinhuashe_test_video/video-scene-00032.mp4\",\"xinhuashe_test_video/video-scene-00033.mp4\",\"video-scene-00034.mp4\"]\n",
+    "    texts = ['a person talking', 'a logo', 'a building']\n",
+    "    for video_path in demo_videos:\n",
+    "        frames, frame_indices, video_duration = read_frames_decord(video_path,8)\n",
+    "        frames = model.transform(frames).unsqueeze(0).to(model.device)\n",
+    "        # 获得视频特征\n",
+    "        with torch.no_grad():\n",
+    "            video_feature = model.encode_vision(frames, test=True)\n",
+    "            video_features.append(video_feature)\n",
+    "    \n",
+    "    # # 获得文本特征\n",
+    "    text_features = get_text_feature(model, texts)\n",
+    "    video_features = torch.cat(video_features, dim=0).to(text_features.dtype).to(config.device)\n",
+    "    results = get_top_videos(model, text_features, video_features, demo_videos, texts)\n",
+    "\n",
+    "\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}