multimodal/tools/add_vg_to_blip2_data.py

import os
import shutil
import glob
import random
from pprint import pprint

DIR_VG = "/gpfs/u/home/LMCG/LMCGljnn/scratch-shared/junyan/raw/vg_0826"
DIR = "/gpfs/u/home/LMCG/LMCGljnn/scratch-shared/junyan/raw/blip2_all_data_ground"
OUT_DIR = "/gpfs/u/home/LMCG/LMCGljnn/scratch-shared/junyan/raw/blip2_all_data_ground_with_vg_0826"


if __name__ == "__main__":
    os.makedirs(OUT_DIR, exist_ok=True)
    blip2_tars = glob.glob(os.path.join(DIR, "*.tar"))
    vg_tars = glob.glob(os.path.join(DIR_VG, "*", "*.tar"))
    tars = []
    tars.extend(blip2_tars)
    tars.extend(vg_tars)
    print(len(tars))
    pprint(tars[:20])
    pprint(tars[-20:])
    for i, tar in enumerate(tars):
        dst = os.path.join(OUT_DIR, f"{str(i).zfill(6)}.tar")
        # print(tar, dst)
        os.symlink(tar, dst)