Update app.py

app.py

@@ -125,6 +125,133 @@ def add_new_eval(
     print("success update", model_name)
     return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)
 
+def add_new_eval_i2v(
+    input_file,
+    model_name_textbox: str,
+    revision_name_textbox: str,
+    model_link: str,
+    team_name: str,
+    contact_email: str,
+    model_publish: str,
+    model_resolution: str,
+    model_fps: str,
+    model_frame: str,
+    model_video_length: str,
+    model_checkpoint: str,
+    model_commit_id: str,
+    model_video_format: str
+):
+    COLNAME2KEY={
+        "Video-Text Camera Motion":"camera_motion",
+        "Video-Image Subject Consistency": "i2v_subject",
+        "Video-Image Background Consistency": "i2v_background",
+        "Subject Consistency": "subject_consistency",
+        "Background Consistency": "background_consistency",
+        "Motion Smoothness": "motion_smoothness",
+        "Dynamic Degree": "dynamic_degree",
+        "Aesthetic Quality": "aesthetic_quality",
+        "Imaging Quality": "imaging_quality",
+        "Temporal Flickering": "temporal_flickering"
+        }
+    if input_file is None:
+        return "Error! Empty file!"
+    if  model_link == '' or model_name_textbox == '' or contact_email == '':
+        return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)
+    
+    upload_content = input_file
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    submission_repo.git_pull()
+    filename = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+    
+    now = datetime.datetime.now()
+    update_time = now.strftime("%Y-%m-%d")  # Capture update time
+    with open(f'{SUBMISSION_NAME}/{filename}.zip','wb') as f:
+        f.write(input_file)
+    # shutil.copyfile(CSV_DIR, os.path.join(SUBMISSION_NAME, f"{input_file}"))
+
+    csv_data = pd.read_csv(I2V_DIR)
+
+    if revision_name_textbox == '':
+        col = csv_data.shape[0]
+        model_name = model_name_textbox.replace(',',' ')
+    else:
+        model_name = revision_name_textbox.replace(',',' ')
+        model_name_list = csv_data['Model Name (clickable)']
+        name_list = [name.split(']')[0][1:] for name in model_name_list]
+        if revision_name_textbox not in name_list:
+            col = csv_data.shape[0]
+        else:
+            col = name_list.index(revision_name_textbox)    
+    if model_link == '':
+        model_name = model_name  # no url
+    else:
+        model_name = '[' + model_name + '](' + model_link + ')'
+
+    os.makedirs(filename, exist_ok=True)
+    with zipfile.ZipFile(io.BytesIO(input_file), 'r') as zip_ref:
+        zip_ref.extractall(filename)
+
+    upload_data = {}
+    for file in os.listdir(filename):
+        if file.startswith('.') or file.startswith('__'):
+            print(f"Skip the file: {file}")
+            continue
+        cur_file = os.path.join(filename, file)
+        if os.path.isdir(cur_file):
+            for subfile in os.listdir(cur_file):
+                if subfile.endswith(".json"):
+                    with open(os.path.join(cur_file, subfile)) as ff:
+                        cur_json = json.load(ff)
+                        print(file, type(cur_json))
+                        if isinstance(cur_json, dict):
+                            print(cur_json.keys())
+                            for key in cur_json:
+                                upload_data[key] = cur_json[key][0]
+                                print(f"{key}:{cur_json[key][0]}")
+        elif cur_file.endswith('json'):
+            with open(cur_file) as ff:
+                cur_json = json.load(ff)
+                print(file, type(cur_json))
+                if isinstance(cur_json, dict):
+                    print(cur_json.keys())
+                    for key in cur_json:
+                        upload_data[key] = cur_json[key][0]
+                        print(f"{key}:{cur_json[key][0]}")
+    # add new data
+    new_data = [model_name]
+    print('upload_data:', upload_data)
+    I2V_HEAD= ["Video-Text Camera Motion",
+    "Video-Image Subject Consistency",
+    "Video-Image Background Consistency",
+    "Subject Consistency",
+    "Background Consistency",
+    "Temporal Flickering",
+    "Motion Smoothness",
+    "Dynamic Degree",
+    "Aesthetic Quality",
+    "Imaging Quality" ]
+    for key in I2V_HEAD :
+        sub_key = COLNAME2KEY[key]
+        if sub_key in upload_data:
+            new_data.append(upload_data[sub_key])
+        else:
+            new_data.append(0)
+    if team_name =='' or 'vbench' in team_name.lower():
+        new_data.append("User Upload")
+    else:
+        new_data.append(team_name)
+
+    new_data.append(contact_email.replace(',',' and ')) # Add contact email [private]
+    new_data.append(update_time)  # Add the update time
+
+    csv_data.loc[col] = new_data
+    csv_data = csv_data.to_csv(I2V_DIR , index=False)
+    with open(INFO_DIR,'a') as f:
+        f.write(f"{model_name}\t{update_time}\t{model_publish}\t{model_resolution}\t{model_fps}\t{model_frame}\t{model_video_length}\t{model_checkpoint}\t{model_commit_id}\t{model_video_format}\n")
+    submission_repo.push_to_hub()
+    print("success update", model_name)
+    return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)
+
 def get_normalized_df(df):
     # final_score = df.drop('name', axis=1).sum(axis=1)
     # df.insert(1, 'Overall Score', final_score)
@@ -137,7 +264,7 @@ def get_normalized_df(df):
 
 def get_normalized_i2v_df(df):
     normalize_df = df.copy().fillna(0.0)
-    for column in normalize_df.columns[1:]:
+    for column in normalize_df.columns[1:-3]:
         min_val = NORMALIZE_DIC_I2V[column]['Min']
         max_val = NORMALIZE_DIC_I2V[column]['Max']
         normalize_df[column] = (normalize_df[column] - min_val) / (max_val - min_val)
@@ -208,7 +335,7 @@ def get_final_score(df, selected_columns):
 def get_final_score_i2v(df, selected_columns):
     normalize_df = get_normalized_i2v_df(df)
     #final_score = normalize_df.drop('name', axis=1).sum(axis=1)
-    for name in normalize_df.drop('Model Name (clickable)', axis=1).drop('Video-Text Camera Motion', axis=1):
+    for name in normalize_df.drop('Model Name (clickable)', axis=1).drop('Video-Text Camera Motion', axis=1).drop('Source', axis=1).drop('Mail', axis=1).drop('Date',axis=1):
         normalize_df[name] = normalize_df[name]*DIM_WEIGHT_I2V[name]
     quality_score = normalize_df[I2V_QUALITY_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_QUALITY_LIST])
     i2v_score = normalize_df[I2V_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_LIST ])
@@ -603,7 +730,7 @@ with block:
             gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text")
         
         # table submission 
-        with gr.TabItem("🚀 Submit here! ", elem_id="mvbench-tab-table", id=6):
+        with gr.TabItem("🚀 [T2V]Submit here! ", elem_id="mvbench-tab-table", id=6):
             gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text")
 
             with gr.Row():
@@ -672,6 +799,77 @@ with block:
                     ],
                     outputs=[submit_button, submit_succ_button, fail_textbox]
                 )
+                
+        with gr.TabItem("🚀 [I2V]Submit here! ", elem_id="mvbench-i2v-tab-table", id=7):
+            gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your i2v model evaluation json file here!", elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown("Here is a required field", elem_classes="markdown-text")
+            with gr.Row():
+                with gr.Column():
+                    model_name_textbox_i2v = gr.Textbox(
+                        label="Model name", placeholder="Required field"
+                        )
+                    revision_name_textbox_i2v = gr.Textbox(
+                        label="Revision Model Name(Optional)", placeholder="If you need to update the previous results, please fill in this line"
+                    )
+
+                with gr.Column():
+                    model_link_i2v = gr.Textbox(
+                        label="Project Page/Paper Link/Github/HuggingFace Repo", placeholder="Required field. If filling in the wrong information, your results may be removed."
+                    )
+                    team_name_i2v = gr.Textbox(
+                        label="Your Team Name(If left blank, it will be user upload)", placeholder="User Upload"
+                    )
+                    contact_email_i2v = gr.Textbox(
+                        label="E-Mail(Will not be displayed)", placeholder="Required field"
+                    )
+            with gr.Row():
+                gr.Markdown("The following is optional and will be synced to [GitHub] (https://github.com/Vchitect/VBench/tree/master/sampled_videos#what-are-the-details-of-the-video-generation-models)", elem_classes="markdown-text")
+            with gr.Row():
+                    release_time_i2v = gr.Textbox(label="Time of Publish", placeholder="1970-01-01")
+                    model_resolution_i2v = gr.Textbox(label="resolution", placeholder="Width x Height")
+                    model_fps_i2v = gr.Textbox(label="model fps", placeholder="FPS(int)")
+                    model_frame_i2v = gr.Textbox(label="model frame count", placeholder="INT")
+                    model_video_length_i2v = gr.Textbox(label="model video length", placeholder="float(2.0)")
+                    model_checkpoint_i2v = gr.Textbox(label="model checkpoint", placeholder="optional")
+                    model_commit_id_i2v = gr.Textbox(label="github commit id", placeholder='main')
+                    model_video_format_i2v = gr.Textbox(label="pipeline format", placeholder='mp4')
+            with gr.Column():
+                input_file_i2v = gr.components.File(label = "Click to Upload a ZIP File", file_count="single", type='binary')
+                submit_button_i2v = gr.Button("Submit Eval")
+                submit_succ_button_i2v = gr.Markdown("Submit Success! Please press refresh and return to LeaderBoard!", visible=False)
+                fail_textbox_i2v = gr.Markdown('<span style="color:red;">Please ensure that the `Model Name`, `Project Page`, and `Email` are filled in correctly.</span>', elem_classes="markdown-text",visible=False)
+                
+    
+                submission_result_i2v = gr.Markdown()
+                submit_button_i2v.click(
+                    add_new_eval_i2v,
+                    inputs = [
+                        input_file_i2v,
+                        model_name_textbox_i2v,
+                        revision_name_textbox_i2v,
+                        model_link_i2v,
+                        team_name_i2v,
+                        contact_email_i2v,
+                        release_time_i2v,
+                        model_resolution_i2v,
+                        model_fps_i2v,
+                        model_frame_i2v,
+                        model_video_length_i2v,
+                        model_checkpoint_i2v,
+                        model_commit_id_i2v,
+                        model_video_format_i2v
+                    ],
+                    outputs=[submit_button_i2v, submit_succ_button_i2v, fail_textbox_i2v]
+                )
+
 
 
     def refresh_data():