update

app.py

@@ -101,4 +101,4 @@ if __name__ == "__main__":
     
     elo_results_file, leaderboard_table_file = load_elo_results(elo_results_dir)
     demo = build_combine_demo(models, elo_results_file, leaderboard_table_file)
-    demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH)
+    demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH)

arena_elo/elo_rating/elo_analysis.py

@@ -171,20 +171,8 @@ def visualize_pairwise_win_fraction(battles, model_order):
         xaxis_side="top",
         title_y=0.07,
         title_x=0.5,
-        # xaxis=dict(
-        #     tickfont=dict(size=16),
-        #     title=dict(font=dict(size=16)),
-        # ),
-        # yaxis=dict(
-        #     tickfont=dict(size=16),
-        #     title=dict(font=dict(size=16)),
-        # ),
     )
     fig.update_traces(
-        # textfont=dict(size=16),
-        # colorbar=dict(
-        #     title=dict(font=dict(size=16))
-        # ),
         hovertemplate="Model A: %{y}<br>Model B: %{x}<br>Fraction of A Wins: %{z}<extra></extra>"
     )
 
@@ -208,20 +196,8 @@ def visualize_battle_count(battles, model_order):
         xaxis_side="top",
         title_y=0.07,
         title_x=0.5,
-        # xaxis=dict(
-        #     tickfont=dict(size=16),
-        #     title=dict(font=dict(size=16)),
-        # ),
-        # yaxis=dict(
-        #     tickfont=dict(size=16),
-        #     title=dict(font=dict(size=16)),
-        # ),
     )
     fig.update_traces(
-        # textfont=dict(size=16),
-        # colorbar=dict(
-        #     title=dict(font=dict(size=16))
-        # ),
         hovertemplate="Model A: %{y}<br>Model B: %{x}<br>Count: %{z}<extra></extra>"
     )
     return fig
@@ -239,14 +215,6 @@ def visualize_average_win_rate(battles, limit_show_number):
     )
     fig.update_layout(
         yaxis_title="Average Win Rate", xaxis_title="Model", showlegend=False,
-        # xaxis=dict(
-        #     tickfont=dict(size=16),
-        #     title=dict(font=dict(size=16)),
-        # ),
-        # yaxis=dict(
-        #     tickfont=dict(size=16),
-        #     title=dict(font=dict(size=16)),
-        # ),
     )
     fig.update_traces(textfont_size=16)
     return fig
@@ -278,16 +246,7 @@ def visualize_bootstrap_elo_rating(df, df_final, limit_show_number):
         height=500,
         width=700,
     )
-    fig.update_layout(xaxis_title="Model", yaxis_title="Rating",
-        # xaxis=dict(
-        #     tickfont=dict(size=16),
-        #     title=dict(font=dict(size=16)),
-        # ),
-        # yaxis=dict(
-        #     tickfont=dict(size=16),
-        #     title=dict(font=dict(size=16)),
-        # ),
-    )
+    fig.update_layout(xaxis_title="Model", yaxis_title="Rating")
     fig.update_traces(textfont_size=16)
     return fig
 
@@ -406,9 +365,6 @@ if __name__ == "__main__":
         battles_with_enough_battles = battles[
             ~battles["model_a"].isin(to_remove_models) & ~battles["model_b"].isin(to_remove_models)
         ]
-        # battles_with_enough_battles = [
-        #     battle for battle in battles if battle["model_a"] not in to_remove_models and battle["model_b"] not in to_remove_models
-        # ]
         print(f"Remove models with less than {args.min_num_battles_per_model} battles: {to_remove_models}")
         print(f"Number of battles: {len(battles)} -> {len(battles_with_enough_battles)}")
         battles = battles_with_enough_battles

arena_elo/elo_rating/generate_leaderboard.py

@@ -43,10 +43,6 @@ def main(
             model_info[model]["Arena Elo rating (full)"] = full_leaderboard_data.loc[model, "rating"]
         else:
             model_info[model]["Arena Elo rating (full)"] = 0
-        # if model in anony_leaderboard_data.index:
-        #     model_info[model]["Arena Elo rating"] = anony_leaderboard_data.loc[model, "rating"]
-        # else:
-        #     model_info[model]["Arena Elo rating"] = 0
 
     final_model_info = {}
     for model in model_info:

arena_elo/results/latest/video_generation_leaderboard.csv

@@ -1,11 +1,11 @@
 key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-StableVideoDiffusion,StableVideoDiffusion,1143.2191298083312,1147.5193110801638,stable-video-diffusion-nc-community,Stability AI,https://fal.ai/models/fal-ai/fast-svd/text-to-video/api
+StableVideoDiffusion,StableVideoDiffusion,1143.2191298083312,1147.5193110801638,SVD-nc-community,Stability AI,https://fal.ai/models/fal-ai/fast-svd/text-to-video/api
 T2VTurbo,T2VTurbo,1092.4551860072625,1093.43771637836,cc-by-nc-4.0,"University of California, Santa Barbara",https://huggingface.co/jiachenli-ucsb/T2V-Turbo-VC2
 VideoCrafter2,VideoCrafter2,1076.7797371219958,1079.3282160297892,Apache 2.0,Tencent AI Lab,https://ailab-cvc.github.io/videocrafter2/
 AnimateDiff,AnimateDiff,1071.9888858847867,1073.0390578256142,creativeml-openrail-m,"The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",https://fal.ai/models/fast-animatediff-t2v
-CogVideoX,CogVideoX,1046.417835182279,1048.070311411077,N/A,N/A,N/A
+CogVideoX,CogVideoX,1046.417835182279,1048.070311411077,CogVideoX License,Tsinghua University,https://huggingface.co/THUDM/CogVideoX-2b
 LaVie,LaVie,999.6270938177271,1001.6816790561326,Apache 2.0,Shanghai AI Lab,https://github.com/Vchitect/LaVie
-OpenSora12,OpenSora12,924.0203683794037,903.7447677401199,N/A,N/A,N/A
+OpenSora12,OpenSora12,924.0203683794037,903.7447677401199,Apache 2.0,HPC-AI Tech,https://github.com/hpcaitech/Open-Sora
 OpenSora,OpenSora,916.0498767986071,917.9961732110512,Apache 2.0,HPC-AI Tech,https://github.com/hpcaitech/Open-Sora
 ModelScope,ModelScope,865.3398542588171,868.9309810684535,cc-by-nc-4.0,Alibaba Group,https://arxiv.org/abs/2308.06571
 AnimateDiffTurbo,AnimateDiffTurbo,864.1020327407906,866.2517861992403,creativeml-openrail-m,"The Chinese University of Hong Kong, Shanghai AI Lab, Stanford University",https://fal.ai/models/fast-animatediff-t2v-turbo

arena_elo/video_generation_model_info.json

@@ -24,11 +24,21 @@
         "License": "cc-by-nc-4.0",
         "Organization": "Alibaba Group"
     },
+    "OpenSora12": {
+        "Link": "https://github.com/hpcaitech/Open-Sora",
+        "License": "Apache 2.0",
+        "Organization": "HPC-AI Tech"
+    },
     "OpenSora": {
         "Link": "https://github.com/hpcaitech/Open-Sora",
         "License": "Apache 2.0",
         "Organization": "HPC-AI Tech"
     },
+    "CogVideoX": {
+        "Link": "https://huggingface.co/THUDM/CogVideoX-2b",
+        "License": "CogVideoX LICENSE",
+        "Organization": "THUDM"
+    },
     "StableVideoDiffusion": {
         "Link": "https://fal.ai/models/fal-ai/fast-svd/text-to-video/api",
         "License": "SVD-nc-community",