update

app.py

@@ -24,7 +24,11 @@ from src.display.utils import (
     fields,
     WeightType,
     Precision,
-    NUMERIC_INTERVALS
+    NUMERIC_INTERVALS,
+    QUOTACOLS,
+    QUOTATYPES,
+    AutoEvalColumnQuota,
+    BENCHMARK_QUOTACOLS
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, DYNAMIC_INFO_REPO, DYNAMIC_INFO_FILE_PATH, DYNAMIC_INFO_PATH, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
@@ -32,10 +36,10 @@ from src.submission.submit import add_new_eval
 from src.scripts.update_all_request_files import update_dynamic_files
 from src.tools.collections import update_collections
 from src.tools.datastatics import get_statics
-from src.tools.plots import (
-    create_plot_df,
-    create_scores_df,
-)
+#from src.tools.plots import (
+#    create_plot_df,
+#    create_scores_df,
+#)
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID, token=TOKEN)
@@ -60,17 +64,18 @@ def init_space():
         restart_space()
 
     raw_data, original_df = get_leaderboard_df(
-    #leaderboard_df = get_leaderboard_df(
         results_path=EVAL_RESULTS_PATH,
         requests_path=EVAL_REQUESTS_PATH,
         dynamic_path=DYNAMIC_INFO_FILE_PATH,
-        cols=COLS,
-        benchmark_cols=BENCHMARK_COLS
+        #cols=COLS,
+        #benchmark_cols=BENCHMARK_COLS,
+        cols=QUOTACOLS,
+        benchmark_cols=BENCHMARK_QUOTACOLS
     )
     update_collections(original_df.copy())
     leaderboard_df = original_df.copy()
 
-    plot_df = create_plot_df(create_scores_df(raw_data))
+    #plot_df = create_plot_df(create_scores_df(raw_data))
 
     (
         finished_eval_queue_df,
@@ -78,9 +83,10 @@ def init_space():
         pending_eval_queue_df,
     ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
-    return leaderboard_df, original_df, plot_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
+    #return leaderboard_df, original_df, plot_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
+    return leaderboard_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
 
-leaderboard_df, original_df, plot_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
+leaderboard_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
     #return leaderboard_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
 
 #leaderboard_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()

src/about.py

@@ -22,6 +22,8 @@ class Tasks(Enum):
     SAT = Task("SAT", "overall", "SAT")
     egoplan_bench2 = Task("egoplan_bench2", "overall", "egoplan_bench2")
     erqa = Task("erqa", "overall", "erqa")
+
+class Quotas(Enum):
     Perception = Task("Perception", "overall", "Perception")
     SpatialReasoning = Task("SpatialReasoning", "overall", "SpatialReasoning")
     Prediction = Task("Prediction", "overall", "Prediction")

src/display/utils.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 import pandas as pd
 
-from src.about import Tasks
+from src.about import Tasks,Quotas
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -44,6 +44,30 @@ auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
+
+## Leaderboard columns
+auto_eval_column_quota_dict = []
+# Init
+auto_eval_column_quota_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_quota_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+#Scores
+auto_eval_column_quota_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+for task in Quotas:
+    auto_eval_column_quota_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+# Model information
+auto_eval_column_quota_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+auto_eval_column_quota_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+auto_eval_column_quota_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+auto_eval_column_quota_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+auto_eval_column_quota_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_quota_dict.append(["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, hidden=True)])
+auto_eval_column_quota_dict.append(["moe", ColumnContent, ColumnContent("MoE", "bool", False, hidden=True)])
+# Dummy column for the search bar (hidden by the custom CSS)
+auto_eval_column_quota_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
+
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumnQuota = make_dataclass("AutoEvalColumnQuota", auto_eval_column_quota_dict, frozen=True)
+
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
@@ -116,10 +140,14 @@ class Precision(Enum):
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
 
+QUOTACOLS = [c.name for c in fields(AutoEvalColumnQuota) if not c.hidden]
+QUOTATYPES = [c.type for c in fields(AutoEvalColumnQuota) if not c.hidden]
+
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
 BENCHMARK_COLS = [t.value.col_name for t in Tasks]
+BENCHMARK_QUOTACOLS = [t.value.col_name for t in Quotas]
 
 NUMERIC_INTERVALS = {
     "?": pd.Interval(-1, 0, closed="right"),