Update space

app.py

@@ -104,7 +104,8 @@ def init_leaderboard(dataframe):
 # model_result_path = "./src/results/models_2024-10-08-03:25:44.801310.jsonl"
 # model_result_path = "./src/results/models_2024-10-08-17:39:21.001582.jsonl"
 # model_result_path = "./src/results/models_2024-10-09-05:17:38.810960.json"
-model_result_path = "./src/results/models_2024-10-09-06:22:21.122422.json"
+# model_result_path = "./src/results/models_2024-10-09-06:22:21.122422.json"
+model_result_path = "./src/results/models_2024-10-10-06:18:54.263527.json"
 # model_leaderboard_df = get_model_leaderboard_df(model_result_path)
 
 
@@ -131,17 +132,33 @@ with demo:
     gr.HTML(TITLE)
     gr.HTML(SUB_TITLE)
     gr.HTML(EXTERNAL_LINKS)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+    # gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+    # gr.HTML('<p style="font-size:15px;">This is a larger text using HTML in Markdown.</p>')
+    INTRODUCTION_TEXT_FONT_SIZE = 16
+    INTRODUCTION_TEXT = (
+        f'<p style="font-size:{INTRODUCTION_TEXT_FONT_SIZE}px;">'
+        '<strong>Decentralized Arena</strong> automates,  scales, and accelerates "<a href="https://lmarena.ai/">Chatbot Arena</a>" '
+        'for large language model (LLM) evaluation across diverse, fine-grained dimensions, '
+        'such as mathematics (algebra, geometry, probability), logical reasoning, social reasoning, biology, chemistry, and more'
+        'The evaluation is decentralized and democratic, with all participating LLMs assessing each other to ensure unbiased and fair results '
+        'With a 95\% correlation to Chatbot Arena\'s overall rankings, the system is fully transparent and reproducible.'
+        '</p>'
+        f'<p style="font-size:{INTRODUCTION_TEXT_FONT_SIZE}px;">'
+        'We actively invite <b>model developers</b> to participate and expedite their benchmarking efforts '
+        'and encourage <b>data stakeholders</b> to freely define and evaluate dimensions of interest for their own objectives.'
+        '</p>'
+    )
+    gr.HTML(INTRODUCTION_TEXT)
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         
         with gr.TabItem("🏅 Overview", elem_id="llm-benchmark-tab-table", id=0):
 
             DESCRIPTION_TEXT = """
-            Total #models: 53 (Last updated: 2024-10-08)
+            Total #models: 53 (Last updated: 2024-10-09)
             
-            This page provids a comprehensive overview of model ranks across various dimensions. Models are sorted based on their averaged rank across all dimensions. 
-            (Some missing values are due to the slow or problemtic model responses, and we will update the leaderboard once we have the complete results.)
+            This page prvovides a comprehensive overview of model ranks across various dimensions, based on their averaged ranks. 
+            (Missing values are due to the slow or problemtic model responses, which will be fixed soom.)
             """
             gr.Markdown(DESCRIPTION_TEXT, elem_classes="markdown-text")
 
@@ -158,6 +175,7 @@ with demo:
                         AutoEvalColumn.rank_reason_logical.name,
                         AutoEvalColumn.rank_reason_social.name,
                         AutoEvalColumn.rank_chemistry.name,
+                        AutoEvalColumn.rank_cpp.name,
                         ],
                     rank_col=[],
                 )
@@ -374,19 +392,31 @@ with demo:
             """
             gr.Markdown(CURRENT_TEXT, elem_classes="markdown-text")
             
-            with gr.TabItem("🐍 Python", elem_id="python_subtab", id=0, elem_classes="subtab"):   
-                CURRENT_TEXT = """
-                # Coming soon!
-                """
-                gr.Markdown(CURRENT_TEXT, elem_classes="markdown-text")
+            with gr.TabItem("➕ C++", elem_id="cpp_subtab", id=0, elem_classes="subtab"):   
+                                
+                leaderboard = overall_leaderboard(
+                    get_model_leaderboard_df(
+                        model_result_path,
+                        benchmark_cols=[
+                            AutoEvalColumn.rank_cpp.name,
+                            AutoEvalColumn.model.name,
+                            AutoEvalColumn.score_cpp.name,
+                            # AutoEvalColumn.sd_cpp.name,
+                            AutoEvalColumn.license.name,
+                            AutoEvalColumn.organization.name,
+                            AutoEvalColumn.knowledge_cutoff.name,
+                            ],
+                        rank_col=[AutoEvalColumn.rank_cpp.name],
+                    )
+                )
 
-            with gr.TabItem("☕ Java", elem_id="java_subtab", id=1, elem_classes="subtab"):   
+            with gr.TabItem("🐍 Python", elem_id="python_subtab", id=1, elem_classes="subtab"):   
                 CURRENT_TEXT = """
                 # Coming soon!
                 """
                 gr.Markdown(CURRENT_TEXT, elem_classes="markdown-text")
 
-            with gr.TabItem("➕ C++", elem_id="cpp_subtab", id=2, elem_classes="subtab"):   
+            with gr.TabItem("☕ Java", elem_id="java_subtab", id=2, elem_classes="subtab"):   
                 CURRENT_TEXT = """
                 # Coming soon!
                 """
@@ -395,6 +425,7 @@ with demo:
 
 
 
+
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=6):
             ABOUT_TEXT = """
             # About Us

src/display/utils.py

@@ -89,6 +89,10 @@ auto_eval_column_dict.append(["score_chemistry", ColumnContent, field(default_fa
 auto_eval_column_dict.append(["sd_chemistry", ColumnContent, field(default_factory=lambda: ColumnContent("Std dev (Chemistry)", "number", True))])
 auto_eval_column_dict.append(["rank_chemistry", ColumnContent, field(default_factory=lambda: ColumnContent("Rank (Chemistry)", "number", True))])
 
+auto_eval_column_dict.append(["score_cpp", ColumnContent, field(default_factory=lambda: ColumnContent("Score (C++)", "number", True))])
+auto_eval_column_dict.append(["sd_cpp", ColumnContent, field(default_factory=lambda: ColumnContent("Std dev (C++)", "number", True))])
+auto_eval_column_dict.append(["rank_cpp", ColumnContent, field(default_factory=lambda: ColumnContent("Rank (C++)", "number", True))])
+
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, field(default_factory=lambda: ColumnContent(task.value.col_name, "number", True))])
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, field(default_factory=lambda: ColumnContent("T", "str", True, never_hidden=True))])

src/leaderboard/read_evals.py

@@ -189,6 +189,10 @@ class ModelResult:
             AutoEvalColumn.sd_chemistry.name: self.results.get("Chemistry").get("Standard Deviation", None) if self.results.get("Chemistry") else None,
             AutoEvalColumn.rank_chemistry.name: self.results.get("Chemistry").get("Rank", None) if self.results.get("Chemistry") else None,
             
+            AutoEvalColumn.score_cpp.name: self.results.get("CPP").get("Average Score", None) if self.results.get("CPP") else None,
+            AutoEvalColumn.sd_cpp.name: self.results.get("CPP").get("Standard Deviation", None) if self.results.get("CPP") else None,
+            AutoEvalColumn.rank_cpp.name: self.results.get("CPP").get("Rank", None) if self.results.get("CPP") else None,
+            
             AutoEvalColumn.license.name: self.license,
             AutoEvalColumn.organization.name: self.org,
             AutoEvalColumn.knowledge_cutoff.name: self.knowledge_cutoff,

src/populate.py

@@ -24,7 +24,7 @@ def get_model_leaderboard_df(results_path: str, requests_path: str="", cols: lis
     if rank_col: # if there is one col in rank_col, sort by that column and remove NaN values
         df = df.dropna(subset=benchmark_cols)
         df = df.sort_values(by=[rank_col[0]], ascending=True)
-        # print(rank_col)
+        # print(rank_col, benchmark_cols)
     else: 
         # when rank_col, the first in benchmark_cols is empty, sort by averaging all the benchmarks, except the first one
         avg_rank = df.iloc[:, 1:].mean(axis=1) 
@@ -43,7 +43,7 @@ def get_model_leaderboard_df(results_path: str, requests_path: str="", cols: lis
         # print(col)
         # if 'Std dev' in col or 'Score' in col:
         if 'Std dev' in col or 'Score' in col:
-            if "Chemistry" in col:
+            if "Chemistry" in col or "C++" in col:
                 df[col] = (df[col]).map('{:.2f}'.format)
             else:
                 df[col] = (df[col]*100).map('{:.2f}'.format)