Separate zero-shot & few-shot results.

app.py

@@ -1,3 +1,4 @@
+import os
 import gradio as gr
 from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
@@ -48,7 +49,8 @@ except Exception:
     restart_space()
 
 
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+LEADERBOARD_0_SHOT_DF = get_leaderboard_df(os.path.join(EVAL_RESULTS_PATH, "zero-shot"), EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+LEADERBOARD_1_SHOT_DF = get_leaderboard_df(os.path.join(EVAL_RESULTS_PATH, "few-shot"), EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
 
 (
     finished_eval_queue_df,
@@ -56,14 +58,14 @@ LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS,
     pending_eval_queue_df,
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
-def init_leaderboard(dataframe):
+def init_leaderboard(dataframe, fewshot=True):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
     return Leaderboard(
         value=dataframe,
         datatype=[c.type for c in fields(AutoEvalColumn)],
         select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not (fewshot and c.hidden_in_fewshot)],
             cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
             label="Select Columns to Display:",
         ),
@@ -104,7 +106,10 @@ with demo:
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
+            with gr.TabItem("Zero-Shot", elem_id="zero-shot"):
+                leaderboard = init_leaderboard(LEADERBOARD_0_SHOT_DF, fewshot=False)
+            with gr.TabItem("Few-Shot", elem_id="few-shot"):
+                leaderboard = init_leaderboard(LEADERBOARD_1_SHOT_DF, fewshot=True)
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")

src/about.py

@@ -23,18 +23,19 @@ class Task:
     url: str
     task_type: TaskType
     is_primary_metric: bool = True
+    zero_shot_only: bool = False
 
 
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
     task0 = Task("sentiment_mlt", "f1", "Sentiment Analysis (F1)", "https://github.com/jerbarnes/typology_of_crosslingual/tree/master/data/sentiment/mt", TaskType.NLU)
     task1 = Task("sib200_mlt", "f1", "SIB200 (F1)", "https://huggingface.co/datasets/Davlan/sib200/viewer/mlt_Latn", TaskType.NLU)
     task2 = Task("taxi1500_mlt", "f1", "Taxi1500 (F1)", "https://github.com/cisnlp/Taxi1500", TaskType.NLU)
     task3 = Task("maltese_news_categories", "loglikelihood", "Maltese News Categories (F1)", "https://huggingface.co/datasets/MLRS/maltese_news_categories", TaskType.NLU)
     task4 = Task("multieurlex_mlt", "loglikelihood", "MultiEURLEX (F1)", "https://huggingface.co/datasets/nlpaueb/multi_eurlex", TaskType.NLU)
-    task5 = Task("belebele_mlt", "acc", "Belebele (Accuracy)", "https://huggingface.co/datasets/facebook/belebele/viewer/mlt_Latn", TaskType.NLU)
+    task5 = Task("belebele_mlt", "acc", "Belebele (Accuracy)", "https://huggingface.co/datasets/facebook/belebele/viewer/mlt_Latn", TaskType.NLU, zero_shot_only=True)
     task6 = Task("opus100_eng-mlt", "bleu", "OPUS-100 EN→MT (BLEU)", "https://huggingface.co/datasets/MLRS/OPUS-MT-EN-Fixed", TaskType.NLG, False)
     task7 = Task("opus100_eng-mlt", "chrf", "OPUS-100 EN→MT (ChrF)", "https://huggingface.co/datasets/MLRS/OPUS-MT-EN-Fixed", TaskType.NLG)
     task8 = Task("flores200_eng-mlt", "bleu", "Flores-200 EN→MT (BLEU)", "https://huggingface.co/datasets/Muennighoff/flores200", TaskType.NLG, False)

src/display/utils.py

@@ -20,20 +20,21 @@ class ColumnContent:
     displayed_by_default: bool
     hidden: bool = False
     never_hidden: bool = False
+    hidden_in_fewshot: bool = False
 
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
 auto_eval_column_dict.append(["model_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-auto_eval_column_dict.append(["n_shot", ColumnContent, ColumnContent("N-Shot", "number", False)])
-auto_eval_column_dict.append(["prompt_version", ColumnContent, ColumnContent("Version", "str", False)])
+auto_eval_column_dict.append(["n_shot", ColumnContent, ColumnContent("N-Shot", "number", True)])
+auto_eval_column_dict.append(["prompt_version", ColumnContent, ColumnContent("Version", "str", True)])
 #Scores
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average (All) ⬆️", "number", True)])
 for task_type in TaskType:
     auto_eval_column_dict.append([task_type.value.name, ColumnContent, ColumnContent(f"Average ({task_type.value.display_name}) {task_type.value.symbol}", "number", True)])
 for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", task.value.is_primary_metric)])
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", task.value.is_primary_metric, hidden_in_fewshot=task.value.zero_shot_only)])
 # Model information
 auto_eval_column_dict.append(["model_training", ColumnContent, ColumnContent("Type", "str", False)])
 auto_eval_column_dict.append(["maltese_training", ColumnContent, ColumnContent("Maltese Training", "str", False)])

src/leaderboard/read_evals.py

@@ -107,7 +107,7 @@ class EvalResult:
                     if task.benchmark not in data or task.metric not in data[task.benchmark]:
                         continue
                     score = data[task.benchmark][task.metric]
-                    if task.metric in ("accuracy", "f1", "loglikelihood", "rouge"):
+                    if task.metric in ("acc", "f1", "loglikelihood", "rouge"):
                         score *= 100
                     results[task.benchmark + "_" + task.metric][seed] = score
 
@@ -185,7 +185,7 @@ class EvalResult:
         for task in Tasks:
             result = self.results.get(task.value.benchmark + "_" + task.value.metric)
             data_dict[task.value.col_name] = result
-            if task.value.is_primary_metric:
+            if task.value.is_primary_metric and not (task.value.zero_shot_only and self.n_shot > 0):
                 results_by_task_type[task.value.task_type].append(result)
         results_averages = []
         for task_type, task_type_results in results_by_task_type.items():
@@ -223,7 +223,7 @@ def get_raw_eval_results(results_path: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
     model_result_filepaths = defaultdict(lambda: defaultdict(list))
 
-    for directory_path in Path(results_path).rglob("*-shot/*/*/"):
+    for directory_path in Path(results_path).rglob("*/*/"):
         for file_path in directory_path.rglob("*-seed/results_*.json"):
             seed = file_path.parent.name.removesuffix("-seed")
             model_result_filepaths[directory_path.relative_to(results_path)][seed].append(file_path)

src/populate.py

@@ -16,6 +16,7 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     df = pd.DataFrame.from_records(all_data_json)
     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     df = df[cols].round(decimals=2)
+    df.dropna(how="all", axis=1, inplace=True)
 
     return df