Field customisations.

app.py

@@ -68,11 +68,11 @@ def init_leaderboard(dataframe):
             cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
             label="Select Columns to Display:",
         ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        search_columns=[AutoEvalColumn.model.name],
         hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
         filter_columns=[
             ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+            ColumnFilter(AutoEvalColumn.maltese_training.name, type="checkboxgroup", label="Maltese training"),
             ColumnFilter(
                 AutoEvalColumn.params.name,
                 type="slider",
@@ -149,7 +149,7 @@ with demo:
                     model_name_textbox = gr.Textbox(label="Model name")
                     revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
                     model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
+                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.NK],
                         label="Model type",
                         multiselect=False,
                         value=None,

src/about.py

@@ -1,33 +1,47 @@
 from dataclasses import dataclass
 from enum import Enum
 
+@dataclass
+class TaskDetails:
+    name: str
+    display_name: str = ""
+    symbol: str = "" # emoji
+
+
+class TaskType(Enum):
+    NLU = TaskDetails("nlu", "NLU", "🧠")
+    NLG = TaskDetails("nlg", "NLG", "✍️")
+
+
 @dataclass
 class Task:
     benchmark: str
     metric: str
     col_name: str
+    task_type: TaskType
+    is_primary_metric: bool = True
 
 
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("sentiment", "f1,none", "Sentiment Analysis (F1)")
-    task1 = Task("sib200", "f1,none", "SIB200 (F1)")
-    task2 = Task("taxi1500", "f1,none", "Taxi1500 (F1)")
-    task3 = Task("maltese_news_categories", "loglikelihood,none", "Maltese News Categories (F1)")
-    task4 = Task("multi_eurlex", "loglikelihood,none", "MultiEURLEX (F1)")
-    task5 = Task("belebele", "acc,none", "Belebele (Accuracy)")
-    task6 = Task("opus100_en-mt", "bleu,none", "OPUS-100 EN→MT (BLEU)")
-    task7 = Task("opus100_en-mt", "chrf,none", "OPUS-100 EN→MT (ChrF)")
-    task8 = Task("flores200_en-mt", "bleu,none", "Flores-200 EN→MT (BLEU)")
-    task9 = Task("flores200_en-mt", "chrf,none", "Flores-200 EN→MT (ChrF)")
-    task10 = Task("webnlg", "chrf,none", "WebNLG (ChrF)")
-    task11 = Task("webnlg", "rouge,none", "WebNLG (Rouge-L)")
-    task12 = Task("eurlex_sum", "chrf,none", "EUR-Lex-Sum (ChrF)")
-    task13 = Task("eurlex_sum", "rouge,none", "EUR-Lex-Sum (Rouge-L)")
-    task14 = Task("maltese_news_headlines", "chrf,none", "Maltese News Headlines (ChrF)")
-    task15 = Task("maltese_news_headlines", "rouge,none", "Maltese News Headlines (Rouge-L)")
+    task0 = Task("sentiment", "f1,none", "Sentiment Analysis (F1)", TaskType.NLU)
+    task1 = Task("sib200", "f1,none", "SIB200 (F1)", TaskType.NLU)
+    task2 = Task("taxi1500", "f1,none", "Taxi1500 (F1)", TaskType.NLU)
+    task3 = Task("maltese_news_categories", "loglikelihood,none", "Maltese News Categories (F1)", TaskType.NLU)
+    task4 = Task("multi_eurlex", "loglikelihood,none", "MultiEURLEX (F1)", TaskType.NLU)
+    task5 = Task("belebele", "acc,none", "Belebele (Accuracy)", TaskType.NLU)
+    task6 = Task("opus100_en-mt", "bleu,none", "OPUS-100 EN→MT (BLEU)", TaskType.NLG, False)
+    task7 = Task("opus100_en-mt", "chrf,none", "OPUS-100 EN→MT (ChrF)", TaskType.NLG)
+    task8 = Task("flores200_en-mt", "bleu,none", "Flores-200 EN→MT (BLEU)", TaskType.NLG, False)
+    task9 = Task("flores200_en-mt", "chrf,none", "Flores-200 EN→MT (ChrF)", TaskType.NLG)
+    task10 = Task("webnlg", "chrf,none", "WebNLG (ChrF)", TaskType.NLG)
+    task11 = Task("webnlg", "rouge,none", "WebNLG (Rouge-L)", TaskType.NLG, False)
+    task12 = Task("eurlex_sum", "chrf,none", "EUR-Lex-Sum (ChrF)", TaskType.NLG, False)
+    task13 = Task("eurlex_sum", "rouge,none", "EUR-Lex-Sum (Rouge-L)", TaskType.NLG)
+    task14 = Task("maltese_news_headlines", "chrf,none", "Maltese News Headlines (ChrF)", TaskType.NLG, False)
+    task15 = Task("maltese_news_headlines", "rouge,none", "Maltese News Headlines (Rouge-L)", TaskType.NLG)
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------

src/display/utils.py

@@ -3,7 +3,8 @@ from enum import Enum
 
 import pandas as pd
 
-from src.about import Tasks
+from src.about import Tasks, TaskType
+
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -23,14 +24,18 @@ class ColumnContent:
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average (All) ⬆️", "number", True)])
+for task_type in TaskType:
+    auto_eval_column_dict.append([task_type.value.name, ColumnContent, ColumnContent(f"Average ({task_type.value.display_name}) {task_type.value.symbol}", "number", True)])
 for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", task.value.is_primary_metric)])
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+auto_eval_column_dict.append(["maltese_training", ColumnContent, ColumnContent("Maltese Training", "str", False)])
+auto_eval_column_dict.append(["num_languages", ColumnContent, ColumnContent("#Languages", "number", False)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
 auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
 auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
@@ -38,7 +43,7 @@ auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub Licen
 auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
 auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
 auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model SHA", "str", False, False)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
@@ -62,26 +67,48 @@ class ModelDetails:
 
 
 class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelDetails(name="", symbol="?")
+    PT = ModelDetails(name="pre-trained", symbol="PT")
+    FT = ModelDetails(name="fine-tuned", symbol="FT")
+    IT = ModelDetails(name="instruction-tuned", symbol="IT")
+    NK = ModelDetails(name="unknown", symbol="?")
 
     def to_str(self, separator=" "):
         return f"{self.value.symbol}{separator}{self.value.name}"
 
     @staticmethod
     def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
+        type = type or ""
+        if type == "PT":
             return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
-        return ModelType.Unknown
+        if type == "FT":
+            return ModelType.FT
+        if type == "IT":
+            return ModelType.IT
+        return ModelType.NK
+
+
+class MalteseTraining(Enum):
+    NO = ModelDetails(name="none", symbol="NO")
+    PT = ModelDetails(name="pre-training", symbol="PT")
+    FT = ModelDetails(name="fine-tuning", symbol="FT")
+    IT = ModelDetails(name="instruction-tuning", symbol="IT")
+    NK = ModelDetails(name="unknown", symbol="?")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+    @staticmethod
+    def from_str(type):
+        type = type or ""
+        if type == "NO":
+            return MalteseTraining.NO
+        if type == "PT":
+            return MalteseTraining.PT
+        if type == "FT":
+            return MalteseTraining.FT
+        if type == "IT":
+            return MalteseTraining.IT
+        return MalteseTraining.NK
 
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")

src/leaderboard/read_evals.py

@@ -1,15 +1,16 @@
 import glob
 import json
-import math
 import os
+from collections import defaultdict
 from dataclasses import dataclass
 
 import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, MalteseTraining
+from src.envs import TOKEN, API
+from src.submission.check_validity import is_model_on_hub, get_model_size
 
 
 @dataclass
@@ -18,12 +19,14 @@ class EvalResult:
     """
     eval_name: str # org_model_precision (uid)
     full_model: str # org/model (path on hub)
-    org: str 
+    org: str
     model: str
     revision: str # commit hash, "" if main
     results: dict
     precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
+    model_type: ModelType = ModelType.NK # Pretrained, fine tuned, ...
+    maltese_training: MalteseTraining = MalteseTraining.NK # none, pre-training, ...
+    num_languages: int = None
     weight_type: WeightType = WeightType.Original # Original or Adapter
     architecture: str = "Unknown" 
     license: str = "?"
@@ -39,10 +42,18 @@ class EvalResult:
             data = json.load(fp)
 
         config = data.get("config")
+        metadata = data.get("metadata")
 
-        # Precision
         precision = Precision.from_str(config.get("model_dtype"))
 
+        model_type = ModelType.from_str(metadata.get("model_type"))
+
+        maltese_training = MalteseTraining.from_str(metadata.get("maltese_training"))
+
+        num_languages = metadata.get("num_languages")
+
+        model_size = config.get("model_num_parameters")
+
         # Get model and org
         org_and_model = config.get("model_name", None)
         org_and_model = org_and_model.split("/", 1)
@@ -98,11 +109,14 @@ class EvalResult:
             org=org,
             model=model,
             results=results,
+            model_type=model_type,
+            maltese_training=maltese_training,
+            num_languages=num_languages or "?",
             precision=precision,  
             revision=revision,
             still_on_hub=still_on_hub,
             architecture=architecture,
-            likes=likes,
+            likes=likes or "?",
             num_params=round(model_size / 1e9, 3),
             license=license,
         )
@@ -130,7 +144,9 @@ class EvalResult:
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
             AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+            AutoEvalColumn.maltese_training.name: self.maltese_training.value.name,
+            AutoEvalColumn.model_symbol.name: self.model_type.value.symbol + "/" + self.maltese_training.value.symbol,
+            AutoEvalColumn.num_languages.name: self.num_languages,
             AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
@@ -142,8 +158,18 @@ class EvalResult:
             AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
 
+        results_by_task_type = defaultdict(list)
         for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+            result = self.results[task.value.benchmark]
+            data_dict[task.value.col_name] = result
+            if task.value.is_primary_metric:
+                results_by_task_type[task.value.task_type].append(result)
+        results_averages = []
+        for task_type, task_type_results in results_by_task_type.items():
+            average = sum([score for score in task_type_results if score is not None]) / len(task_type_results)
+            data_dict[getattr(AutoEvalColumn, task_type.value.name).name] = average
+            results_averages.append(average)
+        data_dict[AutoEvalColumn.average.name] = np.mean(results_averages) if len(results_averages) > 1 else results_averages[0]
 
         return data_dict
 
@@ -192,7 +218,6 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
     for model_result_filepath in model_result_filepaths:
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
 
         # Store results of same eval together
         eval_name = eval_result.eval_name

src/submission/check_validity.py

@@ -1,8 +1,7 @@
 import json
 import os
-import re
 from collections import defaultdict
-from datetime import datetime, timedelta, timezone
+from typing import Any
 
 import huggingface_hub
 from huggingface_hub import ModelCard
@@ -10,6 +9,7 @@ from huggingface_hub.hf_api import ModelInfo
 from transformers import AutoConfig
 from transformers.models.auto.tokenization_auto import AutoTokenizer
 
+
 def check_model_card(repo_id: str) -> tuple[bool, str]:
     """Checks if the model card and license exist and have been filled"""
     try:
@@ -31,7 +31,7 @@ def check_model_card(repo_id: str) -> tuple[bool, str]:
 
     return True, ""
 
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
+def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str, Any]:
     """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
     try:
         config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)