Add new scripts for model processing and tasks management

app.py

@@ -12,6 +12,11 @@ from src.about import (
     LLM_BENCHMARKS_TEXT,
     TITLE,
 )
+
+from src.tasks import (
+    TE_DESCRIPTION,
+)
+
 from src.display.css_html_js import custom_css
 from src.display.utils import (
     BENCHMARK_COLS,
@@ -58,6 +63,7 @@ LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS,
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
 def init_leaderboard(dataframe):
+    print(dataframe)
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
     return Leaderboard(
@@ -89,14 +95,49 @@ def init_leaderboard(dataframe):
     )
 
 
+def init_leaderboard2(dataframe, default_selection=None, hidden_columns=None):
+
+    print("entrato===============================================")
+
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+    return Leaderboard(
+        value=dataframe,
+        datatype=[c.type for c in fields(AutoEvalColumn)],
+        select_columns=SelectColumns(
+            default_selection=default_selection or [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+            label="Select Columns to Display:",
+        ),
+        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        hide_columns=hidden_columns or [c.name for c in fields(AutoEvalColumn) if c.hidden],
+        filter_columns=[
+            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+            ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0.01, max=150, label="Select the number of parameters (B)"),
+            ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True),
+        ],
+        bool_checkboxgroup_label="Hide models",
+        interactive=False,
+    )
+
+
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
+        with gr.TabItem("🏅 EVALITA-LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+            #leaderboard = init_leaderboard(LEADERBOARD_DF)
+
+            leaderboard = init_leaderboard2(
+                LEADERBOARD_DF,
+                default_selection=['T', 'Model', "Average ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"],
+                hidden_columns=[col for col in LEADERBOARD_DF.columns if
+                                col not in ['T', 'Model', "Average ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL" ]]
+            )
+
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
@@ -188,6 +229,85 @@ with demo:
                 submission_result,
             )
 
+
+        with gr.TabItem("TE", elem_id="llm-benchmark-tab-table", id=4):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+            #leaderboard = init_leaderboard(LEADERBOARD_DF)
+
+            LEADERBOARD_DF_TE = LEADERBOARD_DF.rename(columns={"TE Prompt Average": "Prompt Average",
+                                                            "TE Best Prompt": "Best Prompt",
+                                                            "TE Best Prompt Id": "Best Prompt Id",
+                                                            "TE": "Combined Performance"})
+
+            leaderboard = init_leaderboard2(
+                LEADERBOARD_DF_TE,
+                default_selection=['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id'],
+                hidden_columns=[col for col in LEADERBOARD_DF.columns if
+                                col not in ['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id']]
+            )
+
+
+        with gr.TabItem("SA", elem_id="llm-benchmark-tab-table", id=5):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+            LEADERBOARD_DF_SA = LEADERBOARD_DF.rename(columns={"SA Prompt Average": "Prompt Average",
+                                                            "SA Best Prompt": "Best Prompt",
+                                                            "SA Best Prompt Id": "Best Prompt Id",
+                                                            "SA": "Combined Performance"})
+
+            leaderboard = init_leaderboard2(
+                LEADERBOARD_DF_SA,
+                default_selection=['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt',
+                                   'Best Prompt Id'],
+                hidden_columns=[col for col in LEADERBOARD_DF.columns if
+                                col not in ['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt',
+                                            'Best Prompt Id']]
+            )
+
+
+
+
+        with gr.TabItem("HS", elem_id="llm-benchmark-tab-table", id=6):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+            LEADERBOARD_DF_HS = LEADERBOARD_DF.rename(columns={"HS Prompt Average": "Prompt Average",
+                                                               "HS Best Prompt": "Best Prompt",
+                                                               "HS Best Prompt Id": "Best Prompt Id",
+                                                               "HS": "Combined Performance"})
+
+            leaderboard = init_leaderboard2(
+                LEADERBOARD_DF_HS,
+                default_selection=['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt',
+                                   'Best Prompt Id'],
+                hidden_columns=[col for col in LEADERBOARD_DF.columns if
+                                col not in ['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt',
+                                            'Best Prompt Id']]
+            )
+
+
+
+        with gr.TabItem("AT", elem_id="llm-benchmark-tab-table", id=7):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+        with gr.TabItem("WIC", elem_id="llm-benchmark-tab-table", id=8):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+        with gr.TabItem("FAQ", elem_id="llm-benchmark-tab-table", id=9):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+        with gr.TabItem("LS", elem_id="llm-benchmark-tab-table", id=10):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+        with gr.TabItem("SU", elem_id="llm-benchmark-tab-table", id=11):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+        with gr.TabItem("NER", elem_id="llm-benchmark-tab-table", id=12):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+        with gr.TabItem("REL", elem_id="llm-benchmark-tab-table", id=13):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
             citation_button = gr.Textbox(

app2.py

@@ -0,0 +1,153 @@
+import gradio as gr
+from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
+import pandas as pd
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import snapshot_download
+
+from src.about import (
+    CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, EVALUATION_QUEUE_TEXT,
+    INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TITLE
+)
+from src.tasks import TE_DESCRIPTION
+from src.display.css_html_js import custom_css
+from src.display.utils import (
+    BENCHMARK_COLS, COLS, EVAL_COLS, EVAL_TYPES, AutoEvalColumn,
+    ModelType, fields, WeightType, Precision
+)
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.submission.submit import add_new_eval
+
+
+def restart_space():
+    """Restart the Hugging Face space."""
+    API.restart_space(repo_id=REPO_ID)
+
+
+def download_snapshot(repo, local_dir):
+    """Try to download a snapshot from the Hugging Face Hub, restarting space on failure."""
+    try:
+        print(f"Downloading from {repo} to {local_dir}...")
+        snapshot_download(repo_id=repo, local_dir=local_dir, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN)
+    except Exception as e:
+        print(f"Error downloading {repo}: {e}")
+        restart_space()
+
+
+# Space initialization
+download_snapshot(QUEUE_REPO, EVAL_REQUESTS_PATH)
+download_snapshot(RESULTS_REPO, EVAL_RESULTS_PATH)
+
+# Load leaderboard and evaluation queue data
+LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+
+
+def init_leaderboard(dataframe, default_selection=None, hidden_columns=None):
+    """Initialize a leaderboard with specific columns."""
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+
+    return Leaderboard(
+        value=dataframe,
+        datatype=[c.type for c in fields(AutoEvalColumn)],
+        select_columns=SelectColumns(
+            default_selection=default_selection or [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+            label="Select Columns to Display:",
+        ),
+        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        hide_columns=hidden_columns or [c.name for c in fields(AutoEvalColumn) if c.hidden],
+        filter_columns=[
+            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+            ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0.01, max=150, label="Select the number of parameters (B)"),
+            ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True),
+        ],
+        bool_checkboxgroup_label="Hide models",
+        interactive=False,
+    )
+
+
+def prepare_leaderboard_df(df, task_prefix):
+    """Rename columns for a specific task to a standard format."""
+    return df.rename(columns={
+        f"{task_prefix} Prompt Average": "Prompt Average",
+        f"{task_prefix} Best Prompt": "Best Prompt",
+        f"{task_prefix} Best Prompt Id": "Best Prompt Id",
+        task_prefix: "Combined Performance"
+    })
+
+
+demo = gr.Blocks(css=custom_css)
+with demo:
+    gr.HTML(TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        # Main leaderboard tab
+        with gr.TabItem("🏅 EVALITA-LLM Benchmark", elem_id="llm-benchmark-tab-table"):
+            leaderboard = init_leaderboard(
+                LEADERBOARD_DF,
+                default_selection=['T', 'Model', 'Few-Shot', "Average ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"],
+                hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in
+                                ['T', 'Model', 'Few-Shot', "Average ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]]
+            )
+
+        # About tab
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table"):
+            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+
+        # Submission tab
+        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table"):
+            gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+
+            for queue_name, queue_df in [
+                ("✅ Finished Evaluations", finished_eval_queue_df),
+                ("🔄 Running Evaluation Queue", running_eval_queue_df),
+                ("⏳ Pending Evaluation Queue", pending_eval_queue_df)
+            ]:
+                with gr.Accordion(f"{queue_name} ({len(queue_df)})", open=False):
+                    gr.components.Dataframe(value=queue_df, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5)
+
+            gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+            with gr.Row():
+                model_name_textbox = gr.Textbox(label="Model name")
+                revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+                model_type = gr.Dropdown(choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
+                                         label="Model type", multiselect=False, interactive=True)
+                precision = gr.Dropdown(choices=[i.value.name for i in Precision if i != Precision.Unknown],
+                                        label="Precision", multiselect=False, value="float16", interactive=True)
+                weight_type = gr.Dropdown(choices=[i.value.name for i in WeightType],
+                                          label="Weights type", multiselect=False, value="Original", interactive=True)
+                base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+
+            submit_button = gr.Button("Submit Eval")
+            submission_result = gr.Markdown()
+            submit_button.click(
+                add_new_eval,
+                [model_name_textbox, base_model_name_textbox, revision_name_textbox, precision, weight_type, model_type],
+                submission_result,
+            )
+
+        # Task-specific leaderboards
+        for task in ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]:
+            with gr.TabItem(task, elem_id="llm-benchmark-tab-table"):
+                gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+                leaderboard = init_leaderboard(
+                    prepare_leaderboard_df(LEADERBOARD_DF, task),
+                    default_selection=['T', 'Model', 'Few-Shot', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id'],
+                    hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in
+                                    ['T', 'Model', 'Few-Shot', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id']]
+                )
+
+    # Citation section
+    with gr.Accordion("📙 Citation", open=False):
+        gr.Textbox(value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, lines=20, elem_id="citation-button", show_copy_button=True)
+
+# Background job to restart space
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.start()
+
+demo.queue(default_concurrency_limit=40).launch()

get_model_info.py

@@ -8,8 +8,8 @@ from huggingface_hub import HfApi
 api = HfApi()
 
 # Percorsi delle cartelle
-input_folder = "../evalita_llm_results/models_output/"
-output_folder = "../evalita_llm_requests2/"
+input_folder = "../evalita_llm_models_output/"
+output_folder = "../evalita_llm_requests/"
 
 # Creazione della cartella di output se non esiste
 os.makedirs(output_folder, exist_ok=True)

src/about.py

@@ -5,15 +5,88 @@ from enum import Enum
 class Task:
     benchmark: str
     metric: str
+    metric_type: str
     col_name: str
 
 
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
+
+    task1 = Task("text-entailment_1", "acc", "CPS", "TE")
+    task2 = Task("text-entailment_2", "acc", "average_accuracy", "TE Prompt Average")
+    task3 = Task("text-entailment_3", "acc", "best_prompt", "TE Best Prompt")
+    task4 = Task("text-entailment_4", "acc", "prompt_id", "TE Best Prompt Id")
+
+    task5 = Task("sentiment-analysis_1", "acc", "CPS", "SA")
+    task6 = Task("sentiment-analysis_2", "acc", "average_accuracy", "SA Prompt Average")
+    task7 = Task("sentiment-analysis_3", "acc", "best_prompt", "SA Best Prompt")
+    task8 = Task("sentiment-analysis_4", "acc", "prompt_id", "SA Best Prompt Id")
+
+    task9 = Task("hate-speech-detection_1", "acc", "CPS", "HS")
+    task10 = Task("hate-speech-detection_2", "acc", "average_accuracy", "HS Prompt Average")
+    task11 = Task("hate-speech-detection_3", "acc", "best_prompt", "HS Best Prompt")
+    task12 = Task("hate-speech-detection_4", "acc", "prompt_id", "HS Best Prompt Id")
+
+    task13 = Task("admission-test_1", "acc", "CPS", "AT")
+    task14 = Task("admission-test_2", "acc", "average_accuracy", "AT Prompt Average")
+    task15 = Task("admission-test_3", "acc", "best_prompt", "AT Best Prompt")
+    task16 = Task("admission-test_4", "acc", "prompt_id", "AT Best Prompt Id")
+
+    task17 = Task("word-in-context_1", "acc", "CPS", "WIC")
+    task18 = Task("word-in-context_2", "acc", "average_accuracy", "WIC Prompt Average")
+    task19 = Task("word-in-context_3", "acc", "best_prompt", "WIC Best Prompt")
+    task20 = Task("word-in-context_4", "acc", "prompt_id", "WIC Best Prompt Id")
+
+    task21 = Task("faq_1", "acc", "CPS", "FAQ")
+    task22 = Task("faq_2", "acc", "average_accuracy", "FAQ Prompt Average")
+    task23 = Task("faq_3", "acc", "best_prompt", "FAQ Best Prompt")
+    task24 = Task("faq_4", "acc", "prompt_id", "FAQ Best Prompt Id")
+
+    task25 = Task("lexical-substitution_1", "acc", "CPS", "LS")
+    task26 = Task("lexical-substitution_2", "acc", "average_accuracy", "LS Prompt Average")
+    task27 = Task("lexical-substitution_3", "acc", "best_prompt", "LS Best Prompt")
+    task28 = Task("lexical-substitution_4", "acc", "prompt_id", "LS Best Prompt Id")
+
+    task29 = Task("summarization-fanpage_1", "acc", "CPS", "SU")
+    task30 = Task("summarization-fanpage_2", "acc", "average_accuracy", "SU Prompt Average")
+    task31 = Task("summarization-fanpage_3", "acc", "best_prompt", "SU Best Prompt")
+    task32 = Task("summarization-fanpage_4", "acc", "prompt_id", "SU Best Prompt Id")
+
+    task33 = Task("evalita NER_1", "acc", "CPS", "NER")
+    task34 = Task("evalita NER_2", "acc", "average_accuracy", "NER Prompt Average")
+    task35 = Task("evalita NER_3", "acc", "best_prompt", "NER Best Prompt")
+    task36 = Task("evalita NER_4", "acc", "prompt_id", "NER Best Prompt Id")
+
+    task37 = Task("relation-extraction_1", "acc", "CPS", "REL")
+    task38 = Task("relation-extraction_2", "acc", "average_accuracy", "REL Prompt Average")
+    task39 = Task("relation-extraction_3", "acc", "best_prompt", "REL Best Prompt")
+    task40 = Task("relation-extraction_4", "acc", "prompt_id", "REL Best Prompt Id")
+
+
+    '''
+    task0 = Task("TextualEntailment", "acc", "Textual Entailment")
+    task1 = Task("TextualEntailment_best", "acc", "TextualEntailment Best")
+    task2 = Task("Sentiment Analysis", "acc", "Sentiment Analysis")
+    task3 = Task("Sentiment Analysis_best", "acc", "Sentiment Analysis_best")
+    task4 = Task("Hate Speech", "acc", "Hate Speech")
+    task5 = Task("Hate Speech_best", "acc", "Hate Speech_best")
+    task6 = Task("Admission Test", "acc", "Admission Test")
+    task7 = Task("Admission Test_best", "acc", "Admission Test_best")
+    task8 = Task("Word in Context", "acc", "Word in Context")
+    task9 = Task("Word in Context_best", "acc", "Word in Context_best")
+    task10 = Task("FAQ", "acc", "FAQ")
+    task11 = Task("FAQ_best", "acc", "FAQ_best")
+    task12 = Task("Lexical Substitution", "acc", "Lexical Substitution")
+    task13 = Task("Lexical Substitution_best", "acc", "Lexical Substitution_best")
+    task14 = Task("Summarization", "acc", "Summarization")
+    task15 = Task("Summarization_best", "acc", "Summarization_best")
+    task16 = Task("NER", "acc", "NER")
+    task17 = Task("NER_best", "acc", "NER_best")
+    task18 = Task("REL", "acc", "REL")
+    task19 = Task("REL_best", "acc", "REL_best")
+    '''
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
@@ -21,19 +94,54 @@ NUM_FEWSHOT = 0 # Change with your few shot
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">🚀 Evalita Leaderboard 🚀</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
+Evalita-LLM, a new benchmark designed to evaluate Large Language Models (LLMs) on Italian tasks. The distinguishing and innovative features of Evalita-LLM are the following: (i) all tasks are native Italian, avoiding issues of translating from Italian and potential cultural biases; (ii) in addition to well established multiple-choice tasks, the benchmark includes generative tasks, enabling more natural interaction with LLMs; (iii) all tasks are evaluated against multiple prompts, this way mitigating the model sensitivity to specific prompts and allowing a fairer and objective evaluation.
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
 LLM_BENCHMARKS_TEXT = f"""
-## How it works
+### Groups
 
-## Reproducibility
-To reproduce our results, here is the commands you can run:
+- `evalita-mp`: All tasks (perplexity and non-perplexity based).
+- `evalita-mp_gen`: Only generative tasks.
+- `evalita-mp_mc`: Only perplexity-based tasks.
+
+#### Tasks
+
+The following Evalita-LLM tasks can also be evaluated in isolation:
+ - `evalita-mp_te`: Textual Entailment
+ - `evalita-mp_sa`: Sentiment Analysis
+ - `evalita-mp_wic`: Word in Context
+ - `evalita-mp_hs`: Hate Speech Detection
+ - `evalita-mp_at`: Admission Tests
+ - `evalita-mp_faq`: FAQ
+ - `evalita-mp_sum_fp`:  Summarization
+ - `evalita-mp_ls`: Lexical Substitution
+ - `evalita-mp_ner_group`: Named Entity Recognition
+ - `evalita-mp_re`: Relation Extraction
+
+
+### Usage
+
+```bash
+
+lm_eval --model hf --model_args pretrained=meta-llama/Llama-2-7b-hf --tasks evalita-mp --device cuda:0 --batch_size auto 
+```
+
+### Checklist
+
+* [x] Is the task an existing benchmark in the literature?
+ * [x] Have you referenced the original paper that introduced the task?
+ * [x] If yes, does the original paper provide a reference implementation?
+   * [x] Yes, original implementation contributed by author of the benchmark
+
+If other tasks on this dataset are already supported:
+* [x] Is the "Main" variant of this task clearly denoted?
+* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [x] Have you noted which, if any, published evaluation setups are matched by this variant?
 
 """
 
@@ -69,4 +177,13 @@ If everything is done, check you can launch the EleutherAIHarness on your model
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
 CITATION_BUTTON_TEXT = r"""
+@misc{magnini2025evalitallmbenchmarkinglargelanguage,
+     title={Evalita-LLM: Benchmarking Large Language Models on Italian}, 
+     author={Bernardo Magnini and Roberto Zanoli and Michele Resta and Martin Cimmino and Paolo Albano and Marco Madeddu and Viviana Patti},
+     year={2025},
+     eprint={2502.02289},
+     archivePrefix={arXiv},
+     primaryClass={cs.CL},
+     url={https://arxiv.org/abs/2502.02289}, 
+}
 """

src/display/utils.py

@@ -25,6 +25,7 @@ auto_eval_column_dict = []
 # Init
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["fewshot", ColumnContent, ColumnContent("Few-Shot", "str", True)])
 #Scores
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
 for task in Tasks:
@@ -108,3 +109,28 @@ EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
 BENCHMARK_COLS = [t.value.col_name for t in Tasks]
 
+
+# Roberto
+
+# Nuovi valori per CPS, AVERAGE, BEST, e ID nella tabella
+@dataclass
+class NewColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+    never_hidden: bool = False
+
+# Inizializza i nuovi valori
+new_column_dict = []
+# Aggiungi CPS, VERAGE, BEST, ID
+new_column_dict.append(["CPS", NewColumnContent, NewColumnContent("CPS", "number", True)])
+new_column_dict.append(["AVERAGE", NewColumnContent, NewColumnContent("Average ⬆️", "number", True)])
+new_column_dict.append(["BEST", NewColumnContent, NewColumnContent("Best Performance", "number", True)])
+new_column_dict.append(["ID", NewColumnContent, NewColumnContent("ID", "str", True)])
+
+# Puoi usare make_dataclass per creare la classe dinamicamente come per AutoEvalColumn
+NewColumn = make_dataclass("NewColumn", new_column_dict, frozen=True)
+
+# Includi questi nuovi valori nei COLS o in altre variabili di configurazione, se necessario
+NEW_COLS = [c.name for c in fields(NewColumn) if not c.hidden]

src/envs.py

@@ -6,12 +6,17 @@ from huggingface_hub import HfApi
 # ----------------------------------
 TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+#OWNER = "giux78" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "evalitahf"
 # ----------------------------------
 
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
+#REPO_ID = f"{OWNER}/leaderboard-evalita"
+#QUEUE_REPO = f"{OWNER}/evalita-requests"
+#RESULTS_REPO = f"{OWNER}/evalita-results"
+
+REPO_ID = f"{OWNER}/evalita_llm_leaderboard"
+QUEUE_REPO = f"{OWNER}/evalita_llm_requests"
+RESULTS_REPO = f"{OWNER}/evalita_llm_results"
 
 # If you setup a cache later, just change HF_HOME
 CACHE_PATH=os.getenv("HF_HOME", ".")

src/leaderboard/read_evals.py

@@ -22,6 +22,8 @@ class EvalResult:
     model: str
     revision: str # commit hash, "" if main
     results: dict
+    average_CPS: str
+    fewshot: str
     precision: Precision = Precision.Unknown
     model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
     weight_type: WeightType = WeightType.Original # Original or Adapter
@@ -40,21 +42,47 @@ class EvalResult:
 
         config = data.get("config")
 
+        average_CPS = data.get("average_CPS")
+
+        num_fewshot = config.get("num_fewshot", 0)  # Imposta il valore predefinito a 0
+        try:
+            num_fewshot = int(num_fewshot)  # Converte in intero se possibile
+        except ValueError:
+            num_fewshot = 0  # Se la conversione fallisce, assegna 0
+
+
+        precision = config.get("precision")
+
+        print(precision)
+
+        print(config, num_fewshot)
+
         # Precision
         precision = Precision.from_str(config.get("model_dtype"))
 
+        model_type = config.get("model_type")
+        # Modifica: Convertire model_type in un oggetto Enum (se è un Enum)
+        model_type = ModelType.from_str(model_type) if model_type else None
+
+        print("=====================", model_type, config.get("model_name"))
+
+
         # Get model and org
         org_and_model = config.get("model_name", config.get("model_args", None))
         org_and_model = org_and_model.split("/", 1)
 
+        print(precision.value.name)
+
         if len(org_and_model) == 1:
             org = None
             model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
+            #result_key = f"{model}_{precision.value.name}"
+            result_key = f"{model}_{num_fewshot}"
         else:
             org = org_and_model[0]
             model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
+            #result_key = f"{org}_{model}_{precision.value.name}"
+            result_key = f"{org}_{model}_{num_fewshot}"
         full_model = "/".join(org_and_model)
 
         still_on_hub, _, model_config = is_model_on_hub(
@@ -71,6 +99,7 @@ class EvalResult:
         for task in Tasks:
             task = task.value
 
+            '''
             # We average all scores of a given metric (not all metrics are present in all files)
             accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
             if accs.size == 0 or any([acc is None for acc in accs]):
@@ -78,6 +107,29 @@ class EvalResult:
 
             mean_acc = np.mean(accs) * 100.0
             results[task.benchmark] = mean_acc
+            '''
+
+            for k, v in data["tasks"].items():
+                #if task.benchmark == k:
+                if task.benchmark[:-2] == k:
+                    # print(k, "==================", v)
+                    # results[task.benchmark] = v[task.cps]
+
+                    #print(task.benchmark, v[task.metric])
+
+                    if "Best Prompt Id" in task.col_name:
+                        results[task.benchmark] = int(v[task.metric_type][-1:])
+                        #print(results[task.benchmark],v[task.metric_type][-1:])
+                    else:
+                        results[task.benchmark] = v[task.metric_type]
+
+
+                    #results[task.benchmark + "_" + task.metric] = 1.0
+
+
+                    #results[task.benchmark] = v[task.accuracy]
+                    # print("======", results[task.benchmark])
+                    #results[task.benchmark] = 1.0
 
         return self(
             eval_name=result_key,
@@ -85,6 +137,9 @@ class EvalResult:
             org=org,
             model=model,
             results=results,
+            average_CPS=average_CPS,
+            fewshot=num_fewshot,
+            model_type=model_type,
             precision=precision,  
             revision= config.get("model_sha", ""),
             still_on_hub=still_on_hub,
@@ -109,17 +164,25 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        #average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        average = self.average_CPS
+        fewshot = self.fewshot
+        print("?????", fewshot)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+            #AutoEvalColumn.model_type.name: self.model_type.value.name,
+            #AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+
+            AutoEvalColumn.model_type.name: self.model_type.value.name if self.model_type else "Unknown",
+            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol if self.model_type else "Unknown",
+
             AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
             AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average.name: average,
+            AutoEvalColumn.fewshot.name: fewshot,
             AutoEvalColumn.license.name: self.license,
             AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
@@ -176,7 +239,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
     for model_result_filepath in model_result_filepaths:
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
+        #eval_result.update_with_request_file(requests_path)
 
         # Store results of same eval together
         eval_name = eval_result.eval_name