error

app.py

@@ -1,276 +1,192 @@
-import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
-import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-from fastapi import FastAPI
-from src.api_submit_results import router as submission_router
-
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval, add_manual_results
-
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
+import json
+from pathlib import Path
+from typing import List, Dict
 
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
+import gradio as gr
+from pydantic import BaseModel, Field, validator
+
+# --------------- Configuration ---------------
+LEADERBOARD_PATH = Path("leaderboard_data.json")
+DEFAULT_MODEL_NAME = "example/model"
+
+# --------------- Data models ---------------
+class Metrics(BaseModel):
+    readability: float
+    relevance: float
+    explanation_clarity: float = Field(alias="explanation_clarity")
+    problem_identification: float
+    actionability: float
+    completeness: float
+    specificity: float
+    contextual_adequacy: float
+    consistency: float
+    brevity: float
+
+
+class LeaderboardEntry(BaseModel):
+    model_name: str
+    bleu: float
+    llm_pass_1: float
+    llm_pass_5: float
+    llm_pass_10: float
+    metrics: Metrics
+
+    @validator("bleu", "llm_pass_1", "llm_pass_5", "llm_pass_10", each_item=True)
+    def score_range(cls, v: float):
+        if not 0.0 <= v <= 1.0:
+            raise ValueError("Scores should be between 0 and 1")
+        return v
+
+
+# --------------- Persistence helpers ---------------
+
+def _load_leaderboard() -> List[Dict]:
+    if not LEADERBOARD_PATH.exists():
+        return []
+    with LEADERBOARD_PATH.open("r", encoding="utf-8") as f:
+        data = json.load(f)
+        return data.get("leaderboard", [])
+
+
+def _save_leaderboard(data: List[Dict]):
+    to_store = {"leaderboard": data}
+    with LEADERBOARD_PATH.open("w", encoding="utf-8") as f:
+        json.dump(to_store, f, indent=2)
+
+
+# --------------- Utility ---------------
+
+def _flatten_entry(entry: Dict) -> Dict:
+    """Flatten nested metrics so that every metric is a column."""
+    flat = {
+        "Model": entry["model_name"],
+        "BLEU": entry["bleu"],
+        "Pass@1": entry["llm_pass_1"],
+        "Pass@5": entry["llm_pass_5"],
+        "Pass@10": entry["llm_pass_10"],
+    }
+    for metric_name, score in entry["metrics"].items():
+        flat[metric_name.replace("_", " ").title()] = score
+    return flat
+
+
+def _table_data() -> List[Dict]:
+    data = _load_leaderboard()
+    # Sort descending by pass@1 as requested
+    data.sort(key=lambda x: x["llm_pass_1"], reverse=True)
+    return [_flatten_entry(e) for e in data]
+
+
+# --------------- Gradio callbacks ---------------
+
+def submit_model(
+    model_name: str,
+    bleu: float,
+    llm_pass_1: float,
+    llm_pass_5: float,
+    llm_pass_10: float,
+    readability: float,
+    relevance: float,
+    explanation_clarity: float,
+    problem_identification: float,
+    actionability: float,
+    completeness: float,
+    specificity: float,
+    contextual_adequacy: float,
+    consistency: float,
+    brevity: float,
+):
+    """Validate and append a new model entry to the leaderboard."""
+    try:
+        entry = LeaderboardEntry(
+            model_name=model_name.strip(),
+            bleu=bleu,
+            llm_pass_1=llm_pass_1,
+            llm_pass_5=llm_pass_5,
+            llm_pass_10=llm_pass_10,
+            metrics={
+                "readability": readability,
+                "relevance": relevance,
+                "explanation_clarity": explanation_clarity,
+                "problem_identification": problem_identification,
+                "actionability": actionability,
+                "completeness": completeness,
+                "specificity": specificity,
+                "contextual_adequacy": contextual_adequacy,
+                "consistency": consistency,
+                "brevity": brevity,
+            },
+        )
+    except Exception as e:
+        return gr.update(value=_table_data()), gr.update(value=f"❌ Submission failed: {e}")
+
+    data = _load_leaderboard()
+    # Replace existing model entry if any
+    data = [d for d in data if d["model_name"] != entry.model_name]
+    data.append(entry.dict())
+    _save_leaderboard(data)
+
+    return gr.update(value=_table_data()), gr.update(value="✅ Submission recorded!")
+
+
+# --------------- Interface ---------------
+with gr.Blocks(title="Custom LLM Leaderboard") as demo:
+    gr.Markdown("""# 🏆 LLM Leaderboard\nSubmit your model results below. Leaderboard is sorted by **Pass@1**. """)
+
+    leaderboard_df = gr.Dataframe(
+        headers=list(_table_data()[0].keys()) if _table_data() else [],
+        value=_table_data(),
+        label="Current Leaderboard",
         interactive=False,
     )
 
-
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
-
-            # ----------------------------------------------------
-            # Manual metrics submission form
-            # ----------------------------------------------------
-            gr.Markdown("## 📝 Submit metrics manually (advanced)")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_metrics = gr.Textbox(label="Model name", placeholder="org/model")
-                    revision_metrics = gr.Textbox(label="Revision commit", placeholder="main", value="main")
-                    bleu_input = gr.Number(label="BLEU", value=0.5)
-                    pass1_input = gr.Number(label="Pass@1", value=0.5, minimum=0.0, maximum=1.0)
-                    pass5_input = gr.Number(label="Pass@5", value=0.5, minimum=0.0, maximum=1.0)
-                    pass10_input = gr.Number(label="Pass@10", value=0.5, minimum=0.0, maximum=1.0)
-
-                with gr.Column():
-                    # Subjective metrics sliders (0-5)
-                    readability_slider = gr.Slider(0, 5, step=1, value=3, label="Readability")
-                    relevance_slider = gr.Slider(0, 5, step=1, value=3, label="Relevance")
-                    explanation_slider = gr.Slider(0, 5, step=1, value=3, label="Explanation clarity")
-                    problem_slider = gr.Slider(0, 5, step=1, value=3, label="Problem identification")
-                    actionability_slider = gr.Slider(0, 5, step=1, value=3, label="Actionability")
-                    completeness_slider = gr.Slider(0, 5, step=1, value=3, label="Completeness")
-                    specificity_slider = gr.Slider(0, 5, step=1, value=3, label="Specificity")
-                    contextual_slider = gr.Slider(0, 5, step=1, value=3, label="Contextual adequacy")
-                    consistency_slider = gr.Slider(0, 5, step=1, value=3, label="Consistency")
-                    brevity_slider = gr.Slider(0, 5, step=1, value=3, label="Brevity")
-
-            submit_metrics_button = gr.Button("Submit Metrics")
-            metrics_submission_result = gr.Markdown()
-
-            submit_metrics_button.click(
-                add_manual_results,
-                [
-                    model_name_metrics,
-                    revision_metrics,
-                    bleu_input,
-                    readability_slider,
-                    relevance_slider,
-                    explanation_slider,
-                    problem_slider,
-                    actionability_slider,
-                    completeness_slider,
-                    specificity_slider,
-                    contextual_slider,
-                    consistency_slider,
-                    brevity_slider,
-                    pass1_input,
-                    pass5_input,
-                    pass10_input,
-                ],
-                metrics_submission_result,
-            )
-
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
-
-# ------------------------------
-# Start background scheduler
-# ------------------------------
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-
-# ------------------------------
-# Mount Gradio UI into FastAPI application
-# ------------------------------
-# Removed direct .launch(); Gradio UI will be served via the mounted FastAPI `app`.
-
-# ------------------ FastAPI mounting ------------------
-backend = FastAPI()
-backend.include_router(submission_router)
-
-# Enable queuing (same limit as before)
-demo = demo.queue(default_concurrency_limit=40)
-
-# Expose `app` for the HF Spaces runtime
-app = gr.mount_gradio_app(backend, demo, path="/")
+    gr.Markdown("## 🔄 Submit new model results")
+
+    with gr.Accordion("Submission form", open=False):
+        with gr.Row():
+            model_name_inp = gr.Text(label="Model name (org/model)", value="")
+            bleu_inp = gr.Number(label="BLEU", value=0.0, minimum=0.0, maximum=1.0)
+            pass1_inp = gr.Number(label="Pass@1", value=0.0, minimum=0.0, maximum=1.0)
+            pass5_inp = gr.Number(label="Pass@5", value=0.0, minimum=0.0, maximum=1.0)
+            pass10_inp = gr.Number(label="Pass@10", value=0.0, minimum=0.0, maximum=1.0)
+
+        gr.Markdown("### Multi-metric subjective scores (0.0 – 1.0)")
+        with gr.Row():
+            readability_inp = gr.Slider(minimum=0.0, maximum=1.0, value=0.0, step=0.05, label="Readability")
+            relevance_inp = gr.Slider(minimum=0.0, maximum=1.0, value=0.0, step=0.05, label="Relevance")
+            explanation_inp = gr.Slider(minimum=0.0, maximum=1.0, value=0.0, step=0.05, label="Explanation Clarity")
+            problem_inp = gr.Slider(minimum=0.0, maximum=1.0, value=0.0, step=0.05, label="Problem Identification")
+            actionability_inp = gr.Slider(minimum=0.0, maximum=1.0, value=0.0, step=0.05, label="Actionability")
+            completeness_inp = gr.Slider(minimum=0.0, maximum=1.0, value=0.0, step=0.05, label="Completeness")
+            specificity_inp = gr.Slider(minimum=0.0, maximum=1.0, value=0.0, step=0.05, label="Specificity")
+            contextual_inp = gr.Slider(minimum=0.0, maximum=1.0, value=0.0, step=0.05, label="Contextual Adequacy")
+            consistency_inp = gr.Slider(minimum=0.0, maximum=1.0, value=0.0, step=0.05, label="Consistency")
+            brevity_inp = gr.Slider(minimum=0.0, maximum=1.0, value=0.0, step=0.05, label="Brevity")
+
+        submit_btn = gr.Button("Submit")
+        status_markdown = gr.Markdown("")
+
+        submit_btn.click(
+            fn=submit_model,
+            inputs=[
+                model_name_inp,
+                bleu_inp,
+                pass1_inp,
+                pass5_inp,
+                pass10_inp,
+                readability_inp,
+                relevance_inp,
+                explanation_inp,
+                problem_inp,
+                actionability_inp,
+                completeness_inp,
+                specificity_inp,
+                contextual_inp,
+                consistency_inp,
+                brevity_inp,
+            ],
+            outputs=[leaderboard_df, status_markdown],
+        )
+
+# Expose app variable for Spaces
+app = demo

leaderboard_data.json

@@ -0,0 +1,23 @@
+{
+    "leaderboard": [
+        {
+            "model_name": "example/model",
+            "bleu": 0.5,
+            "llm_pass_1": 0.5,
+            "llm_pass_5": 0.5,
+            "llm_pass_10": 0.5,
+            "metrics": {
+                "readability": 0.5,
+                "relevance": 0.5,
+                "explanation_clarity": 0.5,
+                "problem_identification": 0.5,
+                "actionability": 0.5,
+                "completeness": 0.5,
+                "specificity": 0.5,
+                "contextual_adequacy": 0.5,
+                "consistency": 0.5,
+                "brevity": 0.5
+            }
+        }
+    ]
+}

src/about.py

@@ -1,91 +0,0 @@
-from dataclasses import dataclass
-from enum import Enum
-
-@dataclass
-class Task:
-    benchmark: str
-    metric: str
-    col_name: str
-
-
-# Select your metrics here
-# ---------------------------------------------------
-# Each entry: first argument is the key inside "results" dict in the result JSON,
-# second is the metric key inside that sub-dict (we use "score" everywhere for uniformity),
-# third is the column name displayed in the leaderboard.
-
-class Tasks(Enum):
-    bleu = Task("bleu", "score", "BLEU ⬆️")
-    multimetric = Task("multimetric", "score", "Multimetric ⬆️")
-
-    readability = Task("readability", "score", "Readability")
-    relevance = Task("relevance", "score", "Relevance")
-    explanation_clarity = Task("explanation_clarity", "score", "Explanation clarity")
-    problem_identification = Task("problem_identification", "score", "Problem identification")
-    actionability = Task("actionability", "score", "Actionability")
-    completeness = Task("completeness", "score", "Completeness")
-    specificity = Task("specificity", "score", "Specificity")
-    contextual_adequacy = Task("contextual_adequacy", "score", "Contextual adequacy")
-    consistency = Task("consistency", "score", "Consistency")
-    brevity = Task("brevity", "score", "Brevity")
-
-    pass_at_1 = Task("pass_at_1", "score", "Pass@1 ⬆️")
-    pass_at_5 = Task("pass_at_5", "score", "Pass@5")
-    pass_at_10 = Task("pass_at_10", "score", "Pass@10")
-
-
-NUM_FEWSHOT = 0  # Not applicable here but kept for compatibility
-# ---------------------------------------------------
-
-
-
-# Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
-
-# What does your leaderboard evaluate?
-INTRODUCTION_TEXT = """
-Intro text
-"""
-
-# Which evaluations are you running? how can people reproduce what you have?
-LLM_BENCHMARKS_TEXT = f"""
-## How it works
-
-## Reproducibility
-To reproduce our results, here is the commands you can run:
-
-"""
-
-EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
-
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
-"""
-
-CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""
-"""

src/api_submit_results.py

@@ -1,116 +0,0 @@
-from datetime import datetime, timezone
-import json
-import os
-import uuid
-
-from fastapi import APIRouter, HTTPException
-from pydantic import BaseModel, Field, validator
-
-from src.envs import API, RESULTS_REPO, EVAL_RESULTS_PATH, TOKEN
-
-router = APIRouter(prefix="/api", tags=["submission"])
-
-ALL_SUBJECTIVE_FIELDS = [
-    "readability",
-    "relevance",
-    "explanation_clarity",
-    "problem_identification",
-    "actionability",
-    "completeness",
-    "specificity",
-    "contextual_adequacy",
-    "consistency",
-    "brevity",
-]
-
-
-class ResultPayload(BaseModel):
-    model: str = Field(..., description="Model id on the Hub (e.g. org/model)")
-    revision: str = Field("main", description="Commit sha or branch (default: main)")
-    bleu: float = Field(..., ge=0, description="BLEU score (0-100)")
-
-    # 10 subjective metrics 0-5
-    readability: int = Field(..., ge=0, le=5)
-    relevance: int = Field(..., ge=0, le=5)
-    explanation_clarity: int = Field(..., ge=0, le=5)
-    problem_identification: int = Field(..., ge=0, le=5)
-    actionability: int = Field(..., ge=0, le=5)
-    completeness: int = Field(..., ge=0, le=5)
-    specificity: int = Field(..., ge=0, le=5)
-    contextual_adequacy: int = Field(..., ge=0, le=5)
-    consistency: int = Field(..., ge=0, le=5)
-    brevity: int = Field(..., ge=0, le=5)
-
-    pass_at_1: float = Field(..., ge=0, le=1)
-    pass_at_5: float = Field(..., ge=0, le=1)
-    pass_at_10: float = Field(..., ge=0, le=1)
-
-    @validator("pass_at_5")
-    def _p5_ge_p1(cls, v, values):
-        if "pass_at_1" in values and v < values["pass_at_1"]:
-            raise ValueError("pass@5 must be >= pass@1")
-        return v
-
-    @validator("pass_at_10")
-    def _p10_ge_p5(cls, v, values):
-        if "pass_at_5" in values and v < values["pass_at_5"]:
-            raise ValueError("pass@10 must be >= pass@5")
-        return v
-
-    def multimetric(self) -> float:
-        total = sum(getattr(self, f) for f in ALL_SUBJECTIVE_FIELDS)
-        return float(total) / len(ALL_SUBJECTIVE_FIELDS)
-
-
-@router.post("/submit", status_code=200)
-async def submit_results(payload: ResultPayload):
-    """Accept new evaluation results and push them to the results dataset."""
-
-    # Prepare JSON in expected format (compatible with read_evals.py)
-    results_dict = {
-        "config": {
-            "model_dtype": "unknown",
-            "model_name": payload.model,
-            "model_sha": payload.revision,
-        },
-        "results": {},
-    }
-
-    # Primary metrics
-    results_dict["results"]["bleu"] = {"score": payload.bleu}
-    results_dict["results"]["multimetric"] = {"score": payload.multimetric()}
-
-    # Subjective metrics
-    for field in ALL_SUBJECTIVE_FIELDS:
-        results_dict["results"][field] = {"score": getattr(payload, field)}
-
-    # Pass@k metrics
-    results_dict["results"]["pass_at_1"] = {"score": payload.pass_at_1}
-    results_dict["results"]["pass_at_5"] = {"score": payload.pass_at_5}
-    results_dict["results"]["pass_at_10"] = {"score": payload.pass_at_10}
-
-    # File handling
-    os.makedirs(EVAL_RESULTS_PATH, exist_ok=True)
-    ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
-    unique_id = uuid.uuid4().hex[:8]
-    filename = f"results_{payload.model.replace('/', '_')}_{ts}_{unique_id}.json"
-    local_path = os.path.join(EVAL_RESULTS_PATH, filename)
-
-    with open(local_path, "w") as fp:
-        json.dump(results_dict, fp)
-
-    try:
-        API.upload_file(
-            path_or_fileobj=local_path,
-            path_in_repo=filename,
-            repo_id=RESULTS_REPO,
-            repo_type="dataset",
-            commit_message=f"Add results for {payload.model}",
-        )
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Failed to upload results: {e}")
-    finally:
-        if os.path.exists(local_path):
-            os.remove(local_path)
-
-    return {"status": "ok", "detail": "Results submitted."} 

src/display/css_html_js.py

@@ -1,105 +0,0 @@
-custom_css = """
-
-.markdown-text {
-    font-size: 16px !important;
-}
-
-#models-to-add-text {
-    font-size: 18px !important;
-}
-
-#citation-button span {
-    font-size: 16px !important;
-}
-
-#citation-button textarea {
-    font-size: 16px !important;
-}
-
-#citation-button > label > button {
-    margin: 6px;
-    transform: scale(1.3);
-}
-
-#leaderboard-table {
-    margin-top: 15px
-}
-
-#leaderboard-table-lite {
-    margin-top: 15px
-}
-
-#search-bar-table-box > div:first-child {
-    background: none;
-    border: none;
-}
- 
-#search-bar {
-    padding: 0px;
-}
-
-/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
-#leaderboard-table td:nth-child(2),
-#leaderboard-table th:nth-child(2) {
-    max-width: 400px;
-    overflow: auto;
-    white-space: nowrap;
-}
-
-.tab-buttons button {
-    font-size: 20px;
-}
-
-#scale-logo {
-    border-style: none !important;
-    box-shadow: none;
-    display: block;
-    margin-left: auto;
-    margin-right: auto;
-    max-width: 600px;
-}
-
-#scale-logo .download {
-    display: none;
-}
-#filter_type{
-    border: 0;
-    padding-left: 0;
-    padding-top: 0;
-}
-#filter_type label {
-    display: flex;
-}
-#filter_type label > span{
-    margin-top: var(--spacing-lg);
-    margin-right: 0.5em;
-}
-#filter_type label > .wrap{
-    width: 103px;
-}
-#filter_type label > .wrap .wrap-inner{  
-    padding: 2px;
-}
-#filter_type label > .wrap .wrap-inner input{
-    width: 1px
-}
-#filter-columns-type{
-    border:0;
-    padding:0.5;
-}
-#filter-columns-size{
-    border:0;
-    padding:0.5;
-}
-#box-filter > .form{
-    border: 0
-}
-"""
-
-get_window_url_params = """
-    function(url_params) {
-        const params = new URLSearchParams(window.location.search);
-        url_params = Object.fromEntries(params);
-        return url_params;
-    }
-    """

src/display/formatting.py

@@ -1,27 +0,0 @@
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def make_clickable_model(model_name):
-    link = f"https://huggingface.co/{model_name}"
-    return model_hyperlink(link, model_name)
-
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_warning(warn):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
-
-
-def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
-
-
-def has_no_nan_values(df, columns):
-    return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -1,116 +0,0 @@
-from dataclasses import dataclass, make_dataclass
-from enum import Enum
-
-import pandas as pd
-
-from src.about import Tasks
-
-def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
-
-
-# These classes are for user facing column names,
-# to avoid having to change them all around the code
-# when a modif is needed
-@dataclass
-class ColumnContent:
-    name: str
-    type: str
-    displayed_by_default: bool
-    hidden: bool = False
-    never_hidden: bool = False
-
-## Leaderboard columns
-auto_eval_column_dict = []
-# Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-# Average kept but not displayed by default
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", False)])
-
-# Determine which metrics are visible by default
-_DEFAULT_VISIBLE = {"bleu", "multimetric", "pass_at_1", "pass_at_5", "pass_at_10"}
-
-for task in Tasks:
-    show = task.name in _DEFAULT_VISIBLE
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", show)])
-
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-
-# We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
-
-## For the queue columns in the submission tab
-@dataclass(frozen=True)
-class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
-
-## All the model information that we might need
-@dataclass
-class ModelDetails:
-    name: str
-    display_name: str = ""
-    symbol: str = "" # emoji
-
-
-class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelDetails(name="", symbol="?")
-
-    def to_str(self, separator=" "):
-        return f"{self.value.symbol}{separator}{self.value.name}"
-
-    @staticmethod
-    def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
-        return ModelType.Unknown
-
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
-
-class Precision(Enum):
-    float16 = ModelDetails("float16")
-    bfloat16 = ModelDetails("bfloat16")
-    Unknown = ModelDetails("?")
-
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        return Precision.Unknown
-
-# Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-
-EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-

src/envs.py

@@ -1,25 +0,0 @@
-import os
-
-from huggingface_hub import HfApi
-
-# Info to change for your repository
-# ----------------------------------
-TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
-
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
-# ----------------------------------
-
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
-
-# If you setup a cache later, just change HF_HOME
-CACHE_PATH=os.getenv("HF_HOME", ".")
-
-# Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
-
-API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -1,199 +0,0 @@
-import glob
-import json
-import math
-import os
-from dataclasses import dataclass
-
-import dateutil
-import numpy as np
-
-from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
-
-
-@dataclass
-class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
-    """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
-    model: str
-    revision: str # commit hash, "" if main
-    results: dict
-    precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
-    license: str = "?"
-    likes: int = 0
-    num_params: int = 0
-    date: str = "" # submission date of request file
-    still_on_hub: bool = False
-
-    @classmethod
-    def init_from_json_file(self, json_filepath):
-        """Inits the result from the specific model result file"""
-        with open(json_filepath) as fp:
-            data = json.load(fp)
-
-        config = data.get("config")
-
-        # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
-
-        # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
-        org_and_model = org_and_model.split("/", 1)
-
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
-        else:
-            org = org_and_model[0]
-            model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
-        full_model = "/".join(org_and_model)
-
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
-
-        # Extract results available in this file (some results are split in several files)
-        results = {}
-        for task in Tasks:
-            task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            if task.metric == "score":
-                mean_acc = float(np.mean(accs))
-            else:
-                mean_acc = float(np.mean(accs) * 100.0)
-            results[task.benchmark] = mean_acc
-
-        return self(
-            eval_name=result_key,
-            full_model=full_model,
-            org=org,
-            model=model,
-            results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
-            architecture=architecture
-        )
-
-    def update_with_request_file(self, requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
-            self.license = request.get("license", "?")
-            self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
-            self.date = request.get("submitted_time", "")
-        except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
-
-    def to_dict(self):
-        """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-        data_dict = {
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-        }
-
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
-
-        return data_dict
-
-
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file (precision)
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
-                request_file = tmp_request_file
-    return request_file
-
-
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
-    """From the path of the results folder root, extract all needed info for results"""
-    model_result_filepaths = []
-
-    for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
-
-        # Store results of same eval together
-        eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
-
-    results = []
-    for v in eval_results.values():
-        try:
-            v.to_dict() # we test if the dict version is complete
-            results.append(v)
-        except KeyError:  # not all eval values present
-            continue
-
-    return results

src/populate.py

@@ -1,102 +0,0 @@
-import json
-import os
-
-import pandas as pd
-
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
-
-
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
-
-    df = pd.DataFrame.from_records(all_data_json)
-
-    # ------------------------------------------------------------------
-    # Fallback: if no evaluation results are found we populate the
-    # leaderboard with a single example model. This guarantees that a
-    # freshly deployed Space shows a non-empty leaderboard and it serves
-    # as a template for the expected columns/values.
-    # ------------------------------------------------------------------
-    if df.empty:
-        example_row = {}
-
-        # Populate benchmark metrics with the default value 0.5 using internal column names
-        for metric in benchmark_cols:
-            example_row[metric] = 0.5
-
-        # Minimal metadata so that the row displays nicely
-        example_row[AutoEvalColumn.model.name] = make_clickable_model("example/model")
-        example_row[AutoEvalColumn.average.name] = 0.5
-        example_row[AutoEvalColumn.model_type_symbol.name] = "🟢"
-        example_row[AutoEvalColumn.model_type.name] = "pretrained"
-        example_row[AutoEvalColumn.precision.name] = "float16"
-        example_row[AutoEvalColumn.weight_type.name] = "Original"
-        example_row[AutoEvalColumn.still_on_hub.name] = True
-        example_row[AutoEvalColumn.architecture.name] = "Transformer"
-        example_row[AutoEvalColumn.revision.name] = "main"
-        example_row[AutoEvalColumn.license.name] = "apache-2.0"
-
-        # Any missing columns will be created later in the function
-        df = pd.DataFrame([example_row])
-
-    # Sort primarily by LLM exact-match Pass@1 metric; if not present, fall back to average
-    preferred_cols = []
-    if hasattr(AutoEvalColumn, "pass_at_1"):
-        preferred_cols.append(AutoEvalColumn.pass_at_1.name)
-    preferred_cols.append(AutoEvalColumn.average.name)
-
-    for col in preferred_cols:
-        if col in df.columns:
-            df = df.sort_values(by=[col], ascending=False)
-            break
-
-    # Ensure all expected columns exist, add missing ones with NaN so selection does not fail
-    for expected in cols:
-        if expected not in df.columns:
-            df[expected] = pd.NA
-
-    df = df[cols].round(decimals=2)
-
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
-
-
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -1,99 +0,0 @@
-import json
-import os
-import re
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-
-import huggingface_hub
-from huggingface_hub import ModelCard
-from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
-
-def check_model_card(repo_id: str) -> tuple[bool, str]:
-    """Checks if the model card and license exist and have been filled"""
-    try:
-        card = ModelCard.load(repo_id)
-    except huggingface_hub.utils.EntryNotFoundError:
-        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
-
-    # Enforce license metadata
-    if card.data.license is None:
-        if not ("license_name" in card.data and "license_link" in card.data):
-            return False, (
-                "License not found. Please add a license to your model card using the `license` metadata or a"
-                " `license_name`/`license_link` pair."
-            )
-
-    # Enforce card content
-    if len(card.text) < 200:
-        return False, "Please add a description to your model card, it is too short."
-
-    return True, ""
-
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
-    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
-    try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-        if test_tokenizer:
-            try:
-                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-            except ValueError as e:
-                return (
-                    False,
-                    f"uses a tokenizer which is not in a transformers release: {e}",
-                    None
-                )
-            except Exception as e:
-                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
-        return True, None, config
-
-    except ValueError:
-        return (
-            False,
-            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-            None
-        )
-
-    except Exception as e:
-        return False, "was not found on hub!", None
-
-
-def get_model_size(model_info: ModelInfo, precision: str):
-    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
-    try:
-        model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
-    model_size = size_factor * model_size
-    return model_size
-
-def get_model_arch(model_info: ModelInfo):
-    """Gets the model architecture from the configuration"""
-    return model_info.config.get("architectures", "Unknown")
-
-def already_submitted_models(requested_models_dir: str) -> set[str]:
-    """Gather a list of already submitted models to avoid duplicates"""
-    depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
-
-    for root, _, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            for file in files:
-                if not file.endswith(".json"):
-                    continue
-                with open(os.path.join(root, file), "r") as f:
-                    info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
-
-                    # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
-
-    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -1,248 +0,0 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO, EVAL_RESULTS_PATH, RESULTS_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )
-
-# --------------------------------------------------------
-# Manual metrics submission (bypass evaluation queue)
-# --------------------------------------------------------
-
-ALL_SUBJECTIVE_FIELDS = [
-    "readability",
-    "relevance",
-    "explanation_clarity",
-    "problem_identification",
-    "actionability",
-    "completeness",
-    "specificity",
-    "contextual_adequacy",
-    "consistency",
-    "brevity",
-]
-
-def _compute_multimetric(payload: dict) -> float:
-    """Average of the 10 subjective metrics."""
-    total = sum(float(payload[f]) for f in ALL_SUBJECTIVE_FIELDS)
-    return total / len(ALL_SUBJECTIVE_FIELDS)
-
-def add_manual_results(
-    model: str,
-    revision: str,
-    bleu: float,
-    readability: int,
-    relevance: int,
-    explanation_clarity: int,
-    problem_identification: int,
-    actionability: int,
-    completeness: int,
-    specificity: int,
-    contextual_adequacy: int,
-    consistency: int,
-    brevity: int,
-    pass_at_1: float,
-    pass_at_5: float,
-    pass_at_10: float,
-):
-    """Directly submit evaluation metrics for a model and push them to the results dataset."""
-
-    # Basic validation
-    if model == "":
-        return styled_error("Please specify a model name.")
-
-    if revision == "":
-        revision = "main"
-
-    if pass_at_5 < pass_at_1:
-        return styled_error("pass@5 must be greater or equal to pass@1")
-    if pass_at_10 < pass_at_5:
-        return styled_error("pass@10 must be greater or equal to pass@5")
-
-    # Prepare dictionary in the same format used by read_evals.py
-    payload_dict = {
-        "model": model,
-        "revision": revision,
-        "bleu": bleu,
-        "readability": readability,
-        "relevance": relevance,
-        "explanation_clarity": explanation_clarity,
-        "problem_identification": problem_identification,
-        "actionability": actionability,
-        "completeness": completeness,
-        "specificity": specificity,
-        "contextual_adequacy": contextual_adequacy,
-        "consistency": consistency,
-        "brevity": brevity,
-        "pass_at_1": pass_at_1,
-        "pass_at_5": pass_at_5,
-        "pass_at_10": pass_at_10,
-    }
-
-    multimetric = _compute_multimetric(payload_dict)
-
-    # Compose final results file (same structure as api_submit_results)
-    result_json = {
-        "config": {
-            "model_dtype": "unknown",
-            "model_name": model,
-            "model_sha": revision,
-        },
-        "results": {
-            "bleu": {"score": bleu},
-            "multimetric": {"score": multimetric},
-            "pass_at_1": {"score": pass_at_1},
-            "pass_at_5": {"score": pass_at_5},
-            "pass_at_10": {"score": pass_at_10},
-        },
-    }
-
-    # Add subjective metrics
-    for field in ALL_SUBJECTIVE_FIELDS:
-        result_json["results"][field] = {"score": payload_dict[field]}
-
-    # Write file locally then upload
-    try:
-        os.makedirs(EVAL_RESULTS_PATH, exist_ok=True)
-    except Exception:
-        pass
-
-    from datetime import datetime, timezone
-    import uuid
-
-    ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
-    unique_id = uuid.uuid4().hex[:8]
-    filename = f"results_{model.replace('/', '_')}_{ts}_{unique_id}.json"
-    local_path = os.path.join(EVAL_RESULTS_PATH, filename)
-
-    try:
-        with open(local_path, "w") as fp:
-            json.dump(result_json, fp)
-
-        API.upload_file(
-            path_or_fileobj=local_path,
-            path_in_repo=filename,
-            repo_id=RESULTS_REPO,
-            repo_type="dataset",
-            commit_message=f"Add manual results for {model}",
-        )
-    except Exception as e:
-        return styled_error(f"Failed to upload results: {e}")
-    finally:
-        if os.path.exists(local_path):
-            os.remove(local_path)
-
-    return styled_message("Metrics successfully submitted! The leaderboard will refresh shortly.")