sapce

main.py

@@ -0,0 +1,166 @@
+from typing import List
+
+import os
+
+import pandas as pd
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel, Field, validator
+import gradio as gr
+
+# -----------------------------------------------------------------------------
+# Constants
+# -----------------------------------------------------------------------------
+CSV_PATH = os.getenv("LEADERBOARD_CSV", "leaderboard.csv")
+LEADERBOARD_COLUMNS = [
+    "model_name",
+    "bleu",
+    "multimetric",
+    # individual multimetric dimensions
+    "readability",
+    "relevance",
+    "explanation_clarity",
+    "problem_identification",
+    "actionability",
+    "completeness",
+    "specificity",
+    "contextual_adequacy",
+    "consistency",
+    "brevity",
+    # exact-match metrics
+    "pass_at_1",
+    "pass_at_5",
+    "pass_at_10",
+]
+
+
+# -----------------------------------------------------------------------------
+# Pydantic schema for incoming submissions
+# -----------------------------------------------------------------------------
+class Submission(BaseModel):
+    model_name: str = Field(..., description="Arbitrary display name for the submission")
+
+    # automatic metric
+    bleu: float = Field(..., ge=0, description="BLEU score (0-100)")
+
+    # ten subjective dimensions
+    readability: int = Field(..., ge=0, le=5)
+    relevance: int = Field(..., ge=0, le=5)
+    explanation_clarity: int = Field(..., ge=0, le=5)
+    problem_identification: int = Field(..., ge=0, le=5)
+    actionability: int = Field(..., ge=0, le=5)
+    completeness: int = Field(..., ge=0, le=5)
+    specificity: int = Field(..., ge=0, le=5)
+    contextual_adequacy: int = Field(..., ge=0, le=5)
+    consistency: int = Field(..., ge=0, le=5)
+    brevity: int = Field(..., ge=0, le=5)
+
+    # exact-match pass@k
+    pass_at_1: float = Field(..., ge=0, le=1)
+    pass_at_5: float = Field(..., ge=0, le=1)
+    pass_at_10: float = Field(..., ge=0, le=1)
+
+    @validator("pass_at_5")
+    def pass5_ge_pass1(cls, v, values):
+        if "pass_at_1" in values and v < values["pass_at_1"]:
+            raise ValueError("pass@5 must be >= pass@1")
+        return v
+
+    @validator("pass_at_10")
+    def pass10_ge_pass5(cls, v, values):
+        if "pass_at_5" in values and v < values["pass_at_5"]:
+            raise ValueError("pass@10 must be >= pass@5")
+        return v
+
+    # computed property (not part of submission payload)
+    def compute_multimetric(self) -> float:
+        fields = [
+            self.readability,
+            self.relevance,
+            self.explanation_clarity,
+            self.problem_identification,
+            self.actionability,
+            self.completeness,
+            self.specificity,
+            self.contextual_adequacy,
+            self.consistency,
+            self.brevity,
+        ]
+        return float(sum(fields)) / len(fields)
+
+
+# -----------------------------------------------------------------------------
+# Helpers
+# -----------------------------------------------------------------------------
+
+def _init_storage(csv_path: str):
+    """Ensure the CSV exists with the correct header"""
+    if not os.path.exists(csv_path):
+        df = pd.DataFrame(columns=LEADERBOARD_COLUMNS)
+        df.to_csv(csv_path, index=False)
+
+
+def _load_leaderboard() -> pd.DataFrame:
+    _init_storage(CSV_PATH)
+    df = pd.read_csv(CSV_PATH)
+    # sort descending by Pass@1
+    if not df.empty and "pass_at_1" in df.columns:
+        df = df.sort_values("pass_at_1", ascending=False)
+    return df
+
+
+def _save_submission(sub: Submission):
+    _init_storage(CSV_PATH)
+    df = pd.read_csv(CSV_PATH)
+
+    # Remove previous entry for the same model (if any)
+    df = df[df["model_name"] != sub.model_name]
+
+    # Compose new row
+    record = sub.dict()
+    record["multimetric"] = sub.compute_multimetric()
+    df = pd.concat([df, pd.DataFrame([record])], ignore_index=True)
+
+    # keep ordering of columns
+    df = df[LEADERBOARD_COLUMNS]
+    df.to_csv(CSV_PATH, index=False)
+
+
+# -----------------------------------------------------------------------------
+# FastAPI backend
+# -----------------------------------------------------------------------------
+api = FastAPI(title="Leaderboard API", version="0.1.0")
+
+
+@api.post("/submit", tags=["submission"])
+async def submit_results(payload: Submission):
+    """Receive a new result entry and persist it."""
+    try:
+        _save_submission(payload)
+        return {"status": "ok", "detail": "Submission stored."}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@api.get("/leaderboard", tags=["leaderboard"])
+async def get_leaderboard():
+    """Return the current leaderboard as JSON (sorted by Pass@1)."""
+    return _load_leaderboard().to_dict(orient="records")
+
+
+# -----------------------------------------------------------------------------
+# Gradio frontend
+# -----------------------------------------------------------------------------
+
+def _load_leaderboard_df():
+    return _load_leaderboard()
+
+
+with gr.Blocks(title="📊 Leaderboard") as demo:
+    gr.Markdown("# 📊 Leaderboard — sorted by **LLM-based exact-match Pass@1**")
+    df_component = gr.Dataframe(value=_load_leaderboard_df(), interactive=False, wrap=True)
+    refresh_btn = gr.Button("🔄 Refresh")
+    refresh_btn.click(lambda: _load_leaderboard_df(), outputs=df_component)
+
+
+# Mount gradio under "/"
+app = gr.mount_gradio_app(api, demo, path="/") 

requirements.txt

@@ -13,4 +13,5 @@ python-dateutil
 tqdm
 transformers
 tokenizers>=0.15.0
-sentencepiece
+sentencepiece
+fastapi