Spaces:
Running
Running
File size: 3,528 Bytes
362d13b 8b0cd75 362d13b 8b0cd75 362d13b 1c7c01e 8b0cd75 362d13b 1c7c01e 362d13b 8b0cd75 28362c5 1c7c01e 28362c5 1c7c01e 28362c5 362d13b 8b0cd75 362d13b 8b0cd75 362d13b 8b0cd75 4d25207 362d13b 1c7c01e 8b0cd75 1c7c01e 8b0cd75 362d13b 8b0cd75 362d13b bfcd720 157452e bfcd720 157452e 362d13b 8b0cd75 362d13b 8b0cd75 362d13b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 |
import json
import random
from pathlib import Path
import gradio as gr
import pandas as pd
from datasets import load_dataset
abs_path = Path(__file__).parent
submissions = json.load(open(abs_path / "submissions.json"))
TASKS = [("gsm8k", "lighteval|gsm8k|0", "extractive_match")]
TYPES = [
"markdown",
"markdown",
"number",
"number",
"markdown",
]
COLUMNS = ["User", "Model Name", "MMLU", "Average ⬆️", "Results"]
WIDTHS = ["25%", "25%", "15%", "15%", "10%"]
def load_results(dataset):
results = []
try:
output = dataset["latest"]["results"]
output = output[-1]
except KeyError as e:
raise ValueError("Cannot find 'latest' key in the dataset")
try:
output = json.loads(output)
except ValueError as e:
raise ValueError("Cannot parse the output as JSON")
for name, task, metric in TASKS:
try:
output = output[task]
except KeyError as e:
raise ValueError(f"Cannot find '{task}' key in the dataset")
try:
output = (name, output[metric])
except KeyError as e:
raise ValueError("Cannot find 'extractive_match' key in the dataset")
results.append(output)
return results
def load_submissions():
leaderboard = []
for i, submission in enumerate(submissions["submissions"]):
try:
ds = load_dataset(submission["results-dataset"], "results")
results = load_results(ds)
except ValueError as e:
print(f"Cannot load results for {ds['results-dataset']} {e}")
except Exception as e:
print(f"Cannot load dataset for {i} : {submission['results-dataset']} {e}")
continue
leaderboard_row = {}
leaderboard_row["username"] = (
f"[{submission['username']}](https://huggingface.co/{submission['username']})"
)
leaderboard_row["model_name"] = (
f"[{submission['model_name']}](https://huggingface.co/{submission['username']}/{submission['model_name']})"
)
for name, result in results:
leaderboard_row[name] = result
leaderboard_row["Average ⬆️"] = sum(result for _, result in results) / len(
results
)
leaderboard_row["results-dataset"] = (
f"[🔗](https://huggingface.co/datasets/{submission['results-dataset']})"
)
leaderboard.append(leaderboard_row)
df = pd.DataFrame(leaderboard)
for column in df.columns:
if df[column].dtype == "float64":
df[column] = df[column].round(2)
return df.sort_values(by="Average ⬆️", ascending=False)
with gr.Blocks() as demo:
gr.Markdown("""
# 🥇 a smol course leaderboad
A leaderboard of smol course students' submissions.
""")
with gr.Tabs():
with gr.Tab("Demo"):
df = gr.Dataframe(
label="a smol course leaderboard",
value=load_submissions(),
headers=COLUMNS,
show_search="search",
show_copy_button=True,
show_fullscreen_button=True,
show_row_numbers=True,
pinned_columns=1,
static_columns=[0],
datatype=TYPES,
column_widths=WIDTHS,
)
with gr.Tab("Docs"):
gr.Markdown((Path(__file__).parent / "docs.md").read_text())
if __name__ == "__main__":
demo.launch()
|