change structure

app.py

@@ -1,35 +1,95 @@
+from pathlib import Path
+import json
+
 import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
+from huggingface_hub import snapshot_download
+from gradio_leaderboard import Leaderboard, SelectColumns
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
+from ttsds.benchmarks.benchmark import BenchmarkCategory
+from ttsds import BenchmarkSuite
+
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN, TAGS
+from src.texts import LLM_BENCHMARKS_TEXT, EVALUATION_QUEUE_TEXT
+from src.css_html_js import custom_css
 
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    fields,
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
+
+def filter_dfs(tags, lb):
+    global f_b_df, f_a_df
+    is_agg = False
+    if "Environment" in lb.columns:
+        is_agg = True
+    if is_agg:
+        lb = f_a_df.copy()
+    else:
+        lb = f_b_df.copy()
+    if tags and len(lb) > 0:
+        lb = lb[lb["Tags"].apply(lambda x: any(tag in x for tag in tags))]
+    return lb
 
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 
+def submit_eval(model_name, model_tags, web_url, hf_url, code_url, paper_url, inference_details, file_path):
+    model_id = model_name.lower().replace(" ", "_")
+    # check if model already exists
+    if Path(f"{EVAL_REQUESTS_PATH}/{model_id}.json").exists():
+        return "Model already exists in the evaluation queue"
+    # check which urls are valid
+    if web_url and not web_url.startswith("http"):
+        return "Please enter a valid URL"
+    if hf_url and not hf_url.startswith("http"):
+        return "Please enter a valid URL"
+    if code_url and not code_url.startswith("http"):
+        return "Please enter a valid URL"
+    if paper_url and not paper_url.startswith("http"):
+        return "Please enter a valid URL"
+    # move file to correct location
+    if not file_path.endswith(".tar.gz"):
+        return "Please upload a .tar.gz file"
+    Path(file_path).rename(f"{EVAL_REQUESTS_PATH}/{model_id}.tar.gz")
+    # build display name - use web_url to link text if available, and emojis for the other urls
+    display_name = model_name
+    if web_url:
+        display_name = f"[{display_name}]({web_url}) "
+    if hf_url:
+        display_name += f"[🤗]({hf_url})"
+    if code_url:
+        display_name += f"[💻]({code_url})"
+    if paper_url:
+        display_name += f"[📄]({paper_url})"
+    request_obj = {
+        "model_name": model_name,
+        "display_name": display_name,
+        "model_tags": model_tags,
+        "web_url": web_url,
+        "hf_url": hf_url,
+        "code_url": code_url,
+        "paper_url": paper_url,
+        "inference_details": inference_details,
+        "status": "pending",
+    }
+    with open(f"{EVAL_REQUESTS_PATH}/{model_id}.json", "w") as f:
+        json.dump(request_obj, f)
+    API.upload_file(
+        path_or_fileobj=f"{EVAL_REQUESTS_PATH}/{model_id}.json",
+        path_in_repo=f"{model_id}.json",
+        repo_id=QUEUE_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {model_name} to evaluation queue",
+    )
+    API.upload_file(
+        path_or_fileobj=f"{EVAL_REQUESTS_PATH}/{model_id}.tar.gz",
+        path_in_repo=f"{model_id}.tar.gz",
+        repo_id=QUEUE_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {model_name} to evaluation queue",
+    )
+    return "Model submitted successfully 🎉"
+
+
 ### Space initialisation
 try:
     print(EVAL_REQUESTS_PATH)
@@ -57,134 +117,154 @@ except Exception:
     restart_space()
 
 
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+results_df = pd.read_csv(EVAL_RESULTS_PATH + "/results.csv")
+
+agg_df = BenchmarkSuite.aggregate_df(results_df)
+agg_df = agg_df.pivot(index="dataset", columns="benchmark_category", values="score")
+agg_df.rename(columns={"OVERALL": "General"}, inplace=True)
+agg_df.columns = [x.capitalize() for x in agg_df.columns]
+agg_df["Mean"] = agg_df.mean(axis=1)
+# make sure mean is the first column
+agg_df = agg_df[["Mean"] + [col for col in agg_df.columns if col != "Mean"]]
+for col in agg_df.columns:
+    agg_df[col] = agg_df[col].apply(lambda x: round(x, 2))
+agg_df["Tags"] = ""
+agg_df.reset_index(inplace=True)
+agg_df.rename(columns={"dataset": "Model"}, inplace=True)
+agg_df.sort_values("Mean", ascending=False, inplace=True)
+
+benchmark_df = results_df.pivot(index="dataset", columns="benchmark_name", values="score")
+
+# get benchmark name order by category
+benchmark_order = list(results_df.sort_values("benchmark_category")["benchmark_name"].unique())
+benchmark_df = benchmark_df[benchmark_order]
+benchmark_df = benchmark_df.reset_index()
+benchmark_df.rename(columns={"dataset": "Model"}, inplace=True)
+# set index
+benchmark_df.set_index("Model", inplace=True)
+benchmark_df["Mean"] = benchmark_df.mean(axis=1)
+# make sure mean is the first column
+benchmark_df = benchmark_df[["Mean"] + [col for col in benchmark_df.columns if col != "Mean"]]
+# round all
+for col in benchmark_df.columns:
+    benchmark_df[col] = benchmark_df[col].apply(lambda x: round(x, 2))
+benchmark_df["Tags"] = ""
+benchmark_df.reset_index(inplace=True)
+benchmark_df.sort_values("Mean", ascending=False, inplace=True)
 
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+# get details for each model
+model_detail_files = Path(EVAL_REQUESTS_PATH).glob("*.json")
+model_details = {}
+for model_detail_file in model_detail_files:
+    with open(model_detail_file) as f:
+        model_detail = json.load(f)
+    model_details[model_detail_file.stem] = model_detail
+
+# replace .tar.gz
+benchmark_df["Model"] = benchmark_df["Model"].apply(lambda x: x.replace(".tar.gz", ""))
+agg_df["Model"] = agg_df["Model"].apply(lambda x: x.replace(".tar.gz", ""))
+
+benchmark_df["Tags"] = benchmark_df["Model"].apply(lambda x: model_details.get(x, {}).get("model_tags", ""))
+agg_df["Tags"] = agg_df["Model"].apply(lambda x: model_details.get(x, {}).get("model_tags", ""))
+
+benchmark_df["Model"] = benchmark_df["Model"].apply(lambda x: model_details.get(x, {}).get("display_name", x))
+agg_df["Model"] = agg_df["Model"].apply(lambda x: model_details.get(x, {}).get("display_name", x))
+
+f_b_df = benchmark_df.copy()
+f_a_df = agg_df.copy()
 
 
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
+    df_types = []
+    for col in dataframe.columns:
+        if col == "Model":
+            df_types.append("markdown")
+        elif col == "Tags":
+            df_types.append("markdown")
+        else:
+            df_types.append("number")
     return Leaderboard(
         value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
         select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+            default_selection=list(dataframe.columns),
+            cant_deselect=["Model", "Mean"],
             label="Select Columns to Display:",
         ),
-        search_columns=[AutoEvalColumn.model.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
+        search_columns=["Model", "Tags"],
+        filter_columns=[],
+        hide_columns=["Tags"],
         interactive=False,
+        datatype=df_types,
     )
 
 
-def show_leaderboard(profile: gr.OAuthProfile | None, oauth_token: gr.OAuthToken | None):
-    if profile or True:
-        gr.HTML(TITLE)
-        gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+app = gr.Blocks(css=custom_css, title="TTS Benchmark Leaderboard")
 
-        with gr.Tabs(elem_classes="tab-buttons") as tabs:
-            with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-                leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-            with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-                gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-            with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-                with gr.Column():
-                    with gr.Row():
-                        gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                    with gr.Column():
-                        with gr.Accordion(
-                            f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                            open=False,
-                        ):
-                            with gr.Row():
-                                finished_eval_table = gr.components.Dataframe(
-                                    value=finished_eval_queue_df,
-                                    headers=EVAL_COLS,
-                                    datatype=EVAL_TYPES,
-                                    row_count=5,
-                                )
-                        with gr.Accordion(
-                            f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                            open=False,
-                        ):
-                            with gr.Row():
-                                running_eval_table = gr.components.Dataframe(
-                                    value=running_eval_queue_df,
-                                    headers=EVAL_COLS,
-                                    datatype=EVAL_TYPES,
-                                    row_count=5,
-                                )
-
-                        with gr.Accordion(
-                            f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                            open=False,
-                        ):
-                            with gr.Row():
-                                pending_eval_table = gr.components.Dataframe(
-                                    value=pending_eval_queue_df,
-                                    headers=EVAL_COLS,
-                                    datatype=EVAL_TYPES,
-                                    row_count=5,
-                                )
+with app:
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 TTSDB Scores", elem_id="llm-benchmark-tab-table", id=0):
+            tags = gr.Dropdown(
+                TAGS,
+                value=[],
+                multiselect=True,
+                label="Tags",
+                info="Select tags to filter the leaderboard. You can suggest new tags here: https://huggingface.co/spaces/ttsds/benchmark/discussions/1",
+            )
+            leaderboard = init_leaderboard(f_a_df)
+            tags.change(filter_dfs, [tags, leaderboard], [leaderboard])
+        with gr.TabItem("🏅 Individual Benchmarks", elem_id="llm-benchmark-tab-table", id=1):
+            tags = gr.Dropdown(
+                TAGS,
+                value=[],
+                multiselect=True,
+                label="Tags",
+                info="Select tags to filter the leaderboard",
+            )
+            leaderboard = init_leaderboard(f_b_df)
+            tags.change(filter_dfs, [tags, leaderboard], [leaderboard])
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
+            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+        with gr.TabItem("🚀 Submit here!", elem_id="llm-benchmark-tab-table", id=3):
+            with gr.Column():
                 with gr.Row():
-                    gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
+                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+                with gr.Row():
+                    gr.Markdown("# ✉️✨ Submit a TTS dataset here!", elem_classes="markdown-text")
                 with gr.Row():
                     with gr.Column():
                         model_name_textbox = gr.Textbox(label="Model name")
-
-                submit_button = gr.Button("Submit Eval")
-                submission_result = gr.Markdown()
-                submit_button.click(
-                    add_new_eval,
-                    [
-                        model_name_textbox,
-                    ],
-                    submission_result,
-                )
-
-        with gr.Row():
-            with gr.Accordion("📙 Citation", open=False):
-                citation_button = gr.Textbox(
-                    value=CITATION_BUTTON_TEXT,
-                    label=CITATION_BUTTON_LABEL,
-                    lines=20,
-                    elem_id="citation-button",
-                    show_copy_button=True,
-                )
-
-
-demo = gr.Blocks(css=custom_css)
-
-with demo:
-    # gr.LoginButton()
-    m1 = gr.Markdown("Please login to see the leaderboard.")
-    # demo.load(show_leaderboard, inputs=None, outputs=m1)
-    show_leaderboard(None, None)
-
+                        model_tags_dropdown = gr.Dropdown(
+                            label="Model tags",
+                            choices=TAGS,
+                            multiselect=True,
+                        )
+                        website_url_textbox = gr.Textbox(label="Website URL (optional)")
+                        hf_url_textbox = gr.Textbox(label="Huggingface URL (optional)")
+                        code_url_textbox = gr.Textbox(label="Code URL (optional)")
+                        paper_url_textbox = gr.Textbox(label="Paper URL (optional)")
+                        inference_details_textbox = gr.TextArea(label="Inference details (optional)")
+                        file_input = gr.File(file_types=[".gz"], interactive=True, label=".tar.gz TTS dataset")
+                        submit_button = gr.Button("Submit Eval")
+                        submission_result = gr.Markdown()
+                        submit_button.click(
+                            submit_eval,
+                            [
+                                model_name_textbox,
+                                model_tags_dropdown,
+                                website_url_textbox,
+                                hf_url_textbox,
+                                code_url_textbox,
+                                paper_url_textbox,
+                                inference_details_textbox,
+                                file_input,
+                            ],
+                            submission_result,
+                        )
 
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.start()
 
-demo.queue(default_concurrency_limit=40).launch()
-# demo.launch()
+app.queue(default_concurrency_limit=40).launch()

new/app.py

@@ -1,270 +0,0 @@
-from pathlib import Path
-import json
-
-import gradio as gr
-from huggingface_hub import snapshot_download
-from gradio_leaderboard import Leaderboard, SelectColumns
-import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from ttsdb.benchmarks.benchmark import BenchmarkCategory
-from ttsdb import BenchmarkSuite
-
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN, TAGS
-from src.texts import LLM_BENCHMARKS_TEXT, EVALUATION_QUEUE_TEXT
-from src.css_html_js import custom_css
-
-
-def filter_dfs(tags, lb):
-    global f_b_df, f_a_df
-    is_agg = False
-    if "Environment" in lb.columns:
-        is_agg = True
-    if is_agg:
-        lb = f_a_df.copy()
-    else:
-        lb = f_b_df.copy()
-    if tags and len(lb) > 0:
-        lb = lb[lb["Tags"].apply(lambda x: any(tag in x for tag in tags))]
-    return lb
-
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
-
-def submit_eval(model_name, model_tags, web_url, hf_url, code_url, paper_url, inference_details, file_path):
-    model_id = model_name.lower().replace(" ", "_")
-    # check if model already exists
-    if Path(f"{EVAL_REQUESTS_PATH}/{model_id}.json").exists():
-        return "Model already exists in the evaluation queue"
-    # check which urls are valid
-    if web_url and not web_url.startswith("http"):
-        return "Please enter a valid URL"
-    if hf_url and not hf_url.startswith("http"):
-        return "Please enter a valid URL"
-    if code_url and not code_url.startswith("http"):
-        return "Please enter a valid URL"
-    if paper_url and not paper_url.startswith("http"):
-        return "Please enter a valid URL"
-    # move file to correct location
-    if not file_path.endswith(".tar.gz"):
-        return "Please upload a .tar.gz file"
-    Path(file_path).rename(f"{EVAL_REQUESTS_PATH}/{model_id}.tar.gz")
-    # build display name - use web_url to link text if available, and emojis for the other urls
-    display_name = model_name
-    if web_url:
-        display_name = f"[{display_name}]({web_url}) "
-    if hf_url:
-        display_name += f"[🤗]({hf_url})"
-    if code_url:
-        display_name += f"[💻]({code_url})"
-    if paper_url:
-        display_name += f"[📄]({paper_url})"
-    request_obj = {
-        "model_name": model_name,
-        "display_name": display_name,
-        "model_tags": model_tags,
-        "web_url": web_url,
-        "hf_url": hf_url,
-        "code_url": code_url,
-        "paper_url": paper_url,
-        "inference_details": inference_details,
-        "status": "pending",
-    }
-    with open(f"{EVAL_REQUESTS_PATH}/{model_id}.json", "w") as f:
-        json.dump(request_obj, f)
-    API.upload_file(
-        path_or_fileobj=f"{EVAL_REQUESTS_PATH}/{model_id}.json",
-        path_in_repo=f"{model_id}.json",
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model_name} to evaluation queue",
-    )
-    API.upload_file(
-        path_or_fileobj=f"{EVAL_REQUESTS_PATH}/{model_id}.tar.gz",
-        path_in_repo=f"{model_id}.tar.gz",
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model_name} to evaluation queue",
-    )
-    return "Model submitted successfully 🎉"
-
-
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO,
-        local_dir=EVAL_REQUESTS_PATH,
-        repo_type="dataset",
-        tqdm_class=None,
-        etag_timeout=30,
-        token=TOKEN,
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO,
-        local_dir=EVAL_RESULTS_PATH,
-        repo_type="dataset",
-        tqdm_class=None,
-        etag_timeout=30,
-        token=TOKEN,
-    )
-except Exception:
-    restart_space()
-
-
-results_df = pd.read_csv(EVAL_RESULTS_PATH + "/results.csv")
-
-agg_df = BenchmarkSuite.aggregate_df(results_df)
-agg_df = agg_df.pivot(index="dataset", columns="benchmark_category", values="score")
-agg_df.rename(columns={"OVERALL": "General"}, inplace=True)
-agg_df.columns = [x.capitalize() for x in agg_df.columns]
-agg_df["Mean"] = agg_df.mean(axis=1)
-# make sure mean is the first column
-agg_df = agg_df[["Mean"] + [col for col in agg_df.columns if col != "Mean"]]
-for col in agg_df.columns:
-    agg_df[col] = agg_df[col].apply(lambda x: round(x, 2))
-agg_df["Tags"] = ""
-agg_df.reset_index(inplace=True)
-agg_df.rename(columns={"dataset": "Model"}, inplace=True)
-agg_df.sort_values("Mean", ascending=False, inplace=True)
-
-benchmark_df = results_df.pivot(index="dataset", columns="benchmark_name", values="score")
-
-# get benchmark name order by category
-benchmark_order = list(results_df.sort_values("benchmark_category")["benchmark_name"].unique())
-benchmark_df = benchmark_df[benchmark_order]
-benchmark_df = benchmark_df.reset_index()
-benchmark_df.rename(columns={"dataset": "Model"}, inplace=True)
-# set index
-benchmark_df.set_index("Model", inplace=True)
-benchmark_df["Mean"] = benchmark_df.mean(axis=1)
-# make sure mean is the first column
-benchmark_df = benchmark_df[["Mean"] + [col for col in benchmark_df.columns if col != "Mean"]]
-# round all
-for col in benchmark_df.columns:
-    benchmark_df[col] = benchmark_df[col].apply(lambda x: round(x, 2))
-benchmark_df["Tags"] = ""
-benchmark_df.reset_index(inplace=True)
-benchmark_df.sort_values("Mean", ascending=False, inplace=True)
-
-# get details for each model
-model_detail_files = Path(EVAL_REQUESTS_PATH).glob("*.json")
-model_details = {}
-for model_detail_file in model_detail_files:
-    with open(model_detail_file) as f:
-        model_detail = json.load(f)
-    model_details[model_detail_file.stem] = model_detail
-
-# replace .tar.gz
-benchmark_df["Model"] = benchmark_df["Model"].apply(lambda x: x.replace(".tar.gz", ""))
-agg_df["Model"] = agg_df["Model"].apply(lambda x: x.replace(".tar.gz", ""))
-
-benchmark_df["Tags"] = benchmark_df["Model"].apply(lambda x: model_details.get(x, {}).get("model_tags", ""))
-agg_df["Tags"] = agg_df["Model"].apply(lambda x: model_details.get(x, {}).get("model_tags", ""))
-
-benchmark_df["Model"] = benchmark_df["Model"].apply(lambda x: model_details.get(x, {}).get("display_name", x))
-agg_df["Model"] = agg_df["Model"].apply(lambda x: model_details.get(x, {}).get("display_name", x))
-
-f_b_df = benchmark_df.copy()
-f_a_df = agg_df.copy()
-
-
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    df_types = []
-    for col in dataframe.columns:
-        if col == "Model":
-            df_types.append("markdown")
-        elif col == "Tags":
-            df_types.append("markdown")
-        else:
-            df_types.append("number")
-    return Leaderboard(
-        value=dataframe,
-        select_columns=SelectColumns(
-            default_selection=list(dataframe.columns),
-            cant_deselect=["Model", "Mean"],
-            label="Select Columns to Display:",
-        ),
-        search_columns=["Model", "Tags"],
-        filter_columns=[],
-        hide_columns=["Tags"],
-        interactive=False,
-        datatype=df_types,
-    )
-
-
-app = gr.Blocks(css=custom_css, title="TTS Benchmark Leaderboard")
-
-with app:
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 TTSDB Scores", elem_id="llm-benchmark-tab-table", id=0):
-            tags = gr.Dropdown(
-                TAGS,
-                value=[],
-                multiselect=True,
-                label="Tags",
-                info="Select tags to filter the leaderboard. You can suggest new tags here: https://huggingface.co/spaces/ttsds/benchmark/discussions/1",
-            )
-            leaderboard = init_leaderboard(f_a_df)
-            tags.change(filter_dfs, [tags, leaderboard], [leaderboard])
-        with gr.TabItem("🏅 Individual Benchmarks", elem_id="llm-benchmark-tab-table", id=1):
-            tags = gr.Dropdown(
-                TAGS,
-                value=[],
-                multiselect=True,
-                label="Tags",
-                info="Select tags to filter the leaderboard",
-            )
-            leaderboard = init_leaderboard(f_b_df)
-            tags.change(filter_dfs, [tags, leaderboard], [leaderboard])
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-        with gr.TabItem("🚀 Submit here!", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-                with gr.Row():
-                    gr.Markdown("# ✉️✨ Submit a TTS dataset here!", elem_classes="markdown-text")
-                with gr.Row():
-                    with gr.Column():
-                        model_name_textbox = gr.Textbox(label="Model name")
-                        model_tags_dropdown = gr.Dropdown(
-                            label="Model tags",
-                            choices=TAGS,
-                            multiselect=True,
-                        )
-                        website_url_textbox = gr.Textbox(label="Website URL (optional)")
-                        hf_url_textbox = gr.Textbox(label="Huggingface URL (optional)")
-                        code_url_textbox = gr.Textbox(label="Code URL (optional)")
-                        paper_url_textbox = gr.Textbox(label="Paper URL (optional)")
-                        inference_details_textbox = gr.TextArea(label="Inference details (optional)")
-                        file_input = gr.File(file_types=[".gz"], interactive=True, label=".tar.gz TTS dataset")
-                        submit_button = gr.Button("Submit Eval")
-                        submission_result = gr.Markdown()
-                        submit_button.click(
-                            submit_eval,
-                            [
-                                model_name_textbox,
-                                model_tags_dropdown,
-                                website_url_textbox,
-                                hf_url_textbox,
-                                code_url_textbox,
-                                paper_url_textbox,
-                                inference_details_textbox,
-                                file_input,
-                            ],
-                            submission_result,
-                        )
-
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-
-app.queue(default_concurrency_limit=40).launch()

new/requirements.txt

@@ -1,17 +0,0 @@
-APScheduler
-black
-datasets
-gradio
-gradio[oauth]
-gradio_leaderboard==0.0.9
-gradio_client
-huggingface-hub>=0.18.0
-matplotlib
-numpy
-pandas
-python-dateutil
-tqdm
-transformers
-tokenizers>=0.15.0
-sentencepiece
-markdown

new/src/envs.py

@@ -1,38 +0,0 @@
-import os
-
-from huggingface_hub import HfApi
-
-# Info to change for your repository
-# ----------------------------------
-TOKEN = os.environ.get("TOKEN")  # A read/write token for your org
-
-OWNER = "ttsds"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
-# ----------------------------------
-
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
-
-# If you setup a cache later, just change HF_HOME
-CACHE_PATH = os.getenv("HF_HOME", ".")
-
-# Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
-
-API = HfApi(token=TOKEN)
-
-TAGS = [
-    "Normalizing Flow",
-    "Reference-based (Speaker)",
-    "Prompt-based (Speaker)",
-    "Prosodic Correlates",
-    "Adversarial",
-    "Diffusion",
-    "Audio Tokens",
-    "Autoregressive",
-    "Non-autoregressive",
-    "Pretrained Text Encoder",
-]

requirements.txt

@@ -13,4 +13,5 @@ python-dateutil
 tqdm
 transformers
 tokenizers>=0.15.0
-sentencepiece
+sentencepiece
+ttsds

src/about.py

@@ -1,60 +0,0 @@
-from dataclasses import dataclass
-from enum import Enum
-
-
-@dataclass
-class Task:
-    benchmark: str
-    metric: str
-    col_name: str
-    category: str
-
-
-# Select your tasks here
-# ---------------------------------------------------
-class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
-    task0 = Task("anli_r1", "acc", "ANLI", "")
-    task1 = Task("logiqa", "acc_norm", "LogiQA", "")
-
-
-NUM_FEWSHOT = 0  # Change with your few shot
-# ---------------------------------------------------
-
-
-# Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
-
-# What does your leaderboard evaluate?
-INTRODUCTION_TEXT = """
-Intro text
-"""
-
-# Which evaluations are you running? how can people reproduce what you have?
-LLM_BENCHMARKS_TEXT = f"""
-## How it works
-
-## Reproducibility
-To reproduce our results, check out our repository [here](https://github.com/ttsds/ttsds).
-
-"""
-
-EVALUATION_QUEUE_TEXT = """
-## How to submit a TTS model to the leaderboard
-
-### 1) download the evaluation dataset
-The evaluation dataset consists of wav / text pairs.
-You can download it [here](https://huggingface.co/ttsds/eval).
-
-### 2) create your TTS dataset
-Create a dataset with your TTS model and the evaluation dataset.
-Use the wav files as speaker reference and the text as the prompt.
-Create a .tar.gz file with the dataset, and make sure to inlcude .wav files and .txt files.
-
-### 3) submit your TTS dataset
-Submit your dataset below.
-"""
-
-CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""
-"""

src/display/css_html_js.py

@@ -1,105 +0,0 @@
-custom_css = """
-
-.markdown-text {
-    font-size: 16px !important;
-}
-
-#models-to-add-text {
-    font-size: 18px !important;
-}
-
-#citation-button span {
-    font-size: 16px !important;
-}
-
-#citation-button textarea {
-    font-size: 16px !important;
-}
-
-#citation-button > label > button {
-    margin: 6px;
-    transform: scale(1.3);
-}
-
-#leaderboard-table {
-    margin-top: 15px
-}
-
-#leaderboard-table-lite {
-    margin-top: 15px
-}
-
-#search-bar-table-box > div:first-child {
-    background: none;
-    border: none;
-}
- 
-#search-bar {
-    padding: 0px;
-}
-
-/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
-table td:first-child,
-table th:first-child {
-    max-width: 400px;
-    overflow: auto;
-    white-space: nowrap;
-}
-
-.tab-buttons button {
-    font-size: 20px;
-}
-
-#scale-logo {
-    border-style: none !important;
-    box-shadow: none;
-    display: block;
-    margin-left: auto;
-    margin-right: auto;
-    max-width: 600px;
-}
-
-#scale-logo .download {
-    display: none;
-}
-#filter_type{
-    border: 0;
-    padding-left: 0;
-    padding-top: 0;
-}
-#filter_type label {
-    display: flex;
-}
-#filter_type label > span{
-    margin-top: var(--spacing-lg);
-    margin-right: 0.5em;
-}
-#filter_type label > .wrap{
-    width: 103px;
-}
-#filter_type label > .wrap .wrap-inner{  
-    padding: 2px;
-}
-#filter_type label > .wrap .wrap-inner input{
-    width: 1px
-}
-#filter-columns-type{
-    border:0;
-    padding:0.5;
-}
-#filter-columns-size{
-    border:0;
-    padding:0.5;
-}
-#box-filter > .form{
-    border: 0
-}
-"""
-
-get_window_url_params = """
-    function(url_params) {
-        const params = new URLSearchParams(window.location.search);
-        url_params = Object.fromEntries(params);
-        return url_params;
-    }
-    """

src/display/formatting.py

@@ -1,27 +0,0 @@
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def make_clickable_model(model_name):
-    link = f"https://huggingface.co/{model_name}"
-    return model_hyperlink(link, model_name)
-
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_warning(warn):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
-
-
-def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
-
-
-def has_no_nan_values(df, columns):
-    return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -1,58 +0,0 @@
-from dataclasses import dataclass, make_dataclass
-from enum import Enum
-
-import pandas as pd
-
-from src.about import Tasks
-
-
-def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
-
-
-# These classes are for user facing column names,
-# to avoid having to change them all around the code
-# when a modif is needed
-@dataclass
-class ColumnContent:
-    name: str
-    type: str
-    displayed_by_default: bool
-    hidden: bool = False
-    never_hidden: bool = False
-
-
-@dataclass(frozen=True)
-class AutoEvalColumn:
-    model = ColumnContent("model", "markdown", True, never_hidden=True)
-    average = ColumnContent("average", "number", True)
-    general = ColumnContent("general", "number", True)
-    speaker = ColumnContent("speaker", "number", True)
-    prosody = ColumnContent("prosody", "number", True)
-    intelligibility = ColumnContent("intelligibility", "number", True)
-    environment = ColumnContent("environment", "number", True)
-    tags = ColumnContent("tags", "str", False)
-
-
-## For the queue columns in the submission tab
-@dataclass(frozen=True)
-class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    status = ColumnContent("status", "str", True)
-
-
-## All the model information that we might need
-@dataclass
-class ModelDetails:
-    name: str
-    display_name: str = ""
-    symbol: str = ""  # emoji
-
-
-# Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-
-EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-BENCHMARK_COLS = ["general", "speaker", "prosody", "intelligibility", "environment"]

src/envs.py

@@ -23,3 +23,16 @@ EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 
 API = HfApi(token=TOKEN)
+
+TAGS = [
+    "Normalizing Flow",
+    "Reference-based (Speaker)",
+    "Prompt-based (Speaker)",
+    "Prosodic Correlates",
+    "Adversarial",
+    "Diffusion",
+    "Audio Tokens",
+    "Autoregressive",
+    "Non-autoregressive",
+    "Pretrained Text Encoder",
+]

src/leaderboard/read_evals.py

@@ -1,129 +0,0 @@
-import glob
-import json
-import math
-import os
-from dataclasses import dataclass
-
-import dateutil
-import numpy as np
-
-from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, Tasks
-
-
-@dataclass
-class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run."""
-
-    model_id: str
-    results: dict
-    date: str = ""  # submission date of request file
-
-    @classmethod
-    def init_from_json_file(self, json_filepath):
-        """Inits the result from the specific model result file"""
-        with open(json_filepath) as fp:
-            data = json.load(fp)
-
-        config = data.get("config")
-
-        # Extract model info
-        model = config.get("model_name", "")
-
-        # Extract results available in this file (some results are split in several files)
-        results = {}
-        for task in Tasks:
-            task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
-
-        return self(
-            model_id=model,
-            results=results,
-        )
-
-    def update_with_request_file(self, requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model)
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            self.model_id = request.get("model", self.model_id)
-            self.results
-        except Exception:
-            print(
-                f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}"
-            )
-
-    def to_dict(self):
-        """Converts the Eval Result to a dict compatible with our dataframe display"""
-        data_dict = {
-
-
-
-def get_request_file_for_model(requests_path, model_name):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if req_content["status"] in ["FINISHED"]:
-                request_file = tmp_request_file
-    return request_file
-
-
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
-    """From the path of the results folder root, extract all needed info for results"""
-    model_result_filepaths = []
-
-    for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
-
-        # Store results of same eval together
-        eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
-
-    results = []
-    for v in eval_results.values():
-        try:
-            v.to_dict()  # we test if the dict version is complete
-            results.append(v)
-        except KeyError:  # not all eval values present
-            continue
-
-    return results

src/populate.py

@@ -1,58 +0,0 @@
-import json
-import os
-
-import pandas as pd
-
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
-
-
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
-
-    df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
-
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
-
-
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -1,36 +0,0 @@
-import json
-import os
-import re
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-
-import huggingface_hub
-from huggingface_hub import ModelCard
-from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
-
-
-def already_submitted_models(requested_models_dir: str) -> set[str]:
-    """Gather a list of already submitted models to avoid duplicates"""
-    depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
-
-    for root, _, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            for file in files:
-                if not file.endswith(".json"):
-                    continue
-                with open(os.path.join(root, file), "r") as f:
-                    info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}")
-
-                    # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
-
-    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -1,69 +0,0 @@
-import json
-import os
-from datetime import datetime, timezone
-from typing import List
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import already_submitted_models
-
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-
-def add_new_eval(
-    model: str,
-    tags: List[str],
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_name = model
-
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "private": False,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_name}_eval_request_False.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )