nathan-flagged-models-vis (#478)

- Adds a way to hide flagged models (a69dfa979897081c10a30f1be9937a917d93422b)
- remove unnused pprint import (1be35c2d1ffffab552d9d65f826930e4f9f1c273)
- remove unnused pprint import (6adc61160db982ce023039472b8842d21584b367)

(cherry picked from commit 460ecf2f9814163d447819d75dd51e4139b4476b)

app.py

@@ -33,7 +33,6 @@ from src.display.utils import (
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
-from src.submission.check_validity import already_submitted_models
 from src.tools.collections import update_collections
 from src.tools.plots import (
     create_metric_plot_obj,
@@ -82,14 +81,20 @@ def update_table(
     precision_query: str,
     size_query: list,
     show_deleted: bool,
+    show_flagged: bool,
     query: str,
 ):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
+    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted, show_flagged)
     filtered_df = filter_queries(query, filtered_df)
     df = select_columns(filtered_df, columns)
     return df
 
 
+def load_query(request: gr.Request):  # triggered only once at startup => read query parameter if it exists
+    query = request.query_params.get("query") or ""
+    return query, query # return one for the "search_bar", one for a hidden component that triggers a reload only if value has changed
+
+
 def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
     return df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]
 
@@ -127,7 +132,7 @@ def filter_queries(query: str, filtered_df: pd.DataFrame):
 
 
 def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
+    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool, show_flagged: bool
 ) -> pd.DataFrame:
     # Show all models
     if show_deleted:
@@ -135,6 +140,9 @@ def filter_models(
     else:  # Show only still on the hub models
         filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
 
+    if not show_flagged:
+        filtered_df = filtered_df[filtered_df[AutoEvalColumn.flagged.name] == False]
+
     type_emoji = [t[0] for t in type_query]
     filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
     filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
@@ -146,6 +154,7 @@ def filter_models(
 
     return filtered_df
 
+leaderboard_df = filter_models(leaderboard_df, [t.to_str(" : ") for t in ModelType], list(NUMERIC_INTERVALS.keys()), [i.value.name for i in Precision], False, False)
 
 import unicodedata
 
@@ -175,11 +184,11 @@ hidden_leaderboard_table_for_search = gr.components.Dataframe(
 
 def display(x, y):
     # Assuming df is your DataFrame
-    for column in original_df.columns:
-        if original_df[column].dtype == 'object':
-            original_df[column] = original_df[column].apply(remove_invalid_unicode)
+    for column in leaderboard_df.columns:
+        if leaderboard_df[column].dtype == 'object':
+            leaderboard_df[column] = leaderboard_df[column].apply(remove_invalid_unicode)
 
-    subset_df = original_df[COLS]
+    subset_df = leaderboard_df[COLS]
     # Ensure the output directory exists
     #output_dir = 'output'
     #if not os.path.exists(output_dir):

src/display/utils.py

@@ -51,6 +51,7 @@ auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B
 auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub", "number", False)])
 auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, False)])
 # Dummy column for the search bar (hidden by the custom CSS)
 auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
 
@@ -80,6 +81,7 @@ baseline_row = {
     AutoEvalColumn.gsm8k.name: 0.21,
     AutoEvalColumn.dummy.name: "baseline",
     AutoEvalColumn.model_type.name: "",
+    AutoEvalColumn.flagged.name: False,
 }
 
 # Average ⬆️ human baseline is 0.897 (source: averaging human baselines below)

src/leaderboard/filter_models.py

@@ -13,13 +13,26 @@ FLAGGED_MODELS = {
     "AIDC-ai-business/Marcoroni-70B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/287",
     "AIDC-ai-business/Marcoroni-13B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/287",
     "AIDC-ai-business/Marcoroni-7B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/287",
-    "TigerResearch/tigerbot-70b-chat-v4-4k": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/438",
-    "TigerResearch/tigerbot-70b-chat-v2": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/438"
+    "fblgit/una-xaberius-34b-v1beta": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/444",
+    "jan-hq/trinity-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "rwitz2/go-bruins-v2.1.1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "rwitz2/go-bruins-v2.1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "GreenNode/GreenNodeLM-v3olet-7B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "GreenNode/GreenNodeLM-7B-v4leo": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "GreenNode/LeoScorpius-GreenNode-7B-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "viethq188/LeoScorpius-7B-Chat-DPO": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "GreenNode/GreenNodeLM-7B-v2leo": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "janai-hq/trinity-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "ignos/LeoScorpius-GreenNode-Alpaca-7B-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "fblgit/una-cybertron-7b-v3-OMA": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
 }
 
 # Models which have been requested by orgs to not be submitted on the leaderboard
 DO_NOT_SUBMIT_MODELS = [
     "Voicelab/trurl-2-13b",  # trained on MMLU
+    "TigerResearch/tigerbot-70b-chat", # per authors request
+    "TigerResearch/tigerbot-70b-chat-v2", # per authors request
+    "TigerResearch/tigerbot-70b-chat-v4-4k", # per authors request
 ]
 
 
@@ -34,6 +47,9 @@ def flag_models(leaderboard_data: list[dict]):
             model_data[
                 AutoEvalColumn.model.name
             ] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
+            model_data[AutoEvalColumn.flagged.name] = True
+        else:
+            model_data[AutoEvalColumn.flagged.name] = False
 
 
 def remove_forbidden_models(leaderboard_data: list[dict]):