huge speedup in data getter

app.py

@@ -1,5 +1,4 @@
 import pandas as pd
-from tqdm.auto import tqdm
 import streamlit as st
 from huggingface_hub import HfApi, hf_hub_download
 from huggingface_hub.repocard import metadata_load
@@ -10,23 +9,10 @@ from os.path import exists
 import threading
 
 
-def get_model_ids(author=None):
+def get_model_infos():
     api = HfApi()
-    if author is None:
-        models = api.list_models(filter="model-index")
-    else:
-        models = api.list_models(filter="model-index", author="autoevaluate")
-    model_ids = [x.modelId for x in models]
-    return model_ids
-
-
-def get_metadata(model_id):
-    try:
-        readme_path = hf_hub_download(model_id, filename="README.md")
-        return metadata_load(readme_path)
-    except Exception:
-        # 404 README.md not found or problem loading it
-        return None
+    model_infos = api.list_models(filter="model-index", cardData=True)
+    return model_infos
 
 
 def parse_metric_value(value):
@@ -83,27 +69,28 @@ def parse_metrics_rows(meta, only_verified=False):
             continue
         yield row
 
-@st.cache(ttl=3600)
+@st.cache(ttl=10)
 def get_data_wrapper():
 
-    def get_data():
+    def get_data(dataframe=None, verified_dataframe=None):
         data = []
         verified_data = []
-        model_ids = get_model_ids()
-        model_ids_from_autoeval = set(get_model_ids(author="autoevaluate"))
-        for model_id in tqdm(model_ids):
-            meta = get_metadata(model_id)
+        print("getting model infos")
+        model_infos = get_model_infos()
+        print("got model infos")
+        for model_info in model_infos:
+            meta = model_info.cardData
             if meta is None:
                 continue
             for row in parse_metrics_rows(meta):
                 if row is None:
                     continue
-                row["model_id"] = model_id
+                row["model_id"] = model_info.id
                 data.append(row)
             for row in parse_metrics_rows(meta, only_verified=True):
                 if row is None:
                     continue
-                row["model_id"] = model_id
+                row["model_id"] = model_info.id
                 verified_data.append(row)
         dataframe = pd.DataFrame.from_records(data)
         dataframe.to_pickle("cache.pkl")
@@ -136,7 +123,7 @@ only_verified_results = st.sidebar.checkbox(
     "Filter for Verified Results",
 )
 
-selectable_datasets = sorted(list(set(dataframe.dataset.tolist() + verified_dataframe.dataset.tolist())), key=lambda name: name.lower())
+selectable_datasets = sorted(list(set(dataframe.dataset.tolist())), key=lambda name: name.lower())
 
 if only_verified_results:
     dataframe = verified_dataframe