Leaderboard

yall.py

@@ -1,5 +1,4 @@
 import os
-import gistyc
 import requests
 from dataclasses import dataclass
 import re
@@ -19,7 +18,6 @@ class GistInfo:
     bigbench: float
     average: float
 
-
 def update_gist(content, gist_id, access_token):
     """
     Update the content of a GitHub Gist.
@@ -50,67 +48,51 @@ def update_gist(content, gist_id, access_token):
         print("Failed to update gist. Status code:", response.status_code)
         print("Response:", response.json())
 
-
-@st.cache_data 
+@st.cache_data
 def create_yall():
-    # Get token
-    GITHUB_API_TOKEN = os.environ.get("github")
-    
-    # Retrieve all gists
-    gist_api = gistyc.GISTyc(auth_token=GITHUB_API_TOKEN)
-    data = gist_api.get_gists()
-    
-    # List to store the GistInfo objects
-    gist_infos = []
-    
-    for data_dict in data:
-        if 'files' in data_dict and data_dict['files']:
-            file_info = next(iter(data_dict['files'].values()))
-            filename = file_info['filename']
-            if filename.endswith("-Nous.md"):
-                raw_url = file_info['raw_url']
-                response = requests.get(raw_url)
-                if response.status_code == 200:
-                    if "Error: File does not exist" not in response.text:
-                        # Parse the markdown table
-                        lines = response.text.split('\n')
-                        if len(lines) >= 3:
-                            values = lines[2].split('|')[1:-1]
-    
-                            # Extract model name and model id using regular expression
-                            model_match = re.search(r'\[([^\]]+)\]\(https://huggingface.co/([^/]+)/([^)]+)\)', values[0].strip())
-                            if model_match:
-                                model_name = model_match.group(1)
-                                model_id = f"{model_match.group(2)}/{model_match.group(3)}"
-                                print(values[0].strip())
-                                print(model_name)
-                                print(model_id)
-                                print("=============")
-                            else:
-                                model_name = model_id = 'Unknown'
-    
-    
-                        # Parse the markdown table
-                        lines = response.text.split('\n')
-                        if len(lines) >= 3:
-                            values = lines[2].split('|')[1:-1]
-    
-                            # Create a GistInfo object and add it to the list
-                            gist_info = GistInfo(
-                                gist_id=data_dict['id'],
-                                filename=filename,
-                                url=data_dict['html_url'],  # Assuming html_url is the URL of the gist
-                                model_name=model_name,
-                                model_id=model_id,
-                                model=values[0].strip(),
-                                agieval=float(values[1].strip()),
-                                gpt4all=float(values[2].strip()),
-                                truthfulqa=float(values[3].strip()),
-                                bigbench=float(values[4].strip()),
-                                average=float(values[5].strip()),
-                            )
-                            gist_infos.append(gist_info)
-    
+    # Dummy data
+    gist_infos = [
+        GistInfo(
+            gist_id="dummy_gist_id_1",
+            filename="Model-1.md",
+            url="https://gist.github.com/dummy_gist_id_1",
+            model_name="Model 1",
+            model_id="model-1",
+            model="Model 1",
+            agieval=95.4,
+            gpt4all=88.7,
+            truthfulqa=90.3,
+            bigbench=85.6,
+            average=90.0
+        ),
+        GistInfo(
+            gist_id="dummy_gist_id_2",
+            filename="Model-2.md",
+            url="https://gist.github.com/dummy_gist_id_2",
+            model_name="Model 2",
+            model_id="model-2",
+            model="Model 2",
+            agieval=89.1,
+            gpt4all=85.0,
+            truthfulqa=87.5,
+            bigbench=83.0,
+            average=86.2
+        ),
+        GistInfo(
+            gist_id="dummy_gist_id_3",
+            filename="Model-3.md",
+            url="https://gist.github.com/dummy_gist_id_3",
+            model_name="Model 3",
+            model_id="model-3",
+            model="Model 3",
+            agieval=78.2,
+            gpt4all=81.4,
+            truthfulqa=79.5,
+            bigbench=77.0,
+            average=79.0
+        )
+    ]
+
     # Sort the list by average
     gist_infos = sorted(gist_infos, key=lambda x: x.average, reverse=True)
     
@@ -122,7 +104,7 @@ def create_yall():
         model_link = f"[{gist.model_id}](https://huggingface.co/{gist.model_id})"
         markdown_table += f"| {model_link} [📄]({gist.url}) | {gist.average} | {gist.agieval} | {gist.gpt4all} | {gist.truthfulqa} | {gist.bigbench} |\n"
     
-    # Update YALL's gist
-    update_gist(content=markdown_table, gist_id="90294929a2dbcb8877f9696f28105fdf", access_token=GITHUB_API_TOKEN)
+    # Update YALL's gist with dummy gist ID and token
+    update_gist(content=markdown_table, gist_id="dummy_gist_id_yall", access_token="dummy_access_token")
 
-    return markdown_table
+    return markdown_table