add llmonitor & start scoring

pages/index.js

@@ -47,7 +47,7 @@ export default function Home({ prompts, models }) {
         <meta name="viewport" content="width=device-width, initial-scale=1" />
       </Head>
       <main>
-        <h1>Asking 60+ LLMs a set of 20 questions</h1>
+        <h1>Crowdsourced LLM Benchmark</h1>
         <br />
         <p>
           Benchmarks like HellaSwag are a bit too abstract for me to get a sense
@@ -69,13 +69,13 @@ export default function Home({ prompts, models }) {
         <br />
         <p>
           {`view: `}
+          <a href="#" onClick={() => changeView("model")}>
+            models
+          </a>{" "}
+          /
           <a href="#" onClick={() => changeView("prompt")}>
-            all prompts
+            prompts
           </a>{" "}
-          /{" "}
-          <a href="#" onClick={() => changeView("model")}>
-            all models
-          </a>
         </p>
         <br />
         {viewBy === "prompt" ? (
@@ -103,16 +103,22 @@ export default function Home({ prompts, models }) {
           </>
         ) : (
           <ul>
-            {models.map((model, i) => (
-              <li key={i}>
-                {model.name} -{" "}
-                <Link
-                  href={`/model/${model.api_id.split("/").pop().toLowerCase()}`}
-                >
-                  results
-                </Link>
-              </li>
-            ))}
+            {models
+              .score((s) => s.score)
+              .map((model, i) => (
+                <li key={i}>
+                  {model.name} -{" "}
+                  <Link
+                    href={`/model/${model.api_id
+                      .split("/")
+                      .pop()
+                      .toLowerCase()}`}
+                  >
+                    results
+                  </Link>{" "}
+                  - score: {model.score}
+                </li>
+              ))}
           </ul>
         )}
         <br />

run/queriers.py

@@ -4,6 +4,8 @@ import json
 import requests
 from dotenv import load_dotenv
 
+from llmonitor import monitor
+
 load_dotenv()
 
 TOGETHER_API_KEY = os.getenv('TOGETHER_API_KEY')
@@ -15,6 +17,8 @@ OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
 
 MAX_TOKENS = 300
 
+monitor(openai)
+
 def together(model, params):
     def format_prompt(prompt, prompt_type):
       if prompt_type == "language":

run/requirements.txt

@@ -2,4 +2,5 @@ openai
 pandas
 requests
 python-dotenv
-gradio
+gradio
+llmonitor

run/run.py

@@ -1,6 +1,7 @@
 import sqlite3
 import time
-
+from termcolor import colored
+from llmonitor import agent
 from queriers import together, cohere, openai_func, openrouter, ai21, alephalpha
 
 db = sqlite3.connect("./database.db")
@@ -22,6 +23,12 @@ models = [dict(model) for model in models]
 prompts = cursor.execute("SELECT * FROM prompts").fetchall()
 prompts = [dict(prompt) for prompt in prompts]
 
+
+def get_results():
+    results = cursor.execute("SELECT * FROM results").fetchall()
+    print(results[0].keys())
+    return [dict(result) for result in results]
+
 def insert_result(modelId, promptId, result, duration, rate):
     cursor.execute(
         "INSERT INTO results (model, prompt, result, duration, rate) VALUES (?, ?, ?, ?, ?)",
@@ -89,15 +96,85 @@ def ask_prompt(prompt, model):
 total_benchmarks = len(models) * len(prompts)
 print(f"Running {total_benchmarks} benchmarks")
 
+# # Run prompts
+# for model in models:
+#     if model["type"] == "language":
+#         continue
+#     for prompt in prompts:
+#         if prompt["type"] != "code" and model["type"] == "code":
+#             print("Skipping non-code benchmark for code model")
+#             continue
+
+#         ask_prompt(prompt, model)
+
+# Calculate scores
+results = get_results()
+
+@agent(name="RateResult")
+def rate_result(result):
+    rubrics = cursor.execute(
+        "SELECT * FROM rubrics WHERE prompt = ?",
+        (result["prompt"],)
+    ).fetchall()
+
+    has_rubrics = len(rubrics) > 0
+
+    if not has_rubrics:
+        return
+
+    print(colored('---------------------------', 'white'))
+    print(colored('----------RATING-----------', 'white'))
+    print(colored('---------------------------', 'white'))
+    print(colored(result["result"], 'cyan'))
+    print(colored('---------------------------', 'white'))
+    
+    score = None 
+
+    for rubric in rubrics:
+
+        print('Rubric: '+colored(rubric["grading"], 'magenta'))
+        
+        if result["result"].strip() == "":
+            score = 0
+        else:
+            grading_text = (
+                f'You help verify that the following answer match this condition: the answer {rubric["grading"]}. Note: the answer might be imcomplete, in which case do your best to assess based on what the full result would be.\n\n'
+                f'\n\n--START OF THE ANSWER--\n{result["result"]}\n--END OF THE ANSWER--\n\n'
+                f'Take a deep breath and explain step by step how you come to the conclusion.'
+                f'Finally, reply on the last line with YES if the following answer matches this condition (otherwies reply NO).'
+            )
+
+            # get gpt-4 model
+            gpt4 = next((item for item in models if item['api_id'] == 'gpt-4'), None)
+            
+            prompt = { }
+
+            response_text = openai_func(gpt4, {"text": grading_text})
+
+            print(colored(f"-> {response_text}", 'yellow'))
+
+            last_line = response_text.splitlines()[-1]
+
+            # If it includes a yes, then it's valid
+            if "YES" in last_line:
+                print(colored(f'Valid! + {rubric["points"]} points', 'green'))
+                score = rubric["points"] if score is None else score + rubric["points"]
+
+    print('Final score: '+colored(score, 'cyan'))
+    
+    return score
+
+
 
-for model in models:
-    if model["type"] == "language":
-        continue
-    for prompt in prompts:
-        if prompt["type"] != "code" and model["type"] == "code":
-            print("Skipping non-code benchmark for code model")
-            continue
+for result in results:
+    if not result["score"]:
+        score = rate_result(result)
 
-        ask_prompt(prompt, model)
+        if score is not None:
+            cursor.execute(
+                "UPDATE results SET score = ? WHERE id == ?",
+                (score, result["id"])
+            )
+            db.commit()
 
 db.close()