add new models

ZeroEval-main/result_dirs/zebra-grid.summary.json

@@ -21,6 +21,28 @@
     "Total Puzzles": 1000,
     "Reason Lens": "1153.83"
   },
+  {
+    "Model": "Llama-3.1-405B-Instruct-Turbo",
+    "Mode": "greedy",
+    "Puzzle Acc": "32.60",
+    "Cell Acc": "45.80",
+    "No answer": "12.50",
+    "Easy Puzzle Acc": "87.14",
+    "Hard Puzzle Acc": "11.39",
+    "Total Puzzles": 1000,
+    "Reason Lens": "314.66"
+  },
+  {
+    "Model": "Llama-3.1-405B-Instruct-Turbo",
+    "Mode": "sampling",
+    "Puzzle Acc": "32.60",
+    "Cell Acc": "47.04",
+    "No answer": "10.80",
+    "Easy Puzzle Acc": "86.07",
+    "Hard Puzzle Acc": "11.81",
+    "Total Puzzles": 1000,
+    "Reason Lens": "439.96"
+  },
   {
     "Model": "gpt-4o-2024-05-13",
     "Mode": "sampling",
@@ -230,6 +252,17 @@
     "Total Puzzles": 1000,
     "Reason Lens": "809.95"
   },
+  {
+    "Model": "Athene-70B",
+    "Mode": "greedy",
+    "Puzzle Acc": "16.70",
+    "Cell Acc": "32.98",
+    "No answer": "21.10",
+    "Easy Puzzle Acc": "52.50",
+    "Hard Puzzle Acc": "2.78",
+    "Total Puzzles": 1000,
+    "Reason Lens": "391.19"
+  },
   {
     "Model": "gemma-2-27b-it@nvidia",
     "Mode": "greedy",
@@ -252,6 +285,17 @@
     "Total Puzzles": 1000,
     "Reason Lens": "1015.06"
   },
+  {
+    "Model": "command-r-plus",
+    "Mode": "greedy",
+    "Puzzle Acc": "13.90",
+    "Cell Acc": "39.01",
+    "No answer": "0.20",
+    "Easy Puzzle Acc": "44.64",
+    "Hard Puzzle Acc": "1.94",
+    "Total Puzzles": 1000,
+    "Reason Lens": "810.53"
+  },
   {
     "Model": "reka-core-20240501",
     "Mode": "greedy",
@@ -263,17 +307,6 @@
     "Total Puzzles": 1000,
     "Reason Lens": "1078.29"
   },
-  {
-    "Model": "gemma-2-9b-it",
-    "Mode": "greedy",
-    "Puzzle Acc": "12.90",
-    "Cell Acc": "37.07",
-    "No answer": "0.50",
-    "Easy Puzzle Acc": "42.14",
-    "Hard Puzzle Acc": "1.53",
-    "Total Puzzles": 1000,
-    "Reason Lens": "859.14"
-  },
   {
     "Model": "gemma-2-9b-it@nvidia",
     "Mode": "greedy",
@@ -296,6 +329,28 @@
     "Total Puzzles": 1000,
     "Reason Lens": "1216.40"
   },
+  {
+    "Model": "Mistral-Nemo-Instruct-2407",
+    "Mode": "greedy",
+    "Puzzle Acc": "11.80",
+    "Cell Acc": "34.93",
+    "No answer": "1.60",
+    "Easy Puzzle Acc": "38.93",
+    "Hard Puzzle Acc": "1.25",
+    "Total Puzzles": 1000,
+    "Reason Lens": "925.88"
+  },
+  {
+    "Model": "Phi-3-mini-4k-instruct",
+    "Mode": "greedy",
+    "Puzzle Acc": "11.60",
+    "Cell Acc": "13.50",
+    "No answer": "59.00",
+    "Easy Puzzle Acc": "38.21",
+    "Hard Puzzle Acc": "1.25",
+    "Total Puzzles": 1000,
+    "Reason Lens": "790.29"
+  },
   {
     "Model": "Yi-1.5-34B-Chat",
     "Mode": "greedy",
@@ -329,6 +384,17 @@
     "Total Puzzles": 1000,
     "Reason Lens": "820.66"
   },
+  {
+    "Model": "command-r",
+    "Mode": "greedy",
+    "Puzzle Acc": "9.90",
+    "Cell Acc": "32.66",
+    "No answer": "1.50",
+    "Easy Puzzle Acc": "32.14",
+    "Hard Puzzle Acc": "1.25",
+    "Total Puzzles": 1000,
+    "Reason Lens": "1005.17"
+  },
   {
     "Model": "reka-flash-20240226",
     "Mode": "greedy",
@@ -351,6 +417,17 @@
     "Total Puzzles": 1000,
     "Reason Lens": "1148.16"
   },
+  {
+    "Model": "Mixtral-8x7B-Instruct-v0.1",
+    "Mode": "greedy",
+    "Puzzle Acc": "8.70",
+    "Cell Acc": "26.47",
+    "No answer": "20.30",
+    "Easy Puzzle Acc": "28.93",
+    "Hard Puzzle Acc": "0.83",
+    "Total Puzzles": 1000,
+    "Reason Lens": "1177.21"
+  },
   {
     "Model": "Qwen2-7B-Instruct",
     "Mode": "greedy",

model_info.json

@@ -64,5 +64,9 @@
     "SELM-Llama-3-8B-Instruct-iter-3": {"pretty_name": "SELM (Llama3-8B-Inst-iter3)", "hf_model_id": "ZhangShenao/SELM-Llama-3-8B-Instruct-iter-3"},
     "nemotron-4-340b-instruct": {"pretty_name": "Nemotron-4-340B-Instruct", "hf_model_id": "nvidia/Nemotron-4-340B-Instruct"},
     "Llama-3-8B-Magpie-Align-v0.1": {"pretty_name": "Magpie-8B-Align-v0.1", "hf_model_id": "Magpie-Align/Llama-3-8B-Magpie-Align-v0.1"},
-    "mathstral-7B-v0.1":{"pretty_name": "mathstral-7B-v0.1", "hf_model_id": "mistralai/mathstral-7B-v0.1"}
+    "mathstral-7B-v0.1":{"pretty_name": "mathstral-7B-v0.1 🚨", "hf_model_id": "mistralai/mathstral-7B-v0.1"},
+    "Llama-3.1-405B-Instruct-Turbo": {"pretty_name": "Llama-3.1-405B-Instruct-Turbo 🚨", "hf_model_id": "meta-llama/Meta-Llama-3.1-405B-Instruct"},
+    "Mistral-Nemo-Instruct-2407": {"pretty_name": "Mistral-Nemo-Inst (12B) 🚨", "hf_model_id": "Mistral-Nemo-Instruct-2407"},
+    "Phi-3-mini-4k-instruct": {"pretty_name": "Phi-3-mini-4k-instruct 🚨", "hf_model_id": "microsoft/Phi-3-mini-4k-instruct"},
+    "Athene-70B": {"pretty_name": "Athene-70B 🚨", "hf_model_id": "Nexusflow/Athene-70B"}
 }