in progress private

src/leaderboard/read_evals.py

@@ -452,14 +452,14 @@ def get_raw_eval_results(results_path: str, requests_path: str, metadata) -> lis
                 task_name = f"{r['n_shot']}|{task.value.benchmark}"
                 if task_name in missing_results_for_task:
                     missing_results_for_task[task_name].append(f"{v.full_model}|{v.org_and_model}")
-                    if v.still_on_hub and task.value.benchmark in all_tasks:
-                        for_run.append([r["n_shot"], task.value.benchmark, v.full_model])
+                    if task.value.benchmark in all_tasks:
+                        if v.still_on_hub: for_run.append([r["n_shot"], task.value.benchmark, v.full_model])
                         in_progress=True
                         # print(f'sbatch start.sh "bash eval_model_task_bs1.sh {r["n_shot"]} {task.value.benchmark} {v.full_model}"')
                 else:
                     missing_results_for_task[task_name] = [f"{v.full_model}|{v.org_and_model}"]
-                    if v.still_on_hub and task.value.benchmark in all_tasks:
-                        for_run.append([r["n_shot"], task.value.benchmark, v.full_model])
+                    if task.value.benchmark in all_tasks:
+                        if v.still_on_hub: for_run.append([r["n_shot"], task.value.benchmark, v.full_model])
                         in_progress=True
                         # print(f'sbatch start.sh "bash eval_model_task_bs1.sh {r["n_shot"]} {task.value.benchmark} {v.full_model}"')
         if in_progress: