Update app.py

app.py

@@ -109,8 +109,9 @@ def evaluate(
     calibrated: bool = True,
     check_gt_only: bool = False,
     no_gt: bool = False,
+    selective_evaluate: str = "",
 ):
-    pass_k = [int(k.strip()) for k in pass_k.split(',') if k.strip().isdigit()]
+    passk = [int(k.strip()) for k in pass_k.split(',') if k.strip().isdigit()]
     if parallel < 1:
         n_workers = max(1, multiprocessing.cpu_count() // 2)
     else:
@@ -122,6 +123,14 @@ def evaluate(
     extra = subset + "_" if subset != "full" else ""
 
     problems = get_bigcodebench(subset=subset)
+    
+    # Add selective evaluation logic
+    if selective_evaluate:
+        selected_ids = ["BigCodeBench/" + id for id in sorted(set(selective_evaluate.split(",")))]
+        problems = {k: v for k, v in problems.items() if k in selected_ids}
+        if not problems:
+            raise ValueError(f"None of the provided task IDs {selected_ids} were found in the dataset")
+
     dataset_hash = get_bigcodebench_hash(subset=subset)
     
     if not no_gt:
@@ -214,7 +223,7 @@ def evaluate(
 
         pass_at_k.update({
             f"pass@{k}": estimate_pass_at_k(total, base_correct, k).mean()
-            for k in pass_k
+            for k in passk
             if total.min() >= k
         })
 
@@ -247,6 +256,7 @@ interface = gr.Interface(
         gr.Checkbox(label="Calibrated", value=True),
         gr.Checkbox(label="Check GT Only"),
         gr.Checkbox(label="No GT"),
+        gr.Textbox(label="Selective Evaluated Task IDs (comma-separated, e.g. '0,1,2')", value=""),
     ],
     outputs=[
         gr.JSON(label="Results"),