debug

app.py

@@ -30,7 +30,7 @@ from src.display.utils import (
     fields,
 )
 from src.envs import API, EVAL_REQUESTS_PATH, QUEUE_REPO, REPO_ID, TOKEN, RESULTS_REPO_MIB_SUBGRAPH, EVAL_RESULTS_MIB_SUBGRAPH_PATH, RESULTS_REPO_MIB_CAUSALGRAPH, EVAL_RESULTS_MIB_CAUSALGRAPH_PATH
-from src.populate import get_evaluation_queue_df, get_leaderboard_df, get_leaderboard_df_mib
+from src.populate import get_evaluation_queue_df, get_leaderboard_df, get_leaderboard_df_mib_subgraph, get_leaderboard_df_mib_causalgraph
 from src.submission.submit import add_new_eval
 
 
@@ -49,15 +49,6 @@ try:
 except Exception:
     restart_space()
 
-# print("EVAL_RESULTS_PATH")
-# try:
-#     print(EVAL_RESULTS_PATH)
-#     snapshot_download(
-#         repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-#     )
-# except Exception:
-#     restart_space()
-
 
 try:
     print(RESULTS_REPO_MIB_SUBGRAPH)
@@ -78,8 +69,8 @@ except Exception:
 
 
 
-LEADERBOARD_DF_MIB_SUBGRAPH = get_leaderboard_df_mib(EVAL_RESULTS_MIB_SUBGRAPH_PATH, EVAL_REQUESTS_PATH, COLS_MIB, BENCHMARK_COLS_MIB)
-# LEADERBOARD_DF_MIB_CAUSALGRAPH = get_leaderboard_df_mib_causal(EVAL_RESULTS_MIB_CAUSALGRAPH_PATH, EVAL_REQUESTS_PATH, COLS_MIB, BENCHMARK_COLS_MIB)
+LEADERBOARD_DF_MIB_SUBGRAPH = get_leaderboard_df_mib_subgraph(EVAL_RESULTS_MIB_SUBGRAPH_PATH, EVAL_REQUESTS_PATH, COLS_MIB, BENCHMARK_COLS_MIB)
+LEADERBOARD_DF_MIB_CAUSALGRAPH = get_leaderboard_df_mib_causalgraph(EVAL_RESULTS_MIB_CAUSALGRAPH_PATH, EVAL_REQUESTS_PATH, COLS_MIB, BENCHMARK_COLS_MIB)
 
 # LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
 # LEADERBOARD_DF_MULTIMODAL = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS_MULTIMODAL, BENCHMARK_COLS_MULTIMODAL)
@@ -91,7 +82,32 @@ LEADERBOARD_DF_MIB_SUBGRAPH = get_leaderboard_df_mib(EVAL_RESULTS_MIB_SUBGRAPH_P
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
 
-def init_leaderboard_mib(dataframe, track):
+def init_leaderboard_mib_subgraph(dataframe, track):
+    print(f"init_leaderboard_mib: dataframe head before loc is {dataframe.head()}\n")
+    
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+
+    # filter for correct track
+    # dataframe = dataframe.loc[dataframe["Track"] == track]
+
+    print(f"init_leaderboard_mib: dataframe head after loc is {dataframe.head()}\n")
+    
+    return Leaderboard(
+        value=dataframe,
+        datatype=[c.type for c in fields(AutoEvalColumn_mib)],
+        select_columns=SelectColumns(
+            default_selection=[c.name for c in fields(AutoEvalColumn_mib) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(AutoEvalColumn_mib) if c.never_hidden],
+            label="Select Columns to Display:",
+        ),
+        search_columns=["Method"],  # Changed from AutoEvalColumn_mib.model.name to "Method"
+        hide_columns=[c.name for c in fields(AutoEvalColumn_mib) if c.hidden],
+        bool_checkboxgroup_label="Hide models",
+        interactive=False,
+    )
+
+def init_leaderboard_mib_causalgraph(dataframe, track):
     print(f"init_leaderboard_mib: dataframe head before loc is {dataframe.head()}\n")
     
     if dataframe is None or dataframe.empty:
@@ -116,6 +132,7 @@ def init_leaderboard_mib(dataframe, track):
         interactive=False,
     )
 
+
 def init_leaderboard(dataframe, track):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
@@ -180,11 +197,10 @@ with demo:
         #             gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
         
         with gr.TabItem("Subgraph", elem_id="subgraph", id=0):
-            leaderboard = init_leaderboard_mib(LEADERBOARD_DF_MIB_SUBGRAPH, "Subgraph")
-            # leaderboard = init_leaderboard_mib(LEADERBOARD_DF, "mib")
+            leaderboard = init_leaderboard_mib_subgraph(LEADERBOARD_DF_MIB_SUBGRAPH, "Subgraph")
         
         # with gr.TabItem("Causal Graph", elem_id="causalgraph", id=1):
-        #     leaderboard = init_leaderboard_mib_causal(LEADERBOARD_DF_MIB_CAUSALGRAPH, "Causal Graph")
+        #     leaderboard = init_leaderboard_mib_causalgraph(LEADERBOARD_DF_MIB_CAUSALGRAPH, "Causal Graph")
         
     # with gr.Row():
     #     with gr.Accordion("📙 Citation", open=False):

src/about.py

@@ -27,7 +27,7 @@ class Tasks(Enum):
     task3 = Task("ewok", "acc", "EWoK")
 
 
-class TasksMIB(Enum):
+class TasksMib_Subgraph(Enum):
     task0 = TaskMIB("ioi", ["meta_llama", "qwen", "gpt2"], "ioi", ["edge_counts", "faithfulness"])
     task1 = TaskMIB("mcqa", ["meta_llama", "qwen", "gpt2"], "mcqa", ["edge_counts", "faithfulness"])
 

src/display/utils.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 import pandas as pd
 
-from src.about import Tasks, TasksMultimodal, TasksMIB
+from src.about import Tasks, TasksMultimodal, TasksMib_Subgraph
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -35,7 +35,7 @@ auto_eval_column_dict_mib = []
 auto_eval_column_dict_mib.append(["method", ColumnContent, ColumnContent("Method", "markdown", True, never_hidden=True)])
 
 # For each task and model combination
-for task in TasksMIB:
+for task in TasksMib_Subgraph:
     for model in task.value.models:
         col_name = f"{task.value.benchmark}_{model}"  # ioi_meta_llama, mcqa_qwen, etc.
         auto_eval_column_dict_mib.append([
@@ -54,9 +54,9 @@ AutoEvalColumn_mib = make_dataclass("AutoEvalColumn_mib", auto_eval_column_dict_
 # Column selection for display
 COLS_MIB = [c.name for c in fields(AutoEvalColumn_mib) if not c.hidden]
 
-# BENCHMARK_COLS_MIB = [t.value.col_name for t in TasksMIB]
+# BENCHMARK_COLS_MIB = [t.value.col_name for t in TasksMib_Subgraph]
 BENCHMARK_COLS_MIB = []
-for task in TasksMIB:
+for task in TasksMib_Subgraph:
     for model in task.value.models:
         col_name = f"{task.value.col_name}_{model.replace('-', '_')}"
         BENCHMARK_COLS_MIB.append(col_name)

src/leaderboard/read_evals.py

@@ -13,29 +13,9 @@ from src.submission.check_validity import is_model_on_hub
 
 
 from typing import List, Dict
-from src.about import TasksMIB
-
-
-# def compute_area(edge_counts, faithfulnesses, log_scale=True):
-#     percentages = [e / max(edge_counts) for e in edge_counts]
-#     area_under = 0.
-#     area_from_100 = 0.
-#     for i in range(len(faithfulnesses) - 1):
-#         i_1, i_2 = i, i+1
-#         x_1 = percentages[i_1]
-#         x_2 = percentages[i_2]
-#         # area from point to 100
-#         if log_scale:
-#             x_1 = math.log(x_1)
-#             x_2 = math.log(x_2)
-#         trapezoidal = (percentages[i_2] - percentages[i_1]) * \
-#                         (((abs(1. - faithfulnesses[i_1])) + (abs(1. - faithfulnesses[i_2]))) / 2)
-#         area_from_100 += trapezoidal 
-        
-#         trapezoidal = (percentages[i_2] - percentages[i_1]) * ((faithfulnesses[i_1] + faithfulnesses[i_2]) / 2)
-#         area_under += trapezoidal
-#     average = sum(faithfulnesses) / len(faithfulnesses)
-#     return (area_under, area_from_100, average)
+
+
+
 def compute_area(edge_counts, faithfulnesses, log_scale=True):
     # Return None if either list is empty
     if not edge_counts or not faithfulnesses:
@@ -62,7 +42,7 @@ def compute_area(edge_counts, faithfulnesses, log_scale=True):
     return (area_under, area_from_100, average)
         
 @dataclass
-class EvalResult_MIB:
+class EvalResult_MIB_SUBGRAPH:
     """Represents one full evaluation for a method across all models in MIB."""
     eval_name: str        # method name as identifier
     method_name: str      # name of the interpretation method
@@ -104,63 +84,13 @@ class EvalResult_MIB:
                         "faithfulness": scores[task]["faithfulness"]
                     }
 
-        return EvalResult_MIB(
+        return EvalResult_MIB_SUBGRAPH(
             eval_name=method_name,
             method_name=method_name,
             results=results
         )
 
 
-
-    # def to_dict(self):
-    #     """Converts the Eval Result to a dict for dataframe display"""
-    #     data_dict = {
-    #         "eval_name": self.eval_name,
-    #         "Method": self.method_name,
-    #     }
-
-    #     all_scores = []
-    #     required_entries = {
-    #         'ioi_meta_llama': False,
-    #         'ioi_qwen': False,
-    #         'ioi_gpt2': False,
-    #         'mcqa_meta_llama': False,
-    #         'mcqa_qwen': False,
-    #         'mcqa_gpt2': False
-    #     }
-        
-    #     # For each task (ioi, mcqa)
-    #     for task, task_results in self.results.items():
-    #         # Get the models that have results for this task
-    #         models = task_results.keys()
-            
-    #         for model in models:
-    #             col_name = f"{task}_{model}"
-    #             metrics = task_results[model]
-    #             if metrics:
-    #                 edge_counts = metrics["edge_counts"]
-    #                 faithfulness = metrics["faithfulness"]
-    #                 if isinstance(faithfulness[0], list):
-    #                     faithfulness = faithfulness[0]
-                    
-    #                 # Use compute_area
-    #                 area_under, area_from_100, avg = compute_area(edge_counts, faithfulness)
-    #                 score = area_under * 100
-    #                 data_dict[col_name] = round(score, 2)
-    #                 all_scores.append(score)
-    #                 required_entries[col_name] = True
-    #             else:
-    #                 data_dict[col_name] = '-'
-
-    #     # Only show average if all six required entries are present
-    #     if all(required_entries.values()):
-    #         data_dict["Average"] = round(np.mean(all_scores), 2)
-    #     else:
-    #         data_dict["Average"] = '-'
-        
-    #     return data_dict
-
-
     
     def to_dict(self):
         """Converts the Eval Result to a dict for dataframe display"""
@@ -211,13 +141,7 @@ class EvalResult_MIB:
         return data_dict
 
 
-
-
-
-
-
-
-def get_raw_eval_results_mib(results_path: str, requests_path: str) -> List[EvalResult_MIB]:
+def get_raw_eval_results_mib_subgraph(results_path: str, requests_path: str) -> List[EvalResult_MIB_SUBGRAPH]:
     """From the path of the results folder root, extract all needed info for MIB results"""
     model_result_filepaths = []
     
@@ -243,7 +167,7 @@ def get_raw_eval_results_mib(results_path: str, requests_path: str) -> List[Eval
     eval_results = []
     for model_result_filepath in model_result_filepaths:
         try:
-            eval_result = EvalResult_MIB("", "", {})  # Create empty instance
+            eval_result = EvalResult_MIB_SUBGRAPH("", "", {})  # Create empty instance
             result = eval_result.init_from_json_file(model_result_filepath)
             print(f"eval_result.init_from_json_file(model_result_filepath) is {result}")
             # Verify the result can be converted to dict format
@@ -264,6 +188,60 @@ def get_raw_eval_results_mib(results_path: str, requests_path: str) -> List[Eval
 
 
 
+
+
+
+
+
+
+
+
+
+
+@dataclass
+class EvalResult_MIB_CAUSALGRAPH:
+    """Represents one full evaluation for a method across all models in MIB."""
+    eval_name: str        # method name as identifier
+    method_name: str      # name of the interpretation method
+    results: Dict         # nested dict of results {task: {model: {metric: scores}}}
+
+
+    def init_from_json_file(self, json_filepath):
+
+
+
+    
+    def to_dict(self):
+
+        return data_dict
+
+
+def get_raw_eval_results_mib_causalgraph(results_path: str, requests_path: str) -> List[EvalResult_MIB_CAUSALGRAPH]:
+    """From the path of the results folder root, extract all needed info for MIB results"""
+    model_result_filepaths = []
+    
+    print(f"results_path is {results_path}")
+
+
+    return eval_results
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 @dataclass
 class EvalResult:
     """Represents one full evaluation. Built from a combination of the result and request file for a given run.

src/populate.py

@@ -5,7 +5,7 @@ import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
 from src.display.utils import AutoEvalColumn, AutoEvalColumnMultimodal, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results, get_raw_eval_results_mib
+from src.leaderboard.read_evals import get_raw_eval_results, get_raw_eval_results_mib_subgraph, get_raw_eval_results_mib_causalgraph
 
 
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
@@ -42,39 +42,10 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
 
 
 
-# def get_leaderboard_df_mib(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-#     """Creates a dataframe from all the individual experiment results"""
-#     print(f"results_path is {results_path}, requests_path is {requests_path}")
-#     raw_data = get_raw_eval_results(results_path, requests_path)
-#     print(f"raw_data is {raw_data}")
-#     all_data_json = [v.to_dict() for v in raw_data]
-#     print(f"all_data_json is {pd.DataFrame.from_records(all_data_json)}")
-#     all_data_json_filtered = []
-#     for item in all_data_json:
-#         item["Track"] = item["eval_name"].split("_")[-1]
-#         if "VQA" in benchmark_cols and "VQA" in item:
-#             all_data_json_filtered.append(item)
-#         if "VQA" not in benchmark_cols and "VQA" not in item:
-#             all_data_json_filtered.append(item)
-#         all_data_json_filtered.append(item)
-        
-#     all_data_json = all_data_json_filtered
-
-#     df = pd.DataFrame.from_records(all_data_json)
-#     df = df.sort_values(by=[AutoEvalColumn.text_average.name], ascending=False)
-    
-#     print(f"df is {df}")
-    
-#     df = df[cols].round(decimals=1)
-
-#     # filter out if any of the benchmarks have not been produced
-#     df = df[has_no_nan_values(df, benchmark_cols)]
-#     return df
-
-def get_leaderboard_df_mib(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+def get_leaderboard_df_mib_subgraph(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the MIB experiment results"""
     print(f"results_path is {results_path}, requests_path is {requests_path}")
-    raw_data = get_raw_eval_results_mib(results_path, requests_path)
+    raw_data = get_raw_eval_results_mib_subgraph(results_path, requests_path)
     print(f"raw_data is {raw_data}")
     
     # Convert each result to dict format
@@ -94,10 +65,11 @@ def get_leaderboard_df_mib(results_path: str, requests_path: str, cols: list, be
     
     return df
 
-def get_leaderboard_df_mib_causal(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+def get_leaderboard_df_mib_causalgraph(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the MIB experiment results"""
     print(f"results_path is {results_path}, requests_path is {requests_path}")
-    raw_data = get_raw_eval_results_mib(results_path, requests_path)
+    raw_data = get_raw_eval_results_mib_causalgraph(results_path, requests_path)
+    # Implement the rest of the code
     return raw_data