debug

caulsal_metric.py

@@ -0,0 +1,243 @@
+import json
+import os
+import pandas as pd
+import numpy as np
+from typing import List, Dict, Any, Tuple
+from collections import defaultdict
+
+def average_counterfactuals(json_files: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """
+    Averages scores across counterfactuals for each layer.
+    """
+    processed_jsons = []
+    
+    for json_file in json_files:
+        new_json = {
+            'method_name': json_file['method_name'],
+            'results': []
+        }
+        
+        for result in json_file['results']:
+            new_result = {
+                'model_id': result['model_id'],
+                'task_scores': {}
+            }
+            
+            for task, scores in result['task_scores'].items():
+                new_scores = []
+                
+                for layer_data in scores:
+                    new_layer_data = {
+                        'layer': layer_data['layer'],
+                        'layer_scores': []
+                    }
+                    
+                    for intervention_data in layer_data['layer_scores']:
+                        avg_score = np.mean([cf['score'] for cf in intervention_data['counterfactual_scores']])
+                        if np.isnan(avg_score):
+                            avg_score = 0.0
+                        new_layer_data['layer_scores'].append({
+                            'intervention': intervention_data['intervention'],
+                            'average_score': avg_score
+                        })
+                    
+                    new_scores.append(new_layer_data)
+                
+                new_result['task_scores'][task] = new_scores
+            
+            new_json['results'].append(new_result)
+        
+        processed_jsons.append(new_json)
+    
+    return processed_jsons
+
+def find_layer_averages(json_files: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """
+    Averages scores across layers for each intervention.
+    """
+    processed_jsons = []
+    
+    for json_file in json_files:
+        new_json = {
+            'method_name': json_file['method_name'],
+            'results': []
+        }
+        
+        for result in json_file['results']:
+            new_result = {
+                'model_id': result['model_id'],
+                'task_scores': {}
+            }
+            
+            for task, scores in result['task_scores'].items():
+                # Group by intervention first
+                intervention_scores = defaultdict(list)
+                for layer_data in scores:
+                    for intervention_data in layer_data['layer_scores']:
+                        intervention_key = '_'.join(intervention_data['intervention'])
+                        intervention_scores[intervention_key].append(intervention_data['average_score'])
+                
+                # Average across layers for each intervention
+                new_result['task_scores'][task] = [
+                    {
+                        'intervention': intervention.split('_'),
+                        'average_score': np.mean(layer_scores) if layer_scores else 0.0
+                    }
+                    for intervention, layer_scores in intervention_scores.items()
+                ]
+            
+            new_json['results'].append(new_result)
+        
+        processed_jsons.append(new_json)
+    
+    return processed_jsons
+
+def create_summary_dataframe(json_files: List[Dict[str, Any]]) -> pd.DataFrame:
+    """
+    Creates a summary DataFrame with methods as rows and MODEL_TASK_INTERVENTION as columns.
+    Handles duplicate method names by adding a counter suffix.
+    """
+    data = {}
+    method_counters = defaultdict(int)
+    
+    for json_file in json_files:
+        method_name = json_file['method_name']
+        # Increment counter for this method name
+        method_counters[method_name] += 1
+        
+        # If this is a duplicate method name, append a counter
+        unique_method_name = f"{method_name}_{method_counters[method_name]}"
+            
+        method_scores = []
+        column_names = []
+        
+        for result in json_file['results']:
+            model = result['model_id']
+            for task, scores in result['task_scores'].items():
+                for score_data in scores:
+                    intervention = '_'.join(score_data['intervention'])
+                    column = f"{model}_{task}_{intervention}"
+                    score = f"{score_data['average_score']:.3f}"
+                    method_scores.append((column, score))
+        
+        # Sort by column names for consistency
+        method_scores.sort(key=lambda x: x[0])
+        scores_only = [float(score) for _, score in method_scores]
+        avg_score = np.mean(scores_only)
+        
+        # Add average as first column
+        data[unique_method_name] = {
+            **{col: score for col, score in method_scores}
+        }
+    
+    df = pd.DataFrame.from_dict(data, orient='index')
+    
+    
+    return df
+
+def aggregate_methods(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Aggregates rows with the same base method name by taking the max value for each column.
+    """
+    # Create a copy of the DataFrame
+    df_copy = df.copy()
+    
+    # Extract base method names (remove _2, _3, etc. suffixes)
+    base_methods = [name.split('_')[0] if '_' in name and name.split('_')[-1].isdigit() 
+                   else name for name in df_copy.index]
+    df_copy.index = base_methods
+    
+    # Convert scores to numeric values
+    def extract_score(score_str):
+        if isinstance(score_str, str):
+            return float(score_str)
+        return 0.0
+    
+    numeric_df = df_copy.applymap(extract_score)
+    
+    # Group by base method name and take the mean
+    aggregated_df = numeric_df.groupby(level=0).max().round(3)
+    
+    # Convert back to string format
+    aggregated_df = aggregated_df.applymap(lambda x: f"{x:.3f}")
+    
+    return aggregated_df
+
+def create_intervention_averaged_df(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Creates a DataFrame where columns are model_task and cells are averaged over interventions.
+    """
+    # Create a copy of the DataFrame
+    df_copy = df.copy()
+    
+    # Remove the Average column if it exists
+    if 'Average' in df_copy.columns:
+        df_copy = df_copy.drop('Average', axis=1)
+    
+    # Function to extract score value from string
+    def extract_score(score_str):
+        if isinstance(score_str, str):
+            return float(score_str.split()[0])
+        return 0.0
+    
+    # Convert all scores to numeric values
+    numeric_df = df_copy.applymap(extract_score)
+    
+    # Group columns by model_task
+    model_task_groups = {}
+    for col in numeric_df.columns:
+        model_task = '_'.join(col.split('_')[:2])  # Get model_task part
+        if model_task not in model_task_groups:
+            model_task_groups[model_task] = []
+        model_task_groups[model_task].append(col)
+    
+    # Create new DataFrame with averaged intervention scores
+    averaged_df = pd.DataFrame({
+        model_task: numeric_df[cols].mean(axis=1).round(3)
+        for model_task, cols in model_task_groups.items()
+    })
+    
+    # Add overall average column
+    averaged_df['Average'] = averaged_df.mean(axis=1).round(3)
+    
+    # Sort by Average column
+    averaged_df = averaged_df.sort_values('Average', ascending=False)
+    
+    return averaged_df
+
+def process_json_folder(folder_path: str) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+    """
+    Processes all JSON files in a folder and returns three DataFrames:
+    1. Detailed DataFrame showing all results including duplicates (with layer-averaged scores)
+    2. Aggregated DataFrame showing average scores for each base method
+    3. Intervention-averaged DataFrame showing means across interventions
+    """
+    json_files = []
+    
+    # Read all JSON files
+    for filename in os.listdir(folder_path):
+        if filename.endswith('.json'):
+            with open(os.path.join(folder_path, filename), 'r') as f:
+                json_files.append(json.load(f))
+    
+    # Process the files through each step
+    averaged_cf = average_counterfactuals(json_files)
+    layer_averaged = find_layer_averages(averaged_cf)
+    detailed_df = create_summary_dataframe(layer_averaged)
+    aggregated_df = aggregate_methods(detailed_df)
+    intervention_averaged_df = create_intervention_averaged_df(aggregated_df)
+    
+    return detailed_df, aggregated_df, intervention_averaged_df
+
+# Example usage:
+if __name__ == "__main__":
+    # Replace with your folder path
+    folder_path = "./json_files"
+    detailed_df, aggregated_df, intervention_averaged_df = process_json_folder(folder_path)
+    
+    print("Detailed Results (including duplicates):")
+    print(detailed_df)
+    print("\nAggregated Results (max scores per method):")
+    print(aggregated_df)
+    print("\nIntervention-Averaged Results:")
+    print(intervention_averaged_df)

src/about.py

@@ -51,19 +51,19 @@ class TaskMIB_Causalgraph:
     layers: list[str]   # 0-31
     col_name: str       # display name in leaderboard
     interventions: list[str]  # output_token, output_location
-    counterfactuals: list[str]  # symbol_counterfactual, randomLetter_counterfactual, etc.
-    metrics: list[str]  # score <- Added this field
+    counterfactuals: list[str]  # symbol_counterfactual, etc.
+    metrics: list[str]  # score
 
 class TasksMib_Causalgraph(Enum):
     task0 = TaskMIB_Causalgraph(
         "MCQA", 
         ["LlamaForCausalLM"],
-        [str(i) for i in range(32)],  # 0-31 layers
+        [str(i) for i in range(32)],
         "mcqa",
         ["output_token", "output_location"],
         ["symbol_counterfactual", "randomLetter_counterfactual", 
          "answerPosition_counterfactual", "answerPosition_symbol_counterfactual"],
-        ["score"]  # Added this
+        ["score"]
     )
 
 

src/display/utils.py

@@ -98,38 +98,36 @@ COLS_MIB_CAUSALGRAPH = []
 BENCHMARK_COLS_MIB_CAUSALGRAPH = []
 
 
+
+
+
+
 # Initialize the MIB causal graph columns
 auto_eval_column_dict_mib_causalgraph = []
 
 # Method name column
 auto_eval_column_dict_mib_causalgraph.append(["method", ColumnContent, ColumnContent("Method", "markdown", True, never_hidden=True)])
 
-# For each layer-intervention-counterfactual combination
+# For each model-task-intervention combination
 for task in TasksMib_Causalgraph:
     for model in task.value.models:
-        for layer in task.value.layers:
-            for intervention in task.value.interventions:
-                for counterfactual in task.value.counterfactuals:
-                    # Create column name like "layer0_output_token_symbol_counterfactual"
-                    col_name = f"layer{layer}_{intervention}_{counterfactual}"
-                    field_name = col_name.lower()
-                    auto_eval_column_dict_mib_causalgraph.append([
-                        field_name, 
-                        ColumnContent, 
-                        ColumnContent(col_name, "number", True)
-                    ])
-
-# Create the dataclass for MIB causal graph columns
+        for intervention in task.value.interventions:
+            col_name = f"{model}_{task.value.benchmark}_{intervention}".lower()
+            auto_eval_column_dict_mib_causalgraph.append([
+                col_name, 
+                ColumnContent, 
+                ColumnContent(col_name, "number", True)
+            ])
+
+# Create the dataclass
 AutoEvalColumn_mib_causalgraph = make_dataclass("AutoEvalColumn_mib_causalgraph", auto_eval_column_dict_mib_causalgraph, frozen=True)
 
 # Column selection for display
 COLS_MIB_CAUSALGRAPH = [c.name for c in fields(AutoEvalColumn_mib_causalgraph) if not c.hidden]
-BENCHMARK_COLS_MIB_CAUSALGRAPH = [f"layer{layer}_{intervention}_{counterfactual}" 
+BENCHMARK_COLS_MIB_CAUSALGRAPH = [f"{model}_{task.value.benchmark}_{intervention}".lower()
                                  for task in TasksMib_Causalgraph
                                  for model in task.value.models
-                                 for layer in task.value.layers
-                                 for intervention in task.value.interventions
-                                 for counterfactual in task.value.counterfactuals]
+                                 for intervention in task.value.interventions]
 
 
 

src/leaderboard/read_evals.py

@@ -185,43 +185,48 @@ def get_raw_eval_results_mib_subgraph(results_path: str, requests_path: str) ->
 @dataclass
 class EvalResult_MIB_CAUSALGRAPH:
     """Represents one full evaluation for a method in MIB causalgraph."""
-    eval_name: str        # method name as identifier
-    method_name: str      # name of the interpretation method (e.g., "baseline_patching")
-    results: Dict         # nested dict of results {model_id: {task_scores: [{layer, scores}]}}
+    eval_name: str        
+    method_name: str      
+    results: Dict         
 
     def init_from_json_file(self, json_filepath):
         """Inits results from the method result file"""
         with open(json_filepath) as fp:
             data = json.load(fp)
-
+        
         method_name = data.get("method_name")
         results = {}
         
-        # Get results for each model
-        for model_result in data.get("results", []):
-            model_id = model_result.get("model_id", "")
-            task_scores = model_result.get("task_scores", {})
+        # First average across counterfactuals
+        for result in data.get("results", []):
+            model_id = result.get("model_id")
+            task_scores = result.get("task_scores", {})
             
-            # Process MCQA task scores
-            mcqa_scores = {}
-            for layer_data in task_scores.get("MCQA", []):
-                layer = layer_data.get("layer")
-                layer_scores = layer_data.get("layer_scores", [])
-                
-                # Store scores for each intervention and counterfactual
-                for intervention_data in layer_scores:
-                    intervention = intervention_data["intervention"][0]  # e.g., "output_token"
-                    counterfactual_scores = intervention_data["counterfactual_scores"]
+            model_results = {}
+            for task, scores in task_scores.items():
+                layer_scores = []
+                for layer_data in scores:
+                    layer = layer_data.get("layer")
+                    layer_scores_data = []
                     
-                    for cf_score in counterfactual_scores:
-                        counterfactual = cf_score["counterfactual"][0]  # e.g., "symbol_counterfactual"
-                        score = cf_score["score"]
-                        
-                        # Create key for this combination
-                        key = f"layer{layer}_{intervention}_{counterfactual}"
-                        mcqa_scores[key] = score
-
-            results[model_id] = mcqa_scores
+                    for intervention_data in layer_data.get("layer_scores", []):
+                        # Average across counterfactuals
+                        avg_score = np.mean([cf['score'] for cf in intervention_data['counterfactual_scores']])
+                        if np.isnan(avg_score):
+                            avg_score = 0.0
+                        layer_scores_data.append({
+                            'intervention': intervention_data['intervention'][0],
+                            'score': avg_score
+                        })
+                    
+                    layer_scores.append({
+                        'layer': layer,
+                        'scores': layer_scores_data
+                    })
+                
+                model_results[task] = layer_scores
+            
+            results[model_id] = model_results
 
         return EvalResult_MIB_CAUSALGRAPH(
             eval_name=method_name,
@@ -235,27 +240,33 @@ class EvalResult_MIB_CAUSALGRAPH:
             "eval_name": self.eval_name,
             "Method": self.method_name,
         }
-
-        # For each model, add all layer/intervention/counterfactual combinations
-        for model_id, scores in self.results.items():
-            for score_key, score_value in scores.items():
-                data_dict[score_key] = score_value
+        
+        # Process each model's results
+        for model_id, model_results in self.results.items():
+            for task, task_scores in model_results.items():
+                # Calculate layer-averaged scores for each intervention
+                intervention_scores = defaultdict(list)
+                
+                for layer_data in task_scores:
+                    for score_data in layer_data['scores']:
+                        intervention = score_data['intervention']
+                        intervention_scores[intervention].append(score_data['score'])
                 
+                # Average across layers for each intervention
+                for intervention, scores in intervention_scores.items():
+                    col_name = f"{model_id}_{task}_{intervention}".lower()
+                    data_dict[col_name] = round(np.mean(scores), 3)
+        
         return data_dict
 
 def get_raw_eval_results_mib_causalgraph(results_path: str, requests_path: str) -> List[EvalResult_MIB_CAUSALGRAPH]:
-    """From the path of the results folder root, extract all needed info for MIB causalgraph results"""
+    """Extract evaluation results for MIB causalgraph"""
     model_result_filepaths = []
     
-    print(f"results_path is {results_path}")
-    
     for root, dirnames, files in os.walk(results_path):
-        print(f"root is {root}, dirnames is {dirnames}, files is {files}")
-        # We should only have json files in model results
         if len(files) == 0 or any([not f.endswith(".json") for f in files]):
             continue
         
-        # Sort the files by date - keeping original sorting logic
         try:
             files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
         except dateutil.parser._parser.ParserError:
@@ -264,19 +275,15 @@ def get_raw_eval_results_mib_causalgraph(results_path: str, requests_path: str)
         for file in files:
             model_result_filepaths.append(os.path.join(root, file))
 
-    print(f"model_result_filepaths is {model_result_filepaths}")
-
     eval_results = []
-    for model_result_filepath in model_result_filepaths:
+    for filepath in model_result_filepaths:
         try:
-            eval_result = EvalResult_MIB_CAUSALGRAPH("", "", {})  # Create empty instance
-            result = eval_result.init_from_json_file(model_result_filepath)
-            print(f"eval_result.init_from_json_file(model_result_filepath) is {result}")
-            # Verify the result can be converted to dict format
-            result.to_dict()
+            eval_result = EvalResult_MIB_CAUSALGRAPH("", "", {})
+            result = eval_result.init_from_json_file(filepath)
+            result.to_dict()  # Verify conversion works
             eval_results.append(result)
         except Exception as e:
-            print(f"Error processing {model_result_filepath}: {e}")
+            print(f"Error processing {filepath}: {e}")
             continue
 
     return eval_results

src/populate.py

@@ -65,24 +65,71 @@ def get_leaderboard_df_mib_subgraph(results_path: str, requests_path: str, cols:
     
     return df
 
+
+
+
+def aggregate_methods(df: pd.DataFrame) -> pd.DataFrame:
+    """Aggregates rows with the same base method name by taking the max value for each column"""
+    df_copy = df.copy()
+    
+    # Extract base method names (remove _2, _3, etc. suffixes)
+    base_methods = [name.split('_')[0] if '_' in name and name.split('_')[-1].isdigit() 
+                   else name for name in df_copy.index]
+    df_copy.index = base_methods
+    
+    # Convert scores to numeric values
+    numeric_df = df_copy.select_dtypes(include=['float64', 'int64'])
+    
+    # Group by base method name and take the max
+    aggregated_df = numeric_df.groupby(level=0).max().round(3)
+    
+    return aggregated_df
+
+def create_intervention_averaged_df(df: pd.DataFrame) -> pd.DataFrame:
+    """Creates a DataFrame where columns are model_task and cells are averaged over interventions"""
+    df_copy = df.copy()
+    
+    # Remove the Method column and eval_name if present
+    columns_to_drop = ['Method', 'eval_name']
+    df_copy = df_copy.drop(columns=[col for col in columns_to_drop if col in df_copy.columns])
+    
+    # Group columns by model_task
+    model_task_groups = {}
+    for col in df_copy.columns:
+        model_task = '_'.join(col.split('_')[:2])  # Get model_task part
+        if model_task not in model_task_groups:
+            model_task_groups[model_task] = []
+        model_task_groups[model_task].append(col)
+    
+    # Create new DataFrame with averaged intervention scores
+    averaged_df = pd.DataFrame({
+        model_task: df_copy[cols].mean(axis=1).round(3)
+        for model_task, cols in model_task_groups.items()
+    })
+    
+    return averaged_df
+
+
 def get_leaderboard_df_mib_causalgraph(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the MIB causal graph experiment results"""
     print(f"results_path is {results_path}, requests_path is {requests_path}")
     raw_data = get_raw_eval_results_mib_causalgraph(results_path, requests_path)
     print(f"raw_data is {raw_data}")
     
-    # Convert each result to dict format
+    # Convert each result to dict format for detailed df
     all_data_json = [v.to_dict() for v in raw_data]
-    print(f"all_data_json is {pd.DataFrame.from_records(all_data_json)}")
+    detailed_df = pd.DataFrame.from_records(all_data_json)
+    print(f"detailed_df is: {detailed_df}")
 
-    # Convert to dataframe
-    df = pd.DataFrame.from_records(all_data_json)
+    # Create and print other views for debugging/reference
+    aggregated_df = aggregate_methods(detailed_df)
+    print(f"aggregated_df is: {aggregated_df}")
     
-    # Round numeric columns to 2 decimal places
-    numeric_cols = df.select_dtypes(include=['float64', 'int64']).columns
-    df[numeric_cols] = df[numeric_cols].round(2)
+    intervention_averaged_df = create_intervention_averaged_df(aggregated_df)
+    print(f"intervention_averaged_df is: {intervention_averaged_df}")
     
-    return df
+    # Only return detailed_df for display
+    return detailed_df
 
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]: