new metrics

src/display/utils.py

@@ -138,7 +138,7 @@ class GuardBenchColumn:
         name="default_prompts_f1",
         display_name="Default Prompts F1",
         type="number",
-        displayed_by_default=True
+        displayed_by_default=False
     ))
     default_prompts_recall_binary: ColumnInfo = field(default_factory=lambda: ColumnInfo(
         name="default_prompts_recall_binary",
@@ -176,7 +176,7 @@ class GuardBenchColumn:
         name="jailbreaked_prompts_f1",
         display_name="Jailbreaked Prompts F1",
         type="number",
-        displayed_by_default=True
+        displayed_by_default=False
     ))
     jailbreaked_prompts_recall_binary: ColumnInfo = field(default_factory=lambda: ColumnInfo(
         name="jailbreaked_prompts_recall_binary",
@@ -214,7 +214,7 @@ class GuardBenchColumn:
         name="default_answers_f1",
         display_name="Default Answers F1",
         type="number",
-        displayed_by_default=True
+        displayed_by_default=False
     ))
     default_answers_recall_binary: ColumnInfo = field(default_factory=lambda: ColumnInfo(
         name="default_answers_recall_binary",
@@ -279,6 +279,51 @@ class GuardBenchColumn:
         displayed_by_default=False
     ))
 
+    # Calculated overall metrics (renamed)
+    macro_accuracy: ColumnInfo = field(default_factory=lambda: ColumnInfo(
+        name="macro_accuracy",
+        display_name="Macro Accuracy",
+        type="number",
+        displayed_by_default=True
+    ))
+    macro_recall: ColumnInfo = field(default_factory=lambda: ColumnInfo(
+        name="macro_recall",
+        display_name="Macro Recall",
+        type="number",
+        displayed_by_default=True
+    ))
+    macro_precision: ColumnInfo = field(default_factory=lambda: ColumnInfo(
+        name="macro_precision",
+        display_name="Macro Precision",
+        type="number",
+        displayed_by_default=False
+    ))
+    integral_score: ColumnInfo = field(default_factory=lambda: ColumnInfo(
+        name="integral_score",
+        display_name="Integral Score",
+        type="number",
+        displayed_by_default=True
+    ))
+    # NEW Summary Metrics
+    micro_avg_error_ratio: ColumnInfo = field(default_factory=lambda: ColumnInfo(
+        name="micro_avg_error_ratio",
+        display_name="Micro Error %",
+        type="number",
+        displayed_by_default=True
+    ))
+    micro_avg_runtime_ms: ColumnInfo = field(default_factory=lambda: ColumnInfo(
+        name="micro_avg_runtime_ms",
+        display_name="Micro Avg Time (ms)",
+        type="number",
+        displayed_by_default=True
+    ))
+    total_evals_count: ColumnInfo = field(default_factory=lambda: ColumnInfo(
+        name="total_evals_count",
+        display_name="Total Evals Count",
+        type="number",
+        displayed_by_default=True
+    ))
+
 
 # Create instances for easy access
 GUARDBENCH_COLUMN = GuardBenchColumn()

src/leaderboard/processor.py

@@ -7,9 +7,86 @@ import os
 import pandas as pd
 from datetime import datetime
 from typing import Dict, List, Any, Tuple
+import numpy as np
 
 from src.display.utils import CATEGORIES, TEST_TYPES, METRICS
 
+# Constants for Integral Score calculation (mirrors guardbench library)
+MAX_PUNISHABLE_RUNTIME_MS = 6000.0
+MIN_PUNISHABLE_RUNTIME_MS = 200.0
+MAX_RUNTIME_PENALTY = 0.75 # Corresponds to 1.0 - MIN_TIME_FACTOR, library used 0.75
+
+def calculate_integral_score(row: pd.Series) -> float:
+    """
+    Calculate the integral score for a given model entry row.
+    Uses F1-binary as the primary metric, error ratio, and runtime penalty.
+    """
+    integral_score = 1.0
+    metric_count = 0
+
+    # Primary metric (using f1_binary, could be changed to accuracy if needed)
+    for test_type in TEST_TYPES:
+        metric_col = f"{test_type}_f1_binary"
+        if metric_col in row and pd.notna(row[metric_col]):
+            integral_score *= row[metric_col]
+            metric_count += 1
+
+    # If no primary metrics found, return 0
+    if metric_count == 0:
+        # Check for average_f1 as a fallback
+        if "average_f1" in row and pd.notna(row["average_f1"]):
+            integral_score *= row["average_f1"]
+            metric_count += 1
+        else:
+            return 0.0 # Cannot calculate score without primary metrics
+
+    # Account for average errors across all test types (using a simple average for now)
+    # This requires micro-level error data which isn't directly in avg_metrics.
+    # We'll approximate using the average of available error ratios.
+    error_ratios = []
+    for test_type in TEST_TYPES:
+        error_col = f"{test_type}_error_ratio"
+        if error_col in row and pd.notna(row[error_col]):
+            error_ratios.append(row[error_col])
+
+    if error_ratios:
+        avg_error_ratio = np.mean(error_ratios)
+        integral_score *= (1.0 - avg_error_ratio)
+
+    # Account for average runtime across all test types (using a simple average for now)
+    # This requires micro-level runtime data. We'll approximate.
+    runtimes = []
+    for test_type in TEST_TYPES:
+        runtime_col = f"{test_type}_avg_runtime_ms"
+        if runtime_col in row and pd.notna(row[runtime_col]):
+            runtimes.append(row[runtime_col])
+
+    if runtimes:
+        avg_runtime_ms = np.mean(runtimes)
+
+        # Apply penalty based on runtime
+        runtime = max(
+            min(avg_runtime_ms, MAX_PUNISHABLE_RUNTIME_MS),
+            MIN_PUNISHABLE_RUNTIME_MS,
+        )
+
+        if MAX_PUNISHABLE_RUNTIME_MS > MIN_PUNISHABLE_RUNTIME_MS:
+            normalized_time = (runtime - MIN_PUNISHABLE_RUNTIME_MS) / (
+                MAX_PUNISHABLE_RUNTIME_MS - MIN_PUNISHABLE_RUNTIME_MS
+            )
+            time_factor = 1.0 - MAX_RUNTIME_PENALTY * normalized_time
+        else:
+            time_factor = 1.0 if runtime <= MIN_PUNISHABLE_RUNTIME_MS else (1.0 - MAX_RUNTIME_PENALTY) # Assign max penalty if runtime exceeds min when max==min
+
+        # Make sure the factor is not less than the minimum value (1 - MAX_PENALTY)
+        time_factor = max((1.0 - MAX_RUNTIME_PENALTY), time_factor)
+        integral_score *= time_factor
+
+    # Root the score by the number of primary metrics used? (Optional, library did this)
+    # return integral_score ** (1 / metric_count) if metric_count > 0 else 0.0
+    # Let's skip the rooting for now to keep the scale potentially larger.
+    return integral_score
+
 
 def load_leaderboard_data(file_path: str) -> Dict:
     """
@@ -133,29 +210,39 @@ def leaderboard_to_dataframe(leaderboard_data: Dict) -> pd.DataFrame:
                                 row[f"{test_type}_f1"] = metrics[metric]
 
             # Calculate averages if not present
-            if "average_f1" not in row:
+            if "macro_accuracy" not in row:
                 f1_values = []
                 for test_type in TEST_TYPES:
-                    if test_type in avg_metrics and "f1_binary" in avg_metrics[test_type]:
+                    if test_type in avg_metrics and "f1_binary" in avg_metrics[test_type] and pd.notna(avg_metrics[test_type]["f1_binary"]):
                         f1_values.append(avg_metrics[test_type]["f1_binary"])
                 if f1_values:
-                    row["average_f1"] = sum(f1_values) / len(f1_values)
+                    row["macro_accuracy"] = sum(f1_values) / len(f1_values)
 
-            if "average_recall" not in row:
+            if "macro_recall" not in row:
                 recall_values = []
                 for test_type in TEST_TYPES:
-                    if test_type in avg_metrics and "recall_binary" in avg_metrics[test_type]:
+                    if test_type in avg_metrics and "recall_binary" in avg_metrics[test_type] and pd.notna(avg_metrics[test_type]["recall_binary"]):
                         recall_values.append(avg_metrics[test_type]["recall_binary"])
                 if recall_values:
-                    row["average_recall"] = sum(recall_values) / len(recall_values)
+                    row["macro_recall"] = sum(recall_values) / len(recall_values)
 
-            if "average_precision" not in row:
-                precision_values = []
+            if "total_evals_count" not in row:
+                total_samples = 0
+                found_samples = False
                 for test_type in TEST_TYPES:
-                    if test_type in avg_metrics and "precision_binary" in avg_metrics[test_type]:
-                        precision_values.append(avg_metrics[test_type]["precision_binary"])
-                if precision_values:
-                    row["average_precision"] = sum(precision_values) / len(precision_values)
+                    if test_type in avg_metrics and "sample_count" in avg_metrics[test_type] and pd.notna(avg_metrics[test_type]["sample_count"]):
+                        total_samples += avg_metrics[test_type]["sample_count"]
+                        found_samples = True
+                if found_samples:
+                    row["total_evals_count"] = total_samples
+
+        # Extract micro averages directly from entry if they exist (like in guardbench library)
+        row["micro_avg_error_ratio"] = entry.get("micro_avg_error_ratio", pd.NA)
+        row["micro_avg_runtime_ms"] = entry.get("micro_avg_runtime_ms", pd.NA)
+
+        # Convert error ratio to percentage for consistency with display name
+        if pd.notna(row["micro_avg_error_ratio"]):
+            row["micro_avg_error_ratio"] *= 100
 
         rows.append(row)
 
@@ -164,17 +251,34 @@ def leaderboard_to_dataframe(leaderboard_data: Dict) -> pd.DataFrame:
 
     # Ensure all expected columns exist
     for test_type in TEST_TYPES:
-        if f"{test_type}_f1" not in df.columns:
-            df[f"{test_type}_f1"] = None
-        if f"{test_type}_f1_binary" not in df.columns:
-            df[f"{test_type}_f1_binary"] = None
-        if f"{test_type}_recall_binary" not in df.columns:
-            df[f"{test_type}_recall_binary"] = None
-        if f"{test_type}_precision_binary" not in df.columns:
-            df[f"{test_type}_precision_binary"] = None
-
-    if not df.empty and "average_f1" in df.columns:
-        df = df.sort_values(by="average_f1", ascending=False)
+        for metric in METRICS:
+            col_name = f"{test_type}_{metric}"
+            if col_name not in df.columns:
+                df[col_name] = pd.NA # Use pd.NA for missing numeric data
+            # Add non-binary F1 if binary exists
+            if metric == "f1_binary" and f"{test_type}_f1" not in df.columns:
+                df[f"{test_type}_f1"] = df[col_name] # Copy f1_binary to f1 if f1 is missing
+
+    # Calculate Integral Score
+    if not df.empty:
+        df["integral_score"] = df.apply(calculate_integral_score, axis=1)
+        # Sort by Integral Score instead of average_f1
+        df = df.sort_values(by="integral_score", ascending=False, na_position='last')
+    else:
+        # Add the column even if empty
+        df["integral_score"] = pd.NA
+
+    # Ensure summary columns exist
+    summary_cols = ["macro_accuracy", "macro_recall", "micro_avg_error_ratio", "micro_avg_runtime_ms", "total_evals_count"]
+    for col in summary_cols:
+        if col not in df.columns:
+            df[col] = pd.NA
+
+    # Remove old average columns if they somehow snuck in
+    old_avg_cols = ["average_f1", "average_recall", "average_precision"]
+    for col in old_avg_cols:
+        if col in df.columns:
+            df = df.drop(columns=[col])
 
     return df