updates

app.py

@@ -23,6 +23,7 @@ def restart_space():
 print("Pulling evaluation results")
 repo = snapshot_download(
     local_dir=repo_dir_herm,
+    ignore_patterns=["pref-sets-scores/*", "eval-set-scores/*"],
     repo_id=evals_repo,
     use_auth_token=COLLAB_TOKEN,
     tqdm_class=None, 
@@ -31,7 +32,7 @@ repo = snapshot_download(
 )
 
 
-def avg_over_herm(dataframe):
+def avg_over_herm(dataframe_core, dataframe_prefs):
     """
     Averages over the subsets alpacaeval, mt-bench, llmbar, refusals, hep and returns dataframe with only these columns.
 
@@ -42,7 +43,8 @@ def avg_over_herm(dataframe):
     4. Code: Includes the code subsets (hep-cpp, hep-go, hep-java, hep-js, hep-python, hep-rust)
 
     """
-    new_df = dataframe.copy()
+    new_df = dataframe_core.copy()
+    dataframe_prefs = dataframe_prefs.copy()
 
     # for main subsets, keys in subset_mapping, take the weighted avg by example_counts and store for the models
     for subset, sub_subsets in subset_mapping.items():
@@ -52,10 +54,39 @@ def avg_over_herm(dataframe):
         new_df[subset] = np.round(np.average(sub_data, axis=1, weights=sub_counts), 2) # take the weighted average
         # new_df[subset] = np.round(np.nanmean(new_df[subset_cols].values, axis=1), 2)
 
-    keep_columns = ["model",] + ["model_type"] + list(subset_mapping.keys())
+    data_cols = list(subset_mapping.keys())
+    keep_columns = ["model",] + ["model_type"] + data_cols
     # keep_columns = ["model", "average"] + subsets
     new_df = new_df[keep_columns]
 
+    # selected average from pref_sets
+    pref_columns = ["anthropic_helpful", "mtbench_gpt4", "shp", "summarize"]
+    pref_data = dataframe_prefs[pref_columns].values
+
+    # add column test sets knowing the rows are not identical, take superset
+    dataframe_prefs["Test Sets"] = np.round(np.nanmean(pref_data, axis=1), 2)
+
+    # add column Test Sets empty to new_df
+    new_df["Test Sets"] = np.nan
+    # per row in new_df if model is in dataframe_prefs, add the value to new_df["Test Sets"]
+    values = []
+    for i, row in new_df.iterrows():
+        model = row["model"]
+        if model in dataframe_prefs["model"].values:
+            values.append(dataframe_prefs[dataframe_prefs["model"] == model]["Test Sets"].values[0])
+            # new_df.at[i, "Test Sets"] = dataframe_prefs[dataframe_prefs["model"] == model]["Test Sets"].values[0]
+        else:
+            values.append(np.nan)
+    
+    new_df["Test Sets"] = values
+
+    # add total average
+    data_cols += ["Test Sets"]
+    new_df["average"] = np.round(np.nanmean(new_df[data_cols].values, axis=1), 2)
+
+    # make average third column
+    keep_columns = ["model", "model_type", "average"] + data_cols
+    new_df = new_df[keep_columns]
     return new_df
 
 def expand_subsets(dataframe):
@@ -100,11 +131,12 @@ def length_bias_check(dataframe):
 
     
 herm_data = load_all_data(repo_dir_herm, subdir="eval-set").sort_values(by='average', ascending=False)
-herm_data_avg = avg_over_herm(herm_data).sort_values(by='Chat', ascending=False)
 herm_data_length = length_bias_check(herm_data).sort_values(by='Terse Bias', ascending=False)
 prefs_data = load_all_data(repo_dir_herm, subdir="pref-sets").sort_values(by='average', ascending=False)
 # prefs_data_sub = expand_subsets(prefs_data).sort_values(by='average', ascending=False)
 
+herm_data_avg = avg_over_herm(herm_data, prefs_data).sort_values(by='average', ascending=False)
+
 col_types_herm = ["markdown"] + ["str"] + ["number"] * (len(herm_data.columns) - 1)
 col_types_herm_avg = ["markdown"]+ ["str"] + ["number"] * (len(herm_data_avg.columns) - 1)
 cols_herm_data_length = ["markdown"] + ["number"] * (len(herm_data_length.columns) - 1)