update

.gitattributes

@@ -284,3 +284,53 @@ results/gpt-4-1106.pkl filter=lfs diff=lfs merge=lfs -text
 results/Llama-3-70b-chat-hf.pkl filter=lfs diff=lfs merge=lfs -text
 results/dbrx-instruct.pkl filter=lfs diff=lfs merge=lfs -text
 results/gpt-3.5-0613.pkl filter=lfs diff=lfs merge=lfs -text
+final_df.pkl filter=lfs diff=lfs merge=lfs -text
+heatmaps/claude-3-haiku-20240307_vision.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/Mixtral-8x7B-Instruct-v0.1_Textonly.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/claude-3-haiku-20240307_vision-CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gpt-4-1106_Textonly.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gpt-4-vision-preview_vision-CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/dbrx-instruct_Textonly.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/Llama-2-70b-chat-hf_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/Llama-3-70b-chat-hf_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/Llama-3-70b-chat-hf_Textonly.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/Mistral-7B-Instruct-v0.2_Textonly.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/claude-3-sonnet-20240229_vision.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/CodeLlama-70b-Instruct-hf_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/claude-3-haiku-20240307_Textonly.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gemma-7b-it_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gpt-4-0125-preview_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/claude-3-haiku-20240307_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/Mistral-7B-Instruct-v0.2_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/Qwen1.5-72B-Chat_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gpt-3.5-turbo-0125_1shot.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/claude-3-sonnet-20240229_Textonly.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/Mixtral-8x7B-Instruct-v0.1_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gemma-7b-it_Textonly.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gpt-4-0125-preview_Textonly.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gpt-4-turbo-2024-04-09_Textonly.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/claude-3-haiku-20240307_1shot.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/claude-3-opus-20240229_Textonly.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/deepseek-llm-67b-chat_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/claude-3-opus-20240229_vision-CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gemini-pro_vision-CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gpt-3.5-0613_Textonly.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gpt-4-vision-preview_vision.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gemini-pro_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/CodeLlama-70b-Instruct-hf_Textonly.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/Qwen1.5-72B-Chat_Textonly.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/Yi-34B-Chat_Textonly.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/claude-3-sonnet-20240229_vision-CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/dbrx-instruct_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gemini-pro_vision.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gpt-3.5-0613_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gpt-4-1106_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gpt-4-turbo-2024-04-09_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/Llama-2-70b-chat-hf_Textonly.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/claude-3-opus-20240229_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gpt-3.5-turbo-0125_Textonly.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/gpt-3.5-turbo-0125_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/Yi-34B-Chat_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/claude-3-opus-20240229_vision.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/claude-3-sonnet-20240229_CoT.jpg filter=lfs diff=lfs merge=lfs -text
+heatmaps/deepseek-llm-67b-chat_Textonly.jpg filter=lfs diff=lfs merge=lfs -text

app.py

@@ -7,27 +7,50 @@ import pandas as pd
 import seaborn as sns
 from matplotlib.colors import BoundaryNorm, ListedColormap
 
-all_results = pd.read_pickle("all_results.pkl")
+all_results = pd.read_pickle("final_df.pkl")
 
 
-def get_accuracy_dataframe(df):
+def get_accuracy_dataframe(df_mother, category):
     # Calculate overall model accuracy
-    df['parsed_judge_response'] = df['parsed_judge_response'].astype(float)
-    model_accuracy = df.groupby('model_name')['parsed_judge_response'].mean().reset_index()
-    
+    # filter for category only
+    df = df_mother[df_mother["category"] == category].copy()
+    df["is_answer_correct"] = df["is_answer_correct"].astype(float)
+    model_accuracy = df.groupby("model")["is_answer_correct"].mean().reset_index()
+
     # Calculate model accuracy per difficulty level
-    df['difficulty_level'] = df['difficulty_level'].astype(int)
-    model_accuracy_per_level = df.groupby(['model_name', 'difficulty_level'])['parsed_judge_response'].mean().reset_index()
-    model_accuracy_per_level_df = model_accuracy_per_level.pivot(index='model_name', columns='difficulty_level', values='parsed_judge_response')
-    
+    df["difficulty_level"] = df["difficulty_level"].astype(int)
+    model_accuracy_per_level = (
+        df.groupby(["model", "difficulty_level"])["is_answer_correct"]
+        .mean()
+        .reset_index()
+    )
+    model_accuracy_per_level_df = model_accuracy_per_level.pivot(
+        index="model", columns="difficulty_level", values="is_answer_correct"
+    )
+
     # Merge overall accuracy and level-based accuracy into a single DataFrame
-    model_accuracy_df = model_accuracy.merge(model_accuracy_per_level_df, on='model_name')
-    model_accuracy_df.rename(columns={1: 'level_1', 2: 'level_2', 3: 'level_3', 4: 'level_4', 5: 'level_5'}, inplace=True)
-    model_accuracy_df.rename(columns={'parsed_judge_response': 'Accuracy'}, inplace=True)
-    
+    model_accuracy_df = model_accuracy.merge(model_accuracy_per_level_df, on="model")
+    model_accuracy_df.rename(
+        columns={"is_answer_correct": "Overall Accuracy"}, inplace=True
+    )
+
+    # Ensure all expected difficulty levels are present
+    expected_levels = [1, 2, 3, 4]  # Adjust based on your data
+    for level in expected_levels:
+        if level not in model_accuracy_df.columns:
+            model_accuracy_df[
+                level
+            ] = None  # Fill missing levels with None or an appropriate value
+
+    # Rename columns to include levels
+    level_columns = {level: f"Level {level} Accuracy" for level in expected_levels}
+    model_accuracy_df.rename(columns=level_columns, inplace=True)
+
     # Multiply by 100 and format to one decimal point
-    model_accuracy_df = model_accuracy_df.applymap(lambda x: round(x * 100, 1) if isinstance(x, float) else x)
-    
+    model_accuracy_df = model_accuracy_df.applymap(
+        lambda x: round(x * 100, 1) if isinstance(x, float) else x
+    )
+
     # Add headers with icons
     model_accuracy_df.columns = [
         "🤖 Model Name",
@@ -40,13 +63,15 @@ def get_accuracy_dataframe(df):
 
     model_accuracy_df.sort_values(by="⭐ Overall", ascending=False, inplace=True)
     
-    # Add a new column at the beginning for the rank
-    model_accuracy_df.insert(0, '#', range(1, len(model_accuracy_df) + 1))
-    
     return model_accuracy_df
 
 
-accuracy_df = get_accuracy_dataframe(all_results)
+# categories = array(['1shot', 'CoT', 'Textonly', 'vision', 'vision-CoT'], dtype=object)
+accuracy_df_textonly = get_accuracy_dataframe(all_results, "Textonly")
+accuracy_df_cot = get_accuracy_dataframe(all_results, "CoT")
+accuracy_df_vision = get_accuracy_dataframe(all_results, "vision")
+accuracy_df_vision_cot = get_accuracy_dataframe(all_results, "vision-CoT")
+accuracy_df_1shot = get_accuracy_dataframe(all_results, "1shot")
 
 
 # Define the column names with icons
@@ -68,126 +93,83 @@ column_names = [
     "Level 4 Accuracy",
 ]
 
-def load_heatmap(evt: gr.SelectData):
-    heatmap_image = gr.Image(f"results/{evt.value}.jpg")
+
+def load_heatmap_textonly(evt: gr.SelectData):
+    print(f"./heatmaps/{evt.value}_Textonly.jpg")
+    heatmap_image = gr.Image(f"./heatmaps/{evt.value}_Textonly.jpg")
     return heatmap_image
 
 
+def load_heatmap_cot(evt: gr.SelectData):
+    heatmap_image = gr.Image(f"./heatmaps/{evt.value}_CoT.jpg")
+    return heatmap_image
 
-# # Function to process data
-# def process_data(data):
-#     data_for_df = []
-#     for file, df in data.items():
-#         overall_accuracy = round(calculate_accuracy(df), 2)
-#         breakdown_accuracy = [round(acc, 2) for acc in accuracy_breakdown(df)]
-#         model_name = file.split("/")[-1].replace(".pkl", "")
-#         data_for_df.append([model_name, overall_accuracy] + breakdown_accuracy)
-#     return data_for_df
 
+def load_heatmap_vision(evt: gr.SelectData):
+    heatmap_image = gr.Image(f"./heatmaps/{evt.value}_vision.jpg")
+    return heatmap_image
 
-# # Function to finalize DataFrame
-# def finalize_df(df):
-#     df = df.round(1)  # Round to one decimal place
-#     df = df.applymap(lambda x: f"{x:.1f}" if isinstance(x, (int, float)) else x)
-#     df.columns = headers_with_icons
-#     df.sort_values(by="⭐ Overall", ascending=False, inplace=True)
-#     # add a new column with the order (index)
-#     df["#"] = range(1, len(df) + 1)
-#     # bring rank to the first column
-#     cols = df.columns.tolist()
-#     cols = cols[-1:] + cols[:-1]
-#     df = df[cols]
 
-#     return df
+def load_heatmap_vision_cot(evt: gr.SelectData):
+    heatmap_image = gr.Image(f"./heatmaps/{evt.value}_vision-CoT.jpg")
+    return heatmap_image
 
 
-def load_heatmap(evt: gr.SelectData):
-    heatmap_image = gr.Image(f"results/{evt.value}.jpg")
+def load_heatmap_1shot(evt: gr.SelectData):
+    heatmap_image = gr.Image(f"./heatmaps/{evt.value}_1shot.jpg")
     return heatmap_image
 
 
+# Then, use these functions in the corresponding select method calls:
+
 with gr.Blocks() as demo:
     gr.Markdown("# FSM Benchmark Leaderboard")
+
+    # Text-only Benchmark
     with gr.Tab("Text-only Benchmark"):
-        leader_board = gr.Dataframe(accuracy_df, headers=headers_with_icons)
+        leader_board_textonly = gr.Dataframe(
+            accuracy_df_textonly, headers=headers_with_icons
+        )
+        gr.Markdown("## Heatmap")
+        heatmap_image_textonly = gr.Image(label="", show_label=False)
+        leader_board_textonly.select(
+            fn=load_heatmap_textonly, outputs=[heatmap_image_textonly]
+        )
+
+    # CoT Benchmark
+    with gr.Tab("CoT Benchmark"):
+        leader_board_cot = gr.Dataframe(accuracy_df_cot, headers=headers_with_icons)
+        gr.Markdown("## Heatmap")
+        heatmap_image_cot = gr.Image(label="", show_label=False)
+        leader_board_cot.select(fn=load_heatmap_cot, outputs=[heatmap_image_cot])
+
+    # Vision Benchmark
+    with gr.Tab("Vision Benchmark"):
+        leader_board_vision = gr.Dataframe(
+            accuracy_df_vision, headers=headers_with_icons
+        )
+        gr.Markdown("## Heatmap")
+        heatmap_image_vision = gr.Image(label="", show_label=False)
+        leader_board_vision.select(
+            fn=load_heatmap_vision, outputs=[heatmap_image_vision]
+        )
+
+    # Vision-CoT Benchmark
+    with gr.Tab("Vision-CoT Benchmark"):
+        leader_board_vision_cot = gr.Dataframe(
+            accuracy_df_vision_cot, headers=headers_with_icons
+        )
+        gr.Markdown("## Heatmap")
+        heatmap_image_vision_cot = gr.Image(label="", show_label=False)
+        leader_board_vision_cot.select(
+            fn=load_heatmap_vision_cot, outputs=[heatmap_image_vision_cot]
+        )
+
+    # 1shot Benchmark
+    with gr.Tab("1shot Benchmark"):
+        leader_board_1shot = gr.Dataframe(accuracy_df_1shot, headers=headers_with_icons)
         gr.Markdown("## Heatmap")
-        heatmap_image_qwen = gr.Image(label="", show_label=False)
-        leader_board.select(fn=load_heatmap, outputs=[heatmap_image_qwen])
-
-    # with gr.Tab("Vision Benchmark", visible=False):
-    #     gr.Markdown("# Vision Benchmark Leaderboard")
-    #     leader_board_vision = gr.Dataframe(
-    #         vision_accuracy_df, headers=headers_with_icons
-    #     )
-    #     gr.Markdown("## Heatmap")
-    #     heatmap_image_vision = gr.Image(label="", show_label=False)
-    #     leader_board_vision.select(
-    #         fn=load_vision_heatmap, outputs=[heatmap_image_vision]
-    #     )
-
-    # with gr.Tab("Text-only Benchmark (CoT)", visible=False):
-    #     gr.Markdown("# Text-only Leaderboard (CoT)")
-    #     cot_leader_board_text = gr.Dataframe(
-    #         cot_text_accuracy_df, headers=headers_with_icons
-    #     )
-    #     gr.Markdown("## Heatmap")
-    #     cot_heatmap_image_text = gr.Image(label="", show_label=False)
-    #     cot_leader_board_text.select(
-    #         fn=load_cot_heatmap, outputs=[cot_heatmap_image_text]
-    #     )
-
-    # with gr.Tab("Constraint Text-only Results (CoT)", visible=False):
-    #     gr.Markdown("## Constraint Text-only Leaderboard by first substrin (CoT)")
-    #     included_models_cot = gr.CheckboxGroup(
-    #         label="Models to include",
-    #         choices=all_cot_text_only_models,
-    #         value=all_cot_text_only_models,
-    #         interactive=True,
-    #     )
-    #     with gr.Row():
-    #         number_of_queries_cot = gr.Textbox(label="Number of included queries")
-    #         number_of_fsms_cot = gr.Textbox(label="Number of included  FSMs")
-
-    #     constrained_leader_board_text_cot = gr.Dataframe()
-    #     constrained_leader_board_plot_cot = gr.Plot()
-
-    # with gr.Tab("Majority Vote (Subset 1)", visible=False):
-    #     gr.Markdown("## Majority Vote (Subset 1)")
-    #     intersection_leader_board = gr.Dataframe(
-    #         intersection_df_acc, headers=headers_with_icons
-    #     )
-    #     heatmap_image = gr.Plot(label="Model Heatmap")
-
-    # with gr.Tab("Text-only Benchmark (deprecated)", visible=False):
-    #     gr.Markdown("# Text-only Leaderboard")
-    #     leader_board = gr.Dataframe(accuracy_df, headers=headers_with_icons)
-    #     gr.Markdown("## Heatmap")
-    #     heatmap_image = gr.Image(label="", show_label=False)
-    #     leader_board.select(fn=load_heatmap, outputs=[heatmap_image])
-
-    # # ============ Callbacks ============
-
-    # included_models_cot.select(
-    #     fn=calculate_order_by_first_substring_cot,
-    #     inputs=[included_models_cot],
-    #     outputs=[
-    #         constrained_leader_board_text_cot,
-    #         number_of_queries_cot,
-    #         number_of_fsms_cot,
-    #     ],
-    #     queue=True,
-    # )
-
-    # constrained_leader_board_text.select(
-    #     fn=show_constraint_heatmap, outputs=[constrained_leader_board_plot]
-    # )
-
-    # constrained_leader_board_text_cot.select(
-    #     fn=show_constraint_heatmap_cot, outputs=[constrained_leader_board_plot_cot]
-    # )
-
-    # intersection_leader_board.select(
-    #     fn=show_intersection_heatmap, outputs=[heatmap_image]
-    # )
+        heatmap_image_1shot = gr.Image(label="", show_label=False)
+        leader_board_1shot.select(fn=load_heatmap_1shot, outputs=[heatmap_image_1shot])
 
     demo.launch()