update

.gitattributes

@@ -115,3 +115,43 @@ results-cot/gpt-4v-CoT-Azure.pkl filter=lfs diff=lfs merge=lfs -text
 results-cot/gpt-4v-CoT-Azure.csv filter=lfs diff=lfs merge=lfs -text
 results-vision-CoT/gemini-pro-vision-CoT.csv filter=lfs diff=lfs merge=lfs -text
 results-cot/gpt-3.5-CoT.csv filter=lfs diff=lfs merge=lfs -text
+results/claude-3-haiku-20240307.csv filter=lfs diff=lfs merge=lfs -text
+results/claude-3-opus-20240229.csv filter=lfs diff=lfs merge=lfs -text
+results-cot/gemma-7b-it.csv filter=lfs diff=lfs merge=lfs -text
+results-cot/gpt-3.5-turbo-0125.csv filter=lfs diff=lfs merge=lfs -text
+results/gpt-3.5-turbo-0125.csv filter=lfs diff=lfs merge=lfs -text
+results-cot/Mixtral-8x7B-Instruct-v0.1.csv filter=lfs diff=lfs merge=lfs -text
+results/gemma-7b-it.csv filter=lfs diff=lfs merge=lfs -text
+results-cot/Qwen1.5-72B-Chat.csv filter=lfs diff=lfs merge=lfs -text
+results/CodeLlama-70b-Instruct-hf.csv filter=lfs diff=lfs merge=lfs -text
+results/Mixtral-8x7B-Instruct-v0.1.csv filter=lfs diff=lfs merge=lfs -text
+results-cot/gemma-7b-it.pkl filter=lfs diff=lfs merge=lfs -text
+results/claude-3-haiku-20240307.pkl filter=lfs diff=lfs merge=lfs -text
+results/gemma-7b-it.pkl filter=lfs diff=lfs merge=lfs -text
+results-cot/gpt-3.5-turbo-0125.pkl filter=lfs diff=lfs merge=lfs -text
+results-cot/Mixtral-8x7B-Instruct-v0.1.pkl filter=lfs diff=lfs merge=lfs -text
+results/Mixtral-8x7B-Instruct-v0.1.pkl filter=lfs diff=lfs merge=lfs -text
+results/claude-3-opus-20240229.pkl filter=lfs diff=lfs merge=lfs -text
+results-cot/Qwen1.5-72B-Chat.pkl filter=lfs diff=lfs merge=lfs -text
+results/CodeLlama-70b-Instruct-hf.pkl filter=lfs diff=lfs merge=lfs -text
+results/gpt-3.5-turbo-0125.pkl filter=lfs diff=lfs merge=lfs -text
+results/claude-3-haiku-20240307.jpg filter=lfs diff=lfs merge=lfs -text
+results/claude-3-opus-20240229.jpg filter=lfs diff=lfs merge=lfs -text
+results/gpt-3.5-turbo-0125.jpg filter=lfs diff=lfs merge=lfs -text
+results-cot/gpt-3.5-turbo-0125.jpg filter=lfs diff=lfs merge=lfs -text
+results/Mixtral-8x7B-Instruct-v0.1.jpg filter=lfs diff=lfs merge=lfs -text
+results-cot/Qwen1.5-72B-Chat.jpg filter=lfs diff=lfs merge=lfs -text
+results/gemma-7b-it.jpg filter=lfs diff=lfs merge=lfs -text
+results-cot/Mixtral-8x7B-Instruct-v0.1.jpg filter=lfs diff=lfs merge=lfs -text
+results-cot/gemma-7b-it.jpg filter=lfs diff=lfs merge=lfs -text
+results/CodeLlama-70b-Instruct-hf.jpg filter=lfs diff=lfs merge=lfs -text
+results-cot/gemma-7b-it.png filter=lfs diff=lfs merge=lfs -text
+results-cot/gpt-3.5-turbo-0125.png filter=lfs diff=lfs merge=lfs -text
+results/gpt-3.5-turbo-0125.png filter=lfs diff=lfs merge=lfs -text
+results/CodeLlama-70b-Instruct-hf.png filter=lfs diff=lfs merge=lfs -text
+results/Mixtral-8x7B-Instruct-v0.1.png filter=lfs diff=lfs merge=lfs -text
+results/claude-3-opus-20240229.png filter=lfs diff=lfs merge=lfs -text
+results-cot/Mixtral-8x7B-Instruct-v0.1.png filter=lfs diff=lfs merge=lfs -text
+results-cot/Qwen1.5-72B-Chat.png filter=lfs diff=lfs merge=lfs -text
+results/claude-3-haiku-20240307.png filter=lfs diff=lfs merge=lfs -text
+results/gemma-7b-it.png filter=lfs diff=lfs merge=lfs -text

app.py

@@ -10,7 +10,7 @@ vision_results = glob("results-vision/*.pkl")
 # Load CoT text benchmark results
 cot_text_results = glob("results-cot/*.pkl")
 # Load CoT vision benchmark results
-cot_vision_results = glob("results-vision-CoT/*.pkl")
+# cot_vision_results = glob("results-vision-CoT/*.pkl")
 
 # Function to load data, add model type and name
 def load_data(files, model_type):
@@ -27,18 +27,22 @@ def load_data(files, model_type):
 data = load_data(csv_results, "Text Only")
 vision_data = load_data(vision_results, "Vision")
 cot_text_data = load_data(cot_text_results, "CoT Text Only")
-cot_vision_data = load_data(cot_vision_results, "CoT Vision")
+# cot_vision_data = load_data(cot_vision_results, "CoT Vision")
 
 # Combine all data into a single DataFrame
 all_data = pd.concat(
-    [data, vision_data, cot_text_data, cot_vision_data], ignore_index=True
+    [data, vision_data, cot_text_data], ignore_index=True
 )
 
 all_model_names = all_data["Model Name"].unique()
 all_text_only_model_names = list(
     all_data[all_data["Model Type"] == "Text Only"]["Model Name"].unique()
 )
-print(all_text_only_model_names)
+all_cot_text_only_models = list(
+    all_data[all_data["Model Type"] == "CoT Text Only"]["Model Name"].unique()
+)
+
+
 
 ## Continue with the cold code --
 # TODO: Update me to read from all_data for later
@@ -50,7 +54,7 @@ vision_data = {file: pd.read_pickle(file) for file in vision_results}
 # Load the CoT text files into a dict
 cot_text_data = {file: pd.read_pickle(file) for file in cot_text_results}
 # Load the CoT vision files into a dict
-cot_vision_data = {file: pd.read_pickle(file) for file in cot_vision_results}
+# cot_vision_data = {file: pd.read_pickle(file) for file in cot_vision_results}
 
 
 def calculate_accuracy(df):
@@ -96,13 +100,13 @@ def process_data(data):
 text_data_for_df = process_data(data)
 vision_data_for_df = process_data(vision_data)
 cot_text_data_for_df = process_data(cot_text_data)
-cot_vision_data_for_df = process_data(cot_vision_data)
+# cot_vision_data_for_df = process_data(cot_vision_data)
 
 # Create DataFrames
 accuracy_df = pd.DataFrame(text_data_for_df, columns=column_names)
 vision_accuracy_df = pd.DataFrame(vision_data_for_df, columns=column_names)
 cot_text_accuracy_df = pd.DataFrame(cot_text_data_for_df, columns=column_names)
-cot_vision_accuracy_df = pd.DataFrame(cot_vision_data_for_df, columns=column_names)
+# cot_vision_accuracy_df = pd.DataFrame(cot_vision_data_for_df, columns=column_names)
 
 # Function to finalize DataFrame
 def finalize_df(df):
@@ -117,7 +121,7 @@ def finalize_df(df):
 accuracy_df = finalize_df(accuracy_df)
 vision_accuracy_df = finalize_df(vision_accuracy_df)
 cot_text_accuracy_df = finalize_df(cot_text_accuracy_df)
-cot_vision_accuracy_df = finalize_df(cot_vision_accuracy_df)
+# cot_vision_accuracy_df = finalize_df(cot_vision_accuracy_df)
 
 
 def load_heatmap(evt: gr.SelectData):
@@ -176,6 +180,43 @@ def calculate_order_by_first_substring(selected_models):
     return text_only_filtered, number_of_queries, number_of_fsms
 
 
+
+def calculate_order_by_first_substring_cot(selected_models):
+
+    first_columns = all_data[all_data["substring_index"] == 1]
+    query_ids_df = first_columns[first_columns["Model Type"] == "CoT Text Only"]
+    query_ids_df = query_ids_df[query_ids_df["Model Name"].isin(selected_models)]
+
+    query_ids_df = query_ids_df.groupby("query_id").filter(
+        lambda x: x["parsed_judge_response"].eq(1).all()
+    )
+
+    fsm_ids = query_ids_df.fsm_id.unique()
+
+    text_only = all_data[all_data["Model Type"] == "CoT Text Only"]
+    text_only_filtered = text_only[text_only["fsm_id"].isin(fsm_ids)]
+
+    query_ids = text_only_filtered.query_id.unique()
+    text_only_filtered = (
+        text_only_filtered.groupby(["Model Name"])["parsed_judge_response"]
+        .mean()
+        .reset_index()
+    )
+
+    text_only_filtered["Accuracy"] = text_only_filtered["parsed_judge_response"] * 100
+    text_only_filtered.drop("parsed_judge_response", axis=1, inplace=True)
+
+    text_only_filtered["Accuracy"] = text_only_filtered["Accuracy"].apply(
+        lambda x: round(x, 2)
+    )
+    text_only_filtered.sort_values("Accuracy", ascending=False, inplace=True)
+
+    number_of_queries = len(query_ids)
+    number_of_fsms = len(fsm_ids)
+
+    return text_only_filtered, number_of_queries, number_of_fsms
+
+
 with gr.Blocks() as demo:
     gr.Markdown("# FSM Benchmark Leaderboard")
     with gr.Tab("Text-only Benchmark"):
@@ -196,8 +237,8 @@ with gr.Blocks() as demo:
             fn=load_vision_heatmap, outputs=[heatmap_image_vision]
         )
 
-    with gr.Tab("CoT Text-only Benchmark"):
-        gr.Markdown("# CoT Text-only Leaderboard")
+    with gr.Tab("Text-only Benchmark (CoT)"):
+        gr.Markdown("# Text-only Leaderboard (CoT)")
         cot_leader_board_text = gr.Dataframe(
             cot_text_accuracy_df, headers=headers_with_icons
         )
@@ -207,16 +248,16 @@ with gr.Blocks() as demo:
             fn=load_cot_heatmap, outputs=[cot_heatmap_image_text]
         )
 
-    with gr.Tab("CoT Vision Benchmark"):
-        gr.Markdown("# CoT Vision Benchmark Leaderboard")
-        cot_leader_board_vision = gr.Dataframe(
-            cot_vision_accuracy_df, headers=headers_with_icons
-        )
-        gr.Markdown("## Heatmap")
-        cot_heatmap_image_vision = gr.Image(label="", show_label=False)
-        cot_leader_board_vision.select(
-            fn=load_cot_vision_heatmap, outputs=[cot_heatmap_image_vision]
-        )
+    # with gr.Tab("Vision Benchmark (CoT)"):
+    #     gr.Markdown("# Vision Benchmark Leaderboard (CoT)")
+    #     cot_leader_board_vision = gr.Dataframe(
+    #         cot_vision_accuracy_df, headers=headers_with_icons
+    #     )
+    #     gr.Markdown("## Heatmap")
+    #     cot_heatmap_image_vision = gr.Image(label="", show_label=False)
+    #     cot_leader_board_vision.select(
+    #         fn=load_cot_vision_heatmap, outputs=[cot_heatmap_image_vision]
+    #     )
 
     with gr.Tab("Constraint Text-only Results"):
         gr.Markdown("## Constraint Text-only Leaderboard by first substring")
@@ -240,4 +281,26 @@ with gr.Blocks() as demo:
         queue=True,
     )
 
+
+    with gr.Tab("Constraint Text-only Results (CoT)"):
+        gr.Markdown("## Constraint Text-only Leaderboard by first substrin (CoT)")
+        included_models_cot = gr.CheckboxGroup(
+            label="Models to include",
+            choices=all_cot_text_only_models,
+            value=all_cot_text_only_models,
+            interactive=True,
+        )
+        with gr.Row():
+            number_of_queries_cot = gr.Textbox(label="Number of included queries")
+            number_of_fsms_cot = gr.Textbox(label="Number of included  FSMs")
+
+        constrained_leader_board_text_cot = gr.Dataframe()
+
+    included_models_cot.select(
+        fn=calculate_order_by_first_substring_cot,
+        inputs=[included_models_cot],
+        outputs=[constrained_leader_board_text_cot, number_of_queries_cot, number_of_fsms_cot],
+        queue=True,
+    )
+
     demo.launch()

results-cot/Mixtral-8x7B-Instruct-v0.1.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:093e919d90609c3be8d6818cf56ca018214da3a42b78aeaf85f92581b72c5ad4
+size 19494123

results-cot/Mixtral-8x7B-Instruct-v0.1.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:686692584c6ba027c454d699bbf585b95e5c99bfc426810ea74b327a975b9cf3
+size 19489822

results-cot/Qwen1.5-72B-Chat.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32681449776facf1084405001e69ed7926b79c69f9717fb159e3eb064b333636
+size 15795431

results-cot/Qwen1.5-72B-Chat.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c20383298d4b6482ca7c30bf91822e24099dc67b71a3be10271005e25208c40
+size 15778970

results-cot/gemma-7b-it.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8535fa3f2ef5a94b1b552859930e0476ca0f3c77ec4c277893a9ab9ef45d6c3
+size 16793758

results-cot/gemma-7b-it.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c581027f8b78df5934117276cec3e53613f5ac953d045f71af4121b3ec2e1a4
+size 16822239

results-vision-CoT/gemini-pro-vision-CoT.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1ebebe1d6caee19a4f714bf13eaba72e7a0b5d15281c407cd4dc53a2820ad312
-size 6184119

results-vision-CoT/gemini-pro-vision-CoT.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:049d575dbad9da04496fea752e19f915bcec445b13f3010f9c67544012c936ff
-size 6144275

results-vision/gemini-pro-vision-CoT.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1ebebe1d6caee19a4f714bf13eaba72e7a0b5d15281c407cd4dc53a2820ad312
-size 6184119

results-vision/gemini-pro-vision-CoT.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:049d575dbad9da04496fea752e19f915bcec445b13f3010f9c67544012c936ff
-size 6144275

results-vision/gpt-4v-CoT.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:04b4de1a7a4280354c89609d15282109ee60f8f58129960dc0edbb046b12a5c6
-size 6374181

results-vision/gpt-4v-CoT.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:52ae5e417e011db84976acd51a024eae7ccea1e686b7f3f0e8158cd77be4f847
-size 6320889

results/CodeLlama-70b-Instruct-hf.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3726905a1656174f3c29edfced6f2eec63222f6be8965c0d970264901d8cfc75
+size 16476347

results/Llama-2-70b-chat-hf.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42a31de917b05ed5405474a348d072426474a8fb2ce7ff462dbb121e25f4b6ad
+size 20760268

results/Mistral-7B-Instruct-v0.2.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29ad4985661fc41e659a631fc74ba433cd08a571048f11436ccf87ff74f0db09
+size 27242025

results/Mixtral-8x7B-Instruct-v0.1.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a93e2b963a5ac8129b5284f3fd7987964ef96fa0e64194de704a3549c611de1f
+size 17978176

results/Qwen1.5-72B-Chat.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ba395c0b55330f689827527831e57e50ae9d824b6635b2bb569713afcf26d4b
+size 14219193

results/StripedHyena-Nous-7B.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f662367ea0d33a368aaa7a72cfeed41d2f3dc05be6289a6fe485a028c7cb98d5
+size 29219512

results/Yi-34B-Chat.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7f09fb5f46ca144490bcb42ec89dd27f169680493501c211bf2bcfcd908da1c
+size 20485423

results/claude-3-haiku-20240307.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45623535997485afdee5b0312f2b5fdcc26cf531fbb56b6c3af6e126dfbe7b0f
+size 19570166

results/claude-3-opus-20240229.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d902999bcee4798b81644b2ff0ea78280dd46bc310909154c1ef089adf82789
+size 20131397