init

app.py

@@ -0,0 +1,46 @@
+import streamlit as st  
+from parse import retrieve
+
+  
+def main():  
+    st.title("Streamlit App")  
+  
+    model_name = st.selectbox(  
+        "Select Model",  
+        options=["T5", "Vicuna", "UL2", "ChatGPT"],  
+        index=0,  
+    )  
+  
+    dataset_name = st.selectbox(  
+        "Select Dataset",  
+        options=[  
+            "SST-2", "CoLA", "QQP", "MRPC", "MNLI", "QNLI",  
+            "RTE", "WNLI", "MMLU", "SQuAD V2", "IWSLT 2017", "UN Multi", "Math"  
+        ],  
+        index=0,  
+    )
+
+    attack_name = st.selectbox(
+        "Select Attack",
+        options=[
+            "BertAttack", "CheckList", "DeepWordBug", "StressTest", "TextFooler", "TextBugger", "Semantic"
+        ],
+        index=0,
+    )
+
+    prompt_type = st.selectbox(  
+        "Select Prompt Type",  
+        options=["zeroshot-task", "zeroshot-role", "fewshot-task", "fewshot-role"],  
+        index=0,  
+    )  
+  
+    st.write(f"Model: {model_name}")  
+    st.write(f"Dataset: {dataset_name}")  
+    st.write(f"Prompt Type: {prompt_type}")  
+  
+    if st.button("Retrieve"):  
+        output = retrieve(model_name, dataset_name, attack_name, prompt_type)  
+        st.write(f"Output: {output}")  
+  
+if __name__ == "__main__":  
+    main()  

parse.py

@@ -0,0 +1,203 @@
+import numpy as np
+import re
+
+
+def split_markdown_by_title(markdown_file):
+    with open(markdown_file, 'r', encoding='utf-8') as f:
+        content = f.read()
+
+    re_str = "# cola|# mnli|# mrpc|# qnli|# qqp|# rte|# sst2|# wnli|# mmlu|# squad_v2|# iwslt|# un_multi|# math"
+    
+    datasets = ["# cola", "# mnli", "# mrpc", "# qnli", "# qqp", "# rte", "# sst2", "# wnli",
+                "# mmlu", "# squad_v2", "# iwslt", "# un_multi", "# math"]
+    
+    # re_str = "# cola|# mnli|# mrpc|# qnli|# qqp|# rte|# sst2|# wnli"
+    # datasets = ["# cola", "# mnli", "# mrpc", "# qnli", "# qqp", "# rte", "# sst2", "# wnli"]
+    primary_sections = re.split(re_str, content)[1:]
+    assert len(primary_sections) == len(datasets)
+
+    all_sections_dict = {}
+
+    for dataset, primary_section in zip(datasets, primary_sections):
+        re_str = "## "
+        results = re.split(re_str, primary_section)
+        keywords = ["10 prompts", "bertattack", "checklist", "deepwordbug", "stresstest",
+                    "textfooler", "textbugger", "translation"]
+
+        secondary_sections_dict = {}
+        for res in results:
+            for keyword in keywords:
+                if keyword in res.lower():
+                    secondary_sections_dict[keyword] = res
+                    break
+
+        all_sections_dict[dataset] = secondary_sections_dict
+
+    return all_sections_dict
+# def prompts_understanding(sections_dict):
+#     for dataset in sections_dict.keys():
+#         # print(dataset)
+#         for title in sections_dict[dataset].keys():
+#             if title == "10 prompts":
+#                 prompts = sections_dict[dataset][title].split("\n")
+#                 num = 0
+#                 task_prompts_acc = []
+#                 role_prompts_acc = []
+#                 for prompt in prompts:
+#                     if "Acc: " not in prompt:
+#                         continue
+#                     else:
+#                         import re
+#                         num += 1
+#                         match = re.search(r'Acc: (\d+\.\d+)%', prompt)
+#                         if match:
+#                             number = float(match.group(1))
+#                         if num <= 10:
+#                             task_prompts_acc.append(number)
+#                         else:
+#                             role_prompts_acc.append(number)
+
+#                 print(task_prompts_acc)
+#                 print(role_prompts_acc)
+import os
+def list_files(directory):
+    files = [os.path.join(directory, d) for d in os.listdir(directory) if not os.path.isdir(os.path.join(directory, d))]
+    return files
+
+def convert_model_name(attack):
+    attack_name = {
+        "T5": "t5",
+        "UL2": "ul2",
+        "Vicuna": "vicuna",
+        "ChatGPT": "chatgpt",
+    }
+    return attack_name[attack]
+
+def convert_attack_name(attack):
+    attack_name = {
+        "BertAttack": "bertattack",
+        "CheckList": "checklist",
+        "DeepWordBug": "deepwordbug",
+        "StressTest": "stresstest",
+        "TextFooler": "textfooler",
+        "TextBugger": "textbugger",
+        "Semantic": "translation",
+    }
+    return attack_name[attack]
+
+def convert_dataset_name(dataset):
+    dataset_name = {
+        "CoLA": "# cola",
+        "MNLI": "# mnli",
+        "MRPC": "# mrpc",
+        "QNLI": "# qnli",
+        "QQP": "# qqp",
+        "RTE": "# rte",
+        "SST-2": "# sst2",
+        "WNLI": "# wnli",
+        "MMLU": "# mmlu",
+        "SQuAD V2": "# squad_v2",
+        "IWSLT": "# iwslt",
+        "UN Multi": "# un_multi",
+        "Math": "# math",
+        "Avg": "Avg",
+    }
+    return dataset_name[dataset]
+
+
+def retrieve(model_name, dataset_name, attack_name, prompt_type):
+    model_name = convert_model_name(model_name)
+    dataset_name = convert_dataset_name(dataset_name)
+    attack_name = convert_attack_name(attack_name)
+
+    if "zero" in prompt_type:
+        shot = "zeroshot"
+    else:
+        shot = "fewshot"
+    
+    if "task" in prompt_type:
+        prompt_type = "task"
+    else:
+        prompt_type = "role"
+    
+    directory_path = "./db"
+    md_dir = os.path.join(directory_path, model_name + "_" + shot + ".md")
+    sections_dict = split_markdown_by_title(md_dir)
+
+    for cur_dataset in sections_dict.keys():
+        if cur_dataset == dataset_name:
+            dataset_dict = sections_dict[cur_dataset]
+            for cur_attack in dataset_dict.keys():
+                if cur_attack == attack_name:
+                    pass
+
+                    if attack_name == "translation":
+                        results = dataset_dict[attack_name].split("\n")
+
+                        atk_acc = []
+            
+                        for result in results:
+                            if "acc: " not in result:
+                                continue
+
+                            import re
+                        
+                            match_atk = re.search(r'acc: (\d+\.\d+)%', result)
+                            
+                            number_atk = float(match_atk.group(1))
+                            atk_acc.append(number_atk)
+
+                        sorted_atk_acc = sorted(atk_acc)[:6]
+
+                elif title in ["bertattack", "checklist", "deepwordbug", "stresstest", "textfooler", "textbugger"]:
+
+                    results = sections_dict[dataset][title].split("Original prompt: ")
+                    num = 0
+                    
+
+                    for result in results:
+                        if "Attacked prompt: " not in result:
+                            continue
+                        num += 1
+                        import re
+                        match_origin = re.search(r'Original acc: (\d+\.\d+)%', result)
+                        match_atk = re.search(r'attacked acc: (\d+\.\d+)%', result)
+                        if match_origin and match_atk:
+                            number_origin = float(match_origin.group(1))
+                            number_atk = float(match_atk.group(1))
+                            summary[title][dataset].append((number_origin - number_atk)/number_origin)
+                            summary[title]["Avg"].append((number_origin - number_atk)/number_origin)
+                    
+                    # print(model_shot, dataset, title, len(summary[attack][dataset]), num)
+    
+    # for atk in summary.keys():
+    #     for dataset in summary[atk].keys():
+    #         # if atk == "translation":
+    #         print(atk, dataset, len(summary[atk][dataset]))
+    #             # print(summary[atk][dataset][:10])
+    
+    output_dict = {}
+
+    sorted_atk_name = ["TextBugger",  "DeepWordBug", "TextFooler", "BertAttack", "CheckList", "StressTest", "Semantic"]
+    sorted_dataset_name = ["SST-2", "CoLA", "QQP", "MRPC", "MNLI", "QNLI", "RTE",  "WNLI", "MMLU", "SQuAD V2", "IWSLT", "UN Multi", "Math"]
+    
+    for atk in sorted_atk_name:
+        output_dict[atk] = {}
+        for dataset in sorted_dataset_name:
+            output_dict[atk][dataset] = ""
+    
+    for sorted_atk in sorted_atk_name:
+        for attack, dataset_drop_rates in summary.items():
+            # attack = convert_attack_name(attack)
+            if convert_attack_name(attack) == sorted_atk:
+                for sorted_dataset in sorted_dataset_name:
+                    for dataset, drop_rates in dataset_drop_rates.items():
+                        if convert_dataset_name(dataset) == sorted_dataset:
+                            if len(drop_rates) > 0:
+                                output_dict[sorted_atk][sorted_dataset] = "{:.2f}".format(sum(drop_rates)/len(drop_rates)) + "\scriptsize{$\pm$" + "{:.2f}".format(np.std(drop_rates)) + "}"
+                            else:
+                                output_dict[sorted_atk][sorted_dataset] = "-"
+
+                total_drop_rate = summary[attack]["Avg"]
+                output_dict[sorted_atk]["Avg"] = "{:.2f}".format(np.mean(total_drop_rate)) + "\scriptsize{$\pm$" + "{:.2f}".format(np.std(total_drop_rate)) + "}"
+