v0.2

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/precomputed_results.json filter=lfs diff=lfs merge=lfs -text
+precomputed_results.json filter=lfs diff=lfs merge=lfs -text

app.py

@@ -56,14 +56,15 @@ def change_language(lang, randomize_imgs, randomize_labels):
         text_features = None
     correct_text = gr.Text(f"Correct was: ''. Question 1/{len(babel_imagenet[lang][0])} ", label="Game")
     player_score_text = gr.Text(f"Your choice: (Score: 0) ", label="Player")
-    clip_score_text = gr.Text(f"CLIP chose: '' (Score: 0)", label="CLIP")
+    clip_score_text = gr.Text(f"mSigLIP chose: '' (Score: 0)", label="Opponent")
 
     return text_features, -1, class_order, correct_text, player_score_text, clip_score_text, 0, 0
 
-def select(idx, lang, choice, correct, model_choice, player_score, clip_score):
+def select(idx, lang, choice, correct, model_choice, player_score, clip_score, choices):
     # checks if answer choice is correct and updated scores
     correct_name, correct_value = correct
     model_choice_name, model_choice_value = model_choice
+    player_choice = choices[choice][0]
 
     player_correct = choice == correct_value
     model_correct = model_choice_value == correct_value
@@ -72,8 +73,8 @@ def select(idx, lang, choice, correct, model_choice, player_score, clip_score):
     clip_score = clip_score + int(model_correct)
 
     correct_text = gr.Text(f"Correct was: '{correct_name}'. Question {idx+1}/{len(babel_imagenet[lang][0])} ", label="Game")
-    player_score_text = gr.Text(f"Your choice: {'✅' if player_correct else '❌'} (Score: {player_score}) ", label="Player")
-    clip_score_text = gr.Text(f"CLIP chose: '{model_choice_name}' {'✅' if model_correct else '❌'} (Score: {clip_score})", label="CLIP")
+    player_score_text = gr.Text(f"Your choice: {player_choice} {'✅' if player_correct else '❌'} (Score: {player_score}) ", label="Player")
+    clip_score_text = gr.Text(f"mSigLIP chose: '{model_choice_name}' {'✅' if model_correct else '❌'} (Score: {clip_score})", label="Opponent")
 
     return correct_text, player_score_text, clip_score_text, player_score, clip_score
 
@@ -131,9 +132,57 @@ def prepare(raw_idx, lang, text_embeddings, class_order, randomize_images):
     next_radio = gr.Radio(choices=choice_values, interactive=True, label="Select the correct answer:", value=None)
     next_image = gr.Image(value=img_url, width=IMG_WIDTH, height=IMG_WIDTH, label="What class does this image belong to?")
 
-    return next_radio, next_image, raw_idx, correct_choice, model_choice
+    return next_radio, next_image, raw_idx, correct_choice, model_choice, choice_values
 
 
+def reroll(raw_idx, lang, text_embeddings, class_order, randomize_images):
+    #  prepared next question, loads image, and computes choices
+
+    idx = class_order[raw_idx]
+    lang_class_idxs = babel_imagenet[lang][0]
+    class_idx = lang_class_idxs[idx]
+
+    img_idx = 0
+    if randomize_images:
+        img_idx = np.random.choice(len(babelnet_images[class_idx]))
+    img_url = babelnet_images[class_idx][img_idx]["url"]
+    class_labels = babel_imagenet[lang][1] if lang != "EN" else openai_en_classes
+
+    if not precomputed_results:
+        try:
+            image_input = transform(Image.open(requests.get(img_url, stream=True, headers=request_header).raw).convert("RGB")).unsqueeze(0).to(device)
+            with torch.no_grad():
+                image_features = model.encode_image(image_input).float()
+                image_features /= image_features.norm(dim=-1, keepdim=True)
+        except:
+            gr.Warning("There is a problem with the next class. Skipping it.")
+            return prepare(raw_idx, lang, text_embeddings, class_order, randomize_images)
+
+        similarity = (text_embeddings @ image_features.cpu().numpy().T).squeeze()
+        choices = np.argsort(similarity)[-4:].tolist()
+    else:
+        choices = list(reversed(precomputed_results[lang][idx][img_idx]))  # precomputing script uses torch.topk which sorts in reverse here
+    if idx not in choices:
+        choices = [idx] + choices[1:]
+    model_choice_idx = choices[-1]
+
+    numpy.random.shuffle(choices)
+
+    choice_names = [class_labels[idx] for idx in choices]
+    choice_values = [0, 1, 2, 3]
+
+    model_choice_idx = choices.index(model_choice_idx)
+    model_choice = [choice_names[model_choice_idx], choice_values[model_choice_idx]]
+    correct_choice_idx = choices.index(idx)
+    correct_choice = [choice_names[correct_choice_idx], choice_values[correct_choice_idx]]
+
+    choice_values = list(zip(choice_names, choice_values))
+
+    next_radio = gr.Radio(choices=choice_values, interactive=True, label="Select the correct answer:", value=None)
+    next_image = gr.Image(value=img_url, width=IMG_WIDTH, height=IMG_WIDTH, label="What class does this image belong to?")
+
+    return next_radio, next_image, raw_idx, correct_choice, model_choice, choice_values
+
 
 with (gr.Blocks(title="Babel-ImageNet Quiz") as demo):
 
@@ -142,6 +191,7 @@ with (gr.Blocks(title="Babel-ImageNet Quiz") as demo):
     player_score = gr.State(0)
     clip_score = gr.State(0)
     class_order = gr.State([])
+    choices = gr.State([])
 
     text_embeddings = gr.State(None)
     correct_choice = gr.State(["nan", 0]) # 0, 1, 2, 3
@@ -153,7 +203,7 @@ with (gr.Blocks(title="Babel-ImageNet Quiz") as demo):
 
 <small>by Gregor Geigle, WüNLP & Computer Vision Lab, University of Würzburg</small>
 
-In this quiz, you play against a CLIP model and try to correctly classify the images over the 1000 ImageNet classes (in English) or over our (partial) Babel-ImageNet translations of those classes.   
+In this quiz, you play against a CLIP model (specifically: [mSigLIP](https://huggingface.co/timm/ViT-B-16-SigLIP-i18n-256), a multilingual [SigLIP](https://arxiv.org/abs/2303.15343) model) and try to correctly classify the images over the 1000 ImageNet classes (in English) or over our (partial) Babel-ImageNet translations of those classes.   
 Select your language, click 'Start' and start guessing! We'll keep track of your score and of your opponent's.   
 > **Disclaimer:** Translations and images are derived automatically and can be wrong, unusual, or mismatch! This is supposed to be a fun game to explore the dataset and see how a CLIP model would answer the questions and not a product.   
 > We do *not* use the official ImageNet images. Instead, we use images linked in BabelNet for each class, which are often from Wikipedia and have not been checked for suitability.   
@@ -164,7 +214,7 @@ Select your language, click 'Start' and start guessing! We'll keep track of your
 <summary> <b> FAQ</b> (click me to read)</summary>
 <p><b>'Over 1000 classes? I just see 4.'</b> True, you have it easier and you only have to chose between 4 classes. These are the top-4 picks of your opponent (+ the correct class if they are wrong). Your opponent has it harder: they have to deal with all classes.</p>   
 <p><b>'Who is my opponent?'</b> Your opponent CLIP model is [mSigLIP](https://huggingface.co/timm/ViT-B-16-SigLIP-i18n-256), a powerful but small multilingual model with only 370M parameters.</p>      
-<p><b>'My game crashed/ I got an error!'</b> This usually happens because of problems with the image URLs. Unfortunately, there is nothing you can do except start a new round by clicking the 'Start' button.</p>      
+<p><b>'My game crashed/ I got an error!'</b> This usually happens because of problems with the image URLs. You can try the button to reroll the image or start a new round by clicking the 'Start' button again.</p>      
 </details>
         """)
     with gr.Row():
@@ -177,7 +227,7 @@ Select your language, click 'Start' and start guessing! We'll keep track of your
             With CLIP, you can search through, filter, or group large image datasets. The image encoder in CLIP also powers many of the large vision language models like Llava 1.5!
             </p>   
             <p>
-            Your opponent CLIP model in this quiz does 'zero-shot image classification': We encode all possible class labels and the image and we check which class is most similar; this is then the class chosen by CLIP.
+            Your opponent CLIP model [mSigLIP](https://arxiv.org/abs/2303.15343) in this quiz does 'zero-shot image classification': We encode all possible class labels and the image and we check which class is most similar; this is then the class chosen by CLIP.
             </p> 
             </details>
                     """)
@@ -209,8 +259,8 @@ Select your language, click 'Start' and start guessing! We'll keep track of your
     # language select dropdown
     with gr.Row():
         language_select = gr.Dropdown(choices=main_language_values, value="EN", interactive=True, label="Select your language:")
-        randomize_classes = gr.Checkbox(label="Randomize class order (or play in canonic order)")
-        randomize_images = gr.Checkbox(label="Randomize images (if unchecked, will always show the same image). Other images might be less relevant.")
+        randomize_classes = gr.Checkbox(label="Randomize class order (or play in canonic order)", value=True)
+        randomize_images = gr.Checkbox(label="Randomize images (if unchecked, will always show the same image). Other images might be less relevant.", value=True)
         start_btn = gr.Button(value="Start", variant="primary")
 
     # quiz area
@@ -224,26 +274,29 @@ Select your language, click 'Start' and start guessing! We'll keep track of your
             # with gr.Row():
             correct_text = gr.Text("Please click start to begin.")
             player_score_text = gr.Text(f"Player score: 0")
-            clip_score_text = gr.Text(f"CLIP score: 0")
-
+            clip_score_text = gr.Text(f"mSigLIP score: 0")
+            reroll_btn = gr.Button(value="Reroll the image (for bad images or errors)")
 
-    # Explanation area
 
 
 
     options.select(fn=select,
-                   inputs=[class_idx, language_select, options, correct_choice, model_choice, player_score, clip_score],
+                   inputs=[class_idx, language_select, options, correct_choice, model_choice, player_score, clip_score, choices],
                    outputs=[correct_text, player_score_text, clip_score_text, player_score, clip_score]
                    ).then(fn=prepare,
                           inputs=[class_idx, language_select, text_embeddings, class_order, randomize_images],
-                          outputs=[options, image, class_idx, correct_choice, model_choice])
+                          outputs=[options, image, class_idx, correct_choice, model_choice, choices])
 
     start_btn.click(fn=change_language,
                           inputs=[language_select, randomize_images, randomize_classes],
                           outputs=[text_embeddings, class_idx, class_order, correct_text, player_score_text, clip_score_text, player_score, clip_score]
                           ).then(fn=prepare,
                                  inputs=[class_idx, language_select, text_embeddings, class_order, randomize_images],
-                                 outputs=[options, image, class_idx, correct_choice, model_choice])
+                                 outputs=[options, image, class_idx, correct_choice, model_choice, choices])
+
+    reroll_btn.click(fn=reroll,
+                     inputs=[class_idx, language_select, text_embeddings, class_order, randomize_images],
+                     outputs=[options, image, class_idx, correct_choice, model_choice, choices])
 
     # initialization
     # demo.load(fn=change_language,

language_mapping.json

@@ -0,0 +1,300 @@
+{
+  "AB": "Abkhazian",
+  "ACW": "Hijazi Arabic",
+  "ADY": "Adyghe",
+  "AF": "Afrikaans",
+  "AFB": "Gulf Arabic",
+  "AII": "Assyrian Neo-Aramaic",
+  "AK": "Akan",
+  "ALS": "Alemannic",
+  "ALT": "Southern Altai",
+  "AM": "Amharic",
+  "AN": "Aragonese",
+  "ANG": "Anglo-Saxon",
+  "AR": "Arabic",
+  "ARC": "Aramaic",
+  "ARY": "Moroccan Arabic",
+  "ARZ": "Egyptian Arabic",
+  "AS": "Assamese",
+  "AST": "Asturian",
+  "AV": "Avar",
+  "AY": "Aymara",
+  "AZ": "Azerbaijani",
+  "AZB": "South Azerbaijani",
+  "BA": "Bashkir",
+  "BAN": "Balinese",
+  "BAR": "Bavarian",
+  "BAT_SMG": "Samogitian",
+  "BCL": "Central Bicolano",
+  "BE": "Belarusian",
+  "BE_X_OLD": "Belarusian (Tara\u0161kievica)",
+  "BG": "Bulgarian",
+  "BH": "Bihari",
+  "BHO": "Bhojpuri",
+  "BJN": "Banjar",
+  "BM": "Bambara",
+  "BN": "Bengali",
+  "BO": "Tibetan",
+  "BR": "Breton",
+  "BS": "Bosnian",
+  "BXR": "Buryat (Russia)",
+  "CA": "Catalan",
+  "CCC": "Chamicuro",
+  "CDO": "Min Dong",
+  "CE": "Chechen",
+  "CEB": "Cebuano",
+  "CHR": "Cherokee",
+  "CHY": "Cheyenne",
+  "CKB": "Sorani",
+  "CO": "Corsican",
+  "CR": "Cree",
+  "CRH": "Crimean Tatar",
+  "CS": "Czech",
+  "CSB": "Kashubian",
+  "CU": "Old Church Slavonic",
+  "CV": "Chuvash",
+  "CY": "Welsh",
+  "DA": "Danish",
+  "DE": "German",
+  "DIN": "Dinka",
+  "DIQ": "Zazaki",
+  "DLM": "Dalmatian",
+  "DNG": "Dungan",
+  "DSB": "Lower Sorbian",
+  "DV": "Divehi",
+  "DZ": "Dzongkha",
+  "EE": "Ewe",
+  "EL": "Greek",
+  "EML": "Emilian-Romagnol",
+  "EN": "English",
+  "ENM": "Middle English",
+  "EO": "Esperanto",
+  "ES": "Spanish",
+  "ET": "Estonian",
+  "EU": "Basque",
+  "EXT": "Extremaduran",
+  "FA": "Persian",
+  "FI": "Finnish",
+  "FIU_VRO": "V\u00f5ro",
+  "FJ": "Fijian",
+  "FO": "Faroese",
+  "FR": "French",
+  "FRO": "Old French",
+  "FRP": "Franco-Proven\u00e7al/Arpitan",
+  "FRR": "North Frisian",
+  "FUR": "Friulian",
+  "FY": "West Frisian",
+  "GA": "Irish",
+  "GAN": "Gan",
+  "GD": "Scottish Gaelic",
+  "GL": "Galician",
+  "GMQ_BOT": "Westrobothnian",
+  "GN": "Guarani",
+  "GOT": "Gothic",
+  "GRC": "Ancient Greek",
+  "GU": "Gujarati",
+  "GV": "Manx",
+  "HA": "Hausa",
+  "HAK": "Hakka",
+  "HAW": "Hawaiian",
+  "HE": "Hebrew",
+  "HI": "Hindi",
+  "HR": "Croatian",
+  "HRX": "Hunsrik",
+  "HSB": "Upper Sorbian",
+  "HT": "Haitian",
+  "HU": "Hungarian",
+  "HY": "Armenian",
+  "IA": "Interlingua",
+  "ID": "Indonesian",
+  "IG": "Igbo",
+  "IK": "Inupiak",
+  "ILO": "Ilokano",
+  "INH": "Ingush",
+  "IO": "Ido",
+  "IS": "Icelandic",
+  "IT": "Italian",
+  "IU": "Inuktitut",
+  "JA": "Japanese",
+  "JAM": "Patois",
+  "JBO": "Lojban",
+  "JV": "Javanese",
+  "KA": "Georgian",
+  "KAA": "Karakalpak",
+  "KAB": "Kabyle",
+  "KBD": "Kabardian Circassian",
+  "KG": "Kongo",
+  "KHB": "L\u00fc",
+  "KI": "Kikuyu",
+  "KK": "Kazakh",
+  "KL": "Greenlandic",
+  "KM": "Khmer",
+  "KMR": "Northern Kurdish",
+  "KN": "Kannada",
+  "KO": "Korean",
+  "KOI": "Komi-Permyak",
+  "KRC": "Karachay-Balkar",
+  "KS": "Kashmiri",
+  "KSH": "Ripuarian",
+  "KU": "Kurdish",
+  "KUM": "Kumyk",
+  "KV": "Komi",
+  "KW": "Cornish",
+  "KXD": "Brunei Malay",
+  "KY": "Kirghiz",
+  "LA": "Latin",
+  "LAD": "Ladino",
+  "LB": "Luxembourgish",
+  "LBE": "Lak",
+  "LEZ": "Lezgian",
+  "LG": "Luganda",
+  "LI": "Limburgish",
+  "LIJ": "Ligurian",
+  "LMO": "Lombard",
+  "LN": "Lingala",
+  "LO": "Lao",
+  "LT": "Lithuanian",
+  "LTG": "Latgalian",
+  "LV": "Latvian",
+  "LZZ": "Laz",
+  "MAI": "Maithili",
+  "MDF": "Moksha",
+  "MEL": "Central Melanau",
+  "MG": "Malagasy",
+  "MHR": "Meadow Mari",
+  "MI": "Maori",
+  "MIC": "Mi'kmaq",
+  "MIN": "Minangkabau",
+  "MK": "Macedonian",
+  "ML": "Malayalam",
+  "MN": "Mongolian",
+  "MNC": "Manchu",
+  "MNW": "Mon",
+  "MR": "Marathi",
+  "MRJ": "Hill Mari",
+  "MS": "Malay",
+  "MT": "Maltese",
+  "MWL": "Mirandese",
+  "MY": "Burmese",
+  "MYV": "Erzya",
+  "MZN": "Mazandarani",
+  "NAH": "Nahuatl",
+  "NAP": "Neapolitan",
+  "NCI": "Classical Nahuatl",
+  "NDS": "Low Saxon",
+  "NDS_NL": "Dutch Low Saxon",
+  "NE": "Nepali",
+  "NEW": "Newar / Nepal Bhasa",
+  "NL": "Dutch",
+  "NN": "Norwegian (Nynorsk)",
+  "NO": "Norwegian (Bokm\u00e5l)",
+  "NOG": "Nogai",
+  "NON": "Old Norse",
+  "NOV": "Novial",
+  "NRM": "Norman",
+  "NSO": "Northern Sotho",
+  "NV": "Navajo",
+  "NY": "Chichewa",
+  "OC": "Occitan",
+  "OJ": "Ojibwe",
+  "OLO": "Livvinkarjala",
+  "OM": "Oromo",
+  "OR": "Oriya",
+  "ORV": "Old East Slavic",
+  "OS": "Ossetian",
+  "OTA": "Ottoman Turkish",
+  "OVD": "Elfdalian",
+  "PA": "Punjabi",
+  "PAM": "Kapampangan",
+  "PAP": "Papiamentu",
+  "PCD": "Picard",
+  "PDC": "Pennsylvania German",
+  "PDT": "Plautdietsch",
+  "PL": "Polish",
+  "PMS": "Piedmontese",
+  "PNB": "Western Panjabi",
+  "PS": "Pashto",
+  "PT": "Portuguese",
+  "QU": "Quechua",
+  "RM": "Romansh",
+  "RMY": "Romani",
+  "RO": "Romanian",
+  "ROA_RUP": "Aromanian",
+  "RU": "Russian",
+  "RUE": "Rusyn",
+  "SA": "Sanskrit",
+  "SAH": "Sakha",
+  "SAT": "Santali",
+  "SC": "Sardinian",
+  "SCN": "Sicilian",
+  "SCO": "Scots",
+  "SD": "Sindhi",
+  "SE": "Northern Sami",
+  "SH": "Serbo-Croatian",
+  "SHI": "Tashelhit",
+  "SHN": "Shan",
+  "SI": "Sinhalese",
+  "SIMPLE": "Simple English",
+  "SK": "Slovak",
+  "SL": "Slovenian",
+  "SM": "Samoan",
+  "SMJ": "Lule Sami",
+  "SMN": "Inari Sami",
+  "SMS": "Skolt Sami",
+  "SN": "Shona",
+  "SNE": "Bau Bidayuh",
+  "SO": "Somali",
+  "SQ": "Albanian",
+  "SR": "Serbian",
+  "SRN": "Sranan",
+  "ST": "Sesotho",
+  "STQ": "Saterland Frisian",
+  "SU": "Sundanese",
+  "SV": "Swedish",
+  "SVA": "Svan",
+  "SW": "Swahili",
+  "SYC": "Classical Syriac",
+  "SZL": "Silesian",
+  "TA": "Tamil",
+  "TCY": "Tulu",
+  "TE": "Telugu",
+  "TG": "Tajik",
+  "TH": "Thai",
+  "TI": "Tigrinya",
+  "TK": "Turkmen",
+  "TL": "Tagalog",
+  "TN": "Tswana",
+  "TO": "Tongan",
+  "TPI": "Tok Pisin",
+  "TR": "Turkish",
+  "TT": "Tatar",
+  "TYV": "Tuvan",
+  "UDM": "Udmurt",
+  "UG": "Uyghur",
+  "UGA": "Ugaritic",
+  "UK": "Ukrainian",
+  "UR": "Urdu",
+  "UZ": "Uzbek",
+  "VEC": "Venetian",
+  "VEP": "Vepsian",
+  "VI": "Vietnamese",
+  "VLS": "West Flemish",
+  "VO": "Volap\u00fck",
+  "WA": "Walloon",
+  "WAR": "Waray-Waray",
+  "WO": "Wolof",
+  "WUU": "Wu",
+  "XAL": "Kalmyk",
+  "XH": "Xhosa",
+  "XMF": "Mingrelian",
+  "YI": "Yiddish",
+  "YO": "Yoruba",
+  "YUA": "Yucatec Maya",
+  "ZA": "Zhuang",
+  "ZDJ": "Ngazidja Comorian",
+  "ZH": "Chinese",
+  "ZH_CLASSICAL": "Classical Chinese",
+  "ZH_MIN_NAN": "Min Nan",
+  "ZH_YUE": "Cantonese",
+  "ZU": "Zulu"
+}

precomputed_results.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c04522ddd7d79055f5d26c4e60a809f7725e67032dd46bbd0b8303cdc039e882
+size 26844572