Init commit

.gitattributes

@@ -0,0 +1,38 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+embeddings-vit-base-patch32.npy filter=lfs diff=lfs merge=lfs -text
+embeddings-vit-large-patch14-336.npy filter=lfs diff=lfs merge=lfs -text
+embeddings-vit-large-patch14.npy filter=lfs diff=lfs merge=lfs -text
+embeddings2-vit-base-patch32.npy filter=lfs diff=lfs merge=lfs -text
+embeddings2-vit-large-patch14-336.npy filter=lfs diff=lfs merge=lfs -text
+embeddings2-vit-large-patch14.npy filter=lfs diff=lfs merge=lfs -text
+embeddings-vit-base-patch16.npy filter=lfs diff=lfs merge=lfs -text
+embeddings2-flava-full.npy filter=lfs diff=lfs merge=lfs -text
+embeddings2-vit-base-patch16.npy filter=lfs diff=lfs merge=lfs -text
+embeddings-flava-full.npy filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+.vscode/

README.md

@@ -0,0 +1,38 @@
+---
+title: FLAVA Semantic Image Text Search Demo
+emoji: 👁
+colorFrom: indigo
+colorTo: blue
+sdk: streamlit
+sdk_version: 1.2.0
+app_file: app.py
+pinned: false
+---
+
+# Configuration
+
+`title`: _string_  
+Display title for the Space
+
+`emoji`: _string_  
+Space emoji (emoji-only character allowed)
+
+`colorFrom`: _string_  
+Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
+
+`colorTo`: _string_  
+Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
+
+`sdk`: _string_  
+Can be either `gradio` or `streamlit`
+
+`sdk_version` : _string_  
+Only applicable for `streamlit` SDK.  
+See [doc](https://hf.co/docs/hub/spaces) for more info on supported versions.
+
+`app_file`: _string_  
+Path to your main application file (which contains either `gradio` or `streamlit` Python code).  
+Path is relative to the root of the repository.
+
+`pinned`: _boolean_  
+Whether the Space stays on top of your list.

app.py

@@ -0,0 +1,254 @@
+from html import escape
+import re
+import streamlit as st
+import pandas as pd, numpy as np
+from transformers import CLIPProcessor, CLIPModel, FlavaModel, FlavaProcessor
+from st_clickable_images import clickable_images
+
+MODEL_NAMES = ["flava-full", "vit-base-patch32", "vit-base-patch16", "vit-large-patch14", "vit-large-patch14-336"]
+
+
+@st.cache(allow_output_mutation=True)
+def load():
+    df = {0: pd.read_csv("data.csv"), 1: pd.read_csv("data2.csv")}
+    models = {}
+    processors = {}
+    embeddings = {}
+    for name in MODEL_NAMES:
+        if "flava" not in name:
+            model = CLIPModel
+            processor = CLIPProcessor
+            prefix = "openai/clip-"
+        else:
+            model = FlavaModel
+            processor = FlavaProcessor
+            prefix = "facebook/"
+        models[name] = model.from_pretrained(f"{prefix}{name}")
+        processors[name] = processor.from_pretrained(f"{prefix}{name}")
+        embeddings[name] = {
+            0: np.load(f"embeddings-{name}.npy"),
+            1: np.load(f"embeddings2-{name}.npy"),
+        }
+        for k in [0, 1]:
+            embeddings[name][k] = embeddings[name][k] / np.linalg.norm(
+                embeddings[name][k], axis=1, keepdims=True
+            )
+    return models, processors, df, embeddings
+
+
+models, processors, df, embeddings = load()
+source = {0: "\nSource: Unsplash", 1: "\nSource: The Movie Database (TMDB)"}
+
+
+def compute_text_embeddings(list_of_strings, name):
+    inputs = processors[name](text=list_of_strings, return_tensors="pt", padding=True)
+    result = models[name].get_text_features(**inputs)
+    if "flava" in name:
+        result = result[:, 0, :]
+    result = result.detach().numpy()
+    return result / np.linalg.norm(result, axis=1, keepdims=True)
+
+
+def image_search(query, corpus, name, n_results=24):
+    positive_embeddings = None
+
+    def concatenate_embeddings(e1, e2):
+        if e1 is None:
+            return e2
+        else:
+            return np.concatenate((e1, e2), axis=0)
+
+    splitted_query = query.split("EXCLUDING ")
+    dot_product = 0
+    k = 0 if corpus == "Unsplash" else 1
+    if len(splitted_query[0]) > 0:
+        positive_queries = splitted_query[0].split(";")
+        for positive_query in positive_queries:
+            match = re.match(r"\[(Movies|Unsplash):(\d{1,5})\](.*)", positive_query)
+            if match:
+                corpus2, idx, remainder = match.groups()
+                idx, remainder = int(idx), remainder.strip()
+                k2 = 0 if corpus2 == "Unsplash" else 1
+                positive_embeddings = concatenate_embeddings(
+                    positive_embeddings, embeddings[name][k2][idx : idx + 1, :]
+                )
+                if len(remainder) > 0:
+                    positive_embeddings = concatenate_embeddings(
+                        positive_embeddings, compute_text_embeddings([remainder], name)
+                    )
+            else:
+                positive_embeddings = concatenate_embeddings(
+                    positive_embeddings, compute_text_embeddings([positive_query], name)
+                )
+        dot_product = embeddings[name][k] @ positive_embeddings.T
+        dot_product = dot_product - np.median(dot_product, axis=0)
+        dot_product = dot_product / np.max(dot_product, axis=0, keepdims=True)
+        dot_product = np.min(dot_product, axis=1)
+
+    if len(splitted_query) > 1:
+        negative_queries = (" ".join(splitted_query[1:])).split(";")
+        negative_embeddings = compute_text_embeddings(negative_queries, name)
+        dot_product2 = embeddings[name][k] @ negative_embeddings.T
+        dot_product2 = dot_product2 - np.median(dot_product2, axis=0)
+        dot_product2 = dot_product2 / np.max(dot_product2, axis=0, keepdims=True)
+        dot_product -= np.max(np.maximum(dot_product2, 0), axis=1)
+
+    results = np.argsort(dot_product)[-1 : -n_results - 1 : -1]
+    return [
+        (
+            df[k].iloc[i]["path"],
+            df[k].iloc[i]["tooltip"] + source[k],
+            i,
+        )
+        for i in results
+    ]
+
+
+description = """
+# FLAVA Semantic Image-Text Search
+"""
+instruction=  """
+**Enter your query and hit enter**
+"""
+
+credit = """
+*Built with FAIR's [FLAVA](https://arxiv.org/abs/2112.04482) models, 🤗 Hugging Face's [transformers library](https://huggingface.co/transformers/), [Streamlit](https://streamlit.io/), 25k images from [Unsplash](https://unsplash.com/) and 8k images from [The Movie Database (TMDB)](https://www.themoviedb.org/)*
+
+*Forked and inspired from a similar app available [here](https://huggingface.co/spaces/vivien/clip/)*
+"""
+
+options = """
+## Compare
+Check results for a single model or compare two models by using the dropdown below:
+"""
+
+howto = """
+## Advanced Use
+- Click on an image to use it as a query and find similar images
+- Several queries, including one based on an image, can be combined (use "**;**" as a separator). 
+    - Try "sunset at beach; small children".
+- If the input includes "**EXCLUDING**", text following it will be used as a negative query. 
+    - Try "a busy city street with dogs" and "a busy city street EXCLUDING dogs".
+"""
+
+div_style = {
+    "display": "flex",
+    "justify-content": "center",
+    "flex-wrap": "wrap",
+}
+
+
+def main():
+    st.markdown(
+        """
+              <style>
+              .block-container{
+                max-width: 1200px;
+              }
+              div.row-widget.stRadio > div{
+                flex-direction:row;
+                display: flex;
+                justify-content: center;
+              }
+              div.row-widget.stRadio > div > label{
+                margin-left: 5px;
+                margin-right: 5px;
+              }
+              .row-widget {
+                margin-top: -25px;
+              }
+              section>div:first-child {
+                padding-top: 30px;
+              }
+              div.reportview-container > section:first-child{
+                max-width: 320px;
+              }
+              #MainMenu {
+                visibility: hidden;
+              }
+              footer {
+                visibility: hidden;
+              }
+              </style>""",
+        unsafe_allow_html=True,
+    )
+
+    st.sidebar.markdown(description)
+    st.sidebar.markdown(options)
+    mode = st.sidebar.selectbox(
+        "", ["Results for FLAVA full", "Comparison of 2 models"], index=0
+    )
+    st.sidebar.markdown(howto)
+    st.sidebar.markdown(credit)
+    _, c, _ = st.columns((1, 3, 1))
+    c.markdown(instruction)
+    if "query" in st.session_state:
+        query = c.text_input("", value=st.session_state["query"])
+    else:
+        query = c.text_input("", value="a busy city with tall buildings")
+    corpus = st.radio("", ["Unsplash", "Movies"])
+
+    models_dict = {
+        "FLAVA": "flava-full",
+        "ViT-B/32 (quickest)": "vit-base-patch32",
+        "ViT-B/16 (quick)": "vit-base-patch16",
+        "ViT-L/14 (slow)": "vit-large-patch14",
+        "ViT-L/14@336px (slowest)": "vit-large-patch14-336",
+    }
+
+    if "Comparison" in mode:
+        c1, c2 = st.columns((1, 1))
+        selection1 = c1.selectbox("", models_dict.keys(), index=0)
+        selection2 = c2.selectbox("", models_dict.keys(), index=3)
+        name1 = models_dict[selection1]
+        name2 = models_dict[selection2]
+    else:
+        name1 = MODEL_NAMES[0]
+
+    if len(query) > 0:
+        results1 = image_search(query, corpus, name1)
+        if "Comparison" in mode:
+            with c1:
+                clicked1 = clickable_images(
+                    [result[0] for result in results1],
+                    titles=[result[1] for result in results1],
+                    div_style=div_style,
+                    img_style={"margin": "2px", "height": "150px"},
+                    key=query + corpus + name1 + "1",
+                )
+            results2 = image_search(query, corpus, name2)
+            with c2:
+                clicked2 = clickable_images(
+                    [result[0] for result in results2],
+                    titles=[result[1] for result in results2],
+                    div_style=div_style,
+                    img_style={"margin": "2px", "height": "150px"},
+                    key=query + corpus + name2 + "2",
+                )
+        else:
+            clicked1 = clickable_images(
+                [result[0] for result in results1],
+                titles=[result[1] for result in results1],
+                div_style=div_style,
+                img_style={"margin": "2px", "height": "200px"},
+                key=query + corpus + name1 + "1",
+            )
+            clicked2 = -1
+
+        if clicked2 >= 0 or clicked1 >= 0:
+            change_query = False
+            if "last_clicked" not in st.session_state:
+                change_query = True
+            else:
+                if max(clicked2, clicked1) != st.session_state["last_clicked"]:
+                    change_query = True
+            if change_query:
+                if clicked1 >= 0:
+                    st.session_state["query"] = f"[{corpus}:{results1[clicked1][2]}]"
+                elif clicked2 >= 0:
+                    st.session_state["query"] = f"[{corpus}:{results2[clicked2][2]}]"
+                st.experimental_rerun()
+
+
+if __name__ == "__main__":
+    main()

embeddings-flava-full.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17f7b7a1f297f314f3728eb50e16a18780263fa9ec99b8286c58c5fb4b6853df
+size 153354368

embeddings-vit-base-patch16.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:125430e11a4a415ec0c0fc5339f97544f0447e4b0a24c20f2e59f8852e706afc
+size 51200128

embeddings-vit-base-patch32.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f7ebdff24079665faf58d07045056a63b5499753e3ffbda479691d53de3ab38
+size 51200128

embeddings-vit-large-patch14-336.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f79f10ebe267b4ee7acd553dfe0ee31df846123630058a6d58c04bf22e0ad068
+size 76800128

embeddings-vit-large-patch14.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64515f7d3d71137e2944f2c3d72c8df3e684b5d6a6ff7dcebb92370f7326ccfd
+size 76800128

embeddings2-flava-full.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:089b694a9552c65f3fdf81a0d41df299bb00cf199ab0b59fe4dc7ac0ba5e0c31
+size 49545344

embeddings2-vit-base-patch16.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:153cf3fae2385d51fe8729d3a1c059f611ca47a3fc501049708114d1bbf79049
+size 16732288

embeddings2-vit-base-patch32.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7d545bed86121dac1cedcc1de61ea5295f5840c1eb751637e6628ac54faef81
+size 16732288

embeddings2-vit-large-patch14-336.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e66eb377465fbfaa56cec079aa3e214533ceac43646f2ca78028ae4d8ad6d03
+size 25098368

embeddings2-vit-large-patch14.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d730b33e758c2648419a96ac86d39516c59795e613c35700d3a64079e5a9a27
+size 25098368

requirements.txt

@@ -0,0 +1,6 @@
+torch
+transformers
+ftfy
+numpy
+pandas
+st-clickable-images