Upload 6 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+shape_predictor_68_face_landmarks_GTX.dat filter=lfs diff=lfs merge=lfs -text
+shape_predictor_68_face_landmarks.dat filter=lfs diff=lfs merge=lfs -text

face_grab.py

@@ -0,0 +1,67 @@
+import logging
+import cv2 as cv
+import numpy as np
+import dlib
+from typing import Optional
+
+logging.basicConfig(level=logging.INFO)
+
+
+class FaceGrabber:
+    def __init__(self):
+        self.cascades = [
+            "haarcascade_frontalface_default.xml",
+            "haarcascade_frontalface_alt.xml",
+            "haarcascade_frontalface_alt2.xml",
+            "haarcascade_frontalface_alt_tree.xml"
+        ]
+        self.detector = dlib.get_frontal_face_detector() # load face detector
+        self.predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks_GTX.dat") # load face predictor
+        self.mmod = dlib.cnn_face_detection_model_v1("mmod_human_face_detector.dat") # load face detector
+        self.paddingBy = 0.1 # padding by 10%
+
+    def grab_faces(self, img: np.ndarray, bGray: bool = False) -> Optional[np.ndarray]:
+
+        if bGray:
+            img = cv.cvtColor(img, cv.COLOR_BGR2GRAY) # convert to grayscale
+
+        detected = None
+
+        if detected is None:
+            faces = self.detector(img) # detect faces
+            if len(faces) > 0:
+                detected = faces[0]
+                detected = (detected.left(), detected.top(), detected.width(), detected.height())
+                logging.info("Face detected by dlib")
+
+        if detected is None:
+            faces = self.mmod(img)
+            if len(faces) > 0:
+                detected = faces[0]
+                detected = (detected.rect.left(), detected.rect.top(), detected.rect.width(), detected.rect.height())
+                logging.info("Face detected by mmod")
+
+        if detected is None:
+            for cascade in self.cascades:
+                cascadeClassifier = cv.CascadeClassifier(cv.data.haarcascades + cascade)
+                faces = cascadeClassifier.detectMultiScale(img, scaleFactor=1.5, minNeighbors=5) # detect faces
+                if len(faces) > 0:
+                    detected = faces[0]
+                    logging.info(f"Face detected by {cascade}")
+                    break
+
+        if detected is not None: # if face detected
+            x, y, w, h = detected # grab first face
+            padW = int(self.paddingBy * w) # get padding width
+            padH = int(self.paddingBy * h) # get padding height
+            imgH, imgW, _ = img.shape # get image dims
+            x = max(0, x - padW)
+            y = max(0, y - padH)
+            w = min(imgW - x, w + 2 * padW)
+            h = min(imgH - y, h + 2 * padH)
+            x = max(0, x - (w - detected[2]) // 2) # center the face horizontally
+            y = max(0, y - (h - detected[3]) // 2) # center the face vertically
+            face = img[y:y+h, x:x+w] # crop face
+            return face
+
+        return None

gradcam.py

@@ -0,0 +1,138 @@
+from transformers import ViTFeatureExtractor, ViTForImageClassification
+import warnings
+from torchvision import transforms
+from datasets import load_dataset
+from pytorch_grad_cam import run_dff_on_image, GradCAM
+from pytorch_grad_cam.utils.model_targets import ClassifierOutputTarget
+from pytorch_grad_cam.utils.image import show_cam_on_image
+from PIL import Image
+import numpy as np
+import cv2 as cv
+import torch
+from typing import List, Callable, Optional
+import logging
+from face_grab import FaceGrabber
+
+# original borrowed from https://github.com/jacobgil/pytorch-grad-cam/blob/master/tutorials/HuggingFace.ipynb
+# thanks @jacobgil
+# further mods beyond this commit by @simonSlamka
+
+warnings.filterwarnings("ignore")
+
+logging.basicConfig(level=logging.INFO)
+
+
+
+class HuggingfaceToTensorModelWrapper(torch.nn.Module):
+    def __init__(self, model):
+        super(HuggingfaceToTensorModelWrapper, self).__init__()
+        self.model = model
+
+    def forward(self, x):
+        return self.model(x).logits
+
+
+
+class GradCam():
+    def __init__(self):
+        pass
+    
+    def category_name_to_index(self, model, category_name):
+        name_to_index = dict((v, k) for k, v in model.config.id2label.items())
+        return name_to_index[category_name]
+        
+    def run_grad_cam_on_image(self, model: torch.nn.Module,
+                            target_layer: torch.nn.Module,
+                            targets_for_gradcam: List[Callable],
+                            reshape_transform: Optional[Callable],
+                            input_tensor: torch.nn.Module,
+                            input_image: Image,
+                            method: Callable=GradCAM,
+                            threshold: float=0.5):
+        with method(model=HuggingfaceToTensorModelWrapper(model),
+                    target_layers=[target_layer],
+                    reshape_transform=reshape_transform) as cam:
+
+            # Replicate the tensor for each of the categories we want to create Grad-CAM for:
+            repeated_tensor = input_tensor[None, :].repeat(len(targets_for_gradcam), 1, 1, 1)
+
+            batch_results = cam(input_tensor=repeated_tensor,
+                                targets=targets_for_gradcam)
+            results = []
+            for grayscale_cam in batch_results:
+                grayscale_cam[grayscale_cam < threshold] = 0
+                visualization = show_cam_on_image(np.float32(input_image)/255,
+                                                grayscale_cam,
+                                                use_rgb=True)
+                # Make it weight less in the notebook:
+                visualization = cv.resize(visualization,
+                                        (visualization.shape[1]//2, visualization.shape[0]//2))
+                results.append(visualization)
+            return np.hstack(results)
+        
+        
+    def get_top_category(self, model, img_tensor, top_k=5):
+        logits = model(img_tensor.unsqueeze(0)).logits
+        probabilities = torch.nn.functional.softmax(logits, dim=1)
+        topIdx = logits.cpu()[0, :].detach().numpy().argsort()[-1]
+        topClass = model.config.id2label[topIdx]
+        topScore = probabilities[0][topIdx].item()
+        return [{"label": topClass, "score": topScore}]
+
+    def reshape_transform_vit_huggingface(self, x):
+        activations = x[:, 1:, :]
+        activations = activations.view(activations.shape[0],
+                                    14, 14, activations.shape[2])
+        activations = activations.transpose(2, 3).transpose(1, 2)
+        return activations
+
+
+
+if __name__ == "__main__":
+
+    faceGrabber = FaceGrabber()
+    gradCam = GradCam()
+
+    image = Image.open("Feature-Image-74.jpg").convert("RGB")
+    face = faceGrabber.grab_faces(np.array(image))
+    if face is not None:
+        image = Image.fromarray(face)
+
+    img_tensor = transforms.ToTensor()(image)
+
+    model = ViTForImageClassification.from_pretrained("ongkn/attraction-classifier")
+    targets_for_gradcam = [ClassifierOutputTarget(gradCam.category_name_to_index(model, "pos")),
+                        ClassifierOutputTarget(gradCam.category_name_to_index(model, "neg"))]
+    target_layer_dff = model.vit.layernorm
+    target_layer_gradcam = model.vit.encoder.layer[-2].output
+    image_resized = image.resize((224, 224))
+    tensor_resized = transforms.ToTensor()(image_resized)
+
+    dff_image = run_dff_on_image(model=model,
+                                target_layer=target_layer_dff,
+                                classifier=model.classifier,
+                                img_pil=image_resized,
+                                img_tensor=tensor_resized,
+                                reshape_transform=gradCam.reshape_transform_vit_huggingface,
+                                n_components=5,
+                                top_k=10,
+                                threshold=0,
+                                output_size=None) #(500, 500))
+    cv.namedWindow("DFF Image", cv.WINDOW_KEEPRATIO)
+    cv.imshow("DFF Image", cv.cvtColor(dff_image, cv.COLOR_BGR2RGB))
+    cv.resizeWindow("DFF Image", 2500, 700)
+    # cv.waitKey(0)
+    # cv.destroyAllWindows()
+    grad_cam_image = gradCam.run_grad_cam_on_image(model=model,
+                                        target_layer=target_layer_gradcam,
+                                        targets_for_gradcam=targets_for_gradcam,
+                                        input_tensor=tensor_resized,
+                                        input_image=image_resized,
+                                        reshape_transform=gradCam.reshape_transform_vit_huggingface,
+                                        threshold=0)
+    cv.namedWindow("Grad-CAM Image", cv.WINDOW_KEEPRATIO)
+    cv.imshow("Grad-CAM Image", grad_cam_image)
+    cv.resizeWindow("Grad-CAM Image", 2000, 1250)
+    cv.waitKey(0)
+    cv.destroyAllWindows()
+    gradCam.print_top_categories(model, tensor_resized)

requirements.txt

@@ -0,0 +1,9 @@
+gradio
+transformers
+numpy
+Pillow
+opencv-python-headless
+dlib
+torch
+grad-cam
+torchvision

shape_predictor_68_face_landmarks.dat

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbdc2cb80eb9aa7a758672cbfdda32ba6300efe9b6e6c7a299ff7e736b11b92f
+size 99693937

shape_predictor_68_face_landmarks_GTX.dat

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:249a69a1d5f2d7c714a92934d35367d46eb52dc308d46717e82d49e8386b3b80
+size 66435981