Added EAR Plots. TODO: solve color jitter bug

app.py

@@ -100,7 +100,7 @@ def main():
             with st.spinner("Analyzing..."):
 
                 if is_image(file_extension):
-                    input_frames, output_frames, predicted_diameters, face_frames = process_frames(
+                    input_frames, output_frames, predicted_diameters, face_frames, eyes_ratios = process_frames(
                         cols,
                         [input_img],
                         tv_model,
@@ -130,7 +130,7 @@ def main():
 
                 elif is_video(file_extension):
                     output_video_path = f"{root_path}/tmp.webm"
-                    input_frames, output_frames, predicted_diameters, face_frames = process_video(
+                    input_frames, output_frames, predicted_diameters, face_frames, eyes_ratios = process_video(
                         cols,
                         video_frames,
                         tv_model,
@@ -187,6 +187,41 @@ def main():
                             # Display the Altair chart
                             st.altair_chart(chart, use_container_width=True)
 
+                    if eyes_ratios is not None and len(eyes_ratios) > 0:
+                        df = pd.DataFrame(eyes_ratios, columns=["Eyes Aspect Ratio"])
+                        df["Frame"] = range(1, len(eyes_ratios) + 1)  # Create a frame column starting from 1
+
+                        # Create an Altair chart for eyes_ratios
+                        line_chart = (
+                            alt.Chart(df)
+                            .mark_line(point=True, color=colors[-1])  # Set color of the line
+                            .encode(
+                                x=alt.X("Frame:Q", title="Frame Number"),
+                                y=alt.Y("Eyes Aspect Ratio:Q", title="Eyes Aspect Ratio"),
+                                tooltip=[
+                                    alt.Tooltip("Frame:Q", title="Frame Number"),
+                                    alt.Tooltip("Eyes Aspect Ratio:Q", title="Eyes Aspect Ratio"),
+                                ],
+                            )
+                            # .properties(title="Eyes Aspect Ratios (EARs)")
+                            # .configure_axis(grid=True)
+                        )
+
+                        # Create a horizontal rule at y=0.22
+                        line1 = alt.Chart(pd.DataFrame({"y": [0.22]})).mark_rule(color="red").encode(y="y:Q")
+
+                        line2 = alt.Chart(pd.DataFrame({"y": [0.25]})).mark_rule(color="blue").encode(y="y:Q")
+
+                        # Combine line chart and horizontal line, and apply configuration
+                        final_chart = line_chart.properties(title="Eyes Aspect Ratios (EARs)") + line1 + line2
+
+                        # Configure axis properties at the chart level
+                        final_chart = final_chart.configure_axis(grid=True)
+
+                        # Display the Altair chart
+                        st.subheader("Eyes Aspect Ratios (EARs)")
+                        st.altair_chart(final_chart, use_container_width=True)
+
 
 if __name__ == "__main__":
     main()

app_utils.py

@@ -238,6 +238,8 @@ def process_frames(
     ]
     preprocess_function = transforms.Compose(preprocess_steps)
 
+    eyes_ratios = []
+
     for idx, input_img in enumerate(input_imgs):
 
         img = np.array(input_img)
@@ -246,6 +248,7 @@ def process_frames(
         left_eye = None
         right_eye = None
         blinked = False
+        eyes_ratio = None
 
         if ds_results is not None and "face" in ds_results:
             face_img = to_pil_image(ds_results["face"])
@@ -257,6 +260,9 @@ def process_frames(
 
         if ds_results is not None and "eyes" in ds_results.keys():
             blinked = ds_results["eyes"]["blinked"]
+            eyes_ratio = ds_results["eyes"]["eyes_ratio"]
+            if eyes_ratio is not None:
+                eyes_ratios.append(eyes_ratio)
             if "left_eye" in ds_results["eyes"].keys() and ds_results["eyes"]["left_eye"] is not None:
                 left_eye = ds_results["eyes"]["left_eye"]
                 left_eye = to_pil_image(left_eye).convert("RGB")
@@ -368,7 +374,7 @@ def process_frames(
         show_cam_frames(output_frames, output_path, codec, video_output_placeholders)
         show_pred_text_frames(output_frames, output_path, predicted_diameters, codec, video_predictions_placeholders)
 
-    return input_frames, output_frames, predicted_diameters, face_frames
+    return input_frames, output_frames, predicted_diameters, face_frames, eyes_ratios
 
 
 # Function to display video with autoplay and loop
@@ -468,11 +474,11 @@ def process_video(cols, video_frames, tv_model, pupil_selection, output_path, ca
     file_format = output_path.split(".")[-1]
     codec, extension = get_codec_and_extension(file_format)
 
-    input_frames, output_frames, predicted_diameters, face_frames = process_frames(
+    input_frames, output_frames, predicted_diameters, face_frames, eyes_ratios = process_frames(
         cols, resized_frames, tv_model, pupil_selection, cam_method, output_path, codec, blink_detection
     )
 
-    return input_frames, output_frames, predicted_diameters, face_frames
+    return input_frames, output_frames, predicted_diameters, face_frames, eyes_ratios
 
 
 # Function to convert string values to float or None

feature_extraction/extractor_mediapipe.py

@@ -245,6 +245,7 @@ class ExtractorMediaPipe:
         left_eye = self.extract_eyes_regions(image, face_landmarks, self.LEFT_EYE)
         right_eye = self.extract_eyes_regions(image, face_landmarks, self.RIGHT_EYE)
         blinked = False
+        eyes_ratio = None
 
         if blink_detection:
             mesh_coordinates = self.landmarksDetection(image, results, False)
@@ -266,7 +267,7 @@ class ExtractorMediaPipe:
             else:
                 blinked = False
 
-        return {"left_eye": left_eye, "right_eye": right_eye, "blinked": blinked}
+        return {"left_eye": left_eye, "right_eye": right_eye, "blinked": blinked, "eyes_ratio": eyes_ratio}
 
     @staticmethod
     def segment_iris(iris_img):