Update handpose estimation model from MediaPipe (2023feb) (#133)

* update handpose model

* update quantize model

* fix quantize path

* update readme of quantization and benchmark result

* fix document

README.md

@@ -4,11 +4,14 @@ This model estimates 21 hand keypoints per detected hand from [palm detector](..
  
 ![MediaPipe Hands Keypoints](./examples/hand_keypoints.png)
 
-This model is converted from Tensorflow-JS to ONNX using following tools:
-- tfjs to tf_saved_model:  https://github.com/patlevin/tfjs-to-tf/
-- tf_saved_model to ONNX: https://github.com/onnx/tensorflow-onnx
+This model is converted from TFlite to ONNX using following tools:
+- TFLite model to ONNX: https://github.com/onnx/tensorflow-onnx
 - simplified by [onnx-simplifier](https://github.com/daquexian/onnx-simplifier)
 
+**Note**:
+- The int8-quantized model may produce invalid results due to a significant drop of accuracy.
+- Visit https://google.github.io/mediapipe/solutions/models.html#hands for models of larger scale.
+
 ## Demo
 
 Run the following commands to try the demo:
@@ -21,7 +24,7 @@ python demo.py -i /path/to/image
 
 ### Example outputs
 
-![webcam demo](./examples/mphandpose_demo.gif)
+![webcam demo](./examples/mphandpose_demo.webp)
 
 ## License
 
@@ -30,3 +33,5 @@ All files in this directory are licensed under [Apache 2.0 License](./LICENSE).
 ## Reference
 
 - MediaPipe Handpose: https://github.com/tensorflow/tfjs-models/tree/master/handpose
+- MediaPipe hands model and model card: https://google.github.io/mediapipe/solutions/models.html#hands
+- Int8 model quantized with rgb evaluation set of FreiHAND: https://lmb.informatik.uni-freiburg.de/resources/datasets/FreihandDataset.en.html

demo.py

@@ -31,69 +31,126 @@ except:
 
 parser = argparse.ArgumentParser(description='Hand Pose Estimation from MediaPipe')
 parser.add_argument('--input', '-i', type=str, help='Path to the input image. Omit for using default camera.')
-parser.add_argument('--model', '-m', type=str, default='./handpose_estimation_mediapipe_2022may.onnx', help='Path to the model.')
+parser.add_argument('--model', '-m', type=str, default='./handpose_estimation_mediapipe_2023feb.onnx', help='Path to the model.')
 parser.add_argument('--backend', '-b', type=int, default=backends[0], help=help_msg_backends.format(*backends))
 parser.add_argument('--target', '-t', type=int, default=targets[0], help=help_msg_targets.format(*targets))
-parser.add_argument('--conf_threshold', type=float, default=0.8, help='Filter out hands of confidence < conf_threshold.')
+parser.add_argument('--conf_threshold', type=float, default=0.9, help='Filter out hands of confidence < conf_threshold.')
 parser.add_argument('--save', '-s', type=str, default=False, help='Set true to save results. This flag is invalid when using camera.')
 parser.add_argument('--vis', '-v', type=str2bool, default=True, help='Set true to open a window for result visualization. This flag is invalid when using camera.')
 args = parser.parse_args()
 
 
 def visualize(image, hands, print_result=False):
-    output = image.copy()
+    display_screen = image.copy()
+    display_3d = np.zeros((400, 400, 3), np.uint8)
+    cv.line(display_3d, (200, 0), (200, 400), (255, 255, 255), 2)
+    cv.line(display_3d, (0, 200), (400, 200), (255, 255, 255), 2)
+    cv.putText(display_3d, 'Main View', (0, 12), cv.FONT_HERSHEY_DUPLEX, 0.5, (0, 0, 255))
+    cv.putText(display_3d, 'Top View', (200, 12), cv.FONT_HERSHEY_DUPLEX, 0.5, (0, 0, 255))
+    cv.putText(display_3d, 'Left View', (0, 212), cv.FONT_HERSHEY_DUPLEX, 0.5, (0, 0, 255))
+    cv.putText(display_3d, 'Right View', (200, 212), cv.FONT_HERSHEY_DUPLEX, 0.5, (0, 0, 255))
+    is_draw = False  # ensure only one hand is drawn
+
+    def draw_lines(image, landmarks, is_draw_point=True, thickness=2):
+        cv.line(image, landmarks[0], landmarks[1], (255, 255, 255), thickness)
+        cv.line(image, landmarks[1], landmarks[2], (255, 255, 255), thickness)
+        cv.line(image, landmarks[2], landmarks[3], (255, 255, 255), thickness)
+        cv.line(image, landmarks[3], landmarks[4], (255, 255, 255), thickness)
+
+        cv.line(image, landmarks[0], landmarks[5], (255, 255, 255), thickness)
+        cv.line(image, landmarks[5], landmarks[6], (255, 255, 255), thickness)
+        cv.line(image, landmarks[6], landmarks[7], (255, 255, 255), thickness)
+        cv.line(image, landmarks[7], landmarks[8], (255, 255, 255), thickness)
+
+        cv.line(image, landmarks[0], landmarks[9], (255, 255, 255), thickness)
+        cv.line(image, landmarks[9], landmarks[10], (255, 255, 255), thickness)
+        cv.line(image, landmarks[10], landmarks[11], (255, 255, 255), thickness)
+        cv.line(image, landmarks[11], landmarks[12], (255, 255, 255), thickness)
+
+        cv.line(image, landmarks[0], landmarks[13], (255, 255, 255), thickness)
+        cv.line(image, landmarks[13], landmarks[14], (255, 255, 255), thickness)
+        cv.line(image, landmarks[14], landmarks[15], (255, 255, 255), thickness)
+        cv.line(image, landmarks[15], landmarks[16], (255, 255, 255), thickness)
+
+        cv.line(image, landmarks[0], landmarks[17], (255, 255, 255), thickness)
+        cv.line(image, landmarks[17], landmarks[18], (255, 255, 255), thickness)
+        cv.line(image, landmarks[18], landmarks[19], (255, 255, 255), thickness)
+        cv.line(image, landmarks[19], landmarks[20], (255, 255, 255), thickness)
+
+        if is_draw_point:
+            for p in landmarks:
+                cv.circle(image, p, thickness, (0, 0, 255), -1)
 
     for idx, handpose in enumerate(hands):
         conf = handpose[-1]
         bbox = handpose[0:4].astype(np.int32)
-        landmarks = handpose[4:-1].reshape(21, 2).astype(np.int32)
+        handedness = handpose[-2]
+        if handedness <= 0.5:
+            handedness_text = 'Left'
+        else:
+            handedness_text = 'Right'
+        landmarks_screen = handpose[4:67].reshape(21, 3).astype(np.int32)
+        landmarks_word = handpose[67:130].reshape(21, 3)
 
         # Print results
         if print_result:
             print('-----------hand {}-----------'.format(idx + 1))
             print('conf: {:.2f}'.format(conf))
+            print('handedness: {}'.format(handedness_text))
             print('hand box: {}'.format(bbox))
             print('hand landmarks: ')
-            for l in landmarks:
+            for l in landmarks_screen:
+                print('\t{}'.format(l))
+            print('hand world landmarks: ')
+            for l in landmarks_word:
                 print('\t{}'.format(l))
 
+        # draw box
+        cv.rectangle(display_screen, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (0, 255, 0), 2)
+        # draw handedness
+        cv.putText(display_screen, '{}'.format(handedness_text), (bbox[0], bbox[1] + 12), cv.FONT_HERSHEY_DUPLEX, 0.5, (0, 0, 255))
         # Draw line between each key points
-        cv.line(output, landmarks[0], landmarks[1], (255, 255, 255), 2)
-        cv.line(output, landmarks[1], landmarks[2], (255, 255, 255), 2)
-        cv.line(output, landmarks[2], landmarks[3], (255, 255, 255), 2)
-        cv.line(output, landmarks[3], landmarks[4], (255, 255, 255), 2)
-
-        cv.line(output, landmarks[0], landmarks[5], (255, 255, 255), 2)
-        cv.line(output, landmarks[5], landmarks[6], (255, 255, 255), 2)
-        cv.line(output, landmarks[6], landmarks[7], (255, 255, 255), 2)
-        cv.line(output, landmarks[7], landmarks[8], (255, 255, 255), 2)
-
-        cv.line(output, landmarks[0], landmarks[9], (255, 255, 255), 2)
-        cv.line(output, landmarks[9], landmarks[10], (255, 255, 255), 2)
-        cv.line(output, landmarks[10], landmarks[11], (255, 255, 255), 2)
-        cv.line(output, landmarks[11], landmarks[12], (255, 255, 255), 2)
-
-        cv.line(output, landmarks[0], landmarks[13], (255, 255, 255), 2)
-        cv.line(output, landmarks[13], landmarks[14], (255, 255, 255), 2)
-        cv.line(output, landmarks[14], landmarks[15], (255, 255, 255), 2)
-        cv.line(output, landmarks[15], landmarks[16], (255, 255, 255), 2)
-
-        cv.line(output, landmarks[0], landmarks[17], (255, 255, 255), 2)
-        cv.line(output, landmarks[17], landmarks[18], (255, 255, 255), 2)
-        cv.line(output, landmarks[18], landmarks[19], (255, 255, 255), 2)
-        cv.line(output, landmarks[19], landmarks[20], (255, 255, 255), 2)
-
-        for p in landmarks:
-            cv.circle(output, p, 2, (0, 0, 255), 2)
-
-    return output
+        landmarks_xy = landmarks_screen[:, 0:2]
+        draw_lines(display_screen, landmarks_xy, is_draw_point=False)
+
+        # z value is relative to WRIST
+        for p in landmarks_screen:
+            r = max(5 - p[2] // 5, 0)
+            r = min(r, 14)
+            cv.circle(display_screen, np.array([p[0], p[1]]), r, (0, 0, 255), -1)
+
+        if is_draw is False:
+            is_draw = True
+            # Main view
+            landmarks_xy = landmarks_word[:, [0, 1]]
+            landmarks_xy = (landmarks_xy * 1000 + 100).astype(np.int32)
+            draw_lines(display_3d, landmarks_xy, thickness=5)
+
+            # Top view
+            landmarks_xz = landmarks_word[:, [0, 2]]
+            landmarks_xz[:, 1] = -landmarks_xz[:, 1]
+            landmarks_xz = (landmarks_xz * 1000 + np.array([300, 100])).astype(np.int32)
+            draw_lines(display_3d, landmarks_xz, thickness=5)
+
+            # Left view
+            landmarks_yz = landmarks_word[:, [2, 1]]
+            landmarks_yz[:, 0] = -landmarks_yz[:, 0]
+            landmarks_yz = (landmarks_yz * 1000 + np.array([100, 300])).astype(np.int32)
+            draw_lines(display_3d, landmarks_yz, thickness=5)
+
+            # Right view
+            landmarks_zy = landmarks_word[:, [2, 1]]
+            landmarks_zy = (landmarks_zy * 1000 + np.array([300, 300])).astype(np.int32)
+            draw_lines(display_3d, landmarks_zy, thickness=5)
+
+    return display_screen, display_3d
 
 
 if __name__ == '__main__':
     # palm detector
     palm_detector = MPPalmDet(modelPath='../palm_detection_mediapipe/palm_detection_mediapipe_2023feb.onnx',
                               nmsThreshold=0.3,
-                              scoreThreshold=0.8,
+                              scoreThreshold=0.6,
                               backendId=args.backend,
                               targetId=args.target)
     # handpose detector
@@ -108,7 +165,7 @@ if __name__ == '__main__':
 
         # Palm detector inference
         palms = palm_detector.infer(image)
-        hands = np.empty(shape=(0, 47))
+        hands = np.empty(shape=(0, 132))
 
         # Estimate the pose of each hand
         for palm in palms:
@@ -117,10 +174,12 @@ if __name__ == '__main__':
             if handpose is not None:
                 hands = np.vstack((hands, handpose))
         # Draw results on the input image
-        image = visualize(image, hands, True)
+        image, view_3d = visualize(image, hands, True)
 
         if len(palms) == 0:
             print('No palm detected!')
+        else:
+            print('Palm detected!')
 
         # Save results
         if args.save:
@@ -131,6 +190,7 @@ if __name__ == '__main__':
         if args.vis:
             cv.namedWindow(args.input, cv.WINDOW_AUTOSIZE)
             cv.imshow(args.input, image)
+            cv.imshow('3D HandPose Demo', view_3d)
             cv.waitKey(0)
     else:  # Omit input to call default camera
         deviceId = 0
@@ -145,7 +205,7 @@ if __name__ == '__main__':
 
             # Palm detector inference
             palms = palm_detector.infer(frame)
-            hands = np.empty(shape=(0, 47))
+            hands = np.empty(shape=(0, 132))
 
             tm.start()
             # Estimate the pose of each hand
@@ -156,12 +216,14 @@ if __name__ == '__main__':
                     hands = np.vstack((hands, handpose))
             tm.stop()
             # Draw results on the input image
-            frame = visualize(frame, hands)
+            frame, view_3d = visualize(frame, hands)
 
             if len(palms) == 0:
                 print('No palm detected!')
             else:
+                print('Palm detected!')
                 cv.putText(frame, 'FPS: {:.2f}'.format(tm.getFPS()), (0, 15), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255))
 
             cv.imshow('MediaPipe Handpose Detection Demo', frame)
+            cv.imshow('3D HandPose Demo', view_3d)
             tm.reset()

mp_handpose.py

@@ -9,7 +9,7 @@ class MPHandPose:
         self.backend_id = backendId
         self.target_id = targetId
 
-        self.input_size = np.array([256, 256])  # wh
+        self.input_size = np.array([224, 224])  # wh
         self.PALM_LANDMARK_IDS = [0, 5, 9, 13, 17, 1, 2]
         self.PALM_LANDMARKS_INDEX_OF_PALM_BASE = 0
         self.PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE = 2
@@ -115,20 +115,25 @@ class MPHandPose:
         return results # [bbox_coords, landmarks_coords, conf]
 
     def _postprocess(self, blob, rotated_palm_bbox, angle, rotation_matrix):
-        landmarks, conf = blob
+        landmarks, conf, handedness, landmarks_word = blob
 
+        conf = conf[0][0]
         if conf < self.conf_threshold:
             return None
 
-        landmarks = landmarks.reshape(-1, 3)  # shape: (1, 63) -> (21, 3)
+        landmarks = landmarks[0].reshape(-1, 3)  # shape: (1, 63) -> (21, 3)
+        landmarks_word = landmarks_word[0].reshape(-1, 3) # shape: (1, 63) -> (21, 3)
 
         # transform coords back to the input coords
         wh_rotated_palm_bbox = rotated_palm_bbox[1] - rotated_palm_bbox[0]
         scale_factor = wh_rotated_palm_bbox / self.input_size
         landmarks[:, :2] = (landmarks[:, :2] - self.input_size / 2) * scale_factor
+        landmarks[:, 2] = landmarks[:, 2] * max(scale_factor) # depth scaling
         coords_rotation_matrix = cv.getRotationMatrix2D((0, 0), angle, 1.0)
         rotated_landmarks = np.dot(landmarks[:, :2], coords_rotation_matrix[:, :2])
         rotated_landmarks = np.c_[rotated_landmarks, landmarks[:, 2]]
+        rotated_landmarks_world = np.dot(landmarks_word[:, :2], coords_rotation_matrix[:, :2])
+        rotated_landmarks_world = np.c_[rotated_landmarks_world, landmarks_word[:, 2]]
         #  invert rotation
         rotation_component = np.array([
             [rotation_matrix[0][0], rotation_matrix[1][0]],
@@ -144,12 +149,12 @@ class MPHandPose:
         original_center = np.array([
             np.dot(center, inverse_rotation_matrix[0]),
             np.dot(center, inverse_rotation_matrix[1])])
-        landmarks = rotated_landmarks[:, :2] + original_center
+        landmarks[:, :2] = rotated_landmarks[:, :2] + original_center
 
         # get bounding box from rotated_landmarks
         bbox = np.array([
-            np.amin(landmarks, axis=0),
-            np.amax(landmarks, axis=0)])  # [top-left, bottom-right]
+            np.amin(landmarks[:, :2], axis=0),
+            np.amax(landmarks[:, :2], axis=0)])  # [top-left, bottom-right]
         # shift bounding box
         wh_bbox = bbox[1] - bbox[0]
         shift_vector = self.HAND_BOX_SHIFT_VECTOR * wh_bbox
@@ -162,4 +167,9 @@ class MPHandPose:
             center_bbox - new_half_size,
             center_bbox + new_half_size])
 
-        return np.r_[bbox.reshape(-1), landmarks.reshape(-1), conf[0]]
+        # [0: 4]: hand bounding box found in image of format [x1, y1, x2, y2] (top-left and bottom-right points)
+        # [4: 67]: screen landmarks with format [x1, y1, z1, x2, y2 ... x21, y21, z21], z value is relative to WRIST
+        # [67: 130]: world landmarks with format [x1, y1, z1, x2, y2 ... x21, y21, z21], 3D metric x, y, z coordinate
+        # [130]: handedness, (left)[0, 1](right) hand
+        # [131]: confidence
+        return np.r_[bbox.reshape(-1), landmarks.reshape(-1), rotated_landmarks_world.reshape(-1), handedness[0][0], conf]