update

README.md

@@ -2,6 +2,20 @@
 
 pytorch implementation of [openpose](https://github.com/CMU-Perceptual-Computing-Lab/openpose) including **Body and Hand Pose Estimation**, and the pytorch model is directed converted from [openpose](https://github.com/CMU-Perceptual-Computing-Lab/openpose) caffemodel by [caffemodel2pytorch](https://github.com/vadimkantorov/caffemodel2pytorch). You could implement face keypoint detection in the same way if you are interested in.
 
+openpose detects hand by the result of body pose estimation, please refer to the code of [handDetector.cpp](https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/hand/handDetector.cpp).
+In the paper, it states as:
+```
+This is an important detail: to use the keypoint detector in any practical situation, 
+we need a way to generate this bounding box. 
+We directly use the body pose estimation models from [29] and [4], 
+and use the wrist and elbow position to approximate the hand location, 
+assuming the hand extends 0.15 times the length of the forearm in the same direction.
+```
+
+If anybody wants a pure python wrapper, please refer to my [pytorch implementation](https://github.com/Hzzone/pytorch-openpose) of openpose, maybe it helps you to implement a standalone hand keypoint detector.
+
+Don't be mean to star this repo if it helps your research.
+
 ### Model Download
 * [dropbox](https://www.dropbox.com/sh/7xbup2qsn7vvjxo/AABWFksdlgOMXR_r5v3RwKRYa?dl=0)
 
@@ -17,18 +31,39 @@ pytorch implementation of [openpose](https://github.com/CMU-Perceptual-Computing
 ### Demo
 #### Skeleton
 
-<div align='center'>
-<img src='images/skeleton.jpg'>
-</div>
-
+![](images/skeleton.jpg)
 #### Body Pose Estimation
 
-<div align='center'>
-<img src='images/body_preview.jpg'>
-</div>
+![](images/body_preview.jpg)
 
 #### Hand Pose Estimation
+![](images/hand_preview.png)
+
+#### Body + Hand
+![](images/demo_preview.png)
+
+### Citation
+Please cite these papers in your publications if it helps your research (the face keypoint detector was trained using the procedure described in [Simon et al. 2017] for hands):
+
+```
+@inproceedings{cao2017realtime,
+  author = {Zhe Cao and Tomas Simon and Shih-En Wei and Yaser Sheikh},
+  booktitle = {CVPR},
+  title = {Realtime Multi-Person 2D Pose Estimation using Part Affinity Fields},
+  year = {2017}
+}
+
+@inproceedings{simon2017hand,
+  author = {Tomas Simon and Hanbyul Joo and Iain Matthews and Yaser Sheikh},
+  booktitle = {CVPR},
+  title = {Hand Keypoint Detection in Single Images using Multiview Bootstrapping},
+  year = {2017}
+}
 
-<div align='center'>
-<img src='images/hand_preview.png'>
-</div>
+@inproceedings{wei2016cpm,
+  author = {Shih-En Wei and Varun Ramakrishna and Takeo Kanade and Yaser Sheikh},
+  booktitle = {CVPR},
+  title = {Convolutional pose machines},
+  year = {2016}
+}
+```

demo.py

@@ -7,16 +7,16 @@
 from body import Body
 import matplotlib.pyplot as plt
 import copy
+import numpy as np
 
 body_estimation = Body('model/body_pose_model.pth')
 hand_estimation = Hand('model/hand_pose_model.pth')
 
 test_image = 'images/demo.jpg'
-# test_image = '/Users/hzzone/Desktop/1.jpeg'
 oriImg = cv2.imread(test_image)  # B,G,R order
 candidate, subset = body_estimation(oriImg)
-# canvas = util.draw_bodypose(oriImg, candidate, subset)
 canvas = copy.deepcopy(oriImg)
+canvas = util.draw_bodypose(canvas, candidate, subset)
 # detect hand
 hands_list = util.handDetect(candidate, subset, oriImg)
 
@@ -25,14 +25,21 @@
     # cv2.rectangle(canvas, (x, y), (x+w, y+w), (0, 255, 0), 2, lineType=cv2.LINE_AA)
     # cv2.putText(canvas, 'left' if is_left else 'right', (x, y), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
 
-    if is_left:
+    # if is_left:
         # plt.imshow(oriImg[y:y+w, x:x+w, :][:, :, [2, 1, 0]])
         # plt.show()
-        peaks = hand_estimation(oriImg[y:y+w, x:x+w, :], x, y)
-        all_hand_peaks.append(peaks)
+    peaks = hand_estimation(oriImg[y:y+w, x:x+w, :])
+    peaks[:, 0] = np.where(peaks[:, 0]==0, peaks[:, 0], peaks[:, 0]+x)
+    peaks[:, 1] = np.where(peaks[:, 1]==0, peaks[:, 1], peaks[:, 1]+y)
+    # else:
+    #     peaks = hand_estimation(cv2.flip(oriImg[y:y+w, x:x+w, :], 1))
+    #     peaks[:, 0] = np.where(peaks[:, 0]==0, peaks[:, 0], w-peaks[:, 0]-1+x)
+    #     peaks[:, 1] = np.where(peaks[:, 1]==0, peaks[:, 1], peaks[:, 1]+y)
+    #     print(peaks)
+    all_hand_peaks.append(peaks)
 
 canvas = util.draw_handpose(canvas, all_hand_peaks)
 
 plt.imshow(canvas[:, :, [2, 1, 0]])
+plt.axis('off')
 plt.show()
-cv2.imwrite('t.jpg', canvas)

demo_camera.py

@@ -0,0 +1,42 @@
+import sys
+sys.path.insert(0, 'python')
+import cv2
+import model
+import util
+from hand import Hand
+from body import Body
+import matplotlib.pyplot as plt
+import copy
+import numpy as np
+
+body_estimation = Body('model/body_pose_model.pth')
+hand_estimation = Hand('model/hand_pose_model.pth')
+
+cap = cv2.VideoCapture(0)
+cap.set(3, 640)
+cap.set(4, 480)
+while True:
+    ret, oriImg = cap.read()
+    candidate, subset = body_estimation(oriImg)
+    canvas = copy.deepcopy(oriImg)
+    canvas = util.draw_bodypose(canvas, candidate, subset)
+
+    # detect hand
+    hands_list = util.handDetect(candidate, subset, oriImg)
+
+    all_hand_peaks = []
+    for x, y, w, is_left in hands_list:
+        peaks = hand_estimation(oriImg[y:y+w, x:x+w, :])
+        peaks[:, 0] = np.where(peaks[:, 0]==0, peaks[:, 0], peaks[:, 0]+x)
+        peaks[:, 1] = np.where(peaks[:, 1]==0, peaks[:, 1], peaks[:, 1]+y)
+        all_hand_peaks.append(peaks)
+
+    canvas = util.draw_handpose(canvas, all_hand_peaks)
+
+    cv2.imshow('demo', canvas)#一个窗口用以显示原视频
+    if cv2.waitKey(1) & 0xFF == ord('q'):
+        break
+
+cap.release()
+cv2.destroyAllWindows()
+

python/hand.py

@@ -1,4 +1,5 @@
 import cv2
+import json
 import numpy as np
 import util
 import math
@@ -19,7 +20,7 @@ def __init__(self, model_path):
         self.model.load_state_dict(model_dict)
         self.model.eval()
 
-    def __call__(self, oriImg, initial_x=0, initial_y=0):
+    def __call__(self, oriImg):
         scale_search = [0.5, 1.0, 1.5, 2.0]
         # scale_search = [0.5]
         boxsize = 368
@@ -67,7 +68,7 @@ def __call__(self, oriImg, initial_x=0, initial_y=0):
             map_ori[label_img == 0] = 0
 
             y, x = util.npmax(map_ori)
-            all_peaks.append([x+initial_x, y+initial_y])
+            all_peaks.append([x, y])
         return np.array(all_peaks)
 
 if __name__ == "__main__":