From 60584337f81795b1b9fe4f4da5ffe273f6f1266a Mon Sep 17 00:00:00 2001 From: Rong Yu Date: Fri, 1 Jan 2021 19:32:47 +0800 Subject: [PATCH] update joints --- bodymocap/body_mocap_api.py | 2 +- bodymocap/models/smpl.py | 9 +++--- demo/demo_bodymocap.py | 3 +- demo/demo_frankmocap.py | 3 +- demo/demo_handmocap.py | 1 + handmocap/hand_mocap_api.py | 4 +-- handmocap/hand_modules/h3dw_model.py | 45 +++++----------------------- integration/copy_and_paste.py | 28 +++++++++++++++-- renderer/od_renderer.py | 2 +- 9 files changed, 47 insertions(+), 50 deletions(-) diff --git a/bodymocap/body_mocap_api.py b/bodymocap/body_mocap_api.py index 87d96d0..e929d2e 100644 --- a/bodymocap/body_mocap_api.py +++ b/bodymocap/body_mocap_api.py @@ -112,7 +112,7 @@ def regress(self, img_original, body_bbox_list): pred_output['pred_joints_img'] = pred_joints_vis_img # SMPL joints in image space pred_aa_tensor = gu.rotation_matrix_to_angle_axis(pred_rotmat.detach().cpu()[0]) - pred_output['pred_body_pose'] = pred_aa_tensor.cpu().numpy().reshape(1, 72) + pred_output['pred_body_pose'] = pred_aa_tensor.cpu().numpy().reshape(1, 72) # (1, 72) pred_output['pred_rotmat'] = pred_rotmat.detach().cpu().numpy() # (1, 24, 3, 3) pred_output['pred_betas'] = pred_betas.detach().cpu().numpy() # (1, 10) diff --git a/bodymocap/models/smpl.py b/bodymocap/models/smpl.py index 95b59da..bae1123 100644 --- a/bodymocap/models/smpl.py +++ b/bodymocap/models/smpl.py @@ -77,15 +77,14 @@ def forward(self, *args, **kwargs): extra_joints = vertices2joints(self.J_regressor_extra, smpl_output.vertices) # extra_joints = vertices2joints(self.J_regressor_extra, smpl_output.vertices[:,:6890]) *0 #TODO: implement this correctly - #SMPL-X Joint order: https://docs.google.com/spreadsheets/d/1_1dLdaX-sbMkCKr_JzJW_RZCpwBwd7rcKkWT_VgAQ_0/edit#gid=0 - smplx_to_smpl = list(range(0,22)) + [28,43] + list(range(55,76)) #28 left middle finger , 43: right middle finger 1 - smpl_joints = smpl_output.joints[:,smplx_to_smpl,:] #Convert SMPL-X to SMPL 127 ->45 - joints = torch.cat([smpl_joints, extra_joints], dim=1) #[N, 127, 3]->[N, 45, 3] + [N, 9, 3] #SMPL-X has more joints. should convert 45 + smplx_to_smpl = list(range(0,22)) + [28,43] + list(range(55,76)) # 28 left middle finger , 43: right middle finger 1 + smpl_joints = smpl_output.joints[:,smplx_to_smpl,:] # Convert SMPL-X to SMPL 127 ->45 + joints = torch.cat([smpl_joints, extra_joints], dim=1) # [N, 127, 3]->[N, 45, 3] + [N, 9, 3] # SMPL-X has more joints. should convert 45 joints = joints[:, self.joint_map, :] # Hand joints - smplx_hand_to_panoptic = [0, 13,14,15,16, 1,2,3,17, 4,5,6,18, 10,11,12,19, 7,8,9,20] #Wrist Thumb to Pinky + smplx_hand_to_panoptic = [0, 13,14,15,16, 1,2,3,17, 4,5,6,18, 10,11,12,19, 7,8,9,20] #Wrist Thumb to Pinky smplx_lhand = [20] + list(range(25,40)) + list(range(66, 71)) #20 for left wrist. 20 finger joints lhand_joints = smpl_output.joints[:,smplx_lhand, :] #(N,21,3) diff --git a/demo/demo_bodymocap.py b/demo/demo_bodymocap.py index 68fea6e..704b77f 100644 --- a/demo/demo_bodymocap.py +++ b/demo/demo_bodymocap.py @@ -19,6 +19,7 @@ import mocap_utils.general_utils as gnu from mocap_utils.timer import Timer +import renderer.image_utils as imu from renderer.viewer2D import ImShow def run_body_mocap(args, body_bbox_detector, body_mocap, visualizer): @@ -120,7 +121,7 @@ def run_body_mocap(args, body_bbox_detector, body_mocap, visualizer): img_original_bgr, pred_mesh_list = pred_mesh_list, body_bbox_list = body_bbox_list) - + # show result in the screen if not args.no_display: res_img = res_img.astype(np.uint8) diff --git a/demo/demo_frankmocap.py b/demo/demo_frankmocap.py index c2a0c94..e920261 100644 --- a/demo/demo_frankmocap.py +++ b/demo/demo_frankmocap.py @@ -24,6 +24,7 @@ from handmocap.hand_bbox_detector import HandBboxDetector from integration.copy_and_paste import integration_copy_paste +import renderer.image_utils as imu from renderer.viewer2D import ImShow @@ -193,7 +194,7 @@ def run_frank_mocap(args, bbox_detector, body_mocap, hand_mocap, visualizer): pred_mesh_list = pred_mesh_list, body_bbox_list = body_bbox_list, hand_bbox_list = hand_bbox_list) - + # show result in the screen if not args.no_display: res_img = res_img.astype(np.uint8) diff --git a/demo/demo_handmocap.py b/demo/demo_handmocap.py index 2b7d7a9..2262c33 100644 --- a/demo/demo_handmocap.py +++ b/demo/demo_handmocap.py @@ -14,6 +14,7 @@ from handmocap.hand_mocap_api import HandMocap from handmocap.hand_bbox_detector import HandBboxDetector + import renderer.image_utils as imu from renderer.viewer2D import ImShow import time diff --git a/handmocap/hand_mocap_api.py b/handmocap/hand_mocap_api.py index ddf6d55..1390106 100644 --- a/handmocap/hand_mocap_api.py +++ b/handmocap/hand_mocap_api.py @@ -194,9 +194,9 @@ def regress(self, img_original, hand_bbox_list, add_margin=False): pred_output[hand_type]['img_cropped'] = img_cropped # pred hand pose & shape params & hand joints 3d - pred_output[hand_type]['pred_hand_pose'] = pred_pose + pred_output[hand_type]['pred_hand_pose'] = pred_pose # (1, 48): (1, 3) for hand rotation, (1, 45) for finger pose. pred_output[hand_type]['pred_hand_betas'] = pred_res['pred_shape_params'] # (1, 10) - + #Convert vertices into bbox & image space cam_scale = cam[0] cam_trans = cam[1:] diff --git a/handmocap/hand_modules/h3dw_model.py b/handmocap/hand_modules/h3dw_model.py index 68c6240..b337315 100644 --- a/handmocap/hand_modules/h3dw_model.py +++ b/handmocap/hand_modules/h3dw_model.py @@ -32,7 +32,7 @@ def extract_hand_output(output, hand_type, hand_info, top_finger_joints_type='av wrist_idx, hand_start_idx, middle_finger_idx = 20, 25, 28 else: wrist_idx, hand_start_idx, middle_finger_idx = 21, 40, 43 - + vertices = output.vertices joints = output.joints vertices_shift = vertices - joints[:, hand_start_idx:hand_start_idx+1, :] @@ -44,35 +44,13 @@ def extract_hand_output(output, hand_type, hand_info, top_finger_joints_type='av hand_verts = vertices[:, hand_verts_idx, :] hand_verts_shift = hand_verts - joints[:, hand_start_idx:hand_start_idx+1, :] - hand_joints = torch.cat((joints[:, wrist_idx:wrist_idx+1, :], - joints[:, hand_start_idx:hand_start_idx+15, :] ), dim=1) - - # add top hand joints - if len(top_finger_joints_type) > 0: - if top_finger_joints_type in ['long', 'manual']: - key = f'{hand_type}_top_finger_{top_finger_joints_type}_vert_idx' - top_joint_vert_idx = hand_info[key] - hand_joints = torch.cat((hand_joints, vertices[:, top_joint_vert_idx, :]), dim=1) - else: - assert top_finger_joints_type == 'ave' - key1 = f'{hand_type}_top_finger_{top_finger_joints_type}_vert_idx' - key2 = f'{hand_type}_top_finger_{top_finger_joints_type}_vert_weight' - top_joint_vert_idxs = hand_info[key1] - top_joint_vert_weight = hand_info[key2] - bs = vertices.size(0) - - for top_joint_id, selected_verts in enumerate(top_joint_vert_idxs): - top_finger_vert_idx = hand_verts_idx[np.array(selected_verts)] - top_finger_verts = vertices[:, top_finger_vert_idx] - # weights = torch.from_numpy(np.repeat(top_joint_vert_weight[top_joint_id]).reshape(1, -1, 1) - weights = top_joint_vert_weight[top_joint_id].reshape(1, -1, 1) - weights = np.repeat(weights, bs, axis=0) - weights = torch.from_numpy(weights) - if use_cuda: - weights = weights.cuda() - top_joint = torch.sum((weights * top_finger_verts),dim=1).view(bs, 1, 3) - hand_joints = torch.cat((hand_joints, top_joint), dim=1) - + # Hand joints + if hand_type == 'left': + hand_idxs = [20] + list(range(25,40)) + list(range(66, 71)) # 20 for left wrist. 20 finger joints + else: + hand_idxs = [21] + list(range(40,55)) + list(range(71, 76)) # 21 for right wrist. 20 finger joints + smplx_hand_to_panoptic = [0, 13,14,15,16, 1,2,3,17, 4,5,6,18, 10,11,12,19, 7,8,9,20] + hand_joints = joints[:, hand_idxs, :][:, smplx_hand_to_panoptic, :] hand_joints_shift = hand_joints - joints[:, hand_start_idx:hand_start_idx+1, :] output = dict( @@ -262,13 +240,6 @@ def forward(self): self.pred_verts, self.pred_joints_3d = self.get_smplx_output( self.pred_pose_params, self.pred_shape_params) - # generate additional visualization of mesh, with constant camera params - # self.generate_mesh_multi_view() - - # generate predicted joints 2d - self.pred_joints_2d = self.batch_orth_proj_idrot( - self.pred_joints_3d, self.pred_cam_params) - def test(self): with torch.no_grad(): diff --git a/integration/copy_and_paste.py b/integration/copy_and_paste.py index 95db4f6..fd2e6e9 100644 --- a/integration/copy_and_paste.py +++ b/integration/copy_and_paste.py @@ -141,8 +141,12 @@ def integration_copy_paste(pred_body_list, pred_hand_list, smplx_model, image_sh pred_vertices = smplx_output.vertices pred_vertices = pred_vertices[0].detach().cpu().numpy() - pred_joints_3d = smplx_output.joints - pred_joints_3d = pred_joints_3d[0].detach().cpu().numpy() + pred_body_joints = smplx_output.joints + pred_body_joints = pred_body_joints[0].detach().cpu().numpy() + pred_lhand_joints = smplx_output.left_hand_joints + pred_lhand_joints = pred_lhand_joints[0].detach().cpu().numpy() + pred_rhand_joints = smplx_output.right_hand_joints + pred_rhand_joints = pred_rhand_joints[0].detach().cpu().numpy() camScale = body_info['pred_camera'][0] camTrans = body_info['pred_camera'][1:] @@ -167,6 +171,26 @@ def integration_copy_paste(pred_body_list, pred_hand_list, smplx_model, image_sh pred_vertices_bbox, bbox_scale_ratio, bbox_top_left, image_shape[1], image_shape[0]) integral_output['pred_vertices_img'] = pred_vertices_img + # convert joints to original image space (X, Y are aligned to image) + pred_body_joints_bbox = convert_smpl_to_bbox( + pred_body_joints, camScale, camTrans) + pred_body_joints_img = convert_bbox_to_oriIm( + pred_body_joints_bbox, bbox_scale_ratio, bbox_top_left, image_shape[1], image_shape[0]) + integral_output['pred_body_joints_img'] = pred_body_joints_img + + # convert left /right joints to original image space (X, Y are aligned to image) + pred_lhand_joints_bbox = convert_smpl_to_bbox( + pred_lhand_joints, camScale, camTrans) + pred_lhand_joints_img = convert_bbox_to_oriIm( + pred_lhand_joints_bbox, bbox_scale_ratio, bbox_top_left, image_shape[1], image_shape[0]) + integral_output['pred_lhand_joints_img'] = pred_lhand_joints_img + + pred_rhand_joints_bbox = convert_smpl_to_bbox( + pred_rhand_joints, camScale, camTrans) + pred_rhand_joints_img = convert_bbox_to_oriIm( + pred_rhand_joints_bbox, bbox_scale_ratio, bbox_top_left, image_shape[1], image_shape[0]) + integral_output['pred_rhand_joints_img'] = pred_rhand_joints_img + # keep hand info r_hand_local_orient_body = body_info['pred_rotmat'][:, 21] # rot-mat r_hand_global_orient_body = transfer_rotation( diff --git a/renderer/od_renderer.py b/renderer/od_renderer.py index 3bbcc0f..9ae79d7 100644 --- a/renderer/od_renderer.py +++ b/renderer/od_renderer.py @@ -33,7 +33,7 @@ def render(self, verts, faces, bg_img): input_size = 500 - f = 5 + f = 10 verts[:, 0] = (verts[:, 0] - input_size) / input_size verts[:, 1] = (verts[:, 1] - input_size) / input_size