update

src/vid2vid.py

@@ -23,7 +23,7 @@ from src.utils.util import get_fps, read_frames, save_videos_grid
 
 from src.utils.mp_utils  import LMKExtractor
 from src.utils.draw_util import FaceMeshVisualizer
-from src.utils.pose_util import project_points_with_trans,  matrix_to_euler_and_translation
+from src.utils.pose_util import project_points_with_trans, matrix_to_euler_and_translation, euler_and_translation_to_matrix
 from src.audio2vid import smooth_pose_seq
 
 
@@ -153,21 +153,24 @@ def video2video(ref_img, source_video, size=512, steps=25, length=150, seed=42):
     min_bs_idx = np.argmin(bs_arr.sum(1))
     
     # compute delta pose
-    trans_mat_inv_frame_0 = np.linalg.inv(trans_mat_arr[0])
     pose_arr = np.zeros([trans_mat_arr.shape[0], 6])
 
     for i in range(pose_arr.shape[0]):
-        pose_mat = trans_mat_inv_frame_0 @ trans_mat_arr[i]
-        euler_angles, translation_vector = matrix_to_euler_and_translation(pose_mat)
+        euler_angles, translation_vector = matrix_to_euler_and_translation(trans_mat_arr[i]) # real pose of source
         pose_arr[i, :3] =  euler_angles
         pose_arr[i, 3:6] =  translation_vector
+    
+    init_tran_vec = face_result['trans_mat'][:3, 3] # init translation of tgt
+    pose_arr[:, 3:6] = pose_arr[:, 3:6] - pose_arr[0, 3:6] + init_tran_vec # (relative translation of source) + (init translation of tgt)
 
-    pose_arr = smooth_pose_seq(pose_arr)
+    pose_arr_smooth = smooth_pose_seq(pose_arr, window_size=3)
+    pose_mat_smooth = [euler_and_translation_to_matrix(pose_arr_smooth[i][:3], pose_arr_smooth[i][3:6]) for i in range(pose_arr_smooth.shape[0])]    
+    pose_mat_smooth = np.array(pose_mat_smooth)   
 
     # face retarget
     verts_arr = verts_arr - verts_arr[min_bs_idx] + face_result['lmks3d']
     # project 3D mesh to 2D landmark
-    projected_vertices = project_points_with_trans(verts_arr, pose_arr, [frame_height, frame_width])
+    projected_vertices = project_points_with_trans(verts_arr, pose_mat_smooth, [frame_height, frame_width])
     
     pose_list = []
     for i, verts in enumerate(projected_vertices):