diff --git a/Unity/OpenSeeExpression.cs b/Unity/OpenSeeExpression.cs index d4919eb..11c4f30 100644 --- a/Unity/OpenSeeExpression.cs +++ b/Unity/OpenSeeExpression.cs @@ -94,6 +94,8 @@ public class PointSelection { public bool pointsLipLower = true; [Tooltip("When enabled, certain manually designed features will be used to determine the expression.")] public bool features = true; + [Tooltip("When enabled, the depth value of points is also used for training.")] + public bool includeDepth = false; } [Serializable] private class OpenSeeExpressionRepresentation { @@ -171,7 +173,7 @@ static public byte[] ToSerialized(Dictionary> expressions, private int[] indicesBrowLeft = new int[] {22, 23, 24, 25, 26}; private int[] indicesEyeRight = new int[] {36, 37, 38, 39, 40, 41}; private int[] indicesEyeLeft = new int[] {42, 43, 44, 45, 46, 47}; - private int[] indicesNose = new int[] {27, 28, 29, 30, 31, 32, 33, 34, 35}; + private int[] indicesNose = new int[] {27, 28, 29, /*30, */31, 32, 33, 34, 35}; private int[] indicesMouthCorner = new int[] {58, 62}; private int[] indicesLipUpper = new int[] {48, 49, 50, 51, 52, 59, 60, 61}; private int[] indicesLipLower = new int[] {53, 54, 55, 56, 57, 63, 64, 65}; @@ -186,64 +188,91 @@ private void SelectPoints() { foreach (int i in indicesFaceContour) { indexList.Add(colsBase + i * 3); indexList.Add(colsBase + i * 3 + 1); - indexList.Add(colsBase + i * 3 + 2); - cols += 3; + cols += 2; + if (pointSelection.includeDepth) { + indexList.Add(colsBase + i * 3 + 2); + cols++; + } } if (pointSelection.pointsBrowRight) foreach (int i in indicesBrowRight) { indexList.Add(colsBase + i * 3); indexList.Add(colsBase + i * 3 + 1); - indexList.Add(colsBase + i * 3 + 2); - cols += 3; + cols += 2; + if (pointSelection.includeDepth) { + indexList.Add(colsBase + i * 3 + 2); + cols++; + } } if (pointSelection.pointsBrowLeft) foreach (int i in indicesBrowLeft) { indexList.Add(colsBase + i * 3); indexList.Add(colsBase + i * 3 + 1); - indexList.Add(colsBase + i * 3 + 2); - cols += 3; + cols += 2; + if (pointSelection.includeDepth) { + indexList.Add(colsBase + i * 3 + 2); + cols++; + } } if (pointSelection.pointsEyeRight) foreach (int i in indicesEyeRight) { indexList.Add(colsBase + i * 3); indexList.Add(colsBase + i * 3 + 1); - indexList.Add(colsBase + i * 3 + 2); - cols += 3; + cols += 2; + if (pointSelection.includeDepth) { + indexList.Add(colsBase + i * 3 + 2); + cols++; + } } if (pointSelection.pointsEyeLeft) foreach (int i in indicesEyeLeft) { indexList.Add(colsBase + i * 3); indexList.Add(colsBase + i * 3 + 1); - indexList.Add(colsBase + i * 3 + 2); - cols += 3; + cols += 2; + if (pointSelection.includeDepth) { + indexList.Add(colsBase + i * 3 + 2); + cols++; + } } if (pointSelection.pointsNose) foreach (int i in indicesNose) { indexList.Add(colsBase + i * 3); indexList.Add(colsBase + i * 3 + 1); - indexList.Add(colsBase + i * 3 + 2); - cols += 3; + cols += 2; + if (pointSelection.includeDepth) { + indexList.Add(colsBase + i * 3 + 2); + cols++; + } } if (pointSelection.pointsNose) foreach (int i in indicesMouthCorner) { indexList.Add(colsBase + i * 3); indexList.Add(colsBase + i * 3 + 1); - indexList.Add(colsBase + i * 3 + 2); - cols += 3; + cols += 2; + if (pointSelection.includeDepth) { + indexList.Add(colsBase + i * 3 + 2); + cols++; + } } if (pointSelection.pointsLipUpper) foreach (int i in indicesLipUpper) { indexList.Add(colsBase + i * 3); indexList.Add(colsBase + i * 3 + 1); - indexList.Add(colsBase + i * 3 + 2); - cols += 3; + cols += 2; + if (pointSelection.includeDepth) { + indexList.Add(colsBase + i * 3 + 2); + cols++; + } } if (pointSelection.pointsLipLower) foreach (int i in indicesLipLower) { indexList.Add(colsBase + i * 3); indexList.Add(colsBase + i * 3 + 1); - indexList.Add(colsBase + i * 3 + 2); - cols += 3; + cols += 2; + if (pointSelection.includeDepth) { + indexList.Add(colsBase + i * 3 + 2); + cols++; + } } if (pointSelection.features) for (int i = 0; i < 14; i++) { diff --git a/tracker.py b/tracker.py index 6839f65..ca87b09 100644 --- a/tracker.py +++ b/tracker.py @@ -307,22 +307,27 @@ def update(self, result, coord, frame_count): def update_contour(self): self.contour = np.array(self.face_3d[self.contour_pts]) + def normalize_pts3d(self, pts_3d): + # Calculate angle using nose + pts_3d[:, 0:2] -= pts_3d[30, 0:2] + alpha = angle(pts_3d[30, 0:2], pts_3d[27, 0:2]) + alpha -= np.deg2rad(90) + + R = np.matrix([[np.cos(alpha), -np.sin(alpha)], [np.sin(alpha), np.cos(alpha)]]) + pts_3d[:, 0:2] = (pts_3d - pts_3d[30])[:, 0:2].dot(R) + pts_3d[30, 0:2] + + # Vertical scale + pts_3d[:, 1] /= np.mean((pts_3d[27:30, 1] - pts_3d[28:31, 1]) / (self.tracker.face_3d[27:30, 1] - self.tracker.face_3d[28:31, 1])) + + # Horizontal scale + pts_3d[:, 0] /= np.mean(np.abs(pts_3d[[0, 36, 42], 0] - pts_3d[[16, 39, 45], 0]) / np.abs(self.tracker.face_3d[[0, 36, 42], 0] - self.tracker.face_3d[[16, 39, 45], 0])) + + return pts_3d + def adjust_3d(self): if self.conf < 0.4 or self.pnp_error > 300: return - norm = np.array([ - ((self.pts_3d[0, 0] - self.pts_3d[16, 0]) + (self.pts_3d[1, 0] - self.pts_3d[15, 0])) / 2.0, - ((self.pts_3d[27, 1] - self.pts_3d[28, 1]) + (self.pts_3d[28, 1] - self.pts_3d[29, 1])) / 2.0, - np.max(self.pts_3d[0:66, 2]) - np.min(self.pts_3d[0:66, 2]) - ]) - self.pts_3d = (self.pts_3d / norm) * self.tracker.norm - - self.current_features = self.features.update(self.pts_3d[:, 0:2]) - self.eye_blink = [] - self.eye_blink.append(1 - min(max(0, -self.current_features["eye_r"]), 1)) - self.eye_blink.append(1 - min(max(0, -self.current_features["eye_l"]), 1)) - max_runs = 1 eligible = np.arange(0, 66) changed_any = False @@ -392,6 +397,13 @@ def adjust_3d(self): self.face_3d[update_indices] = self.face_3d[update_indices] * weights[update_indices] + updated[update_indices] * (1. - weights[update_indices]) self.update_contour() + self.pts_3d = self.normalize_pts3d(self.pts_3d) + self.current_features = self.features.update(self.pts_3d[:, 0:2]) + self.eye_blink = [] + self.eye_blink.append(1 - min(max(0, -self.current_features["eye_r"]), 1)) + self.eye_blink.append(1 - min(max(0, -self.current_features["eye_l"]), 1)) + + class Tracker(): def __init__(self, width, height, model_type=3, threshold=0.4, max_faces=1, discard_after=5, scan_every=3, bbox_growth=0.0, max_threads=4, silent=False, model_dir=None, no_gaze=False, use_retinaface=False): options = onnxruntime.SessionOptions() @@ -532,12 +544,6 @@ def __init__(self, width, height, model_type=3, threshold=0.4, max_faces=1, disc [-0.25799, 0.27608, -0.24967], ], np.float32) * np.array([1.0, 1.0, 1.3]) - self.norm = np.array([ - ((self.face_3d[0, 0] - self.face_3d[16, 0]) + (self.face_3d[1, 0] - self.face_3d[15, 0])) / 2.0, - ((self.face_3d[27, 1] - self.face_3d[28, 1]) + (self.face_3d[28, 1] - self.face_3d[29, 1])) / 2.0, - np.max(self.face_3d[0:66, 2]) - np.min(self.face_3d[0:66, 2]) - ]) - self.camera = np.array([[width, 0, width/2], [0, width, height/2], [0, 0, 1]], np.float32) self.inverse_camera = np.linalg.inv(self.camera) self.dist_coeffs = np.zeros((4,1))