Skip to content

Commit

Permalink
These changes further improve point normalization to make it easier t…
Browse files Browse the repository at this point in the history 
…o detect expressions under rotations.
  • Loading branch information
@emilianavt
emilianavt committed Mar 8, 2020
1 parent 9c12d29 commit 3153692
Show file tree
Hide file tree
Showing 2 changed files with 72 additions and 37 deletions.
67 changes: 48 additions & 19 deletions Unity/OpenSeeExpression.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,8 @@ public class PointSelection {
public bool pointsLipLower = true;
[Tooltip("When enabled, certain manually designed features will be used to determine the expression.")]
public bool features = true;
[Tooltip("When enabled, the depth value of points is also used for training.")]
public bool includeDepth = false;
}
[Serializable]
private class OpenSeeExpressionRepresentation {
Expand Down Expand Up @@ -171,7 +173,7 @@ static public byte[] ToSerialized(Dictionary<string, List<float[]>> expressions,
private int[] indicesBrowLeft = new int[] {22, 23, 24, 25, 26};
private int[] indicesEyeRight = new int[] {36, 37, 38, 39, 40, 41};
private int[] indicesEyeLeft = new int[] {42, 43, 44, 45, 46, 47};
private int[] indicesNose = new int[] {27, 28, 29, 30, 31, 32, 33, 34, 35};
private int[] indicesNose = new int[] {27, 28, 29, /*30, */31, 32, 33, 34, 35};
private int[] indicesMouthCorner = new int[] {58, 62};
private int[] indicesLipUpper = new int[] {48, 49, 50, 51, 52, 59, 60, 61};
private int[] indicesLipLower = new int[] {53, 54, 55, 56, 57, 63, 64, 65};
Expand All @@ -186,64 +188,91 @@ private void SelectPoints() {
foreach (int i in indicesFaceContour) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
cols += 2;
if (pointSelection.includeDepth) {
indexList.Add(colsBase + i * 3 + 2);
cols++;
}
}
if (pointSelection.pointsBrowRight)
foreach (int i in indicesBrowRight) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
cols += 2;
if (pointSelection.includeDepth) {
indexList.Add(colsBase + i * 3 + 2);
cols++;
}
}
if (pointSelection.pointsBrowLeft)
foreach (int i in indicesBrowLeft) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
cols += 2;
if (pointSelection.includeDepth) {
indexList.Add(colsBase + i * 3 + 2);
cols++;
}
}
if (pointSelection.pointsEyeRight)
foreach (int i in indicesEyeRight) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
cols += 2;
if (pointSelection.includeDepth) {
indexList.Add(colsBase + i * 3 + 2);
cols++;
}
}
if (pointSelection.pointsEyeLeft)
foreach (int i in indicesEyeLeft) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
cols += 2;
if (pointSelection.includeDepth) {
indexList.Add(colsBase + i * 3 + 2);
cols++;
}
}
if (pointSelection.pointsNose)
foreach (int i in indicesNose) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
cols += 2;
if (pointSelection.includeDepth) {
indexList.Add(colsBase + i * 3 + 2);
cols++;
}
}
if (pointSelection.pointsNose)
foreach (int i in indicesMouthCorner) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
cols += 2;
if (pointSelection.includeDepth) {
indexList.Add(colsBase + i * 3 + 2);
cols++;
}
}
if (pointSelection.pointsLipUpper)
foreach (int i in indicesLipUpper) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
cols += 2;
if (pointSelection.includeDepth) {
indexList.Add(colsBase + i * 3 + 2);
cols++;
}
}
if (pointSelection.pointsLipLower)
foreach (int i in indicesLipLower) {
indexList.Add(colsBase + i * 3);
indexList.Add(colsBase + i * 3 + 1);
indexList.Add(colsBase + i * 3 + 2);
cols += 3;
cols += 2;
if (pointSelection.includeDepth) {
indexList.Add(colsBase + i * 3 + 2);
cols++;
}
}
if (pointSelection.features)
for (int i = 0; i < 14; i++) {
Expand Down
42 changes: 24 additions & 18 deletions tracker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,22 +307,27 @@ def update(self, result, coord, frame_count):
def update_contour(self):
self.contour = np.array(self.face_3d[self.contour_pts])

def normalize_pts3d(self, pts_3d):
# Calculate angle using nose
pts_3d[:, 0:2] -= pts_3d[30, 0:2]
alpha = angle(pts_3d[30, 0:2], pts_3d[27, 0:2])
alpha -= np.deg2rad(90)

R = np.matrix([[np.cos(alpha), -np.sin(alpha)], [np.sin(alpha), np.cos(alpha)]])
pts_3d[:, 0:2] = (pts_3d - pts_3d[30])[:, 0:2].dot(R) + pts_3d[30, 0:2]

# Vertical scale
pts_3d[:, 1] /= np.mean((pts_3d[27:30, 1] - pts_3d[28:31, 1]) / (self.tracker.face_3d[27:30, 1] - self.tracker.face_3d[28:31, 1]))

# Horizontal scale
pts_3d[:, 0] /= np.mean(np.abs(pts_3d[[0, 36, 42], 0] - pts_3d[[16, 39, 45], 0]) / np.abs(self.tracker.face_3d[[0, 36, 42], 0] - self.tracker.face_3d[[16, 39, 45], 0]))

return pts_3d

def adjust_3d(self):
if self.conf < 0.4 or self.pnp_error > 300:
return

norm = np.array([
((self.pts_3d[0, 0] - self.pts_3d[16, 0]) + (self.pts_3d[1, 0] - self.pts_3d[15, 0])) / 2.0,
((self.pts_3d[27, 1] - self.pts_3d[28, 1]) + (self.pts_3d[28, 1] - self.pts_3d[29, 1])) / 2.0,
np.max(self.pts_3d[0:66, 2]) - np.min(self.pts_3d[0:66, 2])
])
self.pts_3d = (self.pts_3d / norm) * self.tracker.norm

self.current_features = self.features.update(self.pts_3d[:, 0:2])
self.eye_blink = []
self.eye_blink.append(1 - min(max(0, -self.current_features["eye_r"]), 1))
self.eye_blink.append(1 - min(max(0, -self.current_features["eye_l"]), 1))

max_runs = 1
eligible = np.arange(0, 66)
changed_any = False
Expand Down Expand Up @@ -392,6 +397,13 @@ def adjust_3d(self):
self.face_3d[update_indices] = self.face_3d[update_indices] * weights[update_indices] + updated[update_indices] * (1. - weights[update_indices])
self.update_contour()

self.pts_3d = self.normalize_pts3d(self.pts_3d)
self.current_features = self.features.update(self.pts_3d[:, 0:2])
self.eye_blink = []
self.eye_blink.append(1 - min(max(0, -self.current_features["eye_r"]), 1))
self.eye_blink.append(1 - min(max(0, -self.current_features["eye_l"]), 1))


class Tracker():
def __init__(self, width, height, model_type=3, threshold=0.4, max_faces=1, discard_after=5, scan_every=3, bbox_growth=0.0, max_threads=4, silent=False, model_dir=None, no_gaze=False, use_retinaface=False):
options = onnxruntime.SessionOptions()
Expand Down Expand Up @@ -532,12 +544,6 @@ def __init__(self, width, height, model_type=3, threshold=0.4, max_faces=1, disc
[-0.25799, 0.27608, -0.24967],
], np.float32) * np.array([1.0, 1.0, 1.3])

self.norm = np.array([
((self.face_3d[0, 0] - self.face_3d[16, 0]) + (self.face_3d[1, 0] - self.face_3d[15, 0])) / 2.0,
((self.face_3d[27, 1] - self.face_3d[28, 1]) + (self.face_3d[28, 1] - self.face_3d[29, 1])) / 2.0,
np.max(self.face_3d[0:66, 2]) - np.min(self.face_3d[0:66, 2])
])

self.camera = np.array([[width, 0, width/2], [0, width, height/2], [0, 0, 1]], np.float32)
self.inverse_camera = np.linalg.inv(self.camera)
self.dist_coeffs = np.zeros((4,1))
Expand Down

0 comments on commit 3153692

Please sign in to comment.