-
Notifications
You must be signed in to change notification settings - Fork 0
/
mls_face_warping.py
102 lines (85 loc) · 4 KB
/
mls_face_warping.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
import numpy as np
def mls_affine_deformation(vy, vx, p, q, alpha=1.0, eps=1e-8):
"""
Affine deformation
Parameters
----------
vy, vx: ndarray
coordinate grid, generated by np.meshgrid(gridX, gridY)
p: ndarray
an array with size [n, 2], original control points
q: ndarray
an array with size [n, 2], final control points
alpha: float
parameter used by weights
eps: float
epsilon
Return
------
A deformed image.
"""
# Change (x, y) to (row, col)
q = np.ascontiguousarray(q[:, [1, 0]].astype(np.int16))
p = np.ascontiguousarray(p[:, [1, 0]].astype(np.int16))
# Exchange p and q and hence we transform destination pixels to the corresponding source pixels.
p, q = q, p
grow = vx.shape[0] # grid rows
gcol = vx.shape[1] # grid cols
ctrls = p.shape[0] # control points
# Precompute
reshaped_p = p.reshape(ctrls, 2, 1, 1) # [ctrls, 2, 1, 1]
reshaped_v = np.vstack((vx.reshape(1, grow, gcol), vy.reshape(1, grow, gcol))) # [2, grow, gcol]
w = 1.0 / (np.sum((reshaped_p - reshaped_v).astype(np.float32) ** 2, axis=1) + eps) ** alpha # [ctrls, grow, gcol]
w /= np.sum(w, axis=0, keepdims=True) # [ctrls, grow, gcol]
pstar = np.zeros((2, grow, gcol), np.float32)
for i in range(ctrls):
pstar += w[i] * reshaped_p[i] # [2, grow, gcol]
phat = reshaped_p - pstar # [ctrls, 2, grow, gcol]
phat = phat.reshape(ctrls, 2, 1, grow, gcol) # [ctrls, 2, 1, grow, gcol]
phat1 = phat.reshape(ctrls, 1, 2, grow, gcol) # [ctrls, 1, 2, grow, gcol]
reshaped_w = w.reshape(ctrls, 1, 1, grow, gcol) # [ctrls, 1, 1, grow, gcol]
pTwp = np.zeros((2, 2, grow, gcol), np.float32)
for i in range(ctrls):
pTwp += phat[i] * reshaped_w[i] * phat1[i]
del phat1
try:
inv_pTwp = np.linalg.inv(pTwp.transpose(2, 3, 0, 1)) # [grow, gcol, 2, 2]
flag = False
except np.linalg.linalg.LinAlgError:
flag = True
det = np.linalg.det(pTwp.transpose(2, 3, 0, 1)) # [grow, gcol]
det[det < 1e-8] = np.inf
reshaped_det = det.reshape(1, 1, grow, gcol) # [1, 1, grow, gcol]
adjoint = pTwp[[[1, 0], [1, 0]], [[1, 1], [0, 0]], :, :] # [2, 2, grow, gcol]
adjoint[[0, 1], [1, 0], :, :] = -adjoint[[0, 1], [1, 0], :, :] # [2, 2, grow, gcol]
inv_pTwp = (adjoint / reshaped_det).transpose(2, 3, 0, 1) # [grow, gcol, 2, 2]
mul_left = reshaped_v - pstar # [2, grow, gcol]
reshaped_mul_left = mul_left.reshape(1, 2, grow, gcol).transpose(2, 3, 0, 1) # [grow, gcol, 1, 2]
mul_right = np.multiply(reshaped_w, phat, out=phat) # [ctrls, 2, 1, grow, gcol]
reshaped_mul_right = mul_right.transpose(0, 3, 4, 1, 2) # [ctrls, grow, gcol, 2, 1]
out_A = mul_right.reshape(2, ctrls, grow, gcol, 1, 1)[0] # [ctrls, grow, gcol, 1, 1]
A = np.matmul(np.matmul(reshaped_mul_left, inv_pTwp), reshaped_mul_right, out=out_A) # [ctrls, grow, gcol, 1, 1]
A = A.reshape(ctrls, 1, grow, gcol) # [ctrls, 1, grow, gcol]
del mul_right, reshaped_mul_right, phat
# Calculate q
reshaped_q = q.reshape((ctrls, 2, 1, 1)) # [ctrls, 2, 1, 1]
qstar = np.zeros((2, grow, gcol), np.float32)
for i in range(ctrls):
qstar += w[i] * reshaped_q[i] # [2, grow, gcol]
del w, reshaped_w
# Get final image transfomer -- 3-D array
transformers = np.zeros((2, grow, gcol), np.float32)
for i in range(ctrls):
transformers += A[i] * (reshaped_q[i] - qstar)
transformers += qstar
del A
# Correct the points where pTwp is singular
if flag:
blidx = det == np.inf # bool index
transformers[0][blidx] = vx[blidx] + qstar[0][blidx] - pstar[0][blidx]
transformers[1][blidx] = vy[blidx] + qstar[1][blidx] - pstar[1][blidx]
# Removed the points outside the border
transformers[transformers < 0] = 0
transformers[0][transformers[0] > grow - 1] = vx.shape[0] - 1
transformers[1][transformers[1] > gcol - 1] = vx.shape[0] - 1
return transformers.astype(np.int16)