-
Notifications
You must be signed in to change notification settings - Fork 2
/
filter.py
138 lines (116 loc) · 5.43 KB
/
filter.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
"""
Contains the code for applying the filter.
Author: Cynthia Hom
"""
from fastai import *
from fastai.vision.all import *
import cv2
import imutils
# Include so that learner knows what get_y is.
def get_y(r):
return [
[r['left_eyebrow_outer_end_x'], r['left_eyebrow_outer_end_y']],
[r['right_eyebrow_outer_end_x'], r['right_eyebrow_outer_end_y']],
[r['nose_tip_x'], r['nose_tip_y']],
]
class Filter():
def __init__(self):
self.learn = load_learner('./models/kaggle1.pkl')
# path to cardImgs data
self.cardImgs = untar_data('http://web2.acbl.org/documentlibrary/marketing/Clip_Art/cards_png_zip.zip')
self.spadeAce = PILImage.create(self.cardImgs/"AS.png") # TODO: later change to do all cards.
self.filterImage = self.spadeAce # set filter image to be spadeAce for now.
self.faceCascade = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_frontalface_default.xml")
# These variables are used so that we don't have to recacluate the filter position every single frame
self.rotatedResizedFilter = np.zeros((480, 640, 3))
self.inverseBinary = np.zeros((480, 640, 3))
self.startX, self.startY, self.endX, self.endY = (0, 0, 0, 0)
self.startXC, self.startYC, self.endXC, self.endYC = (0, 0, 0, 0)
def applyFilter(self, img, timeStep):
"""
img: PILImage, image to put filter on
timeStep: int, the current time step of the incoming video
"""
if timeStep % 20 == 0:
img = np.array(img)
tensorPoints = self.getTensorPoints(img)
# calculate position of filter
btwnEyebrows = (tensorPoints[0] + tensorPoints[1])/2
nose = tensorPoints[2]
# Calculate card width and height
# Card height is about twice distance between btwnEyebrows and nose
cardHeight = int(2*np.sqrt(sum(np.square(btwnEyebrows - nose))))
# Overlay the filter
self.getFilterFrame(cardHeight, btwnEyebrows, nose)
# Return final image, after applying the filter
return self.applyFilterFrame(img)
def getTensorPoints(self, img):
# Get only the face, and account for the case where no face is found.
faces = self.faceCascade.detectMultiScale(img, minNeighbors=3, minSize=(int(img.shape[0]/10), int(img.shape[0]/10)))
if len(faces) > 1:
print("faces greater than 1 ")
if len(faces) != 0:
x,y,w,h = faces[0]
onlyFaceImg = img[y:y + h, x:x + w]
else:
x = 0; y = 0; h, w = img.shape[:2]
onlyFaceImg = img
print("no face found ")
# Get tensor points
grayscaleImg = np.array(PILImage.create(onlyFaceImg).convert('L').resize((96, 96)))
tensorPointsGrayscale = self.learn.predict(grayscaleImg)[0]
# Adjust tensor points for original image size. Slice off first two parts of shape.
ratios = tensor(onlyFaceImg.shape[:2])/tensor([96, 96])
ratios = tensor(ratios[1], ratios[0]) # swap because in TensorPoints, first is column index, second is row index (x, y)
# Multiply by ratios and add x, y since tensor points are based on cropped face only image.
return ratios * tensorPointsGrayscale + tensor(x, y)
def getFilterFrame(self, filterHeight, btwnEyebrows, nose):
"""
img: PILImage, image to put the filter over
filterHeight: double, the height of the filter
btwnEyebrows: TensorPoint, contains x, y of the midpoint of the eyebrows
nose: TensorPoint, contains x, y of the tip of the nose
"""
yBtwnEyebrows = int(btwnEyebrows[1])
xBtwnEyebrows = int(btwnEyebrows[0])
# Change in x/change in y , nose first
difference = nose - btwnEyebrows
angle = float(-np.arctan(difference[0]/difference[1]) * 180.0/np.pi)
# Calculations
resizeRatio = filterHeight/self.filterImage.shape[0]
filterWidth = int(self.filterImage.shape[1]*resizeRatio)
resizedFilter = self.filterImage.resize((filterWidth, filterHeight)) # width then height
self.rotatedResizedFilter = imutils.rotate_bound(np.array(resizedFilter), angle=angle)
# Use to black out parts of original image
binaryCard = np.zeros(np.array(resizedFilter).shape) + 1
rotatedBinary = imutils.rotate_bound(binaryCard, angle=angle)
self.inverseBinary = (rotatedBinary < 1).astype(int)
# Calculate positions
# Starting x and y for upper left corner of filter
self.endX = int(xBtwnEyebrows + filterWidth/2 * np.cos(-angle*np.pi/180))
self.endY = int(yBtwnEyebrows + filterWidth/2 * np.sin(-angle*np.pi/180))
self.startY = self.endY - self.inverseBinary.shape[0]
self.startX = self.endX - self.inverseBinary.shape[1]
# Indicies for the card itself.
self.startYC = 0; self.endYC = self.inverseBinary.shape[0]; self.startXC = 0; self.endXC = self.inverseBinary.shape[1]
def applyFilterFrame(self, img):
# Adjust for edge cases
if (self.startY < 0): # top
self.startYC = abs(self.startY)
self.startY = 0
if (self.startX < 0): # left
self.startXC = abs(self.startX)
self.startX = 0
if (self.endY > img.shape[0]): # bottom
self.endYC = self.endY - img.shape[0]
self.endY = img.shape[0]
if (self.endX > img.shape[1]): # right
self.endXC = self.endX - img.shape[1]
self.endX = img.shape[1]
# Create image to multiply by to black out filter area
toMult = np.ones(np.array(img).shape)
toMult[self.startY:self.endY, self.startX:self.endX,:] = self.inverseBinary[self.startYC:self.endYC, self.startXC:self.endXC, :]
# Create image to add to put in filter
toAdd = np.zeros(np.array(img).shape)
toAdd[self.startY:self.endY, self.startX:self.endX,:] = self.rotatedResizedFilter[self.startYC:self.endYC, self.startXC:self.endXC, :]
return (np.array(img) * toMult + toAdd).astype('uint8') # Must be type uint8 for things to work.