-
Notifications
You must be signed in to change notification settings - Fork 0
/
VehicleDetector.py
346 lines (278 loc) · 14.7 KB
/
VehicleDetector.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
'''
Detect vehicles in an image or video frame using a trained VehicleClassifier
'''
import numpy as np
import cv2
from scipy.ndimage.measurements import label
import matplotlib.pyplot as plt
from Config import *
import lane_detection as ld
import time
# from https://github.com/zeeshananjumjunaidi/Advanced-Lane-Detection
# Miscellaneous constants
IMG_SHAPE = (12, 8)
TRAINING_IMAGE_SHAPE = (64, 64)
# Detection constants
HEAT_THRESHOLD = 2 # threshold for a single frame's heatmap
HEAT_SMOOTH_FACTOR = 0.4 # smoothing factor for heatmaps of the current and previous frame
HEAT_MULTI_FRAME_THRESHOLD = 3 # threshold for the heatmap of a number of frames combined
# Detection helper functions
def draw_boxes(img, bboxes, color, thick):
'''Draw one or more bounding boxes in an image'''
imcopy = np.copy(img)
# Iterate through the bounding boxes
for bbox in bboxes:
# Draw a rectangle given bbox coordinates
cv2.rectangle(imcopy, bbox[0], bbox[1], color, thick)
return imcopy
def slide_window(img, x_start_stop, y_start_stop, xy_window, xy_overlap):
'''Create a list of windows covering the image'''
# If x and/or y start/stop positions not defined, set them to the image size
if x_start_stop[0] == None:
x_start_stop[0] = 0
if x_start_stop[1] == None:
x_start_stop[1] = img.shape[1]
if y_start_stop[0] == None:
y_start_stop[0] = 0
if y_start_stop[1] == None:
y_start_stop[1] = img.shape[0]
# Compute the span of the region to be searched
xspan = x_start_stop[1] - x_start_stop[0]
yspan = y_start_stop[1] - y_start_stop[0]
# Compute the number of pixels per step in x/y
nx_pix_per_step = np.int(xy_window[0]*(1 - xy_overlap[0]))
ny_pix_per_step = np.int(xy_window[1]*(1 - xy_overlap[1]))
# Compute the number of windows in x/y
nx_windows = np.int(xspan/nx_pix_per_step) - 1
ny_windows = np.int(yspan/ny_pix_per_step) - 1
# Initialize a list to append window positions to
window_list = []
# Append window position to list
for ys in range(ny_windows):
for xs in range(nx_windows):
# Calculate window position
startx = xs*nx_pix_per_step + x_start_stop[0]
endx = startx + xy_window[0]
starty = ys*ny_pix_per_step + y_start_stop[0]
endy = starty + xy_window[1]
# Append window position to list
window_list.append(((startx, starty), (endx, endy)))
# Return the list of windows
return window_list
def add_heat(heatmap, bbox_list):
for box in bbox_list:
#((x1, y1), (x2, y2))
heatmap[box[0][1]:box[1][1], box[0][0]:box[1][0]] += 1
# Return updated heatmap
return heatmap
def apply_threshold(heatmap, threshold):
new_heatmap = np.copy(heatmap)
new_heatmap[new_heatmap <= threshold] = 0
return new_heatmap
def draw_labeled_bboxes(image, labels):
'''Draw rectangles around labeled regions'''
# Iterate through all detected vehicles
for vehicle_id in range(1, labels[1] + 1):
# Find pixels with each vehicle_id label value
nonzero = (labels[0] == vehicle_id).nonzero()
# Identify x and y values of those pixels
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
# Define a bounding box based on min/max x and y
bbox = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy)))
# Draw the box on the image
cv2.rectangle(image, bbox[0], bbox[1], (0, 0, 255), 6)
# Return the image
return image
class VehicleDetector():
'''Classifier that detects vehicles in an image'''
def __init__(self, classifier, output_path=None):
'''
Initialise the detector
:param classifier: trained VehicleClassifier
:param output_path: path where images are to be save, only relevant when using verbose = True in detect()
'''
self.clf = classifier
self.previous_heatmap = None
self.output_path = output_path
def __search_windows(self, img, windows):
'''
Search each sliding window for images of vehicles using the classifier
:param img: the original image/frame
:param windows: list of sliding windows
:return: list of windows for which the classifier predicts the window contains a vehicle
'''
on_windows = []
# Iterate over all windows in the list
for window in windows:
# Extract the test window from the original image
test_img = cv2.resize(img[window[0][1]:window[1][1], window[0][0]:window[1][0]], TRAINING_IMAGE_SHAPE)
# Extract features for that window
spatial_features, hist_features, hog_features =\
self.clf.feature_extractor.extract_features_single_image(test_img,
cspace=CSPACE,
spatial_size=(SPATIAL_SIZE, SPATIAL_SIZE),
hist_bins=HIST_BIN,
hist_range=HIST_RANGE,
hog_cell_per_block=HOG_CELL_PER_BLOCK,
hog_channel=HOG_CHANNEL,
hog_pix_per_cell=HOG_PIX_PER_CELL,
hog_orient=HOG_ORIENT_BINS)
features = np.concatenate((spatial_features, hist_features, hog_features))
# Scale extracted features to be fed to the classifier
test_features = self.clf.X_scalar.transform(np.array(features).reshape(1, -1))
# Predict using the classifier
#t=time.time()
prediction = self.clf.predict(test_features)
#print(time.time()-t,'sec')
# If prediction is 1 (= vehicle) then save the window
score_predict = self.clf.score(test_features,[1])
#decision_functions=self.clf.decision_function(test_features)
#print(decision_functions)
if prediction == 1 and score_predict >= 1:
#print(score_predict)
on_windows.append(window)
# Return windows for positive detections
return on_windows
preserve_fames=[]
def detect(self, frame, verbose=False, new_image=False):
'''
:param frame: a single image or video frame, must be an RGB with pixel values 0-255
:param verbose: set to True to show and save to disc various stages of the function (do not use with videos)
:param new_image: set to True if the detector is being used on unrelated images, set to False when processing
video frames
:return: original image/frame with bounding boxes that identify vehicles
'''
org_image=lane_detect_copy = np.copy(frame)
lane_detect_copy = ld.img_pipeline(lane_detect_copy)
# plt.imshow(lane_detect_copy)
# plt.show()
all_windows = []
# If the same detector is used on a sequence of individual but unrelated images, as opposed to subsequent
# frames of the same video, the previous_heatmap needs to be reset for every call. For video frames processed
# in succession this parameter should be set to False
if new_image:
self.previous_heatmap = None
# # Grid 0 (closest to the horizon)
# win_size = 30
# y_start = int(frame.shape[0] / 2) + 20
# y_stop = y_start + win_size / 0.75
# x_start = 400
# x_stop = None
# windows_smallest = slide_window(frame, x_start_stop=[x_start, x_stop], y_start_stop=[y_start, y_stop],
# xy_window=(win_size, win_size), xy_overlap=(0.5, 0.5))
#all_windows += windows_smallest
# Grid 1 (closest to the horizon)
win_size = 60
y_start = int(frame.shape[0] / 2) + 20
y_stop = y_start + win_size / 0.75
x_start = 480
x_stop = None
windows_small = slide_window(frame, x_start_stop=[x_start, x_stop], y_start_stop=[y_start, y_stop],
xy_window=(win_size, win_size), xy_overlap=(0.75, 0.75))
all_windows += windows_small
if verbose:
window_img = draw_boxes(frame, windows_small, color=(0, 0, 255), thick=2)
# Grid 2
win_size = 80
y_start = int(frame.shape[0] / 2) + 40
y_stop = y_start + win_size / 0.75
x_start = 400
x_stop = None
windows_med = slide_window(frame, x_start_stop=[x_start, x_stop], y_start_stop=[y_start, y_stop],
xy_window=(win_size, win_size), xy_overlap=(0.75, 0.75))
all_windows += windows_med
if verbose:
window_img = draw_boxes(window_img, windows_med, color=(0, 255, 0), thick=2)
# Grid 3
win_size = 120
y_start = int(frame.shape[0] / 2) + 60
y_stop = y_start + win_size / 0.75
x_start = 380
x_stop = None
windows_bot = slide_window(frame, x_start_stop=[x_start, x_stop], y_start_stop=[y_start, y_stop],
xy_window=(win_size, win_size), xy_overlap=(0.75, 0.75))
all_windows += windows_bot
if verbose:
window_img = draw_boxes(window_img, windows_bot, color=(255, 0, 0), thick=2)
# Grid 4 (closest to the camera car)
win_size = 160
y_start = int(frame.shape[0] / 2) + 80
y_stop = y_start + win_size / 0.75 - 60
x_start = 340
x_stop = None
windows_bot = slide_window(frame, x_start_stop=[x_start, x_stop], y_start_stop=[y_start, y_stop],
xy_window=(win_size, win_size), xy_overlap=(0.75, 0.75))
all_windows += windows_bot
if verbose:
window_img = draw_boxes(window_img, windows_bot, color=(255, 255, 0), thick=2)
if verbose:
print('Number of sliding windows used: {}'.format(len(all_windows)))
# 2. Extract features for each sliding window and predict whther it contains a vehicle using the classifier
hot_windows = self.__search_windows(org_image, all_windows)
# Draw all windows that contain a detected vehicle (may include overlaps and false positives)
if verbose:
window_det = draw_boxes(org_image, hot_windows, color=(255, 255, 255), thick=6)
plt.figure(figsize=IMG_SHAPE)
plt.imshow(window_det)
plt.show()
image_tmp = cv2.resize(window_det, (int(window_det.shape[1] / 2), int(window_det.shape[0] / 2)))
plt.imsave(self.output_path + '/all_detections.jpg', image_tmp)
# 3. Combine duplicate detections by creating a heatmap
current_heatmap = np.zeros_like(frame[:, :, 0]).astype(np.float)
current_heatmap = add_heat(current_heatmap, hot_windows)
# Show heatmap prior to thresholding
if verbose:
plt.figure(figsize=IMG_SHAPE)
plt.imshow(current_heatmap, cmap='hot')
plt.show()
image_tmp = cv2.resize(current_heatmap, (int(current_heatmap.shape[1] / 2), int(current_heatmap.shape[0] /
2)))
plt.imsave(self.output_path + '/heatmap_prior_thresholding.jpg', image_tmp)
# 4. Threshold the heatmap to remove false positives and duplicate detections
current_heatmap_thresh = apply_threshold(current_heatmap, HEAT_THRESHOLD)
# Show heatmap prior to smoothing
if verbose:
plt.figure(figsize=IMG_SHAPE)
plt.imshow(current_heatmap_thresh, cmap='hot')
plt.show()
image_tmp = cv2.resize(current_heatmap_thresh, (int(current_heatmap_thresh.shape[1] / 2),
int(current_heatmap_thresh.shape[0] / 2)))
plt.imsave(self.output_path + '/heatmap_post_thresholding.jpg', image_tmp)
# 5. Determine the number of vehicles and their position by identifying the positions and regions in the heatmap
if self.previous_heatmap is None:
# There is no previous frame heat map so just use blank images
current_heatmap_combined = np.zeros_like(frame[:, :, 0]).astype(np.float)
current_heatmap_combined_thresh = current_heatmap_combined
labels = label(current_heatmap_thresh)
self.previous_heatmap = current_heatmap_thresh
self.preserve_fames.append(current_heatmap)
else:
# Use a smoothing factor to combine the current and previous frame heat map
current_heatmap_combined = self.previous_heatmap * HEAT_SMOOTH_FACTOR +\
current_heatmap_thresh * (1 - HEAT_SMOOTH_FACTOR)
# Apply a different threshold to the combined heatmap
current_heatmap_combined_thresh = apply_threshold(current_heatmap_combined, HEAT_MULTI_FRAME_THRESHOLD)
labels = label(current_heatmap_combined_thresh)
self.previous_heatmap = current_heatmap_combined_thresh
# 6. Draw the bounding boxes of the detected regions in the original image/frame
window_hot = draw_labeled_bboxes(np.copy(frame), labels)
# print(lane_detect_copy.shape,window_hot.shape)
final_img = cv2.addWeighted(lane_detect_copy, 0.5, window_hot,0.5, 0)
#final_img=window_hot
# Show detected car blobs
if verbose:
plt.figure(figsize=IMG_SHAPE)
plt.imshow(labels[0], cmap='gray')
plt.show()
plt.imsave(self.output_path + '/labeled_regions.jpg', labels[0])
# Create a diagnostic view
# if verbose:
# image_diag = plot_diagnostics(window_hot, current_heatmap, current_heatmap_thresh,
# current_heatmap_combined, current_heatmap_combined_thresh)
# image_diag = cv2.resize(image_diag, (int(image_diag.shape[1] / 2), int(image_diag.shape[0] / 2)))
# plt.imsave(self.output_path + '/image_diag.jpg', image_diag)
if verbose:
return window_img, all_windows, window_hot, hot_windows
else:
return final_img#window_hot