-
Notifications
You must be signed in to change notification settings - Fork 24
/
eval.py
executable file
·174 lines (153 loc) · 7.28 KB
/
eval.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
import torch
import numpy as np
import cv2
def sort_by_score(pred_boxes, pred_labels, pred_scores):
score_seq = [(-score).argsort() for index, score in enumerate(pred_scores)]
pred_boxes = [sample_boxes[mask] for sample_boxes, mask in zip(pred_boxes, score_seq)]
pred_labels = [sample_boxes[mask] for sample_boxes, mask in zip(pred_labels, score_seq)]
pred_scores = [sample_boxes[mask] for sample_boxes, mask in zip(pred_scores, score_seq)]
return pred_boxes, pred_labels, pred_scores
def iou_2d(cubes_a, cubes_b):
"""
numpy 计算IoU
:param cubes_a: [N,(x1,y1,x2,y2)]
:param cubes_b: [M,(x1,y1,x2,y2)]
:return: IoU [N,M]
"""
# 扩维
cubes_a = np.expand_dims(cubes_a, axis=1) # [N,1,4]
cubes_b = np.expand_dims(cubes_b, axis=0) # [1,M,4]
# 分别计算高度和宽度的交集
overlap = np.maximum(0.0,
np.minimum(cubes_a[..., 2:], cubes_b[..., 2:]) -
np.maximum(cubes_a[..., :2], cubes_b[..., :2])) # [N,M,(w,h)]
# 交集
overlap = np.prod(overlap, axis=-1) # [N,M]
# 计算面积
area_a = np.prod(cubes_a[..., 2:] - cubes_a[..., :2], axis=-1)
area_b = np.prod(cubes_b[..., 2:] - cubes_b[..., :2], axis=-1)
# 交并比
iou = overlap / (area_a + area_b - overlap)
return iou
def _compute_ap(recall, precision):
""" Compute the average precision, given the recall and precision curves.
Code originally from https://github.com/rbgirshick/py-faster-rcnn.
# Arguments
recall: The recall curve (list).
precision: The precision curve (list).
# Returns
The average precision as computed in py-faster-rcnn.
"""
# correct AP calculation
# first append sentinel values at the end
mrec = np.concatenate(([0.], recall, [1.]))
mpre = np.concatenate(([0.], precision, [0.]))
# compute the precision envelope
for i in range(mpre.size - 1, 0, -1):
mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])
# to calculate area under PR curve, look for points
# where X axis (recall) changes value
i = np.where(mrec[1:] != mrec[:-1])[0]
# and sum (\Delta recall) * prec
ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
return ap
def eval_ap_2d(gt_boxes, gt_labels, pred_boxes, pred_labels, pred_scores, iou_thread, num_cls):
"""
:param gt_boxes: list of 2d array,shape[(a,(x1,y1,x2,y2)),(b,(x1,y1,x2,y2))...]
:param gt_labels: list of 1d array,shape[(a),(b)...],value is sparse label index
:param pred_boxes: list of 2d array, shape[(m,(x1,y1,x2,y2)),(n,(x1,y1,x2,y2))...]
:param pred_labels: list of 1d array,shape[(m),(n)...],value is sparse label index
:param pred_scores: list of 1d array,shape[(m),(n)...]
:param iou_thread: eg. 0.5
:param num_cls: eg. 4, total number of class including background which is equal to 0
:return: a dict containing average precision for each cls
"""
all_ap = {}
for label in range(num_cls)[1:]:
# get samples with specific label
true_label_loc = [sample_labels == label for sample_labels in gt_labels]
gt_single_cls = [sample_boxes[mask] for sample_boxes, mask in zip(gt_boxes, true_label_loc)]
pred_label_loc = [sample_labels == label for sample_labels in pred_labels]
bbox_single_cls = [sample_boxes[mask] for sample_boxes, mask in zip(pred_boxes, pred_label_loc)]
scores_single_cls = [sample_scores[mask] for sample_scores, mask in zip(pred_scores, pred_label_loc)]
fp = np.zeros((0,))
tp = np.zeros((0,))
scores = np.zeros((0,))
total_gts = 0
# loop for each sample
for sample_gts, sample_pred_box, sample_scores in zip(gt_single_cls, bbox_single_cls, scores_single_cls):
total_gts = total_gts + len(sample_gts)
assigned_gt = [] # one gt can only be assigned to one predicted bbox
# loop for each predicted bbox
for index in range(len(sample_pred_box)):
scores = np.append(scores, sample_scores[index])
if len(sample_gts) == 0: # if no gts found for the predicted bbox, assign the bbox to fp
fp = np.append(fp, 1)
tp = np.append(tp, 0)
continue
pred_box = np.expand_dims(sample_pred_box[index], axis=0)
iou = iou_2d(sample_gts, pred_box)
gt_for_box = np.argmax(iou, axis=0)
max_overlap = iou[gt_for_box, 0]
if max_overlap >= iou_thread and gt_for_box not in assigned_gt:
fp = np.append(fp, 0)
tp = np.append(tp, 1)
assigned_gt.append(gt_for_box)
else:
fp = np.append(fp, 1)
tp = np.append(tp, 0)
# sort by score
indices = np.argsort(-scores)
fp = fp[indices]
tp = tp[indices]
# compute cumulative false positives and true positives
fp = np.cumsum(fp)
tp = np.cumsum(tp)
# compute recall and precision
recall = tp / total_gts
precision = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)
ap = _compute_ap(recall, precision)
all_ap[label] = ap
# print(recall, precision)
return all_ap
if __name__=="__main__":
from model.fcos import FCOSDetector
from demo import convertSyncBNtoBN
from dataloader.VOC_dataset import VOCDataset
eval_dataset=VOCDataset("/home/data/voc2007_2012/VOCdevkit/VOC2012",resize_size=[800,1024],split='val2007')
print("INFO===>eval dataset has %d imgs"%len(eval_dataset))
eval_loader=torch.utils.data.DataLoader(eval_dataset,batch_size=1,shuffle=False,collate_fn=eval_dataset.collate_fn)
model=FCOSDetector(mode="inference")
# model=torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)
# print("INFO===>success convert BN to SyncBN")
model.load_state_dict(torch.load("./logs/voc20172012_multigpu_800x1024_epoch27_loss0.5987.pth",map_location=torch.device('cpu')))
# model=convertSyncBNtoBN(model)
# print("INFO===>success convert SyncBN to BN")
model=model.cuda().eval()
print("===>success loading model")
gt_boxes=[]
gt_classes=[]
pred_boxes=[]
pred_classes=[]
pred_scores=[]
num=0
for img,boxes,classes in eval_loader:
with torch.no_grad():
out=model(img.cuda())
pred_boxes.append(out[2][0].cpu().numpy())
pred_classes.append(out[1][0].cpu().numpy())
pred_scores.append(out[0][0].cpu().numpy())
gt_boxes.append(boxes[0].numpy())
gt_classes.append(classes[0].numpy())
num+=1
print(num,end='\r')
# print(gt_boxes[0],gt_classes[0])
# print(pred_boxes[0],pred_classes[0],pred_scores[0])
pred_boxes,pred_classes,pred_scores=sort_by_score(pred_boxes,pred_classes,pred_scores)
all_AP=eval_ap_2d(gt_boxes,gt_classes,pred_boxes,pred_classes,pred_scores,0.5,len(eval_dataset.CLASSES_NAME)+1)
print("all classes AP=====>\n",all_AP)
mAP=0.
for class_id,class_mAP in all_AP.items():
mAP+=float(class_mAP)
mAP/=(len(eval_dataset.CLASSES_NAME)+1)
print("mAP=====>%.3f\n"%mAP)