Skip to content

Commit

Permalink
[Feature] add kitti AP40 evaluation metric (v1.0.0.dev0) (#927)
Browse files Browse the repository at this point in the history 
* Add citation (#901)

* [Feature] Add python3.9 in CI (#900)

* Add python3.0 in CI

* Add python3.0 in CI

* Bump to v0.17.0 (#898)

* Update README.md

* Update README_zh-CN.md

* Update version.py

* Update getting_started.md

* Update getting_started.md

* Update changelog.md

* Remove "recent" in the news

* Remove "recent" in the news

* Fix comments

* [Docs] Fix the version of sphinx (#902)

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* add AP40

* add unitest

* add unitest

* seperate AP11 and AP40

* fix some typos

Co-authored-by: dingchang <hudingchang.vendor@sensetime.com>
Co-authored-by: Tai-Wang <tab_wang@outlook.com>
  • Loading branch information
@ZCMax @DCNSW @Tai-Wang
3 people committed Sep 24, 2021
1 parent d5b7eaa commit 7c27cd7
Show file tree
Hide file tree
Showing 2 changed files with 205 additions and 79 deletions.
210 changes: 156 additions & 54 deletions mmdet3d/core/evaluation/kitti_utils/eval.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -569,13 +569,20 @@ def eval_class(gt_annos,
return ret_dict


def get_mAP(prec):
def get_mAP11(prec):
sums = 0
for i in range(0, prec.shape[-1], 4):
sums = sums + prec[..., i]
return sums / 11 * 100


def get_mAP40(prec):
sums = 0
for i in range(1, prec.shape[-1]):
sums = sums + prec[..., i]
return sums / 40 * 100


def print_str(value, *arg, sstream=None):
if sstream is None:
sstream = sysio.StringIO()
Expand All @@ -592,8 +599,10 @@ def do_eval(gt_annos,
eval_types=['bbox', 'bev', '3d']):
# min_overlaps: [num_minoverlap, metric, num_class]
difficultys = [0, 1, 2]
mAP_bbox = None
mAP_aos = None
mAP11_bbox = None
mAP11_aos = None
mAP40_bbox = None
mAP40_aos = None
if 'bbox' in eval_types:
ret = eval_class(
gt_annos,
Expand All @@ -604,22 +613,29 @@ def do_eval(gt_annos,
min_overlaps,
compute_aos=('aos' in eval_types))
# ret: [num_class, num_diff, num_minoverlap, num_sample_points]
mAP_bbox = get_mAP(ret['precision'])
mAP11_bbox = get_mAP11(ret['precision'])
mAP40_bbox = get_mAP40(ret['precision'])
if 'aos' in eval_types:
mAP_aos = get_mAP(ret['orientation'])
mAP11_aos = get_mAP11(ret['orientation'])
mAP40_aos = get_mAP40(ret['orientation'])

mAP_bev = None
mAP11_bev = None
mAP40_bev = None
if 'bev' in eval_types:
ret = eval_class(gt_annos, dt_annos, current_classes, difficultys, 1,
min_overlaps)
mAP_bev = get_mAP(ret['precision'])
mAP11_bev = get_mAP11(ret['precision'])
mAP40_bev = get_mAP40(ret['precision'])

mAP_3d = None
mAP11_3d = None
mAP40_3d = None
if '3d' in eval_types:
ret = eval_class(gt_annos, dt_annos, current_classes, difficultys, 2,
min_overlaps)
mAP_3d = get_mAP(ret['precision'])
return mAP_bbox, mAP_bev, mAP_3d, mAP_aos
mAP11_3d = get_mAP11(ret['precision'])
mAP40_3d = get_mAP40(ret['precision'])
return (mAP11_bbox, mAP11_bev, mAP11_3d, mAP11_aos, mAP40_bbox, mAP40_bev,
mAP40_3d, mAP40_aos)


def do_coco_style_eval(gt_annos, dt_annos, current_classes, overlap_ranges,
Expand All @@ -629,9 +645,10 @@ def do_coco_style_eval(gt_annos, dt_annos, current_classes, overlap_ranges,
for i in range(overlap_ranges.shape[1]):
for j in range(overlap_ranges.shape[2]):
min_overlaps[:, i, j] = np.linspace(*overlap_ranges[:, i, j])
mAP_bbox, mAP_bev, mAP_3d, mAP_aos = do_eval(gt_annos, dt_annos,
current_classes, min_overlaps,
compute_aos)
mAP_bbox, mAP_bev, mAP_3d, mAP_aos, _, _, \
_, _ = do_eval(gt_annos, dt_annos,
current_classes, min_overlaps,
compute_aos)
# ret: [num_class, num_diff, num_minoverlap]
mAP_bbox = mAP_bbox.mean(-1)
mAP_bev = mAP_bev.mean(-1)
Expand Down Expand Up @@ -703,33 +720,109 @@ def kitti_eval(gt_annos,
if compute_aos:
eval_types.append('aos')

mAPbbox, mAPbev, mAP3d, mAPaos = do_eval(gt_annos, dt_annos,
current_classes, min_overlaps,
eval_types)
mAP11_bbox, mAP11_bev, mAP11_3d, mAP11_aos, mAP40_bbox, mAP40_bev, \
mAP40_3d, mAP40_aos = do_eval(gt_annos, dt_annos,
current_classes, min_overlaps,
eval_types)

ret_dict = {}
difficulty = ['easy', 'moderate', 'hard']

# calculate AP11
result += '\n----------- AP11 Results ------------\n\n'
for j, curcls in enumerate(current_classes):
# mAP threshold array: [num_minoverlap, metric, class]
# mAP result: [num_class, num_diff, num_minoverlap]
curcls_name = class_to_name[curcls]
for i in range(min_overlaps.shape[0]):
# prepare results for print
result += ('{} AP@{:.2f}, {:.2f}, {:.2f}:\n'.format(
result += ('{} AP11@{:.2f}, {:.2f}, {:.2f}:\n'.format(
curcls_name, *min_overlaps[i, :, j]))
if mAPbbox is not None:
result += 'bbox AP:{:.4f}, {:.4f}, {:.4f}\n'.format(
*mAPbbox[j, :, i])
if mAPbev is not None:
result += 'bev AP:{:.4f}, {:.4f}, {:.4f}\n'.format(
*mAPbev[j, :, i])
if mAP3d is not None:
result += '3d AP:{:.4f}, {:.4f}, {:.4f}\n'.format(
*mAP3d[j, :, i])
if mAP11_bbox is not None:
result += 'bbox AP11:{:.4f}, {:.4f}, {:.4f}\n'.format(
*mAP11_bbox[j, :, i])
if mAP11_bev is not None:
result += 'bev AP11:{:.4f}, {:.4f}, {:.4f}\n'.format(
*mAP11_bev[j, :, i])
if mAP11_3d is not None:
result += '3d AP11:{:.4f}, {:.4f}, {:.4f}\n'.format(
*mAP11_3d[j, :, i])
if compute_aos:
result += 'aos AP11:{:.2f}, {:.2f}, {:.2f}\n'.format(
*mAP11_aos[j, :, i])

# prepare results for logger
for idx in range(3):
if i == 0:
postfix = f'{difficulty[idx]}_strict'
else:
postfix = f'{difficulty[idx]}_loose'
prefix = f'KITTI/{curcls_name}'
if mAP11_3d is not None:
ret_dict[f'{prefix}_3D_AP11_{postfix}'] =\
mAP11_3d[j, idx, i]
if mAP11_bev is not None:
ret_dict[f'{prefix}_BEV_AP11_{postfix}'] =\
mAP11_bev[j, idx, i]
if mAP11_bbox is not None:
ret_dict[f'{prefix}_2D_AP11_{postfix}'] =\
mAP11_bbox[j, idx, i]

# calculate mAP11 over all classes if there are multiple classes
if len(current_classes) > 1:
# prepare results for print
result += ('\nOverall AP11@{}, {}, {}:\n'.format(*difficulty))
if mAP11_bbox is not None:
mAP11_bbox = mAP11_bbox.mean(axis=0)
result += 'bbox AP11:{:.4f}, {:.4f}, {:.4f}\n'.format(
*mAP11_bbox[:, 0])
if mAP11_bev is not None:
mAP11_bev = mAP11_bev.mean(axis=0)
result += 'bev AP11:{:.4f}, {:.4f}, {:.4f}\n'.format(
*mAP11_bev[:, 0])
if mAP11_3d is not None:
mAP11_3d = mAP11_3d.mean(axis=0)
result += '3d AP11:{:.4f}, {:.4f}, {:.4f}\n'.format(*mAP11_3d[:,
0])
if compute_aos:
mAP11_aos = mAP11_aos.mean(axis=0)
result += 'aos AP11:{:.2f}, {:.2f}, {:.2f}\n'.format(
*mAP11_aos[:, 0])

# prepare results for logger
for idx in range(3):
postfix = f'{difficulty[idx]}'
if mAP11_3d is not None:
ret_dict[f'KITTI/Overall_3D_AP11_{postfix}'] = mAP11_3d[idx, 0]
if mAP11_bev is not None:
ret_dict[f'KITTI/Overall_BEV_AP11_{postfix}'] =\
mAP11_bev[idx, 0]
if mAP11_bbox is not None:
ret_dict[f'KITTI/Overall_2D_AP11_{postfix}'] =\
mAP11_bbox[idx, 0]

# Calculate AP40
result += '\n----------- AP40 Results ------------\n\n'
for j, curcls in enumerate(current_classes):
# mAP threshold array: [num_minoverlap, metric, class]
# mAP result: [num_class, num_diff, num_minoverlap]
curcls_name = class_to_name[curcls]
for i in range(min_overlaps.shape[0]):
# prepare results for print
result += ('{} AP40@{:.2f}, {:.2f}, {:.2f}:\n'.format(
curcls_name, *min_overlaps[i, :, j]))
if mAP40_bbox is not None:
result += 'bbox AP40:{:.4f}, {:.4f}, {:.4f}\n'.format(
*mAP40_bbox[j, :, i])
if mAP40_bev is not None:
result += 'bev AP40:{:.4f}, {:.4f}, {:.4f}\n'.format(
*mAP40_bev[j, :, i])
if mAP40_3d is not None:
result += '3d AP40:{:.4f}, {:.4f}, {:.4f}\n'.format(
*mAP40_3d[j, :, i])
if compute_aos:
result += 'aos AP:{:.2f}, {:.2f}, {:.2f}\n'.format(
*mAPaos[j, :, i])
result += 'aos AP40:{:.2f}, {:.2f}, {:.2f}\n'.format(
*mAP40_aos[j, :, i])

# prepare results for logger
for idx in range(3):
Expand All @@ -738,39 +831,48 @@ def kitti_eval(gt_annos,
else:
postfix = f'{difficulty[idx]}_loose'
prefix = f'KITTI/{curcls_name}'
if mAP3d is not None:
ret_dict[f'{prefix}_3D_{postfix}'] = mAP3d[j, idx, i]
if mAPbev is not None:
ret_dict[f'{prefix}_BEV_{postfix}'] = mAPbev[j, idx, i]
if mAPbbox is not None:
ret_dict[f'{prefix}_2D_{postfix}'] = mAPbbox[j, idx, i]

# calculate mAP over all classes if there are multiple classes
if mAP40_3d is not None:
ret_dict[f'{prefix}_3D_AP40_{postfix}'] =\
mAP40_3d[j, idx, i]
if mAP40_bev is not None:
ret_dict[f'{prefix}_BEV_AP40_{postfix}'] =\
mAP40_bev[j, idx, i]
if mAP40_bbox is not None:
ret_dict[f'{prefix}_2D_AP40_{postfix}'] =\
mAP40_bbox[j, idx, i]

# calculate mAP40 over all classes if there are multiple classes
if len(current_classes) > 1:
# prepare results for print
result += ('\nOverall AP@{}, {}, {}:\n'.format(*difficulty))
if mAPbbox is not None:
mAPbbox = mAPbbox.mean(axis=0)
result += 'bbox AP:{:.4f}, {:.4f}, {:.4f}\n'.format(*mAPbbox[:, 0])
if mAPbev is not None:
mAPbev = mAPbev.mean(axis=0)
result += 'bev AP:{:.4f}, {:.4f}, {:.4f}\n'.format(*mAPbev[:, 0])
if mAP3d is not None:
mAP3d = mAP3d.mean(axis=0)
result += '3d AP:{:.4f}, {:.4f}, {:.4f}\n'.format(*mAP3d[:, 0])
result += ('\nOverall AP40@{}, {}, {}:\n'.format(*difficulty))
if mAP40_bbox is not None:
mAP40_bbox = mAP40_bbox.mean(axis=0)
result += 'bbox AP40:{:.4f}, {:.4f}, {:.4f}\n'.format(
*mAP40_bbox[:, 0])
if mAP40_bev is not None:
mAP40_bev = mAP40_bev.mean(axis=0)
result += 'bev AP40:{:.4f}, {:.4f}, {:.4f}\n'.format(
*mAP40_bev[:, 0])
if mAP40_3d is not None:
mAP40_3d = mAP40_3d.mean(axis=0)
result += '3d AP40:{:.4f}, {:.4f}, {:.4f}\n'.format(*mAP40_3d[:,
0])
if compute_aos:
mAPaos = mAPaos.mean(axis=0)
result += 'aos AP:{:.2f}, {:.2f}, {:.2f}\n'.format(*mAPaos[:, 0])
mAP40_aos = mAP40_aos.mean(axis=0)
result += 'aos AP40:{:.2f}, {:.2f}, {:.2f}\n'.format(
*mAP40_aos[:, 0])

# prepare results for logger
for idx in range(3):
postfix = f'{difficulty[idx]}'
if mAP3d is not None:
ret_dict[f'KITTI/Overall_3D_{postfix}'] = mAP3d[idx, 0]
if mAPbev is not None:
ret_dict[f'KITTI/Overall_BEV_{postfix}'] = mAPbev[idx, 0]
if mAPbbox is not None:
ret_dict[f'KITTI/Overall_2D_{postfix}'] = mAPbbox[idx, 0]
if mAP40_3d is not None:
ret_dict[f'KITTI/Overall_3D_AP40_{postfix}'] = mAP40_3d[idx, 0]
if mAP40_bev is not None:
ret_dict[f'KITTI/Overall_BEV_AP40_{postfix}'] =\
mAP40_bev[idx, 0]
if mAP40_bbox is not None:
ret_dict[f'KITTI/Overall_2D_AP40_{postfix}'] =\
mAP40_bbox[idx, 0]

return result, ret_dict

Expand Down
74 changes: 49 additions & 25 deletions tests/test_metrics/test_kitti_eval.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,31 +83,49 @@ def test_do_eval():
[[0.5, 0.5, 0.7], [0.25, 0.25, 0.5],
[0.25, 0.25, 0.5]]])
eval_types = ['bbox', 'bev', '3d', 'aos']
mAP_bbox, mAP_bev, mAP_3d, mAP_aos = do_eval([gt_anno], [dt_anno],
mAP11_bbox, mAP11_bev, mAP11_3d, mAP11_aos, mAP40_bbox,\
mAP40_bev, mAP40_3d, mAP40_aos = do_eval([gt_anno], [dt_anno],
current_classes, min_overlaps,
eval_types)
expected_mAP_bbox = np.array([[[0., 0.], [9.09090909, 9.09090909],
[9.09090909, 9.09090909]],
[[0., 0.], [9.09090909, 9.09090909],
[9.09090909, 9.09090909]],
[[0., 0.], [9.09090909, 9.09090909],
[9.09090909, 9.09090909]]])
expected_mAP_bev = np.array([[[0., 0.], [0., 0.], [0., 0.]],
[[0., 0.], [0., 0.], [0., 0.]],
[[0., 0.], [0., 0.], [0., 0.]]])
expected_mAP_3d = np.array([[[0., 0.], [0., 0.], [0., 0.]],
[[0., 0.], [0., 0.], [0., 0.]],
[[0., 0.], [0., 0.], [0., 0.]]])
expected_mAP_aos = np.array([[[0., 0.], [0.55020816, 0.55020816],
[0.55020816, 0.55020816]],
[[0., 0.], [8.36633862, 8.36633862],
[8.36633862, 8.36633862]],
[[0., 0.], [8.63476893, 8.63476893],
[8.63476893, 8.63476893]]])
assert np.allclose(mAP_bbox, expected_mAP_bbox)
assert np.allclose(mAP_bev, expected_mAP_bev)
assert np.allclose(mAP_3d, expected_mAP_3d)
assert np.allclose(mAP_aos, expected_mAP_aos)
expected_mAP11_bbox = np.array([[[0., 0.], [9.09090909, 9.09090909],
[9.09090909, 9.09090909]],
[[0., 0.], [9.09090909, 9.09090909],
[9.09090909, 9.09090909]],
[[0., 0.], [9.09090909, 9.09090909],
[9.09090909, 9.09090909]]])
expected_mAP40_bbox = np.array([[[0., 0.], [0., 0.], [0., 0.]],
[[0., 0.], [0., 0.], [0., 0.]],
[[0., 0.], [2.5, 2.5], [2.5, 2.5]]])
expected_mAP11_bev = np.array([[[0., 0.], [0., 0.], [0., 0.]],
[[0., 0.], [0., 0.], [0., 0.]],
[[0., 0.], [0., 0.], [0., 0.]]])
expected_mAP40_bev = np.array([[[0., 0.], [0., 0.], [0., 0.]],
[[0., 0.], [0., 0.], [0., 0.]],
[[0., 0.], [0., 0.], [0., 0.]]])
expected_mAP11_3d = np.array([[[0., 0.], [0., 0.], [0., 0.]],
[[0., 0.], [0., 0.], [0., 0.]],
[[0., 0.], [0., 0.], [0., 0.]]])
expected_mAP40_3d = np.array([[[0., 0.], [0., 0.], [0., 0.]],
[[0., 0.], [0., 0.], [0., 0.]],
[[0., 0.], [0., 0.], [0., 0.]]])
expected_mAP11_aos = np.array([[[0., 0.], [0.55020816, 0.55020816],
[0.55020816, 0.55020816]],
[[0., 0.], [8.36633862, 8.36633862],
[8.36633862, 8.36633862]],
[[0., 0.], [8.63476893, 8.63476893],
[8.63476893, 8.63476893]]])
expected_mAP40_aos = np.array([[[0., 0.], [0., 0.], [0., 0.]],
[[0., 0.], [0., 0.], [0., 0.]],
[[0., 0.], [1.58140643, 1.58140643],
[1.58140643, 1.58140643]]])
assert np.allclose(mAP11_bbox, expected_mAP11_bbox)
assert np.allclose(mAP11_bev, expected_mAP11_bev)
assert np.allclose(mAP11_3d, expected_mAP11_3d)
assert np.allclose(mAP11_aos, expected_mAP11_aos)
assert np.allclose(mAP40_bbox, expected_mAP40_bbox)
assert np.allclose(mAP40_bev, expected_mAP40_bev)
assert np.allclose(mAP40_3d, expected_mAP40_3d)
assert np.allclose(mAP40_aos, expected_mAP40_aos)


def test_kitti_eval():
Expand Down Expand Up @@ -183,8 +201,14 @@ def test_kitti_eval():

current_classes = [1, 2, 0]
result, ret_dict = kitti_eval([gt_anno], [dt_anno], current_classes)
assert np.isclose(ret_dict['KITTI/Overall_2D_moderate'], 9.090909090909092)
assert np.isclose(ret_dict['KITTI/Overall_2D_hard'], 9.090909090909092)
assert np.isclose(ret_dict['KITTI/Overall_2D_AP11_moderate'],
9.090909090909092)
assert np.isclose(ret_dict['KITTI/Overall_2D_AP11_hard'],
9.090909090909092)
assert np.isclose(ret_dict['KITTI/Overall_2D_AP40_moderate'],
0.8333333333333334)
assert np.isclose(ret_dict['KITTI/Overall_2D_AP40_hard'],
0.8333333333333334)


def test_eval_class():
Expand Down

0 comments on commit 7c27cd7

Please sign in to comment.