Possible to get an MMDetection Confusion Matrix?

[Trotts]

Hi there,

I have an MMDetection model which I can load and run inference with SHAI. The model is trained on tiled imagery (e.g. 250x250, 0.5 overlap). I have ground truth data in COCO JSON format for both the tiled and non-tiled images. I know that it is possible to generate confusion matrices using MMDetection's confusion matrix script, but I was wondering if it was possible to generate a confusion matrix when the model is ran through SAHI when comparing to the non-tiled test set JSON? This would be a big help, as it would allow me to compare model performance with/without SAHI.

Thank you :)

[Trotts]

Hi,

Following on from the above, I have started working on code to run non-tiled images through SAHI via an MMDetection model (trained on tiled imagery), convert the results into a form as required by MMDetection's confusion matrix code, and then outputting a final confusion matrix.

Currently, the code runs without error, however the confusion matrix outputted is not correct. After a fair bit of debugging, I'm wondering if the format of the adapted results JSON from SAHI is correct. Any help with this is greatly appreciated!

import sahi
import glob
import json
import torch
from sahi import AutoDetectionModel
from sahi.predict import predict
import numpy as np
from mmdetection.tools.analysis_tools.confusion_matrix import calculate_confusion_matrix
from mmengine import Config
from mmdet.utils import replace_cfg_vals, update_data_root
from mmengine.registry import init_default_scope
from mmdet.registry import DATASETS
from mmdetection.tools.analysis_tools.confusion_matrix import plot_confusion_matrix


# Locate the trained, tiled, MMDetection model
model_name = 'BENTHIC_PATCHES_OFFLINE_250_250_05_OVERLAP_FASTER_RCNN_25_CLASS_SAHI_TEST'
config_src = './mmdetection/configs/faster_rcnn/' + model_name + '.py'
checkpoints_src = './work_dirs/' + model_name
best_checkpoint = glob.glob(checkpoints_src + '/best_coco_bbox_mAP_epoch*.pth')[0]

# Locate the non-tiled test images JSON
img_src = './whole_images/'
whole_image_test_dataset_json_path = img_src + '/annotations/dataset_test.json'

# Required options
bbox_conf_threshold = 0.75
patch_size = (250, 250)
overlap = 0.5

# Load the MMDetection model
model = AutoDetectionModel.from_pretrained(model_type = 'mmdet',
                           model_path = best_checkpoint,
                           config_path = config_src,
                           confidence_threshold = bbox_conf_threshold,
                           device = 'cuda')

# Run prediction on the non-tiled images, save out in JSON format
output_name = 'sahi_eval_results' # output_dir/output_name

predict(
    model_type = 'mmdet',
    model_path = best_checkpoint,
    model_device = 'cuda',
    model_confidence_threshold = bbox_conf_threshold,
    source = img_src,
    slice_height = patch_size[0],
    slice_width = patch_size[1],
    overlap_height_ratio = overlap,
    overlap_width_ratio = overlap,
    postprocess_type = 'NMM',
    postprocess_match_metric = 'IOU',
    dataset_json_path = whole_image_test_dataset_json_path,
    visual_bbox_thickness = 2,
    project = checkpoints_src,
    name = output_name,
    novisual = True
)

# Load in the predicted results
results_json = checkpoints_src + '/' + output_name + '/result.json'
results_json = json.load(open(results_json))

# Get the results into a format required by the MMDetection conf matrix scripts
def group_detections_by_image_id(json):
    grouped = {}
    for i, item in enumerate(json):
        image_id = item['image_id']
        if image_id not in grouped:
            grouped[image_id] = []
        grouped[image_id].append(item)        
    return grouped

results_grouped = group_detections_by_image_id(results_json)

results_grouped_mmdet = []
for per_img_res in results_grouped:
    per_img_res_mmdet = {}
    per_img_res_mmdet['image_id'] = per_img_res
    per_img_res_mmdet['pred_instances'] = {}
    per_img_res_mmdet['pred_instances']['bboxes'] = torch.tensor([item['bbox'] for item in results_grouped[per_img_res]])
    per_img_res_mmdet['pred_instances']['scores'] = torch.tensor([item['score'] for item in results_grouped[per_img_res]])
    per_img_res_mmdet['pred_instances']['labels'] = torch.tensor([item['category_id'] for item in results_grouped[per_img_res]])
    
    results_grouped_mmdet.append(per_img_res_mmdet)

# Calculate the conf matrix adapting https://github.com/open-mmlab/mmdetection/blob/main/tools/analysis_tools/confusion_matrix.py

def conf_mat(config_src,
                               results,
                               score_thr = 0,
                               tp_iou_thr = 0.5):
    
    cfg = Config.fromfile(config_src)
    cfg = replace_cfg_vals(cfg)
    update_data_root(cfg)
    
    init_default_scope(cfg.get('default_scope', 'mmdet'))
    
    # The MMDetection config test_dataloader points to the non-tiled images (train and val points to tiled)
    dataset = DATASETS.build(cfg.test_dataloader.dataset)
    
    confusion_matrix = calculate_confusion_matrix(
        dataset, results, score_thr= score_thr, tp_iou_thr= tp_iou_thr
    )
    return confusion_matrix

conf = conf_mat(config_src,
                           results_grouped_mmdet,
                           score_thr = bbox_conf_threshold,
                           tp_iou_thr= 0.5)

# Get labels for plotting purposes
cfg = Config.fromfile(config_src)
cfg = replace_cfg_vals(cfg)
update_data_root(cfg)
init_default_scope(cfg.get('default_scope', 'mmdet'))
dataset = DATASETS.build(cfg.test_dataloader.dataset)
labels_dataset = dataset.metainfo['classes']
labels_dataset = labels_dataset + ['background']

# Plot
plot_confusion_matrix(conf, labels_dataset, show = True)

The above code produces a confusion matrix which looks as follows:

Based on the visualised results, the model performs much better than is displayed by the confusion matrix.

[Trotts]

Update:

I managed to fix the above code. The issue was the formatting of the bounding boxes, with SAHI exporting as [x_top_left, y_top_left, w, h] and my ground truth data stored as [x_top_left, y_top_left, x_bottom_right, y_bottom_right]. I assume when SAHI visualises the ground truth data this conversion is done on the fly as needed?

Updated code for those who may come across this thread:

import sahi
import glob
import json
import torch
from sahi import AutoDetectionModel
from sahi.predict import predict

from mmdetection.tools.analysis_tools.confusion_matrix import calculate_confusion_matrix
from mmengine import Config
from mmdet.utils import replace_cfg_vals, update_data_root
from mmengine.registry import init_default_scope
from mmdet.registry import DATASETS
from mmdetection.tools.analysis_tools.confusion_matrix import plot_confusion_matrix

# Locate the trained, tiled, MMDetection model
model_name = 'BENTHIC_PATCHES_OFFLINE_250_250_05_OVERLAP_FASTER_RCNN_25_CLASS_SAHI_TEST'
config_src = './mmdetection/configs/faster_rcnn/' + model_name + '.py'
checkpoints_src = './work_dirs/' + model_name
best_checkpoint = glob.glob(checkpoints_src + '/best_coco_bbox_mAP_epoch*.pth')[0]

# Locate the non-tiled test images JSON
img_src = './whole_images/'
whole_image_test_dataset_json_path = img_src + '/annotations/dataset_test.json'

# Required options
bbox_conf_threshold = 0.75
patch_size = (250, 250)
overlap = 0.5

# Load the MMDetection model
model = AutoDetectionModel.from_pretrained(model_type = 'mmdet',
                           model_path = best_checkpoint,
                           config_path = config_src,
                           confidence_threshold = bbox_conf_threshold,
                           device = 'cuda')

# Run prediction on the non-tiled images, save out in JSON format
output_name = 'sahi_eval_results' # output_dir/output_name

predict(
    model_type = 'mmdet',
    model_path = best_checkpoint,
    model_device = 'cuda',
    model_confidence_threshold = bbox_conf_threshold,
    source = img_src,
    slice_height = patch_size[0],
    slice_width = patch_size[1],
    overlap_height_ratio = overlap,
    overlap_width_ratio = overlap,
    postprocess_type = 'NMM',
    postprocess_match_metric = 'IOU',
    dataset_json_path = whole_image_test_dataset_json_path,
    visual_bbox_thickness = 2,
    project = checkpoints_src,
    name = output_name,
    novisual = True
)

# Load in the predicted results
results_json = checkpoints_src + '/' + output_name + '/result.json'
results_json = json.load(open(results_json))

# Group the detections by image id
def group_detections_by_image_id(json):
    grouped = {}
    for i, item in enumerate(json):
        image_id = item['image_id']
        if image_id not in grouped:
            grouped[image_id] = []
        grouped[image_id].append(item)        
    return grouped

results_grouped = group_detections_by_image_id(results_json)

### Get into format for MMDetection eval scripts
results_grouped_mmdet = []
for per_img_res in results_grouped:
    per_img_res_mmdet = {}
    per_img_res_mmdet['image_id'] = per_img_res
    per_img_res_mmdet['pred_instances'] = {}
    
    # SAHI outputs results in form [x_min, y_min, w, h] but Ground Truth expects [x_min, y_min, x_max, y_max]
    new_bbox = []
    for item in results_grouped[per_img_res]:
        new_bbox.append([item['bbox'][0], item['bbox'][1], item['bbox'][0] + item['bbox'][2], item['bbox'][1] + item['bbox'][3]])
    per_img_res_mmdet['pred_instances']['bboxes'] = torch.tensor(new_bbox)
    
    per_img_res_mmdet['pred_instances']['scores'] = torch.tensor([item['score'] for item in results_grouped[per_img_res]])
    per_img_res_mmdet['pred_instances']['labels'] = torch.tensor([item['category_id'] for item in results_grouped[per_img_res]])
    
    results_grouped_mmdet.append(per_img_res_mmdet)

### Calculate conf matrix adapting from MMDetection analysis_tools/confusion_matrix.py

def build_dataset(cfg_src):
    cfg = Config.fromfile(cfg_src)
    cfg = replace_cfg_vals(cfg)
    update_data_root(cfg)
    init_default_scope(cfg.get('default_scope', 'mmdet'))
    dataset = DATASETS.build(cfg.test_dataloader.dataset)
    return dataset


def conf_mat(dataset, results, score_thr = 0, tp_iou_thr = 0.5):
    
    # Get the confusion matrix
    confusion_matrix = calculate_confusion_matrix(
        dataset, results, score_thr= score_thr, tp_iou_thr= tp_iou_thr
    )
    return confusion_matrix        

dataset = build_dataset(config_src)

conf = conf_mat(dataset,
                results_grouped_mmdet,
                score_thr = bbox_conf_threshold,
                tp_iou_thr= 0.5)

labels_dataset = dataset.metainfo['classes'] + ['background']
plot_confusion_matrix(conf, labels_dataset, show = True)

Produces as output: