From a2a2b4ea357c589d0658d758fd0e22336bf5f3d5 Mon Sep 17 00:00:00 2001 From: Nico Date: Mon, 13 Mar 2023 20:28:54 +0800 Subject: [PATCH] =?UTF-8?q?1.=20updated=20the=20rule=20in=20the=20evaluati?= =?UTF-8?q?on=20of=20different=20sized=20regions=20=20=20=20(=F0=9F=94=A5M?= =?UTF-8?q?ake=20sure=20you=20are=20using=20the=20same=20version=20of=20th?= =?UTF-8?q?e=20protocol=20when=20you=20compare=20detection=20performance?= =?UTF-8?q?=20with=20other=20methods=20on=20different=20size=20regions?= =?UTF-8?q?=F0=9F=94=A5);=202.=20updated=20sample=20threshold=20with=20pix?= =?UTF-8?q?el=20ratio;=203.=20updated=20configure=20of=20dataset=20path;?= =?UTF-8?q?=204.=20unified=20all=20metric=20names=20by=20"F1";?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 ++ baselines/libs/configurer/DEFAULT.yaml | 2 +- baselines/libs/dataset/base_datasets.py | 5 ++- baselines/libs/evaluator/evaluator.py | 3 +- .../models/seg_augment/unet_aug/unet-aug.yaml | 2 +- .../seg_fully/unet/unet-defect-group.yaml | 2 +- .../seg_fully/unet/unet-defect-total.yaml | 2 +- .../models/seg_fully/unet/unet-defect.yaml | 2 +- .../models/seg_fully/unet/unet-small.yaml | 2 +- baselines/models/seg_one_class/cae/cae.yaml | 2 +- .../seg_one_class/ocsvm_feat/ocsvm_feat.yaml | 2 +- baselines/models/seg_one_class/sc/sc.yaml | 2 +- .../seg_transfer/unet_unsup/unet-unsup.yaml | 2 +- .../models/seg_weakly/bbox/unet_bbox.yaml | 2 +- .../models/seg_weakly/label/unet_label.yaml | 2 +- dataset_api/eval_pixel.py | 4 +-- dataset_api/eval_region.py | 35 ++++++++++++------- dataset_api/eval_sample.py | 7 ++-- dataset_api/zl_eval.py | 7 ++-- 19 files changed, 50 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index a1c7861..2596b3d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +saved_models/ # exclude model checkpoints + # Created by .ignore support plugin (hsz.mobi) ### Python template # Byte-compiled / optimized / DLL files diff --git a/baselines/libs/configurer/DEFAULT.yaml b/baselines/libs/configurer/DEFAULT.yaml index 53900cf..eacda94 100644 --- a/baselines/libs/configurer/DEFAULT.yaml +++ b/baselines/libs/configurer/DEFAULT.yaml @@ -1,5 +1,5 @@ DATASET: - dataset_dir: /home/nico/Dataset/FabricFinal + dataset_dir: /home/nico/Dataset/ZJU-Leaper fabrics: all normalization: zscore color_mode: rgb diff --git a/baselines/libs/dataset/base_datasets.py b/baselines/libs/dataset/base_datasets.py index 13edf1c..15567f2 100644 --- a/baselines/libs/dataset/base_datasets.py +++ b/baselines/libs/dataset/base_datasets.py @@ -1,13 +1,12 @@ -from typing import List - import numpy as np import torch from PIL import Image, ImageOps from numpy.lib.stride_tricks import as_strided from torch.utils.data import Dataset from torchvision import transforms +from typing import List -from ..dataset_api.zl_fabric import ZLImage +from dataset_api.zl_fabric import ZLImage ''' FabricFinal RGB diff --git a/baselines/libs/evaluator/evaluator.py b/baselines/libs/evaluator/evaluator.py index 96241c2..66f8780 100644 --- a/baselines/libs/evaluator/evaluator.py +++ b/baselines/libs/evaluator/evaluator.py @@ -5,12 +5,11 @@ from collections import OrderedDict from datetime import datetime from math import isnan - from pandas import DataFrame from tensorboardX import SummaryWriter +from dataset_api import ZLEval from ..configurer import Configurer -from ..dataset_api import ZLEval class _Evaluator(ABC): diff --git a/baselines/models/seg_augment/unet_aug/unet-aug.yaml b/baselines/models/seg_augment/unet_aug/unet-aug.yaml index f3a87bf..98989fe 100644 --- a/baselines/models/seg_augment/unet_aug/unet-aug.yaml +++ b/baselines/models/seg_augment/unet_aug/unet-aug.yaml @@ -1,5 +1,5 @@ DATASET: - dataset_dir: /home/nico/Dataset/FabricFinal + dataset_dir: /home/nico/Dataset/ZJU-Leaper fabrics: all normalization: '[0,1]' color_mode: rgb diff --git a/baselines/models/seg_fully/unet/unet-defect-group.yaml b/baselines/models/seg_fully/unet/unet-defect-group.yaml index 3b3b6d3..6dc779e 100644 --- a/baselines/models/seg_fully/unet/unet-defect-group.yaml +++ b/baselines/models/seg_fully/unet/unet-defect-group.yaml @@ -1,5 +1,5 @@ DATASET: - dataset_dir: /home/nico/Dataset/FabricFinal + dataset_dir: /home/nico/Dataset/ZJU-Leaper fabrics: groups normalization: 'zscore' color_mode: rgb diff --git a/baselines/models/seg_fully/unet/unet-defect-total.yaml b/baselines/models/seg_fully/unet/unet-defect-total.yaml index 3c34a70..3558703 100644 --- a/baselines/models/seg_fully/unet/unet-defect-total.yaml +++ b/baselines/models/seg_fully/unet/unet-defect-total.yaml @@ -1,5 +1,5 @@ DATASET: - dataset_dir: /home/nico/Dataset/FabricFinal + dataset_dir: /home/nico/Dataset/ZJU-Leaper fabrics: total normalization: 'zscore' color_mode: rgb diff --git a/baselines/models/seg_fully/unet/unet-defect.yaml b/baselines/models/seg_fully/unet/unet-defect.yaml index 1958d28..68449d7 100644 --- a/baselines/models/seg_fully/unet/unet-defect.yaml +++ b/baselines/models/seg_fully/unet/unet-defect.yaml @@ -1,5 +1,5 @@ DATASET: - dataset_dir: /home/nico/Dataset/FabricFinal + dataset_dir: /home/nico/Dataset/ZJU-Leaper fabrics: all normalization: 'zscore' color_mode: rgb diff --git a/baselines/models/seg_fully/unet/unet-small.yaml b/baselines/models/seg_fully/unet/unet-small.yaml index b8cdf21..4d9b7ca 100644 --- a/baselines/models/seg_fully/unet/unet-small.yaml +++ b/baselines/models/seg_fully/unet/unet-small.yaml @@ -1,5 +1,5 @@ DATASET: - dataset_dir: /home/nico/Dataset/FabricFinal + dataset_dir: /home/nico/Dataset/ZJU-Leaper fabrics: all normalization: 'zscore' color_mode: rgb diff --git a/baselines/models/seg_one_class/cae/cae.yaml b/baselines/models/seg_one_class/cae/cae.yaml index 4a8fe1a..510f238 100644 --- a/baselines/models/seg_one_class/cae/cae.yaml +++ b/baselines/models/seg_one_class/cae/cae.yaml @@ -1,5 +1,5 @@ DATASET: - dataset_dir: /home/nico/Dataset/FabricFinal + dataset_dir: /home/nico/Dataset/ZJU-Leaper fabrics: all color_mode: gray normalization: '[-1,1]' diff --git a/baselines/models/seg_one_class/ocsvm_feat/ocsvm_feat.yaml b/baselines/models/seg_one_class/ocsvm_feat/ocsvm_feat.yaml index e06a901..9c51aac 100644 --- a/baselines/models/seg_one_class/ocsvm_feat/ocsvm_feat.yaml +++ b/baselines/models/seg_one_class/ocsvm_feat/ocsvm_feat.yaml @@ -1,5 +1,5 @@ DATASET: - dataset_dir: /home/nico/Dataset/FabricFinal + dataset_dir: /home/nico/Dataset/ZJU-Leaper fabrics: all color_mode: gray normalization: '[-1,1]' diff --git a/baselines/models/seg_one_class/sc/sc.yaml b/baselines/models/seg_one_class/sc/sc.yaml index 0206db0..10cdb15 100644 --- a/baselines/models/seg_one_class/sc/sc.yaml +++ b/baselines/models/seg_one_class/sc/sc.yaml @@ -1,5 +1,5 @@ DATASET: - dataset_dir: /home/nico/Dataset/FabricFinal + dataset_dir: /home/nico/Dataset/ZJU-Leaper fabrics: all color_mode: gray normalization: histeq diff --git a/baselines/models/seg_transfer/unet_unsup/unet-unsup.yaml b/baselines/models/seg_transfer/unet_unsup/unet-unsup.yaml index a62a4e2..f25f873 100644 --- a/baselines/models/seg_transfer/unet_unsup/unet-unsup.yaml +++ b/baselines/models/seg_transfer/unet_unsup/unet-unsup.yaml @@ -1,5 +1,5 @@ DATASET: - dataset_dir: /home/nico/Dataset/FabricFinal + dataset_dir: /home/nico/Dataset/ZJU-Leaper fabrics: all normalization: 'zscore' color_mode: rgb diff --git a/baselines/models/seg_weakly/bbox/unet_bbox.yaml b/baselines/models/seg_weakly/bbox/unet_bbox.yaml index 93617ca..ec93129 100644 --- a/baselines/models/seg_weakly/bbox/unet_bbox.yaml +++ b/baselines/models/seg_weakly/bbox/unet_bbox.yaml @@ -1,5 +1,5 @@ DATASET: - dataset_dir: /home/nico/Dataset/FabricFinal + dataset_dir: /home/nico/Dataset/ZJU-Leaper fabrics: all normalization: 'zscore' color_mode: rgb diff --git a/baselines/models/seg_weakly/label/unet_label.yaml b/baselines/models/seg_weakly/label/unet_label.yaml index 157ff37..f41d975 100644 --- a/baselines/models/seg_weakly/label/unet_label.yaml +++ b/baselines/models/seg_weakly/label/unet_label.yaml @@ -1,5 +1,5 @@ DATASET: - dataset_dir: /home/nico/Dataset/FabricFinal + dataset_dir: /home/nico/Dataset/ZJU-Leaper fabrics: all normalization: 'zscore' color_mode: rgb diff --git a/dataset_api/eval_pixel.py b/dataset_api/eval_pixel.py index f1e5b03..b402d88 100644 --- a/dataset_api/eval_pixel.py +++ b/dataset_api/eval_pixel.py @@ -3,9 +3,8 @@ @Time : 2019-05-07 23:53 @Author : Nico """ -from collections import OrderedDict - import numpy as np +from collections import OrderedDict from typing import Tuple @@ -86,6 +85,7 @@ def evaluation_pixel(binary_pixel_pred: np.ndarray, binary_pixel_target: np.ndar Pre=precision, Rec=recall, Dice=dice, + F1=dice, # dice index is just the pixel F1-score # IoU=_iou(confusion), # nMCC=_mcc(confusion, normalized=True), ) diff --git a/dataset_api/eval_region.py b/dataset_api/eval_region.py index 205b022..eb251d6 100644 --- a/dataset_api/eval_region.py +++ b/dataset_api/eval_region.py @@ -5,10 +5,9 @@ """ import itertools -import warnings - import numpy as np import pandas as pd +import warnings from skimage.measure import label, regionprops from skimage.measure._regionprops import RegionProperties @@ -170,21 +169,33 @@ def _precision_for_region_pred(info_pred_overlap: pd.DataFrame, iou_thresh: floa :param iou_thresh: :return: """ + # if scale == 'all': + # pass + # elif scale == 'small': + # info_pred_overlap = info_pred_overlap[ + # (info_pred_overlap['area_target'] <= SMALL_TH) + # | (info_pred_overlap['id_target'].isna() & (info_pred_overlap['area_pred'] <= SMALL_TH))] + # elif scale == 'medium': + # info_pred_overlap = info_pred_overlap[ + # ((SMALL_TH < info_pred_overlap['area_target']) & (info_pred_overlap['area_target'] < MEDIUM_TH)) + # | (info_pred_overlap['id_target'].isna() + # & ((SMALL_TH < info_pred_overlap['area_pred']) & (info_pred_overlap['area_pred'] < MEDIUM_TH)))] + # elif scale == 'large': + # info_pred_overlap = info_pred_overlap[ + # (info_pred_overlap['area_target'] >= MEDIUM_TH) + # | (info_pred_overlap['id_target'].isna() & (info_pred_overlap['area_pred'] >= MEDIUM_TH))] + # else: + # raise NotImplementedError + + # NEW: the size constrains only related to the ground-truth if scale == 'all': pass elif scale == 'small': - info_pred_overlap = info_pred_overlap[ - (info_pred_overlap['area_target'] <= SMALL_TH) - | (info_pred_overlap['id_target'].isna() & (info_pred_overlap['area_pred'] <= SMALL_TH))] + info_pred_overlap = info_pred_overlap[info_pred_overlap['area_target'] < SMALL_TH] elif scale == 'medium': - info_pred_overlap = info_pred_overlap[ - ((SMALL_TH < info_pred_overlap['area_target']) & (info_pred_overlap['area_target'] < MEDIUM_TH)) - | (info_pred_overlap['id_target'].isna() - & ((SMALL_TH < info_pred_overlap['area_pred']) & (info_pred_overlap['area_pred'] < MEDIUM_TH)))] + info_pred_overlap = info_pred_overlap[((SMALL_TH <= info_pred_overlap['area_target']) & (info_pred_overlap['area_target'] <= MEDIUM_TH))] elif scale == 'large': - info_pred_overlap = info_pred_overlap[ - (info_pred_overlap['area_target'] >= MEDIUM_TH) - | (info_pred_overlap['id_target'].isna() & (info_pred_overlap['area_pred'] >= MEDIUM_TH))] + info_pred_overlap = info_pred_overlap[info_pred_overlap['area_target'] > MEDIUM_TH] else: raise NotImplementedError diff --git a/dataset_api/eval_sample.py b/dataset_api/eval_sample.py index b1ff748..d60ecd2 100644 --- a/dataset_api/eval_sample.py +++ b/dataset_api/eval_sample.py @@ -9,7 +9,8 @@ from .eval_pixel import _compute_confusion_matrix, _f1_score, _acc, _fpr -MIN_AREA = 4 +# the size constrain is according to pixel ratio +MIN_AREA_RATIO = 1. / 128. def evaluation_sample(binary_pixel_pred, binary_pixel_target, info_region): @@ -19,10 +20,12 @@ def evaluation_sample(binary_pixel_pred, binary_pixel_target, info_region): info_overlap: pd.DataFrame = info_region["info_overlap"] num_img = len(binary_pixel_target) + _, h, w = binary_pixel_target.squeeze().shape + MIN_AREA_TH = (np.float_(h) * MIN_AREA_RATIO) * (np.float_(w) * MIN_AREA_RATIO) label_pred = np.zeros(num_img, dtype=np.bool) if not info_pred.empty: - img_pred_ids = info_pred[info_pred['area_pred'] > MIN_AREA]['id_img'].unique() + img_pred_ids = info_pred[info_pred['area_pred'] > MIN_AREA_TH]['id_img'].unique() label_pred[img_pred_ids] = 1 label_target = np.zeros(num_img, dtype=np.bool) diff --git a/dataset_api/zl_eval.py b/dataset_api/zl_eval.py index eba80ef..9b7f4ba 100644 --- a/dataset_api/zl_eval.py +++ b/dataset_api/zl_eval.py @@ -4,11 +4,10 @@ @Author : Nico """ import json +import numpy as np import warnings from collections import OrderedDict -import numpy as np - from .eval_pixel import evaluation_pixel from .eval_region import evaluation_region from .eval_sample import evaluation_sample @@ -56,12 +55,12 @@ def evaluate(self) -> dict: diff_size=self.eval_diff_size, return_info=True) sample_metrics = evaluation_sample(binary_pixel_pred=self.binary_pixel_pred, binary_pixel_target=self.binary_pixel_target, info_region=info_region) - summary_score = 0.4 * pixel_metrics['Dice'] + 0.4 * region_metrics['F1'] + 0.2 * sample_metrics['F1'] + summary_score = 0.4 * pixel_metrics['F1'] + 0.4 * region_metrics['F1'] + 0.2 * sample_metrics['F1'] self.metrics_dict = OrderedDict( # Pix_Pre=pixel_metrics['Pre'], # Pix_Rec=pixel_metrics['Rec'], - F1_Pix=pixel_metrics['Dice'], + F1_Pix=pixel_metrics['F1'], # Reg_Pre=region_metrics['Pre'], # Reg_Rec=region_metrics['Rec'],