From eec5f23b26780435bfff417464973da17bf594c5 Mon Sep 17 00:00:00 2001 From: Jiahao Xie <52497952+Jiahao000@users.noreply.github.com> Date: Tue, 29 Mar 2022 15:21:09 +0800 Subject: [PATCH] [Feature] support KNN benchmark (#243) * [Feature] support KNN benchmark * [Fix] add docstring and multi-machine testing * [Fix] fix lint * [Fix] change args format and check init_cfg * [Docs] add benchmark tutorial * [Docs] add benchmark results --- .../benchmarks/classification/knn_imagenet.py | 29 +++ configs/selfsup/byol/README.md | 13 +- configs/selfsup/deepcluster/README.md | 4 +- configs/selfsup/densecl/README.md | 10 +- configs/selfsup/mae/README.md | 2 +- configs/selfsup/mocov2/README.md | 10 +- configs/selfsup/mocov3/README.md | 4 +- configs/selfsup/npid/README.md | 10 +- configs/selfsup/odc/README.md | 12 +- configs/selfsup/relative_loc/README.md | 12 +- configs/selfsup/rotation_pred/README.md | 12 +- configs/selfsup/simclr/README.md | 12 +- configs/selfsup/simsiam/README.md | 13 +- configs/selfsup/swav/README.md | 12 +- docs/en/model_zoo.md | 13 +- docs/en/tutorials/6_benchmarks.md | 78 ++++++-- docs/zh_cn/model_zoo.md | 12 +- docs/zh_cn/tutorials/6_benchmarks.md | 79 +++++--- mmselfsup/models/utils/__init__.py | 8 +- mmselfsup/models/utils/extract_process.py | 44 ++++- mmselfsup/models/utils/knn_classifier.py | 70 +++++++ .../test_utils/test_knn_classifier.py | 17 ++ tests/test_runtime/test_extract_process.py | 20 +- tools/analysis_tools/visualize_tsne.py | 4 +- .../knn_imagenet/dist_test_knn_epoch.sh | 30 +++ .../knn_imagenet/dist_test_knn_pretrain.sh | 27 +++ .../knn_imagenet/slurm_test_knn_epoch.sh | 36 ++++ .../knn_imagenet/slurm_test_knn_pretrain.sh | 33 ++++ .../classification/knn_imagenet/test_knn.py | 186 ++++++++++++++++++ .../classification/svm_voc07/extract.py | 4 +- 30 files changed, 726 insertions(+), 90 deletions(-) create mode 100644 configs/benchmarks/classification/knn_imagenet.py create mode 100644 mmselfsup/models/utils/knn_classifier.py create mode 100644 tests/test_models/test_utils/test_knn_classifier.py create mode 100644 tools/benchmarks/classification/knn_imagenet/dist_test_knn_epoch.sh create mode 100644 tools/benchmarks/classification/knn_imagenet/dist_test_knn_pretrain.sh create mode 100644 tools/benchmarks/classification/knn_imagenet/slurm_test_knn_epoch.sh create mode 100644 tools/benchmarks/classification/knn_imagenet/slurm_test_knn_pretrain.sh create mode 100644 tools/benchmarks/classification/knn_imagenet/test_knn.py diff --git a/configs/benchmarks/classification/knn_imagenet.py b/configs/benchmarks/classification/knn_imagenet.py new file mode 100644 index 000000000..2d694136b --- /dev/null +++ b/configs/benchmarks/classification/knn_imagenet.py @@ -0,0 +1,29 @@ +data_source = 'ImageNet' +dataset_type = 'SingleViewDataset' +img_norm_cfg = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) +pipeline = [ + dict(type='Resize', size=256), + dict(type='CenterCrop', size=224), + dict(type='ToTensor'), + dict(type='Normalize', **img_norm_cfg), +] + +data = dict( + samples_per_gpu=256, + workers_per_gpu=8, + train=dict( + type=dataset_type, + data_source=dict( + type=data_source, + data_prefix='data/imagenet/train', + ann_file='data/imagenet/meta/train.txt', + ), + pipeline=pipeline), + val=dict( + type=dataset_type, + data_source=dict( + type=data_source, + data_prefix='data/imagenet/val', + ann_file='data/imagenet/meta/val.txt', + ), + pipeline=pipeline)) diff --git a/configs/selfsup/byol/README.md b/configs/selfsup/byol/README.md index 93e2e16c9..988972b2d 100644 --- a/configs/selfsup/byol/README.md +++ b/configs/selfsup/byol/README.md @@ -16,11 +16,11 @@ **Back to [model_zoo.md](https://github.com/open-mmlab/mmselfsup/blob/master/docs/en/model_zoo.md) to download models.** -In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models were trained on ImageNet1k dataset. +In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models are pre-trained on ImageNet-1k dataset. ### Classification -The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). +The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). #### VOC SVM / Low-shot SVM @@ -51,6 +51,15 @@ The **Feature1 - Feature5** don't have the GlobalAveragePooling, the feature map | [resnet50_8xb32-accum16-coslr-200e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/byol/byol_resnet50_8xb32-accum16-coslr-200e_in1k.py) | 21.25 | 36.55 | 43.66 | 50.74 | 53.82 | | [resnet50_8xb32-accum16-coslr-300e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/byol/byol_resnet50_8xb32-accum16-coslr-300e_in1k.py) | 21.18 | 36.68 | 43.42 | 51.04 | 54.06 | +#### ImageNet Nearest-Neighbor Classification + +The results are obtained from the features after GlobalAveragePooling. Here, k=10 to 200 indicates different number of nearest neighbors. + +| Self-Supervised Config | k=10 | k=20 | k=100 | k=200 | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---- | ---- | ----- | ----- | +| [resnet50_8xb32-accum16-coslr-200e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/byol/byol_resnet50_8xb32-accum16-coslr-200e_in1k.py) | 63.9 | 64.2 | 62.9 | 61.9 | +| [resnet50_8xb32-accum16-coslr-300e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/byol/byol_resnet50_8xb32-accum16-coslr-300e_in1k.py) | 66.1 | 66.3 | 65.2 | 64.4 | + ### Detection The detection benchmarks includes 2 downstream task datasets, **Pascal VOC 2007 + 2012** and **COCO2017**. This benchmark follows the evluation protocols set up by MoCo. diff --git a/configs/selfsup/deepcluster/README.md b/configs/selfsup/deepcluster/README.md index 976fba298..f11f32a54 100644 --- a/configs/selfsup/deepcluster/README.md +++ b/configs/selfsup/deepcluster/README.md @@ -16,11 +16,11 @@ Clustering is a class of unsupervised learning methods that has been extensively **Back to [model_zoo.md](https://github.com/open-mmlab/mmselfsup/blob/master/docs/en/model_zoo.md) to download models.** -In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models were trained on ImageNet1k dataset. +In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models are pre-trained on ImageNet-1k dataset. ### Classification -The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). +The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). #### VOC SVM / Low-shot SVM diff --git a/configs/selfsup/densecl/README.md b/configs/selfsup/densecl/README.md index adaac4425..ed12476d5 100644 --- a/configs/selfsup/densecl/README.md +++ b/configs/selfsup/densecl/README.md @@ -16,7 +16,7 @@ To date, most existing self-supervised learning methods are designed and optimiz **Back to [model_zoo.md](https://github.com/open-mmlab/mmselfsup/blob/master/docs/en/model_zoo.md) to download models.** -In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models were trained on ImageNet1k dataset. +In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models are pre-trained on ImageNet-1k dataset. ### Classification @@ -50,6 +50,14 @@ The **Feature1 - Feature5** don't have the GlobalAveragePooling, the feature map | -------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -------- | -------- | -------- | -------- | | [resnet50_8xb32-coslr-200e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/densecl/densecl_resnet50_8xb32-coslr-200e_in1k.py) | 21.32 | 36.20 | 43.97 | 51.04 | 50.45 | +#### ImageNet Nearest-Neighbor Classification + +The results are obtained from the features after GlobalAveragePooling. Here, k=10 to 200 indicates different number of nearest neighbors. + +| Self-Supervised Config | k=10 | k=20 | k=100 | k=200 | +| -------------------------------------------------------------------------------------------------------------------------------------------------- | ---- | ---- | ----- | ----- | +| [resnet50_8xb32-coslr-200e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/densecl/densecl_resnet50_8xb32-coslr-200e_in1k.py) | 48.2 | 48.5 | 46.8 | 45.6 | + ### Detection The detection benchmarks includes 2 downstream task datasets, **Pascal VOC 2007 + 2012** and **COCO2017**. This benchmark follows the evluation protocols set up by MoCo. diff --git a/configs/selfsup/mae/README.md b/configs/selfsup/mae/README.md index 88b97d57a..3831bee9b 100644 --- a/configs/selfsup/mae/README.md +++ b/configs/selfsup/mae/README.md @@ -30,7 +30,7 @@ methods that use only ImageNet-1K data. Transfer performance in downstream tasks ## Models and Benchmarks -Here, we report the results of the model, which is pre-trained on ImageNet1K +Here, we report the results of the model, which is pre-trained on ImageNet-1k for 400 epochs, the details are below: diff --git a/configs/selfsup/mocov2/README.md b/configs/selfsup/mocov2/README.md index b42f16511..7014b5a2f 100644 --- a/configs/selfsup/mocov2/README.md +++ b/configs/selfsup/mocov2/README.md @@ -16,7 +16,7 @@ Contrastive unsupervised learning has recently shown encouraging progress, e.g., **Back to [model_zoo.md](https://github.com/open-mmlab/mmselfsup/blob/master/docs/en/model_zoo.md) to download models.** -In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models were trained on ImageNet1k dataset. +In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models are pre-trained on ImageNet-1k dataset. ### Classification @@ -50,6 +50,14 @@ The **Feature1 - Feature5** don't have the GlobalAveragePooling, the feature map | ------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | -------- | -------- | -------- | -------- | | [resnet50_8xb32-coslr-200e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mocov2/mocov2_resnet50_8xb32-coslr-200e_in1k.py) | 20.92 | 35.72 | 42.62 | 49.79 | 52.25 | +#### ImageNet Nearest-Neighbor Classification + +The results are obtained from the features after GlobalAveragePooling. Here, k=10 to 200 indicates different number of nearest neighbors. + +| Self-Supervised Config | k=10 | k=20 | k=100 | k=200 | +| ------------------------------------------------------------------------------------------------------------------------------------------------ | ---- | ---- | ----- | ----- | +| [resnet50_8xb32-coslr-200e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mocov2/mocov2_resnet50_8xb32-coslr-200e_in1k.py) | 55.6 | 55.7 | 53.7 | 52.5 | + ### Detection The detection benchmarks includes 2 downstream task datasets, **Pascal VOC 2007 + 2012** and **COCO2017**. This benchmark follows the evluation protocols set up by MoCo. diff --git a/configs/selfsup/mocov3/README.md b/configs/selfsup/mocov3/README.md index 2030044b8..e80646d57 100644 --- a/configs/selfsup/mocov3/README.md +++ b/configs/selfsup/mocov3/README.md @@ -16,11 +16,11 @@ This paper does not describe a novel method. Instead, it studies a straightforwa **Back to [model_zoo.md](https://github.com/open-mmlab/mmselfsup/blob/master/docs/en/model_zoo.md) to download models.** -In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models were trained on ImageNet1k dataset. +In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models are pre-trained on ImageNet-1k dataset. ### Classification -The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). +The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). #### ImageNet Linear Evaluation diff --git a/configs/selfsup/npid/README.md b/configs/selfsup/npid/README.md index 556387bd9..3e9462b56 100644 --- a/configs/selfsup/npid/README.md +++ b/configs/selfsup/npid/README.md @@ -20,7 +20,7 @@ Our method is also remarkable for consistently improving test performance with m **Back to [model_zoo.md](https://github.com/open-mmlab/mmselfsup/blob/master/docs/en/model_zoo.md) to download models.** -In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models were trained on ImageNet1k dataset. +In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models are pre-trained on ImageNet-1k dataset. ### Classification @@ -54,6 +54,14 @@ The **Feature1 - Feature5** don't have the GlobalAveragePooling, the feature map | ---------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -------- | -------- | -------- | -------- | | [resnet50_8xb32-steplr-200e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/npid/npid_resnet50_8xb32-steplr-200e_in1k.py) | 19.98 | 34.86 | 41.59 | 48.43 | 48.71 | +#### ImageNet Nearest-Neighbor Classification + +The results are obtained from the features after GlobalAveragePooling. Here, k=10 to 200 indicates different number of nearest neighbors. + +| Self-Supervised Config | k=10 | k=20 | k=100 | k=200 | +| ---------------------------------------------------------------------------------------------------------------------------------------------- | ---- | ---- | ----- | ----- | +| [resnet50_8xb32-steplr-200e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/npid/npid_resnet50_8xb32-steplr-200e_in1k.py) | 42.9 | 44.0 | 43.2 | 42.2 | + ### Detection The detection benchmarks includes 2 downstream task datasets, **Pascal VOC 2007 + 2012** and **COCO2017**. This benchmark follows the evluation protocols set up by MoCo. diff --git a/configs/selfsup/odc/README.md b/configs/selfsup/odc/README.md index 473e36183..dc1d9b597 100644 --- a/configs/selfsup/odc/README.md +++ b/configs/selfsup/odc/README.md @@ -16,11 +16,11 @@ Joint clustering and feature learning methods have shown remarkable performance **Back to [model_zoo.md](https://github.com/open-mmlab/mmselfsup/blob/master/docs/en/model_zoo.md) to download models.** -In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models were trained on ImageNet1k dataset. +In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models are pre-trained on ImageNet-1k dataset. ### Classification -The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). +The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). #### VOC SVM / Low-shot SVM @@ -50,6 +50,14 @@ The **Feature1 - Feature5** don't have the GlobalAveragePooling, the feature map | -------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -------- | -------- | -------- | -------- | | [resnet50_8xb64-steplr-440e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/odc/odc_resnet50_8xb64-steplr-440e_in1k.py) | 19.28 | 34.09 | 40.90 | 47.04 | 48.35 | +#### ImageNet Nearest-Neighbor Classification + +The results are obtained from the features after GlobalAveragePooling. Here, k=10 to 200 indicates different number of nearest neighbors. + +| Self-Supervised Config | k=10 | k=20 | k=100 | k=200 | +| -------------------------------------------------------------------------------------------------------------------------------------------- | ---- | ---- | ----- | ----- | +| [resnet50_8xb64-steplr-440e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/odc/odc_resnet50_8xb64-steplr-440e_in1k.py) | 38.5 | 39.1 | 37.8 | 36.9 | + ## Citation ```bibtex diff --git a/configs/selfsup/relative_loc/README.md b/configs/selfsup/relative_loc/README.md index 1b8dc7d31..9529647ad 100644 --- a/configs/selfsup/relative_loc/README.md +++ b/configs/selfsup/relative_loc/README.md @@ -16,11 +16,11 @@ This work explores the use of spatial context as a source of free and plentiful **Back to [model_zoo.md](https://github.com/open-mmlab/mmselfsup/blob/master/docs/en/model_zoo.md) to download models.** -In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models were trained on ImageNet1k dataset. +In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models are pre-trained on ImageNet-1k dataset. ### Classification -The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). +The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). #### VOC SVM / Low-shot SVM @@ -50,6 +50,14 @@ The **Feature1 - Feature5** don't have the GlobalAveragePooling, the feature map | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | -------- | -------- | -------- | -------- | | [resnet50_8xb64-steplr-70e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/relative_loc/relative-loc_resnet50_8xb64-steplr-70e_in1k.py) | 20.69 | 34.72 | 43.01 | 45.97 | 41.96 | +#### ImageNet Nearest-Neighbor Classification + +The results are obtained from the features after GlobalAveragePooling. Here, k=10 to 200 indicates different number of nearest neighbors. + +| Self-Supervised Config | k=10 | k=20 | k=100 | k=200 | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---- | ---- | ----- | ----- | +| [resnet50_8xb64-steplr-70e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/relative_loc/relative-loc_resnet50_8xb64-steplr-70e_in1k.py) | 14.5 | 15.0 | 15.0 | 14.2 | + ### Detection The detection benchmarks includes 2 downstream task datasets, **Pascal VOC 2007 + 2012** and **COCO2017**. This benchmark follows the evluation protocols set up by MoCo. diff --git a/configs/selfsup/rotation_pred/README.md b/configs/selfsup/rotation_pred/README.md index c6961fc74..97e9d5a97 100644 --- a/configs/selfsup/rotation_pred/README.md +++ b/configs/selfsup/rotation_pred/README.md @@ -16,11 +16,11 @@ Over the last years, deep convolutional neural networks (ConvNets) have transfor **Back to [model_zoo.md](https://github.com/open-mmlab/mmselfsup/blob/master/docs/en/model_zoo.md) to download models.** -In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models were trained on ImageNet1k dataset. +In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models are pre-trained on ImageNet-1k dataset. ### Classification -The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). +The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). #### VOC SVM / Low-shot SVM @@ -50,6 +50,14 @@ The **Feature1 - Feature5** don't have the GlobalAveragePooling, the feature map | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -------- | -------- | -------- | -------- | | [resnet50_8xb16-steplr-70e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/rotation_pred/rotation-pred_resnet50_8xb16-steplr-70e_in1k.py) | 18.94 | 34.72 | 44.53 | 46.30 | 44.12 | +#### ImageNet Nearest-Neighbor Classification + +The results are obtained from the features after GlobalAveragePooling. Here, k=10 to 200 indicates different number of nearest neighbors. + +| Self-Supervised Config | k=10 | k=20 | k=100 | k=200 | +| -------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---- | ---- | ----- | ----- | +| [resnet50_8xb16-steplr-70e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/rotation_pred/rotation-pred_resnet50_8xb16-steplr-70e_in1k.py) | 11.0 | 11.9 | 12.6 | 12.4 | + ### Detection The detection benchmarks includes 2 downstream task datasets, **Pascal VOC 2007 + 2012** and **COCO2017**. This benchmark follows the evluation protocols set up by MoCo. diff --git a/configs/selfsup/simclr/README.md b/configs/selfsup/simclr/README.md index c013eafa3..93be60128 100644 --- a/configs/selfsup/simclr/README.md +++ b/configs/selfsup/simclr/README.md @@ -16,11 +16,11 @@ This paper presents SimCLR: a simple framework for contrastive learning of visua **Back to [model_zoo.md](https://github.com/open-mmlab/mmselfsup/blob/master/docs/en/model_zoo.md) to download models.** -In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models were trained on ImageNet1k dataset. +In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models are pre-trained on ImageNet-1k dataset. ### Classification -The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). +The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). #### VOC SVM / Low-shot SVM @@ -50,6 +50,14 @@ The **Feature1 - Feature5** don't have the GlobalAveragePooling, the feature map | ------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | -------- | -------- | -------- | -------- | | [resnet50_8xb32-coslr-200e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k.py) | 20.60 | 33.62 | 38.86 | 45.25 | 50.91 | +#### ImageNet Nearest-Neighbor Classification + +The results are obtained from the features after GlobalAveragePooling. Here, k=10 to 200 indicates different number of nearest neighbors. + +| Self-Supervised Config | k=10 | k=20 | k=100 | k=200 | +| ------------------------------------------------------------------------------------------------------------------------------------------------ | ---- | ---- | ----- | ----- | +| [resnet50_8xb32-coslr-200e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k.py) | 47.8 | 48.4 | 46.7 | 45.2 | + ### Detection The detection benchmarks includes 2 downstream task datasets, **Pascal VOC 2007 + 2012** and **COCO2017**. This benchmark follows the evluation protocols set up by MoCo. diff --git a/configs/selfsup/simsiam/README.md b/configs/selfsup/simsiam/README.md index 3d783e61e..2d4f9356e 100644 --- a/configs/selfsup/simsiam/README.md +++ b/configs/selfsup/simsiam/README.md @@ -16,11 +16,11 @@ Siamese networks have become a common structure in various recent models for uns **Back to [model_zoo.md](https://github.com/open-mmlab/mmselfsup/blob/master/docs/en/model_zoo.md) to download models.** -In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models were trained on ImageNet1k dataset. +In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models are pre-trained on ImageNet-1k dataset. ### Classification -The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). +The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). #### VOC SVM / Low-shot SVM @@ -53,6 +53,15 @@ The **Feature1 - Feature5** don't have the GlobalAveragePooling, the feature map | [resnet50_8xb32-coslr-100e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simsiam/simsiam_resnet50_8xb32-coslr-100e_in1k.py) | 21.32 | 35.66 | 43.05 | 50.79 | 53.27 | | [resnet50_8xb32-coslr-200e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simsiam/simsiam_resnet50_8xb32-coslr-200e_in1k.py) | 21.17 | 35.85 | 43.49 | 50.99 | 54.10 | +#### ImageNet Nearest-Neighbor Classification + +The results are obtained from the features after GlobalAveragePooling. Here, k=10 to 200 indicates different number of nearest neighbors. + +| Self-Supervised Config | k=10 | k=20 | k=100 | k=200 | +| -------------------------------------------------------------------------------------------------------------------------------------------------- | ---- | ---- | ----- | ----- | +| [resnet50_8xb32-coslr-100e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simsiam/simsiam_resnet50_8xb32-coslr-100e_in1k.py) | 57.4 | 57.6 | 55.8 | 54.2 | +| [resnet50_8xb32-coslr-200e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simsiam/simsiam_resnet50_8xb32-coslr-200e_in1k.py) | 60.2 | 60.4 | 58.8 | 57.4 | + ### Detection The detection benchmarks includes 2 downstream task datasets, **Pascal VOC 2007 + 2012** and **COCO2017**. This benchmark follows the evluation protocols set up by MoCo. diff --git a/configs/selfsup/swav/README.md b/configs/selfsup/swav/README.md index b40353665..bf289d99d 100644 --- a/configs/selfsup/swav/README.md +++ b/configs/selfsup/swav/README.md @@ -16,11 +16,11 @@ Unsupervised image representations have significantly reduced the gap with super **Back to [model_zoo.md](https://github.com/open-mmlab/mmselfsup/blob/master/docs/en/model_zoo.md) to download models.** -In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models were trained on ImageNet1k dataset. +In this page, we provide benchmarks as much as possible to evaluate our pre-trained models. If not mentioned, all models are pre-trained on ImageNet-1k dataset. ### Classification -The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). +The classification benchmarks includes 4 downstream task datasets, **VOC**, **ImageNet**, **iNaturalist2018** and **Places205**. If not specified, the results are Top-1 (%). #### VOC SVM / Low-shot SVM @@ -50,6 +50,14 @@ The **Feature1 - Feature5** don't have the GlobalAveragePooling, the feature map | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -------- | -------- | -------- | -------- | | [resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/swav/swav_resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96.py) | 23.33 | 35.45 | 43.13 | 51.98 | 55.09 | +#### ImageNet Nearest-Neighbor Classification + +The results are obtained from the features after GlobalAveragePooling. Here, k=10 to 200 indicates different number of nearest neighbors. + +| Self-Supervised Config | k=10 | k=20 | k=100 | k=200 | +| ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---- | ---- | ----- | ----- | +| [resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/swav/swav_resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96.py) | 60.5 | 60.6 | 59.0 | 57.6 | + ### Detection The detection benchmarks includes 2 downstream task datasets, **Pascal VOC 2007 + 2012** and **COCO2017**. This benchmark follows the evluation protocols set up by MoCo. diff --git a/docs/en/model_zoo.md b/docs/en/model_zoo.md index 1349b25c0..96047bdb1 100644 --- a/docs/en/model_zoo.md +++ b/docs/en/model_zoo.md @@ -30,11 +30,11 @@ Remarks: ## Benchmarks -In following tables, we only displayed ImageNet Linear Evaluation, COCO17 Object Detection and PASCAL VOC12 Aug Segmentation, you can click algorithm name above to check the comprehensive benchmark results. +In the following tables, we only display ImageNet linear evaluation, ImageNet fine-tuning, COCO17 object detection and instance segmentation, and PASCAL VOC12 Aug semantic segmentation. You can click algorithm name above to check more comprehensive benchmark results. ### ImageNet Linear Evaluation -If not specified, we use linear evaluation setting from [MoCo](http://openaccess.thecvf.com/content_CVPR_2020/papers/He_Momentum_Contrast_for_Unsupervised_Visual_Representation_Learning_CVPR_2020_paper.pdf). Or the settings is mentioned in Remark. +If not specified, we use linear evaluation setting from [MoCo](http://openaccess.thecvf.com/content_CVPR_2020/papers/He_Momentum_Contrast_for_Unsupervised_Visual_Representation_Learning_CVPR_2020_paper.pdf) as default. Other settings are mentioned in Remarks. | Algorithm | Config | Remarks | Top-1 (%) | | ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | --------- | @@ -53,15 +53,14 @@ If not specified, we use linear evaluation setting from [MoCo](http://openaccess | SwAV | [swav_resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/swav/swav_resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96.py) | SwAV paper setting | 70.47 | | MoCo v3 | [mocov3_vit-small-p16_32xb128-fp16-coslr-300e_in1k-224](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mocov3/mocov3_vit-small-p16_32xb128-fp16-coslr-300e_in1k-224.py) | MoCo v3 paper setting | 73.19 | - ### ImageNet Fine-tuning | Algorithm | Config | Remarks | Top-1 (%) | | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | --------- | | MAE | [mae_vit-base-p16_8xb512-coslr-400e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mae/mae_vit-base-p16_8xb512-coslr-400e_in1k.py) | | 83.1 | -### COCO17 Object Detection +### COCO17 Object Detection and Instance Segmentation -In COCO17 Object detection task, we choose the evluation protocol from [MoCo](http://openaccess.thecvf.com/content_CVPR_2020/papers/He_Momentum_Contrast_for_Unsupervised_Visual_Representation_Learning_CVPR_2020_paper.pdf), with Mask-RCNN architecture, the results below are trained with the same [config](https://github.com/open-mmlab/mmselfsup/blob/master/configs/benchmarks/mmdetection/coco/mask_rcnn_r50_fpn_mstrain_1x_coco.py). +In COCO17 object detection and instance segmentation task, we choose the evaluation protocol from [MoCo](http://openaccess.thecvf.com/content_CVPR_2020/papers/He_Momentum_Contrast_for_Unsupervised_Visual_Representation_Learning_CVPR_2020_paper.pdf), with Mask-RCNN FPN architecture. The results below are fine-tuned with the same [config](https://github.com/open-mmlab/mmselfsup/blob/master/configs/benchmarks/mmdetection/coco/mask_rcnn_r50_fpn_mstrain_1x_coco.py). | Algorithm | Config | mAP (Box) | mAP (Mask) | | ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | ---------- | @@ -76,9 +75,9 @@ In COCO17 Object detection task, we choose the evluation protocol from [MoCo](ht | | [simsiam_resnet50_8xb32-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simsiam/simsiam_resnet50_8xb32-coslr-200e_in1k.py) | 38.8 | 34.9 | | SwAV | [swav_resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/swav/swav_resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96.py) | 40.2 | 36.3 | -### Pascal VOC12 Aug Segmentation +### Pascal VOC12 Aug Semantic Segmentation -In Pascal VOC12 Aug Segmentation task, we choose the evluation protocol from [MMSeg](https://github.com/open-mmlab/mmsegmentation), with FCN architecture, the results below are trained with the same [config](https://github.com/open-mmlab/mmselfsup/blob/master/configs/benchmarks/mmsegmentation/voc12aug/fcn_r50-d8_512x512_20k_voc12aug.py). +In Pascal VOC12 Aug semantic segmentation task, we choose the evaluation protocol from [MMSeg](https://github.com/open-mmlab/mmsegmentation), with FCN architecture. The results below are fine-tuned with the same [config](https://github.com/open-mmlab/mmselfsup/blob/master/configs/benchmarks/mmsegmentation/voc12aug/fcn_r50-d8_512x512_20k_voc12aug.py). | Algorithm | Config | mIOU | | ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | diff --git a/docs/en/tutorials/6_benchmarks.md b/docs/en/tutorials/6_benchmarks.md index 3019b7d22..6fd16fb6c 100644 --- a/docs/en/tutorials/6_benchmarks.md +++ b/docs/en/tutorials/6_benchmarks.md @@ -6,7 +6,8 @@ In MMSelfSup, we provide many benchmarks, thus the models can be evaluated on di - [Classification](#classification) - [VOC SVM / Low-shot SVM](#voc-svm--low-shot-svm) - [Linear Evaluation](#linear-evaluation) - - [ImageNet Semi-supervised Classification](#imagenet-semi-supervised-classification) + - [ImageNet Semi-Supervised Classification](#imagenet-semi-supervised-classification) + - [ImageNet Nearest-Neighbor Classification](#imagenet-nearest-neighbor-classification) - [Detection](#detection) - [Segmentation](#segmentation) @@ -22,38 +23,38 @@ Arguments: ## Classification -As for classification, we provide scripts in folder `tools/benchmarks/classification/`, which has 4 `.sh` files and 1 folder for VOC SVM related classification task. +As for classification, we provide scripts in folder `tools/benchmarks/classification/`, which has 4 `.sh` files, 1 folder for VOC SVM related classification task and 1 folder for ImageNet nearest-neighbor classification task. ### VOC SVM / Low-shot SVM -To run this benchmarks, you should first prepare your VOC datasets, the details of data prepareation please refer to [data_prepare.md](../data_prepare.md). +To run these benchmarks, you should first prepare your VOC datasets. Please refer to [prepare_data.md](https://github.com/open-mmlab/mmselfsup/blob/master/docs/en/prepare_data.md) for the details of data preparation. -To evaluate the pretrained models, you can run command below. +To evaluate the pre-trained models, you can run command below. ```shell # distributed version bash tools/benchmarks/classification/svm_voc07/dist_test_svm_pretrain.sh ${SELFSUP_CONFIG} ${GPUS} ${PRETRAIN} ${FEATURE_LIST} - # slurm version bash tools/benchmarks/classification/svm_voc07/slurm_test_svm_pretrain.sh ${PARTITION} ${JOB_NAME} ${SELFSUP_CONFIG} ${PRETRAIN} ${FEATURE_LIST} - ``` Besides, if you want to evaluate the ckpt files saved by runner, you can run command below. + ```shell # distributed version bash tools/benchmarks/classification/svm_voc07/dist_test_svm_epoch.sh ${SELFSUP_CONFIG} ${EPOCH} ${FEATURE_LIST} # slurm version -bash tools/benchmarks/classification/svm_voc07/dist_test_svm_epoch.sh ${PARTITION} ${JOB_NAME} ${SELFSUP_CONFIG} ${EPOCH} ${FEATURE_LIST} +bash tools/benchmarks/classification/svm_voc07/slurm_test_svm_epoch.sh ${PARTITION} ${JOB_NAME} ${SELFSUP_CONFIG} ${EPOCH} ${FEATURE_LIST} ``` + **To test with ckpt, the code uses the epoch_*.pth file, there is no need to extract weights.** Remarks: - `${SELFSUP_CONFIG}` is the config file of the self-supervised experiment. -- `${FEATURE_LIST}` is a string to specify features from layer1 to layer5 to evaluate; e.g., if you want to evaluate layer5 only, then `FEATURE_LIST` is "feat5", if you want to evaluate all features, then then `FEATURE_LIST` is "feat1 feat2 feat3 feat4 feat5" (separated by space). If left empty, the default `FEATURE_LIST` is "feat5". -- `PRETRAIN`: the pretrained model file. +- `${FEATURE_LIST}` is a string to specify features from layer1 to layer5 to evaluate; e.g., if you want to evaluate layer5 only, then `FEATURE_LIST` is "feat5", if you want to evaluate all features, then `FEATURE_LIST` is "feat1 feat2 feat3 feat4 feat5" (separated by space). If left empty, the default `FEATURE_LIST` is "feat5". +- `PRETRAIN`: the pre-trained model file. - if you want to change GPU numbers, you could add `GPUS_PER_NODE=4 GPUS=4` at the beginning of the command. - `EPOCH` is the epoch number of the ckpt that you want to test @@ -71,11 +72,10 @@ bash tools/benchmarks/classification/slurm_train_linear.sh ${PARTITION} ${JOB_NA Remarks: - The default GPU number is 8. When changing GPUS, please also change `samples_per_gpu` in the config file accordingly to ensure the total batch size is 256. -- `CONFIG`: Use config files under `configs/benchmarks/classification/`, excluding svm_voc07.py and tsne_imagenet.py and imagenet_*percent folders. -- `PRETRAIN`: the pretrained model file. - +- `CONFIG`: Use config files under `configs/benchmarks/classification/`. Specifically, `imagenet` (excluding `imagenet_*percent` folders), `places205` and `inaturalist2018`. +- `PRETRAIN`: the pre-trained model file. -### ImageNet Semi-supervised Classification +### ImageNet Semi-Supervised Classification To run ImageNet semi-supervised classification, we still use `.sh` script to launch training. @@ -90,19 +90,52 @@ bash tools/benchmarks/classification/slurm_train_semi.sh ${PARTITION} ${JOB_NAME Remarks: - The default GPU number is 4. - `CONFIG`: Use config files under `configs/benchmarks/classification/imagenet/`, named `imagenet_*percent` folders. -- `PRETRAIN`: the pretrained model file. +- `PRETRAIN`: the pre-trained model file. + +### ImageNet Nearest-Neighbor Classification + +To evaluate the pre-trained models using the nearest-neighbor benchmark, you can run command below. + +```shell +# distributed version +bash tools/benchmarks/classification/knn_imagenet/dist_test_knn_pretrain.sh ${SELFSUP_CONFIG} ${PRETRAIN} + +# slurm version +bash tools/benchmarks/classification/knn_imagenet/slurm_test_knn_pretrain.sh ${PARTITION} ${JOB_NAME} ${SELFSUP_CONFIG} ${PRETRAIN} +``` + +Besides, if you want to evaluate the ckpt files saved by runner, you can run command below. + +```shell +# distributed version +bash tools/benchmarks/classification/knn_imagenet/dist_test_knn_epoch.sh ${SELFSUP_CONFIG} ${EPOCH} + +# slurm version +bash tools/benchmarks/classification/knn_imagenet/slurm_test_knn_epoch.sh ${PARTITION} ${JOB_NAME} ${SELFSUP_CONFIG} ${EPOCH} +``` + +**To test with ckpt, the code uses the epoch_*.pth file, there is no need to extract weights.** + +Remarks: +- `${SELFSUP_CONFIG}` is the config file of the self-supervised experiment. +- `PRETRAIN`: the pre-trained model file. +- if you want to change GPU numbers, you could add `GPUS_PER_NODE=4 GPUS=4` at the beginning of the command. +- `EPOCH` is the epoch number of the ckpt that you want to test ## Detection -Here we prefer to use MMDetection to do the detection task. First, make sure you have installed [MIM](https://github.com/open-mmlab/mim), which is also a project of OpenMMLab. +Here, we prefer to use MMDetection to do the detection task. First, make sure you have installed [MIM](https://github.com/open-mmlab/mim), which is also a project of OpenMMLab. + ```shell pip install openmim ``` + It is very easy to install the package. Besides, please refer to MMDet for [installation](https://github.com/open-mmlab/mmdetection/blob/master/docs/en/get_started.md) and [data preparation](https://github.com/open-mmlab/mmdetection/blob/master/docs/en/1_exist_data_model.md) -After installation, you can run MMDet with simple command +After installation, you can run MMDet with simple command. + ```shell # distributed version bash tools/benchmarks/mmdetection/mim_dist_train.sh ${CONFIG} ${PRETRAIN} ${GPUS} @@ -113,10 +146,10 @@ bash tools/benchmarks/mmdetection/mim_slurm_train.sh ${PARTITION} ${CONFIG} ${PR Remarks: - `CONFIG`: Use config files under `configs/benchmarks/mmdetection/` or write your own config files -- `PRETRAIN`: the pretrained model file. +- `PRETRAIN`: the pre-trained model file. Or if you want to do detection task with [detectron2](https://github.com/facebookresearch/detectron2), we also provides some config files. -Please refer [INSTALL.md](https://github.com/facebookresearch/detectron2/blob/main/INSTALL.md) for installation and follow the [directory structure](https://github.com/facebookresearch/detectron2/tree/main/datasets) to prepare your datasets required by detectron2. +Please refer to [INSTALL.md](https://github.com/facebookresearch/detectron2/blob/main/INSTALL.md) for installation and follow the [directory structure](https://github.com/facebookresearch/detectron2/tree/main/datasets) to prepare your datasets required by detectron2. ```shell conda activate detectron2 # use detectron2 environment here, otherwise use open-mmlab environment @@ -124,18 +157,21 @@ cd benchmarks/detection python convert-pretrain-to-detectron2.py ${WEIGHT_FILE} ${OUTPUT_FILE} # must use .pkl as the output extension. bash run.sh ${DET_CFG} ${OUTPUT_FILE} ``` + ## Segmentation -For semantic segmentation task, we are using MMSegmentation. First, make sure you have installed [MIM](https://github.com/open-mmlab/mim), which is also a project of OpenMMLab. +For semantic segmentation task, we use MMSegmentation. First, make sure you have installed [MIM](https://github.com/open-mmlab/mim), which is also a project of OpenMMLab. ```shell pip install openmim ``` + It is very easy to install the package. Besides, please refer to MMSeg for [installation](https://github.com/open-mmlab/mmsegmentation/blob/master/docs/get_started.md) and [data preparation](https://github.com/open-mmlab/mmsegmentation/blob/master/docs/dataset_prepare.md#prepare-datasets). -After installation, you can run MMSeg with simple command +After installation, you can run MMSeg with simple command. + ```shell # distributed version bash tools/benchmarks/mmdetection/mim_dist_train.sh ${CONFIG} ${PRETRAIN} ${GPUS} @@ -146,4 +182,4 @@ bash tools/benchmarks/mmdetection/mim_slurm_train.sh ${PARTITION} ${CONFIG} ${PR Remarks: - `CONFIG`: Use config files under `configs/benchmarks/mmsegmentation/` or write your own config files -- `PRETRAIN`: the pretrained model file. +- `PRETRAIN`: the pre-trained model file. diff --git a/docs/zh_cn/model_zoo.md b/docs/zh_cn/model_zoo.md index 3751fe651..d5b1f286c 100644 --- a/docs/zh_cn/model_zoo.md +++ b/docs/zh_cn/model_zoo.md @@ -30,7 +30,7 @@ ## 基准测试 -在下列表格中，我们只展示了基于 ImageNet 数据集的线性评估，COCO17 数据集的目标检测和 PASCAL VOC12 Aug 数据集的分割任务，您可以点击预训练模型表格中的算法名查看其它基准测试结果。 +在下列表格中，我们只展示了基于 ImageNet 数据集的线性评估，COCO17 数据集的目标检测和实例分割以及 PASCAL VOC12 Aug 数据集的语义分割任务，您可以点击预训练模型表格中的算法名查看更多基准测试结果。 ### ImageNet 线性评估 @@ -53,14 +53,14 @@ | SwAV | [swav_resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/swav/swav_resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96.py) | SwAV 论文设置 | 70.47 | | MoCo v3 | [mocov3_vit-small-p16_32xb128-fp16-coslr-300e_in1k-224](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mocov3/mocov3_vit-small-p16_32xb128-fp16-coslr-300e_in1k-224.py) | MoCo v3 论文设置 | 73.19 | - ### ImageNet 微调 | 算法 | 配置文件 | 备注 | Top-1 (%) | | ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---- | --------- | | MAE | [mae_vit-base-p16_8xb512-coslr-400e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mae/mae_vit-base-p16_8xb512-coslr-400e_in1k.py) | | 83.1 | -### COCO17 目标检测 -在 COCO17 数据集的目标检测任务中，我们选用 [MoCo](http://openaccess.thecvf.com/content_CVPR_2020/papers/He_Momentum_Contrast_for_Unsupervised_Visual_Representation_Learning_CVPR_2020_paper.pdf) 的评估设置，基于 Mask-RCNN 网络架构，下列结果通过同样的 [配置文件](https://github.com/open-mmlab/mmselfsup/blob/master/configs/benchmarks/mmdetection/coco/mask_rcnn_r50_fpn_mstrain_1x_coco.py) 训练得到。 +### COCO17 目标检测和实例分割 + +在 COCO17 数据集的目标检测和实例分割任务中，我们选用 [MoCo](http://openaccess.thecvf.com/content_CVPR_2020/papers/He_Momentum_Contrast_for_Unsupervised_Visual_Representation_Learning_CVPR_2020_paper.pdf) 的评估设置，基于 Mask-RCNN FPN 网络架构，下列结果通过同样的 [配置文件](https://github.com/open-mmlab/mmselfsup/blob/master/configs/benchmarks/mmdetection/coco/mask_rcnn_r50_fpn_mstrain_1x_coco.py) 训练得到。 | 算法 | 配置文件 | mAP (Box) | mAP (Mask) | | ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | ---------- | @@ -75,9 +75,9 @@ | | [simsiam_resnet50_8xb32-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simsiam/simsiam_resnet50_8xb32-coslr-200e_in1k.py) | 38.8 | 34.9 | | SwAV | [swav_resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/swav/swav_resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96.py) | 40.2 | 36.3 | -### Pascal VOC12 Aug 分割 +### Pascal VOC12 Aug 语义分割 -在 Pascal VOC12 Aug 分割任务中，我们选用 [MMSeg](https://github.com/open-mmlab/mmsegmentation) 的评估设置, 基于 FCN 网络架构, 下列结果通过同样的 [配置文件](https://github.com/open-mmlab/mmselfsup/blob/master/configs/benchmarks/mmsegmentation/voc12aug/fcn_r50-d8_512x512_20k_voc12aug.py) 训练得到。 +在 Pascal VOC12 Aug 语义分割任务中，我们选用 [MMSeg](https://github.com/open-mmlab/mmsegmentation) 的评估设置, 基于 FCN 网络架构, 下列结果通过同样的 [配置文件](https://github.com/open-mmlab/mmselfsup/blob/master/configs/benchmarks/mmsegmentation/voc12aug/fcn_r50-d8_512x512_20k_voc12aug.py) 训练得到。 | 算法 | 配置文件 | mIOU | | ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | diff --git a/docs/zh_cn/tutorials/6_benchmarks.md b/docs/zh_cn/tutorials/6_benchmarks.md index fd54f24ac..6fd16fb6c 100644 --- a/docs/zh_cn/tutorials/6_benchmarks.md +++ b/docs/zh_cn/tutorials/6_benchmarks.md @@ -6,7 +6,8 @@ In MMSelfSup, we provide many benchmarks, thus the models can be evaluated on di - [Classification](#classification) - [VOC SVM / Low-shot SVM](#voc-svm--low-shot-svm) - [Linear Evaluation](#linear-evaluation) - - [ImageNet Semi-supervised Classification](#imagenet-semi-supervised-classification) + - [ImageNet Semi-Supervised Classification](#imagenet-semi-supervised-classification) + - [ImageNet Nearest-Neighbor Classification](#imagenet-nearest-neighbor-classification) - [Detection](#detection) - [Segmentation](#segmentation) @@ -22,38 +23,38 @@ Arguments: ## Classification -As for classification, we provide scripts in folder `tools/benchmarks/classification/`, which has 4 `.sh` files and 1 folder for VOC SVM related classification task. +As for classification, we provide scripts in folder `tools/benchmarks/classification/`, which has 4 `.sh` files, 1 folder for VOC SVM related classification task and 1 folder for ImageNet nearest-neighbor classification task. ### VOC SVM / Low-shot SVM -To run this benchmarks, you should first prepare your VOC datasets, the details of data prepareation please refer to [data_prepare.md](../data_prepare.md). +To run these benchmarks, you should first prepare your VOC datasets. Please refer to [prepare_data.md](https://github.com/open-mmlab/mmselfsup/blob/master/docs/en/prepare_data.md) for the details of data preparation. -To evaluate the pretrained models, you can run command below. +To evaluate the pre-trained models, you can run command below. ```shell # distributed version bash tools/benchmarks/classification/svm_voc07/dist_test_svm_pretrain.sh ${SELFSUP_CONFIG} ${GPUS} ${PRETRAIN} ${FEATURE_LIST} - # slurm version bash tools/benchmarks/classification/svm_voc07/slurm_test_svm_pretrain.sh ${PARTITION} ${JOB_NAME} ${SELFSUP_CONFIG} ${PRETRAIN} ${FEATURE_LIST} - ``` Besides, if you want to evaluate the ckpt files saved by runner, you can run command below. + ```shell # distributed version bash tools/benchmarks/classification/svm_voc07/dist_test_svm_epoch.sh ${SELFSUP_CONFIG} ${EPOCH} ${FEATURE_LIST} # slurm version -bash tools/benchmarks/classification/svm_voc07/dist_test_svm_epoch.sh ${PARTITION} ${JOB_NAME} ${SELFSUP_CONFIG} ${EPOCH} ${FEATURE_LIST} +bash tools/benchmarks/classification/svm_voc07/slurm_test_svm_epoch.sh ${PARTITION} ${JOB_NAME} ${SELFSUP_CONFIG} ${EPOCH} ${FEATURE_LIST} ``` + **To test with ckpt, the code uses the epoch_*.pth file, there is no need to extract weights.** Remarks: - `${SELFSUP_CONFIG}` is the config file of the self-supervised experiment. -- `${FEATURE_LIST}` is a string to specify features from layer1 to layer5 to evaluate; e.g., if you want to evaluate layer5 only, then `FEATURE_LIST` is "feat5", if you want to evaluate all features, then then `FEATURE_LIST` is "feat1 feat2 feat3 feat4 feat5" (separated by space). If left empty, the default `FEATURE_LIST` is "feat5". -- `PRETRAIN`: the pretrained model file. +- `${FEATURE_LIST}` is a string to specify features from layer1 to layer5 to evaluate; e.g., if you want to evaluate layer5 only, then `FEATURE_LIST` is "feat5", if you want to evaluate all features, then `FEATURE_LIST` is "feat1 feat2 feat3 feat4 feat5" (separated by space). If left empty, the default `FEATURE_LIST` is "feat5". +- `PRETRAIN`: the pre-trained model file. - if you want to change GPU numbers, you could add `GPUS_PER_NODE=4 GPUS=4` at the beginning of the command. - `EPOCH` is the epoch number of the ckpt that you want to test @@ -71,11 +72,10 @@ bash tools/benchmarks/classification/slurm_train_linear.sh ${PARTITION} ${JOB_NA Remarks: - The default GPU number is 8. When changing GPUS, please also change `samples_per_gpu` in the config file accordingly to ensure the total batch size is 256. -- `CONFIG`: Use config files under `configs/benchmarks/classification/`, excluding svm_voc07.py and tsne_imagenet.py and imagenet_*percent folders. -- `PRETRAIN`: the pretrained model file. - +- `CONFIG`: Use config files under `configs/benchmarks/classification/`. Specifically, `imagenet` (excluding `imagenet_*percent` folders), `places205` and `inaturalist2018`. +- `PRETRAIN`: the pre-trained model file. -### ImageNet Semi-supervised Classification +### ImageNet Semi-Supervised Classification To run ImageNet semi-supervised classification, we still use `.sh` script to launch training. @@ -90,19 +90,52 @@ bash tools/benchmarks/classification/slurm_train_semi.sh ${PARTITION} ${JOB_NAME Remarks: - The default GPU number is 4. - `CONFIG`: Use config files under `configs/benchmarks/classification/imagenet/`, named `imagenet_*percent` folders. -- `PRETRAIN`: the pretrained model file. +- `PRETRAIN`: the pre-trained model file. + +### ImageNet Nearest-Neighbor Classification + +To evaluate the pre-trained models using the nearest-neighbor benchmark, you can run command below. + +```shell +# distributed version +bash tools/benchmarks/classification/knn_imagenet/dist_test_knn_pretrain.sh ${SELFSUP_CONFIG} ${PRETRAIN} + +# slurm version +bash tools/benchmarks/classification/knn_imagenet/slurm_test_knn_pretrain.sh ${PARTITION} ${JOB_NAME} ${SELFSUP_CONFIG} ${PRETRAIN} +``` + +Besides, if you want to evaluate the ckpt files saved by runner, you can run command below. + +```shell +# distributed version +bash tools/benchmarks/classification/knn_imagenet/dist_test_knn_epoch.sh ${SELFSUP_CONFIG} ${EPOCH} + +# slurm version +bash tools/benchmarks/classification/knn_imagenet/slurm_test_knn_epoch.sh ${PARTITION} ${JOB_NAME} ${SELFSUP_CONFIG} ${EPOCH} +``` + +**To test with ckpt, the code uses the epoch_*.pth file, there is no need to extract weights.** + +Remarks: +- `${SELFSUP_CONFIG}` is the config file of the self-supervised experiment. +- `PRETRAIN`: the pre-trained model file. +- if you want to change GPU numbers, you could add `GPUS_PER_NODE=4 GPUS=4` at the beginning of the command. +- `EPOCH` is the epoch number of the ckpt that you want to test ## Detection -Here we prefer to use MMDetection to do the detection task. First, make sure you have installed [MIM](https://github.com/open-mmlab/mim), which is also a project of OpenMMLab. +Here, we prefer to use MMDetection to do the detection task. First, make sure you have installed [MIM](https://github.com/open-mmlab/mim), which is also a project of OpenMMLab. + ```shell pip install openmim ``` + It is very easy to install the package. Besides, please refer to MMDet for [installation](https://github.com/open-mmlab/mmdetection/blob/master/docs/en/get_started.md) and [data preparation](https://github.com/open-mmlab/mmdetection/blob/master/docs/en/1_exist_data_model.md) -After installation, you can run MMDet with simple command +After installation, you can run MMDet with simple command. + ```shell # distributed version bash tools/benchmarks/mmdetection/mim_dist_train.sh ${CONFIG} ${PRETRAIN} ${GPUS} @@ -113,11 +146,10 @@ bash tools/benchmarks/mmdetection/mim_slurm_train.sh ${PARTITION} ${CONFIG} ${PR Remarks: - `CONFIG`: Use config files under `configs/benchmarks/mmdetection/` or write your own config files -- `PRETRAIN`: the pretrained model file. - +- `PRETRAIN`: the pre-trained model file. Or if you want to do detection task with [detectron2](https://github.com/facebookresearch/detectron2), we also provides some config files. -Please refer [INSTALL.md](https://github.com/facebookresearch/detectron2/blob/main/INSTALL.md) for installation and follow the [directory structure](https://github.com/facebookresearch/detectron2/tree/main/datasets) to prepare your datasets required by detectron2. +Please refer to [INSTALL.md](https://github.com/facebookresearch/detectron2/blob/main/INSTALL.md) for installation and follow the [directory structure](https://github.com/facebookresearch/detectron2/tree/main/datasets) to prepare your datasets required by detectron2. ```shell conda activate detectron2 # use detectron2 environment here, otherwise use open-mmlab environment @@ -125,18 +157,21 @@ cd benchmarks/detection python convert-pretrain-to-detectron2.py ${WEIGHT_FILE} ${OUTPUT_FILE} # must use .pkl as the output extension. bash run.sh ${DET_CFG} ${OUTPUT_FILE} ``` + ## Segmentation -For semantic segmentation task, we are using MMSegmentation. First, make sure you have installed [MIM](https://github.com/open-mmlab/mim), which is also a project of OpenMMLab. +For semantic segmentation task, we use MMSegmentation. First, make sure you have installed [MIM](https://github.com/open-mmlab/mim), which is also a project of OpenMMLab. ```shell pip install openmim ``` + It is very easy to install the package. Besides, please refer to MMSeg for [installation](https://github.com/open-mmlab/mmsegmentation/blob/master/docs/get_started.md) and [data preparation](https://github.com/open-mmlab/mmsegmentation/blob/master/docs/dataset_prepare.md#prepare-datasets). -After installation, you can run MMSeg with simple command +After installation, you can run MMSeg with simple command. + ```shell # distributed version bash tools/benchmarks/mmdetection/mim_dist_train.sh ${CONFIG} ${PRETRAIN} ${GPUS} @@ -147,4 +182,4 @@ bash tools/benchmarks/mmdetection/mim_slurm_train.sh ${PARTITION} ${CONFIG} ${PR Remarks: - `CONFIG`: Use config files under `configs/benchmarks/mmsegmentation/` or write your own config files -- `PRETRAIN`: the pretrained model file. +- `PRETRAIN`: the pre-trained model file. diff --git a/mmselfsup/models/utils/__init__.py b/mmselfsup/models/utils/__init__.py index 75b298840..32868a4fb 100644 --- a/mmselfsup/models/utils/__init__.py +++ b/mmselfsup/models/utils/__init__.py @@ -1,13 +1,15 @@ # Copyright (c) OpenMMLab. All rights reserved. from .accuracy import Accuracy, accuracy -from .extract_process import ExtractProcess +from .extract_process import ExtractProcess, MultiExtractProcess from .gather_layer import GatherLayer +from .knn_classifier import knn_classifier from .multi_pooling import MultiPooling from .multi_prototypes import MultiPrototypes from .position_embedding import build_2d_sincos_position_embedding from .sobel import Sobel __all__ = [ - 'Accuracy', 'accuracy', 'ExtractProcess', 'GatherLayer', 'MultiPooling', - 'MultiPrototypes', 'Sobel', 'build_2d_sincos_position_embedding', 'Mixup' + 'Accuracy', 'accuracy', 'ExtractProcess', 'MultiExtractProcess', + 'GatherLayer', 'knn_classifier', 'MultiPooling', 'MultiPrototypes', + 'build_2d_sincos_position_embedding', 'Sobel' ] diff --git a/mmselfsup/models/utils/extract_process.py b/mmselfsup/models/utils/extract_process.py index 354b97b09..2c070451a 100644 --- a/mmselfsup/models/utils/extract_process.py +++ b/mmselfsup/models/utils/extract_process.py @@ -1,4 +1,5 @@ # Copyright (c) OpenMMLab. All rights reserved. +import torch.nn as nn from mmcv.runner import get_dist_info from mmselfsup.utils.collect import (dist_forward_collect, @@ -7,8 +8,47 @@ class ExtractProcess(object): - """Extraction process for `extract.py` and `tsne_visualization.py` in - tools. + """Global average-pooled feature extraction process. + + This process extracts the global average-pooled features from the last + layer of resnet backbone. + """ + + def __init__(self): + self.avg_pool = nn.AdaptiveAvgPool2d((1, 1)) + + def _forward_func(self, model, **x): + """The forward function of extract process.""" + backbone_feat = model(mode='extract', **x) + pooling_feat = self.avg_pool(backbone_feat[-1]) + flat_feat = pooling_feat.view(pooling_feat.size(0), -1) + return dict(feat=flat_feat.cpu()) + + def extract(self, model, data_loader, distributed=False): + """The extract function to apply forward function and choose + distributed or not.""" + model.eval() + + # the function sent to collect function + def func(**x): + return self._forward_func(model, **x) + + if distributed: + rank, world_size = get_dist_info() + results = dist_forward_collect(func, data_loader, rank, + len(data_loader.dataset)) + else: + results = nondist_forward_collect(func, data_loader, + len(data_loader.dataset)) + return results + + +class MultiExtractProcess(object): + """Multi-stage intermediate feature extraction process for `extract.py` and + `tsne_visualization.py` in tools. + + This process extracts feature maps from different stages of backbone, and + average pools each feature map to around 9000 dimensions. Args: pool_type (str): Pooling type in :class:`MultiPooling`. Options are diff --git a/mmselfsup/models/utils/knn_classifier.py b/mmselfsup/models/utils/knn_classifier.py new file mode 100644 index 000000000..e85373481 --- /dev/null +++ b/mmselfsup/models/utils/knn_classifier.py @@ -0,0 +1,70 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +# This file is borrowed from +# https://github.com/facebookresearch/dino/blob/main/eval_knn.py + +import torch +import torch.nn as nn + + +@torch.no_grad() +def knn_classifier(train_features, + train_labels, + test_features, + test_labels, + k, + T, + num_classes=1000): + """Compute accuracy of knn classifier predictions. + + Args: + train_features (Tensor): Extracted features in the training set. + train_labels (Tensor): Labels in the training set. + test_features (Tensor): Extracted features in the testing set. + test_labels (Tensor): Labels in the testing set. + k (int): Number of NN to use. + T (float): Temperature used in the voting coefficient. + num_classes (int): Number of classes. Defaults to 1000. + """ + top1, top5, total = 0.0, 0.0, 0 + train_features = nn.functional.normalize(train_features, dim=1) + test_features = nn.functional.normalize(test_features, dim=1) + train_features = train_features.t() + num_test_images, num_chunks = test_labels.shape[0], 100 + # split all test images into several chunks to prevent out-of-memory + imgs_per_chunk = num_test_images // num_chunks + retrieval_one_hot = torch.zeros(k, num_classes).to(train_features.device) + for idx in range(0, num_test_images, imgs_per_chunk): + # get the features for test images + features = test_features[idx:min((idx + + imgs_per_chunk), num_test_images), :] + targets = test_labels[idx:min((idx + imgs_per_chunk), num_test_images)] + batch_size = targets.shape[0] + + # calculate the dot product and compute top-k neighbors + similarity = torch.mm(features, train_features) + distances, indices = similarity.topk(k, largest=True, sorted=True) + candidates = train_labels.view(1, -1).expand(batch_size, -1) + retrieved_neighbors = torch.gather(candidates, 1, indices) + + retrieval_one_hot.resize_(batch_size * k, num_classes).zero_() + retrieval_one_hot.scatter_(1, retrieved_neighbors.view(-1, 1), 1) + distances_transform = distances.clone().div_(T).exp_() + probs = torch.sum( + torch.mul( + retrieval_one_hot.view(batch_size, -1, num_classes), + distances_transform.view(batch_size, -1, 1), + ), + 1, + ) + _, predictions = probs.sort(1, True) + + # find the predictions that match the target + correct = predictions.eq(targets.data.view(-1, 1)) + top1 = top1 + correct.narrow(1, 0, 1).sum().item() + top5 = top5 + correct.narrow(1, 0, min( + 5, k)).sum().item() # top5 does not make sense if k < 5 + total += targets.size(0) + top1 = top1 * 100.0 / total + top5 = top5 * 100.0 / total + return top1, top5 diff --git a/tests/test_models/test_utils/test_knn_classifier.py b/tests/test_models/test_utils/test_knn_classifier.py new file mode 100644 index 000000000..ec9de2dae --- /dev/null +++ b/tests/test_models/test_utils/test_knn_classifier.py @@ -0,0 +1,17 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch + +from mmselfsup.models.utils import knn_classifier + + +def test_knn_classifier(): + train_feats = torch.ones(200, 3) + train_labels = torch.ones(200).long() + test_feats = torch.ones(200, 3) + test_labels = torch.ones(200).long() + num_knn = [10, 20, 100, 200] + for k in num_knn: + top1, top5 = knn_classifier(train_feats, train_labels, test_feats, + test_labels, k, 0.07) + assert top1 == 100. + assert top5 == 100. diff --git a/tests/test_runtime/test_extract_process.py b/tests/test_runtime/test_extract_process.py index 03529fff0..5aab16e2e 100644 --- a/tests/test_runtime/test_extract_process.py +++ b/tests/test_runtime/test_extract_process.py @@ -7,7 +7,7 @@ from mmcv.parallel import MMDataParallel from torch.utils.data import DataLoader, Dataset -from mmselfsup.models.utils import ExtractProcess +from mmselfsup.models.utils import ExtractProcess, MultiExtractProcess class ExampleDataset(Dataset): @@ -39,8 +39,22 @@ def train_step(self, data_batch, optimizer): def test_extract_process(): + test_dataset = ExampleDataset() + test_dataset.evaluate = MagicMock(return_value=dict(test='success')) + data_loader = DataLoader( + test_dataset, batch_size=1, sampler=None, num_workers=0, shuffle=False) + model = MMDataParallel(ExampleModel()) + + process = ExtractProcess() + + results = process.extract(model, data_loader) + assert 'feat' in results + assert results['feat'].shape == (1, 128 * 1 * 1) + + +def test_multi_extract_process(): with pytest.raises(AssertionError): - process = ExtractProcess( + process = MultiExtractProcess( pool_type='specified', backbone='resnet50', layer_indices=(-1, )) test_dataset = ExampleDataset() @@ -49,7 +63,7 @@ def test_extract_process(): test_dataset, batch_size=1, sampler=None, num_workers=0, shuffle=False) model = MMDataParallel(ExampleModel()) - process = ExtractProcess( + process = MultiExtractProcess( pool_type='specified', backbone='resnet50', layer_indices=(0, 1, 2)) results = process.extract(model, data_loader) diff --git a/tools/analysis_tools/visualize_tsne.py b/tools/analysis_tools/visualize_tsne.py index 41ae9f286..ac6f0ea67 100644 --- a/tools/analysis_tools/visualize_tsne.py +++ b/tools/analysis_tools/visualize_tsne.py @@ -15,7 +15,7 @@ from mmselfsup.apis import set_random_seed from mmselfsup.datasets import build_dataloader, build_dataset from mmselfsup.models import build_algorithm -from mmselfsup.models.utils import ExtractProcess +from mmselfsup.models.utils import MultiExtractProcess from mmselfsup.utils import get_root_logger @@ -208,7 +208,7 @@ def main(): broadcast_buffers=False) # build extraction processor and run - extractor = ExtractProcess( + extractor = MultiExtractProcess( pool_type=args.pool_type, backbone='resnet50', layer_indices=layer_ind) features = extractor.extract(model, data_loader, distributed=distributed) labels = dataset.data_source.get_gt_labels() diff --git a/tools/benchmarks/classification/knn_imagenet/dist_test_knn_epoch.sh b/tools/benchmarks/classification/knn_imagenet/dist_test_knn_epoch.sh new file mode 100644 index 000000000..a8b83b94e --- /dev/null +++ b/tools/benchmarks/classification/knn_imagenet/dist_test_knn_epoch.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -e +set -x + +CFG=$1 +EPOCH=$2 +PY_ARGS=${@:3} +GPUS=${GPUS:-8} +NNODES=${NNODES:-1} +NODE_RANK=${NODE_RANK:-0} +PORT=${PORT:-29500} +MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} + +WORK_DIR=$(echo ${CFG%.*} | sed -e "s/configs/work_dirs/g")/ + +if [ ! -f $WORK_DIR/epoch_${EPOCH}.pth ]; then + echo "ERROR: File not exist: $WORK_DIR/epoch_${EPOCH}.pth" + exit +fi + +python -m torch.distributed.launch \ + --nnodes=$NNODES \ + --node_rank=$NODE_RANK \ + --master_addr=$MASTER_ADDR \ + --nproc_per_node=$GPUS \ + --master_port=$PORT \ + tools/benchmarks/classification/knn_imagenet/test_knn.py $CFG \ + --checkpoint $WORK_DIR/epoch_${EPOCH}.pth \ + --work-dir $WORK_DIR --launcher="pytorch" ${PY_ARGS} diff --git a/tools/benchmarks/classification/knn_imagenet/dist_test_knn_pretrain.sh b/tools/benchmarks/classification/knn_imagenet/dist_test_knn_pretrain.sh new file mode 100644 index 000000000..ae6cc9438 --- /dev/null +++ b/tools/benchmarks/classification/knn_imagenet/dist_test_knn_pretrain.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -e +set -x + +CFG=$1 +PRETRAIN=$2 # pretrained model +PY_ARGS=${@:3} +GPUS=${GPUS:-8} +NNODES=${NNODES:-1} +NODE_RANK=${NODE_RANK:-0} +PORT=${PORT:-29500} +MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} + +# set work_dir according to config path and pretrained model to distinguish different models +WORK_DIR="$(echo ${CFG%.*} | sed -e "s/configs/work_dirs/g")/$(echo $PRETRAIN | rev | cut -d/ -f 1 | rev)" + +python -m torch.distributed.launch \ + --nnodes=$NNODES \ + --node_rank=$NODE_RANK \ + --master_addr=$MASTER_ADDR \ + --nproc_per_node=$GPUS \ + --master_port=$PORT \ + tools/benchmarks/classification/knn_imagenet/test_knn.py $CFG \ + --cfg-options model.backbone.init_cfg.type=Pretrained \ + model.backbone.init_cfg.checkpoint=$PRETRAIN \ + --work-dir $WORK_DIR --launcher="pytorch" ${PY_ARGS} diff --git a/tools/benchmarks/classification/knn_imagenet/slurm_test_knn_epoch.sh b/tools/benchmarks/classification/knn_imagenet/slurm_test_knn_epoch.sh new file mode 100644 index 000000000..d1d152191 --- /dev/null +++ b/tools/benchmarks/classification/knn_imagenet/slurm_test_knn_epoch.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -e +set -x + +PARTITION=$1 +JOB_NAME=$2 +CFG=$3 +EPOCH=$4 +PY_ARGS=${@:5} +GPUS=${GPUS:-8} +GPUS_PER_NODE=${GPUS_PER_NODE:-8} +CPUS_PER_TASK=${CPUS_PER_TASK:-5} +PORT=${PORT:-29500} +SRUN_ARGS=${SRUN_ARGS:-""} + +WORK_DIR=$(echo ${CFG%.*} | sed -e "s/configs/work_dirs/g")/ + +if [ ! -f $WORK_DIR/epoch_${EPOCH}.pth ]; then + echo "ERROR: File not exist: $WORK_DIR/epoch_${EPOCH}.pth" + exit +fi + +PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ +srun -p ${PARTITION} \ + --job-name=${JOB_NAME} \ + --gres=gpu:${GPUS_PER_NODE} \ + --ntasks=${GPUS} \ + --ntasks-per-node=${GPUS_PER_NODE} \ + --cpus-per-task=${CPUS_PER_TASK} \ + --kill-on-bad-exit=1 \ + ${SRUN_ARGS} \ + python -u tools/benchmarks/classification/knn_imagenet/test_knn.py $CFG \ + --checkpoint $WORK_DIR/epoch_${EPOCH}.pth \ + --cfg-options dist_params.port=$PORT \ + --work-dir $WORK_DIR --launcher="slurm" ${PY_ARGS} diff --git a/tools/benchmarks/classification/knn_imagenet/slurm_test_knn_pretrain.sh b/tools/benchmarks/classification/knn_imagenet/slurm_test_knn_pretrain.sh new file mode 100644 index 000000000..e575740ae --- /dev/null +++ b/tools/benchmarks/classification/knn_imagenet/slurm_test_knn_pretrain.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -e +set -x + +PARTITION=$1 +JOB_NAME=$2 +CFG=$3 +PRETRAIN=$4 # pretrained model +PY_ARGS=${@:5} +GPUS=${GPUS:-8} +GPUS_PER_NODE=${GPUS_PER_NODE:-8} +CPUS_PER_TASK=${CPUS_PER_TASK:-5} +PORT=${PORT:-29500} +SRUN_ARGS=${SRUN_ARGS:-""} + +# set work_dir according to config path and pretrained model to distinguish different models +WORK_DIR="$(echo ${CFG%.*} | sed -e "s/configs/work_dirs/g")/$(echo $PRETRAIN | rev | cut -d/ -f 1 | rev)" + +PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ +srun -p ${PARTITION} \ + --job-name=${JOB_NAME} \ + --gres=gpu:${GPUS_PER_NODE} \ + --ntasks=${GPUS} \ + --ntasks-per-node=${GPUS_PER_NODE} \ + --cpus-per-task=${CPUS_PER_TASK} \ + --kill-on-bad-exit=1 \ + ${SRUN_ARGS} \ + python -u tools/benchmarks/classification/knn_imagenet/test_knn.py $CFG \ + --cfg-options model.backbone.init_cfg.type=Pretrained \ + model.backbone.init_cfg.checkpoint=$PRETRAIN \ + dist_params.port=$PORT \ + --work-dir $WORK_DIR --launcher="slurm" ${PY_ARGS} diff --git a/tools/benchmarks/classification/knn_imagenet/test_knn.py b/tools/benchmarks/classification/knn_imagenet/test_knn.py new file mode 100644 index 000000000..51c6f352e --- /dev/null +++ b/tools/benchmarks/classification/knn_imagenet/test_knn.py @@ -0,0 +1,186 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import argparse +import os +import os.path as osp +import time + +import mmcv +import torch +from mmcv import DictAction +from mmcv.parallel import MMDataParallel, MMDistributedDataParallel +from mmcv.runner import get_dist_info, init_dist, load_checkpoint + +from mmselfsup.datasets import build_dataloader, build_dataset +from mmselfsup.models import build_algorithm +from mmselfsup.models.utils import ExtractProcess, knn_classifier +from mmselfsup.utils import get_root_logger + + +def parse_args(): + parser = argparse.ArgumentParser(description='KNN evaluation') + parser.add_argument('config', help='train config file path') + parser.add_argument('--checkpoint', default=None, help='checkpoint file') + parser.add_argument( + '--dataset-config', + default='configs/benchmarks/classification/knn_imagenet.py', + help='knn dataset config file path') + parser.add_argument( + '--work-dir', type=str, default=None, help='the dir to save results') + parser.add_argument( + '--launcher', + choices=['none', 'pytorch', 'slurm', 'mpi'], + default='none', + help='job launcher') + parser.add_argument('--local-rank', type=int, default=0) + parser.add_argument( + '--cfg-options', + nargs='+', + action=DictAction, + help='override some settings in the used config, the key-value pair ' + 'in xxx=yyy format will be merged into config file. If the value to ' + 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' + 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' + 'Note that the quotation marks are necessary and that no white space ' + 'is allowed.') + # KNN settings + parser.add_argument( + '--num-knn', + default=[10, 20, 100, 200], + nargs='+', + type=int, + help='Number of NN to use. 20 usually works the best.') + parser.add_argument( + '--temperature', + default=0.07, + type=float, + help='Temperature used in the voting coefficient.') + parser.add_argument( + '--use-cuda', + default=True, + type=bool, + help='Store the features on GPU. Set to False if you encounter OOM') + args = parser.parse_args() + if 'LOCAL_RANK' not in os.environ: + os.environ['LOCAL_RANK'] = str(args.local_rank) + return args + + +def main(): + args = parse_args() + + cfg = mmcv.Config.fromfile(args.config) + if args.cfg_options is not None: + cfg.merge_from_dict(args.cfg_options) + # set cudnn_benchmark + if cfg.get('cudnn_benchmark', False): + torch.backends.cudnn.benchmark = True + # work_dir is determined in this priority: CLI > segment in file > filename + if args.work_dir is not None: + # update configs according to CLI args if args.work_dir is not None + cfg.work_dir = args.work_dir + elif cfg.get('work_dir', None) is None: + # use config filename as default work_dir if cfg.work_dir is None + work_type = args.config.split('/')[1] + cfg.work_dir = osp.join('./work_dirs', work_type, + osp.splitext(osp.basename(args.config))[0]) + + # init distributed env first, since logger depends on the dist info. + if args.launcher == 'none': + distributed = False + else: + distributed = True + init_dist(args.launcher, **cfg.dist_params) + + # create work_dir and init the logger before other steps + timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime()) + knn_work_dir = osp.join(cfg.work_dir, 'knn/') + mmcv.mkdir_or_exist(osp.abspath(knn_work_dir)) + log_file = osp.join(knn_work_dir, f'knn_{timestamp}.log') + logger = get_root_logger(log_file=log_file, log_level=cfg.log_level) + + # build the dataloader + dataset_cfg = mmcv.Config.fromfile(args.dataset_config) + dataset_train = build_dataset(dataset_cfg.data.train) + dataset_val = build_dataset(dataset_cfg.data.val) + if 'imgs_per_gpu' in cfg.data: + logger.warning('"imgs_per_gpu" is deprecated. ' + 'Please use "samples_per_gpu" instead') + if 'samples_per_gpu' in cfg.data: + logger.warning( + f'Got "imgs_per_gpu"={cfg.data.imgs_per_gpu} and ' + f'"samples_per_gpu"={cfg.data.samples_per_gpu}, "imgs_per_gpu"' + f'={cfg.data.imgs_per_gpu} is used in this experiments') + else: + logger.warning( + 'Automatically set "samples_per_gpu"="imgs_per_gpu"=' + f'{cfg.data.imgs_per_gpu} in this experiments') + cfg.data.samples_per_gpu = cfg.data.imgs_per_gpu + data_loader_train = build_dataloader( + dataset_train, + samples_per_gpu=dataset_cfg.data.samples_per_gpu, + workers_per_gpu=dataset_cfg.data.workers_per_gpu, + dist=distributed, + shuffle=False) + data_loader_val = build_dataloader( + dataset_val, + samples_per_gpu=dataset_cfg.data.samples_per_gpu, + workers_per_gpu=dataset_cfg.data.workers_per_gpu, + dist=distributed, + shuffle=False) + + # build the model + model = build_algorithm(cfg.model) + model.init_weights() + + # model is determined in this priority: init_cfg > checkpoint > random + if hasattr(cfg.model.backbone, 'init_cfg'): + if getattr(cfg.model.backbone.init_cfg, 'type', None) == 'Pretrained': + logger.info( + f'Use pretrained model: ' + f'{cfg.model.backbone.init_cfg.checkpoint} to extract features' + ) + elif args.checkpoint is not None: + logger.info(f'Use checkpoint: {args.checkpoint} to extract features') + load_checkpoint(model, args.checkpoint, map_location='cpu') + else: + logger.info('No pretrained or checkpoint is given, use random init.') + + if not distributed: + model = MMDataParallel(model, device_ids=[0]) + else: + model = MMDistributedDataParallel( + model.cuda(), + device_ids=[torch.cuda.current_device()], + broadcast_buffers=False) + + model.eval() + # build extraction processor and run + extractor = ExtractProcess() + train_feats = extractor.extract( + model, data_loader_train, distributed=distributed)['feat'] + val_feats = extractor.extract( + model, data_loader_val, distributed=distributed)['feat'] + + train_feats = torch.from_numpy(train_feats) + val_feats = torch.from_numpy(val_feats) + train_labels = torch.LongTensor(dataset_train.data_source.get_gt_labels()) + val_labels = torch.LongTensor(dataset_val.data_source.get_gt_labels()) + + logger.info('Features are extracted! Start k-NN classification...') + + rank, _ = get_dist_info() + if rank == 0: + if args.use_cuda: + train_feats = train_feats.cuda() + val_feats = val_feats.cuda() + train_labels = train_labels.cuda() + val_labels = val_labels.cuda() + for k in args.num_knn: + top1, top5 = knn_classifier(train_feats, train_labels, val_feats, + val_labels, k, args.temperature) + logger.info( + f'{k}-NN classifier result: Top1: {top1}, Top5: {top5}') + + +if __name__ == '__main__': + main() diff --git a/tools/benchmarks/classification/svm_voc07/extract.py b/tools/benchmarks/classification/svm_voc07/extract.py index d907d3f01..0fc5d31aa 100644 --- a/tools/benchmarks/classification/svm_voc07/extract.py +++ b/tools/benchmarks/classification/svm_voc07/extract.py @@ -13,7 +13,7 @@ from mmselfsup.datasets import build_dataloader, build_dataset from mmselfsup.models import build_algorithm -from mmselfsup.models.utils import ExtractProcess +from mmselfsup.models.utils import MultiExtractProcess from mmselfsup.utils import get_root_logger @@ -141,7 +141,7 @@ def main(): broadcast_buffers=False) # build extraction processor - extractor = ExtractProcess( + extractor = MultiExtractProcess( pool_type='specified', backbone='resnet50', layer_indices=layer_ind) # run