From 7516b5b7938e5a09893aa090902a343db4f3921a Mon Sep 17 00:00:00 2001 From: Jiahao Xie <52497952+Jiahao000@users.noreply.github.com> Date: Fri, 29 Apr 2022 15:22:25 +0800 Subject: [PATCH] [Enhance] update SimCLR models and results (#295) * [Enhance] update simclr models and results * [Fix] revise comments to indicate settings --- .../resnet50_linear-8xb32-coslr-100e_in1k.py | 2 +- .../resnet50_linear-8xb32-steplr-100e_in1k.py | 2 +- .../resnet50_linear-8xb512-coslr-90e_in1k.py | 8 ++++++-- ...et50_mhead_linear-8xb32-steplr-90e_in1k.py | 7 ++++++- configs/selfsup/_base_/models/simclr.py | 3 ++- configs/selfsup/simclr/README.md | 9 +++++---- configs/selfsup/simclr/metafile.yml | 18 ++++++++++++++--- ...simclr_resnet50_16xb256-coslr-200e_in1k.py | 7 +++++++ .../simclr_resnet50_8xb64-coslr-200e_in1k.py | 3 +++ docs/en/model_zoo.md | 20 ++++++++++--------- docs/zh_cn/model_zoo.md | 18 +++++++++-------- mmselfsup/models/backbones/resnet.py | 4 ++-- mmselfsup/models/backbones/resnext.py | 2 +- 13 files changed, 70 insertions(+), 33 deletions(-) create mode 100644 configs/selfsup/simclr/simclr_resnet50_16xb256-coslr-200e_in1k.py diff --git a/configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-coslr-100e_in1k.py b/configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-coslr-100e_in1k.py index c9d4d546a..e63c2bbd2 100644 --- a/configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-coslr-100e_in1k.py +++ b/configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-coslr-100e_in1k.py @@ -4,10 +4,10 @@ '../_base_/schedules/sgd_coslr-100e.py', '../_base_/default_runtime.py', ] +# SwAV linear evaluation setting model = dict(backbone=dict(frozen_stages=4)) -# swav setting # runtime settings # the max_keep_ckpts controls the max number of ckpt file in your work_dirs # if it is 3, when CheckpointHook (in mmcv) saves the 4th ckpt diff --git a/configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-steplr-100e_in1k.py b/configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-steplr-100e_in1k.py index a0bb07ec2..3a7795a05 100644 --- a/configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-steplr-100e_in1k.py +++ b/configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-steplr-100e_in1k.py @@ -4,12 +4,12 @@ '../_base_/schedules/sgd_steplr-100e.py', '../_base_/default_runtime.py', ] +# MoCo v1/v2 linear evaluation setting model = dict(backbone=dict(frozen_stages=4)) evaluation = dict(interval=1, topk=(1, 5)) -# moco setting # optimizer optimizer = dict(type='SGD', lr=30., momentum=0.9, weight_decay=0.) diff --git a/configs/benchmarks/classification/imagenet/resnet50_linear-8xb512-coslr-90e_in1k.py b/configs/benchmarks/classification/imagenet/resnet50_linear-8xb512-coslr-90e_in1k.py index ea9d8ceea..2742bf133 100644 --- a/configs/benchmarks/classification/imagenet/resnet50_linear-8xb512-coslr-90e_in1k.py +++ b/configs/benchmarks/classification/imagenet/resnet50_linear-8xb512-coslr-90e_in1k.py @@ -4,13 +4,17 @@ '../_base_/schedules/lars_coslr-90e.py', '../_base_/default_runtime.py', ] +# SimSiam linear evaluation setting +# According to SimSiam paper, this setting can also be used to evaluate +# other methods like SimCLR, MoCo, BYOL, SwAV model = dict(backbone=dict(frozen_stages=4)) # dataset summary -data = dict(samples_per_gpu=512) # total 512*8=4096, 8GPU linear cls +data = dict( + samples_per_gpu=512, + workers_per_gpu=8) # total 512*8=4096, 8GPU linear cls -# simsiam setting # runtime settings # the max_keep_ckpts controls the max number of ckpt file in your work_dirs # if it is 3, when CheckpointHook (in mmcv) saves the 4th ckpt diff --git a/configs/benchmarks/classification/imagenet/resnet50_mhead_linear-8xb32-steplr-90e_in1k.py b/configs/benchmarks/classification/imagenet/resnet50_mhead_linear-8xb32-steplr-90e_in1k.py index 16cc862d3..c569c7fd8 100644 --- a/configs/benchmarks/classification/imagenet/resnet50_mhead_linear-8xb32-steplr-90e_in1k.py +++ b/configs/benchmarks/classification/imagenet/resnet50_mhead_linear-8xb32-steplr-90e_in1k.py @@ -4,6 +4,7 @@ '../_base_/schedules/sgd_steplr-100e.py', '../_base_/default_runtime.py', ] +# Multi-head linear evaluation setting model = dict(backbone=dict(frozen_stages=4)) @@ -45,4 +46,8 @@ # runtime settings runner = dict(type='EpochBasedRunner', max_epochs=90) -checkpoint_config = dict(interval=10) + +# the max_keep_ckpts controls the max number of ckpt file in your work_dirs +# if it is 3, when CheckpointHook (in mmcv) saves the 4th ckpt +# it will remove the oldest one to keep the number of total ckpts as 3 +checkpoint_config = dict(interval=10, max_keep_ckpts=3) diff --git a/configs/selfsup/_base_/models/simclr.py b/configs/selfsup/_base_/models/simclr.py index e9f8a9dd9..150f74b5e 100644 --- a/configs/selfsup/_base_/models/simclr.py +++ b/configs/selfsup/_base_/models/simclr.py @@ -6,7 +6,8 @@ depth=50, in_channels=3, out_indices=[4], # 0: conv-1, x: stage-x - norm_cfg=dict(type='SyncBN')), + norm_cfg=dict(type='SyncBN'), + zero_init_residual=True), neck=dict( type='NonLinearNeck', # SimCLR non-linear neck in_channels=2048, diff --git a/configs/selfsup/simclr/README.md b/configs/selfsup/simclr/README.md index d3ed630b2..90b857ca5 100644 --- a/configs/selfsup/simclr/README.md +++ b/configs/selfsup/simclr/README.md @@ -36,11 +36,12 @@ Besides, k=1 to 96 indicates the hyper-parameter of Low-shot SVM. The **Feature1 - Feature5** don't have the GlobalAveragePooling, the feature map is pooled to the specific dimensions and then follows a Linear layer to do the classification. Please refer to [resnet50_mhead_linear-8xb32-steplr-90e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/benchmarks/classification/imagenet/resnet50_mhead_linear-8xb32-steplr-90e_in1k.py) for details of config. -The **AvgPool** result is obtained from Linear Evaluation with GlobalAveragePooling. Please refer to [resnet50_linear-8xb32-steplr-100e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-steplr-100e_in1k.py) for details of config. +The **AvgPool** result is obtained from Linear Evaluation with GlobalAveragePooling. Please refer to [resnet50_linear-8xb512-coslr-90e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/benchmarks/classification/imagenet/resnet50_linear-8xb512-coslr-90e_in1k.py) for details of config. -| Self-Supervised Config | Feature1 | Feature2 | Feature3 | Feature4 | Feature5 | AvgPool | -| ------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | -------- | -------- | -------- | -------- | ------- | -| [resnet50_8xb32-coslr-200e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k.py) | 14.43 | 30.97 | 41.02 | 53.92 | 61.24 | 57.28 | +| Self-Supervised Config | Feature1 | Feature2 | Feature3 | Feature4 | Feature5 | AvgPool | +| ---------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -------- | -------- | -------- | -------- | ------- | +| [resnet50_8xb32-coslr-200e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k.py) | 16.29 | 31.11 | 39.99 | 55.06 | 62.91 | 62.56 | +| [resnet50_16xb256-coslr-200e](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/simclr_resnet50_16xb256-coslr-200e_in1k.py) | 15.44 | 31.47 | 41.83 | 59.44 | 66.41 | 66.66 | #### Places205 Linear Evaluation diff --git a/configs/selfsup/simclr/metafile.yml b/configs/selfsup/simclr/metafile.yml index 914b692d5..e13397137 100644 --- a/configs/selfsup/simclr/metafile.yml +++ b/configs/selfsup/simclr/metafile.yml @@ -4,7 +4,7 @@ Collections: Training Data: ImageNet-1k Training Techniques: - LARS - Training Resources: 8x V100 GPUs + Training Resources: 8x V100 GPUs (b256), 16x A100-80G GPUs (b4096) Architecture: - ResNet - SimCLR @@ -23,6 +23,18 @@ Models: - Task: Self-Supervised Image Classification Dataset: ImageNet-1k Metrics: - Top 1 Accuracy: 57.28 + Top 1 Accuracy: 62.56 Config: configs/selfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k.py - Weights: https://download.openmmlab.com/mmselfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k_20220225-97d2abef.pth + Weights: https://download.openmmlab.com/mmselfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k_20220428-46ef6bb9.pth + - Name: simclr_resnet50_16xb256-coslr-200e_in1k + In Collection: SimCLR + Metadata: + Epochs: 200 + Batch Size: 4096 + Results: + - Task: Self-Supervised Image Classification + Dataset: ImageNet-1k + Metrics: + Top 1 Accuracy: 66.66 + Config: configs/selfsup/simclr/simclr_resnet50_16xb256-coslr-200e_in1k.py + Weights: https://download.openmmlab.com/mmselfsup/simclr/simclr_resnet50_16xb256-coslr-200e_in1k_20220428-8c24b063.pth diff --git a/configs/selfsup/simclr/simclr_resnet50_16xb256-coslr-200e_in1k.py b/configs/selfsup/simclr/simclr_resnet50_16xb256-coslr-200e_in1k.py new file mode 100644 index 000000000..476fe85a0 --- /dev/null +++ b/configs/selfsup/simclr/simclr_resnet50_16xb256-coslr-200e_in1k.py @@ -0,0 +1,7 @@ +_base_ = 'simclr_resnet50_8xb32-coslr-200e_in1k.py' + +# optimizer +optimizer = dict(lr=4.8) + +# dataset summary +data = dict(samples_per_gpu=256, workers_per_gpu=8) # total 256*16 diff --git a/configs/selfsup/simclr/simclr_resnet50_8xb64-coslr-200e_in1k.py b/configs/selfsup/simclr/simclr_resnet50_8xb64-coslr-200e_in1k.py index f6dff5572..054c62a11 100644 --- a/configs/selfsup/simclr/simclr_resnet50_8xb64-coslr-200e_in1k.py +++ b/configs/selfsup/simclr/simclr_resnet50_8xb64-coslr-200e_in1k.py @@ -1,4 +1,7 @@ _base_ = 'simclr_resnet50_8xb32-coslr-200e_in1k.py' +# optimizer +optimizer = dict(lr=0.6) + # dataset summary data = dict(samples_per_gpu=64) # total 64*8 diff --git a/docs/en/model_zoo.md b/docs/en/model_zoo.md index 173cfe497..85a1cf36c 100644 --- a/docs/en/model_zoo.md +++ b/docs/en/model_zoo.md @@ -11,7 +11,8 @@ All models and part of benchmark results are recorded below. | [DeepCluster](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/deepcluster/README.md) | [deepcluster-sobel_resnet50_8xb64-steplr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/deepcluster/deepcluster-sobel_resnet50_8xb64-steplr-200e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/deepcluster/deepcluster-sobel_resnet50_8xb64-steplr-200e_in1k-bb8681e2.pth) | | [NPID](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/npid/README.md) | [npid_resnet50_8xb32-steplr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/npid/npid_resnet50_8xb32-steplr-200e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/npid/npid_resnet50_8xb32-steplr-200e_in1k_20220225-5fbbda2a.pth) | [log](https://download.openmmlab.com/mmselfsup/npid/npid_resnet50_8xb32-steplr-200e_in1k_20220215_185513.log.json) | | [ODC](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/odc/README.md) | [odc_resnet50_8xb64-steplr-440e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/odc/odc_resnet50_8xb64-steplr-440e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/odc/odc_resnet50_8xb64-steplr-440e_in1k_20220225-a755d9c0.pth) | [log](https://download.openmmlab.com/mmselfsup/odc/odc_resnet50_8xb64-steplr-440e_in1k_20220215_235245.log.json) | -| [SimCLR](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/README.md) | [simclr_resnet50_8xb32-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k.py) | [model](simclr_resnet50_8xb32-coslr-200e_in1k_20220225-97d2abef.pth) | [log](https://download.openmmlab.com/mmselfsup/simclr/simclr_resnet50_8xb64-coslr-200e_in1k_20220210_191629.log.json) | +| [SimCLR](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/README.md) | [simclr_resnet50_8xb32-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k_20220428-46ef6bb9.pth) | [log](https://download.openmmlab.com/mmselfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k_20220411_182427.log.json) | +| | [simclr_resnet50_16xb256-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/simclr_resnet50_16xb256-coslr-200e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/simclr/simclr_resnet50_16xb256-coslr-200e_in1k_20220428-8c24b063.pth) | [log](https://download.openmmlab.com/mmselfsup/simclr/simclr_resnet50_16xb256-coslr-200e_in1k_20220423_205520.log.json) | | [MoCo v2](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mocov2/README.md) | [mocov2_resnet50_8xb32-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mocov2/mocov2_resnet50_8xb32-coslr-200e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/moco/mocov2_resnet50_8xb32-coslr-200e_in1k_20220225-89e03af4.pth) | [log](https://download.openmmlab.com/mmselfsup/moco/mocov2_resnet50_8xb32-coslr-200e_in1k_20220210_110905.log.json) | | [BYOL](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/byol/README.md) | [byol_resnet50_8xb32-accum16-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/byol/byol_resnet50_8xb32-accum16-coslr-200e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/byol/byol_resnet50_8xb32-accum16-coslr-200e_in1k_20220225-5c8b2c2e.pth) | [log](https://download.openmmlab.com/mmselfsup/byol/byol_resnet50_8xb32-accum16-coslr-200e_in1k_20220214_115709.log.json) | | | [byol_resnet50_8xb32-accum16-coslr-300e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/byol/byol_resnet50_8xb32-accum16-coslr-300e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/byol/byol_resnet50_8xb32-accum16-coslr-300e_in1k_20220225-a0daa54a.pth) | [log](https://download.openmmlab.com/mmselfsup/byol/byol_resnet50_8xb32-accum16-coslr-300e_in1k_20220210_095852.log.json) | @@ -22,8 +23,8 @@ All models and part of benchmark results are recorded below. | [BarlowTwins](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/barlowtwins/README.md) | [barlowtwins_resnet50_8xb256-coslr-300e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/barlowtwins/barlowtwins_resnet50_8xb256-coslr-300e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/barlowtwins/barlowtwins_resnet50_8xb256-coslr-300e_in1k_20220419-5ae15f89.pth) | [log](https://download.openmmlab.com/mmselfsup/barlowtwins/barlowtwins_resnet50_8xb256-coslr-300e_in1k_20220413_111555.log.json) | | [MoCo v3](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mocov3/README.md) | [mocov3_vit-small-p16_32xb128-fp16-coslr-300e_in1k-224](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mocov3/mocov3_vit-small-p16_32xb128-fp16-coslr-300e_in1k-224.py) | [model](https://download.openmmlab.com/mmselfsup/moco/mocov3_vit-small-p16_32xb128-fp16-coslr-300e_in1k-224_20220225-e31238dd.pth) | [log](https://download.openmmlab.com/mmselfsup/moco/mocov3_vit-small-p16_32xb128-fp16-coslr-300e_in1k-224_20220222_160222.log.json) | | [MAE](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mae/README.md) | [mae_vit-base-p16_8xb512-coslr-400e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mae/mae_vit-base-p16_8xb512-coslr-400e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/mae/mae_vit-base-p16_8xb512-coslr-400e_in1k-224_20220223-85be947b.pth) | [log](https://download.openmmlab.com/mmselfsup/mae/mae_vit-base-p16_8xb512-coslr-300e_in1k-224_20220210_140925.log.json) | -| [SimMIM](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simmim/README.md) | [simmim_swin-base_16xb128-coslr-100e_in1k-192](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simmim/simmim_swin-base_16xb128-coslr-100e_in1k-192.py) | [model](https://download.openmmlab.com/mmselfsup/simmim/simmim_swin-base_16xb128-coslr-100e_in1k-192_20220316-1d090125.pth) | [log](https://download.openmmlab.com/mmselfsup/simmim/simmim_swin-base_16xb128-coslr-100e_in1k-192_20220316-1d090125.log.json) | -| [CAE](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simmim/README.md) | [cae_vit-base-p16_8xb256-fp16-coslr-300e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/cae/cae_vit-base-p16_8xb256-fp16-coslr-300e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/cae/cae_vit-base-p16_16xb256-coslr-300e_in1k-224_20220427-4c786349.pth) | [log](https://download.openmmlab.com/mmselfsup/cae/cae_vit-base-p16_16xb256-coslr-300e_in1k-224_20220427-4c786349.log.json) | +| [SimMIM](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simmim/README.md) | [simmim_swin-base_16xb128-coslr-100e_in1k-192](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simmim/simmim_swin-base_16xb128-coslr-100e_in1k-192.py) | [model](https://download.openmmlab.com/mmselfsup/simmim/simmim_swin-base_16xb128-coslr-100e_in1k-192_20220316-1d090125.pth) | [log](https://download.openmmlab.com/mmselfsup/simmim/simmim_swin-base_16xb128-coslr-100e_in1k-192_20220316-1d090125.log.json) | +| [CAE](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simmim/README.md) | [cae_vit-base-p16_8xb256-fp16-coslr-300e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/cae/cae_vit-base-p16_8xb256-fp16-coslr-300e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/cae/cae_vit-base-p16_16xb256-coslr-300e_in1k-224_20220427-4c786349.pth) | [log](https://download.openmmlab.com/mmselfsup/cae/cae_vit-base-p16_16xb256-coslr-300e_in1k-224_20220427-4c786349.log.json) | Remarks: @@ -46,7 +47,8 @@ If not specified, we use linear evaluation setting from [MoCo](http://openaccess | DeepCluster | [deepcluster-sobel_resnet50_8xb64-steplr-200e_in1k.py](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/deepcluster/deepcluster-sobel_resnet50_8xb64-steplr-200e_in1k.py) | | 46.92 | | NPID | [npid_resnet50_8xb32-steplr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/npid/npid_resnet50_8xb32-steplr-200e_in1k.py) | | 58.97 | | ODC | [odc_resnet50_8xb64-steplr-440e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/odc/odc_resnet50_8xb64-steplr-440e_in1k.py) | | 53.43 | -| SimCLR | [simclr_resnet50_8xb32-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k.py) | | 57.28 | +| SimCLR | [simclr_resnet50_8xb32-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k.py) | SimSiam paper setting | 62.56 | +| | [simclr_resnet50_16xb256-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/simclr_resnet50_16xb256-coslr-200e_in1k.py) | SimSiam paper setting | 66.66 | | MoCo v2 | [mocov2_resnet50_8xb32-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mocov2/mocov2_resnet50_8xb32-coslr-200e_in1k.py) | | 67.58 | | BYOL | [byol_resnet50_8xb32-accum16-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/byol/byol_resnet50_8xb32-accum16-coslr-200e_in1k.py) | | 67.55 | | | [byol_resnet50_8xb32-accum16-coslr-300e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/byol/byol_resnet50_8xb32-accum16-coslr-300e_in1k.py) | | 68.55 | @@ -58,11 +60,11 @@ If not specified, we use linear evaluation setting from [MoCo](http://openaccess | MoCo v3 | [mocov3_vit-small-p16_32xb128-fp16-coslr-300e_in1k-224](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mocov3/mocov3_vit-small-p16_32xb128-fp16-coslr-300e_in1k-224.py) | MoCo v3 paper setting | 73.19 | ### ImageNet Fine-tuning -| Algorithm | Config | Remarks | Top-1 (%) | -| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | --------- | -| MAE | [mae_vit-base-p16_8xb512-coslr-400e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mae/mae_vit-base-p16_8xb512-coslr-400e_in1k.py) | | 83.1 | -| SimMIM | [simmim_swin-base_16xb128-coslr-100e_in1k-192](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simmim/simmim_swin-base_16xb128-coslr-100e_in1k-192.py) | | 82.9 | -| CAE | [cae_vit-base-p16_8xb256-fp16-coslr-300e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/cae/cae_vit-base-p16_8xb256-fp16-coslr-300e_in1k.py) | | 83.2 | +| Algorithm | Config | Remarks | Top-1 (%) | +| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | --------- | +| MAE | [mae_vit-base-p16_8xb512-coslr-400e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mae/mae_vit-base-p16_8xb512-coslr-400e_in1k.py) | | 83.1 | +| SimMIM | [simmim_swin-base_16xb128-coslr-100e_in1k-192](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simmim/simmim_swin-base_16xb128-coslr-100e_in1k-192.py) | | 82.9 | +| CAE | [cae_vit-base-p16_8xb256-fp16-coslr-300e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/cae/cae_vit-base-p16_8xb256-fp16-coslr-300e_in1k.py) | | 83.2 | ### COCO17 Object Detection and Instance Segmentation diff --git a/docs/zh_cn/model_zoo.md b/docs/zh_cn/model_zoo.md index 22314ccea..c9143ef75 100644 --- a/docs/zh_cn/model_zoo.md +++ b/docs/zh_cn/model_zoo.md @@ -11,7 +11,8 @@ | [DeepCluster](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/deepcluster/README.md) | [deepcluster-sobel_resnet50_8xb64-steplr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/deepcluster/deepcluster-sobel_resnet50_8xb64-steplr-200e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/deepcluster/deepcluster-sobel_resnet50_8xb64-steplr-200e_in1k-bb8681e2.pth) | | [NPID](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/npid/README.md) | [npid_resnet50_8xb32-steplr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/npid/npid_resnet50_8xb32-steplr-200e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/npid/npid_resnet50_8xb32-steplr-200e_in1k_20220225-5fbbda2a.pth) | [log](https://download.openmmlab.com/mmselfsup/npid/npid_resnet50_8xb32-steplr-200e_in1k_20220215_185513.log.json) | | [ODC](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/odc/README.md) | [odc_resnet50_8xb64-steplr-440e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/odc/odc_resnet50_8xb64-steplr-440e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/odc/odc_resnet50_8xb64-steplr-440e_in1k_20220225-a755d9c0.pth) | [log](https://download.openmmlab.com/mmselfsup/odc/odc_resnet50_8xb64-steplr-440e_in1k_20220215_235245.log.json) | -| [SimCLR](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/README.md) | [simclr_resnet50_8xb32-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k.py) | [model](simclr_resnet50_8xb32-coslr-200e_in1k_20220225-97d2abef.pth) | [log](https://download.openmmlab.com/mmselfsup/simclr/simclr_resnet50_8xb64-coslr-200e_in1k_20220210_191629.log.json) | +| [SimCLR](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/README.md) | [simclr_resnet50_8xb32-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k_20220428-46ef6bb9.pth) | [log](https://download.openmmlab.com/mmselfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k_20220411_182427.log.json) | +| | [simclr_resnet50_16xb256-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/simclr_resnet50_16xb256-coslr-200e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/simclr/simclr_resnet50_16xb256-coslr-200e_in1k_20220428-8c24b063.pth) | [log](https://download.openmmlab.com/mmselfsup/simclr/simclr_resnet50_16xb256-coslr-200e_in1k_20220423_205520.log.json) | | [MoCo v2](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mocov2/README.md) | [mocov2_resnet50_8xb32-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mocov2/mocov2_resnet50_8xb32-coslr-200e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/moco/mocov2_resnet50_8xb32-coslr-200e_in1k_20220225-89e03af4.pth) | [log](https://download.openmmlab.com/mmselfsup/moco/mocov2_resnet50_8xb32-coslr-200e_in1k_20220210_110905.log.json) | | [BYOL](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/byol/README.md) | [byol_resnet50_8xb32-accum16-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/byol/byol_resnet50_8xb32-accum16-coslr-200e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/byol/byol_resnet50_8xb32-accum16-coslr-200e_in1k_20220225-5c8b2c2e.pth) | [log](https://download.openmmlab.com/mmselfsup/byol/byol_resnet50_8xb32-accum16-coslr-200e_in1k_20220214_115709.log.json) | | | [byol_resnet50_8xb32-accum16-coslr-300e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/byol/byol_resnet50_8xb32-accum16-coslr-300e_in1k.py) | [model](https://download.openmmlab.com/mmselfsup/byol/byol_resnet50_8xb32-accum16-coslr-300e_in1k_20220225-a0daa54a.pth) | [log](https://download.openmmlab.com/mmselfsup/byol/byol_resnet50_8xb32-accum16-coslr-300e_in1k_20220210_095852.log.json) | @@ -39,23 +40,24 @@ 如果没有特殊说明，下列实验采用 [MoCo](http://openaccess.thecvf.com/content_CVPR_2020/papers/He_Momentum_Contrast_for_Unsupervised_Visual_Representation_Learning_CVPR_2020_paper.pdf) 的设置，或者采用的训练设置写在备注中。 -| 算法 | 配置文件 | 备注 | Top-1 (%) | +| 算法 | 配置文件 | 备注 | Top-1 (%) | | ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | --------- | | Relative Location | [relative-loc_resnet50_8xb64-steplr-70e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/relative_loc/relative-loc_resnet50_8xb64-steplr-70e_in1k.py) | | 38.78 | | Rotation Prediction | [rotation-pred_resnet50_8xb16-steplr-70e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/rotation_pred/rotation-pred_resnet50_8xb16-steplr-70e_in1k.py) | | 48.12 | | DeepCluster | [deepcluster-sobel_resnet50_8xb64-steplr-200e_in1k.py](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/deepcluster/deepcluster-sobel_resnet50_8xb64-steplr-200e_in1k.py) | | 46.92 | | NPID | [npid_resnet50_8xb32-steplr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/npid/npid_resnet50_8xb32-steplr-200e_in1k.py) | | 58.97 | | ODC | [odc_resnet50_8xb64-steplr-440e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/odc/odc_resnet50_8xb64-steplr-440e_in1k.py) | | 53.43 | -| SimCLR | [simclr_resnet50_8xb32-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k.py) | | 57.28 | +| SimCLR | [simclr_resnet50_8xb32-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/simclr_resnet50_8xb32-coslr-200e_in1k.py) | SimSiam 论文设置 | 62.56 | +| | [simclr_resnet50_16xb256-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simclr/simclr_resnet50_16xb256-coslr-200e_in1k.py) | SimSiam 论文设置 | 66.66 | | MoCo v2 | [mocov2_resnet50_8xb32-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mocov2/mocov2_resnet50_8xb32-coslr-200e_in1k.py) | | 67.58 | | BYOL | [byol_resnet50_8xb32-accum16-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/byol/byol_resnet50_8xb32-accum16-coslr-200e_in1k.py) | | 67.55 | | | [byol_resnet50_8xb32-accum16-coslr-300e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/byol/byol_resnet50_8xb32-accum16-coslr-300e_in1k.py) | | 68.55 | -| SwAV | [swav_resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/swav/swav_resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96.py) | SwAV 论文设置 | 70.47 | +| SwAV | [swav_resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/swav/swav_resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96.py) | SwAV 论文设置 | 70.47 | | DenseCL | [densecl_resnet50_8xb32-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/densecl/densecl_resnet50_8xb32-coslr-200e_in1k.py) | | 63.62 | -| SimSiam | [simsiam_resnet50_8xb32-coslr-100e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simsiam/simsiam_resnet50_8xb32-coslr-100e_in1k.py) | SimSiam 论文设置 | 68.28 | -| | [simsiam_resnet50_8xb32-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simsiam/simsiam_resnet50_8xb32-coslr-200e_in1k.py) | SimSiam 论文设置 | 69.84 | -| Barlow Twins | [barlowtwins_resnet50_8xb256-coslr-300e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/barlowtwins/barlowtwins_resnet50_8xb256-coslr-300e_in1k.py) | Barlow Twins 论文设置 | 71.66 | -| MoCo v3 | [mocov3_vit-small-p16_32xb128-fp16-coslr-300e_in1k-224](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mocov3/mocov3_vit-small-p16_32xb128-fp16-coslr-300e_in1k-224.py) | MoCo v3 论文设置 | 73.19 | +| SimSiam | [simsiam_resnet50_8xb32-coslr-100e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simsiam/simsiam_resnet50_8xb32-coslr-100e_in1k.py) | SimSiam 论文设置 | 68.28 | +| | [simsiam_resnet50_8xb32-coslr-200e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/simsiam/simsiam_resnet50_8xb32-coslr-200e_in1k.py) | SimSiam 论文设置 | 69.84 | +| Barlow Twins | [barlowtwins_resnet50_8xb256-coslr-300e_in1k](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/barlowtwins/barlowtwins_resnet50_8xb256-coslr-300e_in1k.py) | Barlow Twins 论文设置 | 71.66 | +| MoCo v3 | [mocov3_vit-small-p16_32xb128-fp16-coslr-300e_in1k-224](https://github.com/open-mmlab/mmselfsup/blob/master/configs/selfsup/mocov3/mocov3_vit-small-p16_32xb128-fp16-coslr-300e_in1k-224.py) | MoCo v3 论文设置 | 73.19 | ### ImageNet 微调 | 算法 | 配置文件 | 备注 | Top-1 (%) | diff --git a/mmselfsup/models/backbones/resnet.py b/mmselfsup/models/backbones/resnet.py index 954c90bae..18dcddf07 100644 --- a/mmselfsup/models/backbones/resnet.py +++ b/mmselfsup/models/backbones/resnet.py @@ -43,7 +43,7 @@ class ResNet(_ResNet): with_cp (bool): Use checkpoint or not. Using checkpoint will save some memory while slowing down the training speed. Defaults to False. zero_init_residual (bool): Whether to use zero init for last norm layer - in resblocks to let them behave as identity. Defaults to True. + in resblocks to let them behave as identity. Defaults to False. Probability of the path to be zeroed. Defaults to 0.1 Example: >>> from mmselfsup.models import ResNet @@ -86,7 +86,7 @@ def __init__(self, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=False, with_cp=False, - zero_init_residual=True, + zero_init_residual=False, init_cfg=[ dict(type='Kaiming', layer=['Conv2d']), dict( diff --git a/mmselfsup/models/backbones/resnext.py b/mmselfsup/models/backbones/resnext.py index 6de89a6cf..10ac0ac6a 100644 --- a/mmselfsup/models/backbones/resnext.py +++ b/mmselfsup/models/backbones/resnext.py @@ -51,7 +51,7 @@ class ResNeXt(ResNet): with_cp (bool): Use checkpoint or not. Using checkpoint will save some memory while slowing down the training speed. Defaults to False. zero_init_residual (bool): Whether to use zero init for last norm layer - in resblocks to let them behave as identity. Defaults to True. + in resblocks to let them behave as identity. Defaults to False. Example: >>> from mmselfsup.models import ResNeXt