diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 3e8aee16522..745671cd531 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -27,26 +27,23 @@ jobs:
     strategy:
       matrix:
         python-version: [3.7]
-        torch: [1.3.1, 1.5.1, 1.6.0, 1.7.0, 1.8.0, 1.9.0]
+        torch: [1.5.1, 1.6.0, 1.7.0, 1.8.0, 1.9.0]
         include:
-          - torch: 1.3.1
-            torchvision: 0.4.2
-            mmcv: "latest+torch1.3.0+cpu"
           - torch: 1.5.1
             torchvision: 0.6.1
-            mmcv: "latest+torch1.5.0+cpu"
+            mmcv: 1.5.0
           - torch: 1.6.0
             torchvision: 0.7.0
-            mmcv: "latest+torch1.6.0+cpu"
+            mmcv: 1.6.0
           - torch: 1.7.0
             torchvision: 0.8.1
-            mmcv: "latest+torch1.7.0+cpu"
+            mmcv: 1.7.0
           - torch: 1.8.0
             torchvision: 0.9.0
-            mmcv: "latest+torch1.8.0+cpu"
+            mmcv: 1.8.0
           - torch: 1.9.0
             torchvision: 0.10.0
-            mmcv: "latest+torch1.9.0+cpu"
+            mmcv: 1.9.0
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
@@ -60,7 +57,7 @@ jobs:
         run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
       - name: Install MMCV
         run: |
-          pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch${{matrix.torch}}/index.html
+          pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch${{matrix.mmcv}}/index.html
           python -c 'import mmcv; print(mmcv.__version__)'
       - name: Install unittest dependencies
         run: |
@@ -85,33 +82,28 @@ jobs:
         python-version: [3.7]
         torch:
           [
-            1.3.1,
             1.5.1+cu101,
             1.6.0+cu101,
             1.7.0+cu101,
             1.8.0+cu101,
           ]
         include:
-          - torch: 1.3.1
-            torch_version: torch1.3.1
-            torchvision: 0.4.2
-            mmcv_link: "torch1.3.0"
           - torch: 1.5.1+cu101
             torch_version: torch1.5.1
             torchvision: 0.6.1+cu101
-            mmcv_link: "torch1.5.0"
+            mmcv: 1.5.0
           - torch: 1.6.0+cu101
             torch_version: torch1.6.0
             torchvision: 0.7.0+cu101
-            mmcv_link: "torch1.6.0"
+            mmcv: 1.6.0
           - torch: 1.7.0+cu101
             torch_version: torch1.7.0
             torchvision: 0.8.1+cu101
-            mmcv_link: "torch1.7.0"
+            mmcv: 1.7.0
           - torch: 1.8.0+cu101
             torch_version: torch1.8.0
             torchvision: 0.9.0+cu101
-            mmcv_link: "torch1.8.0"
+            mmcv: 1.8.0
 
     steps:
       - uses: actions/checkout@v2
@@ -135,7 +127,7 @@ jobs:
       - name: Install mmdet dependencies
         run: |
           python -V
-          python -m pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cu101/${{matrix.mmcv_link}}/index.html
+          python -m pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cu101/torch${{matrix.mmcv}}/index.html
           python -m pip install pycocotools
           python -m pip install -r requirements/tests.txt -r requirements/optional.txt
           python -m pip install albumentations>=0.3.2 --no-binary imgaug,albumentations
@@ -173,7 +165,7 @@ jobs:
           - torch: 1.9.0+cu102
             torch_version: torch1.9.0
             torchvision: 0.10.0+cu102
-            mmcv_link: "torch1.9.0"
+            mmcv: 1.9.0
 
     steps:
       - uses: actions/checkout@v2
@@ -208,7 +200,7 @@ jobs:
       - name: Install mmdet dependencies
         run: |
           python -V
-          python -m pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cu102/${{matrix.mmcv_link}}/index.html
+          python -m pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cu102/torch${{matrix.mmcv}}/index.html
           python -m pip install pycocotools
           python -m pip install -r requirements/tests.txt -r requirements/optional.txt
           python -m pip install albumentations>=0.3.2 --no-binary imgaug,albumentations
diff --git a/README.md b/README.md
index e162c510912..ce876adccc4 100644
--- a/README.md
+++ b/README.md
@@ -29,7 +29,7 @@ English | [简体中文](README_zh-CN.md)
 MMDetection is an open source object detection toolbox based on PyTorch. It is
 a part of the [OpenMMLab](https://openmmlab.com/) project.
 
-The master branch works with **PyTorch 1.3+**.
+The master branch works with **PyTorch 1.5+**.
 
 <details open>
 <summary>Major features</summary>
diff --git a/README_zh-CN.md b/README_zh-CN.md
index 42006085182..5ccf06aec2f 100644
--- a/README_zh-CN.md
+++ b/README_zh-CN.md
@@ -28,7 +28,7 @@
 
 MMDetection 是一个基于 PyTorch 的目标检测开源工具箱。它是 [OpenMMLab](https://openmmlab.com/) 项目的一部分。
 
-主分支代码目前支持 PyTorch 1.3 以上的版本。
+主分支代码目前支持 PyTorch 1.5 以上的版本。
 
 
 <details open>