diff --git a/.github/workflows/ci_dockers.yml b/.github/workflows/ci_dockers.yml
index 9f77fb76aa593..897e16a12d44f 100644
--- a/.github/workflows/ci_dockers.yml
+++ b/.github/workflows/ci_dockers.yml
@@ -29,9 +29,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
 
-      # https://github.com/docker/setup-buildx-action
-      # Set up Docker Buildx - to use cache-from and cache-to argument of buildx command
-      - uses: docker/setup-buildx-action@v1
       - name: Build PL Docker
         # publish master/release
         uses: docker/build-push-action@v2
@@ -54,9 +51,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
 
-      # https://github.com/docker/setup-buildx-action
-      # Set up Docker Buildx - to use cache-from and cache-to argument of buildx command
-      - uses: docker/setup-buildx-action@v1
       - name: Build XLA Docker
         # publish master/release
         uses: docker/build-push-action@v2
@@ -93,9 +87,6 @@ jobs:
           echo "::set-output name=CUDA::$cuda"
         id: extend
 
-      # https://github.com/docker/setup-buildx-action
-      # Set up Docker Buildx - to use cache-from and cache-to argument of buildx command
-      - uses: docker/setup-buildx-action@v1
       - name: Build CUDA Docker
         # publish master/release
         uses: docker/build-push-action@v2
@@ -130,9 +121,6 @@ jobs:
           echo "::set-output name=CUDA::$cuda"
         id: extend
 
-      # https://github.com/docker/setup-buildx-action
-      # Set up Docker Buildx - to use cache-from and cache-to argument of buildx command
-      - uses: docker/setup-buildx-action@v1
       - name: Build CUDA Docker
         # publish master/release
         uses: docker/build-push-action@v2
@@ -150,10 +138,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      # https://github.com/docker/setup-buildx-action
-      # Set up Docker Buildx - to use cache-from and cache-to argument of buildx command
-      - uses: docker/setup-buildx-action@v1
-      - name: Build CUDA Docker
+
+      - name: Build NVIDIA Docker
         uses: docker/build-push-action@v2
         with:
           file: dockers/nvidia/Dockerfile
diff --git a/.github/workflows/events-nightly.yml b/.github/workflows/events-nightly.yml
index 24d8ce4002e5d..5ad4396a006f7 100644
--- a/.github/workflows/events-nightly.yml
+++ b/.github/workflows/events-nightly.yml
@@ -126,3 +126,26 @@ jobs:
           push: true
           tags: pytorchlightning/pytorch_lightning:base-conda-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}
         timeout-minutes: 55
+
+#  docker-nvidia:
+#    runs-on: ubuntu-20.04
+#    steps:
+#      - name: Checkout
+#        uses: actions/checkout@v2
+#
+#      # https://github.com/docker/setup-buildx-action
+#      # Set up Docker Buildx - to use cache-from and cache-to argument of buildx command
+#      - uses: docker/setup-buildx-action@v1
+#      - name: Login to DockerHub
+#        uses: docker/login-action@v1
+#        with:
+#          username: ${{ secrets.DOCKER_USERNAME }}
+#          password: ${{ secrets.DOCKER_PASSWORD }}
+#
+#      - name: Publish NVIDIA to Docker Hub
+#        uses: docker/build-push-action@v2
+#        with:
+#          file: dockers/nvidia/Dockerfile
+#          push: true
+#          tags: nvcr.io/pytorchlightning/pytorch_lightning:nvidia
+#        timeout-minutes: 55
diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml
index f285794cbc33b..36ecbe229ac7c 100644
--- a/.github/workflows/release-docker.yml
+++ b/.github/workflows/release-docker.yml
@@ -8,7 +8,7 @@ on:
     types: [created]
 
 jobs:
-  build-PL:
+  cuda-PL:
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
@@ -36,3 +36,27 @@ jobs:
           build_args: PYTHON_VERSION=${{ matrix.python_version }},PYTORCH_VERSION=${{ matrix.pytorch_version }},LIGHTNING_VERSION=${{ steps.get_version.outputs.RELEASE_VERSION }}
           tags: "${{ steps.get_version.outputs.RELEASE_VERSION }}-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }},latest-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}"
         timeout-minutes: 55
+
+#  nvidia-PL:
+#    runs-on: ubuntu-20.04
+#    steps:
+#      - name: Checkout
+#        uses: actions/checkout@v2
+#
+#      - name: Get release version
+#        if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'release'
+#        id: get_version
+#        run: echo "::set-output name=RELEASE_VERSION::$(echo ${GITHUB_REF##*/})"
+#
+#      - name: Publish Releases to Docker
+#        # only on releases
+#        uses: docker/build-push-action@v1.1.0
+#        if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'release'
+#        with:
+#          repository: nvcr.io/pytorchlightning/pytorch_lightning
+#          username: ${{ secrets.DOCKER_USERNAME }}
+#          password: ${{ secrets.DOCKER_PASSWORD }}
+#          dockerfile: dockers/nvidia/Dockerfile
+#          build_args: LIGHTNING_VERSION=${{ steps.get_version.outputs.RELEASE_VERSION }}
+#          tags: "${{ steps.get_version.outputs.RELEASE_VERSION }}-nvidia"
+#        timeout-minutes: 55
diff --git a/dockers/nvidia/Dockerfile b/dockers/nvidia/Dockerfile
index ea567a5306eed..4b04bc9426d4d 100644
--- a/dockers/nvidia/Dockerfile
+++ b/dockers/nvidia/Dockerfile
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM nvcr.io/nvidia/pytorch:20.12-py3
+FROM nvcr.io/nvidia/pytorch:21.02-py3
 
 MAINTAINER PyTorchLightning <https://github.com/PyTorchLightning>
 
@@ -22,16 +22,17 @@ COPY ./ ./pytorch-lightning/
 
 # install dependencies
 RUN \
-    # Disable cache
     #conda install "pip>20.1" && \
-    #pip config set global.cache-dir false && \
-    if [ -z $LIGHTNING_VERSION ] ; then \
-        pip install ./pytorch-lightning --no-cache-dir ; \
+    pip list | grep torch && \
+    if [ ! -z "$LIGHTNING_VERSION" ] ; then \
         rm -rf pytorch-lightning ; \
-    else \
-        rm -rf pytorch-lightning ; \
-        pip install https://github.com/PyTorchLightning/pytorch-lightning/archive/${LIGHTNING_VERSION}.zip --no-cache-dir ; \
-    fi
+        wget https://github.com/PyTorchLightning/pytorch-lightning/archive/${LIGHTNING_VERSION}.zip --progress=bar:force:noscroll ; \
+        unzip ${LIGHTNING_VERSION}.zip ; \
+        mv pytorch-lightning-*/ pytorch-lightning ; \
+        rm *.zip ; \
+    fi && \
+    pip install ./pytorch-lightning["extra"] --no-cache-dir && \
+    rm -rf pytorch-lightning
 
 RUN python --version && \
     pip --version && \
diff --git a/dockers/release/Dockerfile b/dockers/release/Dockerfile
index 3584ee02746e3..0eec1e41a5a3f 100644
--- a/dockers/release/Dockerfile
+++ b/dockers/release/Dockerfile
@@ -25,7 +25,6 @@ COPY ./ ./pytorch-lightning/
 
 # install dependencies
 RUN \
-    # Disable cache
     #conda install "pip>20.1" && \
     if [ ! -z "$LIGHTNING_VERSION" ] ; then \
         rm -rf pytorch-lightning ; \