diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh
index 1a3e5c6f4d2..bec090a491e 100755
--- a/.circleci/unittest/linux/scripts/install.sh
+++ b/.circleci/unittest/linux/scripts/install.sh
@@ -26,5 +26,11 @@ fi
 printf "Installing PyTorch with %s\n" "${cudatoolkit}"
 conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c conda-forge "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}"
 
+if [ $PYTHON_VERSION == "3.6" ]; then
+    printf "Installing minimal PILLOW version\n"
+    # Install the minimal PILLOW version. Otherwise, let setup.py install the latest
+    pip install pillow==5.3.0
+fi
+
 printf "* Installing torchvision\n"
 python setup.py develop
diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh
index 9304b4b9b65..ac5222a7b90 100644
--- a/.circleci/unittest/windows/scripts/install.sh
+++ b/.circleci/unittest/windows/scripts/install.sh
@@ -28,5 +28,11 @@ fi
 printf "Installing PyTorch with %s\n" "${cudatoolkit}"
 conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c conda-forge "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}"
 
+if [ $PYTHON_VERSION == "3.6" ]; then
+    printf "Installing minimal PILLOW version\n"
+    # Install the minimal PILLOW version. Otherwise, let setup.py install the latest
+    pip install pillow==5.3.0
+fi
+
 printf "* Installing torchvision\n"
 "$this_dir/vc_env_helper.bat" python setup.py develop
diff --git a/setup.py b/setup.py
index 23bbdaab378..ff4c48d4cbb 100644
--- a/setup.py
+++ b/setup.py
@@ -67,7 +67,7 @@ def write_version_file():
     pytorch_dep,
 ]
 
-pillow_ver = ' >= 4.1.1'
+pillow_ver = ' >= 5.3.0'
 pillow_req = 'pillow-simd' if get_dist('pillow-simd') is not None else 'pillow'
 requirements.append(pillow_req + pillow_ver)
 
diff --git a/torchvision/transforms/functional.py b/torchvision/transforms/functional.py
index d36e68c2b6f..64ed0b8edd5 100644
--- a/torchvision/transforms/functional.py
+++ b/torchvision/transforms/functional.py
@@ -55,7 +55,6 @@ def _interpolation_modes_from_int(i: int) -> InterpolationMode:
 }
 
 _is_pil_image = F_pil._is_pil_image
-_parse_fill = F_pil._parse_fill
 
 
 def _get_image_size(img: Tensor) -> List[int]:
diff --git a/torchvision/transforms/functional_pil.py b/torchvision/transforms/functional_pil.py
index 9059db03683..3829637fdb7 100644
--- a/torchvision/transforms/functional_pil.py
+++ b/torchvision/transforms/functional_pil.py
@@ -3,7 +3,7 @@
 
 import numpy as np
 import torch
-from PIL import Image, ImageOps, ImageEnhance, __version__ as PILLOW_VERSION
+from PIL import Image, ImageOps, ImageEnhance
 
 try:
     import accimage
@@ -147,7 +147,7 @@ def pad(img, padding, fill=0, padding_mode="constant"):
         raise ValueError("Padding mode should be either constant, edge, reflect or symmetric")
 
     if padding_mode == "constant":
-        opts = _parse_fill(fill, img, "2.3.0", name="fill")
+        opts = _parse_fill(fill, img, name="fill")
         if img.mode == "P":
             palette = img.getpalette()
             image = ImageOps.expand(img, border=padding, **opts)
@@ -242,18 +242,8 @@ def resize(img, size, interpolation=Image.BILINEAR, max_size=None):
 
 
 @torch.jit.unused
-def _parse_fill(fill, img, min_pil_version, name="fillcolor"):
+def _parse_fill(fill, img, name="fillcolor"):
     # Process fill color for affine transforms
-    major_found, minor_found = (int(v) for v in PILLOW_VERSION.split('.')[:2])
-    major_required, minor_required = (int(v) for v in min_pil_version.split('.')[:2])
-    if major_found < major_required or (major_found == major_required and minor_found < minor_required):
-        if fill is None:
-            return {}
-        else:
-            msg = ("The option to fill background area of the transformed image, "
-                   "requires pillow>={}")
-            raise RuntimeError(msg.format(min_pil_version))
-
     num_bands = len(img.getbands())
     if fill is None:
         fill = 0
@@ -276,7 +266,7 @@ def affine(img, matrix, interpolation=0, fill=None):
         raise TypeError('img should be PIL Image. Got {}'.format(type(img)))
 
     output_size = img.size
-    opts = _parse_fill(fill, img, '5.0.0')
+    opts = _parse_fill(fill, img)
     return img.transform(output_size, Image.AFFINE, matrix, interpolation, **opts)
 
 
@@ -285,7 +275,7 @@ def rotate(img, angle, interpolation=0, expand=False, center=None, fill=None):
     if not _is_pil_image(img):
         raise TypeError("img should be PIL Image. Got {}".format(type(img)))
 
-    opts = _parse_fill(fill, img, '5.2.0')
+    opts = _parse_fill(fill, img)
     return img.rotate(angle, interpolation, expand, center, **opts)
 
 
@@ -294,7 +284,7 @@ def perspective(img, perspective_coeffs, interpolation=Image.BICUBIC, fill=None)
     if not _is_pil_image(img):
         raise TypeError('img should be PIL Image. Got {}'.format(type(img)))
 
-    opts = _parse_fill(fill, img, '5.0.0')
+    opts = _parse_fill(fill, img)
 
     return img.transform(img.size, Image.PERSPECTIVE, perspective_coeffs, interpolation, **opts)