diff --git a/changelog.d/4325.feature b/changelog.d/4325.feature
new file mode 100644
index 000000000000..05ad70fe72da
--- /dev/null
+++ b/changelog.d/4325.feature
@@ -0,0 +1 @@
+Synapse can now have its conditional/extra dependencies installed by pip. This functionality can be used by using `pip install matrix-synapse[feature]`, where feature is a comma separated list with the possible values "email.enable_notifs", "ldap3", "postgres", "saml2", "url_preview", and "test". If you want to install all optional dependencies, you can use "all" instead.
diff --git a/debian/build_virtualenv b/debian/build_virtualenv
index 61ffb131921f..e9ef245476e7 100755
--- a/debian/build_virtualenv
+++ b/debian/build_virtualenv
@@ -33,7 +33,8 @@ dh_virtualenv \
     --preinstall="lxml" \
     --preinstall="mock" \
     --extra-pip-arg="--no-cache-dir" \
-    --extra-pip-arg="--compile"
+    --extra-pip-arg="--compile" \
+    --extras="all"
 
 # we copy the tests to a temporary directory so that we can put them on the
 # PYTHONPATH without putting the uninstalled synapse on the pythonpath.
diff --git a/debian/control b/debian/control
index 552a81dcb080..9606a4c135e7 100644
--- a/debian/control
+++ b/debian/control
@@ -5,7 +5,7 @@ Maintainer: Synapse Packaging team <packages@matrix.org>
 Build-Depends:
  debhelper (>= 9),
  dh-systemd,
- dh-virtualenv (>= 1.0),
+ dh-virtualenv (>= 1.1),
  lsb-release,
  python3-dev,
  python3,
diff --git a/docker/Dockerfile-dhvirtualenv b/docker/Dockerfile-dhvirtualenv
index ea6b650af264..ab28e49291dc 100644
--- a/docker/Dockerfile-dhvirtualenv
+++ b/docker/Dockerfile-dhvirtualenv
@@ -11,6 +11,35 @@
 
 # Get the distro we want to pull from as a dynamic build variable
 ARG distro=""
+
+###
+### Stage 0: build a dh-virtualenv
+###
+FROM ${distro} as builder
+
+RUN apt-get update -qq -o Acquire::Languages=none
+RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
+        -yqq --no-install-recommends \
+        build-essential \
+        ca-certificates \
+        devscripts \
+        equivs \
+        wget
+
+# fetch and unpack the package
+RUN wget -q -O /dh-virtuenv-1.1.tar.gz https://github.com/spotify/dh-virtualenv/archive/1.1.tar.gz
+RUN tar xvf /dh-virtuenv-1.1.tar.gz
+
+# install its build deps
+RUN cd dh-virtualenv-1.1/ \
+    && env DEBIAN_FRONTEND=noninteractive mk-build-deps -ri -t "apt-get -yqq --no-install-recommends"
+
+# build it
+RUN cd dh-virtualenv-1.1 && dpkg-buildpackage -us -uc -b
+
+###
+### Stage 1
+###
 FROM ${distro}
 
 # Install the build dependencies
@@ -21,15 +50,15 @@ RUN apt-get update -qq -o Acquire::Languages=none \
         debhelper \
         devscripts \
         dh-systemd \
-        dh-virtualenv \
-        equivs \
         lsb-release \
         python3-dev \
         python3-pip \
         python3-setuptools \
         python3-venv \
-        sqlite3 \
-        wget
+        sqlite3
+
+COPY --from=builder /dh-virtualenv_1.1-1_all.deb /
+RUN apt-get install -yq /dh-virtualenv_1.1-1_all.deb
 
 WORKDIR /synapse/source
 ENTRYPOINT ["bash","/synapse/source/docker/build_debian.sh"]
diff --git a/docker/build_debian.sh b/docker/build_debian.sh
index cea5067fe9f3..6ed2b398986d 100644
--- a/docker/build_debian.sh
+++ b/docker/build_debian.sh
@@ -6,20 +6,6 @@ set -ex
 
 DIST=`lsb_release -c -s`
 
-# We need to build a newer dh_virtualenv on older OSes like Xenial.
-if [ "$DIST" = 'xenial' ]; then
-    mkdir -p /tmp/dhvenv
-    cd /tmp/dhvenv
-    wget https://github.com/spotify/dh-virtualenv/archive/1.1.tar.gz
-    tar xvf 1.1.tar.gz
-    cd dh-virtualenv-1.1/
-    env DEBIAN_FRONTEND=noninteractive mk-build-deps -ri -t "apt-get -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io"
-    dpkg-buildpackage -us -uc -b
-    cd /tmp/dhvenv
-    apt-get install -yqq ./dh-virtualenv_1.1-1_all.deb
-fi
-
-
 # we get a read-only copy of the source: make a writeable copy
 cp -aT /synapse/source /synapse/build
 cd /synapse/build