[CI] Always install into a python venv in ci containers (apache#12663)

This PR changes all ci_ to install TVM Python dependencies in a
virtualenv separate from the system Python dependencies.

 Sets the stage for adding the poetry-based dependency
generator to the CI container build process.

* Always install into a python venv in ci containers.
* Respect Dockerfile ENV PATH modifications in
docker/bash.sh lookups.

docker/Dockerfile.ci_arm

@@ -47,11 +47,12 @@ ENV PATH /opt/sccache:$PATH
 COPY install/ubuntu_install_llvm.sh /install/ubuntu_install_llvm.sh
 RUN bash /install/ubuntu_install_llvm.sh
 
-COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
-RUN bash /install/ubuntu1804_install_python.sh
-
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
+ENV TVM_VENV /venv/apache-tvm-py3.7
+COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+ENV PATH ${TVM_VENV}/bin:$PATH
+ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
 COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh
 RUN bash /install/ubuntu_install_cmake_source.sh

docker/Dockerfile.ci_cortexm

@@ -29,18 +29,15 @@ RUN bash /install/ubuntu_install_core.sh
 COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
 RUN bash /install/ubuntu_install_googletest.sh
 
-COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
-RUN bash /install/ubuntu1804_install_python.sh
-
 COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh
 RUN bash /install/ubuntu_install_cmake_source.sh 3.20.0
 
-COPY install/ubuntu1804_install_python_venv.sh /install/ubuntu1804_install_python_venv.sh
-RUN bash /install/ubuntu1804_install_python_venv.sh
-ENV PATH=/opt/tvm-venv/bin:/opt/zephyr-sdk/sysroots/x86_64-pokysdk-linux/usr/bin:$PATH
-
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
+ENV TVM_VENV /venv/apache-tvm-py3.7
+COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+ENV PATH ${TVM_VENV}/bin:$PATH
+ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
 COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
 RUN bash /install/ubuntu_install_python_package.sh

docker/Dockerfile.ci_cpu

@@ -28,11 +28,12 @@ RUN bash /install/ubuntu_install_core.sh
 COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
 RUN bash /install/ubuntu_install_googletest.sh
 
-COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
-RUN bash /install/ubuntu1804_install_python.sh
-
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
+ENV TVM_VENV /venv/apache-tvm-py3.7
+COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+ENV PATH ${TVM_VENV}/bin:$PATH
+ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
 COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
 RUN bash /install/ubuntu_install_python_package.sh

docker/Dockerfile.ci_gpu

@@ -38,13 +38,15 @@ RUN bash /install/ubuntu_install_cmake_source.sh
 COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
 RUN bash /install/ubuntu_install_googletest.sh
 
-COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
-RUN bash /install/ubuntu1804_install_python.sh
-
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
+ENV TVM_VENV /venv/apache-tvm-py3.7
+COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+ENV PATH ${TVM_VENV}/bin:$PATH
+ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
 COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh
+# Globally disable pip cache
 RUN bash /install/ubuntu_install_cmake_source.sh
 
 COPY install/ubuntu1804_install_llvm.sh /install/ubuntu1804_install_llvm.sh

docker/Dockerfile.ci_hexagon

@@ -31,11 +31,12 @@ RUN bash /install/ubuntu_install_core.sh
 COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
 RUN bash /install/ubuntu_install_googletest.sh
 
-COPY install/ubuntu2004_install_python.sh /install/ubuntu2004_install_python.sh
-RUN bash /install/ubuntu2004_install_python.sh
-
-# Globally disable pip cache
-RUN pip config set global.cache-dir false
+ENV TVM_VENV /venv/apache-tvm-py3.8
+COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+ENV PATH ${TVM_VENV}/bin:$PATH
+ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
 # Rust env (build early; takes a while)
 COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh

docker/Dockerfile.ci_i386

@@ -35,18 +35,19 @@ RUN bash /install/ubuntu_install_googletest.sh
 COPY install/ubuntu_install_llvm.sh /install/ubuntu_install_llvm.sh
 RUN bash /install/ubuntu_install_llvm.sh
 
-COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
-RUN bash /install/ubuntu1804_install_python.sh
-
 # Rust env (build early; takes a while)
 COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh
 RUN bash /install/ubuntu_install_rust.sh
 ENV RUSTUP_HOME /opt/rust
 ENV CARGO_HOME /opt/rust
 ENV PATH $PATH:$CARGO_HOME/bin
+ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
+ENV TVM_VENV /venv/apache-tvm-py3.7
+COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+ENV PATH ${TVM_VENV}/bin:$PATH
 
 COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh
 RUN bash /install/ubuntu_install_cmake_source.sh

docker/Dockerfile.ci_lint

@@ -26,11 +26,12 @@ RUN apt-get update --fix-missing
 
 RUN apt-install-and-clear -y wget git sudo make parallel
 
-COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
-RUN bash /install/ubuntu1804_install_python.sh
-
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
+ENV TVM_VENV /venv/apache-tvm-py3.7
+COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+ENV PATH ${TVM_VENV}/bin:$PATH
+ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
 RUN apt-get update && apt-install-and-clear -y doxygen graphviz curl shellcheck
 

docker/Dockerfile.ci_minimal

@@ -28,11 +28,12 @@ RUN bash /install/ubuntu_install_core.sh
 COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
 RUN bash /install/ubuntu_install_googletest.sh
 
-COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
-RUN bash /install/ubuntu1804_install_python.sh
-
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
+ENV TVM_VENV /venv/apache-tvm-py3.7
+COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+ENV PATH ${TVM_VENV}/bin:$PATH
+ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
 COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
 RUN bash /install/ubuntu_install_python_package.sh

docker/Dockerfile.ci_riscv

@@ -29,18 +29,15 @@ RUN bash /install/ubuntu_install_core.sh
 COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
 RUN bash /install/ubuntu_install_googletest.sh
 
-COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
-RUN bash /install/ubuntu1804_install_python.sh
-
 COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh
 RUN bash /install/ubuntu_install_cmake_source.sh
 
-COPY install/ubuntu1804_install_python_venv.sh /install/ubuntu1804_install_python_venv.sh
-RUN bash /install/ubuntu1804_install_python_venv.sh
-ENV PATH=/opt/tvm-venv/bin:/opt/zephyr-sdk/sysroots/x86_64-pokysdk-linux/usr/bin:$PATH
-
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
+ENV TVM_VENV /venv/apache-tvm-py3.7
+COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+ENV PATH ${TVM_VENV}/bin:$PATH
+ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
 COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
 RUN bash /install/ubuntu_install_python_package.sh

docker/Dockerfile.ci_wasm

@@ -26,11 +26,12 @@ RUN bash /install/ubuntu_install_core.sh
 COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
 RUN bash /install/ubuntu_install_googletest.sh
 
-COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
-RUN bash /install/ubuntu1804_install_python.sh
-
-# Globally disable pip cache
-RUN pip config set global.no-cache-dir false
+ENV TVM_VENV /venv/apache-tvm-py3.7
+COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+ENV PATH ${TVM_VENV}/bin:$PATH
+ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
 COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
 RUN bash /install/ubuntu_install_python_package.sh

docker/install/ubuntu_install_python.sh

@@ -18,28 +18,77 @@
 
 set -e
 set -u
-# Used for debugging RVM build
-set -x
 set -o pipefail
 
-# install python and pip, don't modify this, modify install_python_package.sh
+set -x
+
+if [ -z "${TVM_VENV+x}" ]; then
+    echo "ERROR: expect TVM_VENV env var to be set"
+    exit 2
+fi
+
 apt-get update
-apt-install-and-clear -y python-dev
 
-# python 3.6
+# Ensure lsb-release is installed.
+apt-install-and-clear -y \
+    lsb-core
+
+release=$(lsb_release -sc)
+if [ "${release}" == "bionic" ]; then
+    PYTHON_VERSION=3.7
+elif [ "${release}" == "focal" ]; then
+    PYTHON_VERSION=3.8
+else
+    echo "Don't know which version of python to install for lsb-release ${release}"
+    exit 2
+fi
+
+# Install python and pip. Don't modify this to add Python package dependencies,
+# instead modify install_python_package.sh
 apt-install-and-clear -y software-properties-common
+apt-install-and-clear -y \
+    acl \
+    python${PYTHON_VERSION} \
+    python${PYTHON_VERSION}-dev \
+    python3-pip \
+    python${PYTHON_VERSION}-venv
 
-add-apt-repository -y ppa:deadsnakes/ppa
-apt-get update
-apt-install-and-clear -y python-pip python-dev python3.6 python3.6-dev
+update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1
+
+# Allow disabling user site-packages, even with sudo; this makes it harder to repro CI failures
+# locally because it's hard to tell what might be in this directory.
+echo "Defaults env_keep += \"PYTHONNOUSERSITE\"" >/etc/sudoers.d/91-preserve-python-nousersite
+export PYTHONNOUSERSITE=1
+
+venv_dir="$(python3 -c "import os.path;print(os.path.dirname(\"${TVM_VENV}\"))")"
+mkdir -p "${venv_dir}"
+python3 -mvenv "${TVM_VENV}"
+. "${TVM_VENV}/bin/activate"
+
+# Update pip to match version used to produce requirements-hashed.txt. This step
+# is necessary so that pip's dependency solver is recent.
+pip_spec=$(cat /install/python/bootstrap/lockfiles/constraints-${PYTHON_VERSION}.txt | grep 'pip==')
+pip3 install -U --require-hashes -r <(echo "${pip_spec}") \
+     -c /install/python/bootstrap/lockfiles/constraints-${PYTHON_VERSION}.txt
 
-rm -f /usr/bin/python3 && ln -s /usr/bin/python3.6 /usr/bin/python3
+# Python configuration
+pip3 config set global.no-cache-dir true  # Never cache packages
 
-# python 3.7
-apt-install-and-clear -y python3.7
+# Now install the remaining base packages.
+pip3 install \
+     --require-hashes \
+     -r /install/python/bootstrap/lockfiles/constraints-${PYTHON_VERSION}.txt
 
-# Install pip
-wget -q https://bootstrap.pypa.io/get-pip.py && python3.7 get-pip.py
+addgroup tvm-venv
+chgrp -R tvm-venv "${TVM_VENV}"
+setfacl -R -d -m group:tvm-venv:rwx "${TVM_VENV}"
 
-# Pin pip and setuptools versions
-pip3 install pip==19.3.1 setuptools==58.4.0
+# Prevent further use of pip3 via the system.
+# There may be multiple (i.e. from python3-pip apt package and pip3 install -U).
+deactivate
+while [ "$(which pip3)" != "" ]; do
+    rm "$(which pip3)"
+done
+while [ "$(which pip)" != "" ]; do
+    rm "$(which pip)"
+done