[requirements] Install with compiled requirements as constraints [bui…

…ld_base] (ray-project#36983)

Following up from ray-project#36808, this PR now uses the compiled constraints file in the installation of the dependencies in CI and in the ray-ml docker images. In result, both environments will share the same versions of (sub)dependencies.

Signed-off-by: Kai Fricke <kai@anyscale.com>

ci/build/build-docker-images.py

@@ -424,6 +424,7 @@ def prep_ray_ml():
 
     requirements_files = [
         "python/requirements.txt",
+        "python/requirements_compiled.txt",
     ]
     ml_requirements_files = [
         "python/requirements/docker/ray-docker-requirements.txt",

ci/ci.sh

@@ -158,6 +158,10 @@ compile_pip_dependencies() {
       "${WORKSPACE_DIR}/python/requirements/ml/tune-test-requirements.txt"
   fi
 
+  # Remove some pins from upstream dependencies:
+  # ray, xgboost-ray, lightgbm-ray, tune-sklearn
+  sed -i "/^ray==/d;/^xgboost-ray==/d;/^lightgbm-ray==/d;/^tune-sklearn==/d" "${WORKSPACE_DIR}/python/requirements_compiled.txt"
+
   cat "${WORKSPACE_DIR}/python/requirements_compiled.txt"
 
   if [ "$HAS_TORCH" -eq 0 ]; then

ci/docker/base.ml.Dockerfile

@@ -1,15 +1,4 @@
 ARG DOCKER_IMAGE_BASE_TEST
 FROM $DOCKER_IMAGE_BASE_TEST
 
-# Move out of working dir /ray
-# Delete stale data
-WORKDIR /
-RUN rm -rf /ray
-
-RUN mkdir /ray
-WORKDIR /ray
-
-# Copy new ray files
-COPY . .
-
 RUN RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh

ci/docker/base.test.Dockerfile

@@ -65,4 +65,3 @@ WORKDIR /ray
 COPY . .
 
 RUN ./ci/env/install-dependencies.sh init
-RUN bash --login -i ./ci/ci.sh compile_pip_dependencies

ci/env/install-dependencies.sh

@@ -457,6 +457,16 @@ install_pip_packages() {
 
   # Generate the pip command with collected requirements files
   pip_cmd="pip install -U -c ${WORKSPACE_DIR}/python/requirements.txt"
+
+  if [[ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt"  &&  "${PYTHON-}" != "3.7" && "${OSTYPE}" != msys ]]; then
+    # On Python 3.7, we don't, as the dependencies are compiled for 3.8+
+    # and we don't build ray-ml images. This means we don't have to keep
+    # consistency between CI and docker images.
+    # On Windows, some pinned dependencies are not built for win, so we
+    # skip this until we have a good wy to resolve cross-platform dependencies.
+    pip_cmd+=" -c ${WORKSPACE_DIR}/python/requirements_compiled.txt"
+  fi
+
   for file in "${requirements_files[@]}"; do
      pip_cmd+=" -r ${file}"
   done

doc/requirements-doc.txt

@@ -27,7 +27,7 @@ pytorch-lightning==1.6.5
 scikit-optimize
 redis
 starlette
-uvicorn==0.16.0
+uvicorn==0.22.0
 werkzeug
 wandb
 tensorflow; sys_platform != 'darwin' or platform_machine != 'arm64'

docker/ray-ml/Dockerfile

@@ -3,6 +3,7 @@ FROM rayproject/ray:nightly"$BASE_IMAGE"
 
 # We have to uninstall wrapt this way for Tensorflow compatibility
 COPY requirements.txt ./
+COPY requirements_compiled.txt ./
 COPY dl-cpu-requirements.txt ./
 COPY dl-gpu-requirements.txt ./
 COPY ray-docker-requirements.txt ./

docker/ray-ml/install-ml-docker-requirements.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 # shellcheck disable=SC2139
 alias pip="$HOME/anaconda3/bin/pip"
 
@@ -17,10 +19,10 @@ pip --no-cache-dir install -U pip pip-tools
 # Install requirements
 pip --no-cache-dir install -U -r requirements.txt
 
-
 # Install other requirements. Keep pinned requirements bounds as constraints
 pip --no-cache-dir install -U \
            -c requirements.txt \
+           -c requirements_compiled.txt \
            -r core-requirements.txt \
            -r data-requirements.txt \
            -r rllib-requirements.txt \
@@ -31,11 +33,18 @@ pip --no-cache-dir install -U \
            -r tune-test-requirements.txt \
            -r ray-docker-requirements.txt
 
+
+# Remove any device-specific constraints from requirements_compiled.txt.
+# E.g.: torch-scatter==2.1.1+pt20cpu or torchvision==0.15.2+cpu
+# These are replaced with gpu-specific requirements in dl-gpu-requirements.txt.
+sed "/[0-9]\+cpu/d;/[0-9]\+pt/d" "requirements_compiled.txt" > requirements_compiled_gpu.txt
+
 # explicitly install (overwrite) pytorch with CUDA support
-pip --no-cache-dir install -U \
+pip --no-cache-dir install \
            -c requirements.txt \
+           -c requirements_compiled_gpu.txt \
            -r dl-gpu-requirements.txt
 
 sudo apt-get clean
 
-sudo rm requirements*.txt
+sudo rm ./*requirements*.txt

python/build-wheel-manylinux2014.sh

@@ -81,8 +81,9 @@ for PYTHON_NUMPY in "${PYTHON_NUMPYS[@]}" ; do
   # The -f flag is passed twice to also run git clean in the arrow subdirectory.
   # The -d flag removes directories. The -x flag ignores the .gitignore file,
   # and the -e flag ensures that we don't remove the .whl directory, the
-  # dashboard directory and jars directory.
-  git clean -f -f -x -d -e .whl -e python/ray/dashboard/client -e dashboard/client -e python/ray/jars
+  # dashboard directory and jars directory, as well as the compiled
+  # dependency constraints.
+  git clean -f -f -x -d -e .whl -e python/ray/dashboard/client -e dashboard/client -e python/ray/jars -e python/requirements_compiled.txt
 
   (
     cd python
@@ -122,7 +123,7 @@ for path in .whl/*.whl; do
 done
 
 # Clean the build output so later operations is on a clean directory.
-git clean -f -f -x -d -e .whl -e python/ray/dashboard/client
+git clean -f -f -x -d -e .whl -e python/ray/dashboard/client -e python/requirements_compiled.txt
 
 echo "--- Build JAR"
 if [ "${BUILD_JAR-}" == "1" ]; then

python/requirements/ml/dl-gpu-requirements.txt

@@ -1,7 +1,12 @@
 # If you make changes below this line, please also make the corresponding changes to `dl-cpu-requirements.txt`!
 
---extra-index-url https://download.pytorch.org/whl/cu116  # for GPU versions of torch, torchvision
---find-links https://data.pyg.org/whl/torch-1.13.0+cu116.html  # for GPU versions of torch-scatter, torch-sparse, torch-cluster, torch-spline-conv
+tensorflow==2.11.0; sys_platform != 'darwin' or platform_machine != 'arm64'
+tensorflow-macos==2.11.0; sys_platform == 'darwin' and platform_machine == 'arm64'
+tensorflow-probability==0.19.0
+tensorflow-datasets
+
+--extra-index-url https://download.pytorch.org/whl/cu118  # for GPU versions of torch, torchvision
+--find-links https://data.pyg.org/whl/torch-2.0.1+cu118.html  # for GPU versions of torch-scatter, torch-sparse, torch-cluster, torch-spline-conv
 # specifying explicit plus-notation below so pip overwrites the existing cpu verisons
 torch==1.13.0+cu116
 torchvision==0.14.0+cu116