Skip to content

Commit

Permalink
Revert "Revert "[requirements] Install with compiled requirements as …
Browse files Browse the repository at this point in the history
…constraints [build_base] (ray-project#36983)" (ray-project#37458)" (ray-project#37460)

This reverts commit 38ca6eb.

Signed-off-by: NripeshN <nn2012@hw.ac.uk>
  • Loading branch information
krfricke authored and NripeshN committed Aug 15, 2023
1 parent d1ee653 commit 9ca3c65
Show file tree
Hide file tree
Showing 11 changed files with 587 additions and 19 deletions.
1 change: 1 addition & 0 deletions ci/build/build-docker-images.py
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,7 @@ def prep_ray_ml():

requirements_files = [
"python/requirements.txt",
"python/requirements_compiled.txt",
]
ml_requirements_files = [
"python/requirements/docker/ray-docker-requirements.txt",
Expand Down
4 changes: 4 additions & 0 deletions ci/ci.sh
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,10 @@ compile_pip_dependencies() {
"${WORKSPACE_DIR}/python/requirements/ml/tune-test-requirements.txt"
fi

# Remove some pins from upstream dependencies:
# ray, xgboost-ray, lightgbm-ray, tune-sklearn
sed -i "/^ray==/d;/^xgboost-ray==/d;/^lightgbm-ray==/d;/^tune-sklearn==/d" "${WORKSPACE_DIR}/python/requirements_compiled.txt"

cat "${WORKSPACE_DIR}/python/requirements_compiled.txt"

if [ "$HAS_TORCH" -eq 0 ]; then
Expand Down
11 changes: 0 additions & 11 deletions ci/docker/base.ml.Dockerfile
Original file line number Diff line number Diff line change
@@ -1,15 +1,4 @@
ARG DOCKER_IMAGE_BASE_TEST
FROM $DOCKER_IMAGE_BASE_TEST

# Move out of working dir /ray
# Delete stale data
WORKDIR /
RUN rm -rf /ray

RUN mkdir /ray
WORKDIR /ray

# Copy new ray files
COPY . .

RUN RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
1 change: 0 additions & 1 deletion ci/docker/base.test.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -65,4 +65,3 @@ WORKDIR /ray
COPY . .

RUN ./ci/env/install-dependencies.sh init
RUN bash --login -i ./ci/ci.sh compile_pip_dependencies
10 changes: 10 additions & 0 deletions ci/env/install-dependencies.sh
Original file line number Diff line number Diff line change
Expand Up @@ -457,6 +457,16 @@ install_pip_packages() {

# Generate the pip command with collected requirements files
pip_cmd="pip install -U -c ${WORKSPACE_DIR}/python/requirements.txt"

if [[ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt" && "${PYTHON-}" != "3.7" && "${OSTYPE}" != msys ]]; then
# On Python 3.7, we don't, as the dependencies are compiled for 3.8+
# and we don't build ray-ml images. This means we don't have to keep
# consistency between CI and docker images.
# On Windows, some pinned dependencies are not built for win, so we
# skip this until we have a good wy to resolve cross-platform dependencies.
pip_cmd+=" -c ${WORKSPACE_DIR}/python/requirements_compiled.txt"
fi

for file in "${requirements_files[@]}"; do
pip_cmd+=" -r ${file}"
done
Expand Down
2 changes: 1 addition & 1 deletion doc/requirements-doc.txt
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ pytorch-lightning==1.6.5
scikit-optimize
redis
starlette
uvicorn==0.16.0
uvicorn==0.22.0
werkzeug
wandb
tensorflow; sys_platform != 'darwin' or platform_machine != 'arm64'
Expand Down
1 change: 1 addition & 0 deletions docker/ray-ml/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ FROM rayproject/ray:nightly"$BASE_IMAGE"

# We have to uninstall wrapt this way for Tensorflow compatibility
COPY requirements.txt ./
COPY requirements_compiled.txt ./
COPY dl-cpu-requirements.txt ./
COPY dl-gpu-requirements.txt ./
COPY ray-docker-requirements.txt ./
Expand Down
15 changes: 12 additions & 3 deletions docker/ray-ml/install-ml-docker-requirements.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#!/bin/bash

set -e

# shellcheck disable=SC2139
alias pip="$HOME/anaconda3/bin/pip"

Expand All @@ -17,10 +19,10 @@ pip --no-cache-dir install -U pip pip-tools
# Install requirements
pip --no-cache-dir install -U -r requirements.txt


# Install other requirements. Keep pinned requirements bounds as constraints
pip --no-cache-dir install -U \
-c requirements.txt \
-c requirements_compiled.txt \
-r core-requirements.txt \
-r data-requirements.txt \
-r rllib-requirements.txt \
Expand All @@ -31,11 +33,18 @@ pip --no-cache-dir install -U \
-r tune-test-requirements.txt \
-r ray-docker-requirements.txt


# Remove any device-specific constraints from requirements_compiled.txt.
# E.g.: torch-scatter==2.1.1+pt20cpu or torchvision==0.15.2+cpu
# These are replaced with gpu-specific requirements in dl-gpu-requirements.txt.
sed "/[0-9]\+cpu/d;/[0-9]\+pt/d" "requirements_compiled.txt" > requirements_compiled_gpu.txt

# explicitly install (overwrite) pytorch with CUDA support
pip --no-cache-dir install -U \
pip --no-cache-dir install \
-c requirements.txt \
-c requirements_compiled_gpu.txt \
-r dl-gpu-requirements.txt

sudo apt-get clean

sudo rm requirements*.txt
sudo rm ./*requirements*.txt
7 changes: 4 additions & 3 deletions python/build-wheel-manylinux2014.sh
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,9 @@ for PYTHON_NUMPY in "${PYTHON_NUMPYS[@]}" ; do
# The -f flag is passed twice to also run git clean in the arrow subdirectory.
# The -d flag removes directories. The -x flag ignores the .gitignore file,
# and the -e flag ensures that we don't remove the .whl directory, the
# dashboard directory and jars directory.
git clean -f -f -x -d -e .whl -e python/ray/dashboard/client -e dashboard/client -e python/ray/jars
# dashboard directory and jars directory, as well as the compiled
# dependency constraints.
git clean -f -f -x -d -e .whl -e python/ray/dashboard/client -e dashboard/client -e python/ray/jars -e python/requirements_compiled.txt

(
cd python
Expand Down Expand Up @@ -127,7 +128,7 @@ for path in .whl/*.whl; do
done

# Clean the build output so later operations is on a clean directory.
git clean -f -f -x -d -e .whl -e python/ray/dashboard/client
git clean -f -f -x -d -e .whl -e python/ray/dashboard/client -e python/requirements_compiled.txt

echo "--- Build JAR"
if [ "${BUILD_JAR-}" == "1" ]; then
Expand Down
5 changes: 5 additions & 0 deletions python/requirements/ml/dl-gpu-requirements.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# If you make changes below this line, please also make the corresponding changes to `dl-cpu-requirements.txt`!

tensorflow==2.11.0; sys_platform != 'darwin' or platform_machine != 'arm64'
tensorflow-macos==2.11.0; sys_platform == 'darwin' and platform_machine == 'arm64'
tensorflow-probability==0.19.0
tensorflow-datasets

--extra-index-url https://download.pytorch.org/whl/cu118 # for GPU versions of torch, torchvision
--find-links https://data.pyg.org/whl/torch-2.0.1+cu118.html # for GPU versions of torch-scatter, torch-sparse, torch-cluster, torch-spline-conv
# specifying explicit plus-notation below so pip overwrites the existing cpu verisons
Expand Down
Loading

0 comments on commit 9ca3c65

Please sign in to comment.