[requirements] Split requirements into different tiers [build_base] (#…

…36808)

This PR splits the requirements files into different tiers:

- `air/core-requirements.txt` - these are the most important requirements (tier 1)
- `air/[library]-requirements.txt` - these are pinned dependencies for libraries (tier 2+3)
- `air/[library]-test-requirements.txt` - these are pinned test-dependencies for libraries (tier 4)

(tier 2 and 3 may get merged into one tier, tbd).

This does not change any packages and should not affect installed packages in docker or CI.

The naming format was chosen to be compatible with dependabot.

In a follow-up PR, we will compile the tier 1-3 dependencies to ensure all CI runners and Docker images run on the same dependency versions.

This PR will split existing requirement files. In total, there will be 4 more such files (`rllib-test-requirements.txt`, `tune-test-requirements.txt`, `train-test-requirements.txt`, `docker/ray-docker-requirements.txt`

Signed-off-by: Kai Fricke <kai@anyscale.com>

.buildkite/pipeline.build.yml

@@ -231,9 +231,11 @@
   instance_size: large
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - ./ci/env/env_info.sh
+    # Todo (krfricke): Move mosaicml to train-test-requirements.txt
+    - pip install "mosaicml==0.12.1"
     - DOC_TESTING=1 ./ci/env/install-dependencies.sh
     - ./ci/env/install-horovod.sh
+    - ./ci/env/env_info.sh
     - bazel test --config=ci $(./scripts/bazel_export_options)
       --test_tag_filters=doctest,-gpu
       python/ray/... doc/...

.buildkite/pipeline.gpu.yml

@@ -6,7 +6,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+#    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu python/ray/tune/...
 
@@ -16,7 +16,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     # --jobs 1 is necessary as we only have 1 GPU on the machine and running tests in parallel
     # would cause timeouts as the other scripts would wait for the GPU to become available.
@@ -40,7 +40,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - bazel test --config=ci $(./ci/run/bazel_export_options) --test_tag_filters=gpu python/ray/serve/...
 
 # Todo: enable once tests pass
@@ -49,7 +49,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+#    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_C,examples_D --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1 rllib/...
@@ -60,7 +60,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+#    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_E,examples_F,examples_G,examples_H,examples_I,examples_J,examples_K,examples_L,examples_M,examples_N,examples_O,examples_P --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1
@@ -72,7 +72,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+#    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_Q,examples_R,examples_S,examples_T,examples_U,examples_V,examples_W,examples_X,examples_Y,examples_Z --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1

.buildkite/pipeline.gpu_large.yml

@@ -5,7 +5,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu,gpu_only,-ray_air python/ray/train/...
@@ -15,7 +15,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - TRAIN_TESTING=1 DATA_PROCESSING_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=datasets_train,-doctest doc/...
 
@@ -25,7 +25,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     # --jobs 2 is necessary as we only need to have at least 2 gpus on the machine
     # and running tests in parallel would cause timeouts as the other scripts would
@@ -42,7 +42,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DATA_PROCESSING_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu python/ray/air/... python/ray/train/...
@@ -53,7 +53,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     # Test examples with newer version of `transformers`
     # TODO(amogkam): Remove when https://github.com/ray-project/ray/issues/36011
@@ -65,7 +65,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     # Test examples with newer version of `transformers`
@@ -82,7 +82,7 @@
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - NO_DASHBOARD=1 ./ci/env/install-minimal.sh 3.8
     - PYTHON=3.8 DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - pip uninstall -y pytorch-lightning
     - pip install lightning==2.0.0
     - ./ci/env/env_info.sh

.buildkite/pipeline.ml.yml

@@ -35,6 +35,8 @@
   parallelism: 4
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    # Todo (krfricke): Move mosaicml to train-test-requirements.txt
+    - pip install "mosaicml==0.12.1"
     - TRAIN_TESTING=1 DATA_PROCESSING_TESTING=1 INSTALL_HOROVOD=1 ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh
     - ./ci/run/run_bazel_test_with_sharding.sh

.github/CODEOWNERS

@@ -101,8 +101,8 @@
 /doc/source/ray-air/ @richardliaw @gjoliver @krfricke @xwjiang2010 @amogkam @matthewdeng @Yard1 @maxpumperla @ray-project/ray-docs
 
 # ML Docker Dependencies
-/python/requirements/ml/requirements_dl.txt @amogkam @krfricke @richardliaw @matthewdeng
-/python/requirements/ml/requirements_ml_docker.txt @amogkam @krfricke @richardliaw @matthewdeng
+/python/requirements/ml/dl-cpu-requirements.txt @amogkam @krfricke @richardliaw @matthewdeng
+/python/requirements/ml/dl-gpu-requirements.txt @amogkam @krfricke @richardliaw @matthewdeng
 
 # Ray symbol export
 /src/ray/ray_version_script.lds @iycheng @ericl @scv119

ci/build/build-docker-images.py

@@ -426,20 +426,27 @@ def prep_ray_ml():
         "python/requirements.txt",
     ]
     ml_requirements_files = [
-        "python/requirements/ml/requirements_ml_docker.txt",
-        "python/requirements/ml/requirements_dl.txt",
-        "python/requirements/ml/requirements_tune.txt",
-        "python/requirements/ml/requirements_rllib.txt",
-        "python/requirements/ml/requirements_train.txt",
-        "python/requirements/ml/requirements_upstream.txt",
-        "python/requirements/ml/requirements_no_deps.txt",
+        "python/requirements/docker/ray-docker-requirements.txt",
+        "python/requirements/ml/core-requirements.txt",
+        "python/requirements/ml/data-requirements.txt",
+        "python/requirements/ml/dl-gpu-requirements.txt",
+        "python/requirements/ml/dl-cpu-requirements.txt",
+        "python/requirements/ml/tune-requirements.txt",
+        "python/requirements/ml/tune-test-requirements.txt",
+        "python/requirements/ml/rllib-requirements.txt",
+        "python/requirements/ml/rllib-test-requirements.txt",
+        "python/requirements/ml/train-requirements.txt",
+        "python/requirements/ml/train-test-requirements.txt",
     ]
-    # We don't need these in the ml docker image
+    # We don't need these in the ml docker image (or they are installed elsewhere)
     ignore_requirements = [
         "python/requirements/compat/requirements_legacy_compat.txt",
+        "python/requirements/ml/data-test-requirements.txt",
     ]
 
-    files_on_disk = glob.glob(f"{root_dir}/python/**/requirements*.txt", recursive=True)
+    files_on_disk = glob.glob(
+        f"{root_dir}/python/**/*-requirements.txt", recursive=True
+    )
     for file_on_disk in files_on_disk:
         rel = os.path.relpath(file_on_disk, start=root_dir)
         print(rel)

ci/build/upload_build_info.sh

@@ -35,7 +35,7 @@ if [[ -z "${BUILDKITE-}" ]]; then
     aws s3 cp --recursive /tmp/bazel_event_logs "${DST}"
 else
     # Codepath for Buildkite
-    # Keep cryptography/openssl in sync with `requirements_test.txt`
+    # Keep cryptography/openssl in sync with `requirements/test-requirements.txt`
     pip install -q -c "${RAY_DIR}/python/requirements.txt" docker aws_requests_auth boto3 cryptography==38.0.1 PyOpenSSL==22.1.0
     python .buildkite/copy_files.py --destination logs --path /tmp/bazel_event_logs
 fi

ci/ci.sh

@@ -115,6 +115,56 @@ upload_wheels() {
   )
 }
 
+
+compile_pip_dependencies() {
+  # Compile boundaries
+
+  if [[ "${HOSTTYPE}" == "aarch64" || "${HOSTTYPE}" = "arm64" ]]; then
+    # Resolution currently does not work on aarch64 as some pinned packages
+    # are not available. Once they are reasonably upgraded we should be able
+    # to enable this here.p
+    echo "Skipping for aarch64"
+    return 0
+  fi
+
+  # shellcheck disable=SC2262
+  alias pip="python -m pip"
+  pip install pip-tools
+
+  # Required packages to lookup e.g. dragonfly-opt
+  HAS_TORCH=0
+  python -c "import torch" 2>/dev/null && HAS_TORCH=1
+  pip install --no-cache-dir numpy torch
+
+  if [ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt" ]; then
+    echo requirements_compiled already exists
+  else
+    pip-compile --resolver=backtracking -q \
+       --pip-args --no-deps --strip-extras --no-annotate --no-header -o \
+      "${WORKSPACE_DIR}/python/requirements_compiled.txt" \
+      "${WORKSPACE_DIR}/python/requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/lint-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/test-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/docker/ray-docker-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/core-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/data-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/data-test-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/dl-cpu-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/rllib-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/rllib-test-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/train-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/train-test-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/tune-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/tune-test-requirements.txt"
+  fi
+
+  cat "${WORKSPACE_DIR}/python/requirements_compiled.txt"
+
+  if [ "$HAS_TORCH" -eq 0 ]; then
+    pip uninstall -y torch
+  fi
+}
+
 test_core() {
   local args=(
     "//:*"

ci/docker/base.test.Dockerfile

@@ -65,3 +65,4 @@ WORKDIR /ray
 COPY . .
 
 RUN ./ci/env/install-dependencies.sh init
+RUN bash --login -i ./ci/ci.sh compile_pip_dependencies