[requirements] Split requirements into different tiers [build_base]

.buildkite/pipeline.build.yml

@@ -231,9 +231,11 @@
   instance_size: large
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - ./ci/env/env_info.sh
+    # Todo (krfricke): Move mosaicml to train-test-requirements.txt
+    - pip install "mosaicml==0.12.1"
     - DOC_TESTING=1 ./ci/env/install-dependencies.sh
     - ./ci/env/install-horovod.sh
+    - ./ci/env/env_info.sh
     - bazel test --config=ci $(./scripts/bazel_export_options)
       --test_tag_filters=doctest,-gpu
       python/ray/... doc/...

.buildkite/pipeline.gpu.yml

@@ -6,7 +6,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+#    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu python/ray/tune/...
 
@@ -16,7 +16,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     # --jobs 1 is necessary as we only have 1 GPU on the machine and running tests in parallel
     # would cause timeouts as the other scripts would wait for the GPU to become available.
@@ -40,7 +40,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - bazel test --config=ci $(./ci/run/bazel_export_options) --test_tag_filters=gpu python/ray/serve/...
 
 # Todo: enable once tests pass
@@ -49,7 +49,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+#    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_C,examples_D --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1 rllib/...
@@ -60,7 +60,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+#    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_E,examples_F,examples_G,examples_H,examples_I,examples_J,examples_K,examples_L,examples_M,examples_N,examples_O,examples_P --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1
@@ -72,7 +72,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+#    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_Q,examples_R,examples_S,examples_T,examples_U,examples_V,examples_W,examples_X,examples_Y,examples_Z --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1

.buildkite/pipeline.gpu_large.yml

@@ -5,7 +5,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu,gpu_only,-ray_air python/ray/train/...
@@ -15,7 +15,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - TRAIN_TESTING=1 DATA_PROCESSING_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=datasets_train,-doctest doc/...
 
@@ -25,7 +25,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     # --jobs 2 is necessary as we only need to have at least 2 gpus on the machine
     # and running tests in parallel would cause timeouts as the other scripts would
@@ -42,7 +42,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DATA_PROCESSING_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu python/ray/air/... python/ray/train/...
@@ -53,7 +53,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     # Test examples with newer version of `transformers`
     # TODO(amogkam): Remove when https://github.com/ray-project/ray/issues/36011
@@ -65,7 +65,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     # Test examples with newer version of `transformers`
@@ -82,7 +82,7 @@
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - NO_DASHBOARD=1 ./ci/env/install-minimal.sh 3.8
     - PYTHON=3.8 DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - pip uninstall -y pytorch-lightning
     - pip install lightning==2.0.0
     - ./ci/env/env_info.sh

.buildkite/pipeline.ml.yml

@@ -35,6 +35,8 @@
   parallelism: 4
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    # Todo (krfricke): Move mosaicml to train-test-requirements.txt
+    - pip install "mosaicml==0.12.1"
     - TRAIN_TESTING=1 DATA_PROCESSING_TESTING=1 INSTALL_HOROVOD=1 ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh
     - ./ci/run/run_bazel_test_with_sharding.sh

.github/CODEOWNERS

@@ -101,8 +101,8 @@
 /doc/source/ray-air/ @richardliaw @gjoliver @krfricke @xwjiang2010 @amogkam @matthewdeng @Yard1 @maxpumperla @ray-project/ray-docs
 
 # ML Docker Dependencies
-/python/requirements/ml/requirements_dl.txt @amogkam @krfricke @richardliaw @matthewdeng
-/python/requirements/ml/requirements_ml_docker.txt @amogkam @krfricke @richardliaw @matthewdeng
+/python/requirements/ml/dl-cpu-requirements.txt @amogkam @krfricke @richardliaw @matthewdeng
+/python/requirements/ml/dl-gpu-requirements.txt @amogkam @krfricke @richardliaw @matthewdeng
 
 # Ray symbol export
 /src/ray/ray_version_script.lds @iycheng @ericl @scv119

ci/build/build-docker-images.py

@@ -426,20 +426,27 @@ def prep_ray_ml():
         "python/requirements.txt",
     ]
     ml_requirements_files = [
-        "python/requirements/ml/requirements_ml_docker.txt",
-        "python/requirements/ml/requirements_dl.txt",
-        "python/requirements/ml/requirements_tune.txt",
-        "python/requirements/ml/requirements_rllib.txt",
-        "python/requirements/ml/requirements_train.txt",
-        "python/requirements/ml/requirements_upstream.txt",
-        "python/requirements/ml/requirements_no_deps.txt",
+        "python/requirements/docker/ray-docker-requirements.txt",
+        "python/requirements/ml/core-requirements.txt",
+        "python/requirements/ml/data-requirements.txt",
+        "python/requirements/ml/dl-gpu-requirements.txt",
+        "python/requirements/ml/dl-cpu-requirements.txt",
+        "python/requirements/ml/tune-requirements.txt",
+        "python/requirements/ml/tune-test-requirements.txt",
+        "python/requirements/ml/rllib-requirements.txt",
+        "python/requirements/ml/rllib-test-requirements.txt",
+        "python/requirements/ml/train-requirements.txt",
+        "python/requirements/ml/train-test-requirements.txt",
     ]
-    # We don't need these in the ml docker image
+    # We don't need these in the ml docker image (or they are installed elsewhere)
     ignore_requirements = [
         "python/requirements/compat/requirements_legacy_compat.txt",
+        "python/requirements/ml/data-test-requirements.txt",
     ]
 
-    files_on_disk = glob.glob(f"{root_dir}/python/**/requirements*.txt", recursive=True)
+    files_on_disk = glob.glob(
+        f"{root_dir}/python/**/*-requirements.txt", recursive=True
+    )
     for file_on_disk in files_on_disk:
         rel = os.path.relpath(file_on_disk, start=root_dir)
         print(rel)

ci/build/upload_build_info.sh

@@ -35,7 +35,7 @@ if [[ -z "${BUILDKITE-}" ]]; then
     aws s3 cp --recursive /tmp/bazel_event_logs "${DST}"
 else
     # Codepath for Buildkite
-    # Keep cryptography/openssl in sync with `requirements_test.txt`
+    # Keep cryptography/openssl in sync with `requirements/test-requirements.txt`
     pip install -q -c "${RAY_DIR}/python/requirements.txt" docker aws_requests_auth boto3 cryptography==38.0.1 PyOpenSSL==22.1.0
     python .buildkite/copy_files.py --destination logs --path /tmp/bazel_event_logs
 fi

ci/ci.sh

@@ -115,6 +115,56 @@ upload_wheels() {
   )
 }
 
+
+compile_pip_dependencies() {
+  # Compile boundaries
+
+  if [[ "${HOSTTYPE}" == "aarch64" || "${HOSTTYPE}" = "arm64" ]]; then
+    # Resolution currently does not work on aarch64 as some pinned packages
+    # are not available. Once they are reasonably upgraded we should be able
+    # to enable this here.p
+    echo "Skipping for aarch64"
+    return 0
+  fi
+
+  # shellcheck disable=SC2262
+  alias pip="python -m pip"
+  pip install pip-tools
+
+  # Required packages to lookup e.g. dragonfly-opt
+  HAS_TORCH=0
+  python -c "import torch" 2>/dev/null && HAS_TORCH=1
+  pip install --no-cache-dir numpy torch
+
+  if [ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt" ]; then
+    echo requirements_compiled already exists
+  else
+    pip-compile --resolver=backtracking -q \
+       --pip-args --no-deps --strip-extras --no-annotate --no-header -o \
+      "${WORKSPACE_DIR}/python/requirements_compiled.txt" \
+      "${WORKSPACE_DIR}/python/requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/lint-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/test-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/docker/ray-docker-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/core-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/data-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/data-test-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/dl-cpu-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/rllib-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/rllib-test-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/train-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/train-test-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/tune-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/tune-test-requirements.txt"
+  fi
+
+  cat "${WORKSPACE_DIR}/python/requirements_compiled.txt"
+
+  if [ "$HAS_TORCH" -eq 0 ]; then
+    pip uninstall -y torch
+  fi
+}
+
 test_core() {
   local args=(
     "//:*"

ci/docker/base.test.Dockerfile

@@ -65,3 +65,4 @@ WORKDIR /ray
 COPY . .
 
 RUN ./ci/env/install-dependencies.sh init
+RUN bash --login -i ./ci/ci.sh compile_pip_dependencies