[requirements] Split requirements into different tiers [build_base]

.buildkite/pipeline.gpu.yml

@@ -6,7 +6,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+#    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu python/ray/tune/...
 
@@ -16,7 +16,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     # --jobs 1 is necessary as we only have 1 GPU on the machine and running tests in parallel
     # would cause timeouts as the other scripts would wait for the GPU to become available.
@@ -40,7 +40,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - bazel test --config=ci $(./ci/run/bazel_export_options) --test_tag_filters=gpu python/ray/serve/...
 
 # Todo: enable once tests pass
@@ -49,7 +49,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+#    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_C,examples_D --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1 rllib/...
@@ -60,7 +60,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+#    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_E,examples_F,examples_G,examples_H,examples_I,examples_J,examples_K,examples_L,examples_M,examples_N,examples_O,examples_P --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1
@@ -72,7 +72,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+#    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_Q,examples_R,examples_S,examples_T,examples_U,examples_V,examples_W,examples_X,examples_Y,examples_Z --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1

.buildkite/pipeline.gpu_large.yml

@@ -5,7 +5,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu,gpu_only,-ray_air python/ray/train/...
@@ -15,7 +15,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - TRAIN_TESTING=1 DATA_PROCESSING_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=datasets_train,-doctest doc/...
 
@@ -25,7 +25,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     # --jobs 2 is necessary as we only need to have at least 2 gpus on the machine
     # and running tests in parallel would cause timeouts as the other scripts would
@@ -42,7 +42,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DATA_PROCESSING_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu python/ray/air/... python/ray/train/...
@@ -53,7 +53,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     # Test examples with newer version of `transformers`
     # TODO(amogkam): Remove when https://github.com/ray-project/ray/issues/36011
@@ -65,7 +65,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     # Test examples with newer version of `transformers`
@@ -82,7 +82,7 @@
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - NO_DASHBOARD=1 ./ci/env/install-minimal.sh 3.8
     - PYTHON=3.8 DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - pip uninstall -y pytorch-lightning
     - pip install lightning==2.0.0
     - ./ci/env/env_info.sh

.github/CODEOWNERS

@@ -101,8 +101,8 @@
 /doc/source/ray-air/ @richardliaw @gjoliver @krfricke @xwjiang2010 @amogkam @matthewdeng @Yard1 @maxpumperla @ray-project/ray-docs
 
 # ML Docker Dependencies
-/python/requirements/ml/requirements_dl.txt @amogkam @krfricke @richardliaw @matthewdeng
-/python/requirements/ml/requirements_ml_docker.txt @amogkam @krfricke @richardliaw @matthewdeng
+/python/requirements/air/dl-cpu-requirements.txt @amogkam @krfricke @richardliaw @matthewdeng
+/python/requirements/air/dl-gpu-requirements.txt @amogkam @krfricke @richardliaw @matthewdeng
 
 # Ray symbol export
 /src/ray/ray_version_script.lds @iycheng @ericl @scv119

ci/build/build-docker-images.py

@@ -426,24 +426,32 @@ def prep_ray_ml():
         "python/requirements.txt",
     ]
     ml_requirements_files = [
-        "python/requirements/ml/requirements_ml_docker.txt",
-        "python/requirements/ml/requirements_dl.txt",
-        "python/requirements/ml/requirements_tune.txt",
-        "python/requirements/ml/requirements_rllib.txt",
-        "python/requirements/ml/requirements_train.txt",
-        "python/requirements/ml/requirements_upstream.txt",
-        "python/requirements/ml/requirements_no_deps.txt",
+        "python/requirements/docker/ml-docker-requirements.txt",
+        "python/requirements/docker/ray-docker-requirements.txt",
+        "python/requirements/air/core-requirements.txt",
+        "python/requirements/air/data-requirements.txt",
+        "python/requirements/air/dl-gpu-requirements.txt",
+        "python/requirements/air/dl-cpu-requirements.txt",
+        "python/requirements/air/tune-requirements.txt",
+        "python/requirements/air/tune-test-requirements.txt",
+        "python/requirements/air/rllib-requirements.txt",
+        "python/requirements/air/rllib-test-requirements.txt",
+        "python/requirements/air/train-requirements.txt",
+        "python/requirements/air/train-test-requirements.txt",
     ]
-    # We don't need these in the ml docker image
+    # We don't need these in the ml docker image (or they are installed elsewhere)
     ignore_requirements = [
         "python/requirements/compat/requirements_legacy_compat.txt",
+        "python/requirements/air/data-test-requirements.txt",
     ]
 
-    files_on_disk = glob.glob(f"{root_dir}/python/**/requirements*.txt", recursive=True)
+    files_on_disk = glob.glob(
+        f"{root_dir}/python/**/*-requirements.txt", recursive=True
+    )
     for file_on_disk in files_on_disk:
         rel = os.path.relpath(file_on_disk, start=root_dir)
         print(rel)
-        if not rel.startswith("python/requirements/ml"):
+        if not rel.startswith("python/requirements/air"):
             continue
         elif rel not in ml_requirements_files and rel not in ignore_requirements:
             raise RuntimeError(

ci/build/upload_build_info.sh

@@ -35,7 +35,7 @@ if [[ -z "${BUILDKITE-}" ]]; then
     aws s3 cp --recursive /tmp/bazel_event_logs "${DST}"
 else
     # Codepath for Buildkite
-    # Keep cryptography/openssl in sync with `requirements_test.txt`
+    # Keep cryptography/openssl in sync with `requirements/test-requirements.txt`
     pip install -q -c "${RAY_DIR}/python/requirements.txt" docker aws_requests_auth boto3 cryptography==38.0.1 PyOpenSSL==22.1.0
     python .buildkite/copy_files.py --destination logs --path /tmp/bazel_event_logs
 fi

ci/env/install-dependencies.sh

@@ -192,7 +192,7 @@ install_shellcheck() {
 }
 
 install_linters() {
-  pip install -r "${WORKSPACE_DIR}"/python/requirements_linters.txt
+  pip install -r "${WORKSPACE_DIR}"/python/requirements/lint-requirements.txt
 
   install_shellcheck
 }
@@ -328,7 +328,7 @@ install_pip_packages() {
   # files.
   delayed_packages=()
 
-  requirements_files+=("${WORKSPACE_DIR}/python/requirements_test.txt")
+  requirements_files+=("${WORKSPACE_DIR}/python/requirements/test-requirements.txt")
 
   if [ "${LINT-}" = 1 ]; then
     install_linters
@@ -350,34 +350,26 @@ install_pip_packages() {
 
   # Additional RLlib test dependencies.
   if [ "${RLLIB_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/requirements_rllib.txt")
-    #TODO(amogkam): Add this back to requirements_rllib.txt once mlagents no longer pins torch<1.9.0 version.
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/rllib-requirements.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/rllib-test-requirements.txt")
+    #TODO(amogkam): Add this back to rllib-requirements.txt once mlagents no longer pins torch<1.9.0 version.
     pip install --no-dependencies mlagents==0.28.0
   fi
 
-  # Some Ray Train dependencies have to be installed with --no-deps,
-  # as sub-dependencies conflict. The packages still work for our workflows.
-  # Todo(krfricke): Try to remove once we move to Python 3.8 in CI.
-  local install_ml_no_deps=0
-
   # Additional Train test dependencies.
   if [ "${TRAIN_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/requirements_train.txt")
-    install_ml_no_deps=1
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/train-requirements.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/train-test-requirements.txt")
   fi
 
   # Additional Tune/Doc test dependencies.
   if [ "${TUNE_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/requirements_tune.txt")
-  fi
-
-  # For Tune, install upstream dependencies.
-  if [ "${TUNE_TESTING-}" = 1 ] ||  [ "${DOC_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/requirements_upstream.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/tune-requirements.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/tune-test-requirements.txt")
   fi
 
   # Additional dependency for Ludwig.
-  # This cannot be included in requirements_upstream.txt as it has conflicting
+  # This cannot be included in requirements files as it has conflicting
   # dependencies with Modin.
   if [ "${INSTALL_LUDWIG-}" = 1 ]; then
     # TODO: eventually pin this to master.
@@ -386,7 +378,7 @@ install_pip_packages() {
   fi
 
   # Additional dependency for time series libraries.
-  # This cannot be included in requirements_tune.txt as it has conflicting
+  # This cannot be included in tune-requirements.txt as it has conflicting
   # dependencies.
   if [ "${INSTALL_TIMESERIES_LIBS-}" = 1 ]; then
     requirements_packages+=("statsforecast==1.5.0")
@@ -396,10 +388,10 @@ install_pip_packages() {
 
   # Data processing test dependencies.
   if [ "${DATA_PROCESSING_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/data_processing/requirements.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/data-requirements.txt")
   fi
   if [ "${DATA_PROCESSING_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/data_processing/requirements_dataset.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/data-test-requirements.txt")
     if [ -n "${ARROW_VERSION-}" ]; then
       if [ "${ARROW_VERSION-}" = nightly ]; then
         delayed_packages+=("--extra-index-url")
@@ -416,12 +408,6 @@ install_pip_packages() {
     fi
   fi
 
-  if [ "${install_ml_no_deps}" = 1 ]; then
-    # Install these requirements first. Their dependencies may be overwritten later
-    # by the main install.
-    pip install -r "${WORKSPACE_DIR}/python/requirements/ml/requirements_no_deps.txt"
-  fi
-
   retry_pip_install "CC=gcc pip install -Ur ${WORKSPACE_DIR}/python/requirements.txt"
 
   # Install deeplearning libraries (Torch + TensorFlow)
@@ -431,24 +417,26 @@ install_pip_packages() {
         # Install right away, as some dependencies (e.g. torch-spline-conv) need
         # torch to be installed for their own install.
         pip install -U "torch==${TORCH_VERSION-1.9.0}" "torchvision==${TORCHVISION_VERSION-0.10.0}"
-        # We won't add requirements_dl.txt as it would otherwise overwrite our custom
+        # We won't add dl-cpu-requirements.txt as it would otherwise overwrite our custom
         # torch. Thus we have also have to install tensorflow manually.
-        TF_PACKAGE=$(grep "tensorflow==" "${WORKSPACE_DIR}/python/requirements/ml/requirements_dl.txt")
-        TFPROB_PACKAGE=$(grep "tensorflow-probability==" "${WORKSPACE_DIR}/python/requirements/ml/requirements_dl.txt")
+        TF_PACKAGE=$(grep "tensorflow==" "${WORKSPACE_DIR}/python/requirements/air/dl-cpu-requirements.txt")
+        TFPROB_PACKAGE=$(grep "tensorflow-probability==" "${WORKSPACE_DIR}/python/requirements/air/dl-cpu-requirements.txt")
 
         # %%;* deletes everything after ; to get rid of e.g. python version specifiers
         pip install -U "${TF_PACKAGE%%;*}" "${TFPROB_PACKAGE%%;*}"
       else
         # Otherwise, use pinned default torch version.
         # Again, install right away, as some dependencies (e.g. torch-spline-conv) need
         # torch to be installed for their own install.
-        TORCH_PACKAGE=$(grep "torch==" "${WORKSPACE_DIR}/python/requirements/ml/requirements_dl.txt")
-        TORCHVISION_PACKAGE=$(grep "torchvision==" "${WORKSPACE_DIR}/python/requirements/ml/requirements_dl.txt")
+        TORCH_PACKAGE=$(grep "torch==" "${WORKSPACE_DIR}/python/requirements/air/dl-cpu-requirements.txt")
+        TORCHVISION_PACKAGE=$(grep "torchvision==" "${WORKSPACE_DIR}/python/requirements/air/dl-cpu-requirements.txt")
 
         # %%;* deletes everything after ; to get rid of e.g. python version specifiers
         pip install "${TORCH_PACKAGE%%;*}" "${TORCHVISION_PACKAGE%%;*}"
-        requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/requirements_dl.txt")
+        requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/dl-cpu-requirements.txt")
       fi
+
+      requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/core-requirements.txt")
   fi
 
   # Inject our own mirror for the CIFAR10 dataset

doc/source/ray-contribute/development.rst

@@ -360,13 +360,13 @@ Dependencies for the linter (``scripts/format.sh``) can be installed with:
 
 .. code-block:: shell
 
- pip install -r python/requirements_linters.txt
+ pip install -r python/requirements/lint-requirements.txt
 
 Dependencies for running Ray unit tests under ``python/ray/tests`` can be installed with:
 
 .. code-block:: shell
 
- pip install -c python/requirements.txt -r python/requirements_test.txt
+ pip install -c python/requirements.txt -r python/requirements/test-requirements.txt
 
 Requirement files for running Ray Data / ML library tests are under ``python/requirements/``.
 

doc/source/ray-contribute/docs.ipynb

@@ -66,7 +66,7 @@
     "Install the dependencies for our linters to ensure your changes comply with our style guide.\n",
     "\n",
     "```shell\n",
-    "pip install -r ../python/requirements_linters.txt\n",
+    "pip install -r ../python/requirements/lint-requirements.txt\n",
     "```\n",
     "\n",
     "Build the documentation by running the following command:\n",

doc/source/ray-contribute/getting-involved.rst

@@ -96,7 +96,7 @@ If you are running tests for the first time, you can install the required depend
 
 .. code-block:: shell
 
-    pip install -c python/requirements.txt -r python/requirements_test.txt
+    pip install -c python/requirements.txt -r python/requirements/test-requirements.txt
 
 Testing for Python development
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -222,11 +222,11 @@ Lint and Formatting
 
 We also have tests for code formatting and linting that need to pass before merge.
 
-* For Python formatting, install the `required dependencies <https://github.com/ray-project/ray/blob/master/python/requirements_linters.txt>`_ first with:
+* For Python formatting, install the `required dependencies <https://github.com/ray-project/ray/blob/master/python/requirements/lint-requirements.txt>`_ first with:
 
 .. code-block:: shell
 
-  pip install -r python/requirements_linters.txt
+  pip install -r python/requirements/lint-requirements.txt
 
 * If developing for C++, you will need `clang-format <https://www.kernel.org/doc/html/latest/process/clang-format.html>`_ version ``12`` (download this version of Clang from `here <http://releases.llvm.org/download.html>`_)