Make Ray releases tests use pydantic>=2.5.0 (ray-project#41244)

As a follow-up to ray-project#40451, this change makes all the Ray release tests use pydantic>=2.5.0. Additional changes: The linter now uses mypy==1.7.0. All Ray release tests, including workspace templates, now use deepspeed>=0.12.3. This change fix the serve_resnet_benchmark.py release test, which is broken on master. --------- Signed-off-by: can <can@anyscale.com> Signed-off-by: Shreyas Krishnaswamy <shrekris@anyscale.com> Co-authored-by: Shreyas Krishnaswamy <shrekris@anyscale.com>
ujjawal-khare-27 · Nov 29, 2023 · e0af60c · e0af60c
1 parent 930bf7c
commit e0af60c
Show file tree

Hide file tree

Showing 17 changed files with 1,478 additions and 1,341 deletions.
diff --git a/ci/lint/format.sh b/ci/lint/format.sh
@@ -8,7 +8,7 @@ set -euo pipefail
 FLAKE8_VERSION_REQUIRED="3.9.1"
 BLACK_VERSION_REQUIRED="22.10.0"
 SHELLCHECK_VERSION_REQUIRED="0.7.1"
-MYPY_VERSION_REQUIRED="0.982"
+MYPY_VERSION_REQUIRED="1.7.0"
 ISORT_VERSION_REQUIRED="5.10.1"
 
 check_python_command_exist() {

diff --git a/dashboard/modules/job/tests/backwards_compatibility_scripts/test_backwards_compatibility.sh b/dashboard/modules/job/tests/backwards_compatibility_scripts/test_backwards_compatibility.sh
@@ -37,6 +37,7 @@ do
     conda activate "${env_name}"
 
     # Pin pydantic version due to: https://github.com/ray-project/ray/issues/36990.
+    # ray<2.9 is only compatible with pydantic<2.
     pip install -U "pydantic<2" ray=="${RAY_VERSION}" ray[default]=="${RAY_VERSION}"
 
     printf "\n\n\n"

diff --git a/doc/source/ray-overview/pip_freeze_ray-ml-py39-cpu.txt b/doc/source/ray-overview/pip_freeze_ray-ml-py39-cpu.txt
@@ -319,7 +319,7 @@ pyasn1==0.5.0
 pyasn1-modules==0.3.0
 pycosat @ file:///croot/pycosat_1696536503704/work
 pycparser @ file:///tmp/build/80754af9/pycparser_1636541352034/work
-pydantic==1.10.12
+pydantic==2.5.0
 pyDeprecate==0.3.2
 pygame==2.1.2
 pyglet==1.5.15

diff --git a/doc/source/ray-overview/pip_freeze_ray-py39-cpu.txt b/doc/source/ray-overview/pip_freeze_ray-py39-cpu.txt
@@ -108,7 +108,7 @@ pyasn1==0.5.0
 pyasn1-modules==0.3.0
 pycosat @ file:///croot/pycosat_1696536503704/work
 pycparser @ file:///tmp/build/80754af9/pycparser_1636541352034/work
-pydantic==1.10.12
+pydantic==2.5.0
 Pygments==2.13.0
 PyJWT==2.8.0
 PyNaCl==1.5.0

diff --git a/doc/source/train/examples/deepspeed/gptj_deepspeed_fine_tuning.ipynb b/doc/source/train/examples/deepspeed/gptj_deepspeed_fine_tuning.ipynb
@@ -41,7 +41,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "! pip install \"datasets\" \"evaluate\" \"accelerate==0.18.0\" \"transformers>=4.26.0\" \"torch>=1.12.0\" \"deepspeed==0.8.3\""
+    "! pip install \"datasets\" \"evaluate\" \"accelerate==0.18.0\" \"transformers>=4.26.0\" \"torch>=1.12.0\" \"deepspeed==0.12.3\""
    ]
   },
   {
@@ -111,7 +111,7 @@
     "            \"accelerate==0.16.0\",\n",
     "            \"transformers==4.26.0\",\n",
     "            \"torch>=1.12.0\",\n",
-    "            \"deepspeed==0.9.2\",\n",
+    "            \"deepspeed==0.12.3\",\n",
     "        ],\n",
     "    },\n",
     ")"

diff --git a/doc/source/train/examples/lightning/vicuna_13b_lightning_deepspeed_finetune.ipynb b/doc/source/train/examples/lightning/vicuna_13b_lightning_deepspeed_finetune.ipynb
@@ -98,7 +98,7 @@
     "        \"pip\": [\n",
     "            \"datasets==2.13.1\",\n",
     "            \"torch>=1.13.0\",\n",
-    "            \"deepspeed==0.9.4\",\n",
+    "            \"deepspeed==0.12.3\",\n",
     "            \"accelerate==0.20.3\",\n",
     "            \"transformers==4.30.2\",\n",
     "            \"lightning==2.0.3\",\n",

diff --git a/python/ray/tests/test_serialization.py b/python/ray/tests/test_serialization.py
@@ -678,6 +678,33 @@ def custom_deserializer(b):
     ray.get(ray.put(A(1)))  # success!
 
 
+def test_serialization_pydantic_runtime_env(ray_start_regular):
+    @ray.remote
+    def test(pydantic_model):
+        return pydantic_model.x
+
+    @ray.remote(runtime_env={"pip": ["pydantic<2"]})
+    def py1():
+        from pydantic import BaseModel
+
+        class Foo(BaseModel):
+            x: int
+
+        return ray.get(test.remote(Foo(x=1)))
+
+    @ray.remote(runtime_env={"pip": ["pydantic>=2"]})
+    def py2():
+        from pydantic.v1 import BaseModel
+
+        class Foo(BaseModel):
+            x: int
+
+        return ray.get(test.remote(Foo(x=2)))
+
+    assert ray.get(py1.remote()) == 1
+    assert ray.get(py2.remote()) == 2
+
+
 if __name__ == "__main__":
     import os
     import pytest

diff --git a/release/air_examples/gptj_deepspeed_finetuning/gptj_deepspeed_env.yaml b/release/air_examples/gptj_deepspeed_finetuning/gptj_deepspeed_env.yaml
@@ -10,7 +10,7 @@ python:
     - "accelerate==0.16.0"
     - "transformers==4.26.0"
     - "torch>=1.12.0"
-    - "deepspeed==0.9.2"
+    - "deepspeed==0.12.3"
     - myst-parser==0.15.2
     - myst-nb==0.13.1
     - jupytext==1.13.6

diff --git a/release/air_examples/vicuna_13b_lightning_deepspeed_finetuning/vicuna_13b_deepspeed_env.yaml b/release/air_examples/vicuna_13b_lightning_deepspeed_finetuning/vicuna_13b_deepspeed_env.yaml
@@ -25,4 +25,4 @@ post_build_cmds:
   - echo "sudo mount /dev/nvme1n1 /mnt/local_storage || true" >> ~/.bashrc
   - pip3 uninstall -y pytorch-lightning
   - pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
-  - pip3 install "lightning==2.0.3" "transformers==4.30.2" "accelerate==0.20.3" "deepspeed==0.9.4"
+  - pip3 install "lightning==2.0.3" "transformers==4.30.2" "accelerate==0.20.3" "deepspeed==0.12.3"
diff --git a/release/ray_release/byod/requirements_byod_3.8.in b/release/ray_release/byod/requirements_byod_3.8.in
@@ -1,6 +1,6 @@
 # Python requirements to run release tests from anyscale byod (cpu type)
+
 ale-py
-anyscale
 boto3
 cmake
 crc32c
@@ -17,19 +17,21 @@ openskill
 petastorm
 protobuf
 pyarrow
+pydantic>=2.5.0
 pytest
 requests>=2.31.0
 semidbm
 scikit-learn
 scipy
 tblib
-tensorboardX
-tensorflow
 terminado
+tensorboardX
+tensorflow==2.11.0  # This must be pinned for compatibiliy with pydantic 2 and Python 3.8
 trueskill
 tqdm
 typer
-wandb
+typing-extensions==4.8.0
 xarray
 xgboost
+wandb
 zarr
diff --git a/release/ray_release/byod/requirements_byod_3.8.txt b/release/ray_release/byod/requirements_byod_3.8.txt
diff --git a/release/ray_release/byod/requirements_ml_byod_3.8.in b/release/ray_release/byod/requirements_ml_byod_3.8.in
@@ -1,11 +1,12 @@
 # Python requirements to run release tests from anyscale byod (gpu type)
+
 accelerate
 boto3
 cmake
 crc32c
 cupy-cuda113
 datasets
-deepspeed
+deepspeed>=0.12.3
 diffusers
 evaluate
 fastapi
@@ -20,6 +21,7 @@ numpy
 openskill
 petastorm
 pyarrow
+pydantic>=2.5.0
 pytest
 pytorch_lightning
 scikit-learn

diff --git a/release/ray_release/byod/requirements_ml_byod_3.8.txt b/release/ray_release/byod/requirements_ml_byod_3.8.txt
diff --git a/release/ray_release/byod/requirements_ml_byod_3.9.in b/release/ray_release/byod/requirements_ml_byod_3.9.in
@@ -1,10 +1,11 @@
 # Python requirements to run release tests from anyscale byod (gpu type, python 3.9)
+
 accelerate
 bitsandbytes
 dataset
 datasets
 decord
-deepspeed
+deepspeed>=0.12.3
 diffusers
 evaluate
 fairscale
@@ -15,6 +16,7 @@ matplotlib
 numpy
 openai-whisper
 protobuf
+pydantic>=2.5.0
 pytorch-lightning
 sentencepiece
 statsforecast