From 8beb15ee823ddcde3c1e84d4cb0ad9128a968a99 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Wed, 21 Jun 2023 16:50:48 -0700 Subject: [PATCH] Migrate all serve tests to byod (#36380) Signed-off-by: can Signed-off-by: e428265 --- release/BUILD | 10 + release/ray_release/byod/build.py | 9 +- release/ray_release/byod/byod.Dockerfile | 7 +- release/ray_release/byod/requirements_byod.in | 2 +- .../ray_release/byod/requirements_ml_byod.in | 9 + .../ray_release/byod/requirements_ml_byod.txt | 231 ++++++++++++++++++ .../cluster_manager/cluster_manager.py | 18 +- .../ray_release/tests/test_cluster_manager.py | 4 +- release/release_tests.yaml | 73 ++++-- 9 files changed, 338 insertions(+), 25 deletions(-) create mode 100644 release/ray_release/byod/requirements_ml_byod.in create mode 100644 release/ray_release/byod/requirements_ml_byod.txt diff --git a/release/BUILD b/release/BUILD index ad3e31f03b7c..391cbb8784c8 100644 --- a/release/BUILD +++ b/release/BUILD @@ -23,6 +23,16 @@ compile_pip_requirements( visibility = ["//visibility:private"], ) +compile_pip_requirements( + name = "requirements_ml_byod", + requirements_in = "ray_release/byod/requirements_ml_byod.in", + requirements_txt = "ray_release/byod/requirements_ml_byod.txt", + tags = [ + "team:ci", + ], + visibility = ["//visibility:private"], +) + test_srcs = glob(["**/*.py"]) #### diff --git a/release/ray_release/byod/build.py b/release/ray_release/byod/build.py index 2e163ed278e8..42d4b83b0999 100644 --- a/release/ray_release/byod/build.py +++ b/release/ray_release/byod/build.py @@ -17,6 +17,8 @@ BASE_IMAGE_WAIT_TIMEOUT = 7200 BASE_IMAGE_WAIT_DURATION = 30 RELEASE_BYOD_DIR = os.path.join(RELEASE_PACKAGE_DIR, "ray_release/byod") +REQUIREMENTS_BYOD = "requirements_byod.txt" +REQUIREMENTS_ML_BYOD = "requirements_ml_byod.txt" def build_anyscale_custom_byod_image(test: Test) -> None: @@ -77,6 +79,11 @@ def build_anyscale_base_byod_images(tests: List[Test]) -> None: ): for ray_image, test in to_be_built.items(): byod_image = test.get_anyscale_base_byod_image() + byod_requirements = ( + REQUIREMENTS_BYOD + if test.get_byod_type() == "cpu" + else REQUIREMENTS_ML_BYOD + ) if _byod_image_exist(test): logger.info(f"Image {byod_image} already exists") built.add(ray_image) @@ -114,7 +121,7 @@ def build_anyscale_base_byod_images(tests: List[Test]) -> None: "--build-arg", f"BASE_IMAGE={byod_image}", "--build-arg", - "PIP_REQUIREMENTS=requirements_byod.txt", + f"PIP_REQUIREMENTS={byod_requirements}", "--build-arg", "DEBIAN_REQUIREMENTS=requirements_debian_byod.txt", "-t", diff --git a/release/ray_release/byod/byod.Dockerfile b/release/ray_release/byod/byod.Dockerfile index e0f1390c3823..acbdbd393759 100644 --- a/release/ray_release/byod/byod.Dockerfile +++ b/release/ray_release/byod/byod.Dockerfile @@ -15,8 +15,13 @@ sudo apt-get update -y \ && sudo apt-get install -y --no-install-recommends $(cat requirements_debian_byod.txt) \ && sudo apt-get autoclean +rm -rf /tmp/wrk +git clone --branch 4.2.0 https://github.com/wg/wrk.git /tmp/wrk +make -C /tmp/wrk -j +sudo cp /tmp/wrk/wrk /usr/local/bin/wrk + EOF COPY "$PIP_REQUIREMENTS" . RUN "$HOME"/anaconda3/bin/pip install --no-cache-dir https://ray-ci-deps-wheels.s3.us-west-2.amazonaws.com/AutoROM.accept_rom_license-0.5.4-py3-none-any.whl -RUN "$HOME"/anaconda3/bin/pip install --no-cache-dir -r requirements_byod.txt +RUN "$HOME"/anaconda3/bin/pip install --no-cache-dir -r "${PIP_REQUIREMENTS}" diff --git a/release/ray_release/byod/requirements_byod.in b/release/ray_release/byod/requirements_byod.in index f3c798d1fb9d..d33833f925b0 100644 --- a/release/ray_release/byod/requirements_byod.in +++ b/release/ray_release/byod/requirements_byod.in @@ -1,4 +1,4 @@ -# Python requirements to run release tests from anyscale byod +# Python requirements to run release tests from anyscale byod (cpu type) ale-py anyscale boto3 diff --git a/release/ray_release/byod/requirements_ml_byod.in b/release/ray_release/byod/requirements_ml_byod.in new file mode 100644 index 000000000000..c7d071280edf --- /dev/null +++ b/release/ray_release/byod/requirements_ml_byod.in @@ -0,0 +1,9 @@ +# Python requirements to run release tests from anyscale byod (gpu type) +boto3 +cmake +crc32c +numpy +pyarrow +pytest +tqdm +validators diff --git a/release/ray_release/byod/requirements_ml_byod.txt b/release/ray_release/byod/requirements_ml_byod.txt new file mode 100644 index 000000000000..46688e521af7 --- /dev/null +++ b/release/ray_release/byod/requirements_ml_byod.txt @@ -0,0 +1,231 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# bazel run //release:requirements_ml_byod.update +# +boto3==1.26.152 \ + --hash=sha256:a2778c5729d3dc0b3688c9f0d103543d7ec5ff44a4fd0e84d0d542e2dff05e62 \ + --hash=sha256:ee0b8f8d238d4e1cf50fa6a185e4e066955b6105e9838a80b1b6582cd327dfdf + # via -r release/ray_release/byod/requirements_ml_byod.in +botocore==1.29.152 \ + --hash=sha256:02a3205cc8579d4be6d537e63d72aebbf3f70f3aedcf40b3cae9dc2e24c774d0 \ + --hash=sha256:f6319ecdbe3d325878f837cac2874e461b4d90691bb2d2186f980bce3b3cfcc8 + # via + # boto3 + # s3transfer +cmake==3.26.4 \ + --hash=sha256:05cfd76c637eb22058c95e2dc383cadd4e0615e2643e637bb498a6cc24825790 \ + --hash=sha256:1b92f9f59f48c803106dbdd6750b0f571a0500e25d3a62c42ba84bb7a9240d10 \ + --hash=sha256:1d887be5f1a3f17559a78707a6bc0560f4f8cb93cebb9d823d90a63e68bae09b \ + --hash=sha256:230227bf99f36614de84cdc92ffce3a50eb2803020e946f8da945a08fcf766bf \ + --hash=sha256:235d8eac93a28dcce5a1cd7130412885a2aa53d5735cb2230e0f26f589347b65 \ + --hash=sha256:24110035aff586a04a6a6fcf4609270642e4f503c0620c962dff75b653f81414 \ + --hash=sha256:248a90816abfc10ff6e1109b54b8235c3e62f0ac92da16541753deb3b5ae063d \ + --hash=sha256:3175442985558d5415b97f264a6a1bb0af5ecfe10e3f7510257b1ea66bd33848 \ + --hash=sha256:3e280e81713408987b7053f5b922c9f94e45668ca6efff1f02846309ca0b5b0f \ + --hash=sha256:432837364aa6cab2826a72e8a4cdd3586f5ac9ce495217ccd59aa70f2bba8120 \ + --hash=sha256:93015da6f1c0e1e5f2debf752f1803ea52d742d915ad674043d36e471f937507 \ + --hash=sha256:93a03bad17b9741acaff4a8651f8596496506602fa123e70fe67142f1b21ee2e \ + --hash=sha256:b7a6946c345497c14064e0c9585b30f5aaebbefdfc0b245b6bb5a978eb4fc85f \ + --hash=sha256:c3b0e72750c0f6c0373242c1299bc4ffdbebdd5004966ae6df0b2e9845aa6990 \ + --hash=sha256:d45b30b9ce7280829888c78650177ab525df2b6785e1a5b3d82b4c147d828c0e \ + --hash=sha256:d726671ae7ae4aa6989e73d26b9f8f8e6af45163a26ea243949d72246566fdd8 \ + --hash=sha256:e058e59154a1e490fb9425b420f87e28144292397607638d73e323509f7efae6 + # via -r release/ray_release/byod/requirements_ml_byod.in +crc32c==2.3.post0 \ + --hash=sha256:00de0d157dc17bbf01fef615aff6095a78b3561aad37b4ba4a300b11311aae55 \ + --hash=sha256:01787094f281ae7c8f645d7b3c309a02bac45cb385206eee651aa27d933a87e5 \ + --hash=sha256:04220e1db5567dc234d1e9dc182c5b8241905057ec19967ac3a917bcaf06d70e \ + --hash=sha256:0467261c67377a92ad6665a9590b3820cfb12d59c3c6ccac6326200e032ddda4 \ + --hash=sha256:05ad2f6b6392b2a0af159142e6ec029cddd15d67a76b7762b3316cbb5cc8e22c \ + --hash=sha256:05b69167116680bd40116c8fac847950d1eb170fdd42a814602223b4e002b0bb \ + --hash=sha256:061b084e27d92dc3e1a9efd500e6e3feee9e97e8cefe2fbdcc0011cba7f3242a \ + --hash=sha256:08fdba1351d5cbb428d9ba3ce5c03d43687e7b23c6bc0cf99973306e5549dab9 \ + --hash=sha256:0b5f5a18e9bfe98a273d9618d5c04470fe983dcebcd453fac07c398dfac7db10 \ + --hash=sha256:1101b4e680085ea7c01074d38378610392262bc56936ec17eed61f1372197193 \ + --hash=sha256:1285d33cdeda2d2248994d41706f88f0fe58265ae907d23221c07028e79f9670 \ + --hash=sha256:129b0ac8ee712ce42aae36d7e6e5202ab080f06117f57ba2c894226586e80050 \ + --hash=sha256:176b3c5ff7fa4e2f83c241ab9dc4fd1584d1c9333d4c7295c16c9f6097c29933 \ + --hash=sha256:19d6a505582194ed0b2bb257cf3729c922e7e92f457a9f7f5493cf821cb19afc \ + --hash=sha256:217c1b64be777cf235556066c363f4dec22b29a956a174f6361037b1b2065c63 \ + --hash=sha256:21919937ecac802e436c1a9978e8b27522ca87bf67dc5ce3a5b5622c0b5c3a06 \ + --hash=sha256:21ef9ebdfe4f2e45c94327a1a9a222a899be784d78674065ada6e8e240d3a4a8 \ + --hash=sha256:247ada85bd0a54012e910ff46697871f80bcff9018b59f7de23161726a146b2a \ + --hash=sha256:295053584dc3a11d8f02d6ccc6dd3698331e252cd816d7652b0723c516ef3c41 \ + --hash=sha256:305ca4eb8c399081a68ca0274ae176753be8430fc874d1e7397a2cbc95748733 \ + --hash=sha256:31f59b051a7fa4a2ada3f76a79014be38c45e9d3c906eca381e9007677ffe506 \ + --hash=sha256:335f1fcd5fb346be4ac4c982f89bb66b39c93a2c2d4bcdb3e3188d8adcb225b2 \ + --hash=sha256:363b8f2993f07eb8ac665c7227cb2a569fb1f4eb1551a05695bc2f94c23307e4 \ + --hash=sha256:38f4c8eaa77fa9bbe690b58546dd3f2e244c13d5d0a01fa93076d3817a22bc68 \ + --hash=sha256:3bb11668f75a7f4f699b9a125aaf15259687f691beb95e756e3bea80d7163645 \ + --hash=sha256:3c24ebb561e10a5eada2128a7357e41969155cebe7b34656176fc24412d45c8b \ + --hash=sha256:3e547c06a1dda463daf398661af6bda767debe0097630b48c463605e38ade31a \ + --hash=sha256:3ef1b87ac755e20933bc8136a45ca9993a03c0b0ba16dd946ab287108305332b \ + --hash=sha256:42c0363d68a95d133af02803772395b42bc202840ea70a317e2b46beb9e53af0 \ + --hash=sha256:4459462c732232ffb29b58decd246ed5cdb8c16ae141f57f03cb2e3445dc1d2e \ + --hash=sha256:470bb05c224ede904af0278d18ac26f1132a8cfc7a11cfa0406c26ef75765427 \ + --hash=sha256:585ab3307a2aa73b935f0b0358197f0af5306204d646ac321ecf01f2a3725f94 \ + --hash=sha256:5be4ad72c198c4a22515ef2ad728f9829ee3d75e6c7f3e41030c8266e46c0c7c \ + --hash=sha256:5c9e58f96a0e56e60ea683504f605b76c73b7f885837bfc8bd8346b054133045 \ + --hash=sha256:6072cc60102a8ac86f45ab8d29c8679a8ac0445477eed3d0af6d0becceea1392 \ + --hash=sha256:68941ac55632f231120baeeba72690cdf2ec2531fea3ceab4612dbf855411b05 \ + --hash=sha256:69cb66a0c680ae531df7f32833a3d6df26aeeb144c0f7a8899d2d5bb7c9cdc2c \ + --hash=sha256:69cd27f493a6bcbeb1b59eea4a978734ebf3ce2b6f757a99405d6eebd38af551 \ + --hash=sha256:6bfc277c43fa1dd4c4a91a1b56347008e34c8214dd99b1424b5d636272f2922c \ + --hash=sha256:75e9a588e7241d09de9023dc51174cc2c9ac7c453ae0e26a5718e266b48ae392 \ + --hash=sha256:75eb7cc4200745cbc717bb0b83b538e6582be980d4f8c9f9bb0740a23e93a4d8 \ + --hash=sha256:771ad4abacb89a14e1f5cbbb291ead652a0c9ed4be2c9b579c869957c0b03e3d \ + --hash=sha256:7a61e6754ee54bbae9253035eabb5658d0ada9162eb1b98feabefb044b95e6c0 \ + --hash=sha256:7d02e45cddebfa82694fc9fc7df2f42366431e90b0abd40c5c63758bc9234123 \ + --hash=sha256:7d4b39ca6791830c4f1c053d2d8983627af702f0445535ff53d3220f35cf6ce6 \ + --hash=sha256:7fb366626bf7ef66e55656c8385fcc94f22f8d3847a7a84c810d2e3f63f54c62 \ + --hash=sha256:80ddf6b0594bb980a635ff4818c0c64927193c1a09e8b5b6986769e94a7ba9ee \ + --hash=sha256:85b3395c476916b098a20cfb8686558865ca3ef71caaf9e6b0a548b2049ee87d \ + --hash=sha256:86837a00716056c29f9d84c980cdac050ba3c7610c9edca1b2ac01192715725d \ + --hash=sha256:89bcf1158c577635bdc61b27d29deb2fe0c1191a54a490f976563a73abe3b2e3 \ + --hash=sha256:8a3ff6b893ab482f0841a2b7e394adb749b1a896c854ce92f72c60e2ea3a3553 \ + --hash=sha256:8af10d0c3752db01dfa77c6c4c8fa070bdefe939eb9ab94b4dec1dcc2cc11fa5 \ + --hash=sha256:9ebbcde06765fcde3d2d440153839a9ac675866fb25aa86219595c370e6d3f7c \ + --hash=sha256:a2f6105a430f4f1f393cc660bf8ac3a4860396fa9b5ac8bf0c7ba1de044a3cbc \ + --hash=sha256:a4d98142f6e40dec28994846a6acafd96ba822d81b3c6671633fb11d41692c32 \ + --hash=sha256:a57d1ec8f1aa45c14a51770b011359b511eb7dcc6ffd7c8fc9e918e2aa009416 \ + --hash=sha256:acb0d4a1cf19fdc2946ab9b1dc5d4f1347e97b356a863fbba2d8a3d3c1cbe815 \ + --hash=sha256:b3628ff77ca4cf3c3f0209d5eb824b79d8e324bbb1feaff3fb6bff8adc23ec08 \ + --hash=sha256:b60dd506177d2ea68fb548caa9cc383f46c024947d1990604c11aa615ec9da62 \ + --hash=sha256:b6a4df4a978dbf43e548a008dc4686f6e24d52defb8c03a79b67aebfeaa2caa6 \ + --hash=sha256:b6ad74ec499a3d6981900c1e2873b1e6a19e2ee3c650a3e611c3076ad9167f3a \ + --hash=sha256:b823aacfefe0001a08667d7a4d7dd87133537e3628ed581fa416829a5dad26fd \ + --hash=sha256:bce06be1d9aa7e4b3e3038fe80facafa3526bec9e484ec089c035b8018727c1b \ + --hash=sha256:c002f55429a12ec87a0b33a073b384d26edd46d89b3cf7cfb6ddf5abb6e2bfab \ + --hash=sha256:c102e8988618e3bd15f4297ab95631c6d9e59326f9af17bf7d71c2ad4639a7f7 \ + --hash=sha256:c55e8a45e360aeb3cea2cf9d9fb3771a711ed3c3fce2d91c874d767aae4f5cee \ + --hash=sha256:cb4222a766f59b1cd8cbe56af5dbdfd3a2c0ec40b60c9ee6efe4a5cabc94112d \ + --hash=sha256:cce0c79dbf4d4f2276cb9e32f4d0dfc3ce5d8cc5c3f0a0bc62612505cd779c67 \ + --hash=sha256:cee4275dfb3cfc4a1e4c338089f3223fce878d7151cebb095937c07410371908 \ + --hash=sha256:d18e82f66a0c25c8fb10b21e71cf2f2aa81441fd1a062809249e98a338ef9c81 \ + --hash=sha256:d7bcc07f3a63cc8be7536ca35a4c5b96763b8e0ce5d48f30d9374ec7e381b057 \ + --hash=sha256:d9ea9e6eb2912051a40ac6877646925ee3a058c4aa3868988fe1d8c4577f57d4 \ + --hash=sha256:dbae415e9ec7dfdcdeac981cf4833d9942ce9de175b2be5a21c641c3a88e609b \ + --hash=sha256:de0307898c24a8ae29d4b94ddd3aa81b73d3b2b0e490d226e3a3dd05526dfca1 \ + --hash=sha256:e0925a7a8fcc216744e86c21f1749c22f950f9bdad512cb5c80ee85017625bd5 \ + --hash=sha256:e311f52e24b633e8d588ab9a0e7992bfcfe8284a1655202bdac5aee80254a3fd \ + --hash=sha256:e88bbdeba430dfee6d83192a9e55c8e89884a3c5215d1b7643395ecbbd1b502b \ + --hash=sha256:ec216adbbf7ad1506918c8199a144d26740650b594f79755f5f1affec7e7820b + # via -r release/ray_release/byod/requirements_ml_byod.in +decorator==5.1.1 \ + --hash=sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330 \ + --hash=sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186 + # via validators +exceptiongroup==1.1.1 \ + --hash=sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e \ + --hash=sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785 + # via pytest +iniconfig==2.0.0 \ + --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ + --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 + # via pytest +jmespath==1.0.1 \ + --hash=sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 \ + --hash=sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe + # via + # boto3 + # botocore +numpy==1.24.3 \ + --hash=sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187 \ + --hash=sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812 \ + --hash=sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7 \ + --hash=sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4 \ + --hash=sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6 \ + --hash=sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0 \ + --hash=sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4 \ + --hash=sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570 \ + --hash=sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4 \ + --hash=sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f \ + --hash=sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80 \ + --hash=sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289 \ + --hash=sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385 \ + --hash=sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078 \ + --hash=sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c \ + --hash=sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463 \ + --hash=sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3 \ + --hash=sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950 \ + --hash=sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155 \ + --hash=sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7 \ + --hash=sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c \ + --hash=sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096 \ + --hash=sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17 \ + --hash=sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf \ + --hash=sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4 \ + --hash=sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02 \ + --hash=sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c \ + --hash=sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b + # via + # -r release/ray_release/byod/requirements_ml_byod.in + # pyarrow +packaging==23.1 \ + --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \ + --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f + # via pytest +pluggy==1.0.0 \ + --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 \ + --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3 + # via pytest +pyarrow==12.0.1 \ + --hash=sha256:051f9f5ccf585f12d7de836e50965b3c235542cc896959320d9776ab93f3b33d \ + --hash=sha256:1887bdae17ec3b4c046fcf19951e71b6a619f39fa674f9881216173566c8f718 \ + --hash=sha256:2d3c4cbbf81e6dd23fe921bc91dc4619ea3b79bc58ef10bce0f49bdafb103daf \ + --hash=sha256:345e1828efdbd9aa4d4de7d5676778aba384a2c3add896d995b23d368e60e5af \ + --hash=sha256:3de26da901216149ce086920547dfff5cd22818c9eab67ebc41e863a5883bac7 \ + --hash=sha256:43364daec02f69fec89d2315f7fbfbeec956e0d991cbbef471681bd77875c40f \ + --hash=sha256:459a1c0ed2d68671188b2118c63bac91eaef6fc150c77ddd8a583e3c795737bf \ + --hash=sha256:6251e38470da97a5b2e00de5c6a049149f7b2bd62f12fa5dbb9ac674119ba71a \ + --hash=sha256:6895b5fb74289d055c43db3af0de6e16b07586c45763cb5e558d38b86a91e3a7 \ + --hash=sha256:6d288029a94a9bb5407ceebdd7110ba398a00412c5b0155ee9813a40d246c5df \ + --hash=sha256:749be7fd2ff260683f9cc739cb862fb11be376de965a2a8ccbf2693b098db6c7 \ + --hash=sha256:85e705e33eaf666bbe508a16fd5ba27ca061e177916b7a317ba5a51bee43384c \ + --hash=sha256:8d6009fdf8986332b2169314da482baed47ac053311c8934ac6651e614deacd6 \ + --hash=sha256:9120c3eb2b1f6f516a3b7a9714ed860882d9ef98c4b17edcdc91d95b7528db60 \ + --hash=sha256:a3c63124fc26bf5f95f508f5d04e1ece8cc23a8b0af2a1e6ab2b1ec3fdc91b24 \ + --hash=sha256:b13329f79fa4472324f8d32dc1b1216616d09bd1e77cfb13104dec5463632c36 \ + --hash=sha256:bb656150d3d12ec1396f6dde542db1675a95c0cc8366d507347b0beed96e87ca \ + --hash=sha256:be2757e9275875d2a9c6e6052ac7957fbbfc7bc7370e4a036a9b893e96fedaba \ + --hash=sha256:c780f4dc40460015d80fcd6a6140de80b615349ed68ef9adb653fe351778c9b3 \ + --hash=sha256:cce317fc96e5b71107bf1f9f184d5e54e2bd14bbf3f9a3d62819961f0af86fec \ + --hash=sha256:cdacf515ec276709ac8042c7d9bd5be83b4f5f39c6c037a17a60d7ebfd92c890 \ + --hash=sha256:ce4aebdf412bd0eeb800d8e47db854f9f9f7e2f5a0220440acf219ddfddd4f63 \ + --hash=sha256:cf812306d66f40f69e684300f7af5111c11f6e0d89d6b733e05a3de44961529d \ + --hash=sha256:e0d8730c7f6e893f6db5d5b86eda42c0a130842d101992b581e2138e4d5663d3 \ + --hash=sha256:e2c9cb8eeabbadf5fcfc3d1ddea616c7ce893db2ce4dcef0ac13b099ad7ca082 + # via -r release/ray_release/byod/requirements_ml_byod.in +pytest==7.3.2 \ + --hash=sha256:cdcbd012c9312258922f8cd3f1b62a6580fdced17db6014896053d47cddf9295 \ + --hash=sha256:ee990a3cc55ba808b80795a79944756f315c67c12b56abd3ac993a7b8c17030b + # via -r release/ray_release/byod/requirements_ml_byod.in +python-dateutil==2.8.2 \ + --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ + --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 + # via botocore +s3transfer==0.6.1 \ + --hash=sha256:3c0da2d074bf35d6870ef157158641178a4204a6e689e82546083e31e0311346 \ + --hash=sha256:640bb492711f4c0c0905e1f62b6aaeb771881935ad27884852411f8e9cacbca9 + # via boto3 +six==1.16.0 \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 + # via python-dateutil +tomli==2.0.1 \ + --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ + --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f + # via pytest +tqdm==4.65.0 \ + --hash=sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5 \ + --hash=sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671 + # via -r release/ray_release/byod/requirements_ml_byod.in +urllib3==1.26.16 \ + --hash=sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f \ + --hash=sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14 + # via botocore +validators==0.20.0 \ + --hash=sha256:24148ce4e64100a2d5e267233e23e7afeb55316b47d30faae7eb6e7292bc226a + # via -r release/ray_release/byod/requirements_ml_byod.in diff --git a/release/ray_release/cluster_manager/cluster_manager.py b/release/ray_release/cluster_manager/cluster_manager.py index da61f639b88b..3d5da276117a 100644 --- a/release/ray_release/cluster_manager/cluster_manager.py +++ b/release/ray_release/cluster_manager/cluster_manager.py @@ -62,11 +62,19 @@ def set_cluster_env(self, cluster_env: Dict[str, Any]): "RAY_USAGE_STATS_EXTRA_TAGS" ] = f"test_name={self.test.get_name()};smoke_test={self.smoke_test}" - self.cluster_env_name = ( - f"{self.project_name}_{self.project_id[4:8]}" - f"__env__{self.test.get_name().replace('.', '_')}__" - f"{dict_hash(self.cluster_env)}" - ) + if self.test.is_byod_cluster(): + self.cluster_env_name = ( + self.test.get_anyscale_byod_image() + .replace("/", "_") + .replace(":", "_") + .replace(".", "_") + ) + else: + self.cluster_env_name = ( + f"{self.project_name}_{self.project_id[4:8]}" + f"__env__{self.test.get_name().replace('.', '_')}__" + f"{dict_hash(self.cluster_env)}" + ) def set_cluster_compute( self, diff --git a/release/ray_release/tests/test_cluster_manager.py b/release/ray_release/tests/test_cluster_manager.py index b4f1a5e96c33..4f7508137eb9 100644 --- a/release/ray_release/tests/test_cluster_manager.py +++ b/release/ray_release/tests/test_cluster_manager.py @@ -131,7 +131,7 @@ def testSetClusterEnv(self): sdk.returns["get_project"] = APIDict(result=APIDict(name="release_unit_tests")) sdk.returns["get_cloud"] = APIDict(result=APIDict(provider="AWS")) cluster_manager = self.cls( - test=Test({"name": "test"}), + test=Test({"name": "test", "cluster": {}}), project_id=UNIT_TEST_PROJECT_ID, smoke_test=False, sdk=sdk, @@ -142,7 +142,7 @@ def testSetClusterEnv(self): "test_name=test;smoke_test=False", ) cluster_manager = self.cls( - test=Test({"name": "Test"}), + test=Test({"name": "Test", "cluster": {}}), project_id=UNIT_TEST_PROJECT_ID, smoke_test=True, sdk=sdk, diff --git a/release/release_tests.yaml b/release/release_tests.yaml index f889eb6845ab..ab085894e340 100644 --- a/release/release_tests.yaml +++ b/release/release_tests.yaml @@ -2573,7 +2573,11 @@ frequency: weekly team: serve + python: "3.8" cluster: + byod: + runtime_env: + - RLLIB_TEST_NO_JAX_IMPORT=1 cluster_env: app_config.yaml cluster_compute: tpl_cpu_1.yaml @@ -2611,7 +2615,11 @@ frequency: weekly team: serve + python: "3.8" cluster: + byod: + runtime_env: + - RLLIB_TEST_NO_JAX_IMPORT=1 cluster_env: app_config.yaml cluster_compute: tpl_cpu_1_c5.yaml @@ -2649,7 +2657,11 @@ frequency: weekly team: serve + python: "3.8" cluster: + byod: + runtime_env: + - RLLIB_TEST_NO_JAX_IMPORT=1 cluster_env: app_config.yaml cluster_compute: tpl_cpu_1.yaml @@ -2724,8 +2736,10 @@ frequency: nightly team: serve - + python: "3.8" cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_tpl_4_xlarge.yaml @@ -2753,8 +2767,10 @@ frequency: nightly team: serve - + python: "3.8" cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_tpl_4_xlarge.yaml @@ -2781,7 +2797,10 @@ team: serve frequency: nightly working_dir: jobs_tests + python: "3.8" cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_tpl_4_xlarge.yaml run: @@ -2802,9 +2821,12 @@ - name: jobs_check_cuda_available group: Jobs tests team: serve + python: "3.8" frequency: nightly working_dir: jobs_tests cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_tpl_gpu_node.yaml run: @@ -2825,9 +2847,12 @@ - name: jobs_specify_num_gpus group: Jobs tests team: serve + python: "3.8" frequency: nightly working_dir: jobs_tests cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_tpl_gpu_worker.yaml run: @@ -2854,8 +2879,10 @@ frequency: nightly team: serve + python: "3.8" cluster: + byod: {} cluster_env: app_config.yaml cluster_compute: rte_small.yaml @@ -2883,8 +2910,9 @@ frequency: nightly team: serve - + python: "3.8" cluster: + byod: {} cluster_env: app_config.yaml cluster_compute: rte_minimal.yaml @@ -2938,8 +2966,9 @@ frequency: nightly team: serve - + python: "3.8" cluster: + byod: {} cluster_env: app_config.yaml cluster_compute: compute_tpl_32_cpu.yaml @@ -2965,8 +2994,9 @@ frequency: nightly team: serve - + python: "3.8" cluster: + byod: {} cluster_env: app_config.yaml cluster_compute: compute_tpl_32_cpu.yaml @@ -2992,8 +3022,9 @@ frequency: nightly team: serve - + python: "3.8" cluster: + byod: {} cluster_env: app_config.yaml cluster_compute: compute_tpl_8_cpu_autoscaling.yaml @@ -3019,8 +3050,9 @@ frequency: nightly team: serve - + python: "3.8" cluster: + byod: {} cluster_env: app_config.yaml cluster_compute: compute_tpl_32_cpu_autoscaling.yaml @@ -3046,8 +3078,9 @@ frequency: nightly team: serve - + python: "3.8" cluster: + byod: {} cluster_env: app_config.yaml cluster_compute: compute_tpl_single_node.yaml @@ -3093,8 +3126,9 @@ frequency: nightly team: serve - + python: "3.8" cluster: + byod: {} cluster_env: app_config.yaml cluster_compute: compute_tpl_single_node_32_cpu.yaml @@ -3121,8 +3155,9 @@ frequency: nightly team: serve - + python: "3.8" cluster: + byod: {} cluster_env: app_config.yaml cluster_compute: compute_tpl_single_node_32_cpu.yaml @@ -3149,8 +3184,9 @@ frequency: nightly team: serve - + python: "3.8" cluster: + byod: {} cluster_env: app_config.yaml cluster_compute: compute_tpl_single_node_32_cpu.yaml @@ -3177,8 +3213,9 @@ frequency: nightly team: serve - + python: "3.8" cluster: + byod: {} cluster_env: app_config.yaml cluster_compute: compute_tpl_single_node_32_cpu.yaml @@ -3206,8 +3243,9 @@ frequency: nightly team: serve - + python: "3.8" cluster: + byod: {} cluster_env: app_config.yaml cluster_compute: compute_tpl_single_node.yaml @@ -3233,8 +3271,9 @@ frequency: nightly team: serve - + python: "3.8" cluster: + byod: {} cluster_env: app_config.yaml cluster_compute: compute_tpl_single_node.yaml @@ -3261,8 +3300,10 @@ frequency: nightly team: serve - + python: "3.8" cluster: + byod: + type: gpu cluster_env: gpu_app_config.yaml cluster_compute: compute_tpl_gpu_node.yaml @@ -5977,6 +6018,8 @@ byod: runtime_env: - RAY_worker_killing_policy=retriable_lifo + pip: + - ray[default] cluster_env: shuffle/shuffle_app_config.yaml cluster_compute: shuffle/datasets_large_scale_compute_small_instances.yaml