diff --git a/release/ray_release/byod/requirements_byod.txt b/release/ray_release/byod/requirements_byod.txt index f5e1d91e0e7d..87a00a06a747 100644 --- a/release/ray_release/byod/requirements_byod.txt +++ b/release/ray_release/byod/requirements_byod.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.11 # To update, run: # # bazel run //release:requirements_byod.update @@ -184,24 +184,6 @@ attrs==23.1.0 \ # via # aiohttp # jsonschema -backports-zoneinfo==0.2.1 \ - --hash=sha256:17746bd546106fa389c51dbea67c8b7c8f0d14b5526a579ca6ccf5ed72c526cf \ - --hash=sha256:1b13e654a55cd45672cb54ed12148cd33628f672548f373963b0bff67b217328 \ - --hash=sha256:1c5742112073a563c81f786e77514969acb58649bcdf6cdf0b4ed31a348d4546 \ - --hash=sha256:4a0f800587060bf8880f954dbef70de6c11bbe59c673c3d818921f042f9954a6 \ - --hash=sha256:5c144945a7752ca544b4b78c8c41544cdfaf9786f25fe5ffb10e838e19a27570 \ - --hash=sha256:7b0a64cda4145548fed9efc10322770f929b944ce5cee6c0dfe0c87bf4c0c8c9 \ - --hash=sha256:8439c030a11780786a2002261569bdf362264f605dfa4d65090b64b05c9f79a7 \ - --hash=sha256:8961c0f32cd0336fb8e8ead11a1f8cd99ec07145ec2931122faaac1c8f7fd987 \ - --hash=sha256:89a48c0d158a3cc3f654da4c2de1ceba85263fafb861b98b59040a5086259722 \ - --hash=sha256:a76b38c52400b762e48131494ba26be363491ac4f9a04c1b7e92483d169f6582 \ - --hash=sha256:da6013fd84a690242c310d77ddb8441a559e9cb3d3d59ebac9aca1a57b2e18bc \ - --hash=sha256:e55b384612d93be96506932a786bbcde5a2db7a9e6a4bb4bffe8b733f5b9036b \ - --hash=sha256:e81b76cace8eda1fca50e345242ba977f9be6ae3945af8d46326d776b4cf78d1 \ - --hash=sha256:e8236383a20872c0cdf5a62b554b27538db7fa1bbec52429d8d106effbaeca08 \ - --hash=sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac \ - --hash=sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2 - # via tzlocal boto==2.49.0 \ --hash=sha256:147758d41ae7240dc989f0039f27da8ca0d53734be0eb869ef16e3adcfa462e8 \ --hash=sha256:ea0d3b40a2d852767be77ca343b58a9e3a4b00d9db440efb8da74b4e58025e5a @@ -573,10 +555,6 @@ entrypoints==0.4 \ --hash=sha256:b706eddaa9218a19ebcd67b56818f05bb27589b1ca9e8d797b74affad4ccacd4 \ --hash=sha256:f174b5ff827504fd3cd97cc3f8649f3693f51538c7e4bdf3ef002c8429d42f9f # via numcodecs -exceptiongroup==1.1.1 \ - --hash=sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e \ - --hash=sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785 - # via pytest expiringdict==1.2.2 \ --hash=sha256:09a5d20bc361163e6432a874edd3179676e935eb81b925eccef48d409a8a45e8 \ --hash=sha256:300fb92a7e98f15b05cf9a856c1415b3bc4f2e132be07daa326da6414c23ee09 @@ -931,17 +909,11 @@ idna==3.4 \ importlib-metadata==6.6.0 \ --hash=sha256:43dd286a2cd8995d5eaef7fee2066340423b818ed3fd70adf0bad5f1fac53fed \ --hash=sha256:92501cdf9cc66ebd3e612f1b4f0c0765dfa42f0fa38ffb319b6bd84dd675d705 - # via - # -r release/ray_release/byod/requirements_byod.in - # ale-py - # gym - # markdown + # via -r release/ray_release/byod/requirements_byod.in importlib-resources==5.12.0 \ --hash=sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6 \ --hash=sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a - # via - # ale-py - # jsonschema + # via ale-py iniconfig==2.0.0 \ --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 @@ -1280,10 +1252,6 @@ pathspec==0.11.1 \ pathtools==0.1.2 \ --hash=sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0 # via wandb -pkgutil-resolve-name==1.3.10 \ - --hash=sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174 \ - --hash=sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e - # via jsonschema pluggy==1.0.0 \ --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 \ --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3 @@ -1795,10 +1763,6 @@ threadpoolctl==3.1.0 \ --hash=sha256:8b99adda265feb6773280df41eece7b2e6561b772d21ffd52e372f999024907b \ --hash=sha256:a335baacfaa4400ae1f0d8e3a58d6674d2f8828e3716bb2802c44955ad391380 # via scikit-learn -tomli==2.0.1 \ - --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ - --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f - # via pytest tornado==6.3.2 \ --hash=sha256:05615096845cf50a895026f749195bf0b10b8909f9be672f50b0fe69cba368e4 \ --hash=sha256:0c325e66c8123c606eea33084976c832aa4e766b7dff8aedd7587ea44a604cdf \ @@ -1829,12 +1793,9 @@ typing-extensions==4.6.2 \ --hash=sha256:06006244c70ac8ee83fa8282cb188f697b8db25bc8b4df07be1873c43897060c \ --hash=sha256:3a8b36f13dd5fdc5d1b16fe317f5668545de77fa0b8e02006381fd49d731ab98 # via - # ale-py # pydantic - # rich # tensorflow # typer - # wandb tzdata==2023.3 \ --hash=sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a \ --hash=sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda @@ -2032,9 +1993,7 @@ zarr==2.14.2 \ zipp==3.15.0 \ --hash=sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b \ --hash=sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata # WARNING: The following packages were not pinned, but pip requires them to be # pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag. diff --git a/release/ray_release/byod/requirements_ml_byod.in b/release/ray_release/byod/requirements_ml_byod.in index c7d071280edf..a5951c3363e8 100644 --- a/release/ray_release/byod/requirements_ml_byod.in +++ b/release/ray_release/byod/requirements_ml_byod.in @@ -2,6 +2,7 @@ boto3 cmake crc32c +modin numpy pyarrow pytest diff --git a/release/ray_release/byod/requirements_ml_byod.txt b/release/ray_release/byod/requirements_ml_byod.txt index 46688e521af7..cbe4bc3b5394 100644 --- a/release/ray_release/byod/requirements_ml_byod.txt +++ b/release/ray_release/byod/requirements_ml_byod.txt @@ -121,6 +121,10 @@ exceptiongroup==1.1.1 \ --hash=sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e \ --hash=sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785 # via pytest +fsspec==2023.6.0 \ + --hash=sha256:1cbad1faef3e391fba6dc005ae9b5bdcbf43005c9167ce78c915549c352c869a \ + --hash=sha256:d0b2f935446169753e7a5c5c55681c54ea91996cc67be93c39a154fb3a2742af + # via modin iniconfig==2.0.0 \ --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 @@ -131,6 +135,10 @@ jmespath==1.0.1 \ # via # boto3 # botocore +modin==0.22.1 \ + --hash=sha256:bd6c01ab3714a6c8bb3ae73c34e22264db20f2f8751f4caeef3731573d518501 \ + --hash=sha256:ca7cc6dfd41cf5b405aef9281f136b65a42aa1f2ee41f660f0fdcf30cad2e20a + # via -r release/ray_release/byod/requirements_ml_byod.in numpy==1.24.3 \ --hash=sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187 \ --hash=sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812 \ @@ -162,15 +170,64 @@ numpy==1.24.3 \ --hash=sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b # via # -r release/ray_release/byod/requirements_ml_byod.in + # modin + # pandas # pyarrow packaging==23.1 \ --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \ --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f - # via pytest + # via + # modin + # pytest +pandas==1.5.3 \ + --hash=sha256:14e45300521902689a81f3f41386dc86f19b8ba8dd5ac5a3c7010ef8d2932813 \ + --hash=sha256:26d9c71772c7afb9d5046e6e9cf42d83dd147b5cf5bcb9d97252077118543792 \ + --hash=sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406 \ + --hash=sha256:41179ce559943d83a9b4bbacb736b04c928b095b5f25dd2b7389eda08f46f373 \ + --hash=sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328 \ + --hash=sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996 \ + --hash=sha256:565fa34a5434d38e9d250af3c12ff931abaf88050551d9fbcdfafca50d62babf \ + --hash=sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6 \ + --hash=sha256:5fbcb19d6fceb9e946b3e23258757c7b225ba450990d9ed63ccceeb8cae609f7 \ + --hash=sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc \ + --hash=sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1 \ + --hash=sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23 \ + --hash=sha256:7cec0bee9f294e5de5bbfc14d0573f65526071029d036b753ee6507d2a21480a \ + --hash=sha256:87bd9c03da1ac870a6d2c8902a0e1fd4267ca00f13bc494c9e5a9020920e1d51 \ + --hash=sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572 \ + --hash=sha256:9842b6f4b8479e41968eced654487258ed81df7d1c9b7b870ceea24ed9459b31 \ + --hash=sha256:9f69c4029613de47816b1bb30ff5ac778686688751a5e9c99ad8c7031f6508e5 \ + --hash=sha256:a50d9a4336a9621cab7b8eb3fb11adb82de58f9b91d84c2cd526576b881a0c5a \ + --hash=sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003 \ + --hash=sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d \ + --hash=sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354 \ + --hash=sha256:c4c00e0b0597c8e4f59e8d461f797e5d70b4d025880516a8261b2817c47759ee \ + --hash=sha256:c74a62747864ed568f5a82a49a23a8d7fe171d0c69038b38cedf0976831296fa \ + --hash=sha256:dd05f7783b3274aa206a1af06f0ceed3f9b412cf665b7247eacd83be41cf7bf0 \ + --hash=sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9 \ + --hash=sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae \ + --hash=sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc + # via modin pluggy==1.0.0 \ --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 \ --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3 # via pytest +psutil==5.9.5 \ + --hash=sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d \ + --hash=sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217 \ + --hash=sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4 \ + --hash=sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c \ + --hash=sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f \ + --hash=sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da \ + --hash=sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4 \ + --hash=sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42 \ + --hash=sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5 \ + --hash=sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4 \ + --hash=sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9 \ + --hash=sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f \ + --hash=sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30 \ + --hash=sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48 + # via modin pyarrow==12.0.1 \ --hash=sha256:051f9f5ccf585f12d7de836e50965b3c235542cc896959320d9776ab93f3b33d \ --hash=sha256:1887bdae17ec3b4c046fcf19951e71b6a619f39fa674f9881216173566c8f718 \ @@ -205,7 +262,13 @@ pytest==7.3.2 \ python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 - # via botocore + # via + # botocore + # pandas +pytz==2023.3 \ + --hash=sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588 \ + --hash=sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb + # via pandas s3transfer==0.6.1 \ --hash=sha256:3c0da2d074bf35d6870ef157158641178a4204a6e689e82546083e31e0311346 \ --hash=sha256:640bb492711f4c0c0905e1f62b6aaeb771881935ad27884852411f8e9cacbca9 diff --git a/release/release_tests.yaml b/release/release_tests.yaml index ab085894e340..7d4ed3e6d4ee 100644 --- a/release/release_tests.yaml +++ b/release/release_tests.yaml @@ -87,8 +87,9 @@ frequency: nightly team: ml - + python: "3.8" cluster: + byod: {} cluster_env: app_config.yaml cluster_compute: cpt_autoscaling_1-3_aws.yaml @@ -121,8 +122,12 @@ frequency: nightly-3x team: ml - + python: "3.8" cluster: + byod: + runtime_env: + - RAY_memory_usage_threshold=0.5 + - automatic_object_spilling_enabled=0 cluster_env: frequent_pausing/app_config.yaml cluster_compute: frequent_pausing/compute_config_aws.yaml @@ -186,8 +191,10 @@ frequency: nightly team: ml - + python: "3.8" cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_data_20_nodes_aws.yaml @@ -216,7 +223,6 @@ python: "3.8" frequency: nightly team: ml - cluster: cluster_env: xgboost_app_config.yaml cluster_compute: compute_xgboost_aws.yaml @@ -252,8 +258,10 @@ frequency: nightly team: ml - + python: "3.8" cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_cpu_4_aws.yaml @@ -281,8 +289,10 @@ frequency: weekly team: ml - + python: "3.8" cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_gpu_4x4_aws.yaml @@ -326,8 +336,10 @@ frequency: nightly team: ml - + python: "3.8" cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_cpu_1_aws.yaml @@ -353,8 +365,10 @@ frequency: nightly team: ml - + python: "3.8" cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_gpu_1_cpu_16_aws.yaml @@ -380,10 +394,12 @@ frequency: nightly team: ml - + python: "3.8" stable: false cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_gpu_4x4_aws.yaml @@ -411,8 +427,10 @@ frequency: nightly team: ml - + python: "3.8" cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_cpu_4_aws.yaml @@ -440,8 +458,10 @@ frequency: nightly team: ml - + python: "3.8" cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_cpu_8_aws.yaml @@ -469,7 +489,6 @@ frequency: nightly team: ml - cluster: cluster_env: app_config.yaml cluster_compute: compute_gpu_4x4_aws.yaml @@ -499,8 +518,10 @@ frequency: nightly team: ml - + python: "3.8" cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_cpu_4_aws.yaml @@ -529,8 +550,10 @@ frequency: nightly team: ml - + python: "3.8" cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_cpu_1_aws.yaml @@ -556,10 +579,12 @@ frequency: nightly team: ml - + python: "3.8" stable: false cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_cpu_4_aws.yaml @@ -588,10 +613,12 @@ frequency: weekly team: ml - + python: "3.8" stable: false cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_gpu_4x4_aws.yaml @@ -634,8 +661,10 @@ frequency: nightly team: ml - + python: "3.8" cluster: + byod: + type: gpu cluster_env: app_config.yaml cluster_compute: compute_gpu_1_aws.yaml @@ -661,7 +690,6 @@ frequency: nightly team: ml - stable: false cluster: @@ -5934,7 +5962,12 @@ frequency: nightly team: data + python: "3.8" cluster: + byod: + type: gpu + pip: + - git+https://github.com/ray-project/ray_shuffling_data_loader.git@add-embedding-model cluster_env: pipelined_training_app.yaml cluster_compute: pipelined_training_compute.yaml @@ -5959,7 +5992,15 @@ frequency: nightly team: data + python: "3.8" cluster: + byod: + type: gpu + runtime_env: + - RAY_lineage_pinning_enabled=1 + - RAY_record_ref_creation_sites=1 + pip: + - git+https://github.com/ray-project/ray_shuffling_data_loader.git@add-embedding-model cluster_env: pipelined_ingestion_app.yaml cluster_compute: pipelined_ingestion_compute.yaml @@ -5986,7 +6027,13 @@ frequency: nightly team: data + python: "3.8" cluster: + byod: + runtime_env: + - RAY_worker_killing_policy=retriable_lifo + pip: + - ray[default] cluster_env: shuffle/shuffle_app_config.yaml cluster_compute: shuffle/datasets_large_scale_compute_small_instances.yaml @@ -6253,7 +6300,15 @@ frequency: nightly team: data + python: "3.8" cluster: + byod: + type: gpu + pip: + - git+https://github.com/ray-project/ray_shuffling_data_loader.git@add-embedding-model + runtime_env: + - RAY_lineage_pinning_enabled=1 + - RAY_record_ref_creation_sites=1 cluster_env: dataset/pipelined_ingestion_app.yaml cluster_compute: dataset/pipelined_ingestion_compute.yaml @@ -6282,7 +6337,13 @@ frequency: nightly team: data + python: "3.8" cluster: + byod: + runtime_env: + - RAY_worker_killing_policy=retriable_lifo + pip: + - ray[default] cluster_env: shuffle/shuffle_app_config.yaml cluster_compute: shuffle/datasets_large_scale_compute_small_instances.yaml @@ -6310,7 +6371,11 @@ frequency: nightly team: data + python: "3.8" cluster: + byod: + runtime_env: + - RAY_memory_monitor_refresh_ms=0 cluster_env: shuffle/shuffle_app_config_oom_disabled.yaml cluster_compute: shuffle/datasets_large_scale_compute_small_instances.yaml @@ -6373,8 +6438,12 @@ frequency: nightly team: data + python: "3.8" cluster: # leave oom disabled as test is marked unstable at the moment. + byod: + runtime_env: + - RAY_memory_monitor_refresh_ms=0 cluster_env: shuffle/shuffle_app_config_oom_disabled.yaml cluster_compute: shuffle/datasets_large_scale_compute_small_instances.yaml