[Core] Check that temp_dir must be absolute path. (ray-project#36431)

ray.init accepts a _temp_dir arg. The arg should be absolute path because it makes no sense to have a "relative path" on all nodes. However the code did not check that and proceeds as if it is absolute path. This makes the init code to hang and timeout.

Example issues for treating it as absolute: the latest_session symlink links to the temp_dir and if it is relative path it won't work.

Signed-off-by: Ruiyang Wang <rywang014@gmail.com>
Signed-off-by: e428265 <arvind.chandramouli@lmco.com>

python/ray/_private/parameter.py

@@ -96,7 +96,7 @@ class RayParams:
         raylet_socket_name: If provided, it will specify the socket path
             used by the raylet process.
         temp_dir: If provided, it will specify the root temporary
-            directory for the Ray process.
+            directory for the Ray process. Must be an absolute path.
         storage: Specify a URI for persistent cluster-wide storage. This storage path
             must be accessible by all nodes of the cluster, otherwise an error will be
             raised.
@@ -428,6 +428,9 @@ def build_error(resource, alternative):
                 "serialization. Upgrade numpy if using with ray."
             )
 
+        if self.temp_dir is not None and not os.path.isabs(self.temp_dir):
+            raise ValueError("temp_dir must be absolute path or None.")
+
     def _format_ports(self, pre_selected_ports):
         """Format the pre-selected ports information to be more human-readable."""
         ports = pre_selected_ports.copy()

python/ray/_private/worker.py

@@ -1240,8 +1240,8 @@ def init(
         _redis_password: Prevents external clients without the password
             from connecting to Redis if provided.
         _temp_dir: If provided, specifies the root temporary
-            directory for the Ray process. Defaults to an OS-specific
-            conventional location, e.g., "/tmp/ray".
+            directory for the Ray process. Must be an absolute path. Defaults to an
+            OS-specific conventional location, e.g., "/tmp/ray".
         _metrics_export_port: Port number Ray exposes system metrics
             through a Prometheus endpoint. It is currently under active
             development, and the API is subject to change.

python/ray/tests/test_ray_init_2.py

@@ -325,6 +325,13 @@ def test_get_ray_address_from_environment(monkeypatch):
     )
 
 
+# https://github.com/ray-project/ray/issues/36431
+def test_temp_dir_must_be_absolute(shutdown_only):
+    # This test fails with a relative path _temp_dir.
+    with pytest.raises(ValueError):
+        ray.init(_temp_dir="relative_path")
+
+
 if __name__ == "__main__":
     import sys