diff --git a/WORKSPACE b/WORKSPACE
index eb6aeba907e3..f995b3b422eb 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -21,9 +21,8 @@ load("@bazel_skylib//lib:versions.bzl", "versions")
 # TODO (shrekris-anyscale): Update the min version to 4.2.2 once Windows uses
 # it in CI.
 
-# When the bazel version is updated, make sure to update it
-# in setup.py as well.
-versions.check(minimum_bazel_version = "5.4.0")
+# Please keep this in sync with the .bazeliskrc file.
+versions.check(minimum_bazel_version = "5.4.1")
 
 # Tools to generate `compile_commands.json` to enable awesome tooling of the C language family.
 # Just run `bazel run @hedron_compile_commands//:refresh_all`
diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh
index 0d12d349a992..2e855c3c2daa 100755
--- a/ci/env/install-dependencies.sh
+++ b/ci/env/install-dependencies.sh
@@ -28,7 +28,7 @@ install_bazel() {
       # Only reinstall Bazel if we need to upgrade to a different version.
       python="$(command -v python3 || command -v python || echo python)"
       current_version="$(bazel --version | grep -o "[0-9]\+.[0-9]\+.[0-9]\+")"
-      new_version="$("${python}" -s -c "import runpy, sys; runpy.run_path(sys.argv.pop(), run_name='__api__')" bazel_version "${SCRIPT_DIR}/../../python/setup.py")"
+      new_version="$(grep 'USE_BAZEL_VERSION=' "${WORKSPACE_DIR}/.bazeliskrc" | grep -o "[0-9]\+.[0-9]\+.[0-9]\+")"
       if [[ "$current_version" == "$new_version" ]]; then
         echo "Bazel of the same version already exists, skipping the install"
         export BAZEL_CONFIG_ONLY=1
diff --git a/python/setup.py b/python/setup.py
index 776d09b7b531..01d02d4276e5 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -35,8 +35,6 @@
 # When the bazel version is updated, make sure to update it
 # in WORKSPACE file as well.
 
-SUPPORTED_BAZEL = (5, 4, 0)
-
 ROOT_DIR = os.path.dirname(__file__)
 BUILD_JAVA = os.getenv("RAY_INSTALL_JAVA") == "1"
 SKIP_BAZEL_BUILD = os.getenv("SKIP_BAZEL_BUILD") == "1"
@@ -614,7 +612,7 @@ def build(build_python, build_java, build_cpp):
 
 def walk_directory(directory):
     file_list = []
-    for (root, dirs, filenames) in os.walk(directory):
+    for root, dirs, filenames in os.walk(directory):
         for name in filenames:
             file_list.append(os.path.join(root, name))
     return file_list
@@ -694,7 +692,7 @@ def pip_run(build_ext):
 
 def api_main(program, *args):
     parser = argparse.ArgumentParser()
-    choices = ["build", "bazel_version", "python_versions", "clean", "help"]
+    choices = ["build", "python_versions", "clean", "help"]
     parser.add_argument("command", type=str, choices=choices)
     parser.add_argument(
         "-l",
@@ -721,8 +719,6 @@ def api_main(program, *args):
             else:
                 raise ValueError("invalid language: {!r}".format(lang))
         result = build(**kwargs)
-    elif parsed_args.command == "bazel_version":
-        print(".".join(map(str, SUPPORTED_BAZEL)))
     elif parsed_args.command == "python_versions":
         for version in SUPPORTED_PYTHONS:
             # NOTE: On Windows this will print "\r\n" on the command line.