[RLlib] Make opencv-python-headless default over opencv-python (#48776)

## Why are these changes needed?

Make `opencv-python-headless` defeault requirement for RLlib/
This is necessary, because `opencv-python` may not work in many headless
environments.

python/requirements/ml/rllib-test-requirements.txt

@@ -5,7 +5,7 @@
 # Atari
 ale_py==0.10.1
 imageio==2.34.2
-opencv-python==4.8.1.78
+opencv-python-headless==4.8.1.78
 
 # For testing MuJoCo envs with gymnasium.
 mujoco==3.2.4

rllib/utils/images.py

@@ -1,18 +1,35 @@
 import logging
+import importlib
 
 import numpy as np
 
 from ray.rllib.utils.annotations import DeveloperAPI
 
 logger = logging.getLogger(__name__)
 
+
+@DeveloperAPI
+def is_package_installed(package_name):
+    try:
+        importlib.metadata.version(package_name)
+        return True
+    except importlib.metadata.PackageNotFoundError:
+        return False
+
+
 try:
     import cv2
 
     cv2.ocl.setUseOpenCL(False)
 
     logger.debug("CV2 found for image processing.")
-except ImportError:
+except ImportError as e:
+    if is_package_installed("opencv-python"):
+        raise ImportError(
+            f"OpenCV is installed, but we failed to import it. This may be because "
+            f"you need to install `opencv-python-headless` instead of "
+            f"`opencv-python`. Error message: {e}",
+        )
     cv2 = None