ENH: Compatible with NumPy 2.x (#817)

Co-authored-by: hucorz <chaohui6014@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

.github/workflows/python.yaml

@@ -149,7 +149,7 @@ jobs:
           pip install --upgrade --upgrade-strategy only-if-needed --no-cache-dir ".[doc]"
         else
           pip install -e "git+https://github.com/xorbitsai/xoscar.git@main#subdirectory=python&egg=xoscar"
-          pip install "numpy<2.0.0" scipy cython pyftpdlib coverage flaky numexpr
+          pip install -U numpy scipy cython pyftpdlib coverage flaky numexpr
 
           if [[ "$MODULE" == "xorbits/pandas" ]]; then
             pip install openpyxl
@@ -360,7 +360,7 @@ jobs:
           export YARN_HOME=$HADOOP_HOME
           export HADOOP_COMMON_LIB_NATIVE_DIR="$HADOOP_HOME/lib/native"
           export PATH="$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin"
-          pytest --timeout=1500 -W ignore::PendingDeprecationWarning xorbits/_mars -m hadoop
+          pytest --ignore xorbits/_mars/learn --timeout=1500 -W ignore::PendingDeprecationWarning xorbits/_mars -m hadoop
         elif [[ "$MODULE" == "vineyard" ]]; then
           pytest --timeout=1500 -W ignore::PendingDeprecationWarning \
             --cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/_mars/storage/tests/test_libs.py

doc/source/user_guide/storage_backend.rst

@@ -40,9 +40,9 @@ create a YAML configuration file named `file.yml` which specify `backends` and `
     
     "@inherits": "@default"
     storage:
-    backends: [disk]
-    disk:
-        root_dirs: "/tmp"
+        backends: [disk]
+        disk:
+            root_dirs: "/tmp"
 
 Start the worker using the :code:`-f file.yml` option:
 
@@ -86,9 +86,9 @@ and start the worker by adding :code:`-f file.yml` option.
     
     "@inherits": "@default"
     storage:
-    backends: [disk]
-    disk:
-        root_dirs: "/mnt/xorbits"
+        backends: [disk]
+        disk:
+            root_dirs: "/mnt/xorbits"
 
 
 GPU

python/setup.cfg

@@ -24,7 +24,7 @@ include_package_data = True
 packages = find:
 install_requires =
     xoscar>=0.0.8
-    numpy>=1.14.0,<2.0.0
+    numpy>=1.14.0
     pandas>=1.0.0
     scipy>=1.0.0; sys_platform!="win32" or python_version>="3.10"
     scipy>=1.0.0,<=1.9.1; sys_platform=="win32" and python_version<"3.10"

python/xorbits/_mars/core/operand/core.py

@@ -19,8 +19,13 @@
 import numpy as np
 from xoscar.metrics import Metrics
 
+from ....utils import is_numpy_2
+
 try:
-    from numpy.core._exceptions import UFuncTypeError
+    if is_numpy_2():
+        from numpy._core._exceptions import UFuncTypeError
+    else:
+        from numpy.core._exceptions import UFuncTypeError
 except ImportError:  # pragma: no cover
     UFuncTypeError = None
 

python/xorbits/_mars/dataframe/indexing/align.py

@@ -144,7 +144,7 @@ def _call_dataframe_series(self, lhs: TileableType, rhs: TileableType):
                 series_index = rhs.index_value.to_pandas()
                 dtypes = lhs.dtypes.reindex(
                     lhs.dtypes.index.join(series_index, how=self.join)
-                ).fillna(np.dtype(np.float_))
+                ).fillna(np.dtype(np.float64))
                 l_shape[1] = r_size = len(dtypes)
                 col_val = r_idx_val = parse_index(dtypes.index, store_data=True)
 

python/xorbits/_mars/dataframe/missing/tests/test_missing.py

@@ -241,36 +241,36 @@ def test_isna(setup):
         isna(midx)
 
     # list
-    l = [1, 2, 3, np.Inf, np.NaN, pd.NA, pd.NaT]
+    l = [1, 2, 3, np.inf, np.nan, pd.NA, pd.NaT]
     actual = isna(l).execute().fetch()
     expected = pd.isna(l)
     np.testing.assert_array_equal(expected, actual)
 
     # tuple
-    t = (1, 2, 3, np.Inf, np.NaN, pd.NA, pd.NaT)
+    t = (1, 2, 3, np.inf, np.nan, pd.NA, pd.NaT)
     assert not isna(t)
 
     # numpy ndarray
-    narr = np.array((1, 2, 3, np.Inf, np.NaN))
+    narr = np.array((1, 2, 3, np.inf, np.nan))
     actual = isna(narr).execute().fetch()
     expected = pd.isna(narr)
     np.testing.assert_array_equal(expected, actual)
 
     # pandas index
-    pi = pd.Index((1, 2, 3, np.Inf, np.NaN, pd.NA, pd.NaT))
+    pi = pd.Index((1, 2, 3, np.inf, np.nan, pd.NA, pd.NaT))
     actual = isna(pi).execute().fetch()
     expected = pd.isna(pi)
     np.testing.assert_array_equal(expected, actual)
 
     # pandas series
-    ps = pd.Series((1, 2, 3, np.Inf, np.NaN, pd.NA, pd.NaT))
+    ps = pd.Series((1, 2, 3, np.inf, np.nan, pd.NA, pd.NaT))
     actual = isna(ps).execute().fetch()
     expected = pd.isna(ps)
     pd.testing.assert_series_equal(expected, actual)
 
     # pandas dataframe
     pdf = pd.DataFrame(
-        {"foo": (1, 2, 3, np.Inf, pd.NA), "bar": (4, 5, 6, np.NaN, pd.NaT)}
+        {"foo": (1, 2, 3, np.inf, pd.NA), "bar": (4, 5, 6, np.nan, pd.NaT)}
     )
     actual = isna(pdf).execute().fetch()
     expected = pd.isna(pdf)
@@ -324,36 +324,36 @@ def test_notna(setup):
         notna(midx)
 
     # list
-    l = [1, 2, 3, np.Inf, np.NaN, pd.NA, pd.NaT]
+    l = [1, 2, 3, np.inf, np.nan, pd.NA, pd.NaT]
     actual = notna(l).execute().fetch()
     expected = pd.notna(l)
     np.testing.assert_array_equal(expected, actual)
 
     # tuple
-    t = (1, 2, 3, np.Inf, np.NaN, pd.NA, pd.NaT)
+    t = (1, 2, 3, np.inf, np.nan, pd.NA, pd.NaT)
     assert notna(t)
 
     # numpy ndarray
-    narr = np.array((1, 2, 3, np.Inf, np.NaN))
+    narr = np.array((1, 2, 3, np.inf, np.nan))
     actual = notna(narr).execute().fetch()
     expected = pd.notna(narr)
     np.testing.assert_array_equal(expected, actual)
 
     # pandas index
-    pi = pd.Index((1, 2, 3, np.Inf, np.NaN, pd.NA, pd.NaT))
+    pi = pd.Index((1, 2, 3, np.inf, np.nan, pd.NA, pd.NaT))
     actual = notna(pi).execute().fetch()
     expected = pd.notna(pi)
     np.testing.assert_array_equal(expected, actual)
 
     # pandas series
-    ps = pd.Series((1, 2, 3, np.Inf, np.NaN, pd.NA, pd.NaT))
+    ps = pd.Series((1, 2, 3, np.inf, np.nan, pd.NA, pd.NaT))
     actual = notna(ps).execute().fetch()
     expected = pd.notna(ps)
     pd.testing.assert_series_equal(expected, actual)
 
     # pandas dataframe
     pdf = pd.DataFrame(
-        {"foo": (1, 2, 3, np.Inf, pd.NA), "bar": (4, 5, 6, np.NaN, pd.NaT)}
+        {"foo": (1, 2, 3, np.inf, pd.NA), "bar": (4, 5, 6, np.nan, pd.NaT)}
     )
     actual = notna(pdf).execute().fetch()
     expected = pd.notna(pdf)

python/xorbits/_mars/dataframe/reduction/core.py

@@ -365,7 +365,7 @@ def _call_dataframe(self, df):
                     #  handle pandas Dtypes in the future more carefully.
                     reduced_dtype = np.dtype("O")
                 else:
-                    reduced_dtype = np.find_common_type(dtypes, [])
+                    reduced_dtype = np.result_type(*dtypes)
 
         if level is not None:
             return self._call_groupby_level(df[reduced_cols], level)

python/xorbits/_mars/dataframe/statistics/corr.py

@@ -46,7 +46,7 @@ def _set_inputs(self, inputs):
     def __call__(self, df_or_series):
         if isinstance(df_or_series, SERIES_TYPE):
             inputs = filter_inputs([df_or_series, self.other])
-            return self.new_scalar(inputs, dtype=np.dtype(np.float_))
+            return self.new_scalar(inputs, dtype=np.dtype(np.float64))
         else:
 
             def _filter_numeric(obj):
@@ -63,7 +63,7 @@ def _filter_numeric(obj):
             inputs = filter_inputs([df_or_series, self.other])
             if self.axis is None:
                 dtypes = pd.Series(
-                    [np.dtype(np.float_)] * len(df_or_series.dtypes),
+                    [np.dtype(np.float64)] * len(df_or_series.dtypes),
                     index=df_or_series.dtypes.index,
                 )
                 return self.new_dataframe(
@@ -88,7 +88,7 @@ def _filter_numeric(obj):
                 return self.new_series(
                     inputs,
                     shape=shape,
-                    dtype=np.dtype(np.float_),
+                    dtype=np.dtype(np.float64),
                     index_value=new_index_value,
                 )
 
@@ -115,8 +115,8 @@ def _tile_pearson_cross(left, right, min_periods):
             right.fillna(0).to_tensor(),
         )
 
-        nna_left = left.notna().to_tensor().astype(np.float_)
-        nna_right = right.notna().to_tensor().astype(np.float_)
+        nna_left = left.notna().to_tensor().astype(np.float64)
+        nna_right = right.notna().to_tensor().astype(np.float64)
 
         sum_left = left_tensor.T.dot(nna_right)
         sum_right = right_tensor.T.dot(nna_left)
@@ -143,8 +143,8 @@ def _tile_pearson_align(cls, left, right, axis):
         if has_unknown_shape(left, right):
             yield left.chunks + right.chunks + [left, right]
 
-        nna_left = left.notna().astype(np.float_)
-        nna_right = right.notna().astype(np.float_)
+        nna_left = left.notna().astype(np.float64)
+        nna_right = right.notna().astype(np.float64)
 
         left, right = left.fillna(0), right.fillna(0)
 

python/xorbits/_mars/learn/contrib/lightgbm/_predict.py

@@ -75,7 +75,7 @@ def __call__(self):
             shape = (self.data.shape[0],)
 
         if self._proba:
-            dtype = np.dtype(np.float_)
+            dtype = np.dtype(np.float64)
         elif hasattr(self.model, "classes_"):
             dtype = np.array(self.model.classes_).dtype
         else:

python/xorbits/_mars/learn/decomposition/tests/test_truncated_svd.py

@@ -27,12 +27,10 @@
 shape = 60, 55
 n_samples, n_features = shape
 rng = check_random_state(42)
-X = rng.randint(-100, 20, np.product(shape)).reshape(shape)
+X = rng.randint(-100, 20, np.prod(shape)).reshape(shape)
 X = sp.csr_matrix(np.maximum(X, 0), dtype=np.float64)
 X.data[:] = 1 + np.log(X.data)
-Xdense = X.A
-n_samples = n_samples
-n_features = n_features
+Xdense = X.toarray()
 
 
 def test_attributes(setup):

python/xorbits/_mars/learn/utils/validation.py

@@ -17,7 +17,13 @@
 import warnings
 
 import numpy as np
-from numpy.core.numeric import ComplexWarning
+
+from ....utils import is_numpy_2
+
+if is_numpy_2():
+    from numpy.exceptions import ComplexWarning
+else:
+    from numpy.core.numeric import ComplexWarning
 
 try:
     from sklearn.exceptions import DataConversionWarning

python/xorbits/_mars/tensor/__init__.py

@@ -303,9 +303,9 @@
 # noinspection PyUnresolvedReferences
 from ..core import ExecutableTuple
 
-from numpy import __version__ as np_ver
+from .utils import is_numpy_2
 
-if np_ver >= "2.0.0":
+if is_numpy_2():
     from numpy.exceptions import AxisError
 else:
     from numpy import AxisError

python/xorbits/_mars/tensor/arithmetic/tests/test_arithmetic.py

@@ -357,7 +357,7 @@ def test_unify_chunk_add():
 
 def test_frexp():
     t1 = ones((3, 4, 5), chunk_size=2)
-    t2 = empty((3, 4, 5), dtype=np.float_, chunk_size=2)
+    t2 = empty((3, 4, 5), dtype=np.float64, chunk_size=2)
     op_type = type(t1.op)
 
     o1, o2 = frexp(t1)

python/xorbits/_mars/tensor/array_utils.py

@@ -21,6 +21,7 @@
 from ..lib import sparse
 from ..lib.sparse.core import get_dense_module, issparse
 from ..utils import is_same_module, lazy_import
+from .utils import is_numpy_2
 
 cp = lazy_import("cupy", rename="cp")
 
@@ -43,6 +44,18 @@ def is_cupy(x):
         return False
 
 
+def get_device_id(input_data):
+    # numpy2.x added `device` attribute to `np.ndarray`
+    # `np.ndarray.device` return `cpu` and does not have `id` attribute
+    # while `cupy.ndarray.device.id` return the GPU device id
+    if hasattr(input_data, "device") and not (
+        is_numpy_2() and isinstance(input_data, np.ndarray)
+    ):
+        return input_data.device.id
+    else:
+        return -1
+
+
 def get_array_module(x, nosparse=False):
     if issparse(x):
         if nosparse:
@@ -117,8 +130,7 @@ def as_same_device(inputs, device=None, ret_extra=False, copy_if_not_writeable=F
     if device is None:
         try:
             device = _most_nbytes_device(
-                (i.device.id if hasattr(i, "device") else -1, i.nbytes)
-                for i in input_tensors
+                (get_device_id(i), i.nbytes) for i in input_tensors
             )
         except ValueError:
             device = -1

python/xorbits/_mars/tensor/base/broadcast_to.py

@@ -17,7 +17,7 @@
 
 from ... import opcodes as OperandDef
 from ...serialization.serializables import KeyField, TupleField
-from ..array_utils import device, get_array_module
+from ..array_utils import device, get_array_module, get_device_id
 from ..datasource import tensor as astensor
 from ..operands import TensorHasInput, TensorOperandMixin
 
@@ -80,7 +80,7 @@ def tile(cls, op):
     def execute(cls, ctx, op):
         xp = get_array_module(ctx[op.input.key])
         input_data = ctx[op.input.key]
-        device_id = input_data.device.id if hasattr(input_data, "device") else -1
+        device_id = get_device_id(input_data)
 
         with device(device_id):
             shape = op.shape

python/xorbits/_mars/tensor/base/expand_dims.py

@@ -16,6 +16,12 @@
 import numpy as np
 
 from ..datasource import tensor as astensor
+from ..utils import is_numpy_2
+
+if is_numpy_2():
+    from numpy.exceptions import AxisError
+else:
+    from numpy import AxisError
 
 
 def expand_dims(a, axis):
@@ -77,9 +83,7 @@ def expand_dims(a, axis):
     a = astensor(a)
 
     if axis > a.ndim or axis < -a.ndim - 1:
-        raise np.AxisError(
-            f"Axis must be between -{a.ndim + 1} and {a.ndim}, got {axis}"
-        )
+        raise AxisError(f"Axis must be between -{a.ndim + 1} and {a.ndim}, got {axis}")
 
     axis = axis if axis >= 0 else axis + a.ndim + 1
     indexes = (slice(None),) * axis + (np.newaxis,) + (slice(None),) * (a.ndim - axis)

python/xorbits/_mars/tensor/base/rollaxis.py

@@ -13,9 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import numpy as np
+from ..utils import is_numpy_2, validate_axis
 
-from ..utils import validate_axis
+if is_numpy_2():
+    from numpy.exceptions import AxisError
+else:
+    from numpy import AxisError
 
 
 def rollaxis(tensor, axis, start=0):
@@ -66,7 +69,7 @@ def rollaxis(tensor, axis, start=0):
         start += n
     msg = "'%s' arg requires %d <= %s < %d, but %d was passed in"
     if not (0 <= start < n + 1):
-        raise np.AxisError(msg % ("start", -n, "start", n + 1, start))
+        raise AxisError(msg % ("start", -n, "start", n + 1, start))
     if axis < start:
         # it's been removed
         start -= 1