diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 5f6b6547d..bc6ff5752 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -149,7 +149,7 @@ jobs: pip install --upgrade --upgrade-strategy only-if-needed --no-cache-dir ".[doc]" else pip install -e "git+https://github.com/xorbitsai/xoscar.git@main#subdirectory=python&egg=xoscar" - pip install "numpy<2.0.0" scipy cython pyftpdlib coverage flaky numexpr + pip install -U numpy scipy cython pyftpdlib coverage flaky numexpr if [[ "$MODULE" == "xorbits/pandas" ]]; then pip install openpyxl @@ -360,7 +360,7 @@ jobs: export YARN_HOME=$HADOOP_HOME export HADOOP_COMMON_LIB_NATIVE_DIR="$HADOOP_HOME/lib/native" export PATH="$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin" - pytest --timeout=1500 -W ignore::PendingDeprecationWarning xorbits/_mars -m hadoop + pytest --ignore xorbits/_mars/learn --timeout=1500 -W ignore::PendingDeprecationWarning xorbits/_mars -m hadoop elif [[ "$MODULE" == "vineyard" ]]; then pytest --timeout=1500 -W ignore::PendingDeprecationWarning \ --cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/_mars/storage/tests/test_libs.py diff --git a/doc/source/user_guide/storage_backend.rst b/doc/source/user_guide/storage_backend.rst index d9fcca4f9..d89a1e956 100644 --- a/doc/source/user_guide/storage_backend.rst +++ b/doc/source/user_guide/storage_backend.rst @@ -40,9 +40,9 @@ create a YAML configuration file named `file.yml` which specify `backends` and ` "@inherits": "@default" storage: - backends: [disk] - disk: - root_dirs: "/tmp" + backends: [disk] + disk: + root_dirs: "/tmp" Start the worker using the :code:`-f file.yml` option: @@ -86,9 +86,9 @@ and start the worker by adding :code:`-f file.yml` option. "@inherits": "@default" storage: - backends: [disk] - disk: - root_dirs: "/mnt/xorbits" + backends: [disk] + disk: + root_dirs: "/mnt/xorbits" GPU diff --git a/python/setup.cfg b/python/setup.cfg index 045cb0292..4ee3df4c3 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -24,7 +24,7 @@ include_package_data = True packages = find: install_requires = xoscar>=0.0.8 - numpy>=1.14.0,<2.0.0 + numpy>=1.14.0 pandas>=1.0.0 scipy>=1.0.0; sys_platform!="win32" or python_version>="3.10" scipy>=1.0.0,<=1.9.1; sys_platform=="win32" and python_version<"3.10" diff --git a/python/xorbits/_mars/core/operand/core.py b/python/xorbits/_mars/core/operand/core.py index fb711fc19..7940b7651 100644 --- a/python/xorbits/_mars/core/operand/core.py +++ b/python/xorbits/_mars/core/operand/core.py @@ -19,8 +19,13 @@ import numpy as np from xoscar.metrics import Metrics +from ....utils import is_numpy_2 + try: - from numpy.core._exceptions import UFuncTypeError + if is_numpy_2(): + from numpy._core._exceptions import UFuncTypeError + else: + from numpy.core._exceptions import UFuncTypeError except ImportError: # pragma: no cover UFuncTypeError = None diff --git a/python/xorbits/_mars/dataframe/indexing/align.py b/python/xorbits/_mars/dataframe/indexing/align.py index 195e11941..ffbcd438d 100644 --- a/python/xorbits/_mars/dataframe/indexing/align.py +++ b/python/xorbits/_mars/dataframe/indexing/align.py @@ -144,7 +144,7 @@ def _call_dataframe_series(self, lhs: TileableType, rhs: TileableType): series_index = rhs.index_value.to_pandas() dtypes = lhs.dtypes.reindex( lhs.dtypes.index.join(series_index, how=self.join) - ).fillna(np.dtype(np.float_)) + ).fillna(np.dtype(np.float64)) l_shape[1] = r_size = len(dtypes) col_val = r_idx_val = parse_index(dtypes.index, store_data=True) diff --git a/python/xorbits/_mars/dataframe/missing/tests/test_missing.py b/python/xorbits/_mars/dataframe/missing/tests/test_missing.py index 4f3223a38..321df0fcf 100644 --- a/python/xorbits/_mars/dataframe/missing/tests/test_missing.py +++ b/python/xorbits/_mars/dataframe/missing/tests/test_missing.py @@ -241,36 +241,36 @@ def test_isna(setup): isna(midx) # list - l = [1, 2, 3, np.Inf, np.NaN, pd.NA, pd.NaT] + l = [1, 2, 3, np.inf, np.nan, pd.NA, pd.NaT] actual = isna(l).execute().fetch() expected = pd.isna(l) np.testing.assert_array_equal(expected, actual) # tuple - t = (1, 2, 3, np.Inf, np.NaN, pd.NA, pd.NaT) + t = (1, 2, 3, np.inf, np.nan, pd.NA, pd.NaT) assert not isna(t) # numpy ndarray - narr = np.array((1, 2, 3, np.Inf, np.NaN)) + narr = np.array((1, 2, 3, np.inf, np.nan)) actual = isna(narr).execute().fetch() expected = pd.isna(narr) np.testing.assert_array_equal(expected, actual) # pandas index - pi = pd.Index((1, 2, 3, np.Inf, np.NaN, pd.NA, pd.NaT)) + pi = pd.Index((1, 2, 3, np.inf, np.nan, pd.NA, pd.NaT)) actual = isna(pi).execute().fetch() expected = pd.isna(pi) np.testing.assert_array_equal(expected, actual) # pandas series - ps = pd.Series((1, 2, 3, np.Inf, np.NaN, pd.NA, pd.NaT)) + ps = pd.Series((1, 2, 3, np.inf, np.nan, pd.NA, pd.NaT)) actual = isna(ps).execute().fetch() expected = pd.isna(ps) pd.testing.assert_series_equal(expected, actual) # pandas dataframe pdf = pd.DataFrame( - {"foo": (1, 2, 3, np.Inf, pd.NA), "bar": (4, 5, 6, np.NaN, pd.NaT)} + {"foo": (1, 2, 3, np.inf, pd.NA), "bar": (4, 5, 6, np.nan, pd.NaT)} ) actual = isna(pdf).execute().fetch() expected = pd.isna(pdf) @@ -324,36 +324,36 @@ def test_notna(setup): notna(midx) # list - l = [1, 2, 3, np.Inf, np.NaN, pd.NA, pd.NaT] + l = [1, 2, 3, np.inf, np.nan, pd.NA, pd.NaT] actual = notna(l).execute().fetch() expected = pd.notna(l) np.testing.assert_array_equal(expected, actual) # tuple - t = (1, 2, 3, np.Inf, np.NaN, pd.NA, pd.NaT) + t = (1, 2, 3, np.inf, np.nan, pd.NA, pd.NaT) assert notna(t) # numpy ndarray - narr = np.array((1, 2, 3, np.Inf, np.NaN)) + narr = np.array((1, 2, 3, np.inf, np.nan)) actual = notna(narr).execute().fetch() expected = pd.notna(narr) np.testing.assert_array_equal(expected, actual) # pandas index - pi = pd.Index((1, 2, 3, np.Inf, np.NaN, pd.NA, pd.NaT)) + pi = pd.Index((1, 2, 3, np.inf, np.nan, pd.NA, pd.NaT)) actual = notna(pi).execute().fetch() expected = pd.notna(pi) np.testing.assert_array_equal(expected, actual) # pandas series - ps = pd.Series((1, 2, 3, np.Inf, np.NaN, pd.NA, pd.NaT)) + ps = pd.Series((1, 2, 3, np.inf, np.nan, pd.NA, pd.NaT)) actual = notna(ps).execute().fetch() expected = pd.notna(ps) pd.testing.assert_series_equal(expected, actual) # pandas dataframe pdf = pd.DataFrame( - {"foo": (1, 2, 3, np.Inf, pd.NA), "bar": (4, 5, 6, np.NaN, pd.NaT)} + {"foo": (1, 2, 3, np.inf, pd.NA), "bar": (4, 5, 6, np.nan, pd.NaT)} ) actual = notna(pdf).execute().fetch() expected = pd.notna(pdf) diff --git a/python/xorbits/_mars/dataframe/reduction/core.py b/python/xorbits/_mars/dataframe/reduction/core.py index 9e560b78f..78e766afb 100644 --- a/python/xorbits/_mars/dataframe/reduction/core.py +++ b/python/xorbits/_mars/dataframe/reduction/core.py @@ -365,7 +365,7 @@ def _call_dataframe(self, df): # handle pandas Dtypes in the future more carefully. reduced_dtype = np.dtype("O") else: - reduced_dtype = np.find_common_type(dtypes, []) + reduced_dtype = np.result_type(*dtypes) if level is not None: return self._call_groupby_level(df[reduced_cols], level) diff --git a/python/xorbits/_mars/dataframe/statistics/corr.py b/python/xorbits/_mars/dataframe/statistics/corr.py index 4f6e31320..1f4b37409 100644 --- a/python/xorbits/_mars/dataframe/statistics/corr.py +++ b/python/xorbits/_mars/dataframe/statistics/corr.py @@ -46,7 +46,7 @@ def _set_inputs(self, inputs): def __call__(self, df_or_series): if isinstance(df_or_series, SERIES_TYPE): inputs = filter_inputs([df_or_series, self.other]) - return self.new_scalar(inputs, dtype=np.dtype(np.float_)) + return self.new_scalar(inputs, dtype=np.dtype(np.float64)) else: def _filter_numeric(obj): @@ -63,7 +63,7 @@ def _filter_numeric(obj): inputs = filter_inputs([df_or_series, self.other]) if self.axis is None: dtypes = pd.Series( - [np.dtype(np.float_)] * len(df_or_series.dtypes), + [np.dtype(np.float64)] * len(df_or_series.dtypes), index=df_or_series.dtypes.index, ) return self.new_dataframe( @@ -88,7 +88,7 @@ def _filter_numeric(obj): return self.new_series( inputs, shape=shape, - dtype=np.dtype(np.float_), + dtype=np.dtype(np.float64), index_value=new_index_value, ) @@ -115,8 +115,8 @@ def _tile_pearson_cross(left, right, min_periods): right.fillna(0).to_tensor(), ) - nna_left = left.notna().to_tensor().astype(np.float_) - nna_right = right.notna().to_tensor().astype(np.float_) + nna_left = left.notna().to_tensor().astype(np.float64) + nna_right = right.notna().to_tensor().astype(np.float64) sum_left = left_tensor.T.dot(nna_right) sum_right = right_tensor.T.dot(nna_left) @@ -143,8 +143,8 @@ def _tile_pearson_align(cls, left, right, axis): if has_unknown_shape(left, right): yield left.chunks + right.chunks + [left, right] - nna_left = left.notna().astype(np.float_) - nna_right = right.notna().astype(np.float_) + nna_left = left.notna().astype(np.float64) + nna_right = right.notna().astype(np.float64) left, right = left.fillna(0), right.fillna(0) diff --git a/python/xorbits/_mars/learn/contrib/lightgbm/_predict.py b/python/xorbits/_mars/learn/contrib/lightgbm/_predict.py index 076ad0df1..54351785a 100644 --- a/python/xorbits/_mars/learn/contrib/lightgbm/_predict.py +++ b/python/xorbits/_mars/learn/contrib/lightgbm/_predict.py @@ -75,7 +75,7 @@ def __call__(self): shape = (self.data.shape[0],) if self._proba: - dtype = np.dtype(np.float_) + dtype = np.dtype(np.float64) elif hasattr(self.model, "classes_"): dtype = np.array(self.model.classes_).dtype else: diff --git a/python/xorbits/_mars/learn/decomposition/tests/test_truncated_svd.py b/python/xorbits/_mars/learn/decomposition/tests/test_truncated_svd.py index 58905c6d0..4700789eb 100644 --- a/python/xorbits/_mars/learn/decomposition/tests/test_truncated_svd.py +++ b/python/xorbits/_mars/learn/decomposition/tests/test_truncated_svd.py @@ -27,12 +27,10 @@ shape = 60, 55 n_samples, n_features = shape rng = check_random_state(42) -X = rng.randint(-100, 20, np.product(shape)).reshape(shape) +X = rng.randint(-100, 20, np.prod(shape)).reshape(shape) X = sp.csr_matrix(np.maximum(X, 0), dtype=np.float64) X.data[:] = 1 + np.log(X.data) -Xdense = X.A -n_samples = n_samples -n_features = n_features +Xdense = X.toarray() def test_attributes(setup): diff --git a/python/xorbits/_mars/learn/utils/validation.py b/python/xorbits/_mars/learn/utils/validation.py index 8d8db76f4..dc087d0cf 100644 --- a/python/xorbits/_mars/learn/utils/validation.py +++ b/python/xorbits/_mars/learn/utils/validation.py @@ -17,7 +17,13 @@ import warnings import numpy as np -from numpy.core.numeric import ComplexWarning + +from ....utils import is_numpy_2 + +if is_numpy_2(): + from numpy.exceptions import ComplexWarning +else: + from numpy.core.numeric import ComplexWarning try: from sklearn.exceptions import DataConversionWarning diff --git a/python/xorbits/_mars/tensor/__init__.py b/python/xorbits/_mars/tensor/__init__.py index ee9cd566b..84eeaff5b 100644 --- a/python/xorbits/_mars/tensor/__init__.py +++ b/python/xorbits/_mars/tensor/__init__.py @@ -303,9 +303,9 @@ # noinspection PyUnresolvedReferences from ..core import ExecutableTuple -from numpy import __version__ as np_ver +from .utils import is_numpy_2 -if np_ver >= "2.0.0": +if is_numpy_2(): from numpy.exceptions import AxisError else: from numpy import AxisError diff --git a/python/xorbits/_mars/tensor/arithmetic/tests/test_arithmetic.py b/python/xorbits/_mars/tensor/arithmetic/tests/test_arithmetic.py index 3c36f6618..e1579e3f3 100644 --- a/python/xorbits/_mars/tensor/arithmetic/tests/test_arithmetic.py +++ b/python/xorbits/_mars/tensor/arithmetic/tests/test_arithmetic.py @@ -357,7 +357,7 @@ def test_unify_chunk_add(): def test_frexp(): t1 = ones((3, 4, 5), chunk_size=2) - t2 = empty((3, 4, 5), dtype=np.float_, chunk_size=2) + t2 = empty((3, 4, 5), dtype=np.float64, chunk_size=2) op_type = type(t1.op) o1, o2 = frexp(t1) diff --git a/python/xorbits/_mars/tensor/array_utils.py b/python/xorbits/_mars/tensor/array_utils.py index 929fd9408..6f6f72cdb 100644 --- a/python/xorbits/_mars/tensor/array_utils.py +++ b/python/xorbits/_mars/tensor/array_utils.py @@ -21,6 +21,7 @@ from ..lib import sparse from ..lib.sparse.core import get_dense_module, issparse from ..utils import is_same_module, lazy_import +from .utils import is_numpy_2 cp = lazy_import("cupy", rename="cp") @@ -43,6 +44,18 @@ def is_cupy(x): return False +def get_device_id(input_data): + # numpy2.x added `device` attribute to `np.ndarray` + # `np.ndarray.device` return `cpu` and does not have `id` attribute + # while `cupy.ndarray.device.id` return the GPU device id + if hasattr(input_data, "device") and not ( + is_numpy_2() and isinstance(input_data, np.ndarray) + ): + return input_data.device.id + else: + return -1 + + def get_array_module(x, nosparse=False): if issparse(x): if nosparse: @@ -117,8 +130,7 @@ def as_same_device(inputs, device=None, ret_extra=False, copy_if_not_writeable=F if device is None: try: device = _most_nbytes_device( - (i.device.id if hasattr(i, "device") else -1, i.nbytes) - for i in input_tensors + (get_device_id(i), i.nbytes) for i in input_tensors ) except ValueError: device = -1 diff --git a/python/xorbits/_mars/tensor/base/broadcast_to.py b/python/xorbits/_mars/tensor/base/broadcast_to.py index 338763663..d40465286 100644 --- a/python/xorbits/_mars/tensor/base/broadcast_to.py +++ b/python/xorbits/_mars/tensor/base/broadcast_to.py @@ -17,7 +17,7 @@ from ... import opcodes as OperandDef from ...serialization.serializables import KeyField, TupleField -from ..array_utils import device, get_array_module +from ..array_utils import device, get_array_module, get_device_id from ..datasource import tensor as astensor from ..operands import TensorHasInput, TensorOperandMixin @@ -80,7 +80,7 @@ def tile(cls, op): def execute(cls, ctx, op): xp = get_array_module(ctx[op.input.key]) input_data = ctx[op.input.key] - device_id = input_data.device.id if hasattr(input_data, "device") else -1 + device_id = get_device_id(input_data) with device(device_id): shape = op.shape diff --git a/python/xorbits/_mars/tensor/base/expand_dims.py b/python/xorbits/_mars/tensor/base/expand_dims.py index 5d932eee0..bb98ea1e2 100644 --- a/python/xorbits/_mars/tensor/base/expand_dims.py +++ b/python/xorbits/_mars/tensor/base/expand_dims.py @@ -16,6 +16,12 @@ import numpy as np from ..datasource import tensor as astensor +from ..utils import is_numpy_2 + +if is_numpy_2(): + from numpy.exceptions import AxisError +else: + from numpy import AxisError def expand_dims(a, axis): @@ -77,9 +83,7 @@ def expand_dims(a, axis): a = astensor(a) if axis > a.ndim or axis < -a.ndim - 1: - raise np.AxisError( - f"Axis must be between -{a.ndim + 1} and {a.ndim}, got {axis}" - ) + raise AxisError(f"Axis must be between -{a.ndim + 1} and {a.ndim}, got {axis}") axis = axis if axis >= 0 else axis + a.ndim + 1 indexes = (slice(None),) * axis + (np.newaxis,) + (slice(None),) * (a.ndim - axis) diff --git a/python/xorbits/_mars/tensor/base/rollaxis.py b/python/xorbits/_mars/tensor/base/rollaxis.py index 4881cd362..cafe2f94a 100644 --- a/python/xorbits/_mars/tensor/base/rollaxis.py +++ b/python/xorbits/_mars/tensor/base/rollaxis.py @@ -13,9 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np +from ..utils import is_numpy_2, validate_axis -from ..utils import validate_axis +if is_numpy_2(): + from numpy.exceptions import AxisError +else: + from numpy import AxisError def rollaxis(tensor, axis, start=0): @@ -66,7 +69,7 @@ def rollaxis(tensor, axis, start=0): start += n msg = "'%s' arg requires %d <= %s < %d, but %d was passed in" if not (0 <= start < n + 1): - raise np.AxisError(msg % ("start", -n, "start", n + 1, start)) + raise AxisError(msg % ("start", -n, "start", n + 1, start)) if axis < start: # it's been removed start -= 1 diff --git a/python/xorbits/_mars/tensor/base/tests/test_base.py b/python/xorbits/_mars/tensor/base/tests/test_base.py index fe33dab60..d414cd462 100644 --- a/python/xorbits/_mars/tensor/base/tests/test_base.py +++ b/python/xorbits/_mars/tensor/base/tests/test_base.py @@ -18,6 +18,13 @@ from ....core import tile from ....core.operand import OperandStage +from ...utils import is_numpy_2 + +if is_numpy_2(): + from numpy.exceptions import AxisError +else: + from numpy import AxisError + from ...datasource import arange, ones, tensor from .. import ( TensorCopyTo, @@ -577,7 +584,7 @@ def test_unique(): assert np.isnan(indices.chunks[0].shape[0]) assert indices.chunks[0].dtype == np.dtype(np.intp) - with pytest.raises(np.AxisError): + with pytest.raises(AxisError): unique(0.1, axis=1) raw = np.random.randint(10, size=(10), dtype=np.int64) @@ -684,7 +691,7 @@ def test_sort(): with pytest.raises(ValueError): sort(a, order=["unknown_field"]) - with pytest.raises(np.AxisError): + with pytest.raises(AxisError): sort(np.random.rand(100), axis=1) with pytest.raises(ValueError): @@ -743,7 +750,7 @@ def test_partition(): with pytest.raises(ValueError): partition(a, 4, order=["unknown_field"]) - with pytest.raises(np.AxisError): + with pytest.raises(AxisError): partition(np.random.rand(100), 4, axis=1) with pytest.raises(ValueError): diff --git a/python/xorbits/_mars/tensor/base/tests/test_base_execution.py b/python/xorbits/_mars/tensor/base/tests/test_base_execution.py index 49f1dde70..400fa2555 100644 --- a/python/xorbits/_mars/tensor/base/tests/test_base_execution.py +++ b/python/xorbits/_mars/tensor/base/tests/test_base_execution.py @@ -22,6 +22,13 @@ from .... import execute, fetch from .... import tensor as mt from ....tests.core import require_cupy +from ...utils import is_numpy_2 + +if is_numpy_2(): + from numpy.exceptions import AxisError +else: + from numpy import AxisError + from ...datasource import arange, ones, tensor, zeros from .. import ( argpartition, @@ -424,10 +431,10 @@ def test_expand_dims_execution(setup): res = y.execute().fetch() assert np.array_equal(res, np.expand_dims(raw_data, -4)) is True - with pytest.raises(np.AxisError): + with pytest.raises(AxisError): expand_dims(x, -5) - with pytest.raises(np.AxisError): + with pytest.raises(AxisError): expand_dims(x, 4) diff --git a/python/xorbits/_mars/tensor/datasource/tests/test_datasource_execution.py b/python/xorbits/_mars/tensor/datasource/tests/test_datasource_execution.py index 24059119b..e55ab9e0e 100644 --- a/python/xorbits/_mars/tensor/datasource/tests/test_datasource_execution.py +++ b/python/xorbits/_mars/tensor/datasource/tests/test_datasource_execution.py @@ -968,13 +968,13 @@ def test_index_trick_execution(setup): t = mgrid[0:5, 0:5] res = t.execute().fetch() - expected = np.lib.index_tricks.nd_grid()[0:5, 0:5] + expected = np.mgrid[0:5, 0:5] np.testing.assert_equal(res, expected) t = mgrid[-1:1:5j] res = t.execute().fetch() - expected = np.lib.index_tricks.nd_grid()[-1:1:5j] + expected = np.mgrid[-1:1:5j] np.testing.assert_equal(res, expected) ogrid = nd_grid(sparse=True) @@ -982,7 +982,7 @@ def test_index_trick_execution(setup): t = ogrid[0:5, 0:5] res = [o.execute().fetch() for o in t] - expected = np.lib.index_tricks.nd_grid(sparse=True)[0:5, 0:5] + expected = np.ogrid[0:5, 0:5] [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] diff --git a/python/xorbits/_mars/tensor/fft/fft.py b/python/xorbits/_mars/tensor/fft/fft.py index 496f98f27..f6487bb05 100644 --- a/python/xorbits/_mars/tensor/fft/fft.py +++ b/python/xorbits/_mars/tensor/fft/fft.py @@ -111,5 +111,5 @@ def fft(a, n=None, axis=-1, norm=None): """ a = astensor(a) validate_fft(a, axis, norm) - op = TensorFFT(n=n, axis=axis, norm=norm, dtype=np.dtype(np.complex_)) + op = TensorFFT(n=n, axis=axis, norm=norm, dtype=np.dtype(np.complex128)) return op(a) diff --git a/python/xorbits/_mars/tensor/fft/fft2.py b/python/xorbits/_mars/tensor/fft/fft2.py index c31b66336..a99416421 100644 --- a/python/xorbits/_mars/tensor/fft/fft2.py +++ b/python/xorbits/_mars/tensor/fft/fft2.py @@ -117,5 +117,5 @@ def fft2(a, s=None, axes=(-2, -1), norm=None): raise ValueError("axes length should be 2") a = astensor(a) axes = validate_fftn(a, s=s, axes=axes, norm=norm) - op = TensorFFT2(shape=s, axes=axes, norm=norm, dtype=np.dtype(np.complex_)) + op = TensorFFT2(shape=s, axes=axes, norm=norm, dtype=np.dtype(np.complex128)) return op(a) diff --git a/python/xorbits/_mars/tensor/fft/fftn.py b/python/xorbits/_mars/tensor/fft/fftn.py index da77e1efa..e021522bb 100644 --- a/python/xorbits/_mars/tensor/fft/fftn.py +++ b/python/xorbits/_mars/tensor/fft/fftn.py @@ -122,5 +122,5 @@ def fftn(a, s=None, axes=None, norm=None): """ a = astensor(a) axes = validate_fftn(a, s=s, axes=axes, norm=norm) - op = TensorFFTN(shape=s, axes=axes, norm=norm, dtype=np.dtype(np.complex_)) + op = TensorFFTN(shape=s, axes=axes, norm=norm, dtype=np.dtype(np.complex128)) return op(a) diff --git a/python/xorbits/_mars/tensor/fft/hfft.py b/python/xorbits/_mars/tensor/fft/hfft.py index 29eb119eb..310fd7088 100644 --- a/python/xorbits/_mars/tensor/fft/hfft.py +++ b/python/xorbits/_mars/tensor/fft/hfft.py @@ -112,5 +112,5 @@ def hfft(a, n=None, axis=-1, norm=None): """ a = astensor(a) validate_fft(a, axis=axis, norm=norm) - op = TensorHFFT(n=n, axis=axis, norm=norm, dtype=np.dtype(np.float_)) + op = TensorHFFT(n=n, axis=axis, norm=norm, dtype=np.dtype(np.float64)) return op(a) diff --git a/python/xorbits/_mars/tensor/fft/ifft.py b/python/xorbits/_mars/tensor/fft/ifft.py index 3e6ea6a32..7e48e0c68 100644 --- a/python/xorbits/_mars/tensor/fft/ifft.py +++ b/python/xorbits/_mars/tensor/fft/ifft.py @@ -113,5 +113,5 @@ def ifft(a, n=None, axis=-1, norm=None): """ a = astensor(a) validate_fft(a, axis, norm) - op = TensorIFFT(n=n, axis=axis, norm=norm, dtype=np.dtype(np.complex_)) + op = TensorIFFT(n=n, axis=axis, norm=norm, dtype=np.dtype(np.complex128)) return op(a) diff --git a/python/xorbits/_mars/tensor/fft/ifft2.py b/python/xorbits/_mars/tensor/fft/ifft2.py index 2de9547f3..fc2f6d5a6 100644 --- a/python/xorbits/_mars/tensor/fft/ifft2.py +++ b/python/xorbits/_mars/tensor/fft/ifft2.py @@ -114,5 +114,5 @@ def ifft2(a, s=None, axes=(-2, -1), norm=None): raise ValueError("axes length should be 2") a = astensor(a) axes = validate_fftn(a, s=s, axes=axes, norm=norm) - op = TensorIFFT2(shape=s, axes=axes, norm=norm, dtype=np.dtype(np.complex_)) + op = TensorIFFT2(shape=s, axes=axes, norm=norm, dtype=np.dtype(np.complex128)) return op(a) diff --git a/python/xorbits/_mars/tensor/fft/ifftn.py b/python/xorbits/_mars/tensor/fft/ifftn.py index 4a46f6fdb..b4ca1df29 100644 --- a/python/xorbits/_mars/tensor/fft/ifftn.py +++ b/python/xorbits/_mars/tensor/fft/ifftn.py @@ -122,5 +122,5 @@ def ifftn(a, s=None, axes=None, norm=None): """ a = astensor(a) axes = validate_fftn(a, s=s, axes=axes, norm=norm) - op = TensorIFFTN(shape=s, axes=axes, norm=norm, dtype=np.dtype(np.complex_)) + op = TensorIFFTN(shape=s, axes=axes, norm=norm, dtype=np.dtype(np.complex128)) return op(a) diff --git a/python/xorbits/_mars/tensor/fft/ihfft.py b/python/xorbits/_mars/tensor/fft/ihfft.py index b7427e15f..ffce4a27a 100644 --- a/python/xorbits/_mars/tensor/fft/ihfft.py +++ b/python/xorbits/_mars/tensor/fft/ihfft.py @@ -92,5 +92,5 @@ def ihfft(a, n=None, axis=-1, norm=None): """ a = astensor(a) validate_fft(a, axis=axis, norm=norm) - op = TensorIHFFT(n=n, axis=axis, norm=norm, dtype=np.dtype(np.complex_)) + op = TensorIHFFT(n=n, axis=axis, norm=norm, dtype=np.dtype(np.complex128)) return op(a) diff --git a/python/xorbits/_mars/tensor/fft/irfft.py b/python/xorbits/_mars/tensor/fft/irfft.py index f21c4b6dd..4871148b8 100644 --- a/python/xorbits/_mars/tensor/fft/irfft.py +++ b/python/xorbits/_mars/tensor/fft/irfft.py @@ -118,5 +118,5 @@ def irfft(a, n=None, axis=-1, norm=None): """ a = astensor(a) validate_fft(a, axis=axis, norm=norm) - op = TensorIRFFT(n=n, axis=axis, norm=norm, dtype=np.dtype(np.float_)) + op = TensorIRFFT(n=n, axis=axis, norm=norm, dtype=np.dtype(np.float64)) return op(a) diff --git a/python/xorbits/_mars/tensor/fft/irfft2.py b/python/xorbits/_mars/tensor/fft/irfft2.py index 475116dd0..af85ced9f 100644 --- a/python/xorbits/_mars/tensor/fft/irfft2.py +++ b/python/xorbits/_mars/tensor/fft/irfft2.py @@ -62,5 +62,5 @@ def irfft2(a, s=None, axes=(-2, -1), norm=None): raise ValueError("axes length should be 2") a = astensor(a) axes = validate_fftn(a, s=s, axes=axes, norm=norm) - op = TensorIRFFT2(shape=s, axes=axes, norm=norm, dtype=np.dtype(np.float_)) + op = TensorIRFFT2(shape=s, axes=axes, norm=norm, dtype=np.dtype(np.float64)) return op(a) diff --git a/python/xorbits/_mars/tensor/fft/irfftn.py b/python/xorbits/_mars/tensor/fft/irfftn.py index 05ebab1d7..dd2f4455f 100644 --- a/python/xorbits/_mars/tensor/fft/irfftn.py +++ b/python/xorbits/_mars/tensor/fft/irfftn.py @@ -114,5 +114,5 @@ def irfftn(a, s=None, axes=None, norm=None): """ a = astensor(a) axes = validate_fftn(a, s=s, axes=axes, norm=norm) - op = TensorIRFFTN(shape=s, axes=axes, norm=norm, dtype=np.dtype(np.float_)) + op = TensorIRFFTN(shape=s, axes=axes, norm=norm, dtype=np.dtype(np.float64)) return op(a) diff --git a/python/xorbits/_mars/tensor/fft/rfft.py b/python/xorbits/_mars/tensor/fft/rfft.py index 8cb52e631..45e21ef76 100644 --- a/python/xorbits/_mars/tensor/fft/rfft.py +++ b/python/xorbits/_mars/tensor/fft/rfft.py @@ -115,5 +115,5 @@ def rfft(a, n=None, axis=-1, norm=None): """ a = astensor(a) validate_fft(a, axis=axis, norm=norm) - op = TensorRFFT(n=n, axis=axis, norm=norm, dtype=np.dtype(np.complex_)) + op = TensorRFFT(n=n, axis=axis, norm=norm, dtype=np.dtype(np.complex128)) return op(a) diff --git a/python/xorbits/_mars/tensor/fft/rfft2.py b/python/xorbits/_mars/tensor/fft/rfft2.py index 6394070e6..b98b88f87 100644 --- a/python/xorbits/_mars/tensor/fft/rfft2.py +++ b/python/xorbits/_mars/tensor/fft/rfft2.py @@ -62,5 +62,5 @@ def rfft2(a, s=None, axes=(-2, -1), norm=None): raise ValueError("axes length should be 2") a = astensor(a) axes = validate_fftn(a, s=s, axes=axes, norm=norm) - op = TensorRFFT2(shape=s, axes=axes, norm=norm, dtype=np.dtype(np.complex_)) + op = TensorRFFT2(shape=s, axes=axes, norm=norm, dtype=np.dtype(np.complex128)) return op(a) diff --git a/python/xorbits/_mars/tensor/fft/rfftn.py b/python/xorbits/_mars/tensor/fft/rfftn.py index 316e4a9d0..d4bafce9e 100644 --- a/python/xorbits/_mars/tensor/fft/rfftn.py +++ b/python/xorbits/_mars/tensor/fft/rfftn.py @@ -112,5 +112,5 @@ def rfftn(a, s=None, axes=None, norm=None): """ a = astensor(a) axes = validate_fftn(a, s=s, axes=axes, norm=norm) - op = TensorRFFTN(shape=s, axes=axes, norm=norm, dtype=np.dtype(np.complex_)) + op = TensorRFFTN(shape=s, axes=axes, norm=norm, dtype=np.dtype(np.complex128)) return op(a) diff --git a/python/xorbits/_mars/tensor/indexing/compress.py b/python/xorbits/_mars/tensor/indexing/compress.py index a5765e6d8..37541b030 100644 --- a/python/xorbits/_mars/tensor/indexing/compress.py +++ b/python/xorbits/_mars/tensor/indexing/compress.py @@ -17,7 +17,12 @@ from ..core import Tensor from ..datasource import tensor as astensor -from ..utils import validate_axis +from ..utils import is_numpy_2, validate_axis + +if is_numpy_2(): + from numpy.exceptions import AxisError +else: + from numpy import AxisError def compress(condition, a, axis=None, out=None): @@ -96,7 +101,7 @@ def compress(condition, a, axis=None, out=None): try: axis = validate_axis(a.ndim, axis) except ValueError: - raise np.AxisError( + raise AxisError( f"axis {axis} is out of bounds for tensor of dimension {a.ndim}" ) @@ -118,6 +123,6 @@ def compress(condition, a, axis=None, out=None): out.data = t.astype(out.dtype, order=out.order.value).data return out except IndexError: - raise np.AxisError( + raise AxisError( f"axis {len(condition)} is out of bounds for tensor of dimension 1" ) diff --git a/python/xorbits/_mars/tensor/indexing/tests/test_indexing_execution.py b/python/xorbits/_mars/tensor/indexing/tests/test_indexing_execution.py index c5d4a0e98..f23f99c7a 100644 --- a/python/xorbits/_mars/tensor/indexing/tests/test_indexing_execution.py +++ b/python/xorbits/_mars/tensor/indexing/tests/test_indexing_execution.py @@ -20,6 +20,7 @@ from ....config import options from ... import hstack, mod, stack from ...datasource import arange, tensor, zeros +from ...utils import is_numpy_2 from .. import ( choose, compress, @@ -31,6 +32,11 @@ unravel_index, ) +if is_numpy_2(): + from numpy.exceptions import AxisError +else: + from numpy import AxisError + def test_bool_indexing_execution(setup): raw = np.random.random((11, 8, 12, 14)) @@ -558,7 +564,7 @@ def test_compress_execution(setup): expected = np.compress([False, True], data, axis=1) np.testing.assert_array_equal(res, expected) - with pytest.raises(np.AxisError): + with pytest.raises(AxisError): compress([0, 1, 1], a, axis=1) # test order diff --git a/python/xorbits/_mars/tensor/lib/index_tricks.py b/python/xorbits/_mars/tensor/lib/index_tricks.py index 014d69220..0a7fc695a 100644 --- a/python/xorbits/_mars/tensor/lib/index_tricks.py +++ b/python/xorbits/_mars/tensor/lib/index_tricks.py @@ -16,18 +16,18 @@ import math import numpy as np -from numpy import __version__ as np_ver from numpy.core.numeric import ScalarType -if np_ver >= "2.0.0": - from numpy.lib._index_tricks_impl import ndindex -else: - from numpy.lib.index_tricks import ndindex - from .. import datasource as _nx from ..base import ndim from ..core import Tensor from ..merge import concatenate +from ..utils import is_numpy_2 + +if is_numpy_2(): + from numpy.lib._index_tricks_impl import ndindex +else: + from numpy.lib.index_tricks import ndindex class nd_grid(object): @@ -349,14 +349,8 @@ def __getitem__(self, key): # Ensure that scalars won't up-cast unless warranted # find_common_type is deprecated - if np_ver >= "2.0.0": - from numpy import result_type - - final_dtype = result_type(arraytypes, scalartypes) - else: - from numpy.core.numerictypes import find_common_type + final_dtype = np.result_type(*arraytypes, *scalartypes) - final_dtype = find_common_type(arraytypes, scalartypes) if final_dtype is not None: for k in scalars: objs[k] = objs[k].astype(final_dtype) diff --git a/python/xorbits/_mars/tensor/linalg/norm.py b/python/xorbits/_mars/tensor/linalg/norm.py index 12979be4b..068760bfe 100644 --- a/python/xorbits/_mars/tensor/linalg/norm.py +++ b/python/xorbits/_mars/tensor/linalg/norm.py @@ -337,7 +337,7 @@ def norm(x, ord=None, axis=None, keepdims=False): ord=ord, axis=axis, keepdims=keepdims, - dtype=np.result_type(x.dtype, np.float_), + dtype=np.result_type(x.dtype, np.float64), sparse=x.issparse(), ) return op(x) diff --git a/python/xorbits/_mars/tensor/merge/stack.py b/python/xorbits/_mars/tensor/merge/stack.py index 35ddc943f..4b7cc7c34 100644 --- a/python/xorbits/_mars/tensor/merge/stack.py +++ b/python/xorbits/_mars/tensor/merge/stack.py @@ -24,7 +24,12 @@ from ..core import Tensor, TensorOrder from ..datasource import tensor as astensor from ..operands import TensorOperand, TensorOperandMixin -from ..utils import check_out_param, unify_chunks +from ..utils import check_out_param, is_numpy_2, unify_chunks + +if is_numpy_2(): + from numpy.exceptions import AxisError +else: + from numpy import AxisError class TensorStack(TensorOperand, TensorOperandMixin): @@ -207,7 +212,7 @@ def stack(tensors, axis=0, out=None): if axis < 0: axis = ndim + axis + 1 if axis > ndim or axis < 0: - raise np.AxisError( + raise AxisError( f"axis {raw_axis} is out of bounds for tensor of dimension {ndim}" ) diff --git a/python/xorbits/_mars/tensor/merge/tests/test_merge.py b/python/xorbits/_mars/tensor/merge/tests/test_merge.py index 47c16599d..e51b9aed2 100644 --- a/python/xorbits/_mars/tensor/merge/tests/test_merge.py +++ b/python/xorbits/_mars/tensor/merge/tests/test_merge.py @@ -13,13 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np import pytest from ....core import tile from ...datasource import empty, ones +from ...utils import is_numpy_2 from .. import concatenate, stack +if is_numpy_2(): + from numpy.exceptions import AxisError +else: + from numpy import AxisError + def test_concatenate(): a = ones((10, 20, 30), chunk_size=10) @@ -85,7 +90,7 @@ def test_stack(): raw_arrs2 = [ones((3, 4), chunk_size=2), ones((4, 3), chunk_size=2)] stack(raw_arrs2) - with pytest.raises(np.AxisError): + with pytest.raises(AxisError): stack(raw_arrs, axis=3) arr5 = tile(stack(raw_arrs, -1)) @@ -94,7 +99,7 @@ def test_stack(): arr6 = tile(stack(raw_arrs, -3)) assert arr6.nsplits == ((1,) * 10, (2, 1), (2, 2)) - with pytest.raises(np.AxisError): + with pytest.raises(AxisError): stack(raw_arrs, axis=-4) with pytest.raises(TypeError): diff --git a/python/xorbits/_mars/tensor/random/permutation.py b/python/xorbits/_mars/tensor/random/permutation.py index aedf7b71c..eabfbde07 100644 --- a/python/xorbits/_mars/tensor/random/permutation.py +++ b/python/xorbits/_mars/tensor/random/permutation.py @@ -24,9 +24,14 @@ from ..array_utils import as_same_device, device from ..datasource import tensor as astensor from ..operands import TensorOperandMixin, TensorShuffleProxy -from ..utils import gen_random_seeds, validate_axis +from ..utils import gen_random_seeds, is_numpy_2, validate_axis from .core import TensorRandomMapReduceOperand +if is_numpy_2(): + from numpy.exceptions import AxisError +else: + from numpy import AxisError + def _permutation_on_axis(ar, axis, rs, xp): try: @@ -223,7 +228,7 @@ def permutation(random_state, x, axis=0, chunk_size=None): >>> rng.permutation("abc") Traceback (most recent call last): ... - numpy.AxisError: x must be an integer or at least 1-dimensional + numpy.exceptions.AxisError: x must be an integer or at least 1-dimensional """ if isinstance(x, (Integral, np.integer)): from ..datasource import arange @@ -232,7 +237,7 @@ def permutation(random_state, x, axis=0, chunk_size=None): else: x = astensor(x, chunk_size=chunk_size) if x.ndim < 1: - raise np.AxisError("x must be an integer or at least 1-dimensional") + raise AxisError("x must be an integer or at least 1-dimensional") axis = validate_axis(x.ndim, axis) seed = gen_random_seeds(1, random_state.to_numpy())[0] diff --git a/python/xorbits/_mars/tensor/random/tests/test_random.py b/python/xorbits/_mars/tensor/random/tests/test_random.py index afdf24dc1..d5305bdf3 100644 --- a/python/xorbits/_mars/tensor/random/tests/test_random.py +++ b/python/xorbits/_mars/tensor/random/tests/test_random.py @@ -20,6 +20,7 @@ from ....core import tile from ....serialization.serializables import Serializable from ...datasource import tensor as from_ndarray +from ...utils import is_numpy_2 from .. import ( RandomState, TensorPermutation, @@ -34,6 +35,11 @@ ) from ..core import RandomStateField +if is_numpy_2(): + from numpy.exceptions import AxisError +else: + from numpy import AxisError + class ObjWithRandomStateField(Serializable): random_state = RandomStateField("random_state") @@ -205,7 +211,7 @@ def test_permutation(): == x.cix[1, 0].inputs[0].inputs[0].inputs[0].op.seed ) - with pytest.raises(np.AxisError): + with pytest.raises(AxisError): pytest.raises(permutation("abc")) diff --git a/python/xorbits/_mars/tensor/reduction/core.py b/python/xorbits/_mars/tensor/reduction/core.py index d51dcdded..78959bdc9 100644 --- a/python/xorbits/_mars/tensor/reduction/core.py +++ b/python/xorbits/_mars/tensor/reduction/core.py @@ -444,6 +444,8 @@ def execute_agg(cls, ctx, op): inds = np.ogrid[tuple(map(slice, local_args.shape))] if xp != np: inds = [xp.asarray(it) for it in inds] + # `ogrid()` returns tuple instead of list in numpy2.x + inds = list(inds) inds.insert(axis, local_args) arg = arg[tuple(inds)] ctx[op.outputs[0].key] = arg diff --git a/python/xorbits/_mars/tensor/reduction/tests/test_reduction.py b/python/xorbits/_mars/tensor/reduction/tests/test_reduction.py index 0fd4a2108..a1fdbb7c8 100644 --- a/python/xorbits/_mars/tensor/reduction/tests/test_reduction.py +++ b/python/xorbits/_mars/tensor/reduction/tests/test_reduction.py @@ -13,15 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np import pytest from ....core import tile from ....core.operand import OperandStage from ...datasource import ones, tensor from ...merge import TensorConcatenate +from ...utils import is_numpy_2 from .. import TensorArgmax, TensorArgmin, TensorMean, all +if is_numpy_2(): + from numpy.exceptions import AxisError +else: + from numpy import AxisError + def test_base_reduction(): sum = lambda x, *args, **kwargs: tile(x.sum(*args, **kwargs)) @@ -45,13 +50,13 @@ def test_base_reduction(): res = f(ones((10, 8), chunk_size=3), axis=1) assert res.shape == (10,) - with pytest.raises(np.AxisError): + with pytest.raises(AxisError): f(ones((10, 8), chunk_size=3), axis=2) res = f(ones((10, 8), chunk_size=3), axis=-1) assert res.shape == (10,) - with pytest.raises(np.AxisError): + with pytest.raises(AxisError): f(ones((10, 8), chunk_size=3), axis=-3) res = f(ones((10, 8), chunk_size=3), keepdims=True) @@ -96,13 +101,13 @@ def test_mean_reduction(): res = mean(ones((10, 8), chunk_size=3), axis=1) assert res.shape == (10,) - with pytest.raises(np.AxisError): + with pytest.raises(AxisError): mean(ones((10, 8), chunk_size=3), axis=2) res = mean(ones((10, 8), chunk_size=3), axis=-1) assert res.shape == (10,) - with pytest.raises(np.AxisError): + with pytest.raises(AxisError): mean(ones((10, 8), chunk_size=3), axis=-3) res = mean(ones((10, 8), chunk_size=3), keepdims=True) @@ -161,9 +166,10 @@ def test_arg_reduction(): pytest.raises( TypeError, lambda: argmin(ones((10, 8, 10), chunk_size=3), axis=(0, 1)) ) - pytest.raises(np.AxisError, lambda: argmin(ones((10, 8, 10), chunk_size=3), axis=3)) + pytest.raises(AxisError, lambda: argmin(ones((10, 8, 10), chunk_size=3), axis=3)) pytest.raises( - np.AxisError, lambda: argmin(ones((10, 8, 10), chunk_size=3), axis=-4) + AxisError, + lambda: argmin(ones((10, 8, 10), chunk_size=3), axis=-4), ) @@ -188,9 +194,9 @@ def test_cum_reduction(): assert res1.shape == (10, 8, 8) assert res2.shape == (10, 8, 8) - with pytest.raises(np.AxisError): + with pytest.raises(AxisError): cumsum(ones((10, 8), chunk_size=3), axis=2) - with pytest.raises(np.AxisError): + with pytest.raises(AxisError): cumsum(ones((10, 8), chunk_size=3), axis=-3) diff --git a/python/xorbits/_mars/tensor/reduction/tests/test_reduction_execution.py b/python/xorbits/_mars/tensor/reduction/tests/test_reduction_execution.py index 51562d405..e7af0da66 100644 --- a/python/xorbits/_mars/tensor/reduction/tests/test_reduction_execution.py +++ b/python/xorbits/_mars/tensor/reduction/tests/test_reduction_execution.py @@ -19,6 +19,7 @@ from ....utils import ignore_warning from ...datasource import ones, tensor +from ...utils import is_numpy_2 from .. import ( allclose, array_equal, @@ -153,14 +154,16 @@ def test_all_any_execution(setup): arr = tensor(raw, chunk_size=3) - assert raw.A.all() == arr.all().execute().fetch() - assert raw.A.any() == arr.any().execute().fetch() + assert raw.toarray().all() == arr.all().execute().fetch() + assert raw.toarray().any() == arr.any().execute().fetch() # test string dtype - a = tensor(list("abcdefghi"), dtype=object) - assert a.all().execute().fetch() == "i" - a = tensor(list("abcdefghi"), dtype=object, chunk_size=2) - assert a.any().execute().fetch() == "a" + # `all()` and `any()` now returns booleans for object arrays in numpy2.x + if not is_numpy_2(): + a = tensor(list("abcdefghi"), dtype=object) + assert a.all().execute().fetch() == "i" + a = tensor(list("abcdefghi"), dtype=object, chunk_size=2) + assert a.any().execute().fetch() == "a" def test_mean_execution(setup): diff --git a/python/xorbits/_mars/tensor/spatial/distance/squareform.py b/python/xorbits/_mars/tensor/spatial/distance/squareform.py index d78717ed3..9c1d15e82 100644 --- a/python/xorbits/_mars/tensor/spatial/distance/squareform.py +++ b/python/xorbits/_mars/tensor/spatial/distance/squareform.py @@ -152,7 +152,7 @@ def tile(cls, op): tensor = op.outputs[0] chunk_size = tensor.extra_params.raw_chunk_size or options.chunk_size chunk_size = decide_chunk_sizes(tensor.shape, chunk_size, tensor.dtype.itemsize) - n_chunk = np.product([len(cs) for cs in chunk_size]) + n_chunk = np.prod([len(cs) for cs in chunk_size]) if len(op.input.chunks) == 1 and n_chunk == 1: return cls._tile_one_chunk(op) diff --git a/python/xorbits/_mars/tensor/utils.py b/python/xorbits/_mars/tensor/utils.py index 86bcd483d..afc14fcaa 100644 --- a/python/xorbits/_mars/tensor/utils.py +++ b/python/xorbits/_mars/tensor/utils.py @@ -30,10 +30,16 @@ except (ImportError, OSError): # pragma: no cover tildb = None +from ...utils import is_numpy_2 from ..core import ExecutableTuple, recursive_tile from ..lib.mmh3 import hash_from_buffer from ..utils import lazy_import +if is_numpy_2(): + from numpy.exceptions import AxisError +else: + from numpy import AxisError + cp = lazy_import("cupy", rename="cp") @@ -119,7 +125,7 @@ def gen_random_seeds(n, random_state): def validate_axis(ndim, axis, argname=None): if axis >= ndim or axis < -ndim: - raise np.AxisError(axis, ndim=ndim, msg_prefix=argname) + raise AxisError(axis, ndim=ndim, msg_prefix=argname) return axis if axis >= 0 else ndim + axis diff --git a/python/xorbits/_mars/tests/test_session.py b/python/xorbits/_mars/tests/test_session.py index 28fbcd316..6ebc60511 100644 --- a/python/xorbits/_mars/tests/test_session.py +++ b/python/xorbits/_mars/tests/test_session.py @@ -211,8 +211,8 @@ def test_array_protocol(setup): arr4 = mt.ones((10, 20)).sum() - result = np.asarray(arr4, dtype=np.float_) - np.testing.assert_array_equal(result, np.asarray(200, dtype=np.float_)) + result = np.asarray(arr4, dtype=np.float64) + np.testing.assert_array_equal(result, np.asarray(200, dtype=np.float64)) def test_without_fuse(setup): diff --git a/python/xorbits/numpy/__init__.py b/python/xorbits/numpy/__init__.py index a4f8d436d..40e7546cf 100644 --- a/python/xorbits/numpy/__init__.py +++ b/python/xorbits/numpy/__init__.py @@ -15,9 +15,9 @@ import inspect from typing import Any, Callable, Dict, Optional -from numpy import __version__ as np_ver +from ..utils import is_numpy_2 -if np_ver >= "2.0.0": +if is_numpy_2(): from numpy.exceptions import AxisError from numpy.lib._index_tricks_impl import ndindex else: diff --git a/python/xorbits/numpy/lib/index_tricks.py b/python/xorbits/numpy/lib/index_tricks.py index c49994169..e0c640b4f 100644 --- a/python/xorbits/numpy/lib/index_tricks.py +++ b/python/xorbits/numpy/lib/index_tricks.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import numpy -from numpy import __version__ as np_ver -if np_ver >= "2.0.0": +from ...utils import is_numpy_2 + +if is_numpy_2(): from numpy.lib import _index_tricks_impl as index_tricks else: from numpy.lib import index_tricks diff --git a/python/xorbits/numpy/mars_adapters/tests/test_numpy_examples.py b/python/xorbits/numpy/mars_adapters/tests/test_numpy_examples.py index 1edd35169..066f48aac 100644 --- a/python/xorbits/numpy/mars_adapters/tests/test_numpy_examples.py +++ b/python/xorbits/numpy/mars_adapters/tests/test_numpy_examples.py @@ -21,6 +21,121 @@ from .... import numpy as xnp +SKIPPED_NAMES = [ + "acos", + "acosh", + "asin", + "asinh", + "atan", + "atan2", + "atanh", + "bitwise_count", + "bitwise_invert", + "bitwise_left_shift", + "bitwise_right_shift", + "concat", + "long", + "pow", + "ulong", + "vecdot", + "DataSource", + "RankWarning", + "busday_count", + "add_docstring", + "add_newdoc_ufunc", + "asanyarray", + "broadcast", + "busday_count", + "busdaycalendar", + "busday_offset", + "byte", + "can_cast", + "cdouble", + "cfloat", + "chararray", + "clongdouble", + "clongfloat", + "compare_chararrays", + "complex256", + "complex_", + "csingle", + "datetime_as_string", + "datetime_data", + "divmod", + "expm1x", + "fastCopyAndTranspose", + "flatiter", + "float128", + "float_", + "format_parser", + "from_dlpack", + "frombuffer", + "fromfile", + "fromiter", + "frompyfunc", + "fromstring", + "gcd", + "geterrobj", + "heaviside", + "half", + "heaviside", + "iinfo", + "isnat", + "lcm", + "lexsort", + "longcomplex", + "longdouble", + "longfloat", + "longlong", + "matrix", + "may_share_memory", + "memmap", + "min_scalar_type", + "ndenumerate", + "nditer", + "nested_iters", + "packbits", + "poly1d", + "promote_types", + "putmask", + "ravel_multi_index", + "recarray", + "record", + "set_numeric_ops", + "seterrobj", + "shares_memory", + "short", + "single", + "singlecomplex", + "string_", + "test", + "ubyte", + "ufunc", + "uintc", + "uintp", + "ulonglong", + "unicode_", + "unpackbits", + "ushort", + "is_busday", + "vectorize", + "BitGenerator", + "Generator", + "MT19937", + "PCG64", + "PCG64DXSM", + "Philox", + "RandomState", + "SFC64", + "SeedSequence", + "default_rng", + "get_bit_generator", + "get_state", + "set_bit_generator", + "set_state", + "LinAlgError", +] + def run_docstring( f, @@ -81,104 +196,7 @@ def run_docstring( @pytest.mark.parametrize("obj,name", parameters) def test_docstrings(setup, doctest_namespace, obj, name): - if name in [ - "DataSource", - "RankWarning", - "busday_count", - "add_docstring", - "add_newdoc_ufunc", - "asanyarray", - "broadcast", - "busday_count", - "busdaycalendar", - "busday_offset", - "byte", - "can_cast", - "cdouble", - "cfloat", - "chararray", - "clongdouble", - "clongfloat", - "compare_chararrays", - "complex256", - "complex_", - "csingle", - "datetime_as_string", - "datetime_data", - "divmod", - "expm1x", - "fastCopyAndTranspose", - "flatiter", - "float128", - "float_", - "format_parser", - "from_dlpack", - "frombuffer", - "fromfile", - "fromiter", - "frompyfunc", - "fromstring", - "gcd", - "geterrobj", - "heaviside", - "half", - "heaviside", - "iinfo", - "isnat", - "lcm", - "lexsort", - "longcomplex", - "longdouble", - "longfloat", - "longlong", - "matrix", - "may_share_memory", - "memmap", - "min_scalar_type", - "ndenumerate", - "nditer", - "nested_iters", - "packbits", - "poly1d", - "promote_types", - "putmask", - "ravel_multi_index", - "recarray", - "record", - "set_numeric_ops", - "seterrobj", - "shares_memory", - "short", - "single", - "singlecomplex", - "string_", - "test", - "ubyte", - "ufunc", - "uintc", - "uintp", - "ulonglong", - "unicode_", - "unpackbits", - "ushort", - "is_busday", - "vectorize", - "BitGenerator", - "Generator", - "MT19937", - "PCG64", - "PCG64DXSM", - "Philox", - "RandomState", - "SFC64", - "SeedSequence", - "default_rng", - "get_bit_generator", - "get_state", - "set_bit_generator", - "set_state", - "LinAlgError", - ]: + if name in SKIPPED_NAMES: pytest.skip("Skipping these name as they do not have __code__") results = run_docstring( getattr(obj, name), diff --git a/python/xorbits/utils.py b/python/xorbits/utils.py index 1aa0f9bea..ed69c3ac5 100644 --- a/python/xorbits/utils.py +++ b/python/xorbits/utils.py @@ -82,3 +82,9 @@ def check_signature_compatible(func1, func2, message): match_param_keys = sig1.parameters.keys() & sig2.parameters.keys() for k in match_param_keys: assert sig1.parameters[k] == sig2.parameters[k], message + + +def is_numpy_2(): + import numpy as np + + return np.__version__ >= "2.0.0"