Merge branch 'main' into dependabot/npm_and_yarn/python/xorbits/web/u…

…i/multi-9f37c16f8f
xorbitsai · Dec 12, 2024 · 2061314 · 2061314
2 parents 554a075 + 68a434a
commit 2061314
Show file tree

Hide file tree

Showing 7 changed files with 269 additions and 119 deletions.
diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml
@@ -75,16 +75,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest", "macos-13", "windows-latest"]
+        os: ["ubuntu-latest", "macos-14", "windows-latest"]
         python-version: ["3.9", "3.10", "3.11", "3.12"]
         module: ["xorbits", "xorbits/numpy", "xorbits/pandas"]
         exclude:
-          - { os: macos-13, python-version: 3.10}
-          - { os: macos-13, python-version: 3.9}
+          - { os: macos-14, python-version: 3.10}
+          - { os: macos-14, python-version: 3.9}
           - { os: windows-latest, python-version: 3.10}
           - { os: windows-latest, python-version: 3.9}
           - { os: windows-latest, module: kubernetes}
-          - { os: macos-13, module: kubernetes}
+          - { os: macos-14, module: kubernetes}
         include:
           - { os: ubuntu-latest, module: _mars/dataframe, python-version: 3.9 }
           - { os: ubuntu-latest, module: learn, python-version: 3.9 }
@@ -135,7 +135,7 @@ jobs:
         # conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py
         pip install ucxx-cu12
     - name: Install libomp (macOS)
-      if: ${{ matrix.os == 'macos-latest' || matrix.os == 'macos-13' }}
+      if: ${{ matrix.os == 'macos-latest' || matrix.os == 'macos-14' }}
       run: brew install libomp
 
     # Important for python == 3.12
@@ -155,7 +155,7 @@ jobs:
           pip install --upgrade --no-cache-dir sphinx readthedocs-sphinx-ext
           pip install --upgrade --upgrade-strategy only-if-needed --no-cache-dir ".[doc]"
         else
-          pip install "xoscar<=0.4.1"
+          pip install -e "git+https://github.com/xorbitsai/xoscar.git@main#subdirectory=python&egg=xoscar"
           pip install -U numpy scipy cython pyftpdlib coverage flaky numexpr
 
           if [[ "$MODULE" == "xorbits/pandas" ]]; then
@@ -311,14 +311,7 @@ jobs:
             --timeout=1500 \
             -W ignore::PendingDeprecationWarning \
             --cov-config=setup.cfg --cov-report=xml --cov=xorbits \
-            -k "not test_execution_with_process_exit_message" \
             xorbits
-          # workaround: this case will hang, run it separately.
-          pytest --timeout=1500 \
-            -W ignore::PendingDeprecationWarning \
-            --cov-config=setup.cfg --cov-report=xml --cov=xorbits \
-            -k "test_execution_with_process_exit_message" \
-            xorbits/core/tests/test_execution.py
         elif [[ "$MODULE" == "xorbits/pandas" ]]; then
           pytest --timeout=1500 \
             -W ignore::PendingDeprecationWarning \
@@ -328,15 +321,7 @@ jobs:
           pytest --timeout=1500 \
             -W ignore::PendingDeprecationWarning \
             --cov-config=setup.cfg --cov-report=xml --cov=xorbits \
-            -k "not test_numpy_fallback" \
             xorbits/numpy
-          # workaround: this case will hang, run it separately.
-          pytest --timeout=1500 \
-            -W ignore::PendingDeprecationWarning \
-            --cov-config=setup.cfg --cov-report=xml \
-            --cov=xorbits \
-            -k "test_numpy_fallback" \
-            xorbits/numpy/numpy_adapters/tests/test_numpy_adapters.py
         elif [[ "$MODULE" == "mars-core" ]]; then
           pytest --forked --log-level=DEBUG --ignore xorbits/_mars/dataframe --ignore xorbits/_mars/tensor \
             --ignore xorbits/_mars/learn --ignore xorbits/_mars/remote \

diff --git a/python/xorbits/_mars/deploy/oscar/session.py b/python/xorbits/_mars/deploy/oscar/session.py
@@ -1684,7 +1684,7 @@ def get_cluster_versions(self) -> List[str]:
 
     def destroy(self):
         coro = self._isolated_session.destroy()
-        asyncio.run_coroutine_threadsafe(coro, self._loop).result()
+        asyncio.run_coroutine_threadsafe(coro, self._loop)
         self.reset_default()
 
     def stop_server(self, isolation=True):

diff --git a/python/xorbits/_mars/deploy/oscar/tests/test_local.py b/python/xorbits/_mars/deploy/oscar/tests/test_local.py
@@ -143,9 +143,7 @@ async def _assert(session_id: str, addr: str, level: StorageLevel):
         assert info.used_size == 0
 
     isolation = new_isolation()
-    asyncio.run_coroutine_threadsafe(
-        _assert(session_id, addr, level), isolation.loop
-    ).result()
+    asyncio.run_coroutine_threadsafe(_assert(session_id, addr, level), isolation.loop)
 
 
 @pytest.mark.parametrize("backend", ["mars"])

diff --git a/python/xorbits/_mars/lib/aio/isolation.py b/python/xorbits/_mars/lib/aio/isolation.py
@@ -36,6 +36,18 @@ def _run(self):
         asyncio.set_event_loop(self.loop)
         self._stopped = asyncio.Event()
         self.loop.run_until_complete(self._stopped.wait())
+        self._cancel_all_tasks(self.loop)
+
+    @staticmethod
+    def _cancel_all_tasks(loop):
+        to_cancel = asyncio.all_tasks(loop)
+        if not to_cancel:
+            return
+
+        for task in to_cancel:
+            task.cancel()
+
+        # Waiting for the tasks to be complete at exit may hang.
 
     def start(self):
         if self._threaded:

diff --git a/python/xorbits/core/tests/test_execution.py b/python/xorbits/core/tests/test_execution.py
@@ -175,7 +175,7 @@ def test_bool_conversion(setup, dummy_df, dummy_int_2d_array, dummy_str_series):
     assert test == 2
     import xorbits.numpy as np
 
-    if np.zeros(0):
+    if np.zeros(0).size > 0:
         test += 1
     assert test == 2
     import xorbits.pandas as pd
@@ -339,20 +339,21 @@ def test_getitem(setup):
     pd.testing.assert_frame_equal(result.to_pandas(), expected)
 
 
-def test_execution_with_process_exit_message(mocker):
-    import numpy as np
-    from xoscar.errors import ServerClosed
+# TODO: process exit cause hang
+# def test_execution_with_process_exit_message(mocker):
+#     import numpy as np
+#     from xoscar.errors import ServerClosed
 
-    import xorbits
-    import xorbits.remote as xr
+#     import xorbits
+#     import xorbits.remote as xr
 
-    mocker.patch(
-        "xorbits._mars.services.subtask.api.SubtaskAPI.run_subtask_in_slot",
-        side_effect=ServerClosed,
-    )
+#     mocker.patch(
+#         "xorbits._mars.services.subtask.api.SubtaskAPI.run_subtask_in_slot",
+#         side_effect=ServerClosed,
+#     )
 
-    with pytest.raises(
-        ServerClosed,
-        match=r".*?\(.*?\) with address .*? Out-of-Memory \(OOM\) problem",
-    ):
-        xorbits.run(xr.spawn(lambda *_: np.random.rand(10**4, 10**4)))
+#     with pytest.raises(
+#         ServerClosed,
+#         match=r".*?\(.*?\) with address .*? Out-of-Memory \(OOM\) problem",
+#     ):
+#         xorbits.run(xr.spawn(lambda *_: np.random.rand(10**4, 10**4)))
diff --git a/python/xorbits/numpy/mars_adapters/tests/test_numpy_examples.py b/python/xorbits/numpy/mars_adapters/tests/test_numpy_examples.py
@@ -88,6 +88,7 @@
     "longfloat",
     "longlong",
     "matrix",
+    "matvec",
     "may_share_memory",
     "memmap",
     "min_scalar_type",
@@ -119,6 +120,7 @@
     "ushort",
     "is_busday",
     "vectorize",
+    "vecmat",
     "BitGenerator",
     "Generator",
     "MT19937",