From 6f52b9bb91e77fa691b4dd216856354124b21873 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 27 Mar 2024 18:27:28 +0000 Subject: [PATCH 1/4] Enable all pytests for arm jobs --- ci/test_wheel_pylibraft.sh | 7 +------ ci/test_wheel_raft_dask.sh | 7 +------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index d990a0e6c2..d6b1266b69 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -10,9 +10,4 @@ RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/pylibraft*.whl)[test] -# Run smoke tests for aarch64 pull requests -if [[ "$(arch)" == "aarch64" && "${RAPIDS_BUILD_TYPE}" == "pull-request" ]]; then - python ./ci/wheel_smoke_test_pylibraft.py -else - python -m pytest ./python/pylibraft/pylibraft/test -fi +python -m pytest ./python/pylibraft/pylibraft/test diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index b70563b7a1..4a3e963ace 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -14,9 +14,4 @@ python -m pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/raft_dask*.whl)[test] -# Run smoke tests for aarch64 pull requests -if [[ "$(arch)" == "aarch64" && "${RAPIDS_BUILD_TYPE}" == "pull-request" ]]; then - python ./ci/wheel_smoke_test_raft_dask.py -else - python -m pytest ./python/raft-dask/raft_dask/test -fi +python -m pytest ./python/raft-dask/raft_dask/test From 32f215ce652df93ea95f519b4609f11d8d25f4a4 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Fri, 5 Apr 2024 17:25:53 +0000 Subject: [PATCH 2/4] drop files --- ci/wheel_smoke_test_pylibraft.py | 53 --------------- ci/wheel_smoke_test_raft_dask.py | 107 ------------------------------- 2 files changed, 160 deletions(-) delete mode 100644 ci/wheel_smoke_test_pylibraft.py delete mode 100644 ci/wheel_smoke_test_raft_dask.py diff --git a/ci/wheel_smoke_test_pylibraft.py b/ci/wheel_smoke_test_pylibraft.py deleted file mode 100644 index c0df2fe45c..0000000000 --- a/ci/wheel_smoke_test_pylibraft.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -from scipy.spatial.distance import cdist - -from pylibraft.common import Handle, Stream, device_ndarray -from pylibraft.distance import pairwise_distance - - -if __name__ == "__main__": - metric = "euclidean" - n_rows = 1337 - n_cols = 1337 - - input1 = np.random.random_sample((n_rows, n_cols)) - input1 = np.asarray(input1, order="C").astype(np.float64) - - output = np.zeros((n_rows, n_rows), dtype=np.float64) - - expected = cdist(input1, input1, metric) - - expected[expected <= 1e-5] = 0.0 - - input1_device = device_ndarray(input1) - output_device = None - - s2 = Stream() - handle = Handle(stream=s2) - ret_output = pairwise_distance( - input1_device, input1_device, output_device, metric, handle=handle - ) - handle.sync() - - output_device = ret_output - - actual = output_device.copy_to_host() - - actual[actual <= 1e-5] = 0.0 - - assert np.allclose(expected, actual, rtol=1e-4) diff --git a/ci/wheel_smoke_test_raft_dask.py b/ci/wheel_smoke_test_raft_dask.py deleted file mode 100644 index 5709ac901c..0000000000 --- a/ci/wheel_smoke_test_raft_dask.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from dask.distributed import Client, get_worker, wait -from dask_cuda import LocalCUDACluster, initialize - -from raft_dask.common import ( - Comms, - local_handle, - perform_test_comm_split, - perform_test_comms_allgather, - perform_test_comms_allreduce, - perform_test_comms_bcast, - perform_test_comms_device_multicast_sendrecv, - perform_test_comms_device_send_or_recv, - perform_test_comms_device_sendrecv, - perform_test_comms_gather, - perform_test_comms_gatherv, - perform_test_comms_reduce, - perform_test_comms_reducescatter, - perform_test_comms_send_recv, -) - -import os -os.environ["UCX_LOG_LEVEL"] = "error" - - -def func_test_send_recv(sessionId, n_trials): - handle = local_handle(sessionId, dask_worker=get_worker()) - return perform_test_comms_send_recv(handle, n_trials) - - -def func_test_collective(func, sessionId, root): - handle = local_handle(sessionId, dask_worker=get_worker()) - return func(handle, root) - - -if __name__ == "__main__": - # initial setup - cluster = LocalCUDACluster(protocol="tcp", scheduler_port=0) - client = Client(cluster) - - n_trials = 5 - root_location = "client" - - # p2p test for ucx - cb = Comms(comms_p2p=True, verbose=True) - cb.init() - - dfs = [ - client.submit( - func_test_send_recv, - cb.sessionId, - n_trials, - pure=False, - workers=[w], - ) - for w in cb.worker_addresses - ] - - wait(dfs, timeout=5) - - assert list(map(lambda x: x.result(), dfs)) - - cb.destroy() - - # collectives test for nccl - - cb = Comms( - verbose=True, client=client, nccl_root_location=root_location - ) - cb.init() - - for k, v in cb.worker_info(cb.worker_addresses).items(): - - dfs = [ - client.submit( - func_test_collective, - perform_test_comms_allgather, - cb.sessionId, - v["rank"], - pure=False, - workers=[w], - ) - for w in cb.worker_addresses - ] - wait(dfs, timeout=5) - - assert all([x.result() for x in dfs]) - - cb.destroy() - - # final client and cluster teardown - client.close() - cluster.close() From 28b1270a3bc3124bca7d313fe371dd5c0ac25e8f Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 8 Apr 2024 14:35:07 -0500 Subject: [PATCH 3/4] Update test_wheel_pylibraft.sh --- ci/test_wheel_pylibraft.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index d6b1266b69..b38f5a690b 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. set -euo pipefail From c9a209de9dcbd29c95e76ff516e398d0f6ff15bf Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 8 Apr 2024 14:35:20 -0500 Subject: [PATCH 4/4] Update test_wheel_raft_dask.sh --- ci/test_wheel_raft_dask.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index 4a3e963ace..76bb62e859 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. set -euo pipefail