Skip to content

BLD: Fix upload PyPI #3672

BLD: Fix upload PyPI

BLD: Fix upload PyPI #3672

Workflow file for this run

name: Python CI
on:
push:
branches:
- '*'
pull_request:
types: ['opened', 'reopened', 'synchronize']
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
lint:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ "ubuntu-latest" ]
python-version: [ "3.10" ]
steps:
- name: Check out code
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: recursive
- name: Set up Python environment
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: flake8 Lint
uses: py-actions/flake8@v2
with:
path: "python/xorbits"
args: "--config python/setup.cfg"
- name: black
uses: psf/black@stable
with:
src: "python/xorbits"
options: "--check"
version: "23.12.0"
- uses: isort/isort-action@master
with:
sortPaths: "python/xorbits"
configuration: "--check-only --diff --sp python/setup.cfg"
- name: mypy
run: pip install mypy && cd python && mypy xorbits
- name: codespell
run: pip install codespell==2.2.6 && cd python && codespell xorbits
- name: Set up Node.js
uses: actions/setup-node@v1
with:
node-version: 16
# ESLint and Prettier must be in `package.json`
- name: Install Node.js dependencies
run: cd python/xorbits/web/ui && npm ci
- name: ESLint Check
run: cd python/xorbits/web/ui && npx eslint .
- name: Prettier Check
run: cd python/xorbits/web/ui && ./node_modules/.bin/prettier --check .
build_test_job:
if: github.repository == 'xorbitsai/xorbits'
runs-on: ${{ matrix.os }}
needs: lint
env:
CONDA_ENV: xorbits-test
SELF_HOST_PYTHON: /root/miniconda3/envs/xorbits-test/bin/python
SELF_HOST_CONDA: /root/miniconda3/condabin/conda
defaults:
run:
shell: bash -l {0}
strategy:
fail-fast: false
matrix:
os: ["ubuntu-latest", "macos-13", "windows-latest"]
python-version: ["3.9", "3.10", "3.11"]
module: ["xorbits", "xorbits/numpy", "xorbits/pandas"]
exclude:
- { os: macos-13, python-version: 3.10}
- { os: macos-13, python-version: 3.9}
- { os: windows-latest, python-version: 3.10}
- { os: windows-latest, python-version: 3.9}
- { os: windows-latest, module: kubernetes}
- { os: macos-13, module: kubernetes}
include:
- { os: ubuntu-latest, module: _mars/dataframe, python-version: 3.9 }
- { os: ubuntu-latest, module: learn, python-version: 3.9 }
- { os: ubuntu-latest, module: mars-core, python-version: 3.9 }
- { os: ubuntu-20.04, module: hadoop, python-version: 3.9 }
- { os: ubuntu-latest, module: vineyard, python-version: 3.11 }
- { os: ubuntu-latest, module: external-storage, python-version: 3.11 }
# always test compatibility with the latest version
# - { os: ubuntu-latest, module: compatibility, python-version: 3.9 }
- { os: ubuntu-latest, module: doc-build, python-version: 3.9 }
- { os: self-hosted, module: gpu, python-version: 3.11}
- { os: ubuntu-latest, module: jax, python-version: 3.9 }
# a self-hosted runner which needs computing resources, activate when necessary
# - { os: juicefs-ci, module: kubernetes-juicefs, python-version: 3.9 }
- { os: ubuntu-latest, module: slurm, python-version: 3.9 }
- { os: ubuntu-latest, module: datasets, python-version: 3.9 }
- { os: ubuntu-latest, module: kubernetes, python-version: 3.11 }
steps:
- name: Check out code
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: recursive
- name: Add msbuild to PATH
if: ${{ matrix.os == 'windows-latest'}}
uses: microsoft/setup-msbuild@v1.1
- name: Set up conda ${{ matrix.python-version }}
uses: conda-incubator/setup-miniconda@v3
if: ${{ matrix.module != 'gpu' && matrix.module != 'kubernetes-juicefs' }}
with:
python-version: ${{ matrix.python-version }}
activate-environment: ${{ env.CONDA_ENV }}
- name: Start minikube
uses: medyagh/setup-minikube@master
if: ${{ matrix.module == 'kubernetes' }}
with:
driver: none
kubernetes-version: v1.23.12
minikube-version: 1.31.2
- name: Install ucx dependencies
if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.module != 'doc-build') }}
run: |
# ucx-py move to ucxx and ucxx-cu12 can be run on CPU
# conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py
pip install ucxx-cu12
- name: Install libomp (macOS)
if: ${{ matrix.os == 'macos-latest' || matrix.os == 'macos-13' }}
run: brew install libomp
- name: Install dependencies
env:
MODULE: ${{ matrix.module }}
PYTHON: ${{ matrix.python-version }}
if: ${{ matrix.module != 'gpu' }}
run: |
if [[ "$MODULE" == "doc-build" ]]; then
pip install --upgrade --no-cache-dir pip setuptools
pip install --upgrade --no-cache-dir sphinx readthedocs-sphinx-ext
pip install --upgrade --upgrade-strategy only-if-needed --no-cache-dir ".[doc]"
else
pip install -e "git+https://github.com/xorbitsai/xoscar.git@main#subdirectory=python&egg=xoscar"
pip install "numpy<2.0.0" scipy cython pyftpdlib coverage flaky numexpr
if [[ "$MODULE" == "xorbits/pandas" ]]; then
pip install openpyxl
fi
if [[ "$MODULE" == "mars-core" ]]; then
pip install oss2
fi
if [[ "$MODULE" == "kubernetes" ]]; then
pip install kubernetes
fi
if [[ "$MODULE" == "kubernetes-juicefs" ]]; then
pip install kubernetes
if helm version --short >/dev/null 2>&1; then
echo "Helm installed. Skipping installation."
else
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
fi
if helm repo list | grep "juicefs" >/dev/null 2>&1; then
echo "Repo juicefs existed. Skipping adding to helm repos."
else
helm repo add juicefs https://juicedata.github.io/charts/
helm repo update
helm fetch --untar juicefs/juicefs-csi-driver
fi
if kubectl get pods -n kube-system | grep "juicefs-csi" >/dev/null 2>&1; then
echo "Juicefs-csi-driver installed. Skipping installation."
else
helm install juicefs-csi-driver juicefs/juicefs-csi-driver -n kube-system -f xorbits/deploy/kubernetes/external_storage/juicefs/tests/values.yaml
fi
sleep 60
kubectl apply -f xorbits/deploy/kubernetes/external_storage/juicefs/tests/example-redis-config.yaml
sleep 60
fi
if [[ "$MODULE" == "hadoop" ]]; then
../CI/install-hadoop.sh
echo "import coverage; coverage.process_startup()" > \
$(python -c "import site; print(site.getsitepackages()[-1])")/coverage.pth
conda install --quiet --yes conda-forge::libffi==3.4.2 conda-forge::skein==0.8.1 conda-forge::conda-pack==0.8.0 conda-forge::protobuf==3.20.1 conda-forge::grpcio==1.42.0
fi
if [[ "$MODULE" == "vineyard" ]]; then
pip install vineyard
mkdir -p /tmp/etcd-download-test
export ETCD_VER=v3.4.13
export ETCD_DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
curl -L $ETCD_DOWNLOAD_URL/$ETCD_VER/etcd-$ETCD_VER-linux-amd64.tar.gz -o /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz
tar xzvf /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1
sudo mv /tmp/etcd-download-test/etcd /usr/local/bin/
sudo mv /tmp/etcd-download-test/etcdctl /usr/local/bin/
rm -fr /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz /tmp/etcd-download-test
fi
if [[ "$MODULE" == "external-storage" ]]; then
export CUR_DIR=$(pwd)
mkdir -p /tmp/alluxio-download-test
cd /tmp/alluxio-download-test
wget -q https://downloads.alluxio.io/downloads/files/2.9.3/alluxio-2.9.3-bin.tar.gz
tar -xzf alluxio-2.9.3-bin.tar.gz
cd alluxio-2.9.3
cp conf/alluxio-env.sh.template conf/alluxio-env.sh
cp conf/alluxio-site.properties.template conf/alluxio-site.properties
echo "alluxio.master.hostname=localhost" >> conf/alluxio-site.properties
cd $CUR_DIR
sudo mv /tmp/alluxio-download-test/alluxio-2.9.3 /usr/local/bin/
rm -R /tmp/alluxio-download-test
unset CUR_DIR
export ALLUXIO_HOME="/usr/local/bin/alluxio-2.9.3"
${ALLUXIO_HOME}/bin/alluxio format
${ALLUXIO_HOME}/bin/alluxio-start.sh local SudoMount
curl -sSL https://d.juicefs.com/install | sh -
curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list
sudo apt-get update
sudo apt-get install redis
export JUICEFS_HOME="/usr/local/bin/juicefs"
fi
if [[ "$PYTHON" == '3.9' ]]; then
pip install h5py zarr matplotlib fastparquet
conda install -n test --quiet --yes -c conda-forge python=$PYTHON \
"tiledb-py>=0.4.3,<0.6.0" "tiledb<2.0.0" || true
fi
if [[ "$MODULE" == "_mars/learn" ]]; then
pip install scikit-learn
pip install xgboost lightgbm keras tensorflow faiss-cpu\<1.7.3 torch torchvision \
statsmodels tsfresh dask[complete] mimesis\<9.0.0
fi
if [[ "$MODULE" == "learn" ]]; then
pip install xgboost lightgbm
fi
if [[ "$MODULE" == "compatibility" ]]; then
# test if compatible with older versions
pip install "pandas==1.5.3" "scipy<=1.10.1" "numpy<=1.24.1" "matplotlib<=3.7.0" "pyarrow<12.0.0" "sqlalchemy<2"
fi
if [[ "$MODULE" == "jax" ]]; then
# test jax
pip install "jax>=0.4.0" "jaxlib>=0.4.0"
fi
if [[ "$MODULE" == "datasets" ]]; then
# test huggingface datasets
pip install datasets torch
fi
pip install -e ".[dev,extra,aws]"
fi
working-directory: ./python
- name: Install on JuiceFsCI
if: ${{ matrix.module == 'kubernetes-juicefs' }}
run: |
pip install -U xoscar
python setup.py build_ext -i
working-directory: ./python
- name: Slurm Setup Job queuing system
if: ${{ matrix.module == 'slurm' }}
run: |
source CI/slurm/${{ matrix.module }}.sh
jobqueue_before_install
- name: Slurm Install xorbits
if: ${{ matrix.module == 'slurm' }}
run: |
source CI/slurm/${{ matrix.module }}.sh
jobqueue_install
- name: Install on GPU
if: ${{ matrix.module == 'gpu' }}
run: |
source activate ${{ env.CONDA_ENV }}
pip install --extra-index-url=https://pypi.nvidia.com cudf-cu12==24.8.*
pip install ucxx-cu12 cython "numpy>=1.14.0,<2.0.0" cloudpickle scikit-learn \
pyyaml psutil tornado sqlalchemy defusedxml tqdm uvloop coverage \
pytest pytest-cov pytest-timeout pytest-forked pytest-asyncio pytest-mock
pip install -U xoscar
pip install -e ".[dev,extra,aws]"
working-directory: ./python
- name: Build doc
if: ${{ matrix.module == 'doc-build' }}
run: |
make html
make html_zh_cn
working-directory: ./doc
- name: Test with pytest
if: ${{ matrix.module != 'doc-build' && matrix.module != 'gpu' }}
env:
MODULE: ${{ matrix.module }}
run: |
if [[ "$MODULE" == "xorbits" ]]; then
pytest --ignore xorbits/_mars/ --ignore xorbits/pandas --ignore xorbits/numpy \
--ignore xorbits/xgboost --ignore xorbits/lightgbm --ignore xorbits/sklearn \
--ignore xorbits/datasets \
--ignore xorbits/core/tests/test_execution_exit.py \
--timeout=1500 \
-W ignore::PendingDeprecationWarning \
--cov-config=setup.cfg --cov-report=xml --cov=xorbits \
-k "not test_execution_with_process_exit_message" \
xorbits
# workaround: this case will hang, run it separately.
pytest --timeout=1500 \
-W ignore::PendingDeprecationWarning \
--cov-config=setup.cfg --cov-report=xml --cov=xorbits \
-k "test_execution_with_process_exit_message" \
xorbits/core/tests/test_execution.py
elif [[ "$MODULE" == "xorbits/pandas" ]]; then
pytest --timeout=1500 \
-W ignore::PendingDeprecationWarning \
--cov-config=setup.cfg --cov-report=xml \
--cov=xorbits xorbits/pandas
elif [[ "$MODULE" == "xorbits/numpy" ]]; then
pytest --timeout=1500 \
-W ignore::PendingDeprecationWarning \
--cov-config=setup.cfg --cov-report=xml --cov=xorbits \
-k "not test_numpy_fallback" \
xorbits/numpy
# workaround: this case will hang, run it separately.
pytest --timeout=1500 \
-W ignore::PendingDeprecationWarning \
--cov-config=setup.cfg --cov-report=xml \
--cov=xorbits \
-k "test_numpy_fallback" \
xorbits/numpy/numpy_adapters/tests/test_numpy_adapters.py
elif [[ "$MODULE" == "mars-core" ]]; then
pytest --forked --log-level=DEBUG --ignore xorbits/_mars/dataframe --ignore xorbits/_mars/tensor \
--ignore xorbits/_mars/learn --ignore xorbits/_mars/remote \
--timeout=1500 \
-W ignore::PendingDeprecationWarning \
--cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/_mars
elif [[ "$MODULE" == "kubernetes" ]]; then
pytest --ignore xorbits/_mars/ --timeout=1500 \
-W ignore::PendingDeprecationWarning \
--cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/deploy/kubernetes
elif [[ "$MODULE" == "kubernetes-juicefs" ]]; then
pytest --ignore xorbits/_mars/ --timeout=1500 \
-W ignore::PendingDeprecationWarning \
--cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/deploy/kubernetes/external_storage/juicefs
elif [[ "$MODULE" == "slurm" ]]; then
docker exec c1 /bin/bash -c "pip install xorbits"
docker exec c2 /bin/bash -c "pip install xorbits"
docker exec slurmctld /bin/bash -c \
"pytest /xorbits/python/xorbits/deploy/slurm/tests/test_slurm.py "
elif [[ "$MODULE" == "hadoop" ]]; then
export WITH_HADOOP="1"
export HADOOP_HOME="/usr/local/hadoop"
export CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath --glob`
export HADOOP_INSTALL=$HADOOP_HOME
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export HADOOP_COMMON_LIB_NATIVE_DIR="$HADOOP_HOME/lib/native"
export PATH="$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin"
pytest --timeout=1500 -W ignore::PendingDeprecationWarning xorbits/_mars -m hadoop
elif [[ "$MODULE" == "vineyard" ]]; then
pytest --timeout=1500 -W ignore::PendingDeprecationWarning \
--cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/_mars/storage/tests/test_libs.py
pytest --timeout=1500 -W ignore::PendingDeprecationWarning \
--cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/_mars/deploy/oscar/tests/test_local.py
pytest --timeout=1500 -W ignore::PendingDeprecationWarning \
--cov-config=setup.cfg --cov-report=xml --cov=xorbits -k "vineyard" \
xorbits/_mars/tensor/datastore/tests/test_datastore_execution.py \
xorbits/_mars/dataframe/datastore/tests/test_datastore_execution.py
elif [[ "$MODULE" == "external-storage" ]]; then
export ALLUXIO_HOME="/usr/local/bin/alluxio-2.9.3"
export JUICEFS_HOME="/usr/local/bin/juicefs"
pytest --timeout=1500 -W ignore::PendingDeprecationWarning \
--cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/_mars/storage/tests/test_libs.py
elif [[ "$MODULE" == "_mars/learn" ]]; then
pytest --timeout=1500 \
-W ignore::PendingDeprecationWarning \
--cov-config=setup.cfg --cov-report=xml --cov=xorbits \
xorbits/$MODULE xorbits/_mars/contrib/dask/tests/test_dask.py
elif [[ "$MODULE" == "learn" ]]; then
pytest --timeout=1500 \
-W ignore::PendingDeprecationWarning \
--cov-config=setup.cfg --cov-report=xml --cov=xorbits \
xorbits/xgboost xorbits/lightgbm
elif [ "$MODULE" == "jax" ]; then
pytest --cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/_mars/tensor/fuse/tests/test_runtime_fusion.py
pytest --cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/_mars/tensor/
elif [ "$MODULE" == "datasets" ]; then
pytest --cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/datasets/backends
pytest --cov-config=setup.cfg --cov-report=xml --cov=xorbits xorbits/datasets/tests
elif [ "$MODULE" == "compatibility" ]; then
pytest --timeout=1500 \
-W ignore::PendingDeprecationWarning \
--cov-config=setup.cfg --cov-report=xml --cov=xorbits/deploy --cov=xorbits xorbits/_mars/dataframe
pytest --timeout=1500 \
-W ignore::PendingDeprecationWarning \
--cov-config=setup.cfg --cov-report=xml --cov=xorbits/deploy --cov=xorbits xorbits/_mars/services/storage
else
pytest --timeout=1500 \
-W ignore::PendingDeprecationWarning \
--cov-config=setup.cfg --cov-report=xml --cov=xorbits/deploy --cov=xorbits xorbits/$MODULE
fi
working-directory: ./python
- name: Test with pytest GPU
if: ${{ matrix.module == 'gpu' }}
run: |
source activate ${{ env.CONDA_ENV }}
pytest -m cuda --gpu --ignore xorbits/datasets \
--ignore xorbits/sklearn --ignore xorbits/_mars/learn \
--cov-config=setup.cfg --cov-report=xml --cov=xorbits \
xorbits
working-directory: ./python
- name: Cleanup on slurm
if: ${{ matrix.module == 'slurm' }}
run: |
source CI/slurm/${{ matrix.module }}.sh
jobqueue_after_script
- name: Report coverage data
uses: codecov/codecov-action@v4
with:
working-directory: ./python
flags: unittests
token: ${{ secrets.CODECOV_TOKEN }}