diff --git a/.travis.yml b/.travis.yml index a11cd469e9b9c..5d00684f93552 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,6 +36,13 @@ matrix: - env: - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network)" + - dist: trusty + os: linux + sudo: true + arch: arm64 + env: + - JOB="3.7" ENV_FILE="ci/deps/travis-37-aarch64.yaml" PATTERN="(not slow and not network)" + - env: - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1" services: @@ -89,6 +96,6 @@ script: after_script: - echo "after_script start" - - source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - - ci/print_skipped.py + - source activate pandas-dev && pushd /tmp && if [ `uname -m` = 'aarch64' ];then VAR="python3.7" ; else VAR="python";fi && $VAR -c "import pandas; pandas.show_versions();" && popd + - ci/print_skipped.py - echo "after_script done" diff --git a/ci/deps/travis-37-aarch64.yaml b/ci/deps/travis-37-aarch64.yaml new file mode 100644 index 0000000000000..779d7600b1ca2 --- /dev/null +++ b/ci/deps/travis-37-aarch64.yaml @@ -0,0 +1,22 @@ +name: pandas-dev +channels: + - defaults + - conda-forge + - c3i_test +dependencies: + - python=3.7.* + + # tools + - cython>=0.29.13 + - pytest>=5.0.1 + - pytest-xdist>=1.21 + - hypothesis>=3.58.0 + + # pandas dependencies + - botocore>=1.11 + - numpy + - python-dateutil + - pytz + - pip + - pip: + - moto diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 8020680d617d7..2bbf79ae0c746 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -5,6 +5,17 @@ # https://github.com/pytest-dev/pytest/issues/1075 export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))') +if [ -n "$LOCALE_OVERRIDE" ]; then + export LC_ALL="$LOCALE_OVERRIDE" + export LANG="$LOCALE_OVERRIDE" + PANDAS_LOCALE=`python -c 'import pandas; pandas.get_option("display.encoding")'` + if [[ "$LOCALE_OVERRIDE" != "$PANDAS_LOCALE" ]]; then + echo "pandas could not detect the locale. System locale: $LOCALE_OVERRIDE, pandas detected: $PANDAS_LOCALE" + # TODO Not really aborting the tests until https://github.com/pandas-dev/pandas/issues/23923 is fixed + # exit 1 + fi +fi + if [[ "not network" == *"$PATTERN"* ]]; then export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4; fi @@ -19,14 +30,22 @@ PYTEST_CMD="pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --duratio # Travis does not have have an X server if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then DISPLAY=DISPLAY=:99.0 - PYTEST_CMD="xvfb-run -e /dev/stdout $PYTEST_CMD" + if [ `uname -m` = 'aarch64' ]; then + PYTEST_CMD="xvfb-run -e /dev/stdout pytest -m \"$PATTERN\" -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" + else + PYTEST_CMD="xvfb-run -e /dev/stdout $PYTEST_CMD" + fi fi echo $PYTEST_CMD -sh -c "$PYTEST_CMD" +if [ `uname -m` = 'aarch64' ]; then + sudo sh -c "$PYTEST_CMD" +else + sh -c "$PYTEST_CMD" +fi if [[ "$COVERAGE" && $? == 0 && "$TRAVIS_BRANCH" == "master" ]]; then echo "uploading coverage" - echo "bash <(curl -s https://codecov.io/bash) -Z -c -f $COVERAGE_FNAME" - bash <(curl -s https://codecov.io/bash) -Z -c -f $COVERAGE_FNAME + echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME" + bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME fi diff --git a/ci/setup_env.sh b/ci/setup_env.sh index db28eaea8956e..25f0603944173 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -1,5 +1,7 @@ #!/bin/bash -e +IS_SUDO="" +ARCHICONDA_PYTHON="python3.7" # edit the locale file if needed if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then echo "Adding locale to the first line of pandas/__init__.py" @@ -12,8 +14,13 @@ if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then echo fi -MINICONDA_DIR="$HOME/miniconda3" +if [ `uname -m` = 'aarch64' ]; then + MINICONDA_DIR="$HOME/archiconda3" + IS_SUDO="sudo" +else + MINICONDA_DIR="$HOME/miniconda3" +fi if [ -d "$MINICONDA_DIR" ]; then echo @@ -36,9 +43,24 @@ else exit 1 fi -wget -q "https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh" -O miniconda.sh -chmod +x miniconda.sh -./miniconda.sh -b +if [ `uname -m` = 'aarch64' ]; then + wget -q "https://github.com/Archiconda/build-tools/releases/download/0.2.3/Archiconda3-0.2.3-Linux-aarch64.sh" -O archiconda.sh + chmod +x archiconda.sh + $IS_SUDO apt-get install python-dev + $IS_SUDO apt-get install python3-pip + $IS_SUDO apt-get install lib$ARCHICONDA_PYTHON-dev + $IS_SUDO apt-get install xvfb + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib:/usr/local/lib:/usr/local/bin/python + ./archiconda.sh -b + echo "chmod MINICONDA_DIR" + $IS_SUDO chmod -R 777 $MINICONDA_DIR + $IS_SUDO cp $MINICONDA_DIR/bin/* /usr/bin/ + $IS_SUDO rm /usr/bin/lsb_release +else + wget -q "https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh" -O miniconda.sh + chmod +x miniconda.sh + ./miniconda.sh -b +fi export PATH=$MINICONDA_DIR/bin:$PATH @@ -50,8 +72,8 @@ echo echo "update conda" conda config --set ssl_verify false conda config --set quiet true --set always_yes true --set changeps1 false -conda install pip # create conda to create a historical artifact for pip & setuptools -conda update -n base conda +$IS_SUDO conda install pip # create conda to create a historical artifact for pip & setuptools +$IS_SUDO conda update -n base conda echo "conda info -a" conda info -a @@ -91,8 +113,18 @@ conda list conda remove --all -q -y -n pandas-dev echo +if [ `uname -m` = 'aarch64' ]; then + $IS_SUDO chmod -R 777 $MINICONDA_DIR + $IS_SUDO conda install botocore + $IS_SUDO conda install numpy + $IS_SUDO conda install python-dateutil=2.8.0 + $IS_SUDO conda install hypothesis + $IS_SUDO conda install pytz + $IS_SUDO chmod -R 777 $MINICONDA_DIR +fi + echo "conda env create -q --file=${ENV_FILE}" -time conda env create -q --file="${ENV_FILE}" +time $IS_SUDO conda env create -q --file="${ENV_FILE}" if [[ "$BITS32" == "yes" ]]; then @@ -106,13 +138,17 @@ source activate pandas-dev echo echo "remove any installed pandas package" echo "w/o removing anything else" -conda remove pandas -y --force || true -pip uninstall -y pandas || true +$IS_SUDO conda remove pandas -y --force || true +if [ `uname -m` = 'aarch64' ]; then + $IS_SUDO $ARCHICONDA_PYTHON -m pip uninstall -y pandas || true +else + pip uninstall -y pandas || true +fi echo echo "remove postgres if has been installed with conda" echo "we use the one from the CI" -conda remove postgresql -y --force || true +$IS_SUDO conda remove postgresql -y --force || true echo echo "conda list pandas" @@ -121,7 +157,10 @@ conda list pandas # Make sure any error below is reported as such echo "[Build extensions]" -python setup.py build_ext -q -i -j2 +if [ `uname -m` = 'aarch64' ]; then + sudo chmod -R 777 /home/travis/.ccache +fi +python setup.py build_ext -q -i # XXX: Some of our environments end up with old versions of pip (10.x) # Adding a new enough version of pip to the requirements explodes the @@ -130,21 +169,40 @@ python setup.py build_ext -q -i -j2 # - py35_compat # - py36_32bit echo "[Updating pip]" -python -m pip install --no-deps -U pip wheel setuptools +if [ `uname -m` = 'aarch64' ]; then + sudo chmod -R 777 /home/travis/archiconda3/envs/pandas-dev/lib/$ARCHICONDA_PYTHON/site-packages + $IS_SUDO $ARCHICONDA_PYTHON -m pip install pytest-forked + $IS_SUDO $ARCHICONDA_PYTHON -m pip install pytest-xdist + $IS_SUDO $ARCHICONDA_PYTHON -m pip install --no-deps -U pip wheel setuptools + sudo chmod -R 777 $MINICONDA_DIR +else + python -m pip install --no-deps -U pip wheel setuptools +fi echo "[Install pandas]" -python -m pip install --no-build-isolation -e . +if [ `uname -m` = 'aarch64' ]; then + $IS_SUDO chmod -R 777 $MINICONDA_DIR + $IS_SUDO $ARCHICONDA_PYTHON -m pip install numpy + $IS_SUDO $ARCHICONDA_PYTHON -m pip install hypothesis + $IS_SUDO chmod -R 777 /home/travis/.cache/ + $IS_SUDO $ARCHICONDA_PYTHON -m pip install --no-build-isolation -e . +else + python -m pip install --no-build-isolation -e . +fi echo echo "conda list" conda list # Install DB for Linux - if [[ -n ${SQL:0} ]]; then echo "installing dbs" + if [ `uname -m` = 'aarch64' ]; then + sudo systemctl start mysql + else + psql -c 'create database pandas_nosetest;' -U postgres + fi mysql -e 'create database pandas_nosetest;' - psql -c 'create database pandas_nosetest;' -U postgres else echo "not using dbs on non-linux Travis builds or Azure Pipelines" fi