diff --git a/.travis.yml b/.travis.yml
index e49f01a21bdbd7..ef9d54b41f3f97 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,7 +11,6 @@ cache:
   directories:
     - $HOME/.cache # cython cache
     - $HOME/.ccache # compiler cache
-
 env:
   global:
     # create a github personal access token
@@ -33,12 +32,9 @@ matrix:
     - env:
         - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network and not clipboard)"
 
-    - dist: trusty
-      os: linux
-      sudo: true
-      arch: arm64
+    - arch: arm64
       env:
-        - JOB="3.7" ENV_FILE="ci/deps/travis-37-aarch64.yaml" PATTERN="(not slow and not network)"
+        - JOB="3.7" ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)"
 
     - env:
         - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
@@ -95,6 +91,6 @@ script:
 
 after_script:
   - echo "after_script start"
-  - source activate pandas-dev && pushd /tmp && if [ `uname -m` = 'aarch64' ];then VAR="python3.7" ; else VAR="python";fi && $VAR -c "import pandas; pandas.show_versions();" && popd
+  - source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
   - ci/print_skipped.py
   - echo "after_script done"
diff --git a/ci/run_tests.sh b/ci/run_tests.sh
index 584a5b2eefe6f6..a9294bc52a4c56 100755
--- a/ci/run_tests.sh
+++ b/ci/run_tests.sh
@@ -14,27 +14,23 @@ if [ "$COVERAGE" ]; then
     COVERAGE="-s --cov=pandas --cov-report=xml:$COVERAGE_FNAME"
 fi
 
-# Travis does not have have an X server
-if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-    DISPLAY=DISPLAY=:99.0
-    if [ `uname -m` = 'aarch64' ]; then
-        PYTEST_CMD="xvfb-run -e /dev/stdout pytest -m \"$PATTERN\" -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"
-    else
-        PYTEST_CMD="xvfb-run -e /dev/stdout $PYTEST_CMD"
-    fi
+# If no X server is found, we use xvfb to emulate it
+if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then
+    export DISPLAY=":0"
+    XVFB="xvfb-run "
 fi
 
-PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"
-
-echo $PYTEST_CMD
-if [ `uname -m` = 'aarch64' ]; then
-   sudo sh -c "$PYTEST_CMD"
+if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then
+    PYTEST_CMD="pytest -m \"$PATTERN\" --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas";
 else
-    sh -c "$PYTEST_CMD"
+    PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"
 fi
 
+echo $PYTEST_CMD
+sh -c "$PYTEST_CMD"
+
 if [[ "$COVERAGE" && $? == 0 && "$TRAVIS_BRANCH" == "master" ]]; then
     echo "uploading coverage"
-    echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME"
+    echo "bash <(curl -s https://codecov.io/bash) -Z -c -f $COVERAGE_FNAME"
           bash <(curl -s https://codecov.io/bash) -Z -c -f $COVERAGE_FNAME
 fi
diff --git a/ci/setup_env.sh b/ci/setup_env.sh
index 46a14ed09d327e..a61440382245ed 100755
--- a/ci/setup_env.sh
+++ b/ci/setup_env.sh
@@ -1,7 +1,5 @@
 #!/bin/bash -e
 
-IS_SUDO=""
-ARCHICONDA_PYTHON="python3.7"
 # edit the locale file if needed
 if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then
     echo "Adding locale to the first line of pandas/__init__.py"
@@ -14,13 +12,8 @@ if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then
     echo
 fi
 
+MINICONDA_DIR="$HOME/miniconda3"
 
-if [ `uname -m` = 'aarch64' ]; then
-   MINICONDA_DIR="$HOME/archiconda3"
-   IS_SUDO="sudo"
-else
-   MINICONDA_DIR="$HOME/miniconda3"
-fi
 
 if [ -d "$MINICONDA_DIR" ]; then
     echo
@@ -43,24 +36,16 @@ else
   exit 1
 fi
 
-if [ `uname -m` = 'aarch64' ]; then
-   wget -q "https://github.com/Archiconda/build-tools/releases/download/0.2.3/Archiconda3-0.2.3-Linux-aarch64.sh" -O archiconda.sh
-   chmod +x archiconda.sh
-   $IS_SUDO apt-get install python-dev
-   $IS_SUDO apt-get install python3-pip
-   $IS_SUDO apt-get install lib$ARCHICONDA_PYTHON-dev
-   $IS_SUDO apt-get install xvfb
-   export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib:/usr/local/lib:/usr/local/bin/python
-   ./archiconda.sh -b
-   echo "chmod MINICONDA_DIR"
-   $IS_SUDO chmod -R 777 $MINICONDA_DIR
-   $IS_SUDO cp $MINICONDA_DIR/bin/* /usr/bin/
-   $IS_SUDO rm /usr/bin/lsb_release
+if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then
+  CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.2-1/Miniforge3-4.8.2-1-Linux-aarch64.sh";
 else
-   wget -q "https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh" -O miniconda.sh
-   chmod +x miniconda.sh
-   ./miniconda.sh -b
+  CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh";
 fi
+wget -q $CONDA_URL -O miniconda.sh;
+chmod +x miniconda.sh
+
+# Installation path is required for ARM64 platform as miniforge script installs in path $HOME/miniforge3.
+./miniconda.sh -b -p $MINICONDA_DIR
 
 export PATH=$MINICONDA_DIR/bin:$PATH
 
@@ -72,8 +57,8 @@ echo
 echo "update conda"
 conda config --set ssl_verify false
 conda config --set quiet true --set always_yes true --set changeps1 false
-$IS_SUDO conda install pip  # create conda to create a historical artifact for pip & setuptools
-$IS_SUDO conda update -n base conda
+conda install pip conda  # create conda to create a historical artifact for pip & setuptools
+conda update -n base conda
 
 echo "conda info -a"
 conda info -a
@@ -113,18 +98,8 @@ conda list
 conda remove --all -q -y -n pandas-dev
 
 echo
-if [ `uname -m` = 'aarch64' ]; then
-    $IS_SUDO chmod -R 777 $MINICONDA_DIR
-    $IS_SUDO conda install botocore
-    $IS_SUDO conda install numpy
-    $IS_SUDO conda install python-dateutil=2.8.0
-    $IS_SUDO conda install hypothesis
-    $IS_SUDO conda install pytz
-    $IS_SUDO chmod -R 777 $MINICONDA_DIR
-fi
-
 echo "conda env create -q --file=${ENV_FILE}"
-time $IS_SUDO conda env create -q --file="${ENV_FILE}"
+time conda env create -q --file="${ENV_FILE}"
 
 
 if [[ "$BITS32" == "yes" ]]; then
@@ -138,17 +113,13 @@ source activate pandas-dev
 echo
 echo "remove any installed pandas package"
 echo "w/o removing anything else"
-$IS_SUDO conda remove pandas -y --force || true
-if [ `uname -m` = 'aarch64' ]; then
-    $IS_SUDO $ARCHICONDA_PYTHON -m pip uninstall -y pandas || true
-else
-    pip uninstall -y pandas || true
-fi
+conda remove pandas -y --force || true
+pip uninstall -y pandas || true
 
 echo
 echo "remove postgres if has been installed with conda"
 echo "we use the one from the CI"
-$IS_SUDO conda remove postgresql -y --force || true
+conda remove postgresql -y --force || true
 
 echo
 echo "remove qt"
@@ -162,10 +133,7 @@ conda list pandas
 # Make sure any error below is reported as such
 
 echo "[Build extensions]"
-if [ `uname -m` = 'aarch64' ]; then
-    sudo chmod -R 777 /home/travis/.ccache
-fi
-python setup.py build_ext -q -i
+python setup.py build_ext -q -i -j2
 
 # XXX: Some of our environments end up with old versions of pip (10.x)
 # Adding a new enough version of pip to the requirements explodes the
@@ -174,40 +142,21 @@ python setup.py build_ext -q -i
 # - py35_compat
 # - py36_32bit
 echo "[Updating pip]"
-if [ `uname -m` = 'aarch64' ]; then
-    sudo chmod -R 777 /home/travis/archiconda3/envs/pandas-dev/lib/$ARCHICONDA_PYTHON/site-packages
-    $IS_SUDO $ARCHICONDA_PYTHON -m pip install pytest-forked
-    $IS_SUDO $ARCHICONDA_PYTHON -m pip install pytest-xdist
-    $IS_SUDO $ARCHICONDA_PYTHON -m pip install --no-deps -U pip wheel setuptools
-    sudo chmod -R 777 $MINICONDA_DIR
-else
-    python -m pip install --no-deps -U pip wheel setuptools
-fi
+python -m pip install --no-deps -U pip wheel setuptools
 
 echo "[Install pandas]"
-if [ `uname -m` = 'aarch64' ]; then
-    $IS_SUDO chmod -R 777 $MINICONDA_DIR
-    $IS_SUDO $ARCHICONDA_PYTHON -m pip install numpy
-    $IS_SUDO $ARCHICONDA_PYTHON -m pip install hypothesis
-    $IS_SUDO chmod -R 777 /home/travis/.cache/
-    $IS_SUDO $ARCHICONDA_PYTHON -m pip install --no-build-isolation -e .
-else
-    python -m pip install --no-build-isolation -e .
-fi
+python -m pip install --no-build-isolation -e .
 
 echo
 echo "conda list"
 conda list
 
 # Install DB for Linux
+
 if [[ -n ${SQL:0} ]]; then
   echo "installing dbs"
-  if [ `uname -m` = 'aarch64' ]; then
-    sudo systemctl start mysql
-  else
-    psql -c 'create database pandas_nosetest;' -U postgres
-  fi
   mysql -e 'create database pandas_nosetest;'
+  psql -c 'create database pandas_nosetest;' -U postgres
 else
    echo "not using dbs on non-linux Travis builds or Azure Pipelines"
 fi