diff --git a/.travis.yml b/.travis.yml
index 4fdcae485c61f1..425ecd63adfcb7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,8 +11,11 @@ cache:
   directories:
     - $HOME/.cache # cython cache
     - $HOME/.ccache # compiler cache
+
 env:
   global:
+    # Variable for test workers
+    - PYTEST_WORKERS="auto"
     # create a github personal access token
     # cd pandas-dev/pandas
     # travis encrypt 'PANDAS_GH_TOKEN=personal_access_token' -r pandas-dev/pandas
@@ -39,7 +42,7 @@ matrix:
 
     - arch: arm64
       env:
-        - JOB="3.7" ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)"
+        - JOB="3.7" PYTEST_WORKERS=4 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)"
 
     - env:
         - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index d042bda77d4e8b..e45cafc02cb610 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -5,6 +5,9 @@ trigger:
 pr:
 - master
 
+variables:
+  PYTEST_WORKERS: auto
+
 jobs:
 # Mac and Linux use the same template
 - template: ci/azure/posix.yml
diff --git a/ci/deps/travis-37-aarch64.yaml b/ci/deps/travis-37-arm64.yaml
similarity index 95%
rename from ci/deps/travis-37-aarch64.yaml
rename to ci/deps/travis-37-arm64.yaml
index 779d7600b1ca25..5cb53489be225a 100644
--- a/ci/deps/travis-37-aarch64.yaml
+++ b/ci/deps/travis-37-arm64.yaml
@@ -2,7 +2,6 @@ name: pandas-dev
 channels:
   - defaults
   - conda-forge
-  - c3i_test
 dependencies:
   - python=3.7.*
 
diff --git a/ci/run_tests.sh b/ci/run_tests.sh
index a9294bc52a4c56..b198b39e277b94 100755
--- a/ci/run_tests.sh
+++ b/ci/run_tests.sh
@@ -20,11 +20,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then
     XVFB="xvfb-run "
 fi
 
-if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then
-    PYTEST_CMD="pytest -m \"$PATTERN\" --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas";
-else
-    PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"
-fi
+PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"
 
 echo $PYTEST_CMD
 sh -c "$PYTEST_CMD"
diff --git a/ci/setup_env.sh b/ci/setup_env.sh
index 6c264fe8b1de20..4d551294dbb211 100755
--- a/ci/setup_env.sh
+++ b/ci/setup_env.sh
@@ -42,11 +42,12 @@ else
 fi
 
 if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then
-  CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.2-1/Miniforge3-4.8.2-1-Linux-aarch64.sh";
+  sudo apt-get -y install xvfb
+  CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.2-1/Miniforge3-4.8.2-1-Linux-aarch64.sh"
 else
-  CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh";
+  CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh"
 fi
-wget -q $CONDA_URL -O miniconda.sh;
+wget -q $CONDA_URL -O miniconda.sh
 chmod +x miniconda.sh
 
 # Installation path is required for ARM64 platform as miniforge script installs in path $HOME/miniforge3.
diff --git a/pandas/_libs/src/ujson/lib/ultrajson.h b/pandas/_libs/src/ujson/lib/ultrajson.h
index b40ac9856d6a61..acb66b668e8dc4 100644
--- a/pandas/_libs/src/ujson/lib/ultrajson.h
+++ b/pandas/_libs/src/ujson/lib/ultrajson.h
@@ -108,7 +108,7 @@ typedef uint32_t JSUINT32;
 
 #define FASTCALL_MSVC
 
-#if !defined __x86_64__
+#if !defined __x86_64__ && !defined __aarch64__
 #define FASTCALL_ATTR __attribute__((fastcall))
 #else
 #define FASTCALL_ATTR