diff --git a/.github/workflows/testing_ci.yml b/.github/workflows/testing_ci.yml
index 16ecb5d2..0026bd01 100644
--- a/.github/workflows/testing_ci.yml
+++ b/.github/workflows/testing_ci.yml
@@ -49,8 +49,6 @@ jobs:
             - name: Install other dependencies
               run: |
                   pip install -r requirements/requirements.txt
-                  pip install numpy==1.24  # many libs not compatible with numpy 2.0. Note 3.12 requests for numpy>=2.0
-                  pip install pandas==1.5  # fix pandas version to avoid installing pandas 2.0, the same reason with numpy
 
             - name: Test building package
               # we need to know if the package can be built successfully without optional dependencies
@@ -61,6 +59,8 @@ jobs:
             - name: Continue to install torch-geometric dependencies
               run: |
                   pip install torch-geometric torch-scatter torch-sparse -f "https://data.pyg.org/whl/torch-${{ matrix.pytorch-version }}+cpu.html"
+                  pip install numpy==1.24  # many libs not compatible with numpy 2.0. Note 3.12 requests for numpy>=2.0
+                  pip install pandas==1.5  # fix pandas version to avoid installing pandas 2.0, the same reason with numpy
                   pip install pypots[dev]
                   python_site_path=`python -c "import site; print(site.getsitepackages()[0])"`
                   echo "python site-packages path: $python_site_path"