BLD: Split out tests into pandas_tests package

.circleci/config.yml

@@ -9,7 +9,7 @@ jobs:
       ENV_FILE: ci/deps/circle-310-arm64.yaml
       PYTEST_WORKERS: auto
       PATTERN: "not single_cpu and not slow and not network and not clipboard and not arm_slow and not db"
-      PYTEST_TARGET: "pandas"
+      PYTEST_TARGET: "pandas/tests"
       PANDAS_CI: "1"
     steps:
       - checkout
@@ -40,7 +40,7 @@ jobs:
       - run: |
           . ~/virtualenvs/pandas-dev/bin/activate
           export PANDAS_CI=1
-          python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml
+          python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas/tests --junitxml=test-data.xml
   build-aarch64:
     parameters:
       cibw-build:
@@ -67,6 +67,28 @@ jobs:
             elif ! (curl https://api.github.com/repos/pandas-dev/pandas/issues/$CIRCLE_PR_NUMBER | jq '.labels' | grep -q 'Build'); then
                 circleci-agent step halt
             fi
+      - run:
+          # The process here is more complicated than on Github Actions
+          # since we need to copy the tests wheel into the unzipped sdist
+          # so that it gets copied (along with the contents of the unzipped sdist)
+          # into the Linux docker image used for building the wheel
+          # (unlike Github Actions there is no host access using cibuildwheel with CircleCI)
+          name: Build the sdist
+          command: |
+            pip3 install build setuptools-scm wheel
+            python3 -m build --sdist
+            sdist_name=$(ls ./dist/*.tar.gz)
+            mkdir unzipped-sdist
+            tar -xzf $sdist_name -C unzipped-sdist --strip-components=1
+            cd pandas
+            # Have to disable isolation otherwise
+            # we won't be able to pull the version correctly
+            python3 -m build --wheel --no-isolation --outdir ../unzipped-sdist
+            cd ..
+            echo "sdist_name=./unzipped-sdist" >> "$BASH_ENV"
+            echo "pandas_tests_loc=$(ls ./unzipped-sdist/*.whl)" >> "$BASH_ENV"
+            ls ./unzipped-sdist/*.whl
+
       - run:
           name: Build aarch64 wheels
           no_output_timeout: 30m # Sometimes the tests won't generate any output, make sure the job doesn't get killed by that
@@ -76,7 +98,8 @@ jobs:
             if [[ "$IS_SCHEDULE_DISPATCH" == "true" || "$IS_PUSH" != 'true' ]]; then
                 export CIBW_ENVIRONMENT="PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
             fi
-            cibuildwheel --prerelease-pythons --output-dir wheelhouse
+            export CIBW_BEFORE_TEST="pip install {project}/${pandas_tests_loc}"
+            cibuildwheel ${sdist_name} --prerelease-pythons --output-dir wheelhouse
 
           environment:
             CIBW_BUILD: << parameters.cibw-build >>

.gitattributes

@@ -77,8 +77,5 @@ Dockerfile export-ignore
 environment.yml export-ignore
 setup.py export-ignore
 
-
-# GH 39321
-# csv_dir_path fixture checks the existence of the directory
-# exclude the whole directory to avoid running related tests in sdist
-pandas/tests/io/parser/data export-ignore
+# Strip tests from the distribution
+pandas/tests export-ignore

.github/workflows/unit-tests.yml

@@ -77,7 +77,7 @@ jobs:
       PANDAS_CI: ${{ matrix.pandas_ci || '1' }}
       TEST_ARGS: ${{ matrix.test_args || '' }}
       PYTEST_WORKERS: ${{ matrix.pytest_workers || 'auto' }}
-      PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
+      PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas/tests' }}
       NPY_PROMOTION_STATE: ${{ matrix.env_file == 'actions-311-numpydev.yaml' && 'weak' || 'legacy' }}
       # Clipboard tests
       QT_QPA_PLATFORM: offscreen
@@ -180,7 +180,7 @@ jobs:
       cancel-in-progress: true
     env:
       PANDAS_CI: 1
-      PYTEST_TARGET: pandas
+      PYTEST_TARGET: pandas/tests
       PATTERN: "not slow and not db and not network and not single_cpu"
       PYTEST_WORKERS: 'auto'
 
@@ -232,7 +232,7 @@ jobs:
           python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
           python -m pip list --no-cache-dir
           export PANDAS_CI=1
-          python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml
+          python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas/tests --junitxml=test-data.xml
     concurrency:
       # https://git.luolix.topmunity/t/concurrecy-not-work-for-push/183068/7
       group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-32bit
@@ -274,7 +274,7 @@ jobs:
         run: |
           . ~/virtualenvs/pandas-dev/bin/activate
           export PANDAS_CI=1
-          python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml
+          python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas/tests --junitxml=test-data.xml
     concurrency:
       # https://git.luolix.topmunity/t/concurrecy-not-work-for-push/183068/7
       group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-musl
@@ -322,7 +322,7 @@ jobs:
       PYTEST_WORKERS: "auto"
       PANDAS_CI: 1
       PATTERN: "not slow and not network and not clipboard and not single_cpu"
-      PYTEST_TARGET: pandas
+      PYTEST_TARGET: pandas/tests
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/wheels.yml

@@ -46,6 +46,7 @@ jobs:
       IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
     outputs:
       sdist_file: ${{ steps.save-path.outputs.sdist_name }}
+      pandas_tests_loc: ${{ steps.save-tests-path.outputs.pandas_tests_loc }}
     steps:
       - name: Checkout pandas
         uses: actions/checkout@v4
@@ -59,22 +60,34 @@ jobs:
 
       - name: Build sdist
         run: |
-          python -m pip install build
+          python -m pip install build wheel setuptools-scm
           python -m build --sdist
 
+      - name: Build pandas_tests
+        run: |
+          cd pandas
+          # we want to place the pandas_tests wheel in the same
+          # dist directory as the sdist
+          python -m build --wheel --outdir ../dist --no-isolation
+
       - uses: actions/upload-artifact@v4
         with:
-          name: sdist
+          name: sdist-and-tests
           path: ./dist/*
 
-      - name: Sanity check sdist files
+      - name: Sanity check built files
         run: |
           ls ./dist
 
       - name: Output sdist name
         id: save-path
         shell: bash -el {0}
-        run: echo "sdist_name=$(ls ./dist)" >> "$GITHUB_OUTPUT"
+        run: echo "sdist_name=$(ls ./dist/*.tar.gz)" >> "$GITHUB_OUTPUT"
+
+      - name: Output pandas_tests location
+        id: save-tests-path
+        shell: bash -el {0}
+        run: echo "pandas_tests_loc=$(ls ./dist/*.whl)" >> "$GITHUB_OUTPUT"
 
   build_wheels:
     needs: build_sdist
@@ -109,52 +122,48 @@ jobs:
         with:
           fetch-depth: 0
 
-      # TODO: Build wheels from sdist again
-      # There's some sort of weird race condition?
-      # within Github that makes the sdist be missing files
-
-      # We need to build wheels from the sdist since the sdist
-      # removes unnecessary files from the release
-      - name: Download sdist (not macOS)
-        #if: ${{ matrix.buildplat[1] != 'macosx_*' }}
+      - name: Download sdist and pandas_tests
         uses: actions/download-artifact@v4
         with:
-          name: sdist
+          name: sdist-and-tests
           path: ./dist
 
-      - name: Output sdist name (macOS)
-        id: save-path
-        shell: bash -el {0}
-        run: echo "sdist_name=$(ls ./dist)" >> "$GITHUB_ENV"
-
-      # Python version used to build sdist doesn't matter
-      # wheel will be built from sdist with the correct version
+      # The sdist will be corrupted by cibuildwheel/macOS sometimes somehow
+      # (I think this is a Github Actions bug)
       - name: Unzip sdist (macOS)
         if: ${{ startsWith(matrix.buildplat[1], 'macosx') }}
         run: |
-          tar -xzf ./dist/${{ env.sdist_name }} -C ./dist
+          tar -xzf ${{ needs.build_sdist.outputs.sdist_file }} -C ./dist
 
-      - name: Output sdist name (macOS)
+      - name: Output new sdist name (macOS)
         id: save-path2
+        if: ${{ matrix.buildplat[1] == 'macosx_*' }}
         shell: bash -el {0}
-        run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
+        run: echo "sdist_name=./dist/$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
       - name: Build normal wheels
         if: ${{ (env.IS_SCHEDULE_DISPATCH != 'true' || env.IS_PUSH == 'true') }}
         uses: pypa/cibuildwheel@v2.17.0
         with:
-         package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
+         package-dir: ${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:
           CIBW_PRERELEASE_PYTHONS: True
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
+          # Note: Since the sdist is the project directory, it does not contain the tests
+          # We need to manually install tests from the host. This isn't ideal since it breaks
+          # isolation but should be no big deal
+          # (On Linux, need to prefix with /host, since we run builds in a container and the root directory is
+          # mounted to host)
+          CIBW_BEFORE_TEST: >
+            pip install ${{ startsWith(matrix.buildplat[0], 'ubuntu') && '/host'|| '' }}${{ github.workspace }}/${{ needs.build_sdist.outputs.pandas_tests_loc }}
 
       - name: Build nightly wheels (with NumPy pre-release)
         if: ${{ (env.IS_SCHEDULE_DISPATCH == 'true' && env.IS_PUSH != 'true') }}
         uses: pypa/cibuildwheel@v2.17.0
         with:
          package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:
-          # The nightly wheels should be build witht he NumPy 2.0 pre-releases
+          # The nightly wheels should be build with the NumPy 2.0 pre-releases
           # which requires the additional URL.
           CIBW_ENVIRONMENT: PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
           CIBW_PRERELEASE_PYTHONS: True
@@ -185,13 +194,15 @@ jobs:
           $TST_CMD = @"
           python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0;
           python -m pip install `$(Get-Item pandas\wheelhouse\*.whl);
+          python -m pip install `$(Get-Item pandas\dist\pandas_tests*.whl);
           python -c `'import pandas as pd; pd.test(extra_args=[`\"--no-strict-data-files`\", `\"-m not clipboard and not single_cpu and not slow and not network and not db`\"])`';
           "@
           # add rc to the end of the image name if the Python version is unreleased
           docker pull python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }}
           docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }} powershell -Command $TST_CMD
 
       - uses: actions/upload-artifact@v4
+        if: always()
         with:
           name: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
           path: ./wheelhouse/*.whl

MANIFEST.in

@@ -54,10 +54,9 @@ global-exclude *.h
 global-exclude *.py[ocd]
 global-exclude *.pxi
 
-# GH 39321
-# csv_dir_path fixture checks the existence of the directory
-# exclude the whole directory to avoid running related tests in sdist
-prune pandas/tests/io/parser/data
+# Remove tests
+# Those will be distributed in a separate package
+prune pandas/tests
 
 # Selectively re-add *.cxx files that were excluded above
 graft pandas/_libs/src

meson.build

@@ -46,7 +46,7 @@ endif
 
 # Needed by pandas.test() when it looks for the pytest ini options
 py.install_sources(
-    'pyproject.toml',
+    'pandas/pyproject.toml',
     subdir: 'pandas'
 )
 

pandas/__init__.py

@@ -189,6 +189,19 @@
     __git_version__ = v.get("full-revisionid")
     del get_versions, v
 
+import sys
+
+try:
+    import pandas_tests  # pyright: ignore [reportMissingImports]
+
+    sys.modules["pandas.tests"] = pandas_tests
+
+    del pandas_tests
+except ImportError:
+    pass
+
+del sys
+
 
 # module level doc-string
 __doc__ = """