From 2308e7c32b95f8704138f885cd436100c134f47a Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 25 Jun 2024 03:40:22 +0200 Subject: [PATCH 1/9] Backport PR #59065: ENH: Fix Python 3.13 test failures & enable CI --- .github/workflows/unit-tests.yml | 4 ++-- pandas/_libs/src/vendored/ujson/python/objToJSON.c | 12 ++++++------ pandas/_libs/tslibs/offsets.pyx | 7 ++++++- pandas/tests/groupby/test_groupby.py | 4 +++- pandas/tests/io/parser/test_dialect.py | 2 +- pandas/tests/io/test_common.py | 5 ++++- pandas/tests/io/xml/test_xml.py | 2 +- pandas/tests/scalar/timedelta/test_arithmetic.py | 1 + 8 files changed, 24 insertions(+), 13 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index c1965fcbd9236..bc23637727e1d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -329,7 +329,7 @@ jobs: # To freeze this file, uncomment out the ``if: false`` condition, and migrate the jobs # to the corresponding posix/windows-macos/sdist etc. workflows. # Feel free to modify this comment as necessary. - if: false # Uncomment this to freeze the workflow, comment it to unfreeze + # if: false # Uncomment this to freeze the workflow, comment it to unfreeze defaults: run: shell: bash -eou pipefail {0} @@ -361,7 +361,7 @@ jobs: - name: Set up Python Dev Version uses: actions/setup-python@v5 with: - python-version: '3.12-dev' + python-version: '3.13-dev' - name: Build Environment run: | diff --git a/pandas/_libs/src/vendored/ujson/python/objToJSON.c b/pandas/_libs/src/vendored/ujson/python/objToJSON.c index fa91db5fe34e3..5f35860c59cb7 100644 --- a/pandas/_libs/src/vendored/ujson/python/objToJSON.c +++ b/pandas/_libs/src/vendored/ujson/python/objToJSON.c @@ -410,8 +410,8 @@ static void NpyArr_iterBegin(JSOBJ _obj, JSONTypeContext *tc) { npyarr->type_num = PyArray_DESCR(obj)->type_num; if (GET_TC(tc)->transpose) { - npyarr->dim = PyArray_DIM(obj, npyarr->ndim); - npyarr->stride = PyArray_STRIDE(obj, npyarr->ndim); + npyarr->dim = PyArray_DIM(obj, (int)npyarr->ndim); + npyarr->stride = PyArray_STRIDE(obj, (int)npyarr->ndim); npyarr->stridedim = npyarr->ndim; npyarr->index[npyarr->ndim] = 0; npyarr->inc = -1; @@ -452,8 +452,8 @@ static void NpyArrPassThru_iterEnd(JSOBJ obj, JSONTypeContext *tc) { return; } const PyArrayObject *arrayobj = (const PyArrayObject *)npyarr->array; - npyarr->dim = PyArray_DIM(arrayobj, npyarr->stridedim); - npyarr->stride = PyArray_STRIDE(arrayobj, npyarr->stridedim); + npyarr->dim = PyArray_DIM(arrayobj, (int)npyarr->stridedim); + npyarr->stride = PyArray_STRIDE(arrayobj, (int)npyarr->stridedim); npyarr->dataptr += npyarr->stride; NpyArr_freeItemValue(obj, tc); @@ -524,8 +524,8 @@ static int NpyArr_iterNext(JSOBJ _obj, JSONTypeContext *tc) { } const PyArrayObject *arrayobj = (const PyArrayObject *)npyarr->array; - npyarr->dim = PyArray_DIM(arrayobj, npyarr->stridedim); - npyarr->stride = PyArray_STRIDE(arrayobj, npyarr->stridedim); + npyarr->dim = PyArray_DIM(arrayobj, (int)npyarr->stridedim); + npyarr->stride = PyArray_STRIDE(arrayobj, (int)npyarr->stridedim); npyarr->index[npyarr->stridedim] = 0; ((PyObjectEncoder *)tc->encoder)->npyCtxtPassthru = npyarr; diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index c37a4b285daef..5dacd7dd55231 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -4960,7 +4960,12 @@ cpdef to_offset(freq, bint is_period=False): if result is None: raise ValueError(INVALID_FREQ_ERR_MSG.format(freq)) - if is_period and not hasattr(result, "_period_dtype_code"): + try: + has_period_dtype_code = hasattr(result, "_period_dtype_code") + except ValueError: + has_period_dtype_code = False + + if is_period and not has_period_dtype_code: if isinstance(freq, str): raise ValueError(f"{result.name} is not supported as period frequency") else: diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index ed9acdd0c9dde..44d6340e55507 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -2816,7 +2816,9 @@ def test_rolling_wrong_param_min_period(): test_df = DataFrame([name_l, val_l]).T test_df.columns = ["name", "val"] - result_error_msg = r"__init__\(\) got an unexpected keyword argument 'min_period'" + result_error_msg = ( + r"^[a-zA-Z._]*\(\) got an unexpected keyword argument 'min_period'" + ) with pytest.raises(TypeError, match=result_error_msg): test_df.groupby("name")["val"].rolling(window=2, min_period=1).sum() diff --git a/pandas/tests/io/parser/test_dialect.py b/pandas/tests/io/parser/test_dialect.py index 7a72e66996d43..803114723bc74 100644 --- a/pandas/tests/io/parser/test_dialect.py +++ b/pandas/tests/io/parser/test_dialect.py @@ -26,7 +26,7 @@ def custom_dialect(): "escapechar": "~", "delimiter": ":", "skipinitialspace": False, - "quotechar": "~", + "quotechar": "`", "quoting": 3, } return dialect_name, dialect_kwargs diff --git a/pandas/tests/io/test_common.py b/pandas/tests/io/test_common.py index 074033868635a..e51f86563081b 100644 --- a/pandas/tests/io/test_common.py +++ b/pandas/tests/io/test_common.py @@ -485,7 +485,10 @@ def test_warning_missing_utf_bom(self, encoding, compression_): df.to_csv(path, compression=compression_, encoding=encoding) # reading should fail (otherwise we wouldn't need the warning) - msg = r"UTF-\d+ stream does not start with BOM" + msg = ( + r"UTF-\d+ stream does not start with BOM|" + r"'utf-\d+' codec can't decode byte" + ) with pytest.raises(UnicodeError, match=msg): pd.read_csv(path, compression=compression_, encoding=encoding) diff --git a/pandas/tests/io/xml/test_xml.py b/pandas/tests/io/xml/test_xml.py index 6f429c1ecbf8a..900734e9f0fdf 100644 --- a/pandas/tests/io/xml/test_xml.py +++ b/pandas/tests/io/xml/test_xml.py @@ -1044,7 +1044,7 @@ def test_utf16_encoding(xml_baby_names, parser): UnicodeError, match=( "UTF-16 stream does not start with BOM|" - "'utf-16-le' codec can't decode byte" + "'utf-16(-le)?' codec can't decode byte" ), ): read_xml(xml_baby_names, encoding="UTF-16", parser=parser) diff --git a/pandas/tests/scalar/timedelta/test_arithmetic.py b/pandas/tests/scalar/timedelta/test_arithmetic.py index 4fc59880c49dd..a4d846f068d00 100644 --- a/pandas/tests/scalar/timedelta/test_arithmetic.py +++ b/pandas/tests/scalar/timedelta/test_arithmetic.py @@ -622,6 +622,7 @@ def test_td_floordiv_invalid_scalar(self): [ r"Invalid dtype datetime64\[D\] for __floordiv__", "'dtype' is an invalid keyword argument for this function", + "this function got an unexpected keyword argument 'dtype'", r"ufunc '?floor_divide'? cannot use operands with types", ] ) From 799ab4997f93d12453f78e10e3f6a9c1e6d0f5bc Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Tue, 30 Apr 2024 13:55:25 -0400 Subject: [PATCH 2/9] Remove deprecated plot_date calls (#58484) * Remove deprecated plot_date calls These were deprecated in Matplotlib 3.9. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> (cherry picked from commit c9bc4809528998313a609ab16168ca237bc186b6) --- pandas/tests/plotting/test_datetimelike.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pandas/tests/plotting/test_datetimelike.py b/pandas/tests/plotting/test_datetimelike.py index 112172656b6ec..6c318402ea226 100644 --- a/pandas/tests/plotting/test_datetimelike.py +++ b/pandas/tests/plotting/test_datetimelike.py @@ -1451,13 +1451,19 @@ def test_mpl_nopandas(self): values1 = np.arange(10.0, 11.0, 0.5) values2 = np.arange(11.0, 12.0, 0.5) - kw = {"fmt": "-", "lw": 4} - _, ax = mpl.pyplot.subplots() - ax.plot_date([x.toordinal() for x in dates], values1, **kw) - ax.plot_date([x.toordinal() for x in dates], values2, **kw) - - line1, line2 = ax.get_lines() + ( + line1, + line2, + ) = ax.plot( + [x.toordinal() for x in dates], + values1, + "-", + [x.toordinal() for x in dates], + values2, + "-", + linewidth=4, + ) exp = np.array([x.toordinal() for x in dates], dtype=np.float64) tm.assert_numpy_array_equal(line1.get_xydata()[:, 0], exp) From 941549b236706e2ff25ad16f654aa4b69d6c2a6d Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 12 Sep 2024 15:55:56 -0400 Subject: [PATCH 3/9] Pick out fastparquet xfails for green CI --- pandas/tests/io/test_parquet.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/test_parquet.py b/pandas/tests/io/test_parquet.py index 8771793672263..d030ec12f882d 100644 --- a/pandas/tests/io/test_parquet.py +++ b/pandas/tests/io/test_parquet.py @@ -1314,7 +1314,9 @@ def test_empty_dataframe(self, fp): expected = df.copy() check_round_trip(df, fp, expected=expected) - @pytest.mark.skipif(using_copy_on_write(), reason="fastparquet writes into Index") + @pytest.mark.xfail( + reason="fastparquet bug, see https://github.com/dask/fastparquet/issues/929" + ) def test_timezone_aware_index(self, fp, timezone_aware_date_list): idx = 5 * [timezone_aware_date_list] From 3ec3c5e75d7e04557f8c9f1a73dcff44c25657be Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Fri, 13 Sep 2024 15:14:01 -0400 Subject: [PATCH 4/9] pin pytz to fix test_arrays.py --- ci/deps/actions-310.yaml | 3 ++- ci/deps/actions-311-downstream_compat.yaml | 3 ++- ci/deps/actions-311-numpydev.yaml | 3 ++- ci/deps/actions-311-pyarrownightly.yaml | 3 ++- ci/deps/actions-311.yaml | 3 ++- ci/deps/actions-312.yaml | 1 + ci/deps/actions-39.yaml | 3 ++- ci/deps/actions-pypy-39.yaml | 1 + ci/deps/circle-310-arm64.yaml | 1 + 9 files changed, 15 insertions(+), 6 deletions(-) diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index a3e44e6373145..d0e788d1b124f 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -20,7 +20,8 @@ dependencies: # required dependencies - python-dateutil - numpy - - pytz + # pytz 2024.2 timezones cause wrong results + - pytz<2024.2 # optional dependencies - beautifulsoup4>=4.11.2 diff --git a/ci/deps/actions-311-downstream_compat.yaml b/ci/deps/actions-311-downstream_compat.yaml index d6bf9ec7843de..7fda383dd9e1d 100644 --- a/ci/deps/actions-311-downstream_compat.yaml +++ b/ci/deps/actions-311-downstream_compat.yaml @@ -22,7 +22,8 @@ dependencies: # required dependencies - python-dateutil - numpy - - pytz + # pytz 2024.2 timezones cause wrong results + - pytz<2024.2 # optional dependencies - beautifulsoup4>=4.11.2 diff --git a/ci/deps/actions-311-numpydev.yaml b/ci/deps/actions-311-numpydev.yaml index b62e8630f2059..21791e3a9c2eb 100644 --- a/ci/deps/actions-311-numpydev.yaml +++ b/ci/deps/actions-311-numpydev.yaml @@ -21,7 +21,8 @@ dependencies: # pandas dependencies - python-dateutil - - pytz + # pytz 2024.2 timezones cause wrong results + - pytz<2024.2 - pip - pip: diff --git a/ci/deps/actions-311-pyarrownightly.yaml b/ci/deps/actions-311-pyarrownightly.yaml index 5455b9b84b034..b90fa2e044cd6 100644 --- a/ci/deps/actions-311-pyarrownightly.yaml +++ b/ci/deps/actions-311-pyarrownightly.yaml @@ -19,7 +19,8 @@ dependencies: # required dependencies - python-dateutil - numpy<2 - - pytz + # pytz 2024.2 timezones cause wrong results + - pytz<2024.2 - pip - pip: diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml index 95cd1a4d46ef4..c72d743bf3375 100644 --- a/ci/deps/actions-311.yaml +++ b/ci/deps/actions-311.yaml @@ -20,7 +20,8 @@ dependencies: # required dependencies - python-dateutil - numpy - - pytz + # pytz 2024.2 timezones cause wrong results + - pytz<2024.2 # optional dependencies - beautifulsoup4>=4.11.2 diff --git a/ci/deps/actions-312.yaml b/ci/deps/actions-312.yaml index a442ed6feeb5d..9612bc0601290 100644 --- a/ci/deps/actions-312.yaml +++ b/ci/deps/actions-312.yaml @@ -20,6 +20,7 @@ dependencies: # required dependencies - python-dateutil - numpy + # pytz 2024.2 timezones cause wrong results - pytz # optional dependencies diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index b162a78e7f115..4320e9060fb4a 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -20,7 +20,8 @@ dependencies: # required dependencies - python-dateutil - numpy - - pytz + # pytz 2024.2 timezones cause wrong results + - pytz<2024.2 # optional dependencies - beautifulsoup4>=4.11.2 diff --git a/ci/deps/actions-pypy-39.yaml b/ci/deps/actions-pypy-39.yaml index d9c8dd81b7c33..bdc07931988d1 100644 --- a/ci/deps/actions-pypy-39.yaml +++ b/ci/deps/actions-pypy-39.yaml @@ -22,6 +22,7 @@ dependencies: # required - numpy - python-dateutil + # pytz 2024.2 timezones cause wrong results - pytz - pip: - tzdata>=2022.7 diff --git a/ci/deps/circle-310-arm64.yaml b/ci/deps/circle-310-arm64.yaml index a19ffd485262d..cc2bf9877ab7f 100644 --- a/ci/deps/circle-310-arm64.yaml +++ b/ci/deps/circle-310-arm64.yaml @@ -21,6 +21,7 @@ dependencies: # required dependencies - python-dateutil - numpy + # pytz 2024.2 timezones cause wrong results - pytz # optional dependencies From 7a967fc56c0bad6ab465a91e3978e5c2d36f7164 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Fri, 13 Sep 2024 15:18:19 -0400 Subject: [PATCH 5/9] more workflow tweaks for pytz and Python 3.13 --- .circleci/config.yml | 8 ++++---- .github/workflows/unit-tests.yml | 6 +++--- .github/workflows/wheels.yml | 6 +++--- ci/deps/actions-312.yaml | 2 +- pyproject.toml | 14 +++++--------- 5 files changed, 16 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0748d6550fe2d..9ef3f9e2857a0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,10 +14,10 @@ jobs: steps: - checkout - run: .circleci/setup_env.sh - - run: > - PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH - LD_PRELOAD=$HOME/miniconda3/envs/pandas-dev/lib/libgomp.so.1:$LD_PRELOAD + - run: | sudo apt-get update && sudo apt-get install -y libegl1 libopengl0 + PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH \ + LD_PRELOAD=$HOME/miniconda3/envs/pandas-dev/lib/libgomp.so.1:$LD_PRELOAD \ ci/run_tests.sh linux-musl: docker: @@ -35,7 +35,7 @@ jobs: /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev . ~/virtualenvs/pandas-dev/bin/activate python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1 - python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1 + python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil "pytz<2024.2" pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1 python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror" python -m pip list --no-cache-dir - run: | diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index bc23637727e1d..ad63908e4682d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -257,7 +257,7 @@ jobs: . ~/virtualenvs/pandas-dev/bin/activate python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1 python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true" - python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1 + python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil "pytz<2024.2" pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1 python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror" python -m pip list --no-cache-dir export PANDAS_CI=1 @@ -295,7 +295,7 @@ jobs: /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev . ~/virtualenvs/pandas-dev/bin/activate python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1 - python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1 + python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil "pytz<2024.2" pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1 python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror" python -m pip list --no-cache-dir @@ -369,7 +369,7 @@ jobs: python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1 python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy python -m pip install versioneer[toml] - python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov + python -m pip install python-dateutil "pytz<2024.2" tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov python -m pip install -ve . --no-build-isolation --no-index --no-deps --config-settings=setup-args="--werror" python -m pip list diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 4bd9068e91b67..3d4fbfb995fb3 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -170,13 +170,13 @@ jobs: shell: pwsh run: | $TST_CMD = @" - python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0; + python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytz<2024.2; python -m pip install `$(Get-Item pandas\wheelhouse\*.whl); python -c `'import pandas as pd; pd.test(extra_args=[`\"--no-strict-data-files`\", `\"-m not clipboard and not single_cpu and not slow and not network and not db`\"])`'; "@ # add rc to the end of the image name if the Python version is unreleased - docker pull python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }} - docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }} powershell -Command $TST_CMD + docker pull python:${{ matrix.python[1] == '3.13' && '3.13-rc' || format('{0}-windowsservercore', matrix.python[1]) }} + docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] == '3.13' && '3.13-rc' || format('{0}-windowsservercore', matrix.python[1]) }} powershell -Command $TST_CMD - uses: actions/upload-artifact@v4 with: diff --git a/ci/deps/actions-312.yaml b/ci/deps/actions-312.yaml index 9612bc0601290..032bd68c09ad6 100644 --- a/ci/deps/actions-312.yaml +++ b/ci/deps/actions-312.yaml @@ -21,7 +21,7 @@ dependencies: - python-dateutil - numpy # pytz 2024.2 timezones cause wrong results - - pytz + - pytz<2024.2 # optional dependencies - beautifulsoup4>=4.11.2 diff --git a/pyproject.toml b/pyproject.toml index db9f055799ab0..2b1490e26d9ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,9 +6,9 @@ requires = [ "meson==1.2.1", "wheel", "Cython==3.0.5", # Note: sync with setup.py, environment.yml and asv.conf.json - # Force numpy higher than 2.0rc1, so that built wheels are compatible + # Force numpy higher than 2.0, so that built wheels are compatible # with both numpy 1 and 2 - "numpy>=2.0.0rc1", + "numpy>=2.0", "versioneer[toml]" ] @@ -153,10 +153,8 @@ setup = ['--vsenv'] # For Windows skip = "cp36-* cp37-* cp38-* pp* *_i686 *_ppc64le *_s390x" build-verbosity = "3" environment = {LDFLAGS="-Wl,--strip-all"} -# TODO: remove this once numpy 2.0 proper releases -# and specify numpy 2.0 as a dependency in [build-system] requires in pyproject.toml -before-build = "pip install numpy==2.0.0rc1" -test-requires = "hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0" +# pytz 2024.2 causing some failures +test-requires = "hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytz<2024.2" test-command = """ PANDAS_CI='1' python -c 'import pandas as pd; \ pd.test(extra_args=["-m not clipboard and not single_cpu and not slow and not network and not db", "-n 2", "--no-strict-data-files"]); \ @@ -164,9 +162,7 @@ test-command = """ """ [tool.cibuildwheel.windows] -# TODO: remove this once numpy 2.0 proper releases -# and specify numpy 2.0 as a dependency in [build-system] requires in pyproject.toml -before-build = "pip install delvewheel numpy==2.0.0rc1" +before-build = "pip install delvewheel" repair-wheel-command = "delvewheel repair -w {dest_dir} {wheel}" [[tool.cibuildwheel.overrides]] From 679e45ee4d9b30417f5403d504eac588623930c9 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 14 Sep 2024 08:20:30 -0400 Subject: [PATCH 6/9] fix typing and tune tests for copy on write --- pandas/io/gbq.py | 6 +++--- pandas/tests/io/test_parquet.py | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index 350002bf461ff..24e4e0b7cef0a 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -11,7 +11,7 @@ from pandas.util._exceptions import find_stack_level if TYPE_CHECKING: - import google.auth + from google.auth.credentials import Credentials from pandas import DataFrame @@ -37,7 +37,7 @@ def read_gbq( dialect: str | None = None, location: str | None = None, configuration: dict[str, Any] | None = None, - credentials: google.auth.credentials.Credentials | None = None, + credentials: Credentials | None = None, use_bqstorage_api: bool | None = None, max_results: int | None = None, progress_bar_type: str | None = None, @@ -230,7 +230,7 @@ def to_gbq( table_schema: list[dict[str, str]] | None = None, location: str | None = None, progress_bar: bool = True, - credentials: google.auth.credentials.Credentials | None = None, + credentials: Credentials | None = None, ) -> None: warnings.warn( "to_gbq is deprecated and will be removed in a future version. " diff --git a/pandas/tests/io/test_parquet.py b/pandas/tests/io/test_parquet.py index d030ec12f882d..fd3b74dfa1a17 100644 --- a/pandas/tests/io/test_parquet.py +++ b/pandas/tests/io/test_parquet.py @@ -1315,7 +1315,10 @@ def test_empty_dataframe(self, fp): check_round_trip(df, fp, expected=expected) @pytest.mark.xfail( - reason="fastparquet bug, see https://github.com/dask/fastparquet/issues/929" + not using_copy_on_write() + and _HAVE_FASTPARQUET + and Version(fastparquet.__version__) > Version("2022.12"), + reason="fastparquet bug, see https://github.com/dask/fastparquet/issues/929", ) def test_timezone_aware_index(self, fp, timezone_aware_date_list): idx = 5 * [timezone_aware_date_list] From c1a448ba28f538f72b9d47751d59facbeb24268f Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 18 Sep 2024 07:28:06 -0400 Subject: [PATCH 7/9] remove WASM stuff --- pandas/tests/series/indexing/test_setitem.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py index 29ad674d1cadf..182de980d311d 100644 --- a/pandas/tests/series/indexing/test_setitem.py +++ b/pandas/tests/series/indexing/test_setitem.py @@ -8,7 +8,6 @@ import numpy as np import pytest -from pandas.compat import WASM from pandas.compat.numpy import np_version_gte1p24 from pandas.errors import IndexingError @@ -1449,7 +1448,6 @@ def obj(self): np_version_gte1p24 and os.environ.get("NPY_PROMOTION_STATE", "weak") != "weak" ) - or WASM ), reason="np.float32(1.1) ends up as 1.100000023841858, so " "np_can_hold_element raises and we cast to float64", From e3ef4c662bd8ddf8616a39f48171e88f4c9d275f Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 18 Sep 2024 07:37:28 -0400 Subject: [PATCH 8/9] more arm skips --- pandas/tests/indexes/test_common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/indexes/test_common.py b/pandas/tests/indexes/test_common.py index 80c39322b9b81..05b2aa584674c 100644 --- a/pandas/tests/indexes/test_common.py +++ b/pandas/tests/indexes/test_common.py @@ -452,6 +452,7 @@ def test_sort_values_invalid_na_position(index_with_missing, na_position): index_with_missing.sort_values(na_position=na_position) +@pytest.mark.fails_arm_wheels @pytest.mark.filterwarnings(r"ignore:PeriodDtype\[B\] is deprecated:FutureWarning") @pytest.mark.parametrize("na_position", ["first", "last"]) def test_sort_values_with_missing(index_with_missing, na_position, request): From e459de63de63d4f5d79f9de5d87f727f027d9089 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:28:25 -0400 Subject: [PATCH 9/9] go for green --- ci/deps/circle-310-arm64.yaml | 2 +- .../tests/indexes/interval/test_interval_tree.py | 1 - pandas/tests/indexing/interval/test_interval.py | 2 -- .../tests/indexing/interval/test_interval_new.py | 3 --- pandas/tests/io/test_parquet.py | 14 ++------------ pandas/tests/series/indexing/test_setitem.py | 8 ++++++-- 6 files changed, 9 insertions(+), 21 deletions(-) diff --git a/ci/deps/circle-310-arm64.yaml b/ci/deps/circle-310-arm64.yaml index cc2bf9877ab7f..36c584bf1fd10 100644 --- a/ci/deps/circle-310-arm64.yaml +++ b/ci/deps/circle-310-arm64.yaml @@ -22,7 +22,7 @@ dependencies: - python-dateutil - numpy # pytz 2024.2 timezones cause wrong results - - pytz + - pytz < 2024.2 # optional dependencies - beautifulsoup4>=4.11.2 diff --git a/pandas/tests/indexes/interval/test_interval_tree.py b/pandas/tests/indexes/interval/test_interval_tree.py index 45b25f2533afd..78388e84fc6dc 100644 --- a/pandas/tests/indexes/interval/test_interval_tree.py +++ b/pandas/tests/indexes/interval/test_interval_tree.py @@ -190,7 +190,6 @@ def test_construction_overflow(self): expected = (50 + np.iinfo(np.int64).max) / 2 assert result == expected - @pytest.mark.xfail(not IS64, reason="GH 23440") @pytest.mark.parametrize( "left, right, expected", [ diff --git a/pandas/tests/indexing/interval/test_interval.py b/pandas/tests/indexing/interval/test_interval.py index cabfee9aa040a..dd51917b85a59 100644 --- a/pandas/tests/indexing/interval/test_interval.py +++ b/pandas/tests/indexing/interval/test_interval.py @@ -2,7 +2,6 @@ import pytest from pandas._libs import index as libindex -from pandas.compat import IS64 import pandas as pd from pandas import ( @@ -210,7 +209,6 @@ def test_mi_intervalindex_slicing_with_scalar(self): expected = Series([1, 6, 2, 8, 7], index=expected_index, name="value") tm.assert_series_equal(result, expected) - @pytest.mark.xfail(not IS64, reason="GH 23440") @pytest.mark.parametrize( "base", [101, 1010], diff --git a/pandas/tests/indexing/interval/test_interval_new.py b/pandas/tests/indexing/interval/test_interval_new.py index 283921a23e368..018db5846f4e2 100644 --- a/pandas/tests/indexing/interval/test_interval_new.py +++ b/pandas/tests/indexing/interval/test_interval_new.py @@ -3,8 +3,6 @@ import numpy as np import pytest -from pandas.compat import IS64 - from pandas import ( Index, Interval, @@ -211,7 +209,6 @@ def test_loc_getitem_missing_key_error_message( obj.loc[[4, 5, 6]] -@pytest.mark.xfail(not IS64, reason="GH 23440") @pytest.mark.parametrize( "intervals", [ diff --git a/pandas/tests/io/test_parquet.py b/pandas/tests/io/test_parquet.py index fd3b74dfa1a17..760a64c8d4c33 100644 --- a/pandas/tests/io/test_parquet.py +++ b/pandas/tests/io/test_parquet.py @@ -16,7 +16,6 @@ pa_version_under11p0, pa_version_under13p0, pa_version_under15p0, - pa_version_under17p0, ) import pandas as pd @@ -449,12 +448,8 @@ def test_read_filters(self, engine, tmp_path): repeat=1, ) - def test_write_index(self, engine, using_copy_on_write, request): + def test_write_index(self, engine): check_names = engine != "fastparquet" - if using_copy_on_write and engine == "fastparquet": - request.applymarker( - pytest.mark.xfail(reason="fastparquet write into index") - ) df = pd.DataFrame({"A": [1, 2, 3]}) check_round_trip(df, engine) @@ -1064,9 +1059,6 @@ def test_read_dtype_backend_pyarrow_config_index(self, pa): expected=expected, ) - @pytest.mark.xfail( - pa_version_under17p0, reason="pa.pandas_compat passes 'datetime64' to .astype" - ) def test_columns_dtypes_not_invalid(self, pa): df = pd.DataFrame({"string": list("abc"), "int": list(range(1, 4))}) @@ -1315,9 +1307,7 @@ def test_empty_dataframe(self, fp): check_round_trip(df, fp, expected=expected) @pytest.mark.xfail( - not using_copy_on_write() - and _HAVE_FASTPARQUET - and Version(fastparquet.__version__) > Version("2022.12"), + _HAVE_FASTPARQUET and Version(fastparquet.__version__) > Version("2022.12"), reason="fastparquet bug, see https://github.com/dask/fastparquet/issues/929", ) def test_timezone_aware_index(self, fp, timezone_aware_date_list): diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py index 182de980d311d..ed681563f6fcd 100644 --- a/pandas/tests/series/indexing/test_setitem.py +++ b/pandas/tests/series/indexing/test_setitem.py @@ -8,7 +8,10 @@ import numpy as np import pytest -from pandas.compat.numpy import np_version_gte1p24 +from pandas.compat.numpy import ( + np_version_gt2, + np_version_gte1p24, +) from pandas.errors import IndexingError from pandas.core.dtypes.common import is_list_like @@ -1446,7 +1449,8 @@ def obj(self): not np_version_gte1p24 or ( np_version_gte1p24 - and os.environ.get("NPY_PROMOTION_STATE", "weak") != "weak" + and not np_version_gt2 + and os.environ.get("NPY_PROMOTION_STATE", "legacy") != "weak" ) ), reason="np.float32(1.1) ends up as 1.100000023841858, so "