diff --git a/Dockerfile b/Dockerfile index 03f76f39b8cc7..0fcbcee92295c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,4 +11,5 @@ RUN apt-get install -y libhdf5-dev libgles2-mesa-dev RUN python -m pip install --upgrade pip COPY requirements-dev.txt /tmp RUN python -m pip install -r /tmp/requirements-dev.txt +RUN git config --global --add safe.directory /home/pandas CMD ["/bin/bash"] diff --git a/asv_bench/benchmarks/categoricals.py b/asv_bench/benchmarks/categoricals.py index 1716110b619d6..69697906e493e 100644 --- a/asv_bench/benchmarks/categoricals.py +++ b/asv_bench/benchmarks/categoricals.py @@ -88,7 +88,7 @@ def setup(self): ) for col in ("int", "float", "timestamp"): - self.df[col + "_as_str"] = self.df[col].astype(str) + self.df[f"{col}_as_str"] = self.df[col].astype(str) for col in self.df.columns: self.df[col] = self.df[col].astype("category") diff --git a/asv_bench/benchmarks/join_merge.py b/asv_bench/benchmarks/join_merge.py index ce64304731116..a6c6990892d38 100644 --- a/asv_bench/benchmarks/join_merge.py +++ b/asv_bench/benchmarks/join_merge.py @@ -328,6 +328,23 @@ def time_i8merge(self, how): merge(self.left, self.right, how=how) +class UniqueMerge: + params = [4_000_000, 1_000_000] + param_names = ["unique_elements"] + + def setup(self, unique_elements): + N = 1_000_000 + self.left = DataFrame({"a": np.random.randint(1, unique_elements, (N,))}) + self.right = DataFrame({"a": np.random.randint(1, unique_elements, (N,))}) + uniques = self.right.a.drop_duplicates() + self.right["a"] = concat( + [uniques, Series(np.arange(0, -(N - len(uniques)), -1))], ignore_index=True + ) + + def time_unique_merge(self, unique_elements): + merge(self.left, self.right, how="inner") + + class MergeDatetime: params = [ [ diff --git a/asv_bench/benchmarks/timeseries.py b/asv_bench/benchmarks/timeseries.py index 8e1deb99a66a4..06f488f7baaaf 100644 --- a/asv_bench/benchmarks/timeseries.py +++ b/asv_bench/benchmarks/timeseries.py @@ -183,7 +183,7 @@ def setup(self): self.dt_ts = Series(5, rng3, dtype="datetime64[ns]") def time_resample(self): - self.dt_ts.resample("1S").last() + self.dt_ts.resample("1s").last() class AsOf: diff --git a/ci/code_checks.sh b/ci/code_checks.sh index a9967dcb8efe6..0c4e6641444f1 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -504,7 +504,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then -i "pandas.Timedelta.to_timedelta64 SA01" \ -i "pandas.Timedelta.total_seconds SA01" \ -i "pandas.Timedelta.view SA01" \ - -i "pandas.TimedeltaIndex PR01" \ -i "pandas.TimedeltaIndex.as_unit RT03,SA01" \ -i "pandas.TimedeltaIndex.ceil SA01" \ -i "pandas.TimedeltaIndex.components SA01" \ @@ -797,8 +796,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then -i "pandas.io.formats.style.Styler.clear SA01" \ -i "pandas.io.formats.style.Styler.concat RT03,SA01" \ -i "pandas.io.formats.style.Styler.export RT03" \ - -i "pandas.io.formats.style.Styler.format RT03" \ - -i "pandas.io.formats.style.Styler.format_index RT03" \ -i "pandas.io.formats.style.Styler.from_custom_template SA01" \ -i "pandas.io.formats.style.Styler.hide RT03,SA01" \ -i "pandas.io.formats.style.Styler.highlight_between RT03" \ @@ -808,7 +805,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then -i "pandas.io.formats.style.Styler.highlight_quantile RT03" \ -i "pandas.io.formats.style.Styler.map RT03" \ -i "pandas.io.formats.style.Styler.map_index RT03" \ - -i "pandas.io.formats.style.Styler.relabel_index RT03" \ -i "pandas.io.formats.style.Styler.set_caption RT03,SA01" \ -i "pandas.io.formats.style.Styler.set_properties RT03,SA01" \ -i "pandas.io.formats.style.Styler.set_sticky RT03,SA01" \ @@ -1022,7 +1018,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then -i "pandas.tseries.offsets.DateOffset.rule_code GL08" \ -i "pandas.tseries.offsets.Day PR02" \ -i "pandas.tseries.offsets.Day.copy SA01" \ - -i "pandas.tseries.offsets.Day.delta GL08" \ -i "pandas.tseries.offsets.Day.freqstr SA01" \ -i "pandas.tseries.offsets.Day.is_on_offset GL08" \ -i "pandas.tseries.offsets.Day.kwds SA01" \ @@ -1075,7 +1070,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then -i "pandas.tseries.offsets.FY5253Quarter.year_has_extra_week GL08" \ -i "pandas.tseries.offsets.Hour PR02" \ -i "pandas.tseries.offsets.Hour.copy SA01" \ - -i "pandas.tseries.offsets.Hour.delta GL08" \ -i "pandas.tseries.offsets.Hour.freqstr SA01" \ -i "pandas.tseries.offsets.Hour.is_on_offset GL08" \ -i "pandas.tseries.offsets.Hour.kwds SA01" \ @@ -1098,7 +1092,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then -i "pandas.tseries.offsets.LastWeekOfMonth.weekday GL08" \ -i "pandas.tseries.offsets.Micro PR02" \ -i "pandas.tseries.offsets.Micro.copy SA01" \ - -i "pandas.tseries.offsets.Micro.delta GL08" \ -i "pandas.tseries.offsets.Micro.freqstr SA01" \ -i "pandas.tseries.offsets.Micro.is_on_offset GL08" \ -i "pandas.tseries.offsets.Micro.kwds SA01" \ @@ -1109,7 +1102,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then -i "pandas.tseries.offsets.Micro.rule_code GL08" \ -i "pandas.tseries.offsets.Milli PR02" \ -i "pandas.tseries.offsets.Milli.copy SA01" \ - -i "pandas.tseries.offsets.Milli.delta GL08" \ -i "pandas.tseries.offsets.Milli.freqstr SA01" \ -i "pandas.tseries.offsets.Milli.is_on_offset GL08" \ -i "pandas.tseries.offsets.Milli.kwds SA01" \ @@ -1120,7 +1112,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then -i "pandas.tseries.offsets.Milli.rule_code GL08" \ -i "pandas.tseries.offsets.Minute PR02" \ -i "pandas.tseries.offsets.Minute.copy SA01" \ - -i "pandas.tseries.offsets.Minute.delta GL08" \ -i "pandas.tseries.offsets.Minute.freqstr SA01" \ -i "pandas.tseries.offsets.Minute.is_on_offset GL08" \ -i "pandas.tseries.offsets.Minute.kwds SA01" \ @@ -1151,7 +1142,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then -i "pandas.tseries.offsets.MonthEnd.rule_code GL08" \ -i "pandas.tseries.offsets.Nano PR02" \ -i "pandas.tseries.offsets.Nano.copy SA01" \ - -i "pandas.tseries.offsets.Nano.delta GL08" \ -i "pandas.tseries.offsets.Nano.freqstr SA01" \ -i "pandas.tseries.offsets.Nano.is_on_offset GL08" \ -i "pandas.tseries.offsets.Nano.kwds SA01" \ @@ -1184,7 +1174,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then -i "pandas.tseries.offsets.QuarterEnd.startingMonth GL08" \ -i "pandas.tseries.offsets.Second PR02" \ -i "pandas.tseries.offsets.Second.copy SA01" \ - -i "pandas.tseries.offsets.Second.delta GL08" \ -i "pandas.tseries.offsets.Second.freqstr SA01" \ -i "pandas.tseries.offsets.Second.is_on_offset GL08" \ -i "pandas.tseries.offsets.Second.kwds SA01" \ @@ -1217,7 +1206,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then -i "pandas.tseries.offsets.SemiMonthEnd.rule_code GL08" \ -i "pandas.tseries.offsets.Tick GL08" \ -i "pandas.tseries.offsets.Tick.copy SA01" \ - -i "pandas.tseries.offsets.Tick.delta GL08" \ -i "pandas.tseries.offsets.Tick.freqstr SA01" \ -i "pandas.tseries.offsets.Tick.is_on_offset GL08" \ -i "pandas.tseries.offsets.Tick.kwds SA01" \ diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 85ee5230b31be..ed7dfe1a3c17e 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -26,7 +26,7 @@ dependencies: - beautifulsoup4>=4.11.2 - blosc>=1.21.3 - bottleneck>=1.3.6 - - fastparquet>=2023.04.0 + - fastparquet>=2023.10.0 - fsspec>=2022.11.0 - html5lib>=1.1 - hypothesis>=6.46.1 @@ -57,7 +57,7 @@ dependencies: - zstandard>=0.19.0 - pip: - - adbc-driver-postgresql>=0.8.0 + - adbc-driver-postgresql>=0.10.0 - adbc-driver-sqlite>=0.8.0 - tzdata>=2022.7 - pytest-localserver>=0.7.1 diff --git a/ci/deps/actions-311-downstream_compat.yaml b/ci/deps/actions-311-downstream_compat.yaml index efd790d77afbb..dd1d341c70a9b 100644 --- a/ci/deps/actions-311-downstream_compat.yaml +++ b/ci/deps/actions-311-downstream_compat.yaml @@ -28,7 +28,7 @@ dependencies: - beautifulsoup4>=4.11.2 - blosc>=1.21.3 - bottleneck>=1.3.6 - - fastparquet>=2023.04.0 + - fastparquet>=2023.10.0 - fsspec>=2022.11.0 - html5lib>=1.1 - hypothesis>=6.46.1 @@ -72,6 +72,6 @@ dependencies: - pyyaml - py - pip: - - adbc-driver-postgresql>=0.8.0 + - adbc-driver-postgresql>=0.10.0 - adbc-driver-sqlite>=0.8.0 - tzdata>=2022.7 diff --git a/ci/deps/actions-311-numpydev.yaml b/ci/deps/actions-311-numpydev.yaml index b62e8630f2059..61a0eabbf133c 100644 --- a/ci/deps/actions-311-numpydev.yaml +++ b/ci/deps/actions-311-numpydev.yaml @@ -13,10 +13,7 @@ dependencies: # test dependencies - pytest>=7.3.2 - pytest-cov - # Once pytest-cov > 4 comes out, unpin this - # Right now, a DeprecationWarning related to rsyncdir - # causes an InternalError within pytest - - pytest-xdist>=2.2.0, <3 + - pytest-xdist>=2.2.0 - hypothesis>=6.46.1 # pandas dependencies diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml index 535c260582eec..388116439f944 100644 --- a/ci/deps/actions-311.yaml +++ b/ci/deps/actions-311.yaml @@ -26,7 +26,7 @@ dependencies: - beautifulsoup4>=4.11.2 - blosc>=1.21.3 - bottleneck>=1.3.6 - - fastparquet>=2023.04.0 + - fastparquet>=2023.10.0 - fsspec>=2022.11.0 - html5lib>=1.1 - hypothesis>=6.46.1 @@ -57,6 +57,6 @@ dependencies: - zstandard>=0.19.0 - pip: - - adbc-driver-postgresql>=0.8.0 + - adbc-driver-postgresql>=0.10.0 - adbc-driver-sqlite>=0.8.0 - pytest-localserver>=0.7.1 diff --git a/ci/deps/actions-312.yaml b/ci/deps/actions-312.yaml index 8b3f19f55e4b6..1d9f8aa3b092a 100644 --- a/ci/deps/actions-312.yaml +++ b/ci/deps/actions-312.yaml @@ -26,7 +26,7 @@ dependencies: - beautifulsoup4>=4.11.2 - blosc>=1.21.3 - bottleneck>=1.3.6 - - fastparquet>=2023.04.0 + - fastparquet>=2023.10.0 - fsspec>=2022.11.0 - html5lib>=1.1 - hypothesis>=6.46.1 @@ -34,7 +34,7 @@ dependencies: - jinja2>=3.1.2 - lxml>=4.9.2 - matplotlib>=3.6.3 - # - numba>=0.56.4 + - numba>=0.56.4 - numexpr>=2.8.4 - odfpy>=1.4.1 - qtpy>=2.3.0 @@ -44,7 +44,7 @@ dependencies: - pyarrow>=10.0.1 - pymysql>=1.0.2 - pyreadstat>=1.2.0 - # - pytables>=3.8.0 + - pytables>=3.8.0 - python-calamine>=0.1.7 - pyxlsb>=1.0.10 - s3fs>=2022.11.0 @@ -57,7 +57,7 @@ dependencies: - zstandard>=0.19.0 - pip: - - adbc-driver-postgresql>=0.8.0 + - adbc-driver-postgresql>=0.10.0 - adbc-driver-sqlite>=0.8.0 - tzdata>=2022.7 - pytest-localserver>=0.7.1 diff --git a/ci/deps/actions-39-minimum_versions.yaml b/ci/deps/actions-39-minimum_versions.yaml index 94cb21d1621b6..b760f27a3d4d3 100644 --- a/ci/deps/actions-39-minimum_versions.yaml +++ b/ci/deps/actions-39-minimum_versions.yaml @@ -29,7 +29,7 @@ dependencies: - beautifulsoup4=4.11.2 - blosc=1.21.3 - bottleneck=1.3.6 - - fastparquet=2023.04.0 + - fastparquet=2023.10.0 - fsspec=2022.11.0 - html5lib=1.1 - hypothesis=6.46.1 @@ -60,6 +60,6 @@ dependencies: - zstandard=0.19.0 - pip: - - adbc-driver-postgresql==0.8.0 + - adbc-driver-postgresql==0.10.0 - adbc-driver-sqlite==0.8.0 - tzdata==2022.7 diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 4cc9b1fbe2491..8f235a836bb3d 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -26,7 +26,7 @@ dependencies: - beautifulsoup4>=4.11.2 - blosc>=1.21.3 - bottleneck>=1.3.6 - - fastparquet>=2023.04.0 + - fastparquet>=2023.10.0 - fsspec>=2022.11.0 - html5lib>=1.1 - hypothesis>=6.46.1 @@ -57,7 +57,7 @@ dependencies: - zstandard>=0.19.0 - pip: - - adbc-driver-postgresql>=0.8.0 + - adbc-driver-postgresql>=0.10.0 - adbc-driver-sqlite>=0.8.0 - tzdata>=2022.7 - pytest-localserver>=0.7.1 diff --git a/ci/deps/circle-310-arm64.yaml b/ci/deps/circle-310-arm64.yaml index 869aae8596681..ed4d139714e71 100644 --- a/ci/deps/circle-310-arm64.yaml +++ b/ci/deps/circle-310-arm64.yaml @@ -27,7 +27,7 @@ dependencies: - beautifulsoup4>=4.11.2 - blosc>=1.21.3 - bottleneck>=1.3.6 - - fastparquet>=2023.04.0 + - fastparquet>=2023.10.0 - fsspec>=2022.11.0 - html5lib>=1.1 - hypothesis>=6.46.1 diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 39ab0890a32d1..d2c2f58427a23 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -10,7 +10,7 @@ echo PYTHONHASHSEED=$PYTHONHASHSEED COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=pyproject.toml" -PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fE -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fE -n $PYTEST_WORKERS --dist=worksteal $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" diff --git a/doc/source/development/contributing_docstring.rst b/doc/source/development/contributing_docstring.rst index e2881c1087e60..0b8c1e16dce0e 100644 --- a/doc/source/development/contributing_docstring.rst +++ b/doc/source/development/contributing_docstring.rst @@ -940,7 +940,7 @@ Finally, docstrings can also be appended to with the ``doc`` decorator. In this example, we'll create a parent docstring normally (this is like ``pandas.core.generic.NDFrame``). Then we'll have two children (like -``pandas.core.series.Series`` and ``pandas.core.frame.DataFrame``). We'll +``pandas.core.series.Series`` and ``pandas.DataFrame``). We'll substitute the class names in this docstring. .. code-block:: python diff --git a/doc/source/development/maintaining.rst b/doc/source/development/maintaining.rst index 5d833dca50732..f6ff95aa72c6c 100644 --- a/doc/source/development/maintaining.rst +++ b/doc/source/development/maintaining.rst @@ -151,7 +151,7 @@ and then run:: git bisect start git bisect good v1.4.0 git bisect bad v1.5.0 - git bisect run bash -c "python setup.py build_ext -j 4; python t.py" + git bisect run bash -c "python -m pip install -ve . --no-build-isolation --config-settings editable-verbose=true; python t.py" This finds the first commit that changed the behavior. The C extensions have to be rebuilt at every step, so the search can take a while. @@ -159,7 +159,7 @@ rebuilt at every step, so the search can take a while. Exit bisect and rebuild the current version:: git bisect reset - python setup.py build_ext -j 4 + python -m pip install -ve . --no-build-isolation --config-settings editable-verbose=true Report your findings under the corresponding issue and ping the commit author to get their input. diff --git a/doc/source/getting_started/install.rst b/doc/source/getting_started/install.rst index 77e273d8c81fe..3cd9e030d6b3c 100644 --- a/doc/source/getting_started/install.rst +++ b/doc/source/getting_started/install.rst @@ -346,7 +346,7 @@ SQLAlchemy 2.0.0 postgresql, SQL support for dat sql-other psycopg2 2.9.6 postgresql PostgreSQL engine for sqlalchemy pymysql 1.0.2 mysql MySQL engine for sqlalchemy -adbc-driver-postgresql 0.8.0 postgresql ADBC Driver for PostgreSQL +adbc-driver-postgresql 0.10.0 postgresql ADBC Driver for PostgreSQL adbc-driver-sqlite 0.8.0 sql-other ADBC Driver for SQLite ========================= ================== =============== ============================================================= @@ -361,7 +361,7 @@ Dependency Minimum Version pip extra Notes PyTables 3.8.0 hdf5 HDF5-based reading / writing blosc 1.21.3 hdf5 Compression for HDF5; only available on ``conda`` zlib hdf5 Compression for HDF5 -fastparquet 2023.04.0 - Parquet reading / writing (pyarrow is default) +fastparquet 2023.10.0 - Parquet reading / writing (pyarrow is default) pyarrow 10.0.1 parquet, feather Parquet, ORC, and feather reading / writing pyreadstat 1.2.0 spss SPSS files (.sav) reading odfpy 1.4.1 excel Open document format (.odf, .ods, .odt) reading / writing diff --git a/doc/source/reference/offset_frequency.rst b/doc/source/reference/offset_frequency.rst index 37eff247899be..8bb2c6ffe73be 100644 --- a/doc/source/reference/offset_frequency.rst +++ b/doc/source/reference/offset_frequency.rst @@ -1042,7 +1042,6 @@ Properties .. autosummary:: :toctree: api/ - Tick.delta Tick.freqstr Tick.kwds Tick.name @@ -1077,7 +1076,6 @@ Properties .. autosummary:: :toctree: api/ - Day.delta Day.freqstr Day.kwds Day.name @@ -1112,7 +1110,6 @@ Properties .. autosummary:: :toctree: api/ - Hour.delta Hour.freqstr Hour.kwds Hour.name @@ -1147,7 +1144,6 @@ Properties .. autosummary:: :toctree: api/ - Minute.delta Minute.freqstr Minute.kwds Minute.name @@ -1182,7 +1178,6 @@ Properties .. autosummary:: :toctree: api/ - Second.delta Second.freqstr Second.kwds Second.name @@ -1217,7 +1212,6 @@ Properties .. autosummary:: :toctree: api/ - Milli.delta Milli.freqstr Milli.kwds Milli.name @@ -1252,7 +1246,6 @@ Properties .. autosummary:: :toctree: api/ - Micro.delta Micro.freqstr Micro.kwds Micro.name @@ -1287,7 +1280,6 @@ Properties .. autosummary:: :toctree: api/ - Nano.delta Nano.freqstr Nano.kwds Nano.name diff --git a/doc/source/reference/style.rst b/doc/source/reference/style.rst index 2256876c93e01..0e1d93841d52f 100644 --- a/doc/source/reference/style.rst +++ b/doc/source/reference/style.rst @@ -41,6 +41,7 @@ Style application Styler.map_index Styler.format Styler.format_index + Styler.format_index_names Styler.relabel_index Styler.hide Styler.concat diff --git a/doc/source/user_guide/enhancingperf.rst b/doc/source/user_guide/enhancingperf.rst index 8c510173819e0..c4721f3a6b09c 100644 --- a/doc/source/user_guide/enhancingperf.rst +++ b/doc/source/user_guide/enhancingperf.rst @@ -453,7 +453,7 @@ by evaluate arithmetic and boolean expression all at once for large :class:`~pan :func:`~pandas.eval` is many orders of magnitude slower for smaller expressions or objects than plain Python. A good rule of thumb is to only use :func:`~pandas.eval` when you have a - :class:`.DataFrame` with more than 10,000 rows. + :class:`~pandas.core.frame.DataFrame` with more than 10,000 rows. Supported syntax ~~~~~~~~~~~~~~~~ diff --git a/doc/source/user_guide/io.rst b/doc/source/user_guide/io.rst index 9c48e66daacf0..db2326d5b9754 100644 --- a/doc/source/user_guide/io.rst +++ b/doc/source/user_guide/io.rst @@ -6400,7 +6400,7 @@ ignored. In [2]: df = pd.DataFrame({'A': np.random.randn(sz), 'B': [1] * sz}) In [3]: df.info() - + RangeIndex: 1000000 entries, 0 to 999999 Data columns (total 2 columns): A 1000000 non-null float64 diff --git a/doc/source/user_guide/timeseries.rst b/doc/source/user_guide/timeseries.rst index ecdfb3c565d33..37413722de96f 100644 --- a/doc/source/user_guide/timeseries.rst +++ b/doc/source/user_guide/timeseries.rst @@ -1468,11 +1468,16 @@ or some other non-observed day. Defined observance rules are: :header: "Rule", "Description" :widths: 15, 70 + "next_workday", "move Saturday and Sunday to Monday" + "previous_workday", "move Saturday and Sunday to Friday" "nearest_workday", "move Saturday to Friday and Sunday to Monday" + "before_nearest_workday", "apply ``nearest_workday`` and then move to previous workday before that day" + "after_nearest_workday", "apply ``nearest_workday`` and then move to next workday after that day" "sunday_to_monday", "move Sunday to following Monday" "next_monday_or_tuesday", "move Saturday to Monday and Sunday/Monday to Tuesday" "previous_friday", move Saturday and Sunday to previous Friday" "next_monday", "move Saturday and Sunday to following Monday" + "weekend_to_monday", "same as ``next_monday``" An example of how holidays and holiday calendars are defined: diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index cb12962256a55..c63d047f03823 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -840,7 +840,7 @@ then all the columns are dummy-encoded, and a :class:`SparseDataFrame` was retur In [2]: df = pd.DataFrame({"A": [1, 2], "B": ['a', 'b'], "C": ['a', 'a']}) In [3]: type(pd.get_dummies(df, sparse=True)) - Out[3]: pandas.core.frame.DataFrame + Out[3]: pandas.DataFrame In [4]: type(pd.get_dummies(df[['B', 'C']], sparse=True)) Out[4]: pandas.core.sparse.frame.SparseDataFrame diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 94a8ee7cd1a5d..5dbf6f1c60598 100755 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -414,7 +414,7 @@ Extended verbose info output for :class:`~pandas.DataFrame` ... "text_col": ["a", "b", "c"], ... "float_col": [0.0, 0.1, 0.2]}) In [2]: df.info(verbose=True) - + RangeIndex: 3 entries, 0 to 2 Data columns (total 3 columns): int_col 3 non-null int64 diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index 10d5a518f686d..4b7b075ceafaf 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -34,6 +34,7 @@ Other enhancements - Allow dictionaries to be passed to :meth:`pandas.Series.str.replace` via ``pat`` parameter (:issue:`51748`) - Support passing a :class:`Series` input to :func:`json_normalize` that retains the :class:`Series` :class:`Index` (:issue:`51452`) - Users can globally disable any ``PerformanceWarning`` by setting the option ``mode.performance_warnings`` to ``False`` (:issue:`56920`) +- :meth:`Styler.format_index_names` can now be used to format the index and column names (:issue:`48936` and :issue:`47489`) - .. --------------------------------------------------------------------------- @@ -130,11 +131,13 @@ For `optional libraries ` (:issue:`54710`) - In :meth:`DataFrame.stack`, the default value of ``future_stack`` is now ``True``; specifying ``False`` will raise a ``FutureWarning`` (:issue:`55448`) - Iterating over a :class:`.DataFrameGroupBy` or :class:`.SeriesGroupBy` will return tuples of length 1 for the groups when grouping by ``level`` a list of length 1 (:issue:`50064`) @@ -254,7 +273,10 @@ Removal of prior version deprecations/changes - Removed unused arguments ``*args`` and ``**kwargs`` in :class:`Resampler` methods (:issue:`50977`) - Unrecognized timezones when parsing strings to datetimes now raises a ``ValueError`` (:issue:`51477`) - Removed the :class:`Grouper` attributes ``ax``, ``groups``, ``indexer``, and ``obj`` (:issue:`51206`, :issue:`51182`) +- Removed deprecated keyword ``verbose`` on :func:`read_csv` and :func:`read_table` (:issue:`56556`) +- Removed the ``method`` keyword in ``ExtensionArray.fillna``, implement ``ExtensionArray._pad_or_backfill`` instead (:issue:`53621`) - Removed the attribute ``dtypes`` from :class:`.DataFrameGroupBy` (:issue:`51997`) +- Enforced deprecation of ``argmin``, ``argmax``, ``idxmin``, and ``idxmax`` returning a result when ``skipna=False`` and an NA value is encountered or all values are NA values; these operations will now raise in such cases (:issue:`33941`, :issue:`51276`) .. --------------------------------------------------------------------------- .. _whatsnew_300.performance: @@ -262,6 +284,7 @@ Removal of prior version deprecations/changes Performance improvements ~~~~~~~~~~~~~~~~~~~~~~~~ - :attr:`Categorical.categories` returns a :class:`RangeIndex` columns instead of an :class:`Index` if the constructed ``values`` was a ``range``. (:issue:`57787`) +- :class:`DataFrame` returns a :class:`RangeIndex` columns when possible when ``data`` is a ``dict`` (:issue:`57943`) - :func:`concat` returns a :class:`RangeIndex` level in the :class:`MultiIndex` result when ``keys`` is a ``range`` or :class:`RangeIndex` (:issue:`57542`) - :meth:`RangeIndex.append` returns a :class:`RangeIndex` instead of a :class:`Index` when appending values that could continue the :class:`RangeIndex` (:issue:`57467`) - :meth:`Series.str.extract` returns a :class:`RangeIndex` columns instead of an :class:`Index` column when possible (:issue:`57542`) @@ -282,6 +305,7 @@ Performance improvements - Performance improvement in :meth:`RangeIndex.join` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57651`, :issue:`57752`) - Performance improvement in :meth:`RangeIndex.reindex` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57647`, :issue:`57752`) - Performance improvement in :meth:`RangeIndex.take` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57445`, :issue:`57752`) +- Performance improvement in :func:`merge` if hash-join can be used (:issue:`57970`) - Performance improvement in ``DataFrameGroupBy.__len__`` and ``SeriesGroupBy.__len__`` (:issue:`57595`) - Performance improvement in indexing operations for string dtypes (:issue:`56997`) - Performance improvement in unary methods on a :class:`RangeIndex` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57825`) @@ -292,6 +316,7 @@ Performance improvements Bug fixes ~~~~~~~~~ - Fixed bug in :class:`SparseDtype` for equal comparison with na fill value. (:issue:`54770`) +- Fixed bug in :meth:`.DataFrameGroupBy.median` where nat values gave an incorrect result. (:issue:`57926`) - Fixed bug in :meth:`DataFrame.join` inconsistently setting result index name (:issue:`55815`) - Fixed bug in :meth:`DataFrame.to_string` that raised ``StopIteration`` with nested DataFrames. (:issue:`16098`) - Fixed bug in :meth:`DataFrame.update` bool dtype being converted to object (:issue:`55509`) @@ -313,7 +338,7 @@ Datetimelike Timedelta ^^^^^^^^^ -- +- Accuracy improvement in :meth:`Timedelta.to_pytimedelta` to round microseconds consistently for large nanosecond based Timedelta (:issue:`57841`) - Timezones diff --git a/environment.yml b/environment.yml index e7bf2556d27f8..186d7e1d703df 100644 --- a/environment.yml +++ b/environment.yml @@ -30,7 +30,7 @@ dependencies: - beautifulsoup4>=4.11.2 - blosc - bottleneck>=1.3.6 - - fastparquet>=2023.04.0 + - fastparquet>=2023.10.0 - fsspec>=2022.11.0 - html5lib>=1.1 - hypothesis>=6.46.1 @@ -116,7 +116,7 @@ dependencies: - pygments # Code highlighting - pip: - - adbc-driver-postgresql>=0.8.0 + - adbc-driver-postgresql>=0.10.0 - adbc-driver-sqlite>=0.8.0 - typing_extensions; python_version<"3.11" - tzdata>=2022.7 diff --git a/pandas/__init__.py b/pandas/__init__.py index da06222b19924..ce42a1d67f757 100644 --- a/pandas/__init__.py +++ b/pandas/__init__.py @@ -28,7 +28,8 @@ raise ImportError( f"C extension: {_module} not built. If you want to import " "pandas from the source directory, you may need to run " - "'python setup.py build_ext' to build the C extensions first." + "'python -m pip install -ve . --no-build-isolation --config-settings " + "editable-verbose=true' to build the C extensions first." ) from _err from pandas._config import ( diff --git a/pandas/_libs/groupby.pyi b/pandas/_libs/groupby.pyi index 95ac555303221..53f5f73624232 100644 --- a/pandas/_libs/groupby.pyi +++ b/pandas/_libs/groupby.pyi @@ -12,6 +12,7 @@ def group_median_float64( min_count: int = ..., # Py_ssize_t mask: np.ndarray | None = ..., result_mask: np.ndarray | None = ..., + is_datetimelike: bool = ..., # bint ) -> None: ... def group_cumprod( out: np.ndarray, # float64_t[:, ::1] diff --git a/pandas/_libs/groupby.pyx b/pandas/_libs/groupby.pyx index 2ff45038d6a3e..c0b9ed42cb535 100644 --- a/pandas/_libs/groupby.pyx +++ b/pandas/_libs/groupby.pyx @@ -101,7 +101,11 @@ cdef float64_t median_linear_mask(float64_t* a, int n, uint8_t* mask) noexcept n return result -cdef float64_t median_linear(float64_t* a, int n) noexcept nogil: +cdef float64_t median_linear( + float64_t* a, + int n, + bint is_datetimelike=False +) noexcept nogil: cdef: int i, j, na_count = 0 float64_t* tmp @@ -111,9 +115,14 @@ cdef float64_t median_linear(float64_t* a, int n) noexcept nogil: return NaN # count NAs - for i in range(n): - if a[i] != a[i]: - na_count += 1 + if is_datetimelike: + for i in range(n): + if a[i] == NPY_NAT: + na_count += 1 + else: + for i in range(n): + if a[i] != a[i]: + na_count += 1 if na_count: if na_count == n: @@ -124,10 +133,16 @@ cdef float64_t median_linear(float64_t* a, int n) noexcept nogil: raise MemoryError() j = 0 - for i in range(n): - if a[i] == a[i]: - tmp[j] = a[i] - j += 1 + if is_datetimelike: + for i in range(n): + if a[i] != NPY_NAT: + tmp[j] = a[i] + j += 1 + else: + for i in range(n): + if a[i] == a[i]: + tmp[j] = a[i] + j += 1 a = tmp n -= na_count @@ -170,6 +185,7 @@ def group_median_float64( Py_ssize_t min_count=-1, const uint8_t[:, :] mask=None, uint8_t[:, ::1] result_mask=None, + bint is_datetimelike=False, ) -> None: """ Only aggregates on axis=0 @@ -228,7 +244,7 @@ def group_median_float64( ptr += _counts[0] for j in range(ngroups): size = _counts[j + 1] - out[j, i] = median_linear(ptr, size) + out[j, i] = median_linear(ptr, size, is_datetimelike) ptr += size diff --git a/pandas/_libs/hashtable.pxd b/pandas/_libs/hashtable.pxd index 29ace4a339ced..a5a3edad63403 100644 --- a/pandas/_libs/hashtable.pxd +++ b/pandas/_libs/hashtable.pxd @@ -183,7 +183,7 @@ cdef class Int64Vector(Vector): cdef Int64VectorData data cdef ndarray ao - cdef resize(self) + cdef resize(self, Py_ssize_t new_size) cpdef ndarray to_array(self) cdef void append(self, int64_t x) noexcept cdef extend(self, int64_t[:] x) diff --git a/pandas/_libs/hashtable.pyi b/pandas/_libs/hashtable.pyi index 3725bfa3362d9..7a810a988e50e 100644 --- a/pandas/_libs/hashtable.pyi +++ b/pandas/_libs/hashtable.pyi @@ -16,7 +16,7 @@ def unique_label_indices( class Factorizer: count: int uniques: Any - def __init__(self, size_hint: int) -> None: ... + def __init__(self, size_hint: int, uses_mask: bool = False) -> None: ... def get_count(self) -> int: ... def factorize( self, @@ -25,6 +25,9 @@ class Factorizer: na_value=..., mask=..., ) -> npt.NDArray[np.intp]: ... + def hash_inner_join( + self, values: np.ndarray, mask=... + ) -> tuple[np.ndarray, np.ndarray]: ... class ObjectFactorizer(Factorizer): table: PyObjectHashTable @@ -216,6 +219,9 @@ class HashTable: mask=..., ignore_na: bool = True, ) -> tuple[np.ndarray, npt.NDArray[np.intp]]: ... # np.ndarray[subclass-specific] + def hash_inner_join( + self, values: np.ndarray, mask=... + ) -> tuple[np.ndarray, np.ndarray]: ... class Complex128HashTable(HashTable): ... class Complex64HashTable(HashTable): ... diff --git a/pandas/_libs/hashtable.pyx b/pandas/_libs/hashtable.pyx index 8250d0242c31f..97fae1d6480ce 100644 --- a/pandas/_libs/hashtable.pyx +++ b/pandas/_libs/hashtable.pyx @@ -7,6 +7,7 @@ from libc.stdlib cimport ( free, malloc, ) +from libc.string cimport memcpy import numpy as np @@ -69,7 +70,7 @@ cdef class Factorizer: cdef readonly: Py_ssize_t count - def __cinit__(self, size_hint: int): + def __cinit__(self, size_hint: int, uses_mask: bool = False): self.count = 0 def get_count(self) -> int: @@ -78,13 +79,16 @@ cdef class Factorizer: def factorize(self, values, na_sentinel=-1, na_value=None, mask=None) -> np.ndarray: raise NotImplementedError + def hash_inner_join(self, values, mask=None): + raise NotImplementedError + cdef class ObjectFactorizer(Factorizer): cdef public: PyObjectHashTable table ObjectVector uniques - def __cinit__(self, size_hint: int): + def __cinit__(self, size_hint: int, uses_mask: bool = False): self.table = PyObjectHashTable(size_hint) self.uniques = ObjectVector() diff --git a/pandas/_libs/hashtable_class_helper.pxi.in b/pandas/_libs/hashtable_class_helper.pxi.in index f37a32ed61555..e3a9102fec395 100644 --- a/pandas/_libs/hashtable_class_helper.pxi.in +++ b/pandas/_libs/hashtable_class_helper.pxi.in @@ -163,8 +163,9 @@ ctypedef fused vector_data: Complex64VectorData StringVectorData -cdef bint needs_resize(vector_data *data) noexcept nogil: - return data.size == data.capacity + +cdef bint needs_resize(Py_ssize_t nelems, Py_ssize_t capacity) noexcept nogil: + return nelems >= capacity # ---------------------------------------------------------------------- # Vector @@ -214,8 +215,8 @@ cdef class {{name}}Vector(Vector): self.ao = np.empty(self.data.capacity, dtype=np.{{dtype}}) self.data.data = <{{c_type}}*>self.ao.data - cdef resize(self): - self.data.capacity = max(self.data.capacity * 4, _INIT_VEC_CAP) + cdef resize(self, Py_ssize_t new_size): + self.data.capacity = max(new_size, _INIT_VEC_CAP) self.ao.resize(self.data.capacity, refcheck=False) self.data.data = <{{c_type}}*>self.ao.data @@ -234,17 +235,28 @@ cdef class {{name}}Vector(Vector): cdef void append(self, {{c_type}} x) noexcept: - if needs_resize(&self.data): + if needs_resize(self.data.size, self.data.capacity): if self.external_view_exists: raise ValueError("external reference but " "Vector.resize() needed") - self.resize() + self.resize(self.data.capacity * 4) append_data_{{dtype}}(&self.data, x) cdef extend(self, const {{c_type}}[:] x): - for i in range(len(x)): - self.append(x[i]) + cdef Py_ssize_t x_size = len(x) + if x_size == 0: + return + + cdef Py_ssize_t needed_size = self.data.size + x_size + if needs_resize(needed_size, self.data.capacity): + if self.external_view_exists: + raise ValueError("external reference but " + "Vector.resize() needed") + self.resize(needed_size) + + memcpy(self.data.data + self.data.size, &x[0], x_size * sizeof({{c_type}})) + self.data.size = needed_size {{endfor}} @@ -260,7 +272,7 @@ cdef class StringVector(Vector): if self.data.data is NULL: raise MemoryError() - cdef resize(self): + cdef resize(self, Py_ssize_t new_size): cdef: char **orig_data Py_ssize_t i, orig_capacity @@ -297,8 +309,8 @@ cdef class StringVector(Vector): cdef void append(self, char *x) noexcept: - if needs_resize(&self.data): - self.resize() + if needs_resize(self.data.size, self.data.capacity): + self.resize(self.data.capacity * 4) append_data_string(&self.data, x) @@ -545,6 +557,49 @@ cdef class {{name}}HashTable(HashTable): self.table.vals[k] = i self.na_position = na_position + @cython.wraparound(False) + @cython.boundscheck(False) + def hash_inner_join(self, const {{dtype}}_t[:] values, const uint8_t[:] mask = None) -> tuple[ndarray, ndarray]: + cdef: + Py_ssize_t i, n = len(values) + {{c_type}} val + khiter_t k + Int64Vector locs = Int64Vector() + Int64Vector self_locs = Int64Vector() + Int64VectorData *l + Int64VectorData *sl + int8_t na_position = self.na_position + + l = &locs.data + sl = &self_locs.data + + if self.uses_mask and mask is None: + raise NotImplementedError # pragma: no cover + + with nogil: + for i in range(n): + if self.uses_mask and mask[i]: + if self.na_position == -1: + continue + if needs_resize(l.size, l.capacity): + with gil: + locs.resize(locs.data.capacity * 4) + self_locs.resize(locs.data.capacity * 4) + append_data_int64(l, i) + append_data_int64(sl, na_position) + else: + val = {{to_c_type}}(values[i]) + k = kh_get_{{dtype}}(self.table, val) + if k != self.table.n_buckets: + if needs_resize(l.size, l.capacity): + with gil: + locs.resize(locs.data.capacity * 4) + self_locs.resize(locs.data.capacity * 4) + append_data_int64(l, i) + append_data_int64(sl, self.table.vals[k]) + + return self_locs.to_array(), locs.to_array() + @cython.boundscheck(False) def lookup(self, const {{dtype}}_t[:] values, const uint8_t[:] mask = None) -> ndarray: # -> np.ndarray[np.intp] @@ -684,18 +739,18 @@ cdef class {{name}}HashTable(HashTable): continue seen_na = True - if needs_resize(ud): + if needs_resize(ud.size, ud.capacity): with gil: if uniques.external_view_exists: raise ValueError("external reference to " "uniques held, but " "Vector.resize() needed") - uniques.resize() + uniques.resize(uniques.data.capacity * 4) if result_mask.external_view_exists: raise ValueError("external reference to " "result_mask held, but " "Vector.resize() needed") - result_mask.resize() + result_mask.resize(result_mask.data.capacity * 4) append_data_{{dtype}}(ud, val) append_data_uint8(rmd, 1) continue @@ -706,19 +761,19 @@ cdef class {{name}}HashTable(HashTable): # k hasn't been seen yet k = kh_put_{{dtype}}(self.table, val, &ret) - if needs_resize(ud): + if needs_resize(ud.size, ud.capacity): with gil: if uniques.external_view_exists: raise ValueError("external reference to " "uniques held, but " "Vector.resize() needed") - uniques.resize() + uniques.resize(uniques.data.capacity * 4) if use_result_mask: if result_mask.external_view_exists: raise ValueError("external reference to " "result_mask held, but " "Vector.resize() needed") - result_mask.resize() + result_mask.resize(result_mask.data.capacity * 4) append_data_{{dtype}}(ud, val) if use_result_mask: append_data_uint8(rmd, 0) @@ -849,9 +904,9 @@ cdef class {{name}}HashTable(HashTable): k = kh_put_{{dtype}}(self.table, val, &ret) self.table.vals[k] = count - if needs_resize(ud): + if needs_resize(ud.size, ud.capacity): with gil: - uniques.resize() + uniques.resize(uniques.data.capacity * 4) append_data_{{dtype}}(ud, val) labels[i] = count count += 1 @@ -867,8 +922,8 @@ cdef class {{name}}Factorizer(Factorizer): {{name}}HashTable table {{name}}Vector uniques - def __cinit__(self, size_hint: int): - self.table = {{name}}HashTable(size_hint) + def __cinit__(self, size_hint: int, uses_mask: bool = False): + self.table = {{name}}HashTable(size_hint, uses_mask=uses_mask) self.uniques = {{name}}Vector() def factorize(self, const {{c_type}}[:] values, @@ -899,6 +954,9 @@ cdef class {{name}}Factorizer(Factorizer): self.count = len(self.uniques) return labels + def hash_inner_join(self, const {{c_type}}[:] values, const uint8_t[:] mask = None) -> tuple[np.ndarray, np.ndarray]: + return self.table.hash_inner_join(values, mask) + {{endfor}} diff --git a/pandas/_libs/hashtable_func_helper.pxi.in b/pandas/_libs/hashtable_func_helper.pxi.in index ca1b28b9442ca..5500fadb73b6d 100644 --- a/pandas/_libs/hashtable_func_helper.pxi.in +++ b/pandas/_libs/hashtable_func_helper.pxi.in @@ -480,9 +480,9 @@ def _unique_label_indices_{{dtype}}(const {{c_type}}[:] labels) -> ndarray: for i in range(n): kh_put_{{ttype}}(table, labels[i], &ret) if ret != 0: - if needs_resize(ud): + if needs_resize(ud.size, ud.capacity): with gil: - idx.resize() + idx.resize(idx.data.capacity * 4) append_data_{{ttype}}(ud, i) kh_destroy_{{ttype}}(table) diff --git a/pandas/_libs/lib.pyi b/pandas/_libs/lib.pyi index 34193a9b1d231..b39d32d069619 100644 --- a/pandas/_libs/lib.pyi +++ b/pandas/_libs/lib.pyi @@ -231,3 +231,7 @@ def is_range_indexer( left: np.ndarray, n: int, # np.ndarray[np.int64, ndim=1] ) -> bool: ... +def is_sequence_range( + sequence: np.ndarray, + step: int, # np.ndarray[np.int64, ndim=1] +) -> bool: ... diff --git a/pandas/_libs/lib.pyx b/pandas/_libs/lib.pyx index 00668576d5d53..a2205454a5a46 100644 --- a/pandas/_libs/lib.pyx +++ b/pandas/_libs/lib.pyx @@ -678,6 +678,28 @@ def is_range_indexer(ndarray[int6432_t, ndim=1] left, Py_ssize_t n) -> bool: return True +@cython.wraparound(False) +@cython.boundscheck(False) +def is_sequence_range(ndarray[int6432_t, ndim=1] sequence, int64_t step) -> bool: + """ + Check if sequence is equivalent to a range with the specified step. + """ + cdef: + Py_ssize_t i, n = len(sequence) + int6432_t first_element + + if step == 0: + return False + if n == 0: + return True + + first_element = sequence[0] + for i in range(1, n): + if sequence[i] != first_element + i * step: + return False + return True + + ctypedef fused ndarr_object: ndarray[object, ndim=1] ndarray[object, ndim=2] diff --git a/pandas/_libs/parsers.pyx b/pandas/_libs/parsers.pyx index 82e9812094af2..c29cdbcf5975e 100644 --- a/pandas/_libs/parsers.pyx +++ b/pandas/_libs/parsers.pyx @@ -6,7 +6,6 @@ from csv import ( QUOTE_NONE, QUOTE_NONNUMERIC, ) -import time import warnings from pandas.util._exceptions import find_stack_level @@ -344,10 +343,9 @@ cdef class TextReader: object true_values, false_values object handle object orig_header - bint na_filter, keep_default_na, verbose, has_usecols, has_mi_columns + bint na_filter, keep_default_na, has_usecols, has_mi_columns bint allow_leading_cols uint64_t parser_start # this is modified after __init__ - list clocks const char *encoding_errors kh_str_starts_t *false_set kh_str_starts_t *true_set @@ -400,7 +398,6 @@ cdef class TextReader: bint allow_leading_cols=True, skiprows=None, skipfooter=0, # int64_t - bint verbose=False, float_precision=None, bint skip_blank_lines=True, encoding_errors=b"strict", @@ -417,9 +414,6 @@ cdef class TextReader: self.parser = parser_new() self.parser.chunksize = tokenize_chunksize - # For timekeeping - self.clocks = [] - self.parser.usecols = (usecols is not None) self._setup_parser_source(source) @@ -507,8 +501,6 @@ cdef class TextReader: self.converters = converters self.na_filter = na_filter - self.verbose = verbose - if float_precision == "round_trip": # see gh-15140 self.parser.double_converter = round_trip_wrapper @@ -896,8 +888,6 @@ cdef class TextReader: int64_t buffered_lines int64_t irows - self._start_clock() - if rows is not None: irows = rows buffered_lines = self.parser.lines - self.parser_start @@ -915,12 +905,8 @@ cdef class TextReader: if self.parser_start >= self.parser.lines: raise StopIteration - self._end_clock("Tokenization") - self._start_clock() columns = self._convert_column_data(rows) - self._end_clock("Type conversion") - self._start_clock() if len(columns) > 0: rows_read = len(list(columns.values())[0]) # trim @@ -929,18 +915,8 @@ cdef class TextReader: parser_trim_buffers(self.parser) self.parser_start -= rows_read - self._end_clock("Parser memory cleanup") - return columns - cdef _start_clock(self): - self.clocks.append(time.time()) - - cdef _end_clock(self, str what): - if self.verbose: - elapsed = time.time() - self.clocks.pop(-1) - print(f"{what} took: {elapsed * 1000:.2f} ms") - def set_noconvert(self, i: int) -> None: self.noconvert.add(i) @@ -1603,7 +1579,7 @@ cdef _categorical_convert(parser_t *parser, int64_t col, # -> ndarray[f'|S{width}'] cdef _to_fw_string(parser_t *parser, int64_t col, int64_t line_start, - int64_t line_end, int64_t width) noexcept: + int64_t line_end, int64_t width): cdef: char *data ndarray result diff --git a/pandas/_libs/tslibs/dtypes.pyx b/pandas/_libs/tslibs/dtypes.pyx index 906842d322e91..5bfbe211bfd14 100644 --- a/pandas/_libs/tslibs/dtypes.pyx +++ b/pandas/_libs/tslibs/dtypes.pyx @@ -313,15 +313,7 @@ cdef dict c_DEPR_ABBREVS = { "H": "h", "BH": "bh", "CBH": "cbh", - "T": "min", - "t": "min", "S": "s", - "L": "ms", - "l": "ms", - "U": "us", - "u": "us", - "N": "ns", - "n": "ns", } @@ -415,13 +407,17 @@ class Resolution(Enum): """ cdef: str abbrev + if freq in {"T", "t", "L", "l", "U", "u", "N", "n"}: + raise ValueError( + f"Frequency \'{freq}\' is no longer supported." + ) try: if freq in c_DEPR_ABBREVS: abbrev = c_DEPR_ABBREVS[freq] warnings.warn( f"\'{freq}\' is deprecated and will be removed in a future " f"version. Please use \'{abbrev}\' " - "instead of \'{freq}\'.", + f"instead of \'{freq}\'.", FutureWarning, stacklevel=find_stack_level(), ) diff --git a/pandas/_libs/tslibs/offsets.pyi b/pandas/_libs/tslibs/offsets.pyi index 791ebc0fbb245..3f942d6aa3622 100644 --- a/pandas/_libs/tslibs/offsets.pyi +++ b/pandas/_libs/tslibs/offsets.pyi @@ -20,8 +20,6 @@ from pandas._typing import ( npt, ) -from .timedeltas import Timedelta - _BaseOffsetT = TypeVar("_BaseOffsetT", bound=BaseOffset) _DatetimeT = TypeVar("_DatetimeT", bound=datetime) _TimedeltaT = TypeVar("_TimedeltaT", bound=timedelta) @@ -114,8 +112,6 @@ class Tick(SingleConstructorOffset): _prefix: str def __init__(self, n: int = ..., normalize: bool = ...) -> None: ... @property - def delta(self) -> Timedelta: ... - @property def nanos(self) -> int: ... def delta_to_tick(delta: timedelta) -> Tick: ... diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index fd18ae5908f10..e36abdf0ad971 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -957,22 +957,6 @@ cdef class Tick(SingleConstructorOffset): def _as_pd_timedelta(self): return Timedelta(self) - @property - def delta(self): - warnings.warn( - # GH#55498 - f"{type(self).__name__}.delta is deprecated and will be removed in " - "a future version. Use pd.Timedelta(obj) instead", - FutureWarning, - stacklevel=find_stack_level(), - ) - try: - return self.n * Timedelta(self._nanos_inc) - except OverflowError as err: - # GH#55503 as_unit will raise a more useful OutOfBoundsTimedelta - Timedelta(self).as_unit("ns") - raise AssertionError("This should not be reached.") - @property def nanos(self) -> int64_t: """ diff --git a/pandas/_libs/tslibs/parsing.pyx b/pandas/_libs/tslibs/parsing.pyx index 94c549cbd3db0..384df1cac95eb 100644 --- a/pandas/_libs/tslibs/parsing.pyx +++ b/pandas/_libs/tslibs/parsing.pyx @@ -45,7 +45,6 @@ from decimal import InvalidOperation from dateutil.parser import DEFAULTPARSER from dateutil.tz import ( - tzlocal as _dateutil_tzlocal, tzoffset, tzutc as _dateutil_tzutc, ) @@ -703,17 +702,12 @@ cdef datetime dateutil_parse( if res.tzname and res.tzname in time.tzname: # GH#50791 if res.tzname != "UTC": - # If the system is localized in UTC (as many CI runs are) - # we get tzlocal, once the deprecation is enforced will get - # timezone.utc, not raise. - warnings.warn( + raise ValueError( f"Parsing '{res.tzname}' as tzlocal (dependent on system timezone) " - "is deprecated and will raise in a future version. Pass the 'tz' " + "is no longer supported. Pass the 'tz' " "keyword or call tz_localize after construction instead", - FutureWarning, - stacklevel=find_stack_level() ) - ret = ret.replace(tzinfo=_dateutil_tzlocal()) + ret = ret.replace(tzinfo=timezone.utc) elif res.tzoffset == 0: ret = ret.replace(tzinfo=_dateutil_tzutc()) elif res.tzoffset: diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 18ead7d5381df..9078fd4116899 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -1376,7 +1376,10 @@ cdef class _Timedelta(timedelta): datetime.timedelta(days=3) """ if self._creso == NPY_FR_ns: - return timedelta(microseconds=int(self._value) / 1000) + us, remainder = divmod(self._value, 1000) + if remainder >= 500: + us += 1 + return timedelta(microseconds=us) # TODO(@WillAyd): is this the right way to use components? self._ensure_components() diff --git a/pandas/compat/_optional.py b/pandas/compat/_optional.py index f9273ba4bbc62..f4e717c26d6fd 100644 --- a/pandas/compat/_optional.py +++ b/pandas/compat/_optional.py @@ -20,12 +20,12 @@ # deps_minimum.toml & pyproject.toml when updating versions! VERSIONS = { - "adbc-driver-postgresql": "0.8.0", + "adbc-driver-postgresql": "0.10.0", "adbc-driver-sqlite": "0.8.0", "bs4": "4.11.2", "blosc": "1.21.3", "bottleneck": "1.3.6", - "fastparquet": "2023.04.0", + "fastparquet": "2023.10.0", "fsspec": "2022.11.0", "html5lib": "1.1", "hypothesis": "6.46.1", diff --git a/pandas/conftest.py b/pandas/conftest.py index d5e3d6ff72aad..04a81e21185b8 100644 --- a/pandas/conftest.py +++ b/pandas/conftest.py @@ -30,7 +30,7 @@ def ignore_doctest_warning(item: pytest.Item, path: str, message: str) -> None: item : pytest.Item pytest test item. path : str - Module path to Python object, e.g. "pandas.core.frame.DataFrame.append". A + Module path to Python object, e.g. "pandas.DataFrame.append". A warning will be filtered when item.name ends with in given path. So it is sufficient to specify e.g. "DataFrame.append". message : str diff --git a/pandas/core/algorithms.py b/pandas/core/algorithms.py index 344314d829c19..8620aafd97528 100644 --- a/pandas/core/algorithms.py +++ b/pandas/core/algorithms.py @@ -892,26 +892,9 @@ def value_counts_internal( if keys.dtype == np.float16: keys = keys.astype(np.float32) - # For backwards compatibility, we let Index do its normal type - # inference, _except_ for if if infers from object to bool. - idx = Index(keys) - if idx.dtype == bool and keys.dtype == object: - idx = idx.astype(object) - elif ( - idx.dtype != keys.dtype # noqa: PLR1714 # # pylint: disable=R1714 - and idx.dtype != "string[pyarrow_numpy]" - ): - warnings.warn( - # GH#56161 - "The behavior of value_counts with object-dtype is deprecated. " - "In a future version, this will *not* perform dtype inference " - "on the resulting index. To retain the old behavior, use " - "`result.index = result.index.infer_objects()`", - FutureWarning, - stacklevel=find_stack_level(), - ) - idx.name = index_name - + # Starting in 3.0, we no longer perform dtype inference on the + # Index object we construct here, xref GH#56161 + idx = Index(keys, dtype=keys.dtype, name=index_name) result = Series(counts, index=idx, name=name, copy=False) if sort: @@ -1606,16 +1589,8 @@ def union_with_duplicates( """ from pandas import Series - with warnings.catch_warnings(): - # filter warning from object dtype inference; we will end up discarding - # the index here, so the deprecation does not affect the end result here. - warnings.filterwarnings( - "ignore", - "The behavior of value_counts with object-dtype is deprecated", - category=FutureWarning, - ) - l_count = value_counts_internal(lvals, dropna=False) - r_count = value_counts_internal(rvals, dropna=False) + l_count = value_counts_internal(lvals, dropna=False) + r_count = value_counts_internal(rvals, dropna=False) l_count, r_count = l_count.align(r_count, fill_value=0) final_count = np.maximum(l_count.values, r_count.values) final_count = Series(final_count, index=l_count.index, dtype="int", copy=False) diff --git a/pandas/core/arrays/_mixins.py b/pandas/core/arrays/_mixins.py index c1d0ade572e8a..7f4e6f6666382 100644 --- a/pandas/core/arrays/_mixins.py +++ b/pandas/core/arrays/_mixins.py @@ -33,7 +33,6 @@ from pandas.util._decorators import doc from pandas.util._validators import ( validate_bool_kwarg, - validate_fillna_kwargs, validate_insert_loc, ) @@ -336,13 +335,7 @@ def _pad_or_backfill( return new_values @doc(ExtensionArray.fillna) - def fillna( - self, value=None, method=None, limit: int | None = None, copy: bool = True - ) -> Self: - value, method = validate_fillna_kwargs( - value, method, validate_scalar_dict_value=False - ) - + def fillna(self, value=None, limit: int | None = None, copy: bool = True) -> Self: mask = self.isna() # error: Argument 2 to "check_value_size" has incompatible type # "ExtensionArray"; expected "ndarray" @@ -353,25 +346,12 @@ def fillna( ) if mask.any(): - if method is not None: - # (for now) when self.ndim == 2, we assume axis=0 - func = missing.get_fill_func(method, ndim=self.ndim) - npvalues = self._ndarray.T - if copy: - npvalues = npvalues.copy() - func(npvalues, limit=limit, mask=mask.T) - npvalues = npvalues.T - - # TODO: NumpyExtensionArray didn't used to copy, need tests - # for this - new_values = self._from_backing_data(npvalues) + # fill with value + if copy: + new_values = self.copy() else: - # fill with value - if copy: - new_values = self.copy() - else: - new_values = self[:] - new_values[mask] = value + new_values = self[:] + new_values[mask] = value else: # We validate the fill_value even if there is nothing to fill if value is not None: diff --git a/pandas/core/arrays/arrow/array.py b/pandas/core/arrays/arrow/array.py index aaf43662ebde2..84b62563605ac 100644 --- a/pandas/core/arrays/arrow/array.py +++ b/pandas/core/arrays/arrow/array.py @@ -29,7 +29,6 @@ pa_version_under13p0, ) from pandas.util._decorators import doc -from pandas.util._validators import validate_fillna_kwargs from pandas.core.dtypes.cast import ( can_hold_element, @@ -1068,6 +1067,7 @@ def _pad_or_backfill( # a kernel for duration types. pass + # TODO: Why do we no longer need the above cases? # TODO(3.0): after EA.fillna 'method' deprecation is enforced, we can remove # this method entirely. return super()._pad_or_backfill( @@ -1078,21 +1078,15 @@ def _pad_or_backfill( def fillna( self, value: object | ArrayLike | None = None, - method: FillnaOptions | None = None, limit: int | None = None, copy: bool = True, ) -> Self: - value, method = validate_fillna_kwargs(value, method) - if not self._hasna: # TODO(CoW): Not necessary anymore when CoW is the default return self.copy() if limit is not None: - return super().fillna(value=value, method=method, limit=limit, copy=copy) - - if method is not None: - return super().fillna(method=method, limit=limit, copy=copy) + return super().fillna(value=value, limit=limit, copy=copy) if isinstance(value, (np.ndarray, ExtensionArray)): # Similar to check_value_size, but we do not mask here since we may @@ -1118,7 +1112,7 @@ def fillna( # a kernel for duration types. pass - return super().fillna(value=value, method=method, limit=limit, copy=copy) + return super().fillna(value=value, limit=limit, copy=copy) def isin(self, values: ArrayLike) -> npt.NDArray[np.bool_]: # short-circuit to return all False array. diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py index 86831f072bb8f..76615704f2e33 100644 --- a/pandas/core/arrays/base.py +++ b/pandas/core/arrays/base.py @@ -38,7 +38,6 @@ from pandas.util._exceptions import find_stack_level from pandas.util._validators import ( validate_bool_kwarg, - validate_fillna_kwargs, validate_insert_loc, ) @@ -1007,31 +1006,6 @@ def _pad_or_backfill( [, 2, 2, 3, , ] Length: 6, dtype: Int64 """ - - # If a 3rd-party EA has implemented this functionality in fillna, - # we warn that they need to implement _pad_or_backfill instead. - if ( - type(self).fillna is not ExtensionArray.fillna - and type(self)._pad_or_backfill is ExtensionArray._pad_or_backfill - ): - # Check for _pad_or_backfill here allows us to call - # super()._pad_or_backfill without getting this warning - warnings.warn( - "ExtensionArray.fillna 'method' keyword is deprecated. " - "In a future version. arr._pad_or_backfill will be called " - "instead. 3rd-party ExtensionArray authors need to implement " - "_pad_or_backfill.", - DeprecationWarning, - stacklevel=find_stack_level(), - ) - if limit_area is not None: - raise NotImplementedError( - f"{type(self).__name__} does not implement limit_area " - "(added in pandas 2.2). 3rd-party ExtnsionArray authors " - "need to add this argument to _pad_or_backfill." - ) - return self.fillna(method=method, limit=limit) - mask = self.isna() if mask.any(): @@ -1057,8 +1031,7 @@ def _pad_or_backfill( def fillna( self, - value: object | ArrayLike | None = None, - method: FillnaOptions | None = None, + value: object | ArrayLike, limit: int | None = None, copy: bool = True, ) -> Self: @@ -1071,14 +1044,6 @@ def fillna( If a scalar value is passed it is used to fill all missing values. Alternatively, an array-like "value" can be given. It's expected that the array-like have the same length as 'self'. - method : {'backfill', 'bfill', 'pad', 'ffill', None}, default None - Method to use for filling holes in reindexed Series: - - * pad / ffill: propagate last valid observation forward to next valid. - * backfill / bfill: use NEXT valid observation to fill gap. - - .. deprecated:: 2.1.0 - limit : int, default None If method is specified, this is the maximum number of consecutive NaN values to forward/backward fill. In other words, if there is @@ -1086,9 +1051,6 @@ def fillna( be partially filled. If method is not specified, this is the maximum number of entries along the entire axis where NaNs will be filled. - - .. deprecated:: 2.1.0 - copy : bool, default True Whether to make a copy of the data before filling. If False, then the original should be modified and no new memory should be allocated. @@ -1110,16 +1072,6 @@ def fillna( [0, 0, 2, 3, 0, 0] Length: 6, dtype: Int64 """ - if method is not None: - warnings.warn( - f"The 'method' keyword in {type(self).__name__}.fillna is " - "deprecated and will be removed in a future version.", - FutureWarning, - stacklevel=find_stack_level(), - ) - - value, method = validate_fillna_kwargs(value, method) - mask = self.isna() # error: Argument 2 to "check_value_size" has incompatible type # "ExtensionArray"; expected "ndarray" @@ -1130,24 +1082,12 @@ def fillna( ) if mask.any(): - if method is not None: - meth = missing.clean_fill_method(method) - - npmask = np.asarray(mask) - if meth == "pad": - indexer = libalgos.get_fill_indexer(npmask, limit=limit) - return self.take(indexer, allow_fill=True) - else: - # i.e. meth == "backfill" - indexer = libalgos.get_fill_indexer(npmask[::-1], limit=limit)[::-1] - return self[::-1].take(indexer, allow_fill=True) + # fill with value + if not copy: + new_values = self[:] else: - # fill with value - if not copy: - new_values = self[:] - else: - new_values = self.copy() - new_values[mask] = value + new_values = self.copy() + new_values[mask] = value else: if not copy: new_values = self[:] diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py index 60529c1c2251b..416331a260e9f 100644 --- a/pandas/core/arrays/categorical.py +++ b/pandas/core/arrays/categorical.py @@ -626,19 +626,6 @@ def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike: return result - def to_list(self) -> list: - """ - Alias for tolist. - """ - # GH#51254 - warnings.warn( - "Categorical.to_list is deprecated and will be removed in a future " - "version. Use obj.tolist() instead", - FutureWarning, - stacklevel=find_stack_level(), - ) - return self.tolist() - @classmethod def _from_inferred_categories( cls, inferred_categories, inferred_codes, dtype, true_values=None @@ -2622,7 +2609,7 @@ def describe(self) -> DataFrame: from pandas import Index from pandas.core.reshape.concat import concat - result = concat([counts, freqs], axis=1) + result = concat([counts, freqs], ignore_index=True, axis=1) result.columns = Index(["counts", "freqs"]) result.index.name = "categories" diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 745774b34a3ad..3dc2d77bb5a19 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -26,7 +26,6 @@ algos, lib, ) -from pandas._libs.arrays import NDArrayBacked from pandas._libs.tslibs import ( BaseOffset, IncompatibleFrequency, @@ -1936,100 +1935,6 @@ class TimelikeOps(DatetimeLikeArrayMixin): Common ops for TimedeltaIndex/DatetimeIndex, but not PeriodIndex. """ - _default_dtype: np.dtype - - def __init__( - self, values, dtype=None, freq=lib.no_default, copy: bool = False - ) -> None: - warnings.warn( - # GH#55623 - f"{type(self).__name__}.__init__ is deprecated and will be " - "removed in a future version. Use pd.array instead.", - FutureWarning, - stacklevel=find_stack_level(), - ) - if dtype is not None: - dtype = pandas_dtype(dtype) - - values = extract_array(values, extract_numpy=True) - if isinstance(values, IntegerArray): - values = values.to_numpy("int64", na_value=iNaT) - - inferred_freq = getattr(values, "_freq", None) - explicit_none = freq is None - freq = freq if freq is not lib.no_default else None - - if isinstance(values, type(self)): - if explicit_none: - # don't inherit from values - pass - elif freq is None: - freq = values.freq - elif freq and values.freq: - freq = to_offset(freq) - freq = _validate_inferred_freq(freq, values.freq) - - if dtype is not None and dtype != values.dtype: - # TODO: we only have tests for this for DTA, not TDA (2022-07-01) - raise TypeError( - f"dtype={dtype} does not match data dtype {values.dtype}" - ) - - dtype = values.dtype - values = values._ndarray - - elif dtype is None: - if isinstance(values, np.ndarray) and values.dtype.kind in "Mm": - dtype = values.dtype - else: - dtype = self._default_dtype - if isinstance(values, np.ndarray) and values.dtype == "i8": - values = values.view(dtype) - - if not isinstance(values, np.ndarray): - raise ValueError( - f"Unexpected type '{type(values).__name__}'. 'values' must be a " - f"{type(self).__name__}, ndarray, or Series or Index " - "containing one of those." - ) - if values.ndim not in [1, 2]: - raise ValueError("Only 1-dimensional input arrays are supported.") - - if values.dtype == "i8": - # for compat with datetime/timedelta/period shared methods, - # we can sometimes get here with int64 values. These represent - # nanosecond UTC (or tz-naive) unix timestamps - if dtype is None: - dtype = self._default_dtype - values = values.view(self._default_dtype) - elif lib.is_np_dtype(dtype, "mM"): - values = values.view(dtype) - elif isinstance(dtype, DatetimeTZDtype): - kind = self._default_dtype.kind - new_dtype = f"{kind}8[{dtype.unit}]" - values = values.view(new_dtype) - - dtype = self._validate_dtype(values, dtype) - - if freq == "infer": - raise ValueError( - f"Frequency inference not allowed in {type(self).__name__}.__init__. " - "Use 'pd.array()' instead." - ) - - if copy: - values = values.copy() - if freq: - freq = to_offset(freq) - if values.dtype.kind == "m" and not isinstance(freq, Tick): - raise TypeError("TimedeltaArray/Index freq must be a Tick") - - NDArrayBacked.__init__(self, values=values, dtype=dtype) - self._freq = freq - - if inferred_freq is None and freq is not None: - type(self)._validate_frequency(self, freq) - @classmethod def _validate_dtype(cls, values, dtype): raise AbstractMethodError(cls) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index ad4611aac9e35..d446407ec3d01 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -186,7 +186,7 @@ class DatetimeArray(dtl.TimelikeOps, dtl.DatelikeOps): # type: ignore[misc] Parameters ---------- - values : Series, Index, DatetimeArray, ndarray + data : Series, Index, DatetimeArray, ndarray The datetime data. For DatetimeArray `values` (or a Series or Index boxing one), @@ -287,7 +287,6 @@ def _scalar_type(self) -> type[Timestamp]: _dtype: np.dtype[np.datetime64] | DatetimeTZDtype _freq: BaseOffset | None = None - _default_dtype = DT64NS_DTYPE # used in TimeLikeOps.__init__ @classmethod def _from_scalars(cls, scalars, *, dtype: DtypeObj) -> Self: diff --git a/pandas/core/arrays/interval.py b/pandas/core/arrays/interval.py index 1ea32584403ba..af666a591b1bc 100644 --- a/pandas/core/arrays/interval.py +++ b/pandas/core/arrays/interval.py @@ -13,7 +13,6 @@ Union, overload, ) -import warnings import numpy as np @@ -29,7 +28,6 @@ ArrayLike, AxisInt, Dtype, - FillnaOptions, IntervalClosedType, NpDtype, PositionalIndexer, @@ -894,23 +892,7 @@ def max(self, *, axis: AxisInt | None = None, skipna: bool = True) -> IntervalOr indexer = obj.argsort()[-1] return obj[indexer] - def _pad_or_backfill( # pylint: disable=useless-parent-delegation - self, - *, - method: FillnaOptions, - limit: int | None = None, - limit_area: Literal["inside", "outside"] | None = None, - copy: bool = True, - ) -> Self: - # TODO(3.0): after EA.fillna 'method' deprecation is enforced, we can remove - # this method entirely. - return super()._pad_or_backfill( - method=method, limit=limit, limit_area=limit_area, copy=copy - ) - - def fillna( - self, value=None, method=None, limit: int | None = None, copy: bool = True - ) -> Self: + def fillna(self, value=None, limit: int | None = None, copy: bool = True) -> Self: """ Fill NA/NaN values using the specified method. @@ -921,9 +903,6 @@ def fillna( Alternatively, a Series or dict can be used to fill in different values for each index. The value should not be a list. The value(s) passed should be either Interval objects or NA/NaN. - method : {'backfill', 'bfill', 'pad', 'ffill', None}, default None - (Not implemented yet for IntervalArray) - Method to use for filling holes in reindexed Series limit : int, default None (Not implemented yet for IntervalArray) If method is specified, this is the maximum number of consecutive @@ -944,8 +923,6 @@ def fillna( """ if copy is False: raise NotImplementedError - if method is not None: - return super().fillna(value=value, method=method, limit=limit) value_left, value_right = self._validate_scalar(value) @@ -1239,15 +1216,8 @@ def value_counts(self, dropna: bool = True) -> Series: Series.value_counts """ # TODO: implement this is a non-naive way! - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", - "The behavior of value_counts with object-dtype is deprecated", - category=FutureWarning, - ) - result = value_counts(np.asarray(self), dropna=dropna) - # Once the deprecation is enforced, we will need to do - # `result.index = result.index.astype(self.dtype)` + result = value_counts(np.asarray(self), dropna=dropna) + result.index = result.index.astype(self.dtype) return result # --------------------------------------------------------------------- diff --git a/pandas/core/arrays/masked.py b/pandas/core/arrays/masked.py index 108202f5e510b..d20d7f98b8aa8 100644 --- a/pandas/core/arrays/masked.py +++ b/pandas/core/arrays/masked.py @@ -38,7 +38,6 @@ ) from pandas.errors import AbstractMethodError from pandas.util._decorators import doc -from pandas.util._validators import validate_fillna_kwargs from pandas.core.dtypes.base import ExtensionDtype from pandas.core.dtypes.common import ( @@ -237,32 +236,18 @@ def _pad_or_backfill( return new_values @doc(ExtensionArray.fillna) - def fillna( - self, value=None, method=None, limit: int | None = None, copy: bool = True - ) -> Self: - value, method = validate_fillna_kwargs(value, method) - + def fillna(self, value=None, limit: int | None = None, copy: bool = True) -> Self: mask = self._mask value = missing.check_value_size(value, mask, len(self)) if mask.any(): - if method is not None: - func = missing.get_fill_func(method, ndim=self.ndim) - npvalues = self._data.T - new_mask = mask.T - if copy: - npvalues = npvalues.copy() - new_mask = new_mask.copy() - func(npvalues, limit=limit, mask=new_mask) - return self._simple_new(npvalues.T, new_mask.T) + # fill with value + if copy: + new_values = self.copy() else: - # fill with value - if copy: - new_values = self.copy() - else: - new_values = self[:] - new_values[mask] = value + new_values = self[:] + new_values[mask] = value else: if copy: new_values = self.copy() diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index d05f857f46179..e73eba710ec39 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -847,19 +847,6 @@ def _pad_or_backfill( else: return self - def fillna( - self, value=None, method=None, limit: int | None = None, copy: bool = True - ) -> Self: - if method is not None: - # view as dt64 so we get treated as timelike in core.missing, - # similar to dtl._period_dispatch - dta = self.view("M8[ns]") - result = dta.fillna(value=value, method=method, limit=limit, copy=copy) - # error: Incompatible return value type (got "Union[ExtensionArray, - # ndarray[Any, Any]]", expected "PeriodArray") - return result.view(self.dtype) # type: ignore[return-value] - return super().fillna(value=value, method=method, limit=limit, copy=copy) - # ------------------------------------------------------------------ # Arithmetic Methods diff --git a/pandas/core/arrays/sparse/array.py b/pandas/core/arrays/sparse/array.py index bf44e5e099530..bdcb3219a9875 100644 --- a/pandas/core/arrays/sparse/array.py +++ b/pandas/core/arrays/sparse/array.py @@ -98,10 +98,7 @@ class ellipsis(Enum): from scipy.sparse import spmatrix - from pandas._typing import ( - FillnaOptions, - NumpySorter, - ) + from pandas._typing import NumpySorter SparseIndexKind = Literal["integer", "block"] @@ -717,24 +714,9 @@ def isna(self) -> Self: # type: ignore[override] mask[self.sp_index.indices] = isna(self.sp_values) return type(self)(mask, fill_value=False, dtype=dtype) - def _pad_or_backfill( # pylint: disable=useless-parent-delegation - self, - *, - method: FillnaOptions, - limit: int | None = None, - limit_area: Literal["inside", "outside"] | None = None, - copy: bool = True, - ) -> Self: - # TODO(3.0): We can remove this method once deprecation for fillna method - # keyword is enforced. - return super()._pad_or_backfill( - method=method, limit=limit, limit_area=limit_area, copy=copy - ) - def fillna( self, value=None, - method: FillnaOptions | None = None, limit: int | None = None, copy: bool = True, ) -> Self: @@ -743,17 +725,8 @@ def fillna( Parameters ---------- - value : scalar, optional - method : str, optional - - .. warning:: - - Using 'method' will result in high memory use, - as all `fill_value` methods will be converted to - an in-memory ndarray - + value : scalar limit : int, optional - copy: bool, default True Ignored for SparseArray. @@ -773,22 +746,15 @@ def fillna( When ``self.fill_value`` is not NA, the result dtype will be ``self.dtype``. Again, this preserves the amount of memory used. """ - if (method is None and value is None) or ( - method is not None and value is not None - ): - raise ValueError("Must specify one of 'method' or 'value'.") - - if method is not None: - return super().fillna(method=method, limit=limit) + if value is None: + raise ValueError("Must specify 'value'.") + new_values = np.where(isna(self.sp_values), value, self.sp_values) + if self._null_fill_value: + # This is essentially just updating the dtype. + new_dtype = SparseDtype(self.dtype.subtype, fill_value=value) else: - new_values = np.where(isna(self.sp_values), value, self.sp_values) - - if self._null_fill_value: - # This is essentially just updating the dtype. - new_dtype = SparseDtype(self.dtype.subtype, fill_value=value) - else: - new_dtype = self.dtype + new_dtype = self.dtype return self._simple_new(new_values, self._sparse_index, new_dtype) diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index c41e078095feb..6eb4d234b349d 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -113,7 +113,7 @@ class TimedeltaArray(dtl.TimelikeOps): Parameters ---------- - values : array-like + data : array-like The timedelta data. dtype : numpy.dtype @@ -196,7 +196,6 @@ def dtype(self) -> np.dtype[np.timedelta64]: # type: ignore[override] # Constructors _freq = None - _default_dtype = TD64NS_DTYPE # used in TimeLikeOps.__init__ @classmethod def _validate_dtype(cls, values, dtype): diff --git a/pandas/core/base.py b/pandas/core/base.py index 987136ffdff7d..263265701691b 100644 --- a/pandas/core/base.py +++ b/pandas/core/base.py @@ -14,7 +14,6 @@ final, overload, ) -import warnings import numpy as np @@ -35,7 +34,6 @@ cache_readonly, doc, ) -from pandas.util._exceptions import find_stack_level from pandas.core.dtypes.cast import can_hold_element from pandas.core.dtypes.common import ( @@ -686,7 +684,8 @@ def argmax( axis : {{None}} Unused. Parameter needed for compatibility with DataFrame. skipna : bool, default True - Exclude NA/null values when showing the result. + Exclude NA/null values. If the entire Series is NA, or if ``skipna=False`` + and there is an NA value, this method will raise a ``ValueError``. *args, **kwargs Additional arguments and keywords for compatibility with NumPy. @@ -736,28 +735,15 @@ def argmax( nv.validate_minmax_axis(axis) skipna = nv.validate_argmax_with_skipna(skipna, args, kwargs) + if skipna and len(delegate) > 0 and isna(delegate).all(): + raise ValueError("Encountered all NA values") + elif not skipna and isna(delegate).any(): + raise ValueError("Encountered an NA value with skipna=False") + if isinstance(delegate, ExtensionArray): - if not skipna and delegate.isna().any(): - warnings.warn( - f"The behavior of {type(self).__name__}.argmax/argmin " - "with skipna=False and NAs, or with all-NAs is deprecated. " - "In a future version this will raise ValueError.", - FutureWarning, - stacklevel=find_stack_level(), - ) - return -1 - else: - return delegate.argmax() + return delegate.argmax() else: result = nanops.nanargmax(delegate, skipna=skipna) - if result == -1: - warnings.warn( - f"The behavior of {type(self).__name__}.argmax/argmin " - "with skipna=False and NAs, or with all-NAs is deprecated. " - "In a future version this will raise ValueError.", - FutureWarning, - stacklevel=find_stack_level(), - ) # error: Incompatible return value type (got "Union[int, ndarray]", expected # "int") return result # type: ignore[return-value] @@ -770,28 +756,15 @@ def argmin( nv.validate_minmax_axis(axis) skipna = nv.validate_argmin_with_skipna(skipna, args, kwargs) + if skipna and len(delegate) > 0 and isna(delegate).all(): + raise ValueError("Encountered all NA values") + elif not skipna and isna(delegate).any(): + raise ValueError("Encountered an NA value with skipna=False") + if isinstance(delegate, ExtensionArray): - if not skipna and delegate.isna().any(): - warnings.warn( - f"The behavior of {type(self).__name__}.argmax/argmin " - "with skipna=False and NAs, or with all-NAs is deprecated. " - "In a future version this will raise ValueError.", - FutureWarning, - stacklevel=find_stack_level(), - ) - return -1 - else: - return delegate.argmin() + return delegate.argmin() else: result = nanops.nanargmin(delegate, skipna=skipna) - if result == -1: - warnings.warn( - f"The behavior of {type(self).__name__}.argmax/argmin " - "with skipna=False and NAs, or with all-NAs is deprecated. " - "In a future version this will raise ValueError.", - FutureWarning, - stacklevel=find_stack_level(), - ) # error: Incompatible return value type (got "Union[int, ndarray]", expected # "int") return result # type: ignore[return-value] @@ -1065,7 +1038,7 @@ def nunique(self, dropna: bool = True) -> int: @property def is_unique(self) -> bool: """ - Return boolean if values in the object are unique. + Return True if values in the object are unique. Returns ------- @@ -1086,7 +1059,7 @@ def is_unique(self) -> bool: @property def is_monotonic_increasing(self) -> bool: """ - Return boolean if values in the object are monotonically increasing. + Return True if values in the object are monotonically increasing. Returns ------- @@ -1109,7 +1082,7 @@ def is_monotonic_increasing(self) -> bool: @property def is_monotonic_decreasing(self) -> bool: """ - Return boolean if values in the object are monotonically decreasing. + Return True if values in the object are monotonically decreasing. Returns ------- diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 8fd0cd8c66e3c..b218dd899c8f8 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -65,6 +65,7 @@ Appender, Substitution, doc, + set_module, ) from pandas.util._exceptions import ( find_stack_level, @@ -356,7 +357,7 @@ of a string to indicate that the column name from `left` or `right` should be left as-is, with no suffix. At least one of the values must not be None. -copy : bool, default True +copy : bool, default False If False, avoid copy if possible. .. note:: @@ -370,6 +371,8 @@ You can already get the future behavior and improvements through enabling copy on write ``pd.options.mode.copy_on_write = True`` + + .. deprecated:: 3.0.0 indicator : bool or str, default False If True, adds a column to the output DataFrame called "_merge" with information on the source of each row. The column can be given a different @@ -498,6 +501,7 @@ # DataFrame class +@set_module("pandas") class DataFrame(NDFrame, OpsMixin): """ Two-dimensional, size-mutable, potentially heterogeneous tabular data. @@ -651,25 +655,36 @@ def _constructor(self) -> type[DataFrame]: return DataFrame def _constructor_from_mgr(self, mgr, axes) -> DataFrame: - if self._constructor is DataFrame: - # we are pandas.DataFrame (or a subclass that doesn't override _constructor) - return DataFrame._from_mgr(mgr, axes=axes) - else: - assert axes is mgr.axes + df = DataFrame._from_mgr(mgr, axes=axes) + + if type(self) is DataFrame: + # This would also work `if self._constructor is DataFrame`, but + # this check is slightly faster, benefiting the most-common case. + return df + + elif type(self).__name__ == "GeoDataFrame": + # Shim until geopandas can override their _constructor_from_mgr + # bc they have different behavior for Managers than for DataFrames return self._constructor(mgr) - _constructor_sliced: Callable[..., Series] = Series + # We assume that the subclass __init__ knows how to handle a + # pd.DataFrame object. + return self._constructor(df) - def _sliced_from_mgr(self, mgr, axes) -> Series: - return Series._from_mgr(mgr, axes) + _constructor_sliced: Callable[..., Series] = Series def _constructor_sliced_from_mgr(self, mgr, axes) -> Series: - if self._constructor_sliced is Series: - ser = self._sliced_from_mgr(mgr, axes) - ser._name = None # caller is responsible for setting real name + ser = Series._from_mgr(mgr, axes) + ser._name = None # caller is responsible for setting real name + + if type(self) is DataFrame: + # This would also work `if self._constructor_sliced is Series`, but + # this check is slightly faster, benefiting the most-common case. return ser - assert axes is mgr.axes - return self._constructor_sliced(mgr) + + # We assume that the subclass __init__ knows how to handle a + # pd.Series object. + return self._constructor_sliced(ser) # ---------------------------------------------------------------------- # Constructors @@ -3563,7 +3578,11 @@ def memory_usage(self, index: bool = True, deep: bool = False) -> Series: result = index_memory_usage._append(result) return result - def transpose(self, *args, copy: bool = False) -> DataFrame: + def transpose( + self, + *args, + copy: bool | lib.NoDefault = lib.no_default, + ) -> DataFrame: """ Transpose index and columns. @@ -3594,6 +3613,8 @@ def transpose(self, *args, copy: bool = False) -> DataFrame: You can already get the future behavior and improvements through enabling copy on write ``pd.options.mode.copy_on_write = True`` + .. deprecated:: 3.0.0 + Returns ------- DataFrame @@ -3674,6 +3695,7 @@ def transpose(self, *args, copy: bool = False) -> DataFrame: 1 object dtype: object """ + self._check_copy_deprecation(copy) nv.validate_transpose(args, {}) # construct the args @@ -5049,9 +5071,9 @@ def set_axis( labels, *, axis: Axis = 0, - copy: bool | None = None, + copy: bool | lib.NoDefault = lib.no_default, ) -> DataFrame: - return super().set_axis(labels, axis=axis) + return super().set_axis(labels, axis=axis, copy=copy) @doc( NDFrame.reindex, @@ -5300,7 +5322,7 @@ def rename( index: Renamer | None = ..., columns: Renamer | None = ..., axis: Axis | None = ..., - copy: bool | None = ..., + copy: bool | lib.NoDefault = lib.no_default, inplace: Literal[True], level: Level = ..., errors: IgnoreRaise = ..., @@ -5314,7 +5336,7 @@ def rename( index: Renamer | None = ..., columns: Renamer | None = ..., axis: Axis | None = ..., - copy: bool | None = ..., + copy: bool | lib.NoDefault = lib.no_default, inplace: Literal[False] = ..., level: Level = ..., errors: IgnoreRaise = ..., @@ -5328,7 +5350,7 @@ def rename( index: Renamer | None = ..., columns: Renamer | None = ..., axis: Axis | None = ..., - copy: bool | None = ..., + copy: bool | lib.NoDefault = lib.no_default, inplace: bool = ..., level: Level = ..., errors: IgnoreRaise = ..., @@ -5341,7 +5363,7 @@ def rename( index: Renamer | None = None, columns: Renamer | None = None, axis: Axis | None = None, - copy: bool | None = None, + copy: bool | lib.NoDefault = lib.no_default, inplace: bool = False, level: Level | None = None, errors: IgnoreRaise = "ignore", @@ -5371,7 +5393,7 @@ def rename( axis : {0 or 'index', 1 or 'columns'}, default 0 Axis to target with ``mapper``. Can be either the axis name ('index', 'columns') or number (0, 1). The default is 'index'. - copy : bool, default True + copy : bool, default False Also copy underlying data. .. note:: @@ -5385,6 +5407,8 @@ def rename( You can already get the future behavior and improvements through enabling copy on write ``pd.options.mode.copy_on_write = True`` + + .. deprecated:: 3.0.0 inplace : bool, default False Whether to modify the DataFrame rather than creating a new one. If True then value of copy is ignored. @@ -5465,6 +5489,7 @@ def rename( 2 2 5 4 3 6 """ + self._check_copy_deprecation(copy) return super()._rename( mapper=mapper, index=index, @@ -9286,10 +9311,9 @@ def stack( DataFrame. The new inner-most levels are created by pivoting the columns of the current dataframe: - - if the columns have a single level, the output is a Series; - - if the columns have multiple levels, the new index - level(s) is (are) taken from the prescribed level(s) and - the output is a DataFrame. + - if the columns have a single level, the output is a Series; + - if the columns have multiple levels, the new index level(s) is (are) + taken from the prescribed level(s) and the output is a DataFrame. Parameters ---------- @@ -10645,10 +10669,12 @@ def merge( right_index: bool = False, sort: bool = False, suffixes: Suffixes = ("_x", "_y"), - copy: bool | None = None, + copy: bool | lib.NoDefault = lib.no_default, indicator: str | bool = False, validate: MergeValidate | None = None, ) -> DataFrame: + self._check_copy_deprecation(copy) + from pandas.core.reshape.merge import merge return merge( @@ -10701,6 +10727,12 @@ def round( numpy.around : Round a numpy array to the given number of decimals. Series.round : Round a Series to the given number of decimals. + Notes + ----- + For values exactly halfway between rounded decimal values, pandas rounds + to the nearest even value (e.g. -0.5 and 0.5 round to 0.0, 1.5 and 2.5 + round to 2.0, etc.). + Examples -------- >>> df = pd.DataFrame( @@ -12444,7 +12476,7 @@ def to_timestamp( freq: Frequency | None = None, how: ToTimestampHow = "start", axis: Axis = 0, - copy: bool | None = None, + copy: bool | lib.NoDefault = lib.no_default, ) -> DataFrame: """ Cast to DatetimeIndex of timestamps, at *beginning* of period. @@ -12458,7 +12490,7 @@ def to_timestamp( vs. end. axis : {0 or 'index', 1 or 'columns'}, default 0 The axis to convert (the index by default). - copy : bool, default True + copy : bool, default False If False then underlying input data is not copied. .. note:: @@ -12473,6 +12505,8 @@ def to_timestamp( You can already get the future behavior and improvements through enabling copy on write ``pd.options.mode.copy_on_write = True`` + .. deprecated:: 3.0.0 + Returns ------- DataFrame @@ -12509,6 +12543,7 @@ def to_timestamp( >>> df2.index DatetimeIndex(['2023-01-31', '2024-01-31'], dtype='datetime64[ns]', freq=None) """ + self._check_copy_deprecation(copy) new_obj = self.copy(deep=False) axis_name = self._get_axis_name(axis) @@ -12522,7 +12557,10 @@ def to_timestamp( return new_obj def to_period( - self, freq: Frequency | None = None, axis: Axis = 0, copy: bool | None = None + self, + freq: Frequency | None = None, + axis: Axis = 0, + copy: bool | lib.NoDefault = lib.no_default, ) -> DataFrame: """ Convert DataFrame from DatetimeIndex to PeriodIndex. @@ -12536,7 +12574,7 @@ def to_period( Frequency of the PeriodIndex. axis : {0 or 'index', 1 or 'columns'}, default 0 The axis to convert (the index by default). - copy : bool, default True + copy : bool, default False If False then underlying input data is not copied. .. note:: @@ -12551,6 +12589,8 @@ def to_period( You can already get the future behavior and improvements through enabling copy on write ``pd.options.mode.copy_on_write = True`` + .. deprecated:: 3.0.0 + Returns ------- DataFrame @@ -12578,6 +12618,7 @@ def to_period( >>> idx.to_period("Y") PeriodIndex(['2001', '2002', '2003'], dtype='period[Y-DEC]') """ + self._check_copy_deprecation(copy) new_obj = self.copy(deep=False) axis_name = self._get_axis_name(axis) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index c9e6ffe1d7dc6..ebcb700e656f6 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -99,7 +99,6 @@ check_dtype_backend, validate_ascending, validate_bool_kwarg, - validate_fillna_kwargs, validate_inclusive, ) @@ -288,6 +287,7 @@ def _init_mgr( mgr = mgr.astype(dtype=dtype) return mgr + @final @classmethod def _from_mgr(cls, mgr: Manager, axes: list[Index]) -> Self: """ @@ -398,7 +398,7 @@ def flags(self) -> Flags: def set_flags( self, *, - copy: bool = False, + copy: bool | lib.NoDefault = lib.no_default, allows_duplicate_labels: bool | None = None, ) -> Self: """ @@ -420,6 +420,8 @@ def set_flags( You can already get the future behavior and improvements through enabling copy on write ``pd.options.mode.copy_on_write = True`` + + .. deprecated:: 3.0.0 allows_duplicate_labels : bool, optional Whether the returned object allows duplicate labels. @@ -454,6 +456,7 @@ def set_flags( >>> df2.flags.allows_duplicate_labels False """ + self._check_copy_deprecation(copy) df = self.copy(deep=False) if allows_duplicate_labels is not None: df.flags["allows_duplicate_labels"] = allows_duplicate_labels @@ -679,7 +682,7 @@ def set_axis( labels, *, axis: Axis = 0, - copy: bool | None = None, + copy: bool | lib.NoDefault = lib.no_default, ) -> Self: """ Assign desired index to given axis. @@ -696,7 +699,7 @@ def set_axis( The axis to update. The value 0 identifies the rows. For `Series` this parameter is unused and defaults to 0. - copy : bool, default True + copy : bool, default False Whether to make a copy of the underlying data. .. note:: @@ -711,6 +714,8 @@ def set_axis( You can already get the future behavior and improvements through enabling copy on write ``pd.options.mode.copy_on_write = True`` + .. deprecated:: 3.0.0 + Returns ------- %(klass)s @@ -720,6 +725,7 @@ def set_axis( -------- %(klass)s.rename_axis : Alter the name of the index%(see_also_sub)s. """ + self._check_copy_deprecation(copy) return self._set_axis_nocheck(labels, axis, inplace=False) @overload @@ -948,7 +954,6 @@ def _rename( index: Renamer | None = ..., columns: Renamer | None = ..., axis: Axis | None = ..., - copy: bool | None = ..., inplace: Literal[False] = ..., level: Level | None = ..., errors: str = ..., @@ -962,7 +967,6 @@ def _rename( index: Renamer | None = ..., columns: Renamer | None = ..., axis: Axis | None = ..., - copy: bool | None = ..., inplace: Literal[True], level: Level | None = ..., errors: str = ..., @@ -976,7 +980,6 @@ def _rename( index: Renamer | None = ..., columns: Renamer | None = ..., axis: Axis | None = ..., - copy: bool | None = ..., inplace: bool, level: Level | None = ..., errors: str = ..., @@ -990,7 +993,6 @@ def _rename( index: Renamer | None = None, columns: Renamer | None = None, axis: Axis | None = None, - copy: bool | None = None, inplace: bool = False, level: Level | None = None, errors: str = "ignore", @@ -1061,7 +1063,7 @@ def rename_axis( index=..., columns=..., axis: Axis = ..., - copy: bool | None = ..., + copy: bool | lib.NoDefault = lib.no_default, inplace: Literal[False] = ..., ) -> Self: ... @@ -1073,7 +1075,7 @@ def rename_axis( index=..., columns=..., axis: Axis = ..., - copy: bool | None = ..., + copy: bool | lib.NoDefault = lib.no_default, inplace: Literal[True], ) -> None: ... @@ -1085,7 +1087,7 @@ def rename_axis( index=..., columns=..., axis: Axis = ..., - copy: bool | None = ..., + copy: bool | lib.NoDefault = lib.no_default, inplace: bool = ..., ) -> Self | None: ... @@ -1096,7 +1098,7 @@ def rename_axis( index=lib.no_default, columns=lib.no_default, axis: Axis = 0, - copy: bool | None = None, + copy: bool | lib.NoDefault = lib.no_default, inplace: bool = False, ) -> Self | None: """ @@ -1118,7 +1120,7 @@ def rename_axis( apply to that axis' values. axis : {0 or 'index', 1 or 'columns'}, default 0 The axis to rename. - copy : bool, default None + copy : bool, default False Also copy underlying data. .. note:: @@ -1132,6 +1134,8 @@ def rename_axis( You can already get the future behavior and improvements through enabling copy on write ``pd.options.mode.copy_on_write = True`` + + .. deprecated:: 3.0.0 inplace : bool, default False Modifies the object directly, instead of creating a new Series or DataFrame. @@ -1219,6 +1223,7 @@ class name cat 4 0 monkey 2 2 """ + self._check_copy_deprecation(copy) axes = {"index": index, "columns": columns} if axis is not None: @@ -6327,7 +6332,7 @@ def astype( return self.copy(deep=False) # GH 19920: retain column metadata after concat - result = concat(results, axis=1, copy=False) + result = concat(results, axis=1) # GH#40810 retain subclass # error: Incompatible types in assignment # (expression has type "Self", variable has type "DataFrame") @@ -7624,7 +7629,6 @@ def interpolate( * 'time': Works on daily and higher resolution data to interpolate given length of interval. * 'index', 'values': use the actual numerical values of the index. - * 'pad': Fill in NaNs using existing values. * 'nearest', 'zero', 'slinear', 'quadratic', 'cubic', 'barycentric', 'polynomial': Passed to `scipy.interpolate.interp1d`, whereas 'spline' is passed to @@ -7648,23 +7652,9 @@ def interpolate( 0. inplace : bool, default False Update the data in place if possible. - limit_direction : {{'forward', 'backward', 'both'}}, Optional + limit_direction : {{'forward', 'backward', 'both'}}, optional, default 'forward' Consecutive NaNs will be filled in this direction. - If limit is specified: - * If 'method' is 'pad' or 'ffill', 'limit_direction' must be 'forward'. - * If 'method' is 'backfill' or 'bfill', 'limit_direction' must be - 'backwards'. - - If 'limit' is not specified: - * If 'method' is 'backfill' or 'bfill', the default is 'backward' - * else the default is 'forward' - - raises ValueError if `limit_direction` is 'forward' or 'both' and - method is 'backfill' or 'bfill'. - raises ValueError if `limit_direction` is 'backward' or 'both' and - method is 'pad' or 'ffill'. - limit_area : {{`None`, 'inside', 'outside'}}, default None If limit is specified, consecutive NaNs will be filled with this restriction. @@ -7797,30 +7787,11 @@ def interpolate( if not isinstance(method, str): raise ValueError("'method' should be a string, not None.") - fillna_methods = ["ffill", "bfill", "pad", "backfill"] - if method.lower() in fillna_methods: - # GH#53581 - warnings.warn( - f"{type(self).__name__}.interpolate with method={method} is " - "deprecated and will raise in a future version. " - "Use obj.ffill() or obj.bfill() instead.", - FutureWarning, - stacklevel=find_stack_level(), - ) - obj, should_transpose = self, False - else: - obj, should_transpose = (self.T, True) if axis == 1 else (self, False) - # GH#53631 - if np.any(obj.dtypes == object): - raise TypeError( - f"{type(self).__name__} cannot interpolate with object dtype." - ) - - if method in fillna_methods and "fill_value" in kwargs: - raise ValueError( - "'fill_value' is not a valid keyword for " - f"{type(self).__name__}.interpolate with method from " - f"{fillna_methods}" + obj, should_transpose = (self.T, True) if axis == 1 else (self, False) + # GH#53631 + if np.any(obj.dtypes == object): + raise TypeError( + f"{type(self).__name__} cannot interpolate with object dtype." ) if isinstance(obj.index, MultiIndex) and method != "linear": @@ -7830,34 +7801,16 @@ def interpolate( limit_direction = missing.infer_limit_direction(limit_direction, method) - if method.lower() in fillna_methods: - # TODO(3.0): remove this case - # TODO: warn/raise on limit_direction or kwargs which are ignored? - # as of 2023-06-26 no tests get here with either - if not self._mgr.is_single_block and axis == 1: - # GH#53898 - if inplace: - raise NotImplementedError() - obj, axis, should_transpose = self.T, 1 - axis, True - - new_data = obj._mgr.pad_or_backfill( - method=method, - axis=self._get_block_manager_axis(axis), - limit=limit, - limit_area=limit_area, - inplace=inplace, - ) - else: - index = missing.get_interp_index(method, obj.index) - new_data = obj._mgr.interpolate( - method=method, - index=index, - limit=limit, - limit_direction=limit_direction, - limit_area=limit_area, - inplace=inplace, - **kwargs, - ) + index = missing.get_interp_index(method, obj.index) + new_data = obj._mgr.interpolate( + method=method, + index=index, + limit=limit, + limit_direction=limit_direction, + limit_area=limit_area, + inplace=inplace, + **kwargs, + ) result = self._constructor_from_mgr(new_data, axes=new_data.axes) if should_transpose: @@ -9630,7 +9583,6 @@ def _align_series( # fill fill_na = notna(fill_value) if fill_na: - fill_value, _ = validate_fillna_kwargs(fill_value, None) left = left.fillna(fill_value) right = right.fillna(fill_value) diff --git a/pandas/core/groupby/generic.py b/pandas/core/groupby/generic.py index 3b20b854b344e..361e9e87fadb8 100644 --- a/pandas/core/groupby/generic.py +++ b/pandas/core/groupby/generic.py @@ -574,7 +574,7 @@ def _transform_general( if results: from pandas.core.reshape.concat import concat - concatenated = concat(results) + concatenated = concat(results, ignore_index=True) result = self._set_result_index_ordered(concatenated) else: result = self.obj._constructor(dtype=np.float64) @@ -1803,7 +1803,9 @@ def _transform_general(self, func, engine, engine_kwargs, *args, **kwargs): applied.append(res) concat_index = obj.columns - concatenated = concat(applied, axis=0, verify_integrity=False) + concatenated = concat( + applied, axis=0, verify_integrity=False, ignore_index=True + ) concatenated = concatenated.reindex(concat_index, axis=1) return self._set_result_index_ordered(concatenated) @@ -2797,7 +2799,7 @@ def _wrap_transform_general_frame( # other dimension; this will preserve dtypes # GH14457 if res.index.is_(obj.index): - res_frame = concat([res] * len(group.columns), axis=1) + res_frame = concat([res] * len(group.columns), axis=1, ignore_index=True) res_frame.columns = group.columns res_frame.index = group.index else: diff --git a/pandas/core/groupby/ops.py b/pandas/core/groupby/ops.py index acf4c7bebf52d..8585ae3828247 100644 --- a/pandas/core/groupby/ops.py +++ b/pandas/core/groupby/ops.py @@ -415,6 +415,7 @@ def _call_cython_op( "last", "first", "sum", + "median", ]: func( out=result, @@ -427,7 +428,7 @@ def _call_cython_op( is_datetimelike=is_datetimelike, **kwargs, ) - elif self.how in ["sem", "std", "var", "ohlc", "prod", "median"]: + elif self.how in ["sem", "std", "var", "ohlc", "prod"]: if self.how in ["std", "sem"]: kwargs["is_datetimelike"] = is_datetimelike func( diff --git a/pandas/core/indexes/api.py b/pandas/core/indexes/api.py index a8887a21afa34..9b05eb42c6d6e 100644 --- a/pandas/core/indexes/api.py +++ b/pandas/core/indexes/api.py @@ -1,6 +1,5 @@ from __future__ import annotations -import textwrap from typing import ( TYPE_CHECKING, cast, @@ -23,6 +22,7 @@ ensure_index, ensure_index_from_sequences, get_unanimous_names, + maybe_sequence_to_range, ) from pandas.core.indexes.category import CategoricalIndex from pandas.core.indexes.datetimes import DatetimeIndex @@ -34,16 +34,6 @@ if TYPE_CHECKING: from pandas._typing import Axis -_sort_msg = textwrap.dedent( - """\ -Sorting because non-concatenation axis is not aligned. A future version -of pandas will change to not sort by default. - -To accept the future behavior, pass 'sort=False'. - -To retain the current behavior and silence the warning, pass 'sort=True'. -""" -) __all__ = [ @@ -66,6 +56,7 @@ "all_indexes_same", "default_index", "safe_sort_index", + "maybe_sequence_to_range", ] diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 62facb89a2f16..76dd19a9424f5 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -6976,16 +6976,10 @@ def argmin(self, axis=None, skipna: bool = True, *args, **kwargs) -> int: if not self._is_multi and self.hasnans: # Take advantage of cache - mask = self._isnan - if not skipna or mask.all(): - warnings.warn( - f"The behavior of {type(self).__name__}.argmax/argmin " - "with skipna=False and NAs, or with all-NAs is deprecated. " - "In a future version this will raise ValueError.", - FutureWarning, - stacklevel=find_stack_level(), - ) - return -1 + if self._isnan.all(): + raise ValueError("Encountered all NA values") + elif not skipna: + raise ValueError("Encountered an NA value with skipna=False") return super().argmin(skipna=skipna) @Appender(IndexOpsMixin.argmax.__doc__) @@ -6995,16 +6989,10 @@ def argmax(self, axis=None, skipna: bool = True, *args, **kwargs) -> int: if not self._is_multi and self.hasnans: # Take advantage of cache - mask = self._isnan - if not skipna or mask.all(): - warnings.warn( - f"The behavior of {type(self).__name__}.argmax/argmin " - "with skipna=False and NAs, or with all-NAs is deprecated. " - "In a future version this will raise ValueError.", - FutureWarning, - stacklevel=find_stack_level(), - ) - return -1 + if self._isnan.all(): + raise ValueError("Encountered all NA values") + elif not skipna: + raise ValueError("Encountered an NA value with skipna=False") return super().argmax(skipna=skipna) def min(self, axis=None, skipna: bool = True, *args, **kwargs): @@ -7169,24 +7157,17 @@ def maybe_sequence_to_range(sequence) -> Any | range: ------- Any : input or range """ - if isinstance(sequence, (ABCSeries, Index)): + if isinstance(sequence, (ABCSeries, Index, range, ExtensionArray)): return sequence - np_sequence = np.asarray(sequence) - if np_sequence.dtype.kind != "i" or len(np_sequence) == 1: + elif len(sequence) == 1 or lib.infer_dtype(sequence, skipna=False) != "integer": return sequence - elif len(np_sequence) == 0: + elif len(sequence) == 0: return range(0) - diff = np_sequence[1] - np_sequence[0] + diff = sequence[1] - sequence[0] if diff == 0: return sequence - elif len(np_sequence) == 2: - return range(np_sequence[0], np_sequence[1] + diff, diff) - maybe_range_indexer, remainder = np.divmod(np_sequence - np_sequence[0], diff) - if ( - lib.is_range_indexer(maybe_range_indexer, len(maybe_range_indexer)) - and not remainder.any() - ): - return range(np_sequence[0], np_sequence[-1] + diff, diff) + elif len(sequence) == 2 or lib.is_sequence_range(np.asarray(sequence), diff): + return range(sequence[0], sequence[-1] + diff, diff) else: return sequence diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 2d773c04b8ea9..cefdc14145d1f 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -28,7 +28,6 @@ cache_readonly, doc, ) -from pandas.util._exceptions import find_stack_level from pandas.core.dtypes.common import is_scalar from pandas.core.dtypes.dtypes import DatetimeTZDtype @@ -150,17 +149,6 @@ class DatetimeIndex(DatetimeTimedeltaMixin): inferred frequency upon creation. tz : pytz.timezone or dateutil.tz.tzfile or datetime.tzinfo or str Set the Timezone of the data. - normalize : bool, default False - Normalize start/end dates to midnight before generating date range. - - .. deprecated:: 2.1.0 - - closed : {'left', 'right'}, optional - Set whether to include `start` and `end` that are on the - boundary. The default includes boundary points on either end. - - .. deprecated:: 2.1.0 - ambiguous : 'infer', bool-ndarray, 'NaT', default 'raise' When clocks moved backward due to DST, ambiguous times may arise. For example in Central European Time (UTC+01), when going from 03:00 @@ -322,8 +310,6 @@ def __new__( data=None, freq: Frequency | lib.NoDefault = lib.no_default, tz=lib.no_default, - normalize: bool | lib.NoDefault = lib.no_default, - closed=lib.no_default, ambiguous: TimeAmbiguous = "raise", dayfirst: bool = False, yearfirst: bool = False, @@ -331,23 +317,6 @@ def __new__( copy: bool = False, name: Hashable | None = None, ) -> Self: - if closed is not lib.no_default: - # GH#52628 - warnings.warn( - f"The 'closed' keyword in {cls.__name__} construction is " - "deprecated and will be removed in a future version.", - FutureWarning, - stacklevel=find_stack_level(), - ) - if normalize is not lib.no_default: - # GH#52628 - warnings.warn( - f"The 'normalize' keyword in {cls.__name__} construction is " - "deprecated and will be removed in a future version.", - FutureWarning, - stacklevel=find_stack_level(), - ) - if is_scalar(data): cls._raise_scalar_data_error(data) diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py index 6a2c04b0ddf51..8af5a56f43c57 100644 --- a/pandas/core/indexes/timedeltas.py +++ b/pandas/core/indexes/timedeltas.py @@ -3,7 +3,6 @@ from __future__ import annotations from typing import TYPE_CHECKING -import warnings from pandas._libs import ( index as libindex, @@ -14,8 +13,6 @@ Timedelta, to_offset, ) -from pandas._libs.tslibs.timedeltas import disallow_ambiguous_unit -from pandas.util._exceptions import find_stack_level from pandas.core.dtypes.common import ( is_scalar, @@ -63,12 +60,6 @@ class TimedeltaIndex(DatetimeTimedeltaMixin): ---------- data : array-like (1-dimensional), optional Optional timedelta-like data to construct index with. - unit : {'D', 'h', 'm', 's', 'ms', 'us', 'ns'}, optional - The unit of ``data``. - - .. deprecated:: 2.2.0 - Use ``pd.to_timedelta`` instead. - freq : str or pandas offset object, optional One of pandas date offset strings or corresponding objects. The string ``'infer'`` can be passed in order to set the frequency of the index as @@ -151,40 +142,16 @@ def _resolution_obj(self) -> Resolution | None: # type: ignore[override] def __new__( cls, data=None, - unit=lib.no_default, freq=lib.no_default, - closed=lib.no_default, dtype=None, copy: bool = False, name=None, ): - if closed is not lib.no_default: - # GH#52628 - warnings.warn( - f"The 'closed' keyword in {cls.__name__} construction is " - "deprecated and will be removed in a future version.", - FutureWarning, - stacklevel=find_stack_level(), - ) - - if unit is not lib.no_default: - # GH#55499 - warnings.warn( - f"The 'unit' keyword in {cls.__name__} construction is " - "deprecated and will be removed in a future version. " - "Use pd.to_timedelta instead.", - FutureWarning, - stacklevel=find_stack_level(), - ) - else: - unit = None - name = maybe_extract_name(name, data, cls) if is_scalar(data): cls._raise_scalar_data_error(data) - disallow_ambiguous_unit(unit) if dtype is not None: dtype = pandas_dtype(dtype) @@ -211,7 +178,7 @@ def __new__( # - Cases checked above all return/raise before reaching here - # tdarr = TimedeltaArray._from_sequence_not_strict( - data, freq=freq, unit=unit, dtype=dtype, copy=copy + data, freq=freq, unit=None, dtype=dtype, copy=copy ) refs = None if not copy and isinstance(data, (ABCSeries, Index)): diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index c8a2e11dce3d7..6b4070ed6349c 100644 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -227,7 +227,7 @@ def iloc(self) -> _iLocIndexer: a b c d 0 1 2 3 4 >>> type(df.iloc[[0]]) - + >>> df.iloc[[0, 1]] a b c d diff --git a/pandas/core/interchange/dataframe.py b/pandas/core/interchange/dataframe.py index 1abacddfc7e3b..0a116af567e59 100644 --- a/pandas/core/interchange/dataframe.py +++ b/pandas/core/interchange/dataframe.py @@ -33,7 +33,7 @@ def __init__(self, df: DataFrame, allow_copy: bool = True) -> None: Constructor - an instance of this (private) class is returned from `pd.DataFrame.__dataframe__`. """ - self._df = df.rename(columns=str, copy=False) + self._df = df.rename(columns=str) self._allow_copy = allow_copy for i, _col in enumerate(self._df.columns): rechunked = maybe_rechunk(self._df.iloc[:, i], allow_copy=allow_copy) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index f6bf5dffb5f48..a7cdc7c39754d 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -1873,13 +1873,11 @@ def fillna( copy, refs = self._get_refs_and_copy(inplace) try: - new_values = self.values.fillna( - value=value, method=None, limit=limit, copy=copy - ) + new_values = self.values.fillna(value=value, limit=limit, copy=copy) except TypeError: # 3rd party EA that has not implemented copy keyword yet refs = None - new_values = self.values.fillna(value=value, method=None, limit=limit) + new_values = self.values.fillna(value=value, limit=limit) # issue the warning *after* retrying, in case the TypeError # was caused by an invalid fill_value warnings.warn( diff --git a/pandas/core/internals/construction.py b/pandas/core/internals/construction.py index 93f1674fbd328..73b93110c9018 100644 --- a/pandas/core/internals/construction.py +++ b/pandas/core/internals/construction.py @@ -60,6 +60,7 @@ default_index, ensure_index, get_objs_combined_axis, + maybe_sequence_to_range, union_indexes, ) from pandas.core.internals.blocks import ( @@ -403,7 +404,7 @@ def dict_to_mgr( arrays[i] = arr else: - keys = list(data.keys()) + keys = maybe_sequence_to_range(list(data.keys())) columns = Index(keys) if keys else default_index(0) arrays = [com.maybe_iterable_to_list(data[k]) for k in keys] diff --git a/pandas/core/methods/describe.py b/pandas/core/methods/describe.py index 380bf9ce55659..ef20d4c509732 100644 --- a/pandas/core/methods/describe.py +++ b/pandas/core/methods/describe.py @@ -175,6 +175,7 @@ def describe(self, percentiles: Sequence[float] | np.ndarray) -> DataFrame: d = concat( [x.reindex(col_names) for x in ldesc], axis=1, + ignore_index=True, sort=False, ) d.columns = data.columns.copy() diff --git a/pandas/core/missing.py b/pandas/core/missing.py index de26ad14a7b7a..9fef78d9f8c3d 100644 --- a/pandas/core/missing.py +++ b/pandas/core/missing.py @@ -322,13 +322,17 @@ def get_interp_index(method, index: Index) -> Index: or isinstance(index.dtype, DatetimeTZDtype) or lib.is_np_dtype(index.dtype, "mM") ) - if method not in methods and not is_numeric_or_datetime: - raise ValueError( - "Index column must be numeric or datetime type when " - f"using {method} method other than linear. " - "Try setting a numeric or datetime index column before " - "interpolating." - ) + valid = NP_METHODS + SP_METHODS + if method in valid: + if method not in methods and not is_numeric_or_datetime: + raise ValueError( + "Index column must be numeric or datetime type when " + f"using {method} method other than linear. " + "Try setting a numeric or datetime index column before " + "interpolating." + ) + else: + raise ValueError(f"Can not interpolate with method={method}.") if isna(index).any(): raise NotImplementedError( @@ -467,20 +471,20 @@ def _interpolate_1d( if valid.all(): return - # These are sets of index pointers to invalid values... i.e. {0, 1, etc... - all_nans = set(np.flatnonzero(invalid)) + # These index pointers to invalid values... i.e. {0, 1, etc... + all_nans = np.flatnonzero(invalid) first_valid_index = find_valid_index(how="first", is_valid=valid) if first_valid_index is None: # no nan found in start first_valid_index = 0 - start_nans = set(range(first_valid_index)) + start_nans = np.arange(first_valid_index) last_valid_index = find_valid_index(how="last", is_valid=valid) if last_valid_index is None: # no nan found in end last_valid_index = len(yvalues) - end_nans = set(range(1 + last_valid_index, len(valid))) + end_nans = np.arange(1 + last_valid_index, len(valid)) - # Like the sets above, preserve_nans contains indices of invalid values, + # preserve_nans contains indices of invalid values, # but in this case, it is the final set of indices that need to be # preserved as NaN after the interpolation. @@ -489,27 +493,25 @@ def _interpolate_1d( # are more than 'limit' away from the prior non-NaN. # set preserve_nans based on direction using _interp_limit - preserve_nans: list | set if limit_direction == "forward": - preserve_nans = start_nans | set(_interp_limit(invalid, limit, 0)) + preserve_nans = np.union1d(start_nans, _interp_limit(invalid, limit, 0)) elif limit_direction == "backward": - preserve_nans = end_nans | set(_interp_limit(invalid, 0, limit)) + preserve_nans = np.union1d(end_nans, _interp_limit(invalid, 0, limit)) else: # both directions... just use _interp_limit - preserve_nans = set(_interp_limit(invalid, limit, limit)) + preserve_nans = np.unique(_interp_limit(invalid, limit, limit)) # if limit_area is set, add either mid or outside indices # to preserve_nans GH #16284 if limit_area == "inside": # preserve NaNs on the outside - preserve_nans |= start_nans | end_nans + preserve_nans = np.union1d(preserve_nans, start_nans) + preserve_nans = np.union1d(preserve_nans, end_nans) elif limit_area == "outside": # preserve NaNs on the inside - mid_nans = all_nans - start_nans - end_nans - preserve_nans |= mid_nans - - # sort preserve_nans and convert to list - preserve_nans = sorted(preserve_nans) + mid_nans = np.setdiff1d(all_nans, start_nans, assume_unique=True) + mid_nans = np.setdiff1d(mid_nans, end_nans, assume_unique=True) + preserve_nans = np.union1d(preserve_nans, mid_nans) is_datetimelike = yvalues.dtype.kind in "mM" @@ -611,7 +613,9 @@ def _interpolate_scipy_wrapper( y = y.copy() if not new_x.flags.writeable: new_x = new_x.copy() - terp = alt_methods[method] + terp = alt_methods.get(method, None) + if terp is None: + raise ValueError(f"Can not interpolate with method={method}.") new_y = terp(x, y, new_x, **kwargs) return new_y @@ -1021,7 +1025,7 @@ def clean_reindex_fill_method(method) -> ReindexMethod | None: def _interp_limit( invalid: npt.NDArray[np.bool_], fw_limit: int | None, bw_limit: int | None -): +) -> np.ndarray: """ Get indexers of values that won't be filled because they exceed the limits. @@ -1053,20 +1057,23 @@ def _interp_limit(invalid, fw_limit, bw_limit): # 1. operate on the reversed array # 2. subtract the returned indices from N - 1 N = len(invalid) - f_idx = set() - b_idx = set() + f_idx = np.array([], dtype=np.int64) + b_idx = np.array([], dtype=np.int64) + assume_unique = True def inner(invalid, limit: int): limit = min(limit, N) - windowed = _rolling_window(invalid, limit + 1).all(1) - idx = set(np.where(windowed)[0] + limit) | set( - np.where((~invalid[: limit + 1]).cumsum() == 0)[0] + windowed = np.lib.stride_tricks.sliding_window_view(invalid, limit + 1).all(1) + idx = np.union1d( + np.where(windowed)[0] + limit, + np.where((~invalid[: limit + 1]).cumsum() == 0)[0], ) return idx if fw_limit is not None: if fw_limit == 0: - f_idx = set(np.where(invalid)[0]) + f_idx = np.where(invalid)[0] + assume_unique = False else: f_idx = inner(invalid, fw_limit) @@ -1076,26 +1083,8 @@ def inner(invalid, limit: int): # just use forwards return f_idx else: - b_idx_inv = list(inner(invalid[::-1], bw_limit)) - b_idx = set(N - 1 - np.asarray(b_idx_inv)) + b_idx = N - 1 - inner(invalid[::-1], bw_limit) if fw_limit == 0: return b_idx - return f_idx & b_idx - - -def _rolling_window(a: npt.NDArray[np.bool_], window: int) -> npt.NDArray[np.bool_]: - """ - [True, True, False, True, False], 2 -> - - [ - [True, True], - [True, False], - [False, True], - [True, False], - ] - """ - # https://stackoverflow.com/a/6811241 - shape = a.shape[:-1] + (a.shape[-1] - window + 1, window) - strides = a.strides + (a.strides[-1],) - return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides) + return np.intersect1d(f_idx, b_idx, assume_unique=assume_unique) diff --git a/pandas/core/nanops.py b/pandas/core/nanops.py index 6cb825e9b79a2..b68337d9e0de9 100644 --- a/pandas/core/nanops.py +++ b/pandas/core/nanops.py @@ -1441,17 +1441,18 @@ def _maybe_arg_null_out( if axis is None or not getattr(result, "ndim", False): if skipna: if mask.all(): - return -1 + raise ValueError("Encountered all NA values") else: if mask.any(): - return -1 + raise ValueError("Encountered an NA value with skipna=False") else: - if skipna: - na_mask = mask.all(axis) - else: - na_mask = mask.any(axis) + na_mask = mask.all(axis) if na_mask.any(): - result[na_mask] = -1 + raise ValueError("Encountered all NA values") + elif not skipna: + na_mask = mask.any(axis) + if na_mask.any(): + raise ValueError("Encountered an NA value with skipna=False") return result diff --git a/pandas/core/reshape/concat.py b/pandas/core/reshape/concat.py index 35a08e0167924..0868f711093d6 100644 --- a/pandas/core/reshape/concat.py +++ b/pandas/core/reshape/concat.py @@ -12,10 +12,13 @@ cast, overload, ) +import warnings import numpy as np +from pandas._libs import lib from pandas.util._decorators import cache_readonly +from pandas.util._exceptions import find_stack_level from pandas.core.dtypes.common import is_bool from pandas.core.dtypes.concat import concat_compat @@ -75,7 +78,7 @@ def concat( names: list[HashableT] | None = ..., verify_integrity: bool = ..., sort: bool = ..., - copy: bool | None = ..., + copy: bool | lib.NoDefault = ..., ) -> DataFrame: ... @@ -91,7 +94,7 @@ def concat( names: list[HashableT] | None = ..., verify_integrity: bool = ..., sort: bool = ..., - copy: bool | None = ..., + copy: bool | lib.NoDefault = ..., ) -> Series: ... @@ -107,7 +110,7 @@ def concat( names: list[HashableT] | None = ..., verify_integrity: bool = ..., sort: bool = ..., - copy: bool | None = ..., + copy: bool | lib.NoDefault = ..., ) -> DataFrame | Series: ... @@ -123,7 +126,7 @@ def concat( names: list[HashableT] | None = ..., verify_integrity: bool = ..., sort: bool = ..., - copy: bool | None = ..., + copy: bool | lib.NoDefault = ..., ) -> DataFrame: ... @@ -139,7 +142,7 @@ def concat( names: list[HashableT] | None = ..., verify_integrity: bool = ..., sort: bool = ..., - copy: bool | None = ..., + copy: bool | lib.NoDefault = ..., ) -> DataFrame | Series: ... @@ -154,7 +157,7 @@ def concat( names: list[HashableT] | None = None, verify_integrity: bool = False, sort: bool = False, - copy: bool | None = None, + copy: bool | lib.NoDefault = lib.no_default, ) -> DataFrame | Series: """ Concatenate pandas objects along a particular axis. @@ -198,9 +201,23 @@ def concat( non-concatentation axis is a DatetimeIndex and join='outer' and the axis is not already aligned. In that case, the non-concatenation axis is always sorted lexicographically. - copy : bool, default True + copy : bool, default False If False, do not copy data unnecessarily. + .. note:: + The `copy` keyword will change behavior in pandas 3.0. + `Copy-on-Write + `__ + will be enabled by default, which means that all methods with a + `copy` keyword will use a lazy copy mechanism to defer the copy and + ignore the `copy` keyword. The `copy` keyword will be removed in a + future version of pandas. + + You can already get the future behavior and improvements through + enabling copy on write ``pd.options.mode.copy_on_write = True`` + + .. deprecated:: 3.0.0 + Returns ------- object, type of objs @@ -359,6 +376,15 @@ def concat( 0 1 2 1 3 4 """ + if copy is not lib.no_default: + warnings.warn( + "The copy keyword is deprecated and will be removed in a future " + "version. Copy-on-Write is active in pandas since 3.0 which utilizes " + "a lazy copy mechanism that defers copies until necessary. Use " + ".copy() to make an eager copy if necessary.", + DeprecationWarning, + stacklevel=find_stack_level(), + ) op = _Concatenator( objs, @@ -635,16 +661,13 @@ def _get_concat_axis(self) -> Index: indexes, self.keys, self.levels, self.names ) - self._maybe_check_integrity(concat_axis) - - return concat_axis - - def _maybe_check_integrity(self, concat_index: Index) -> None: if self.verify_integrity: - if not concat_index.is_unique: - overlap = concat_index[concat_index.duplicated()].unique() + if not concat_axis.is_unique: + overlap = concat_axis[concat_axis.duplicated()].unique() raise ValueError(f"Indexes have overlapping values: {overlap}") + return concat_axis + def _clean_keys_and_objs( objs: Iterable[Series | DataFrame] | Mapping[HashableT, Series | DataFrame], @@ -742,6 +765,12 @@ def _concat_indexes(indexes) -> Index: return indexes[0].append(indexes[1:]) +def validate_unique_levels(levels: list[Index]) -> None: + for level in levels: + if not level.is_unique: + raise ValueError(f"Level values not unique: {level.tolist()}") + + def _make_concat_multiindex(indexes, keys, levels=None, names=None) -> MultiIndex: if (levels is None and isinstance(keys[0], tuple)) or ( levels is not None and len(levels) > 1 @@ -754,6 +783,7 @@ def _make_concat_multiindex(indexes, keys, levels=None, names=None) -> MultiInde _, levels = factorize_from_iterables(zipped) else: levels = [ensure_index(x) for x in levels] + validate_unique_levels(levels) else: zipped = [keys] if names is None: @@ -763,12 +793,9 @@ def _make_concat_multiindex(indexes, keys, levels=None, names=None) -> MultiInde levels = [ensure_index(keys).unique()] else: levels = [ensure_index(x) for x in levels] + validate_unique_levels(levels) - for level in levels: - if not level.is_unique: - raise ValueError(f"Level values not unique: {level.tolist()}") - - if not all_indexes_same(indexes) or not all(level.is_unique for level in levels): + if not all_indexes_same(indexes): codes_list = [] # things are potentially different sizes, so compute the exact codes diff --git a/pandas/core/reshape/melt.py b/pandas/core/reshape/melt.py index 24a070a536150..f51a833e5f906 100644 --- a/pandas/core/reshape/melt.py +++ b/pandas/core/reshape/melt.py @@ -243,7 +243,7 @@ def melt( not isinstance(dt, np.dtype) and dt._supports_2d for dt in frame.dtypes ): mdata[value_name] = concat( - [frame.iloc[:, i] for i in range(frame.shape[1])] + [frame.iloc[:, i] for i in range(frame.shape[1])], ignore_index=True ).values else: mdata[value_name] = frame._values.ravel("F") diff --git a/pandas/core/reshape/merge.py b/pandas/core/reshape/merge.py index 8ea2ac24e13c8..dcb638cfee97b 100644 --- a/pandas/core/reshape/merge.py +++ b/pandas/core/reshape/merge.py @@ -145,11 +145,12 @@ def merge( right_index: bool = False, sort: bool = False, suffixes: Suffixes = ("_x", "_y"), - copy: bool | None = None, + copy: bool | lib.NoDefault = lib.no_default, indicator: str | bool = False, validate: str | None = None, ) -> DataFrame: left_df = _validate_operand(left) + left._check_copy_deprecation(copy) right_df = _validate_operand(right) if how == "cross": return _cross_merge( @@ -1780,7 +1781,10 @@ def get_join_indexers_non_unique( np.ndarray[np.intp] Indexer into right. """ - lkey, rkey, count = _factorize_keys(left, right, sort=sort) + lkey, rkey, count = _factorize_keys(left, right, sort=sort, how=how) + if count == -1: + # hash join + return lkey, rkey if how == "left": lidx, ridx = libjoin.left_outer_join(lkey, rkey, count, sort=sort) elif how == "right": @@ -2385,7 +2389,10 @@ def _left_join_on_index( def _factorize_keys( - lk: ArrayLike, rk: ArrayLike, sort: bool = True + lk: ArrayLike, + rk: ArrayLike, + sort: bool = True, + how: str | None = None, ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp], int]: """ Encode left and right keys as enumerated types. @@ -2401,6 +2408,9 @@ def _factorize_keys( sort : bool, defaults to True If True, the encoding is done such that the unique elements in the keys are sorted. + how: str, optional + Used to determine if we can use hash-join. If not given, then just factorize + keys. Returns ------- @@ -2409,7 +2419,8 @@ def _factorize_keys( np.ndarray[np.intp] Right (resp. left if called with `key='right'`) labels, as enumerated type. int - Number of unique elements in union of left and right labels. + Number of unique elements in union of left and right labels. -1 if we used + a hash-join. See Also -------- @@ -2527,28 +2538,41 @@ def _factorize_keys( klass, lk, rk = _convert_arrays_and_get_rizer_klass(lk, rk) - rizer = klass(max(len(lk), len(rk))) + rizer = klass( + max(len(lk), len(rk)), + uses_mask=isinstance(rk, (BaseMaskedArray, ArrowExtensionArray)), + ) if isinstance(lk, BaseMaskedArray): assert isinstance(rk, BaseMaskedArray) - llab = rizer.factorize(lk._data, mask=lk._mask) - rlab = rizer.factorize(rk._data, mask=rk._mask) + lk_data, lk_mask = lk._data, lk._mask + rk_data, rk_mask = rk._data, rk._mask elif isinstance(lk, ArrowExtensionArray): assert isinstance(rk, ArrowExtensionArray) # we can only get here with numeric dtypes # TODO: Remove when we have a Factorizer for Arrow - llab = rizer.factorize( - lk.to_numpy(na_value=1, dtype=lk.dtype.numpy_dtype), mask=lk.isna() - ) - rlab = rizer.factorize( - rk.to_numpy(na_value=1, dtype=lk.dtype.numpy_dtype), mask=rk.isna() - ) + lk_data = lk.to_numpy(na_value=1, dtype=lk.dtype.numpy_dtype) + rk_data = rk.to_numpy(na_value=1, dtype=lk.dtype.numpy_dtype) + lk_mask, rk_mask = lk.isna(), rk.isna() else: # Argument 1 to "factorize" of "ObjectFactorizer" has incompatible type # "Union[ndarray[Any, dtype[signedinteger[_64Bit]]], # ndarray[Any, dtype[object_]]]"; expected "ndarray[Any, dtype[object_]]" - llab = rizer.factorize(lk) # type: ignore[arg-type] - rlab = rizer.factorize(rk) # type: ignore[arg-type] + lk_data, rk_data = lk, rk # type: ignore[assignment] + lk_mask, rk_mask = None, None + + hash_join_available = how == "inner" and not sort and lk.dtype.kind in "iufb" + if hash_join_available: + rlab = rizer.factorize(rk_data, mask=rk_mask) + if rizer.get_count() == len(rlab): + ridx, lidx = rizer.hash_inner_join(lk_data, lk_mask) + return lidx, ridx, -1 + else: + llab = rizer.factorize(lk_data, mask=lk_mask) + else: + llab = rizer.factorize(lk_data, mask=lk_mask) + rlab = rizer.factorize(rk_data, mask=rk_mask) + assert llab.dtype == np.dtype(np.intp), llab.dtype assert rlab.dtype == np.dtype(np.intp), rlab.dtype diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 7b2fbb54f7d35..b62f550662f5d 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -835,7 +835,7 @@ def _normalize( elif normalize == "index": index_margin = index_margin / index_margin.sum() - table = table._append(index_margin) + table = table._append(index_margin, ignore_index=True) table = table.fillna(0) table.index = table_index @@ -844,7 +844,7 @@ def _normalize( index_margin = index_margin / index_margin.sum() index_margin.loc[margins_name] = 1 table = concat([table, column_margin], axis=1) - table = table._append(index_margin) + table = table._append(index_margin, ignore_index=True) table = table.fillna(0) table.index = table_index diff --git a/pandas/core/reshape/reshape.py b/pandas/core/reshape/reshape.py index ff358e8ba346c..afb0c489c9c94 100644 --- a/pandas/core/reshape/reshape.py +++ b/pandas/core/reshape/reshape.py @@ -953,7 +953,7 @@ def stack_v3(frame: DataFrame, level: list[int]) -> Series | DataFrame: result: Series | DataFrame if len(buf) > 0 and not frame.empty: - result = concat(buf) + result = concat(buf, ignore_index=True) ratio = len(result) // len(frame) else: # input is empty diff --git a/pandas/core/series.py b/pandas/core/series.py index 08e56cb4925b3..0761dc17ab147 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -576,14 +576,17 @@ def _constructor(self) -> type[Series]: return Series def _constructor_from_mgr(self, mgr, axes): - if self._constructor is Series: - # we are pandas.Series (or a subclass that doesn't override _constructor) - ser = Series._from_mgr(mgr, axes=axes) - ser._name = None # caller is responsible for setting real name + ser = Series._from_mgr(mgr, axes=axes) + ser._name = None # caller is responsible for setting real name + + if type(self) is Series: + # This would also work `if self._constructor is Series`, but + # this check is slightly faster, benefiting the most-common case. return ser - else: - assert axes is mgr.axes - return self._constructor(mgr) + + # We assume that the subclass __init__ knows how to handle a + # pd.Series object. + return self._constructor(ser) @property def _constructor_expanddim(self) -> Callable[..., DataFrame]: @@ -595,18 +598,19 @@ def _constructor_expanddim(self) -> Callable[..., DataFrame]: return DataFrame - def _expanddim_from_mgr(self, mgr, axes) -> DataFrame: + def _constructor_expanddim_from_mgr(self, mgr, axes): from pandas.core.frame import DataFrame - return DataFrame._from_mgr(mgr, axes=mgr.axes) + df = DataFrame._from_mgr(mgr, axes=mgr.axes) - def _constructor_expanddim_from_mgr(self, mgr, axes): - from pandas.core.frame import DataFrame + if type(self) is Series: + # This would also work `if self._constructor_expanddim is DataFrame`, + # but this check is slightly faster, benefiting the most-common case. + return df - if self._constructor_expanddim is DataFrame: - return self._expanddim_from_mgr(mgr, axes) - assert axes is mgr.axes - return self._constructor_expanddim(mgr) + # We assume that the subclass __init__ knows how to handle a + # pd.DataFrame object. + return self._constructor_expanddim(df) # types @property @@ -2333,8 +2337,8 @@ def idxmin(self, axis: Axis = 0, skipna: bool = True, *args, **kwargs) -> Hashab axis : {0 or 'index'} Unused. Parameter needed for compatibility with DataFrame. skipna : bool, default True - Exclude NA/null values. If the entire Series is NA, the result - will be NA. + Exclude NA/null values. If the entire Series is NA, or if ``skipna=False`` + and there is an NA value, this method will raise a ``ValueError``. *args, **kwargs Additional arguments and keywords have no effect but might be accepted for compatibility with NumPy. @@ -2376,32 +2380,10 @@ def idxmin(self, axis: Axis = 0, skipna: bool = True, *args, **kwargs) -> Hashab >>> s.idxmin() 'A' - - If `skipna` is False and there is an NA value in the data, - the function returns ``nan``. - - >>> s.idxmin(skipna=False) - nan """ axis = self._get_axis_number(axis) - with warnings.catch_warnings(): - # TODO(3.0): this catching/filtering can be removed - # ignore warning produced by argmin since we will issue a different - # warning for idxmin - warnings.simplefilter("ignore") - i = self.argmin(axis, skipna, *args, **kwargs) - - if i == -1: - # GH#43587 give correct NA value for Index. - warnings.warn( - f"The behavior of {type(self).__name__}.idxmin with all-NA " - "values, or any-NA and skipna=False, is deprecated. In a future " - "version this will raise ValueError", - FutureWarning, - stacklevel=find_stack_level(), - ) - return self.index._na_value - return self.index[i] + iloc = self.argmin(axis, skipna, *args, **kwargs) + return self.index[iloc] def idxmax(self, axis: Axis = 0, skipna: bool = True, *args, **kwargs) -> Hashable: """ @@ -2415,8 +2397,8 @@ def idxmax(self, axis: Axis = 0, skipna: bool = True, *args, **kwargs) -> Hashab axis : {0 or 'index'} Unused. Parameter needed for compatibility with DataFrame. skipna : bool, default True - Exclude NA/null values. If the entire Series is NA, the result - will be NA. + Exclude NA/null values. If the entire Series is NA, or if ``skipna=False`` + and there is an NA value, this method will raise a ``ValueError``. *args, **kwargs Additional arguments and keywords have no effect but might be accepted for compatibility with NumPy. @@ -2459,32 +2441,10 @@ def idxmax(self, axis: Axis = 0, skipna: bool = True, *args, **kwargs) -> Hashab >>> s.idxmax() 'C' - - If `skipna` is False and there is an NA value in the data, - the function returns ``nan``. - - >>> s.idxmax(skipna=False) - nan """ axis = self._get_axis_number(axis) - with warnings.catch_warnings(): - # TODO(3.0): this catching/filtering can be removed - # ignore warning produced by argmax since we will issue a different - # warning for argmax - warnings.simplefilter("ignore") - i = self.argmax(axis, skipna, *args, **kwargs) - - if i == -1: - # GH#43587 give correct NA value for Index. - warnings.warn( - f"The behavior of {type(self).__name__}.idxmax with all-NA " - "values, or any-NA and skipna=False, is deprecated. In a future " - "version this will raise ValueError", - FutureWarning, - stacklevel=find_stack_level(), - ) - return self.index._na_value - return self.index[i] + iloc = self.argmax(axis, skipna, *args, **kwargs) + return self.index[iloc] def round(self, decimals: int = 0, *args, **kwargs) -> Series: """ @@ -2509,13 +2469,21 @@ def round(self, decimals: int = 0, *args, **kwargs) -> Series: numpy.around : Round values of an np.array. DataFrame.round : Round values of a DataFrame. + Notes + ----- + For values exactly halfway between rounded decimal values, pandas rounds + to the nearest even value (e.g. -0.5 and 0.5 round to 0.0, 1.5 and 2.5 + round to 2.0, etc.). + Examples -------- - >>> s = pd.Series([0.1, 1.3, 2.7]) + >>> s = pd.Series([-0.5, 0.1, 2.5, 1.3, 2.7]) >>> s.round() - 0 0.0 - 1 1.0 - 2 3.0 + 0 -0.0 + 1 0.0 + 2 2.0 + 3 1.0 + 4 3.0 dtype: float64 """ nv.validate_round(args, kwargs) @@ -4125,7 +4093,7 @@ def nsmallest( ), ) def swaplevel( - self, i: Level = -2, j: Level = -1, copy: bool | None = None + self, i: Level = -2, j: Level = -1, copy: bool | lib.NoDefault = lib.no_default ) -> Series: """ Swap levels i and j in a :class:`MultiIndex`. @@ -4145,6 +4113,7 @@ def swaplevel( {examples} """ + self._check_copy_deprecation(copy) assert isinstance(self.index, MultiIndex) result = self.copy(deep=False) result.index = self.index.swaplevel(i, j) @@ -4643,7 +4612,7 @@ def rename( index: Renamer | Hashable | None = ..., *, axis: Axis | None = ..., - copy: bool = ..., + copy: bool | lib.NoDefault = ..., inplace: Literal[True], level: Level | None = ..., errors: IgnoreRaise = ..., @@ -4655,7 +4624,7 @@ def rename( index: Renamer | Hashable | None = ..., *, axis: Axis | None = ..., - copy: bool = ..., + copy: bool | lib.NoDefault = ..., inplace: Literal[False] = ..., level: Level | None = ..., errors: IgnoreRaise = ..., @@ -4667,7 +4636,7 @@ def rename( index: Renamer | Hashable | None = ..., *, axis: Axis | None = ..., - copy: bool = ..., + copy: bool | lib.NoDefault = ..., inplace: bool = ..., level: Level | None = ..., errors: IgnoreRaise = ..., @@ -4678,7 +4647,7 @@ def rename( index: Renamer | Hashable | None = None, *, axis: Axis | None = None, - copy: bool | None = None, + copy: bool | lib.NoDefault = lib.no_default, inplace: bool = False, level: Level | None = None, errors: IgnoreRaise = "ignore", @@ -4703,7 +4672,7 @@ def rename( attribute. axis : {0 or 'index'} Unused. Parameter needed for compatibility with DataFrame. - copy : bool, default True + copy : bool, default False Also copy underlying data. .. note:: @@ -4717,6 +4686,8 @@ def rename( You can already get the future behavior and improvements through enabling copy on write ``pd.options.mode.copy_on_write = True`` + + .. deprecated:: 3.0.0 inplace : bool, default False Whether to return a new Series. If True the value of copy is ignored. level : int or level name, default None @@ -4760,6 +4731,7 @@ def rename( 5 3 dtype: int64 """ + self._check_copy_deprecation(copy) if axis is not None: # Make sure we raise if an invalid 'axis' is passed. axis = self._get_axis_number(axis) @@ -4809,9 +4781,9 @@ def set_axis( labels, *, axis: Axis = 0, - copy: bool | None = None, + copy: bool | lib.NoDefault = lib.no_default, ) -> Series: - return super().set_axis(labels, axis=axis) + return super().set_axis(labels, axis=axis, copy=copy) # error: Cannot determine type of 'reindex' @doc( @@ -4848,7 +4820,7 @@ def rename_axis( *, index=..., axis: Axis = ..., - copy: bool = ..., + copy: bool | lib.NoDefault = ..., inplace: Literal[True], ) -> None: ... @@ -4859,7 +4831,7 @@ def rename_axis( *, index=..., axis: Axis = ..., - copy: bool = ..., + copy: bool | lib.NoDefault = ..., inplace: Literal[False] = ..., ) -> Self: ... @@ -4870,7 +4842,7 @@ def rename_axis( *, index=..., axis: Axis = ..., - copy: bool = ..., + copy: bool | lib.NoDefault = ..., inplace: bool = ..., ) -> Self | None: ... @@ -4880,7 +4852,7 @@ def rename_axis( *, index=lib.no_default, axis: Axis = 0, - copy: bool = True, + copy: bool | lib.NoDefault = lib.no_default, inplace: bool = False, ) -> Self | None: """ @@ -4899,7 +4871,7 @@ def rename_axis( apply to that axis' values. axis : {0 or 'index'}, default 0 The axis to rename. For `Series` this parameter is unused and defaults to 0. - copy : bool, default None + copy : bool, default False Also copy underlying data. .. note:: @@ -4949,6 +4921,7 @@ def rename_axis( index=index, axis=axis, inplace=inplace, + copy=copy, ) @overload @@ -5672,7 +5645,7 @@ def to_timestamp( self, freq: Frequency | None = None, how: Literal["s", "e", "start", "end"] = "start", - copy: bool | None = None, + copy: bool | lib.NoDefault = lib.no_default, ) -> Series: """ Cast to DatetimeIndex of Timestamps, at *beginning* of period. @@ -5684,7 +5657,7 @@ def to_timestamp( how : {'s', 'e', 'start', 'end'} Convention for converting period to timestamp; start of period vs. end. - copy : bool, default True + copy : bool, default False Whether or not to return a copy. .. note:: @@ -5699,6 +5672,8 @@ def to_timestamp( You can already get the future behavior and improvements through enabling copy on write ``pd.options.mode.copy_on_write = True`` + .. deprecated:: 3.0.0 + Returns ------- Series with DatetimeIndex @@ -5732,6 +5707,7 @@ def to_timestamp( 2025-01-31 3 Freq: YE-JAN, dtype: int64 """ + self._check_copy_deprecation(copy) if not isinstance(self.index, PeriodIndex): raise TypeError(f"unsupported Type {type(self.index).__name__}") @@ -5740,7 +5716,11 @@ def to_timestamp( setattr(new_obj, "index", new_index) return new_obj - def to_period(self, freq: str | None = None, copy: bool | None = None) -> Series: + def to_period( + self, + freq: str | None = None, + copy: bool | lib.NoDefault = lib.no_default, + ) -> Series: """ Convert Series from DatetimeIndex to PeriodIndex. @@ -5748,7 +5728,7 @@ def to_period(self, freq: str | None = None, copy: bool | None = None) -> Series ---------- freq : str, default None Frequency associated with the PeriodIndex. - copy : bool, default True + copy : bool, default False Whether or not to return a copy. .. note:: @@ -5763,6 +5743,8 @@ def to_period(self, freq: str | None = None, copy: bool | None = None) -> Series You can already get the future behavior and improvements through enabling copy on write ``pd.options.mode.copy_on_write = True`` + .. deprecated:: 3.0.0 + Returns ------- Series @@ -5784,6 +5766,7 @@ def to_period(self, freq: str | None = None, copy: bool | None = None) -> Series >>> s.index PeriodIndex(['2023', '2024', '2025'], dtype='period[Y-DEC]') """ + self._check_copy_deprecation(copy) if not isinstance(self.index, DatetimeIndex): raise TypeError(f"unsupported Type {type(self.index).__name__}") diff --git a/pandas/core/shared_docs.py b/pandas/core/shared_docs.py index 15aa210a09d6d..a2b5439f9e12f 100644 --- a/pandas/core/shared_docs.py +++ b/pandas/core/shared_docs.py @@ -692,8 +692,8 @@ axis : {{0 or 'index', 1 or 'columns'}}, default 0 The axis to use. 0 or 'index' for row-wise, 1 or 'columns' for column-wise. skipna : bool, default True - Exclude NA/null values. If an entire row/column is NA, the result - will be NA. + Exclude NA/null values. If the entire Series is NA, or if ``skipna=False`` + and there is an NA value, this method will raise a ``ValueError``. numeric_only : bool, default {numeric_only_default} Include only `float`, `int` or `boolean` data. @@ -757,8 +757,8 @@ axis : {{0 or 'index', 1 or 'columns'}}, default 0 The axis to use. 0 or 'index' for row-wise, 1 or 'columns' for column-wise. skipna : bool, default True - Exclude NA/null values. If an entire row/column is NA, the result - will be NA. + Exclude NA/null values. If the entire Series is NA, or if ``skipna=False`` + and there is an NA value, this method will raise a ``ValueError``. numeric_only : bool, default {numeric_only_default} Include only `float`, `int` or `boolean` data. diff --git a/pandas/core/tools/timedeltas.py b/pandas/core/tools/timedeltas.py index 409a27ea64488..296168fe7e725 100644 --- a/pandas/core/tools/timedeltas.py +++ b/pandas/core/tools/timedeltas.py @@ -112,18 +112,17 @@ def to_timedelta( * 'W' * 'D' / 'days' / 'day' * 'hours' / 'hour' / 'hr' / 'h' / 'H' - * 'm' / 'minute' / 'min' / 'minutes' / 'T' + * 'm' / 'minute' / 'min' / 'minutes' * 's' / 'seconds' / 'sec' / 'second' / 'S' - * 'ms' / 'milliseconds' / 'millisecond' / 'milli' / 'millis' / 'L' - * 'us' / 'microseconds' / 'microsecond' / 'micro' / 'micros' / 'U' - * 'ns' / 'nanoseconds' / 'nano' / 'nanos' / 'nanosecond' / 'N' + * 'ms' / 'milliseconds' / 'millisecond' / 'milli' / 'millis' + * 'us' / 'microseconds' / 'microsecond' / 'micro' / 'micros' + * 'ns' / 'nanoseconds' / 'nano' / 'nanos' / 'nanosecond' Must not be specified when `arg` contains strings and ``errors="raise"``. .. deprecated:: 2.2.0 - Units 'H', 'T', 'S', 'L', 'U' and 'N' are deprecated and will be removed - in a future version. Please use 'h', 'min', 's', 'ms', 'us', and 'ns' - instead of 'H', 'T', 'S', 'L', 'U' and 'N'. + Units 'H'and 'S' are deprecated and will be removed + in a future version. Please use 'h' and 's'. errors : {'raise', 'coerce'}, default 'raise' - If 'raise', then invalid parsing will raise an exception. diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 8566751b9f33e..c503121328f53 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -855,7 +855,7 @@ class DataFrameRenderer: - to_csv - to_latex - Called in pandas.core.frame.DataFrame: + Called in pandas.DataFrame: - to_html - to_string diff --git a/pandas/io/formats/info.py b/pandas/io/formats/info.py index ad595a2be8374..bb156f0fbf826 100644 --- a/pandas/io/formats/info.py +++ b/pandas/io/formats/info.py @@ -72,7 +72,7 @@ Prints information of all columns: >>> df.info(verbose=True) - + RangeIndex: 5 entries, 0 to 4 Data columns (total 3 columns): # Column Non-Null Count Dtype @@ -87,7 +87,7 @@ information: >>> df.info(verbose=False) - + RangeIndex: 5 entries, 0 to 4 Columns: 3 entries, int_col to float_col dtypes: float64(1), int64(1), object(1) @@ -115,7 +115,7 @@ ... 'column_3': np.random.choice(['a', 'b', 'c'], 10 ** 6) ... }) >>> df.info() - + RangeIndex: 1000000 entries, 0 to 999999 Data columns (total 3 columns): # Column Non-Null Count Dtype @@ -127,7 +127,7 @@ memory usage: 22.9+ MB >>> df.info(memory_usage='deep') - + RangeIndex: 1000000 entries, 0 to 999999 Data columns (total 3 columns): # Column Non-Null Count Dtype diff --git a/pandas/io/formats/style.py b/pandas/io/formats/style.py index 7247e11be874e..ab5f1c039b7ca 100644 --- a/pandas/io/formats/style.py +++ b/pandas/io/formats/style.py @@ -1683,6 +1683,8 @@ def _copy(self, deepcopy: bool = False) -> Styler: "_display_funcs", "_display_funcs_index", "_display_funcs_columns", + "_display_funcs_index_names", + "_display_funcs_column_names", "hidden_rows", "hidden_columns", "ctx", diff --git a/pandas/io/formats/style_render.py b/pandas/io/formats/style_render.py index 2c93dbe74eace..92afbc0e150ef 100644 --- a/pandas/io/formats/style_render.py +++ b/pandas/io/formats/style_render.py @@ -140,9 +140,15 @@ def __init__( self._display_funcs_index: DefaultDict[ # maps (row, level) -> format func tuple[int, int], Callable[[Any], str] ] = defaultdict(lambda: partial(_default_formatter, precision=precision)) + self._display_funcs_index_names: DefaultDict[ # maps index level -> format func + int, Callable[[Any], str] + ] = defaultdict(lambda: partial(_default_formatter, precision=precision)) self._display_funcs_columns: DefaultDict[ # maps (level, col) -> format func tuple[int, int], Callable[[Any], str] ] = defaultdict(lambda: partial(_default_formatter, precision=precision)) + self._display_funcs_column_names: DefaultDict[ # maps col level -> format func + int, Callable[[Any], str] + ] = defaultdict(lambda: partial(_default_formatter, precision=precision)) def _render( self, @@ -460,6 +466,12 @@ def _generate_col_header_row( ] * (self.index.nlevels - sum(self.hide_index_) - 1) name = self.data.columns.names[r] + + is_display = name is not None and not self.hide_column_names + value = name if is_display else self.css["blank_value"] + display_value = ( + self._display_funcs_column_names[r](value) if is_display else None + ) column_name = [ _element( "th", @@ -468,10 +480,9 @@ def _generate_col_header_row( if name is None else f"{self.css['index_name']} {self.css['level']}{r}" ), - name - if (name is not None and not self.hide_column_names) - else self.css["blank_value"], + value, not all(self.hide_index_), + display_value=display_value, ) ] @@ -553,6 +564,9 @@ def _generate_index_names_row( f"{self.css['index_name']} {self.css['level']}{c}", self.css["blank_value"] if name is None else name, not self.hide_index_[c], + display_value=( + None if name is None else self._display_funcs_index_names[c](name) + ), ) for c, name in enumerate(self.data.index.names) ] @@ -1005,6 +1019,7 @@ def format( Returns ------- Styler + Returns itself for chaining. See Also -------- @@ -1261,6 +1276,7 @@ def format_index( Returns ------- Styler + Returns itself for chaining. See Also -------- @@ -1425,6 +1441,7 @@ def relabel_index( Returns ------- Styler + Returns itself for chaining. See Also -------- @@ -1560,6 +1577,140 @@ def alias_(x, value): return self + def format_index_names( + self, + formatter: ExtFormatter | None = None, + axis: Axis = 0, + level: Level | list[Level] | None = None, + na_rep: str | None = None, + precision: int | None = None, + decimal: str = ".", + thousands: str | None = None, + escape: str | None = None, + hyperlinks: str | None = None, + ) -> StylerRenderer: + r""" + Format the text display value of index names or column names. + + .. versionadded:: 3.0 + + Parameters + ---------- + formatter : str, callable, dict or None + Object to define how values are displayed. See notes. + axis : {0, "index", 1, "columns"} + Whether to apply the formatter to the index or column headers. + level : int, str, list + The level(s) over which to apply the generic formatter. + na_rep : str, optional + Representation for missing values. + If ``na_rep`` is None, no special formatting is applied. + precision : int, optional + Floating point precision to use for display purposes, if not determined by + the specified ``formatter``. + decimal : str, default "." + Character used as decimal separator for floats, complex and integers. + thousands : str, optional, default None + Character used as thousands separator for floats, complex and integers. + escape : str, optional + Use 'html' to replace the characters ``&``, ``<``, ``>``, ``'``, and ``"`` + in cell display string with HTML-safe sequences. + Use 'latex' to replace the characters ``&``, ``%``, ``$``, ``#``, ``_``, + ``{``, ``}``, ``~``, ``^``, and ``\`` in the cell display string with + LaTeX-safe sequences. + Escaping is done before ``formatter``. + hyperlinks : {"html", "latex"}, optional + Convert string patterns containing https://, http://, ftp:// or www. to + HTML tags as clickable URL hyperlinks if "html", or LaTeX \href + commands if "latex". + + Returns + ------- + Styler + Returns itself for chaining. + + Raises + ------ + ValueError + If the `formatter` is a string and the dtypes are incompatible. + + See Also + -------- + Styler.format_index: Format the text display value of index labels + or column headers. + + Notes + ----- + This method has a similar signature to :meth:`Styler.format_index`. Since + `names` are generally label based, and often not numeric, the typical features + expected to be more frequently used here are ``escape`` and ``hyperlinks``. + + .. warning:: + `Styler.format_index_names` is ignored when using the output format + `Styler.to_excel`, since Excel and Python have inherrently different + formatting structures. + + Examples + -------- + >>> df = pd.DataFrame( + ... [[1, 2], [3, 4]], + ... index=pd.Index(["a", "b"], name="idx"), + ... ) + >>> df # doctest: +SKIP + 0 1 + idx + a 1 2 + b 3 4 + >>> df.style.format_index_names(lambda x: x.upper(), axis=0) # doctest: +SKIP + 0 1 + IDX + a 1 2 + b 3 4 + """ + axis = self.data._get_axis_number(axis) + if axis == 0: + display_funcs_, obj = self._display_funcs_index_names, self.index + else: + display_funcs_, obj = self._display_funcs_column_names, self.columns + levels_ = refactor_levels(level, obj) + + if all( + ( + formatter is None, + level is None, + precision is None, + decimal == ".", + thousands is None, + na_rep is None, + escape is None, + hyperlinks is None, + ) + ): + display_funcs_.clear() + return self # clear the formatter / revert to default and avoid looping + + if not isinstance(formatter, dict): + formatter = {level: formatter for level in levels_} + else: + formatter = { + obj._get_level_number(level): formatter_ + for level, formatter_ in formatter.items() + } + + for lvl in levels_: + format_func = _maybe_wrap_formatter( + formatter.get(lvl), + na_rep=na_rep, + precision=precision, + decimal=decimal, + thousands=thousands, + escape=escape, + hyperlinks=hyperlinks, + ) + display_funcs_[lvl] = format_func + + return self + def _element( html_element: str, @@ -1571,7 +1722,7 @@ def _element( """ Template to return container with information for a or element. """ - if "display_value" not in kwargs: + if "display_value" not in kwargs or kwargs["display_value"] is None: kwargs["display_value"] = value return { "type": html_element, diff --git a/pandas/io/parsers/base_parser.py b/pandas/io/parsers/base_parser.py index 7b06c6b6b0d39..5a7d117b0543e 100644 --- a/pandas/io/parsers/base_parser.py +++ b/pandas/io/parsers/base_parser.py @@ -174,7 +174,7 @@ def __init__(self, kwds) -> None: and all(map(is_integer, self.index_col)) ): raise ValueError( - "index_col must only contain row numbers " + "index_col must only contain integers of column positions " "when specifying a multi-index header" ) else: @@ -519,7 +519,6 @@ def _convert_to_ndarrays( dct: Mapping, na_values, na_fvalues, - verbose: bool = False, converters=None, dtypes=None, ) -> dict[Any, np.ndarray]: @@ -596,8 +595,6 @@ def _convert_to_ndarrays( cvals = self._cast_types(cvals, cast_type, c) result[c] = cvals - if verbose and na_count: - print(f"Filled {na_count} NA values in column {c!s}") return result @final @@ -1236,7 +1233,6 @@ def converter(*date_cols, col: Hashable): "usecols": None, # 'iterator': False, "chunksize": None, - "verbose": False, "encoding": None, "compression": None, "skip_blank_lines": True, diff --git a/pandas/io/parsers/python_parser.py b/pandas/io/parsers/python_parser.py index dbda47172f6ac..44210b6979827 100644 --- a/pandas/io/parsers/python_parser.py +++ b/pandas/io/parsers/python_parser.py @@ -110,8 +110,6 @@ def __init__(self, f: ReadCsvBuffer[str] | list, **kwds) -> None: if "has_index_names" in kwds: self.has_index_names = kwds["has_index_names"] - self.verbose = kwds["verbose"] - self.thousands = kwds["thousands"] self.decimal = kwds["decimal"] @@ -372,7 +370,6 @@ def _convert_data( data, clean_na_values, clean_na_fvalues, - self.verbose, clean_conv, clean_dtypes, ) diff --git a/pandas/io/parsers/readers.py b/pandas/io/parsers/readers.py index 1ef2e65617c9b..b234a6b78e051 100644 --- a/pandas/io/parsers/readers.py +++ b/pandas/io/parsers/readers.py @@ -116,7 +116,6 @@ class _read_shared(TypedDict, Generic[HashableT], total=False): ) keep_default_na: bool na_filter: bool - verbose: bool | lib.NoDefault skip_blank_lines: bool parse_dates: bool | Sequence[Hashable] | None infer_datetime_format: bool | lib.NoDefault @@ -295,10 +294,6 @@ class _read_shared(TypedDict, Generic[HashableT], total=False): Detect missing value markers (empty strings and the value of ``na_values``). In data without any ``NA`` values, passing ``na_filter=False`` can improve the performance of reading a large file. -verbose : bool, default False - Indicate number of ``NA`` values placed in non-numeric columns. - - .. deprecated:: 2.2.0 skip_blank_lines : bool, default True If ``True``, skip over blank lines rather than interpreting as ``NaN`` values. parse_dates : bool, None, list of Hashable, list of lists or dict of {{Hashable : \ @@ -556,7 +551,6 @@ class _Fwf_Defaults(TypedDict): "converters", "iterator", "dayfirst", - "verbose", "skipinitialspace", "low_memory", } @@ -755,7 +749,6 @@ def read_csv( | None = None, keep_default_na: bool = True, na_filter: bool = True, - verbose: bool | lib.NoDefault = lib.no_default, skip_blank_lines: bool = True, # Datetime Handling parse_dates: bool | Sequence[Hashable] | None = None, @@ -845,17 +838,6 @@ def read_csv( else: delim_whitespace = False - if verbose is not lib.no_default: - # GH#55569 - warnings.warn( - "The 'verbose' keyword in pd.read_csv is deprecated and " - "will be removed in a future version.", - FutureWarning, - stacklevel=find_stack_level(), - ) - else: - verbose = False - # locals() should never be modified kwds = locals().copy() del kwds["filepath_or_buffer"] @@ -958,7 +940,6 @@ def read_table( | None = None, keep_default_na: bool = True, na_filter: bool = True, - verbose: bool | lib.NoDefault = lib.no_default, skip_blank_lines: bool = True, # Datetime Handling parse_dates: bool | Sequence[Hashable] | None = None, @@ -1039,17 +1020,6 @@ def read_table( else: delim_whitespace = False - if verbose is not lib.no_default: - # GH#55569 - warnings.warn( - "The 'verbose' keyword in pd.read_table is deprecated and " - "will be removed in a future version.", - FutureWarning, - stacklevel=find_stack_level(), - ) - else: - verbose = False - # locals() should never be modified kwds = locals().copy() del kwds["filepath_or_buffer"] @@ -1139,7 +1109,7 @@ def read_fwf( ``file://localhost/path/to/table.csv``. colspecs : list of tuple (int, int) or 'infer'. optional A list of tuples giving the extents of the fixed-width - fields of each line as half-open intervals (i.e., [from, to[ ). + fields of each line as half-open intervals (i.e., [from, to] ). String value 'infer' can be used to instruct the parser to try detecting the column specifications from the first 100 rows of the data which are not being skipped via skiprows (default='infer'). diff --git a/pandas/tests/api/test_api.py b/pandas/tests/api/test_api.py index 15b6c9abaea8f..82c5c305b574c 100644 --- a/pandas/tests/api/test_api.py +++ b/pandas/tests/api/test_api.py @@ -401,3 +401,7 @@ def test_pandas_array_alias(): res = pd.arrays.PandasArray assert res is pd.arrays.NumpyExtensionArray + + +def test_set_module(): + assert pd.DataFrame.__module__ == "pandas" diff --git a/pandas/tests/arrays/categorical/test_api.py b/pandas/tests/arrays/categorical/test_api.py index cff8afaa17516..2791fd55f54d7 100644 --- a/pandas/tests/arrays/categorical/test_api.py +++ b/pandas/tests/arrays/categorical/test_api.py @@ -18,13 +18,6 @@ class TestCategoricalAPI: - def test_to_list_deprecated(self): - # GH#51254 - cat1 = Categorical(list("acb"), ordered=False) - msg = "Categorical.to_list is deprecated and will be removed" - with tm.assert_produces_warning(FutureWarning, match=msg): - cat1.to_list() - def test_ordered_api(self): # GH 9347 cat1 = Categorical(list("acb"), ordered=False) diff --git a/pandas/tests/arrays/categorical/test_missing.py b/pandas/tests/arrays/categorical/test_missing.py index 332d31e9e3fc2..9d4b78ce9944e 100644 --- a/pandas/tests/arrays/categorical/test_missing.py +++ b/pandas/tests/arrays/categorical/test_missing.py @@ -62,34 +62,6 @@ def test_set_item_nan(self): exp = Categorical([1, np.nan, 3], categories=[1, 2, 3]) tm.assert_categorical_equal(cat, exp) - @pytest.mark.parametrize( - "fillna_kwargs, msg", - [ - ( - {"value": 1, "method": "ffill"}, - "Cannot specify both 'value' and 'method'.", - ), - ({}, "Must specify a fill 'value' or 'method'."), - ({"method": "bad"}, "Invalid fill method. Expecting .* bad"), - ( - {"value": Series([1, 2, 3, 4, "a"])}, - "Cannot setitem on a Categorical with a new category", - ), - ], - ) - def test_fillna_raises(self, fillna_kwargs, msg): - # https://github.com/pandas-dev/pandas/issues/19682 - # https://github.com/pandas-dev/pandas/issues/13628 - cat = Categorical([1, 2, 3, None, None]) - - if len(fillna_kwargs) == 1 and "value" in fillna_kwargs: - err = TypeError - else: - err = ValueError - - with pytest.raises(err, match=msg): - cat.fillna(**fillna_kwargs) - @pytest.mark.parametrize("named", [True, False]) def test_fillna_iterable_category(self, named): # https://github.com/pandas-dev/pandas/issues/21097 diff --git a/pandas/tests/arrays/datetimes/test_constructors.py b/pandas/tests/arrays/datetimes/test_constructors.py index 3d22427d41985..d7264c002c67f 100644 --- a/pandas/tests/arrays/datetimes/test_constructors.py +++ b/pandas/tests/arrays/datetimes/test_constructors.py @@ -16,34 +16,6 @@ def test_from_sequence_invalid_type(self): with pytest.raises(TypeError, match="Cannot create a DatetimeArray"): DatetimeArray._from_sequence(mi, dtype="M8[ns]") - def test_only_1dim_accepted(self): - arr = np.array([0, 1, 2, 3], dtype="M8[h]").astype("M8[ns]") - - depr_msg = "DatetimeArray.__init__ is deprecated" - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(ValueError, match="Only 1-dimensional"): - # 3-dim, we allow 2D to sneak in for ops purposes GH#29853 - DatetimeArray(arr.reshape(2, 2, 1)) - - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(ValueError, match="Only 1-dimensional"): - # 0-dim - DatetimeArray(arr[[0]].squeeze()) - - def test_freq_validation(self): - # GH#24623 check that invalid instances cannot be created with the - # public constructor - arr = np.arange(5, dtype=np.int64) * 3600 * 10**9 - - msg = ( - "Inferred frequency h from passed values does not " - "conform to passed frequency W-SUN" - ) - depr_msg = "DatetimeArray.__init__ is deprecated" - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(ValueError, match=msg): - DatetimeArray(arr, freq="W") - @pytest.mark.parametrize( "meth", [ @@ -76,42 +48,9 @@ def test_from_pandas_array(self): expected = pd.date_range("1970-01-01", periods=5, freq="h")._data tm.assert_datetime_array_equal(result, expected) - def test_mismatched_timezone_raises(self): - depr_msg = "DatetimeArray.__init__ is deprecated" - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - arr = DatetimeArray( - np.array(["2000-01-01T06:00:00"], dtype="M8[ns]"), - dtype=DatetimeTZDtype(tz="US/Central"), - ) - dtype = DatetimeTZDtype(tz="US/Eastern") - msg = r"dtype=datetime64\[ns.*\] does not match data dtype datetime64\[ns.*\]" - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(TypeError, match=msg): - DatetimeArray(arr, dtype=dtype) - - # also with mismatched tzawareness - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(TypeError, match=msg): - DatetimeArray(arr, dtype=np.dtype("M8[ns]")) - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(TypeError, match=msg): - DatetimeArray(arr.tz_localize(None), dtype=arr.dtype) - - def test_non_array_raises(self): - depr_msg = "DatetimeArray.__init__ is deprecated" - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(ValueError, match="list"): - DatetimeArray([1, 2, 3]) - def test_bool_dtype_raises(self): arr = np.array([1, 2, 3], dtype="bool") - depr_msg = "DatetimeArray.__init__ is deprecated" - msg = "Unexpected value for 'dtype': 'bool'. Must be" - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(ValueError, match=msg): - DatetimeArray(arr) - msg = r"dtype bool cannot be converted to datetime64\[ns\]" with pytest.raises(TypeError, match=msg): DatetimeArray._from_sequence(arr, dtype="M8[ns]") @@ -122,41 +61,6 @@ def test_bool_dtype_raises(self): with pytest.raises(TypeError, match=msg): pd.to_datetime(arr) - def test_incorrect_dtype_raises(self): - depr_msg = "DatetimeArray.__init__ is deprecated" - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(ValueError, match="Unexpected value for 'dtype'."): - DatetimeArray(np.array([1, 2, 3], dtype="i8"), dtype="category") - - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(ValueError, match="Unexpected value for 'dtype'."): - DatetimeArray(np.array([1, 2, 3], dtype="i8"), dtype="m8[s]") - - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(ValueError, match="Unexpected value for 'dtype'."): - DatetimeArray(np.array([1, 2, 3], dtype="i8"), dtype="M8[D]") - - def test_mismatched_values_dtype_units(self): - arr = np.array([1, 2, 3], dtype="M8[s]") - dtype = np.dtype("M8[ns]") - msg = "Values resolution does not match dtype." - depr_msg = "DatetimeArray.__init__ is deprecated" - - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(ValueError, match=msg): - DatetimeArray(arr, dtype=dtype) - - dtype2 = DatetimeTZDtype(tz="UTC", unit="ns") - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(ValueError, match=msg): - DatetimeArray(arr, dtype=dtype2) - - def test_freq_infer_raises(self): - depr_msg = "DatetimeArray.__init__ is deprecated" - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(ValueError, match="Frequency inference"): - DatetimeArray(np.array([1, 2, 3], dtype="i8"), freq="infer") - def test_copy(self): data = np.array([1, 2, 3], dtype="M8[ns]") arr = DatetimeArray._from_sequence(data, copy=False) diff --git a/pandas/tests/arrays/test_datetimelike.py b/pandas/tests/arrays/test_datetimelike.py index b6ae1a9df0e65..971c5bf487104 100644 --- a/pandas/tests/arrays/test_datetimelike.py +++ b/pandas/tests/arrays/test_datetimelike.py @@ -1320,12 +1320,6 @@ def test_from_pandas_array(dtype): cls = {"M8[ns]": DatetimeArray, "m8[ns]": TimedeltaArray}[dtype] - depr_msg = f"{cls.__name__}.__init__ is deprecated" - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - result = cls(arr) - expected = cls(data) - tm.assert_extension_array_equal(result, expected) - result = cls._from_sequence(arr, dtype=dtype) expected = cls._from_sequence(data, dtype=dtype) tm.assert_extension_array_equal(result, expected) diff --git a/pandas/tests/arrays/timedeltas/test_constructors.py b/pandas/tests/arrays/timedeltas/test_constructors.py index 91b6f7fa222f9..ee29f505fd7b1 100644 --- a/pandas/tests/arrays/timedeltas/test_constructors.py +++ b/pandas/tests/arrays/timedeltas/test_constructors.py @@ -1,45 +1,10 @@ import numpy as np import pytest -import pandas._testing as tm from pandas.core.arrays import TimedeltaArray class TestTimedeltaArrayConstructor: - def test_only_1dim_accepted(self): - # GH#25282 - arr = np.array([0, 1, 2, 3], dtype="m8[h]").astype("m8[ns]") - - depr_msg = "TimedeltaArray.__init__ is deprecated" - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(ValueError, match="Only 1-dimensional"): - # 3-dim, we allow 2D to sneak in for ops purposes GH#29853 - TimedeltaArray(arr.reshape(2, 2, 1)) - - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(ValueError, match="Only 1-dimensional"): - # 0-dim - TimedeltaArray(arr[[0]].squeeze()) - - def test_freq_validation(self): - # ensure that the public constructor cannot create an invalid instance - arr = np.array([0, 0, 1], dtype=np.int64) * 3600 * 10**9 - - msg = ( - "Inferred frequency None from passed values does not " - "conform to passed frequency D" - ) - depr_msg = "TimedeltaArray.__init__ is deprecated" - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(ValueError, match=msg): - TimedeltaArray(arr.view("timedelta64[ns]"), freq="D") - - def test_non_array_raises(self): - depr_msg = "TimedeltaArray.__init__ is deprecated" - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(ValueError, match="list"): - TimedeltaArray([1, 2, 3]) - def test_other_type_raises(self): msg = r"dtype bool cannot be converted to timedelta64\[ns\]" with pytest.raises(TypeError, match=msg): @@ -78,16 +43,6 @@ def test_incorrect_dtype_raises(self): np.array([1, 2, 3], dtype="i8"), dtype=np.dtype("m8[Y]") ) - def test_mismatched_values_dtype_units(self): - arr = np.array([1, 2, 3], dtype="m8[s]") - dtype = np.dtype("m8[ns]") - msg = r"Values resolution does not match dtype" - depr_msg = "TimedeltaArray.__init__ is deprecated" - - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - with pytest.raises(ValueError, match=msg): - TimedeltaArray(arr, dtype=dtype) - def test_copy(self): data = np.array([1, 2, 3], dtype="m8[ns]") arr = TimedeltaArray._from_sequence(data, copy=False) diff --git a/pandas/tests/base/test_value_counts.py b/pandas/tests/base/test_value_counts.py index a0b0bdfdb46d8..ac40e48f3d523 100644 --- a/pandas/tests/base/test_value_counts.py +++ b/pandas/tests/base/test_value_counts.py @@ -347,9 +347,8 @@ def test_value_counts_object_inference_deprecated(): dti = pd.date_range("2016-01-01", periods=3, tz="UTC") idx = dti.astype(object) - msg = "The behavior of value_counts with object-dtype is deprecated" - with tm.assert_produces_warning(FutureWarning, match=msg): - res = idx.value_counts() + res = idx.value_counts() exp = dti.value_counts() + exp.index = exp.index.astype(object) tm.assert_series_equal(res, exp) diff --git a/pandas/tests/copy_view/test_copy_deprecation.py b/pandas/tests/copy_view/test_copy_deprecation.py index ca57c02112131..8ee37213b92ab 100644 --- a/pandas/tests/copy_view/test_copy_deprecation.py +++ b/pandas/tests/copy_view/test_copy_deprecation.py @@ -1,6 +1,10 @@ import pytest import pandas as pd +from pandas import ( + concat, + merge, +) import pandas._testing as tm @@ -13,20 +17,33 @@ ("infer_objects", {}), ("astype", {"dtype": "float64"}), ("reindex", {"index": [2, 0, 1]}), + ("transpose", {}), + ("set_axis", {"labels": [1, 2, 3]}), + ("rename", {"index": {1: 2}}), + ("set_flags", {}), + ("to_period", {}), + ("to_timestamp", {}), + ("swaplevel", {"i": 0, "j": 1}), ], ) def test_copy_deprecation(meth, kwargs): - df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) + df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": 1}) - if meth in ("tz_convert", "tz_localize"): - tz = None if meth == "tz_localize" else "US/Eastern" + if meth in ("tz_convert", "tz_localize", "to_period"): + tz = None if meth in ("tz_localize", "to_period") else "US/Eastern" df.index = pd.date_range("2020-01-01", freq="D", periods=len(df), tz=tz) + elif meth == "to_timestamp": + df.index = pd.period_range("2020-01-01", freq="D", periods=len(df)) + elif meth == "swaplevel": + df = df.set_index(["b", "c"]) - with tm.assert_produces_warning(DeprecationWarning, match="copy"): - getattr(df, meth)(copy=False, **kwargs) + if meth != "swaplevel": + with tm.assert_produces_warning(DeprecationWarning, match="copy"): + getattr(df, meth)(copy=False, **kwargs) - with tm.assert_produces_warning(DeprecationWarning, match="copy"): - getattr(df.a, meth)(copy=False, **kwargs) + if meth != "transpose": + with tm.assert_produces_warning(DeprecationWarning, match="copy"): + getattr(df.a, meth)(copy=False, **kwargs) def test_copy_deprecation_reindex_like_align(): @@ -51,3 +68,22 @@ def test_copy_deprecation_reindex_like_align(): DeprecationWarning, match="copy", check_stacklevel=False ): df.a.align(df.a, copy=False) + + +def test_copy_deprecation_merge_concat(): + df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) + + with tm.assert_produces_warning( + DeprecationWarning, match="copy", check_stacklevel=False + ): + df.merge(df, copy=False) + + with tm.assert_produces_warning( + DeprecationWarning, match="copy", check_stacklevel=False + ): + merge(df, df, copy=False) + + with tm.assert_produces_warning( + DeprecationWarning, match="copy", check_stacklevel=False + ): + concat([df, df], copy=False) diff --git a/pandas/tests/copy_view/test_functions.py b/pandas/tests/copy_view/test_functions.py index eeb19103f7bd5..196d908a44a46 100644 --- a/pandas/tests/copy_view/test_functions.py +++ b/pandas/tests/copy_view/test_functions.py @@ -139,12 +139,11 @@ def test_concat_mixed_series_frame(): tm.assert_frame_equal(result, expected) -@pytest.mark.parametrize("copy", [True, None, False]) -def test_concat_copy_keyword(copy): +def test_concat_copy_keyword(): df = DataFrame({"a": [1, 2]}) df2 = DataFrame({"b": [1.5, 2.5]}) - result = concat([df, df2], axis=1, copy=copy) + result = concat([df, df2], axis=1) assert np.shares_memory(get_array(df, "a"), get_array(result, "a")) assert np.shares_memory(get_array(df2, "b"), get_array(result, "b")) @@ -234,12 +233,11 @@ def test_merge_on_key_enlarging_one(func, how): tm.assert_frame_equal(df2, df2_orig) -@pytest.mark.parametrize("copy", [True, None, False]) -def test_merge_copy_keyword(copy): +def test_merge_copy_keyword(): df = DataFrame({"a": [1, 2]}) df2 = DataFrame({"b": [3, 4.5]}) - result = df.merge(df2, copy=copy, left_index=True, right_index=True) + result = df.merge(df2, left_index=True, right_index=True) assert np.shares_memory(get_array(df, "a"), get_array(result, "a")) assert np.shares_memory(get_array(df2, "b"), get_array(result, "b")) diff --git a/pandas/tests/copy_view/test_interp_fillna.py b/pandas/tests/copy_view/test_interp_fillna.py index 8fe58e59b9cfd..abd87162ec32e 100644 --- a/pandas/tests/copy_view/test_interp_fillna.py +++ b/pandas/tests/copy_view/test_interp_fillna.py @@ -19,19 +19,18 @@ def test_interpolate_no_op(method): df = DataFrame({"a": [1, 2]}) df_orig = df.copy() - warn = None if method == "pad": - warn = FutureWarning - msg = "DataFrame.interpolate with method=pad is deprecated" - with tm.assert_produces_warning(warn, match=msg): + msg = f"Can not interpolate with method={method}" + with pytest.raises(ValueError, match=msg): + df.interpolate(method=method) + else: result = df.interpolate(method=method) + assert np.shares_memory(get_array(result, "a"), get_array(df, "a")) - assert np.shares_memory(get_array(result, "a"), get_array(df, "a")) + result.iloc[0, 0] = 100 - result.iloc[0, 0] = 100 - - assert not np.shares_memory(get_array(result, "a"), get_array(df, "a")) - tm.assert_frame_equal(df, df_orig) + assert not np.shares_memory(get_array(result, "a"), get_array(df, "a")) + tm.assert_frame_equal(df, df_orig) @pytest.mark.parametrize("func", ["ffill", "bfill"]) @@ -122,9 +121,6 @@ def test_interpolate_cannot_with_object_dtype(): def test_interpolate_object_convert_no_op(): df = DataFrame({"a": ["a", "b", "c"], "b": 1}) arr_a = get_array(df, "a") - msg = "DataFrame.interpolate with method=pad is deprecated" - with tm.assert_produces_warning(FutureWarning, match=msg): - df.interpolate(method="pad", inplace=True) # Now CoW makes a copy, it should not! assert df._mgr._has_no_reference(0) @@ -134,8 +130,8 @@ def test_interpolate_object_convert_no_op(): def test_interpolate_object_convert_copies(): df = DataFrame({"a": [1, np.nan, 2.5], "b": 1}) arr_a = get_array(df, "a") - msg = "DataFrame.interpolate with method=pad is deprecated" - with tm.assert_produces_warning(FutureWarning, match=msg): + msg = "Can not interpolate with method=pad" + with pytest.raises(ValueError, match=msg): df.interpolate(method="pad", inplace=True, downcast="infer") assert df._mgr._has_no_reference(0) @@ -147,12 +143,13 @@ def test_interpolate_downcast_reference_triggers_copy(): df_orig = df.copy() arr_a = get_array(df, "a") view = df[:] - msg = "DataFrame.interpolate with method=pad is deprecated" - with tm.assert_produces_warning(FutureWarning, match=msg): + + msg = "Can not interpolate with method=pad" + with pytest.raises(ValueError, match=msg): df.interpolate(method="pad", inplace=True, downcast="infer") + assert df._mgr._has_no_reference(0) + assert not np.shares_memory(arr_a, get_array(df, "a")) - assert df._mgr._has_no_reference(0) - assert not np.shares_memory(arr_a, get_array(df, "a")) tm.assert_frame_equal(df_orig, view) diff --git a/pandas/tests/copy_view/test_methods.py b/pandas/tests/copy_view/test_methods.py index 8bf0e81e74e25..3712a74fe54ed 100644 --- a/pandas/tests/copy_view/test_methods.py +++ b/pandas/tests/copy_view/test_methods.py @@ -176,13 +176,6 @@ def test_methods_series_copy_keyword(request, method, copy): assert np.shares_memory(get_array(ser2), get_array(ser)) -@pytest.mark.parametrize("copy", [True, None, False]) -def test_transpose_copy_keyword(copy): - df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) - result = df.transpose(copy=copy) - assert np.shares_memory(get_array(df, "a"), get_array(result, 0)) - - # ----------------------------------------------------------------------------- # DataFrame methods returning new DataFrame using shallow copy @@ -1415,11 +1408,10 @@ def test_inplace_arithmetic_series_with_reference(): tm.assert_series_equal(ser_orig, view) -@pytest.mark.parametrize("copy", [True, False]) -def test_transpose(copy): +def test_transpose(): df = DataFrame({"a": [1, 2, 3], "b": 1}) df_orig = df.copy() - result = df.transpose(copy=copy) + result = df.transpose() assert np.shares_memory(get_array(df, "a"), get_array(result, 0)) result.iloc[0, 0] = 100 diff --git a/pandas/tests/dtypes/test_concat.py b/pandas/tests/dtypes/test_concat.py index 4f7ae6fa2a0a0..1652c9254061b 100644 --- a/pandas/tests/dtypes/test_concat.py +++ b/pandas/tests/dtypes/test_concat.py @@ -20,14 +20,13 @@ def test_concat_mismatched_categoricals_with_empty(): tm.assert_categorical_equal(result, expected) -@pytest.mark.parametrize("copy", [True, False]) -def test_concat_single_dataframe_tz_aware(copy): +def test_concat_single_dataframe_tz_aware(): # https://github.com/pandas-dev/pandas/issues/25257 df = pd.DataFrame( {"timestamp": [pd.Timestamp("2020-04-08 09:00:00.709949+0000", tz="UTC")]} ) expected = df.copy() - result = pd.concat([df], copy=copy) + result = pd.concat([df]) tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/extension/base/methods.py b/pandas/tests/extension/base/methods.py index c803a8113b4a4..26638c6160b7b 100644 --- a/pandas/tests/extension/base/methods.py +++ b/pandas/tests/extension/base/methods.py @@ -169,8 +169,8 @@ def test_argmin_argmax_all_na(self, method, data, na_value): ("idxmin", True, 2), ("argmax", True, 0), ("argmin", True, 2), - ("idxmax", False, np.nan), - ("idxmin", False, np.nan), + ("idxmax", False, -1), + ("idxmin", False, -1), ("argmax", False, -1), ("argmin", False, -1), ], @@ -179,17 +179,13 @@ def test_argreduce_series( self, data_missing_for_sorting, op_name, skipna, expected ): # data_missing_for_sorting -> [B, NA, A] with A < B and NA missing. - warn = None - msg = "The behavior of Series.argmax/argmin" - if op_name.startswith("arg") and expected == -1: - warn = FutureWarning - if op_name.startswith("idx") and np.isnan(expected): - warn = FutureWarning - msg = f"The behavior of Series.{op_name}" ser = pd.Series(data_missing_for_sorting) - with tm.assert_produces_warning(warn, match=msg): + if expected == -1: + with pytest.raises(ValueError, match="Encountered an NA value"): + getattr(ser, op_name)(skipna=skipna) + else: result = getattr(ser, op_name)(skipna=skipna) - tm.assert_almost_equal(result, expected) + tm.assert_almost_equal(result, expected) def test_argmax_argmin_no_skipna_notimplemented(self, data_missing_for_sorting): # GH#38733 diff --git a/pandas/tests/extension/base/reshaping.py b/pandas/tests/extension/base/reshaping.py index 4550e3b055cfe..489cd15644d04 100644 --- a/pandas/tests/extension/base/reshaping.py +++ b/pandas/tests/extension/base/reshaping.py @@ -106,7 +106,7 @@ def test_concat_extension_arrays_copy_false(self, data, na_value): "B": data[3:7], } ) - result = pd.concat([df1, df2], axis=1, copy=False) + result = pd.concat([df1, df2], axis=1) tm.assert_frame_equal(result, expected) def test_concat_with_reindex(self, data): diff --git a/pandas/tests/extension/conftest.py b/pandas/tests/extension/conftest.py index 5ae0864190f10..97fb5a0bc5066 100644 --- a/pandas/tests/extension/conftest.py +++ b/pandas/tests/extension/conftest.py @@ -189,7 +189,7 @@ def use_numpy(request): def fillna_method(request): """ Parametrized fixture giving method parameters 'ffill' and 'bfill' for - Series.fillna(method=) testing. + Series. testing. """ return request.param diff --git a/pandas/tests/extension/decimal/array.py b/pandas/tests/extension/decimal/array.py index 709cff59cd824..59f313b4c9edb 100644 --- a/pandas/tests/extension/decimal/array.py +++ b/pandas/tests/extension/decimal/array.py @@ -287,17 +287,10 @@ def value_counts(self, dropna: bool = True): return value_counts(self.to_numpy(), dropna=dropna) # We override fillna here to simulate a 3rd party EA that has done so. This - # lets us test the deprecation telling authors to implement _pad_or_backfill - # Simulate a 3rd-party EA that has not yet updated to include a "copy" + # lets us test a 3rd-party EA that has not yet updated to include a "copy" # keyword in its fillna method. - # error: Signature of "fillna" incompatible with supertype "ExtensionArray" - def fillna( # type: ignore[override] - self, - value=None, - method=None, - limit: int | None = None, - ): - return super().fillna(value=value, method=method, limit=limit, copy=True) + def fillna(self, value=None, limit=None): + return super().fillna(value=value, limit=limit, copy=True) def to_decimal(values, context=None): diff --git a/pandas/tests/extension/decimal/test_decimal.py b/pandas/tests/extension/decimal/test_decimal.py index bed3ec62f43da..a2721908e858f 100644 --- a/pandas/tests/extension/decimal/test_decimal.py +++ b/pandas/tests/extension/decimal/test_decimal.py @@ -137,86 +137,6 @@ def test_fillna_frame(self, data_missing): ): super().test_fillna_frame(data_missing) - def test_fillna_limit_pad(self, data_missing): - msg = "ExtensionArray.fillna 'method' keyword is deprecated" - with tm.assert_produces_warning( - DeprecationWarning, - match=msg, - check_stacklevel=False, - raise_on_extra_warnings=False, - ): - super().test_fillna_limit_pad(data_missing) - - msg = "The 'method' keyword in DecimalArray.fillna is deprecated" - with tm.assert_produces_warning( - FutureWarning, - match=msg, - check_stacklevel=False, - raise_on_extra_warnings=False, - ): - super().test_fillna_limit_pad(data_missing) - - @pytest.mark.parametrize( - "limit_area, input_ilocs, expected_ilocs", - [ - ("outside", [1, 0, 0, 0, 1], [1, 0, 0, 0, 1]), - ("outside", [1, 0, 1, 0, 1], [1, 0, 1, 0, 1]), - ("outside", [0, 1, 1, 1, 0], [0, 1, 1, 1, 1]), - ("outside", [0, 1, 0, 1, 0], [0, 1, 0, 1, 1]), - ("inside", [1, 0, 0, 0, 1], [1, 1, 1, 1, 1]), - ("inside", [1, 0, 1, 0, 1], [1, 1, 1, 1, 1]), - ("inside", [0, 1, 1, 1, 0], [0, 1, 1, 1, 0]), - ("inside", [0, 1, 0, 1, 0], [0, 1, 1, 1, 0]), - ], - ) - def test_ffill_limit_area( - self, data_missing, limit_area, input_ilocs, expected_ilocs - ): - # GH#56616 - msg = "ExtensionArray.fillna 'method' keyword is deprecated" - with tm.assert_produces_warning( - DeprecationWarning, - match=msg, - check_stacklevel=False, - raise_on_extra_warnings=False, - ): - msg = "DecimalArray does not implement limit_area" - with pytest.raises(NotImplementedError, match=msg): - super().test_ffill_limit_area( - data_missing, limit_area, input_ilocs, expected_ilocs - ) - - def test_fillna_limit_backfill(self, data_missing): - msg = "ExtensionArray.fillna 'method' keyword is deprecated" - with tm.assert_produces_warning( - DeprecationWarning, - match=msg, - check_stacklevel=False, - raise_on_extra_warnings=False, - ): - super().test_fillna_limit_backfill(data_missing) - - msg = "The 'method' keyword in DecimalArray.fillna is deprecated" - with tm.assert_produces_warning( - FutureWarning, - match=msg, - check_stacklevel=False, - raise_on_extra_warnings=False, - ): - super().test_fillna_limit_backfill(data_missing) - - def test_fillna_no_op_returns_copy(self, data): - msg = "|".join( - [ - "ExtensionArray.fillna 'method' keyword is deprecated", - "The 'method' keyword in DecimalArray.fillna is deprecated", - ] - ) - with tm.assert_produces_warning( - (FutureWarning, DeprecationWarning), match=msg, check_stacklevel=False - ): - super().test_fillna_no_op_returns_copy(data) - def test_fillna_series(self, data_missing): msg = "ExtensionArray.fillna added a 'copy' keyword" with tm.assert_produces_warning( @@ -224,18 +144,6 @@ def test_fillna_series(self, data_missing): ): super().test_fillna_series(data_missing) - def test_fillna_series_method(self, data_missing, fillna_method): - msg = "|".join( - [ - "ExtensionArray.fillna 'method' keyword is deprecated", - "The 'method' keyword in DecimalArray.fillna is deprecated", - ] - ) - with tm.assert_produces_warning( - (FutureWarning, DeprecationWarning), match=msg, check_stacklevel=False - ): - super().test_fillna_series_method(data_missing, fillna_method) - @pytest.mark.parametrize("dropna", [True, False]) def test_value_counts(self, all_data, dropna): all_data = all_data[:10] diff --git a/pandas/tests/extension/test_arrow.py b/pandas/tests/extension/test_arrow.py index 11a9f4f22167f..9b2251d0b7d4a 100644 --- a/pandas/tests/extension/test_arrow.py +++ b/pandas/tests/extension/test_arrow.py @@ -706,10 +706,6 @@ def test_fillna_no_op_returns_copy(self, data): assert result is not data tm.assert_extension_array_equal(result, data) - result = data.fillna(method="backfill") - assert result is not data - tm.assert_extension_array_equal(result, data) - @pytest.mark.xfail( reason="GH 45419: pyarrow.ChunkedArray does not support views", run=False ) diff --git a/pandas/tests/extension/test_string.py b/pandas/tests/extension/test_string.py index c09d4d315451f..49ad3fce92a5c 100644 --- a/pandas/tests/extension/test_string.py +++ b/pandas/tests/extension/test_string.py @@ -136,10 +136,6 @@ def test_fillna_no_op_returns_copy(self, data): assert result is not data tm.assert_extension_array_equal(result, data) - result = data.fillna(method="backfill") - assert result is not data - tm.assert_extension_array_equal(result, data) - def _get_expected_exception( self, op_name: str, obj, other ) -> type[Exception] | None: diff --git a/pandas/tests/frame/methods/test_info.py b/pandas/tests/frame/methods/test_info.py index fcb7677f03f27..4e3726f4dc51d 100644 --- a/pandas/tests/frame/methods/test_info.py +++ b/pandas/tests/frame/methods/test_info.py @@ -40,7 +40,7 @@ def test_info_empty(): result = buf.getvalue() expected = textwrap.dedent( """\ - + RangeIndex: 0 entries Empty DataFrame\n""" ) @@ -208,7 +208,7 @@ def test_info_memory(): bytes = float(df.memory_usage().sum()) expected = textwrap.dedent( f"""\ - + RangeIndex: 2 entries, 0 to 1 Data columns (total 1 columns): # Column Non-Null Count Dtype @@ -501,7 +501,7 @@ def test_info_int_columns(): result = buf.getvalue() expected = textwrap.dedent( """\ - + Index: 2 entries, A to B Data columns (total 2 columns): # Column Non-Null Count Dtype diff --git a/pandas/tests/frame/methods/test_interpolate.py b/pandas/tests/frame/methods/test_interpolate.py index 2ba3bbd3109a2..0a9d059736e6f 100644 --- a/pandas/tests/frame/methods/test_interpolate.py +++ b/pandas/tests/frame/methods/test_interpolate.py @@ -129,13 +129,7 @@ def test_interp_bad_method(self): "C": [1, 2, 3, 5], } ) - msg = ( - r"method must be one of \['linear', 'time', 'index', 'values', " - r"'nearest', 'zero', 'slinear', 'quadratic', 'cubic', " - r"'barycentric', 'krogh', 'spline', 'polynomial', " - r"'from_derivatives', 'piecewise_polynomial', 'pchip', 'akima', " - r"'cubicspline'\]. Got 'not_a_method' instead." - ) + msg = "Can not interpolate with method=not_a_method" with pytest.raises(ValueError, match=msg): df.interpolate(method="not_a_method") @@ -398,12 +392,9 @@ def test_interp_fillna_methods(self, axis, multiblock, method): df["D"] = np.nan df["E"] = 1.0 - method2 = method if method != "pad" else "ffill" - expected = getattr(df, method2)(axis=axis) - msg = f"DataFrame.interpolate with method={method} is deprecated" - with tm.assert_produces_warning(FutureWarning, match=msg): - result = df.interpolate(method=method, axis=axis) - tm.assert_frame_equal(result, expected) + msg = f"Can not interpolate with method={method}" + with pytest.raises(ValueError, match=msg): + df.interpolate(method=method, axis=axis) def test_interpolate_empty_df(self): # GH#53199 diff --git a/pandas/tests/frame/methods/test_rename.py b/pandas/tests/frame/methods/test_rename.py index 996fc30552bc4..6153a168476d4 100644 --- a/pandas/tests/frame/methods/test_rename.py +++ b/pandas/tests/frame/methods/test_rename.py @@ -165,7 +165,7 @@ def test_rename_multiindex(self): tm.assert_index_equal(renamed.index, new_index) def test_rename_nocopy(self, float_frame): - renamed = float_frame.rename(columns={"C": "foo"}, copy=False) + renamed = float_frame.rename(columns={"C": "foo"}) assert np.shares_memory(renamed["foo"]._values, float_frame["C"]._values) diff --git a/pandas/tests/frame/methods/test_set_axis.py b/pandas/tests/frame/methods/test_set_axis.py index 8c42498b45621..1967941bca9f0 100644 --- a/pandas/tests/frame/methods/test_set_axis.py +++ b/pandas/tests/frame/methods/test_set_axis.py @@ -29,10 +29,7 @@ def test_set_axis_copy(self, obj): expected = obj.copy() expected.index = new_index - result = obj.set_axis(new_index, axis=0, copy=True) - tm.assert_equal(expected, result) - assert result is not obj - result = obj.set_axis(new_index, axis=0, copy=False) + result = obj.set_axis(new_index, axis=0) tm.assert_equal(expected, result) assert result is not obj # check we did NOT make a copy @@ -44,7 +41,6 @@ def test_set_axis_copy(self, obj): for i in range(obj.shape[1]) ) - # copy defaults to True result = obj.set_axis(new_index, axis=0) tm.assert_equal(expected, result) assert result is not obj @@ -57,7 +53,7 @@ def test_set_axis_copy(self, obj): for i in range(obj.shape[1]) ) - res = obj.set_axis(new_index, copy=False) + res = obj.set_axis(new_index) tm.assert_equal(expected, res) # check we did NOT make a copy if res.ndim == 1: diff --git a/pandas/tests/frame/test_api.py b/pandas/tests/frame/test_api.py index 680800d7f5e4c..48f51dfa981ca 100644 --- a/pandas/tests/frame/test_api.py +++ b/pandas/tests/frame/test_api.py @@ -356,9 +356,7 @@ def test_set_flags( assert obj.iloc[key] == 1 # Now we do copy. - result = obj.set_flags( - copy=True, allows_duplicate_labels=allows_duplicate_labels - ) + result = obj.set_flags(allows_duplicate_labels=allows_duplicate_labels) result.iloc[key] = 10 assert obj.iloc[key] == 1 diff --git a/pandas/tests/frame/test_constructors.py b/pandas/tests/frame/test_constructors.py index 7d1a5b4492740..12d8269b640fc 100644 --- a/pandas/tests/frame/test_constructors.py +++ b/pandas/tests/frame/test_constructors.py @@ -2709,6 +2709,11 @@ def test_inference_on_pandas_objects(self): result = DataFrame({"a": ser}) assert result.dtypes.iloc[0] == np.object_ + def test_dict_keys_returns_rangeindex(self): + result = DataFrame({0: [1], 1: [2]}).columns + expected = RangeIndex(2) + tm.assert_index_equal(result, expected, exact=True) + class TestDataFrameConstructorIndexInference: def test_frame_from_dict_of_series_overlapping_monthly_period_indexes(self): diff --git a/pandas/tests/frame/test_reductions.py b/pandas/tests/frame/test_reductions.py index 63c15fab76562..408cb0ab6fc5c 100644 --- a/pandas/tests/frame/test_reductions.py +++ b/pandas/tests/frame/test_reductions.py @@ -1065,18 +1065,20 @@ def test_idxmin(self, float_frame, int_frame, skipna, axis): frame.iloc[5:10] = np.nan frame.iloc[15:20, -2:] = np.nan for df in [frame, int_frame]: - warn = None - if skipna is False or axis == 1: - warn = None if df is int_frame else FutureWarning - msg = "The behavior of DataFrame.idxmin with all-NA values" - with tm.assert_produces_warning(warn, match=msg): + if (not skipna or axis == 1) and df is not int_frame: + if axis == 1: + msg = "Encountered all NA values" + else: + msg = "Encountered an NA value" + with pytest.raises(ValueError, match=msg): + df.idxmin(axis=axis, skipna=skipna) + with pytest.raises(ValueError, match=msg): + df.idxmin(axis=axis, skipna=skipna) + else: result = df.idxmin(axis=axis, skipna=skipna) - - msg2 = "The behavior of Series.idxmin" - with tm.assert_produces_warning(warn, match=msg2): expected = df.apply(Series.idxmin, axis=axis, skipna=skipna) - expected = expected.astype(df.index.dtype) - tm.assert_series_equal(result, expected) + expected = expected.astype(df.index.dtype) + tm.assert_series_equal(result, expected) @pytest.mark.parametrize("axis", [0, 1]) @pytest.mark.filterwarnings(r"ignore:PeriodDtype\[B\] is deprecated:FutureWarning") @@ -1113,16 +1115,17 @@ def test_idxmax(self, float_frame, int_frame, skipna, axis): frame.iloc[5:10] = np.nan frame.iloc[15:20, -2:] = np.nan for df in [frame, int_frame]: - warn = None - if skipna is False or axis == 1: - warn = None if df is int_frame else FutureWarning - msg = "The behavior of DataFrame.idxmax with all-NA values" - with tm.assert_produces_warning(warn, match=msg): - result = df.idxmax(axis=axis, skipna=skipna) + if (skipna is False or axis == 1) and df is frame: + if axis == 1: + msg = "Encountered all NA values" + else: + msg = "Encountered an NA value" + with pytest.raises(ValueError, match=msg): + df.idxmax(axis=axis, skipna=skipna) + return - msg2 = "The behavior of Series.idxmax" - with tm.assert_produces_warning(warn, match=msg2): - expected = df.apply(Series.idxmax, axis=axis, skipna=skipna) + result = df.idxmax(axis=axis, skipna=skipna) + expected = df.apply(Series.idxmax, axis=axis, skipna=skipna) expected = expected.astype(df.index.dtype) tm.assert_series_equal(result, expected) @@ -2118,15 +2121,16 @@ def test_numeric_ea_axis_1(method, skipna, min_count, any_numeric_ea_dtype): if method in ("prod", "product", "sum"): kwargs["min_count"] = min_count - warn = None - msg = None if not skipna and method in ("idxmax", "idxmin"): - warn = FutureWarning + # GH#57745 - EAs use groupby for axis=1 which still needs a proper deprecation. msg = f"The behavior of DataFrame.{method} with all-NA values" - with tm.assert_produces_warning(warn, match=msg): - result = getattr(df, method)(axis=1, **kwargs) - with tm.assert_produces_warning(warn, match=msg): - expected = getattr(expected_df, method)(axis=1, **kwargs) + with tm.assert_produces_warning(FutureWarning, match=msg): + getattr(df, method)(axis=1, **kwargs) + with pytest.raises(ValueError, match="Encountered an NA value"): + getattr(expected_df, method)(axis=1, **kwargs) + return + result = getattr(df, method)(axis=1, **kwargs) + expected = getattr(expected_df, method)(axis=1, **kwargs) if method not in ("idxmax", "idxmin"): expected = expected.astype(expected_dtype) tm.assert_series_equal(result, expected) diff --git a/pandas/tests/frame/test_subclass.py b/pandas/tests/frame/test_subclass.py index 316216e69a587..355953eac9d51 100644 --- a/pandas/tests/frame/test_subclass.py +++ b/pandas/tests/frame/test_subclass.py @@ -16,6 +16,17 @@ class TestDataFrameSubclassing: + def test_no_warning_on_mgr(self): + # GH#57032 + df = tm.SubclassedDataFrame( + {"X": [1, 2, 3], "Y": [1, 2, 3]}, index=["a", "b", "c"] + ) + with tm.assert_produces_warning(None): + # df.isna() goes through _constructor_from_mgr, which we want to + # *not* pass a Manager do __init__ + df.isna() + df["X"].isna() + def test_frame_subclassing_and_slicing(self): # Subclass frame and ensure it returns the right class on slicing it # In reference to PR 9632 diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index 00e781e6a7f07..7ec1598abf403 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -145,6 +145,15 @@ def test_len_nan_group(): assert len(df.groupby(["a", "b"])) == 0 +def test_groupby_timedelta_median(): + # issue 57926 + expected = Series(data=Timedelta("1d"), index=["foo"]) + df = DataFrame({"label": ["foo", "foo"], "timedelta": [pd.NaT, Timedelta("1d")]}) + gb = df.groupby("label")["timedelta"] + actual = gb.median() + tm.assert_series_equal(actual, expected, check_names=False) + + @pytest.mark.parametrize("keys", [["a"], ["a", "b"]]) def test_len_categorical(dropna, observed, keys): # GH#57595 diff --git a/pandas/tests/groupby/test_grouping.py b/pandas/tests/groupby/test_grouping.py index 699fffe5d0488..9ce7a0818ac02 100644 --- a/pandas/tests/groupby/test_grouping.py +++ b/pandas/tests/groupby/test_grouping.py @@ -509,7 +509,7 @@ def test_groupby_with_datetime_key(self): assert len(gb.groups.keys()) == 4 def test_grouping_error_on_multidim_input(self, df): - msg = "Grouper for '' not 1-dimensional" + msg = "Grouper for '' not 1-dimensional" with pytest.raises(ValueError, match=msg): Grouping(df.index, df[["A", "A"]]) diff --git a/pandas/tests/indexes/datetimes/test_constructors.py b/pandas/tests/indexes/datetimes/test_constructors.py index 48bbfc1a9f646..4be45e834ce31 100644 --- a/pandas/tests/indexes/datetimes/test_constructors.py +++ b/pandas/tests/indexes/datetimes/test_constructors.py @@ -35,18 +35,6 @@ class TestDatetimeIndex: - def test_closed_deprecated(self): - # GH#52628 - msg = "The 'closed' keyword" - with tm.assert_produces_warning(FutureWarning, match=msg): - DatetimeIndex([], closed=True) - - def test_normalize_deprecated(self): - # GH#52628 - msg = "The 'normalize' keyword" - with tm.assert_produces_warning(FutureWarning, match=msg): - DatetimeIndex([], normalize=True) - def test_from_dt64_unsupported_unit(self): # GH#49292 val = np.datetime64(1, "D") diff --git a/pandas/tests/indexes/datetimes/test_date_range.py b/pandas/tests/indexes/datetimes/test_date_range.py index ddbeecf150a5e..43fcfd1e59670 100644 --- a/pandas/tests/indexes/datetimes/test_date_range.py +++ b/pandas/tests/indexes/datetimes/test_date_range.py @@ -772,30 +772,11 @@ def test_freq_dateoffset_with_relateivedelta_nanos(self): ) tm.assert_index_equal(result, expected) - @pytest.mark.parametrize( - "freq,freq_depr", - [ - ("h", "H"), - ("2min", "2T"), - ("1s", "1S"), - ("2ms", "2L"), - ("1us", "1U"), - ("2ns", "2N"), - ], - ) - def test_frequencies_H_T_S_L_U_N_deprecated(self, freq, freq_depr): - # GH#52536 - freq_msg = re.split("[0-9]*", freq, maxsplit=1)[1] - freq_depr_msg = re.split("[0-9]*", freq_depr, maxsplit=1)[1] - msg = ( - f"'{freq_depr_msg}' is deprecated and will be removed in a future version, " - ) - f"please use '{freq_msg}' instead" - - expected = date_range("1/1/2000", periods=2, freq=freq) - with tm.assert_produces_warning(FutureWarning, match=msg): - result = date_range("1/1/2000", periods=2, freq=freq_depr) - tm.assert_index_equal(result, expected) + @pytest.mark.parametrize("freq", ["2T", "2L", "1l", "1U", "2N", "2n"]) + def test_frequency_H_T_S_L_U_N_raises(self, freq): + msg = f"Invalid frequency: {freq}" + with pytest.raises(ValueError, match=msg): + date_range("1/1/2000", periods=2, freq=freq) @pytest.mark.parametrize( "freq,freq_depr", diff --git a/pandas/tests/indexes/period/test_constructors.py b/pandas/tests/indexes/period/test_constructors.py index 519c09015427e..ec2216c102c3f 100644 --- a/pandas/tests/indexes/period/test_constructors.py +++ b/pandas/tests/indexes/period/test_constructors.py @@ -60,6 +60,16 @@ def test_period_index_from_datetime_index_invalid_freq(self, freq): with pytest.raises(ValueError, match=msg): rng.to_period() + @pytest.mark.parametrize("freq_depr", ["2T", "1l", "2U", "n"]) + def test_period_index_T_L_U_N_raises(self, freq_depr): + # GH#9586 + msg = f"Invalid frequency: {freq_depr}" + + with pytest.raises(ValueError, match=msg): + period_range("2020-01", "2020-05", freq=freq_depr) + with pytest.raises(ValueError, match=msg): + PeriodIndex(["2020-01", "2020-05"], freq=freq_depr) + class TestPeriodIndex: def test_from_ordinals(self): diff --git a/pandas/tests/indexes/timedeltas/test_constructors.py b/pandas/tests/indexes/timedeltas/test_constructors.py index 0510700bb64d7..895ea110c8ad5 100644 --- a/pandas/tests/indexes/timedeltas/test_constructors.py +++ b/pandas/tests/indexes/timedeltas/test_constructors.py @@ -15,12 +15,6 @@ class TestTimedeltaIndex: - def test_closed_deprecated(self): - # GH#52628 - msg = "The 'closed' keyword" - with tm.assert_produces_warning(FutureWarning, match=msg): - TimedeltaIndex([], closed=True) - def test_array_of_dt64_nat_raises(self): # GH#39462 nat = np.datetime64("NaT", "ns") @@ -36,14 +30,6 @@ def test_array_of_dt64_nat_raises(self): with pytest.raises(TypeError, match=msg): to_timedelta(arr) - @pytest.mark.parametrize("unit", ["Y", "y", "M"]) - def test_unit_m_y_raises(self, unit): - msg = "Units 'M', 'Y', and 'y' are no longer supported" - depr_msg = "The 'unit' keyword in TimedeltaIndex construction is deprecated" - with pytest.raises(ValueError, match=msg): - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - TimedeltaIndex([1, 3, 7], unit) - def test_int64_nocopy(self): # GH#23539 check that a copy isn't made when we pass int64 data # and copy=False @@ -70,7 +56,6 @@ def test_infer_from_tdi_mismatch(self): # has one and it does not match the `freq` input tdi = timedelta_range("1 second", periods=100, freq="1s") - depr_msg = "TimedeltaArray.__init__ is deprecated" msg = ( "Inferred frequency .* from passed values does " "not conform to passed frequency" @@ -78,18 +63,9 @@ def test_infer_from_tdi_mismatch(self): with pytest.raises(ValueError, match=msg): TimedeltaIndex(tdi, freq="D") - with pytest.raises(ValueError, match=msg): - # GH#23789 - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - TimedeltaArray(tdi, freq="D") - with pytest.raises(ValueError, match=msg): TimedeltaIndex(tdi._data, freq="D") - with pytest.raises(ValueError, match=msg): - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - TimedeltaArray(tdi._data, freq="D") - def test_dt64_data_invalid(self): # GH#23539 # passing tz-aware DatetimeIndex raises, naive or ndarray[datetime64] @@ -138,9 +114,6 @@ def test_construction_base_constructor(self): tm.assert_index_equal(pd.Index(arr), TimedeltaIndex(arr)) tm.assert_index_equal(pd.Index(np.array(arr)), TimedeltaIndex(np.array(arr))) - @pytest.mark.filterwarnings( - "ignore:The 'unit' keyword in TimedeltaIndex construction:FutureWarning" - ) def test_constructor(self): expected = TimedeltaIndex( [ @@ -162,22 +135,6 @@ def test_constructor(self): ) tm.assert_index_equal(result, expected) - expected = TimedeltaIndex( - ["0 days 00:00:00", "0 days 00:00:01", "0 days 00:00:02"] - ) - result = TimedeltaIndex(range(3), unit="s") - tm.assert_index_equal(result, expected) - expected = TimedeltaIndex( - ["0 days 00:00:00", "0 days 00:00:05", "0 days 00:00:09"] - ) - result = TimedeltaIndex([0, 5, 9], unit="s") - tm.assert_index_equal(result, expected) - expected = TimedeltaIndex( - ["0 days 00:00:00.400", "0 days 00:00:00.450", "0 days 00:00:01.200"] - ) - result = TimedeltaIndex([400, 450, 1200], unit="ms") - tm.assert_index_equal(result, expected) - def test_constructor_iso(self): # GH #21877 expected = timedelta_range("1s", periods=9, freq="s") @@ -273,11 +230,6 @@ def test_explicit_none_freq(self): result = TimedeltaIndex(tdi._data, freq=None) assert result.freq is None - msg = "TimedeltaArray.__init__ is deprecated" - with tm.assert_produces_warning(FutureWarning, match=msg): - tda = TimedeltaArray(tdi, freq=None) - assert tda.freq is None - def test_from_categorical(self): tdi = timedelta_range(1, periods=5) diff --git a/pandas/tests/indexes/timedeltas/test_timedelta_range.py b/pandas/tests/indexes/timedeltas/test_timedelta_range.py index f22bdb7a90516..1b645e2bc607f 100644 --- a/pandas/tests/indexes/timedeltas/test_timedelta_range.py +++ b/pandas/tests/indexes/timedeltas/test_timedelta_range.py @@ -43,32 +43,24 @@ def test_timedelta_range(self): result = timedelta_range("0 days", freq="30min", periods=50) tm.assert_index_equal(result, expected) - @pytest.mark.parametrize( - "depr_unit, unit", - [ - ("H", "hour"), - ("T", "minute"), - ("t", "minute"), - ("S", "second"), - ("L", "millisecond"), - ("l", "millisecond"), - ("U", "microsecond"), - ("u", "microsecond"), - ("N", "nanosecond"), - ("n", "nanosecond"), - ], - ) - def test_timedelta_units_H_T_S_L_U_N_deprecated(self, depr_unit, unit): + @pytest.mark.parametrize("depr_unit, unit", [("H", "hour"), ("S", "second")]) + def test_timedelta_units_H_S_deprecated(self, depr_unit, unit): # GH#52536 depr_msg = ( f"'{depr_unit}' is deprecated and will be removed in a future version." ) - expected = to_timedelta(np.arange(5), unit=unit) with tm.assert_produces_warning(FutureWarning, match=depr_msg): result = to_timedelta(np.arange(5), unit=depr_unit) tm.assert_index_equal(result, expected) + @pytest.mark.parametrize("unit", ["T", "t", "L", "l", "U", "u", "N", "n"]) + def test_timedelta_unit_T_L_U_N_raises(self, unit): + msg = f"invalid unit abbreviation: {unit}" + + with pytest.raises(ValueError, match=msg): + to_timedelta(np.arange(5), unit=unit) + @pytest.mark.parametrize( "periods, freq", [(3, "2D"), (5, "D"), (6, "19h12min"), (7, "16h"), (9, "12h")] ) @@ -78,16 +70,21 @@ def test_linspace_behavior(self, periods, freq): expected = timedelta_range(start="0 days", end="4 days", freq=freq) tm.assert_index_equal(result, expected) - @pytest.mark.parametrize("msg_freq, freq", [("H", "19H12min"), ("T", "19h12T")]) - def test_timedelta_range_H_T_deprecated(self, freq, msg_freq): + def test_timedelta_range_H_deprecated(self): # GH#52536 - msg = f"'{msg_freq}' is deprecated and will be removed in a future version." + msg = "'H' is deprecated and will be removed in a future version." result = timedelta_range(start="0 days", end="4 days", periods=6) with tm.assert_produces_warning(FutureWarning, match=msg): - expected = timedelta_range(start="0 days", end="4 days", freq=freq) + expected = timedelta_range(start="0 days", end="4 days", freq="19H12min") tm.assert_index_equal(result, expected) + def test_timedelta_range_T_raises(self): + msg = "Invalid frequency: T" + + with pytest.raises(ValueError, match=msg): + timedelta_range(start="0 days", end="4 days", freq="19h12T") + def test_errors(self): # not enough params msg = ( @@ -143,18 +140,6 @@ def test_timedelta_range_infer_freq(self): ["0 days 05:03:01", "0 days 05:03:04.500000", "0 days 05:03:08"], "3500ms", ), - ( - "2.5T", - "5 hours", - "5 hours 8 minutes", - [ - "0 days 05:00:00", - "0 days 05:02:30", - "0 days 05:05:00", - "0 days 05:07:30", - ], - "150s", - ), ], ) def test_timedelta_range_deprecated_freq( @@ -171,3 +156,23 @@ def test_timedelta_range_deprecated_freq( expected_values, dtype="timedelta64[ns]", freq=expected_freq ) tm.assert_index_equal(result, expected) + + @pytest.mark.parametrize( + "freq_depr, start, end", + [ + ( + "3.5l", + "05:03:01", + "05:03:10", + ), + ( + "2.5T", + "5 hours", + "5 hours 8 minutes", + ), + ], + ) + def test_timedelta_range_removed_freq(self, freq_depr, start, end): + msg = f"Invalid frequency: {freq_depr}" + with pytest.raises(ValueError, match=msg): + timedelta_range(start=start, end=end, freq=freq_depr) diff --git a/pandas/tests/io/formats/style/test_format.py b/pandas/tests/io/formats/style/test_format.py index 1c84816ead140..ae68fcf9ef1fc 100644 --- a/pandas/tests/io/formats/style/test_format.py +++ b/pandas/tests/io/formats/style/test_format.py @@ -32,10 +32,14 @@ def styler(df): @pytest.fixture def df_multi(): - return DataFrame( - data=np.arange(16).reshape(4, 4), - columns=MultiIndex.from_product([["A", "B"], ["a", "b"]]), - index=MultiIndex.from_product([["X", "Y"], ["x", "y"]]), + return ( + DataFrame( + data=np.arange(16).reshape(4, 4), + columns=MultiIndex.from_product([["A", "B"], ["a", "b"]]), + index=MultiIndex.from_product([["X", "Y"], ["x", "y"]]), + ) + .rename_axis(["0_0", "0_1"], axis=0) + .rename_axis(["1_0", "1_1"], axis=1) ) @@ -560,3 +564,98 @@ def test_relabel_roundtrip(styler): ctx = styler._translate(True, True) assert {"value": "x", "display_value": "x"}.items() <= ctx["body"][0][0].items() assert {"value": "y", "display_value": "y"}.items() <= ctx["body"][1][0].items() + + +@pytest.mark.parametrize("axis", [0, 1]) +@pytest.mark.parametrize( + "level, expected", + [ + (0, ["X", "one"]), # level int + ("zero", ["X", "one"]), # level name + (1, ["zero", "X"]), # other level int + ("one", ["zero", "X"]), # other level name + ([0, 1], ["X", "X"]), # both levels + ([0, "zero"], ["X", "one"]), # level int and name simultaneous + ([0, "one"], ["X", "X"]), # both levels as int and name + (["one", "zero"], ["X", "X"]), # both level names, reversed + ], +) +def test_format_index_names_level(axis, level, expected): + midx = MultiIndex.from_arrays([["_", "_"], ["_", "_"]], names=["zero", "one"]) + df = DataFrame([[1, 2], [3, 4]]) + if axis == 0: + df.index = midx + else: + df.columns = midx + + styler = df.style.format_index_names(lambda v: "X", level=level, axis=axis) + ctx = styler._translate(True, True) + + if axis == 0: # compare index + result = [ctx["head"][1][s]["display_value"] for s in range(2)] + else: # compare columns + result = [ctx["head"][s][0]["display_value"] for s in range(2)] + assert expected == result + + +@pytest.mark.parametrize( + "attr, kwargs", + [ + ("_display_funcs_index_names", {"axis": 0}), + ("_display_funcs_column_names", {"axis": 1}), + ], +) +def test_format_index_names_clear(styler, attr, kwargs): + assert 0 not in getattr(styler, attr) # using default + styler.format_index_names("{:.2f}", **kwargs) + assert 0 in getattr(styler, attr) # formatter is specified + styler.format_index_names(**kwargs) + assert 0 not in getattr(styler, attr) # formatter cleared to default + + +@pytest.mark.parametrize("axis", [0, 1]) +def test_format_index_names_callable(styler_multi, axis): + ctx = styler_multi.format_index_names( + lambda v: v.replace("_", "A"), axis=axis + )._translate(True, True) + result = [ + ctx["head"][2][0]["display_value"], + ctx["head"][2][1]["display_value"], + ctx["head"][0][1]["display_value"], + ctx["head"][1][1]["display_value"], + ] + if axis == 0: + expected = ["0A0", "0A1", "1_0", "1_1"] + else: + expected = ["0_0", "0_1", "1A0", "1A1"] + assert result == expected + + +def test_format_index_names_dict(styler_multi): + ctx = ( + styler_multi.format_index_names({"0_0": "{:<<5}"}) + .format_index_names({"1_1": "{:>>4}"}, axis=1) + ._translate(True, True) + ) + assert ctx["head"][2][0]["display_value"] == "0_0<<" + assert ctx["head"][1][1]["display_value"] == ">1_1" + + +def test_format_index_names_with_hidden_levels(styler_multi): + ctx = styler_multi._translate(True, True) + full_head_height = len(ctx["head"]) + full_head_width = len(ctx["head"][0]) + assert full_head_height == 3 + assert full_head_width == 6 + + ctx = ( + styler_multi.hide(axis=0, level=1) + .hide(axis=1, level=1) + .format_index_names("{:>>4}", axis=1) + .format_index_names("{:!<5}") + ._translate(True, True) + ) + assert len(ctx["head"]) == full_head_height - 1 + assert len(ctx["head"][0]) == full_head_width - 1 + assert ctx["head"][0][0]["display_value"] == ">1_0" + assert ctx["head"][1][0]["display_value"] == "0_0!!" diff --git a/pandas/tests/io/formats/style/test_html.py b/pandas/tests/io/formats/style/test_html.py index 8cb06e3b7619d..2306324efb974 100644 --- a/pandas/tests/io/formats/style/test_html.py +++ b/pandas/tests/io/formats/style/test_html.py @@ -34,6 +34,16 @@ def styler_mi(): return Styler(DataFrame(np.arange(16).reshape(4, 4), index=midx, columns=midx)) +@pytest.fixture +def styler_multi(): + df = DataFrame( + data=np.arange(16).reshape(4, 4), + columns=MultiIndex.from_product([["A", "B"], ["a", "b"]], names=["A&", "b&"]), + index=MultiIndex.from_product([["X", "Y"], ["x", "y"]], names=["X>", "y_"]), + ) + return Styler(df) + + @pytest.fixture def tpl_style(env): return env.get_template("html_style.tpl") @@ -1003,3 +1013,23 @@ def test_to_html_na_rep_non_scalar_data(datapath): """ assert result == expected + + +@pytest.mark.parametrize("escape_axis_0", [True, False]) +@pytest.mark.parametrize("escape_axis_1", [True, False]) +def test_format_index_names(styler_multi, escape_axis_0, escape_axis_1): + if escape_axis_0: + styler_multi.format_index_names(axis=0, escape="html") + expected_index = ["X>", "y_"] + else: + expected_index = ["X>", "y_"] + + if escape_axis_1: + styler_multi.format_index_names(axis=1, escape="html") + expected_columns = ["A&", "b&"] + else: + expected_columns = ["A&", "b&"] + + result = styler_multi.to_html(table_uuid="test") + for expected_str in expected_index + expected_columns: + assert f"{expected_str}" in result diff --git a/pandas/tests/io/formats/style/test_style.py b/pandas/tests/io/formats/style/test_style.py index 6fa72bd48031c..89addbbbc1ded 100644 --- a/pandas/tests/io/formats/style/test_style.py +++ b/pandas/tests/io/formats/style/test_style.py @@ -77,6 +77,8 @@ def mi_styler_comp(mi_styler): columns=mi_styler.columns, ) ) + mi_styler.format_index_names(escape="html", axis=0) + mi_styler.format_index_names(escape="html", axis=1) return mi_styler diff --git a/pandas/tests/io/parser/common/test_verbose.py b/pandas/tests/io/parser/common/test_verbose.py deleted file mode 100644 index c5490afba1e04..0000000000000 --- a/pandas/tests/io/parser/common/test_verbose.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -Tests that work on both the Python and C engines but do not have a -specific classification into the other test modules. -""" - -from io import StringIO - -import pytest - -import pandas._testing as tm - -depr_msg = "The 'verbose' keyword in pd.read_csv is deprecated" - - -def test_verbose_read(all_parsers, capsys): - parser = all_parsers - data = """a,b,c,d -one,1,2,3 -one,1,2,3 -,1,2,3 -one,1,2,3 -,1,2,3 -,1,2,3 -one,1,2,3 -two,1,2,3""" - - if parser.engine == "pyarrow": - msg = "The 'verbose' option is not supported with the 'pyarrow' engine" - with pytest.raises(ValueError, match=msg): - with tm.assert_produces_warning( - FutureWarning, match=depr_msg, check_stacklevel=False - ): - parser.read_csv(StringIO(data), verbose=True) - return - - # Engines are verbose in different ways. - with tm.assert_produces_warning( - FutureWarning, match=depr_msg, check_stacklevel=False - ): - parser.read_csv(StringIO(data), verbose=True) - captured = capsys.readouterr() - - if parser.engine == "c": - assert "Tokenization took:" in captured.out - assert "Parser memory cleanup took:" in captured.out - else: # Python engine - assert captured.out == "Filled 3 NA values in column a\n" - - -def test_verbose_read2(all_parsers, capsys): - parser = all_parsers - data = """a,b,c,d -one,1,2,3 -two,1,2,3 -three,1,2,3 -four,1,2,3 -five,1,2,3 -,1,2,3 -seven,1,2,3 -eight,1,2,3""" - - if parser.engine == "pyarrow": - msg = "The 'verbose' option is not supported with the 'pyarrow' engine" - with pytest.raises(ValueError, match=msg): - with tm.assert_produces_warning( - FutureWarning, match=depr_msg, check_stacklevel=False - ): - parser.read_csv(StringIO(data), verbose=True, index_col=0) - return - - with tm.assert_produces_warning( - FutureWarning, match=depr_msg, check_stacklevel=False - ): - parser.read_csv(StringIO(data), verbose=True, index_col=0) - captured = capsys.readouterr() - - # Engines are verbose in different ways. - if parser.engine == "c": - assert "Tokenization took:" in captured.out - assert "Parser memory cleanup took:" in captured.out - else: # Python engine - assert captured.out == "Filled 1 NA values in column a\n" diff --git a/pandas/tests/io/parser/test_header.py b/pandas/tests/io/parser/test_header.py index d185e83bfc027..85ce55b3bcf83 100644 --- a/pandas/tests/io/parser/test_header.py +++ b/pandas/tests/io/parser/test_header.py @@ -162,7 +162,7 @@ def test_header_multi_index(all_parsers): {"index_col": ["foo", "bar"]}, ( "index_col must only contain " - "row numbers when specifying " + "integers of column positions when specifying " "a multi-index header" ), ), diff --git a/pandas/tests/io/pytables/test_append.py b/pandas/tests/io/pytables/test_append.py index b722a7f179479..7f7f7eccb2382 100644 --- a/pandas/tests/io/pytables/test_append.py +++ b/pandas/tests/io/pytables/test_append.py @@ -6,6 +6,7 @@ import pytest from pandas._libs.tslibs import Timestamp +from pandas.compat import PY312 import pandas as pd from pandas import ( @@ -283,7 +284,7 @@ def test_append_all_nans(setup_path): tm.assert_frame_equal(store["df2"], df, check_index_type=True) -def test_append_frame_column_oriented(setup_path): +def test_append_frame_column_oriented(setup_path, request): with ensure_clean_store(setup_path) as store: # column oriented df = DataFrame( @@ -303,6 +304,13 @@ def test_append_frame_column_oriented(setup_path): tm.assert_frame_equal(expected, result) # selection on the non-indexable + request.applymarker( + pytest.mark.xfail( + PY312, + reason="AST change in PY312", + raises=ValueError, + ) + ) result = store.select("df1", ("columns=A", "index=df.index[0:4]")) expected = df.reindex(columns=["A"], index=df.index[0:4]) tm.assert_frame_equal(expected, result) diff --git a/pandas/tests/io/pytables/test_select.py b/pandas/tests/io/pytables/test_select.py index 0e303d1c890c5..752e2fc570023 100644 --- a/pandas/tests/io/pytables/test_select.py +++ b/pandas/tests/io/pytables/test_select.py @@ -2,6 +2,7 @@ import pytest from pandas._libs.tslibs import Timestamp +from pandas.compat import PY312 import pandas as pd from pandas import ( @@ -168,7 +169,7 @@ def test_select(setup_path): tm.assert_frame_equal(expected, result) -def test_select_dtypes(setup_path): +def test_select_dtypes(setup_path, request): with ensure_clean_store(setup_path) as store: # with a Timestamp data column (GH #2637) df = DataFrame( @@ -279,6 +280,13 @@ def test_select_dtypes(setup_path): expected = df[df["A"] > 0] store.append("df", df, data_columns=True) + request.applymarker( + pytest.mark.xfail( + PY312, + reason="AST change in PY312", + raises=ValueError, + ) + ) np_zero = np.float64(0) # noqa: F841 result = store.select("df", where=["A>np_zero"]) tm.assert_frame_equal(expected, result) @@ -607,7 +615,7 @@ def test_select_iterator_many_empty_frames(setup_path): assert len(results) == 0 -def test_frame_select(setup_path): +def test_frame_select(setup_path, request): df = DataFrame( np.random.default_rng(2).standard_normal((10, 4)), columns=Index(list("ABCD"), dtype=object), @@ -624,6 +632,13 @@ def test_frame_select(setup_path): crit2 = "columns=['A', 'D']" crit3 = "columns=A" + request.applymarker( + pytest.mark.xfail( + PY312, + reason="AST change in PY312", + raises=TypeError, + ) + ) result = store.select("frame", [crit1, crit2]) expected = df.loc[date:, ["A", "D"]] tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/io/pytables/test_store.py b/pandas/tests/io/pytables/test_store.py index fda385685da19..e62df0bc1c977 100644 --- a/pandas/tests/io/pytables/test_store.py +++ b/pandas/tests/io/pytables/test_store.py @@ -7,6 +7,8 @@ import numpy as np import pytest +from pandas.compat import PY312 + import pandas as pd from pandas import ( DataFrame, @@ -866,7 +868,7 @@ def test_start_stop_fixed(setup_path): df.iloc[8:10, -2] = np.nan -def test_select_filter_corner(setup_path): +def test_select_filter_corner(setup_path, request): df = DataFrame(np.random.default_rng(2).standard_normal((50, 100))) df.index = [f"{c:3d}" for c in df.index] df.columns = [f"{c:3d}" for c in df.columns] @@ -874,6 +876,13 @@ def test_select_filter_corner(setup_path): with ensure_clean_store(setup_path) as store: store.put("frame", df, format="table") + request.applymarker( + pytest.mark.xfail( + PY312, + reason="AST change in PY312", + raises=ValueError, + ) + ) crit = "columns=df.columns[:75]" result = store.select("frame", [crit]) tm.assert_frame_equal(result, df.loc[:, df.columns[:75]]) diff --git a/pandas/tests/plotting/frame/test_frame_subplots.py b/pandas/tests/plotting/frame/test_frame_subplots.py index 16853114d93cd..511266d5786c5 100644 --- a/pandas/tests/plotting/frame/test_frame_subplots.py +++ b/pandas/tests/plotting/frame/test_frame_subplots.py @@ -187,9 +187,9 @@ def test_subplots_timeseries_y_axis_not_supported(self): data = { "numeric": np.array([1, 2, 5]), "period": [ - pd.Period("2017-08-01 00:00:00", freq="H"), - pd.Period("2017-08-01 02:00", freq="H"), - pd.Period("2017-08-02 00:00:00", freq="H"), + pd.Period("2017-08-01 00:00:00", freq="h"), + pd.Period("2017-08-01 02:00", freq="h"), + pd.Period("2017-08-02 00:00:00", freq="h"), ], "categorical": pd.Categorical( ["c", "b", "a"], categories=["a", "b", "c"], ordered=False diff --git a/pandas/tests/plotting/test_datetimelike.py b/pandas/tests/plotting/test_datetimelike.py index 7164b7a046ff2..6b709522bab70 100644 --- a/pandas/tests/plotting/test_datetimelike.py +++ b/pandas/tests/plotting/test_datetimelike.py @@ -927,7 +927,7 @@ def test_mixed_freq_shared_ax_twin_x(self): @pytest.mark.xfail(reason="TODO (GH14330, GH14322)") def test_mixed_freq_shared_ax_twin_x_irregular_first(self): # GH13341, using sharex=True - idx1 = date_range("2015-01-01", periods=3, freq="M") + idx1 = date_range("2015-01-01", periods=3, freq="ME") idx2 = idx1[:1].union(idx1[2:]) s1 = Series(range(len(idx1)), idx1) s2 = Series(range(len(idx2)), idx2) diff --git a/pandas/tests/reductions/test_reductions.py b/pandas/tests/reductions/test_reductions.py index 91ee13ecd87dd..b10319f5380e7 100644 --- a/pandas/tests/reductions/test_reductions.py +++ b/pandas/tests/reductions/test_reductions.py @@ -128,28 +128,14 @@ def test_nanargminmax(self, opname, index_or_series): obj = klass([NaT, datetime(2011, 11, 1)]) assert getattr(obj, arg_op)() == 1 - msg = ( - "The behavior of (DatetimeIndex|Series).argmax/argmin with " - "skipna=False and NAs" - ) - if klass is Series: - msg = "The behavior of Series.(idxmax|idxmin) with all-NA" - with tm.assert_produces_warning(FutureWarning, match=msg): - result = getattr(obj, arg_op)(skipna=False) - if klass is Series: - assert np.isnan(result) - else: - assert result == -1 + with pytest.raises(ValueError, match="Encountered an NA value"): + getattr(obj, arg_op)(skipna=False) obj = klass([NaT, datetime(2011, 11, 1), NaT]) # check DatetimeIndex non-monotonic path assert getattr(obj, arg_op)() == 1 - with tm.assert_produces_warning(FutureWarning, match=msg): - result = getattr(obj, arg_op)(skipna=False) - if klass is Series: - assert np.isnan(result) - else: - assert result == -1 + with pytest.raises(ValueError, match="Encountered an NA value"): + getattr(obj, arg_op)(skipna=False) @pytest.mark.parametrize("opname", ["max", "min"]) @pytest.mark.parametrize("dtype", ["M8[ns]", "datetime64[ns, UTC]"]) @@ -175,40 +161,38 @@ def test_argminmax(self): obj = Index([np.nan, 1, np.nan, 2]) assert obj.argmin() == 1 assert obj.argmax() == 3 - msg = "The behavior of Index.argmax/argmin with skipna=False and NAs" - with tm.assert_produces_warning(FutureWarning, match=msg): - assert obj.argmin(skipna=False) == -1 - with tm.assert_produces_warning(FutureWarning, match=msg): - assert obj.argmax(skipna=False) == -1 + with pytest.raises(ValueError, match="Encountered an NA value"): + obj.argmin(skipna=False) + with pytest.raises(ValueError, match="Encountered an NA value"): + obj.argmax(skipna=False) obj = Index([np.nan]) - with tm.assert_produces_warning(FutureWarning, match=msg): - assert obj.argmin() == -1 - with tm.assert_produces_warning(FutureWarning, match=msg): - assert obj.argmax() == -1 - with tm.assert_produces_warning(FutureWarning, match=msg): - assert obj.argmin(skipna=False) == -1 - with tm.assert_produces_warning(FutureWarning, match=msg): - assert obj.argmax(skipna=False) == -1 + with pytest.raises(ValueError, match="Encountered all NA values"): + obj.argmin() + with pytest.raises(ValueError, match="Encountered all NA values"): + obj.argmax() + with pytest.raises(ValueError, match="Encountered all NA values"): + obj.argmin(skipna=False) + with pytest.raises(ValueError, match="Encountered all NA values"): + obj.argmax(skipna=False) - msg = "The behavior of DatetimeIndex.argmax/argmin with skipna=False and NAs" obj = Index([NaT, datetime(2011, 11, 1), datetime(2011, 11, 2), NaT]) assert obj.argmin() == 1 assert obj.argmax() == 2 - with tm.assert_produces_warning(FutureWarning, match=msg): - assert obj.argmin(skipna=False) == -1 - with tm.assert_produces_warning(FutureWarning, match=msg): - assert obj.argmax(skipna=False) == -1 + with pytest.raises(ValueError, match="Encountered an NA value"): + obj.argmin(skipna=False) + with pytest.raises(ValueError, match="Encountered an NA value"): + obj.argmax(skipna=False) obj = Index([NaT]) - with tm.assert_produces_warning(FutureWarning, match=msg): - assert obj.argmin() == -1 - with tm.assert_produces_warning(FutureWarning, match=msg): - assert obj.argmax() == -1 - with tm.assert_produces_warning(FutureWarning, match=msg): - assert obj.argmin(skipna=False) == -1 - with tm.assert_produces_warning(FutureWarning, match=msg): - assert obj.argmax(skipna=False) == -1 + with pytest.raises(ValueError, match="Encountered all NA values"): + obj.argmin() + with pytest.raises(ValueError, match="Encountered all NA values"): + obj.argmax() + with pytest.raises(ValueError, match="Encountered all NA values"): + obj.argmin(skipna=False) + with pytest.raises(ValueError, match="Encountered all NA values"): + obj.argmax(skipna=False) @pytest.mark.parametrize("op, expected_col", [["max", "a"], ["min", "b"]]) def test_same_tz_min_max_axis_1(self, op, expected_col): @@ -841,26 +825,16 @@ def test_idxmin_dt64index(self, unit): # GH#43587 should have NaT instead of NaN dti = DatetimeIndex(["NaT", "2015-02-08", "NaT"]).as_unit(unit) ser = Series([1.0, 2.0, np.nan], index=dti) - msg = "The behavior of Series.idxmin with all-NA values" - with tm.assert_produces_warning(FutureWarning, match=msg): - res = ser.idxmin(skipna=False) - assert res is NaT - msg = "The behavior of Series.idxmax with all-NA values" - with tm.assert_produces_warning(FutureWarning, match=msg): - res = ser.idxmax(skipna=False) - assert res is NaT + with pytest.raises(ValueError, match="Encountered an NA value"): + ser.idxmin(skipna=False) + with pytest.raises(ValueError, match="Encountered an NA value"): + ser.idxmax(skipna=False) df = ser.to_frame() - msg = "The behavior of DataFrame.idxmin with all-NA values" - with tm.assert_produces_warning(FutureWarning, match=msg): - res = df.idxmin(skipna=False) - assert res.dtype == f"M8[{unit}]" - assert res.isna().all() - msg = "The behavior of DataFrame.idxmax with all-NA values" - with tm.assert_produces_warning(FutureWarning, match=msg): - res = df.idxmax(skipna=False) - assert res.dtype == f"M8[{unit}]" - assert res.isna().all() + with pytest.raises(ValueError, match="Encountered an NA value"): + df.idxmin(skipna=False) + with pytest.raises(ValueError, match="Encountered an NA value"): + df.idxmax(skipna=False) def test_idxmin(self): # test idxmin @@ -872,9 +846,8 @@ def test_idxmin(self): # skipna or no assert string_series[string_series.idxmin()] == string_series.min() - msg = "The behavior of Series.idxmin" - with tm.assert_produces_warning(FutureWarning, match=msg): - assert isna(string_series.idxmin(skipna=False)) + with pytest.raises(ValueError, match="Encountered an NA value"): + string_series.idxmin(skipna=False) # no NaNs nona = string_series.dropna() @@ -883,8 +856,8 @@ def test_idxmin(self): # all NaNs allna = string_series * np.nan - with tm.assert_produces_warning(FutureWarning, match=msg): - assert isna(allna.idxmin()) + with pytest.raises(ValueError, match="Encountered all NA values"): + allna.idxmin() # datetime64[ns] s = Series(date_range("20130102", periods=6)) @@ -905,8 +878,7 @@ def test_idxmax(self): # skipna or no assert string_series[string_series.idxmax()] == string_series.max() - msg = "The behavior of Series.idxmax with all-NA values" - with tm.assert_produces_warning(FutureWarning, match=msg): + with pytest.raises(ValueError, match="Encountered an NA value"): assert isna(string_series.idxmax(skipna=False)) # no NaNs @@ -916,9 +888,8 @@ def test_idxmax(self): # all NaNs allna = string_series * np.nan - msg = "The behavior of Series.idxmax with all-NA values" - with tm.assert_produces_warning(FutureWarning, match=msg): - assert isna(allna.idxmax()) + with pytest.raises(ValueError, match="Encountered all NA values"): + allna.idxmax() s = Series(date_range("20130102", periods=6)) result = s.idxmax() @@ -1175,12 +1146,12 @@ def test_idxminmax_object_dtype(self, using_infer_string): msg = "'>' not supported between instances of 'float' and 'str'" with pytest.raises(TypeError, match=msg): ser3.idxmax() - with pytest.raises(TypeError, match=msg): + with pytest.raises(ValueError, match="Encountered an NA value"): ser3.idxmax(skipna=False) msg = "'<' not supported between instances of 'float' and 'str'" with pytest.raises(TypeError, match=msg): ser3.idxmin() - with pytest.raises(TypeError, match=msg): + with pytest.raises(ValueError, match="Encountered an NA value"): ser3.idxmin(skipna=False) def test_idxminmax_object_frame(self): @@ -1228,14 +1199,12 @@ def test_idxminmax_with_inf(self): s = Series([0, -np.inf, np.inf, np.nan]) assert s.idxmin() == 1 - msg = "The behavior of Series.idxmin with all-NA values" - with tm.assert_produces_warning(FutureWarning, match=msg): - assert np.isnan(s.idxmin(skipna=False)) + with pytest.raises(ValueError, match="Encountered an NA value"): + s.idxmin(skipna=False) assert s.idxmax() == 2 - msg = "The behavior of Series.idxmax with all-NA values" - with tm.assert_produces_warning(FutureWarning, match=msg): - assert np.isnan(s.idxmax(skipna=False)) + with pytest.raises(ValueError, match="Encountered an NA value"): + s.idxmax(skipna=False) def test_sum_uint64(self): # GH 53401 diff --git a/pandas/tests/resample/test_period_index.py b/pandas/tests/resample/test_period_index.py index 6fdc398b13835..dd058ada60974 100644 --- a/pandas/tests/resample/test_period_index.py +++ b/pandas/tests/resample/test_period_index.py @@ -982,22 +982,20 @@ def test_sum_min_count(self): def test_resample_t_l_deprecated(self): # GH#52536 - msg_t = "'T' is deprecated and will be removed in a future version." - msg_l = "'L' is deprecated and will be removed in a future version." + msg_t = "Invalid frequency: T" + msg_l = "Invalid frequency: L" - with tm.assert_produces_warning(FutureWarning, match=msg_l): - rng_l = period_range( + with pytest.raises(ValueError, match=msg_l): + period_range( "2020-01-01 00:00:00 00:00", "2020-01-01 00:00:00 00:01", freq="L" ) + rng_l = period_range( + "2020-01-01 00:00:00 00:00", "2020-01-01 00:00:00 00:01", freq="ms" + ) ser = Series(np.arange(len(rng_l)), index=rng_l) - rng = period_range( - "2020-01-01 00:00:00 00:00", "2020-01-01 00:00:00 00:01", freq="min" - ) - expected = Series([29999.5, 60000.0], index=rng) - with tm.assert_produces_warning(FutureWarning, match=msg_t): - result = ser.resample("T").mean() - tm.assert_series_equal(result, expected) + with pytest.raises(ValueError, match=msg_t): + ser.resample("T").mean() @pytest.mark.parametrize( "freq, freq_depr, freq_res, freq_depr_res, data", diff --git a/pandas/tests/reshape/concat/test_concat.py b/pandas/tests/reshape/concat/test_concat.py index cf11bf237f615..b986aa8182219 100644 --- a/pandas/tests/reshape/concat/test_concat.py +++ b/pandas/tests/reshape/concat/test_concat.py @@ -50,12 +50,12 @@ def test_concat_copy(self): df3 = DataFrame({5: "foo"}, index=range(4)) # These are actual copies. - result = concat([df, df2, df3], axis=1, copy=True) + result = concat([df, df2, df3], axis=1) for arr in result._mgr.arrays: assert arr.base is not None # These are the same. - result = concat([df, df2, df3], axis=1, copy=False) + result = concat([df, df2, df3], axis=1) for arr in result._mgr.arrays: if arr.dtype.kind == "f": @@ -67,7 +67,7 @@ def test_concat_copy(self): # Float block was consolidated. df4 = DataFrame(np.random.default_rng(2).standard_normal((4, 1))) - result = concat([df, df2, df3, df4], axis=1, copy=False) + result = concat([df, df2, df3, df4], axis=1) for arr in result._mgr.arrays: if arr.dtype.kind == "f": # this is a view on some array in either df or df4 diff --git a/pandas/tests/reshape/concat/test_index.py b/pandas/tests/reshape/concat/test_index.py index ca544c5d42a25..68d77b79a59e7 100644 --- a/pandas/tests/reshape/concat/test_index.py +++ b/pandas/tests/reshape/concat/test_index.py @@ -101,7 +101,7 @@ def test_concat_rename_index(self): def test_concat_copy_index_series(self, axis): # GH 29879 ser = Series([1, 2]) - comb = concat([ser, ser], axis=axis, copy=True) + comb = concat([ser, ser], axis=axis) if axis in [0, "index"]: assert comb.index is not ser.index else: @@ -110,7 +110,7 @@ def test_concat_copy_index_series(self, axis): def test_concat_copy_index_frame(self, axis): # GH 29879 df = DataFrame([[1, 2], [3, 4]], columns=["a", "b"]) - comb = concat([df, df], axis=axis, copy=True) + comb = concat([df, df], axis=axis) if axis in [0, "index"]: assert not comb.index.is_(df.index) assert comb.columns.is_(df.columns) diff --git a/pandas/tests/reshape/merge/test_merge.py b/pandas/tests/reshape/merge/test_merge.py index f063f333ac889..1cd52ab1ae8b4 100644 --- a/pandas/tests/reshape/merge/test_merge.py +++ b/pandas/tests/reshape/merge/test_merge.py @@ -260,7 +260,7 @@ def test_merge_copy(self): left = DataFrame({"a": 0, "b": 1}, index=range(10)) right = DataFrame({"c": "foo", "d": "bar"}, index=range(10)) - merged = merge(left, right, left_index=True, right_index=True, copy=True) + merged = merge(left, right, left_index=True, right_index=True) merged["a"] = 6 assert (left["a"] == 0).all() @@ -272,7 +272,7 @@ def test_merge_nocopy(self, using_infer_string): left = DataFrame({"a": 0, "b": 1}, index=range(10)) right = DataFrame({"c": "foo", "d": "bar"}, index=range(10)) - merged = merge(left, right, left_index=True, right_index=True, copy=False) + merged = merge(left, right, left_index=True, right_index=True) assert np.shares_memory(merged["a"]._values, left["a"]._values) if not using_infer_string: diff --git a/pandas/tests/reshape/test_pivot.py b/pandas/tests/reshape/test_pivot.py index 99250dc929997..f750d5e7fa919 100644 --- a/pandas/tests/reshape/test_pivot.py +++ b/pandas/tests/reshape/test_pivot.py @@ -1738,6 +1738,7 @@ def test_daily(self): mask = ts.index.year == y expected[y] = Series(ts.values[mask], index=doy[mask]) expected = DataFrame(expected, dtype=float).T + expected.index = expected.index.astype(np.int32) tm.assert_frame_equal(result, expected) def test_monthly(self): @@ -1753,6 +1754,7 @@ def test_monthly(self): mask = ts.index.year == y expected[y] = Series(ts.values[mask], index=month[mask]) expected = DataFrame(expected, dtype=float).T + expected.index = expected.index.astype(np.int32) tm.assert_frame_equal(result, expected) def test_pivot_table_with_iterator_values(self, data): diff --git a/pandas/tests/scalar/period/test_asfreq.py b/pandas/tests/scalar/period/test_asfreq.py index 73c4d8061c257..1a21d234f1d50 100644 --- a/pandas/tests/scalar/period/test_asfreq.py +++ b/pandas/tests/scalar/period/test_asfreq.py @@ -116,8 +116,8 @@ def test_conv_annual(self): assert ival_A.asfreq("H", "E") == ival_A_to_H_end assert ival_A.asfreq("min", "s") == ival_A_to_T_start assert ival_A.asfreq("min", "E") == ival_A_to_T_end - msg = "'T' is deprecated and will be removed in a future version." - with tm.assert_produces_warning(FutureWarning, match=msg): + msg = "Invalid frequency: T" + with pytest.raises(ValueError, match=msg): assert ival_A.asfreq("T", "s") == ival_A_to_T_start assert ival_A.asfreq("T", "E") == ival_A_to_T_end msg = "'S' is deprecated and will be removed in a future version." diff --git a/pandas/tests/scalar/timedelta/test_constructors.py b/pandas/tests/scalar/timedelta/test_constructors.py index e680ca737b546..5509216f4daf4 100644 --- a/pandas/tests/scalar/timedelta/test_constructors.py +++ b/pandas/tests/scalar/timedelta/test_constructors.py @@ -32,24 +32,14 @@ def test_unit_m_y_raises(self, unit): with pytest.raises(ValueError, match=msg): to_timedelta([1, 2], unit) - @pytest.mark.parametrize( - "unit,unit_depr", - [ - ("h", "H"), - ("min", "T"), - ("s", "S"), - ("ms", "L"), - ("ns", "N"), - ("us", "U"), - ], - ) - def test_units_H_T_S_L_N_U_deprecated(self, unit, unit_depr): + @pytest.mark.parametrize("unit", ["h", "s"]) + def test_units_H_S_deprecated(self, unit): # GH#52536 - msg = f"'{unit_depr}' is deprecated and will be removed in a future version." + msg = f"'{unit.upper()}' is deprecated and will be removed in a future version." expected = Timedelta(1, unit=unit) with tm.assert_produces_warning(FutureWarning, match=msg): - result = Timedelta(1, unit=unit_depr) + result = Timedelta(1, unit=unit.upper()) tm.assert_equal(result, expected) @pytest.mark.parametrize( @@ -103,13 +93,11 @@ def test_units_H_T_S_L_N_U_deprecated(self, unit, unit_depr): "microsecond", "micro", "micros", - "u", "US", "Microseconds", "Microsecond", "Micro", "Micros", - "U", ] ] + [ @@ -120,13 +108,11 @@ def test_units_H_T_S_L_N_U_deprecated(self, unit, unit_depr): "nanosecond", "nano", "nanos", - "n", "NS", "Nanoseconds", "Nanosecond", "Nano", "Nanos", - "N", ] ], ) @@ -139,36 +125,39 @@ def test_unit_parser(self, unit, np_unit, wrapper): dtype="m8[ns]", ) # TODO(2.0): the desired output dtype may have non-nano resolution - msg = f"'{unit}' is deprecated and will be removed in a future version." - - if (unit, np_unit) in (("u", "us"), ("U", "us"), ("n", "ns"), ("N", "ns")): - warn = FutureWarning - else: - warn = FutureWarning - msg = "The 'unit' keyword in TimedeltaIndex construction is deprecated" - with tm.assert_produces_warning(warn, match=msg): - result = to_timedelta(wrapper(range(5)), unit=unit) - tm.assert_index_equal(result, expected) - result = TimedeltaIndex(wrapper(range(5)), unit=unit) - tm.assert_index_equal(result, expected) - - str_repr = [f"{x}{unit}" for x in np.arange(5)] - result = to_timedelta(wrapper(str_repr)) - tm.assert_index_equal(result, expected) - result = to_timedelta(wrapper(str_repr)) - tm.assert_index_equal(result, expected) - - # scalar - expected = Timedelta(np.timedelta64(2, np_unit).astype("timedelta64[ns]")) - result = to_timedelta(2, unit=unit) - assert result == expected - result = Timedelta(2, unit=unit) - assert result == expected - - result = to_timedelta(f"2{unit}") - assert result == expected - result = Timedelta(f"2{unit}") - assert result == expected + + result = to_timedelta(wrapper(range(5)), unit=unit) + tm.assert_index_equal(result, expected) + + str_repr = [f"{x}{unit}" for x in np.arange(5)] + result = to_timedelta(wrapper(str_repr)) + tm.assert_index_equal(result, expected) + result = to_timedelta(wrapper(str_repr)) + tm.assert_index_equal(result, expected) + + # scalar + expected = Timedelta(np.timedelta64(2, np_unit).astype("timedelta64[ns]")) + result = to_timedelta(2, unit=unit) + assert result == expected + result = Timedelta(2, unit=unit) + assert result == expected + + result = to_timedelta(f"2{unit}") + assert result == expected + result = Timedelta(f"2{unit}") + assert result == expected + + @pytest.mark.parametrize("unit", ["T", "t", "L", "l", "U", "u", "N", "n"]) + def test_unit_T_L_N_U_raises(self, unit): + msg = f"invalid unit abbreviation: {unit}" + with pytest.raises(ValueError, match=msg): + Timedelta(1, unit=unit) + + with pytest.raises(ValueError, match=msg): + to_timedelta(10, unit) + + with pytest.raises(ValueError, match=msg): + to_timedelta([1, 2], unit) def test_construct_from_kwargs_overflow(): diff --git a/pandas/tests/scalar/timedelta/test_timedelta.py b/pandas/tests/scalar/timedelta/test_timedelta.py index 06a0f3324c2cf..73b2da0f7dd50 100644 --- a/pandas/tests/scalar/timedelta/test_timedelta.py +++ b/pandas/tests/scalar/timedelta/test_timedelta.py @@ -665,3 +665,10 @@ def test_timedelta_attribute_precision(): result += td.nanoseconds expected = td._value assert result == expected + + +def test_to_pytimedelta_large_values(): + td = Timedelta(1152921504609987375, unit="ns") + result = td.to_pytimedelta() + expected = timedelta(days=13343, seconds=86304, microseconds=609987) + assert result == expected diff --git a/pandas/tests/series/methods/test_interpolate.py b/pandas/tests/series/methods/test_interpolate.py index e4726f3ec6b32..c5df1fd498938 100644 --- a/pandas/tests/series/methods/test_interpolate.py +++ b/pandas/tests/series/methods/test_interpolate.py @@ -344,7 +344,7 @@ def test_interpolate_invalid_float_limit(self, nontemporal_method): def test_interp_invalid_method(self, invalid_method): s = Series([1, 3, np.nan, 12, np.nan, 25]) - msg = f"method must be one of.* Got '{invalid_method}' instead" + msg = "Can not interpolate with method=nonexistent_method" if invalid_method is None: msg = "'method' should be a string, not None" with pytest.raises(ValueError, match=msg): @@ -355,16 +355,6 @@ def test_interp_invalid_method(self, invalid_method): with pytest.raises(ValueError, match=msg): s.interpolate(method=invalid_method, limit=-1) - def test_interp_invalid_method_and_value(self): - # GH#36624 - ser = Series([1, 3, np.nan, 12, np.nan, 25]) - - msg = "'fill_value' is not a valid keyword for Series.interpolate" - msg2 = "Series.interpolate with method=pad" - with pytest.raises(ValueError, match=msg): - with tm.assert_produces_warning(FutureWarning, match=msg2): - ser.interpolate(fill_value=3, method="pad") - def test_interp_limit_forward(self): s = Series([1, 3, np.nan, np.nan, np.nan, 11]) @@ -455,107 +445,70 @@ def test_interp_limit_area(self): s.interpolate(method="linear", limit_area="abc") @pytest.mark.parametrize( - "method, limit_direction, expected", - [ - ("pad", "backward", "forward"), - ("ffill", "backward", "forward"), - ("backfill", "forward", "backward"), - ("bfill", "forward", "backward"), - ("pad", "both", "forward"), - ("ffill", "both", "forward"), - ("backfill", "both", "backward"), - ("bfill", "both", "backward"), - ], - ) - def test_interp_limit_direction_raises(self, method, limit_direction, expected): - # https://github.com/pandas-dev/pandas/pull/34746 - s = Series([1, 2, 3]) - - msg = f"`limit_direction` must be '{expected}' for method `{method}`" - msg2 = "Series.interpolate with method=" - with pytest.raises(ValueError, match=msg): - with tm.assert_produces_warning(FutureWarning, match=msg2): - s.interpolate(method=method, limit_direction=limit_direction) - - @pytest.mark.parametrize( - "data, expected_data, kwargs", + "data, kwargs", ( ( [np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan], - [np.nan, np.nan, 3.0, 3.0, 3.0, 3.0, 7.0, np.nan, np.nan], {"method": "pad", "limit_area": "inside"}, ), ( [np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan], - [np.nan, np.nan, 3.0, 3.0, np.nan, np.nan, 7.0, np.nan, np.nan], {"method": "pad", "limit_area": "inside", "limit": 1}, ), ( [np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan], - [np.nan, np.nan, 3.0, np.nan, np.nan, np.nan, 7.0, 7.0, 7.0], {"method": "pad", "limit_area": "outside"}, ), ( [np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan], - [np.nan, np.nan, 3.0, np.nan, np.nan, np.nan, 7.0, 7.0, np.nan], {"method": "pad", "limit_area": "outside", "limit": 1}, ), ( - [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan], {"method": "pad", "limit_area": "outside", "limit": 1}, ), ( - range(5), range(5), {"method": "pad", "limit_area": "outside", "limit": 1}, ), ), ) - def test_interp_limit_area_with_pad(self, data, expected_data, kwargs): + def test_interp_limit_area_with_pad(self, data, kwargs): # GH26796 s = Series(data) - expected = Series(expected_data) - msg = "Series.interpolate with method=pad" - with tm.assert_produces_warning(FutureWarning, match=msg): - result = s.interpolate(**kwargs) - tm.assert_series_equal(result, expected) + msg = "Can not interpolate with method=pad" + with pytest.raises(ValueError, match=msg): + s.interpolate(**kwargs) @pytest.mark.parametrize( - "data, expected_data, kwargs", + "data, kwargs", ( ( [np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan], - [np.nan, np.nan, 3.0, 7.0, 7.0, 7.0, 7.0, np.nan, np.nan], {"method": "bfill", "limit_area": "inside"}, ), ( [np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan], - [np.nan, np.nan, 3.0, np.nan, np.nan, 7.0, 7.0, np.nan, np.nan], {"method": "bfill", "limit_area": "inside", "limit": 1}, ), ( [np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan], - [3.0, 3.0, 3.0, np.nan, np.nan, np.nan, 7.0, np.nan, np.nan], {"method": "bfill", "limit_area": "outside"}, ), ( [np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan], - [np.nan, 3.0, 3.0, np.nan, np.nan, np.nan, 7.0, np.nan, np.nan], {"method": "bfill", "limit_area": "outside", "limit": 1}, ), ), ) - def test_interp_limit_area_with_backfill(self, data, expected_data, kwargs): + def test_interp_limit_area_with_backfill(self, data, kwargs): # GH26796 - s = Series(data) - expected = Series(expected_data) - msg = "Series.interpolate with method=bfill" - with tm.assert_produces_warning(FutureWarning, match=msg): - result = s.interpolate(**kwargs) - tm.assert_series_equal(result, expected) + + msg = "Can not interpolate with method=bfill" + with pytest.raises(ValueError, match=msg): + s.interpolate(**kwargs) def test_interp_limit_direction(self): # These tests are for issue #9218 -- fill NaNs in both directions. @@ -650,20 +603,18 @@ def test_interp_datetime64(self, method, tz_naive_fixture): df = Series( [1, np.nan, 3], index=date_range("1/1/2000", periods=3, tz=tz_naive_fixture) ) - warn = None if method == "nearest" else FutureWarning - msg = "Series.interpolate with method=pad is deprecated" - with tm.assert_produces_warning(warn, match=msg): - result = df.interpolate(method=method) - if warn is not None: - # check the "use ffill instead" is equivalent - alt = df.ffill() - tm.assert_series_equal(result, alt) - expected = Series( - [1.0, 1.0, 3.0], - index=date_range("1/1/2000", periods=3, tz=tz_naive_fixture), - ) - tm.assert_series_equal(result, expected) + if method == "nearest": + result = df.interpolate(method=method) + expected = Series( + [1.0, 1.0, 3.0], + index=date_range("1/1/2000", periods=3, tz=tz_naive_fixture), + ) + tm.assert_series_equal(result, expected) + else: + msg = "Can not interpolate with method=pad" + with pytest.raises(ValueError, match=msg): + df.interpolate(method=method) def test_interp_pad_datetime64tz_values(self): # GH#27628 missing.interpolate_2d should handle datetimetz values @@ -671,16 +622,9 @@ def test_interp_pad_datetime64tz_values(self): ser = Series(dti) ser[1] = pd.NaT - msg = "Series.interpolate with method=pad is deprecated" - with tm.assert_produces_warning(FutureWarning, match=msg): - result = ser.interpolate(method="pad") - # check the "use ffill instead" is equivalent - alt = ser.ffill() - tm.assert_series_equal(result, alt) - - expected = Series(dti) - expected[1] = expected[0] - tm.assert_series_equal(result, expected) + msg = "Can not interpolate with method=pad" + with pytest.raises(ValueError, match=msg): + ser.interpolate(method="pad") def test_interp_limit_no_nans(self): # GH 7173 diff --git a/pandas/tests/series/methods/test_rename.py b/pandas/tests/series/methods/test_rename.py index c67298b777f6d..1da98b3a273be 100644 --- a/pandas/tests/series/methods/test_rename.py +++ b/pandas/tests/series/methods/test_rename.py @@ -173,7 +173,7 @@ def test_rename_copy_false(self): # GH 46889 ser = Series(["foo", "bar"]) ser_orig = ser.copy() - shallow_copy = ser.rename({1: 9}, copy=False) + shallow_copy = ser.rename({1: 9}) ser[0] = "foobar" assert ser_orig[0] == shallow_copy[0] assert ser_orig[1] == shallow_copy[9] diff --git a/pandas/tests/test_downstream.py b/pandas/tests/test_downstream.py index a4fd29878a2d1..ee26fdae74960 100644 --- a/pandas/tests/test_downstream.py +++ b/pandas/tests/test_downstream.py @@ -20,10 +20,6 @@ TimedeltaIndex, ) import pandas._testing as tm -from pandas.core.arrays import ( - DatetimeArray, - TimedeltaArray, -) @pytest.fixture @@ -284,14 +280,6 @@ def test_from_obscure_array(dtype, box): else: data = box(arr) - cls = {"M8[ns]": DatetimeArray, "m8[ns]": TimedeltaArray}[dtype] - - depr_msg = f"{cls.__name__}.__init__ is deprecated" - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - expected = cls(arr) - result = cls._from_sequence(data, dtype=dtype) - tm.assert_extension_array_equal(result, expected) - if not isinstance(data, memoryview): # FIXME(GH#44431) these raise on memoryview and attempted fix # fails on py3.10 diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index ed125ece349a9..ce41f1e76de79 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -296,6 +296,7 @@ def check_fun_data( self, testfunc, targfunc, + testar, testarval, targarval, skipna, @@ -319,6 +320,13 @@ def check_fun_data( else: targ = bool(targ) + if testfunc.__name__ in ["nanargmax", "nanargmin"] and ( + testar.startswith("arr_nan") + or (testar.endswith("nan") and (not skipna or axis == 1)) + ): + with pytest.raises(ValueError, match="Encountered .* NA value"): + testfunc(testarval, axis=axis, skipna=skipna, **kwargs) + return res = testfunc(testarval, axis=axis, skipna=skipna, **kwargs) if ( @@ -350,6 +358,7 @@ def check_fun_data( self.check_fun_data( testfunc, targfunc, + testar, testarval2, targarval2, skipna=skipna, @@ -370,6 +379,7 @@ def check_fun( self.check_fun_data( testfunc, targfunc, + testar, testarval, targarval, skipna=skipna, diff --git a/pandas/tests/tseries/offsets/test_ticks.py b/pandas/tests/tseries/offsets/test_ticks.py index c8fbdfa11991a..f91230e1460c4 100644 --- a/pandas/tests/tseries/offsets/test_ticks.py +++ b/pandas/tests/tseries/offsets/test_ticks.py @@ -16,7 +16,6 @@ import pytest from pandas._libs.tslibs.offsets import delta_to_tick -from pandas.errors import OutOfBoundsTimedelta from pandas import ( Timedelta, @@ -239,16 +238,6 @@ def test_tick_addition(kls, expected): assert result == expected -def test_tick_delta_overflow(): - # GH#55503 raise OutOfBoundsTimedelta, not OverflowError - tick = offsets.Day(10**9) - msg = "Cannot cast 1000000000 days 00:00:00 to unit='ns' without overflow" - depr_msg = "Day.delta is deprecated" - with pytest.raises(OutOfBoundsTimedelta, match=msg): - with tm.assert_produces_warning(FutureWarning, match=depr_msg): - tick.delta - - @pytest.mark.parametrize("cls", tick_classes) def test_tick_division(cls): off = cls(10) diff --git a/pandas/tests/tslibs/test_parsing.py b/pandas/tests/tslibs/test_parsing.py index d1b0595dd50e6..52af5adb686a7 100644 --- a/pandas/tests/tslibs/test_parsing.py +++ b/pandas/tests/tslibs/test_parsing.py @@ -6,7 +6,6 @@ import re from dateutil.parser import parse as du_parse -from dateutil.tz import tzlocal from hypothesis import given import numpy as np import pytest @@ -22,6 +21,10 @@ ) import pandas.util._test_decorators as td +# Usually we wouldn't want this import in this test file (which is targeted at +# tslibs.parsing), but it is convenient to test the Timestamp constructor at +# the same time as the other parsing functions. +from pandas import Timestamp import pandas._testing as tm from pandas._testing._hypothesis import DATETIME_NO_TZ @@ -33,20 +36,21 @@ def test_parsing_tzlocal_deprecated(): # GH#50791 msg = ( - "Parsing 'EST' as tzlocal.*" + r"Parsing 'EST' as tzlocal \(dependent on system timezone\) " + r"is no longer supported\. " "Pass the 'tz' keyword or call tz_localize after construction instead" ) dtstr = "Jan 15 2004 03:00 EST" with tm.set_timezone("US/Eastern"): - with tm.assert_produces_warning(FutureWarning, match=msg): - res, _ = parse_datetime_string_with_reso(dtstr) + with pytest.raises(ValueError, match=msg): + parse_datetime_string_with_reso(dtstr) - assert isinstance(res.tzinfo, tzlocal) + with pytest.raises(ValueError, match=msg): + parsing.py_parse_datetime_string(dtstr) - with tm.assert_produces_warning(FutureWarning, match=msg): - res = parsing.py_parse_datetime_string(dtstr) - assert isinstance(res.tzinfo, tzlocal) + with pytest.raises(ValueError, match=msg): + Timestamp(dtstr) def test_parse_datetime_string_with_reso(): diff --git a/pandas/tests/tslibs/test_resolution.py b/pandas/tests/tslibs/test_resolution.py index c91e7bd6574ff..e9da6b3cf991c 100644 --- a/pandas/tests/tslibs/test_resolution.py +++ b/pandas/tests/tslibs/test_resolution.py @@ -48,10 +48,17 @@ def test_get_attrname_from_abbrev(freqstr, expected): assert reso.attrname == expected -@pytest.mark.parametrize("freq", ["H", "T", "S", "L", "U", "N"]) -def test_units_H_T_S_L_U_N_deprecated_from_attrname_to_abbrevs(freq): +@pytest.mark.parametrize("freq", ["H", "S"]) +def test_units_H_S_deprecated_from_attrname_to_abbrevs(freq): # GH#52536 msg = f"'{freq}' is deprecated and will be removed in a future version." with tm.assert_produces_warning(FutureWarning, match=msg): Resolution.get_reso_from_freqstr(freq) + + +@pytest.mark.parametrize("freq", ["T", "t", "L", "U", "N", "n"]) +def test_reso_abbrev_T_L_U_N_raises(freq): + msg = f"Frequency '{freq}' is no longer supported." + with pytest.raises(ValueError, match=msg): + Resolution.get_reso_from_freqstr(freq) diff --git a/pandas/tseries/holiday.py b/pandas/tseries/holiday.py index 50d0d33f0339f..cc9e2e3be8c38 100644 --- a/pandas/tseries/holiday.py +++ b/pandas/tseries/holiday.py @@ -108,7 +108,7 @@ def nearest_workday(dt: datetime) -> datetime: def next_workday(dt: datetime) -> datetime: """ - returns next weekday used for observances + returns next workday used for observances """ dt += timedelta(days=1) while dt.weekday() > 4: @@ -119,7 +119,7 @@ def next_workday(dt: datetime) -> datetime: def previous_workday(dt: datetime) -> datetime: """ - returns previous weekday used for observances + returns previous workday used for observances """ dt -= timedelta(days=1) while dt.weekday() > 4: @@ -130,7 +130,7 @@ def previous_workday(dt: datetime) -> datetime: def before_nearest_workday(dt: datetime) -> datetime: """ - returns previous workday after nearest workday + returns previous workday before nearest workday """ return previous_workday(nearest_workday(dt)) diff --git a/pandas/util/_decorators.py b/pandas/util/_decorators.py index 4f6fc0f3d8de3..d287fa72d552d 100644 --- a/pandas/util/_decorators.py +++ b/pandas/util/_decorators.py @@ -503,3 +503,24 @@ def indent(text: str | None, indents: int = 1) -> str: "future_version_msg", "Substitution", ] + + +def set_module(module): + """Private decorator for overriding __module__ on a function or class. + + Example usage:: + + @set_module("pandas") + def example(): + pass + + + assert example.__module__ == "pandas" + """ + + def decorator(func): + if module is not None: + func.__module__ = module + return func + + return decorator diff --git a/pyproject.toml b/pyproject.toml index e301ad304065c..ecd494523000c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,22 +76,22 @@ hdf5 = [# blosc only available on conda (https://github.com/Blosc/python-blosc/i #'blosc>=1.20.1', 'tables>=3.8.0'] spss = ['pyreadstat>=1.2.0'] -postgresql = ['SQLAlchemy>=2.0.0', 'psycopg2>=2.9.6', 'adbc-driver-postgresql>=0.8.0'] +postgresql = ['SQLAlchemy>=2.0.0', 'psycopg2>=2.9.6', 'adbc-driver-postgresql>=0.10.0'] mysql = ['SQLAlchemy>=2.0.0', 'pymysql>=1.0.2'] -sql-other = ['SQLAlchemy>=2.0.0', 'adbc-driver-postgresql>=0.8.0', 'adbc-driver-sqlite>=0.8.0'] +sql-other = ['SQLAlchemy>=2.0.0', 'adbc-driver-postgresql>=0.10.0', 'adbc-driver-sqlite>=0.8.0'] html = ['beautifulsoup4>=4.11.2', 'html5lib>=1.1', 'lxml>=4.9.2'] xml = ['lxml>=4.9.2'] plot = ['matplotlib>=3.6.3'] output-formatting = ['jinja2>=3.1.2', 'tabulate>=0.9.0'] clipboard = ['PyQt5>=5.15.9', 'qtpy>=2.3.0'] compression = ['zstandard>=0.19.0'] -all = ['adbc-driver-postgresql>=0.8.0', +all = ['adbc-driver-postgresql>=0.10.0', 'adbc-driver-sqlite>=0.8.0', 'beautifulsoup4>=4.11.2', # blosc only available on conda (https://github.com/Blosc/python-blosc/issues/297) #'blosc>=1.21.3', 'bottleneck>=1.3.6', - 'fastparquet>=2023.04.0', + 'fastparquet>=2023.10.0', 'fsspec>=2022.11.0', 'gcsfs>=2022.11.0', 'html5lib>=1.1', diff --git a/requirements-dev.txt b/requirements-dev.txt index 0cc064d2660bb..a42ee1587961a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -19,7 +19,7 @@ pytz beautifulsoup4>=4.11.2 blosc bottleneck>=1.3.6 -fastparquet>=2023.04.0 +fastparquet>=2023.10.0 fsspec>=2022.11.0 html5lib>=1.1 hypothesis>=6.46.1 @@ -84,7 +84,7 @@ feedparser pyyaml requests pygments -adbc-driver-postgresql>=0.8.0 +adbc-driver-postgresql>=0.10.0 adbc-driver-sqlite>=0.8.0 typing_extensions; python_version<"3.11" tzdata>=2022.7 diff --git a/scripts/run_stubtest.py b/scripts/run_stubtest.py index 6307afa1bc822..df88c61061f12 100644 --- a/scripts/run_stubtest.py +++ b/scripts/run_stubtest.py @@ -44,6 +44,7 @@ "pandas._libs.hashtable.HashTable.set_na", "pandas._libs.hashtable.HashTable.sizeof", "pandas._libs.hashtable.HashTable.unique", + "pandas._libs.hashtable.HashTable.hash_inner_join", # stubtest might be too sensitive "pandas._libs.lib.NoDefault", "pandas._libs.lib._NoDefault.no_default", diff --git a/web/pandas/config.yml b/web/pandas/config.yml index 05fdea13cab43..74e7fda2e7983 100644 --- a/web/pandas/config.yml +++ b/web/pandas/config.yml @@ -72,11 +72,9 @@ blog: - https://phofl.github.io/feeds/pandas.atom.xml maintainers: active: - - wesm - jorisvandenbossche - TomAugspurger - jreback - - gfyoung - WillAyd - mroeschke - jbrockmendel @@ -93,7 +91,6 @@ maintainers: - fangchenli - twoertwein - lithomas1 - - mzeitlin11 - lukemanley - noatamir inactive: @@ -108,6 +105,9 @@ maintainers: - jschendel - charlesdong1991 - dsaxton + - wesm + - gfyoung + - mzeitlin11 workgroups: coc: name: Code of Conduct @@ -121,13 +121,12 @@ workgroups: finance: name: Finance contact: finance@pandas.pydata.org - responsibilities: "Approve the project expenses." + responsibilities: "Manage the funding. Coordinate the request of grants. Approve the project expenses." members: - - Wes McKinney + - Matthew Roeschke - Jeff Reback - Joris Van den Bossche - - Tom Augspurger - - Matthew Roeschke + - Patrick Hoefler infrastructure: name: Infrastructure contact: infrastructure@pandas.pydata.org diff --git a/web/pandas/pdeps/0007-copy-on-write.md b/web/pandas/pdeps/0007-copy-on-write.md index e45fbaf555bc1..f5adb6a571120 100644 --- a/web/pandas/pdeps/0007-copy-on-write.md +++ b/web/pandas/pdeps/0007-copy-on-write.md @@ -1,7 +1,7 @@ # PDEP-7: Consistent copy/view semantics in pandas with Copy-on-Write - Created: July 2021 -- Status: Accepted +- Status: Implemented - Discussion: [#36195](https://github.com/pandas-dev/pandas/issues/36195) - Author: [Joris Van den Bossche](https://github.com/jorisvandenbossche) - Revision: 1