Merge branch 'main' into sas/shlookup

.github/workflows/codeql.yml

@@ -0,0 +1,31 @@
+name: CodeQL
+on:
+  schedule:
+    # every day at midnight
+    - cron: "0 0 * * *"
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language:
+          - python
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+      - uses: github/codeql-action/autobuild@v2
+      - uses: github/codeql-action/analyze@v2

.github/workflows/sdist.yml

@@ -79,9 +79,9 @@ jobs:
       run: |
         case "${{matrix.python-version}}" in
         3.8)
-          pip install numpy==1.19.5 ;;
+          pip install numpy==1.20.3 ;;
         3.9)
-          pip install numpy==1.19.5 ;;
+          pip install numpy==1.20.3 ;;
         3.10)
           pip install numpy==1.21.2 ;;
         esac

.pre-commit-config.yaml

@@ -85,32 +85,42 @@ repos:
 -   repo: local
     hooks:
     -   id: pyright
+        # note: assumes python env is setup and activated
         name: pyright
         entry: pyright
-        # note: assumes python env is setup and activated
         language: node
         pass_filenames: false
         types: [python]
         stages: [manual]
         additional_dependencies: &pyright_dependencies
-        - pyright@1.1.258
+        - pyright@1.1.264
     -   id: pyright_reportGeneralTypeIssues
+        # note: assumes python env is setup and activated
         name: pyright reportGeneralTypeIssues
         entry: pyright --skipunannotated -p pyright_reportGeneralTypeIssues.json
-        # note: assumes python env is setup and activated
         language: node
         pass_filenames: false
         types: [python]
         stages: [manual]
         additional_dependencies: *pyright_dependencies
     -   id: mypy
+        # note: assumes python env is setup and activated
         name: mypy
         entry: mypy
-        # note: assumes python env is setup and activated
         language: system
         pass_filenames: false
         types: [python]
         stages: [manual]
+    -   id: stubtest
+        # note: assumes python env is setup and activated
+        # note: requires pandas dev to be installed
+        name: mypy (stubtest)
+        entry: python
+        language: system
+        pass_filenames: false
+        types: [pyi]
+        args: [scripts/run_stubtest.py]
+        stages: [manual]
     -   id: flake8-rst
         name: flake8-rst
         description: Run flake8 on code snippets in docstrings or RST files
@@ -230,7 +240,7 @@ repos:
         language: python
         additional_dependencies:
         - flake8==4.0.1
-        - flake8-pyi==22.5.1
+        - flake8-pyi==22.7.0
     -   id: future-annotations
         name: import annotations from __future__
         entry: 'from __future__ import annotations'

ci/code_checks.sh

@@ -78,8 +78,8 @@ fi
 ### DOCSTRINGS ###
 if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
-    MSG='Validate docstrings (EX04, GL01, GL02, GL03, GL04, GL05, GL06, GL07, GL09, GL10, PR03, PR04, PR05, PR06, PR08, PR09, PR10, RT01, RT04, RT05, SA02, SA03, SA04, SS01, SS02, SS03, SS04, SS05)' ; echo $MSG
-    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX04,GL01,GL02,GL03,GL04,GL05,GL06,GL07,GL09,GL10,PR03,PR04,PR05,PR06,PR08,PR09,PR10,RT01,RT04,RT05,SA02,SA03,SA04,SS01,SS02,SS03,SS04,SS05
+    MSG='Validate docstrings (EX04, GL01, GL02, GL03, GL04, GL05, GL06, GL07, GL09, GL10, PR03, PR04, PR05, PR06, PR08, PR09, PR10, RT01, RT04, RT05, SA02, SA03, SA04, SS01, SS02, SS03, SS04, SS05, SS06)' ; echo $MSG
+    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX04,GL01,GL02,GL03,GL04,GL05,GL06,GL07,GL09,GL10,PR03,PR04,PR05,PR06,PR08,PR09,PR10,RT01,RT04,RT05,SA02,SA03,SA04,SS01,SS02,SS03,SS04,SS05,SS06
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

ci/deps/actions-310-numpydev.yaml

@@ -16,8 +16,7 @@ dependencies:
   - pytz
   - pip
   - pip:
-    #- cython # TODO: don't install from master after Cython 3.0.0a11 is released
-    - "git+https://github.com/cython/cython.git@master"
+    - "cython"
     - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple"
     - "--pre"
     - "numpy"

ci/deps/actions-310.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.10
 
   # test dependencies
-  - cython=0.29.30
+  - cython>=0.29.30
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-38-downstream_compat.yaml

@@ -6,7 +6,7 @@ dependencies:
   - python=3.8
 
   # test dependencies
-  - cython=0.29.30
+  - cython>=0.29.30
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-38-minimum_versions.yaml

@@ -7,7 +7,7 @@ dependencies:
   - python=3.8.0
 
   # test dependencies
-  - cython=0.29.30
+  - cython>=0.29.30
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
@@ -17,7 +17,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil=2.8.1
-  - numpy=1.19.5
+  - numpy=1.20.3
   - pytz=2020.1
 
   # optional dependencies

ci/deps/actions-38.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.8
 
   # test dependencies
-  - cython=0.29.30
+  - cython>=0.29.30
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-39.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.9
 
   # test dependencies
-  - cython=0.29.30
+  - cython>=0.29.30
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/circle-38-arm64.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.8
 
   # test dependencies
-  - cython=0.29.30
+  - cython>=0.29.30
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

doc/source/conf.py

@@ -50,23 +50,25 @@
 # sphinxext.
 
 extensions = [
-    "sphinx.ext.autodoc",
-    "sphinx.ext.autosummary",
-    "sphinx.ext.doctest",
-    "sphinx.ext.extlinks",
-    "sphinx.ext.todo",
-    "numpydoc",  # handle NumPy documentation formatted docstrings
+    "contributors",  # custom pandas extension
     "IPython.sphinxext.ipython_directive",
     "IPython.sphinxext.ipython_console_highlighting",
     "matplotlib.sphinxext.plot_directive",
-    "sphinx.ext.intersphinx",
+    "numpydoc",
+    "sphinx_copybutton",
+    "sphinx_panels",
+    "sphinx_toggleprompt",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
     "sphinx.ext.coverage",
-    "sphinx.ext.mathjax",
+    "sphinx.ext.doctest",
+    "sphinx.ext.extlinks",
     "sphinx.ext.ifconfig",
+    "sphinx.ext.intersphinx",
     "sphinx.ext.linkcode",
+    "sphinx.ext.mathjax",
+    "sphinx.ext.todo",
     "nbsphinx",
-    "sphinx_panels",
-    "contributors",  # custom pandas extension
 ]
 
 exclude_patterns = [
@@ -144,6 +146,9 @@
 # already loads it
 panels_add_bootstrap_css = False
 
+# https://sphinx-toggleprompt.readthedocs.io/en/stable/#offset
+toggleprompt_offset_right = 35
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["../_templates"]
 
@@ -453,7 +458,6 @@
 # extlinks alias
 extlinks = {
     "issue": ("https://github.com/pandas-dev/pandas/issues/%s", "GH"),
-    "wiki": ("https://github.com/pandas-dev/pandas/wiki/%s", "wiki "),
 }
 
 

doc/source/ecosystem.rst

@@ -161,10 +161,10 @@ A good implementation for Python users is `has2k1/plotnine <https://github.com/h
 `IPython Vega <https://github.com/vega/ipyvega>`__ leverages `Vega
 <https://github.com/vega/vega>`__ to create plots within Jupyter Notebook.
 
-`Plotly <https://poltly.com/python>`__
+`Plotly <https://plotly.com/python>`__
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-`Plotly’s <https://poltly.com/>`__ `Python API <https://poltly.com/python/>`__ enables interactive figures and web shareability. Maps, 2D, 3D, and live-streaming graphs are rendered with WebGL and `D3.js <https://d3js.org/>`__. The library supports plotting directly from a pandas DataFrame and cloud-based collaboration. Users of `matplotlib, ggplot for Python, and Seaborn <https://poltly.com/python/matplotlib-to-plotly-tutorial/>`__ can convert figures into interactive web-based plots. Plots can be drawn in `IPython Notebooks <https://plotly.com/ipython-notebooks/>`__ , edited with R or MATLAB, modified in a GUI, or embedded in apps and dashboards. Plotly is free for unlimited sharing, and has `offline <https://poltly.com/python/offline/>`__, or `on-premise <https://poltly.com/product/enterprise/>`__ accounts for private use.
+`Plotly’s <https://plotly.com/>`__ `Python API <https://plotly.com/python/>`__ enables interactive figures and web shareability. Maps, 2D, 3D, and live-streaming graphs are rendered with WebGL and `D3.js <https://d3js.org/>`__. The library supports plotting directly from a pandas DataFrame and cloud-based collaboration. Users of `matplotlib, ggplot for Python, and Seaborn <https://plotly.com/python/matplotlib-to-plotly-tutorial/>`__ can convert figures into interactive web-based plots. Plots can be drawn in `IPython Notebooks <https://plotly.com/ipython-notebooks/>`__ , edited with R or MATLAB, modified in a GUI, or embedded in apps and dashboards. Plotly is free for unlimited sharing, and has `offline <https://plotly.com/python/offline/>`__, or `on-premise <https://plotly.com/product/enterprise/>`__ accounts for private use.
 
 `Lux <https://github.com/lux-org/lux>`__
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -591,12 +591,12 @@ Library            Accessor     Classes                              Description
 Development tools
 -----------------
 
-`pandas-stubs <https://github.com/VirtusLab/pandas-stubs>`__
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+`pandas-stubs <https://github.com/pandas-dev/pandas-stubs>`__
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 While pandas repository is partially typed, the package itself doesn't expose this information for external use.
 Install pandas-stubs to enable basic type coverage of pandas API.
 
 Learn more by reading through :issue:`14468`, :issue:`26766`, :issue:`28142`.
 
-See installation and usage instructions on the `github page <https://github.com/VirtusLab/pandas-stubs>`__.
+See installation and usage instructions on the `github page <https://github.com/pandas-dev/pandas-stubs>`__.

doc/source/getting_started/install.rst

@@ -235,7 +235,7 @@ Dependencies
 ================================================================ ==========================
 Package                                                          Minimum supported version
 ================================================================ ==========================
-`NumPy <https://numpy.org>`__                                    1.19.5
+`NumPy <https://numpy.org>`__                                    1.20.3
 `python-dateutil <https://dateutil.readthedocs.io/en/stable/>`__ 2.8.1
 `pytz <https://pypi.org/project/pytz/>`__                        2020.1
 ================================================================ ==========================

doc/source/getting_started/intro_tutorials/10_text_data.rst

@@ -179,7 +179,7 @@ applied to integers, so no ``str`` is used.
 
 Based on the index name of the row (``307``) and the column (``Name``),
 we can do a selection using the ``loc`` operator, introduced in the
-`tutorial on subsetting <3_subset_data.ipynb>`__.
+:ref:`tutorial on subsetting <10min_tut_03_subset>`.
 
 .. raw:: html
 

doc/source/reference/general_functions.rst

@@ -85,4 +85,4 @@ Importing from other DataFrame libraries
 .. autosummary::
    :toctree: api/
 
-   api.exchange.from_dataframe
+   api.interchange.from_dataframe

doc/source/reference/style.rst

@@ -41,6 +41,7 @@ Style application
    Styler.applymap_index
    Styler.format
    Styler.format_index
+   Styler.relabel_index
    Styler.hide
    Styler.concat
    Styler.set_td_classes

doc/source/user_guide/io.rst

@@ -107,9 +107,10 @@ index_col : int, str, sequence of int / str, or False, optional, default ``None`
   string name or column index. If a sequence of int / str is given, a
   MultiIndex is used.
 
-  Note: ``index_col=False`` can be used to force pandas to *not* use the first
-  column as the index, e.g. when you have a malformed file with delimiters at
-  the end of each line.
+  .. note::
+     ``index_col=False`` can be used to force pandas to *not* use the first
+     column as the index, e.g. when you have a malformed file with delimiters at
+     the end of each line.
 
   The default value of ``None`` instructs pandas to guess. If the number of
   fields in the column header row is equal to the number of fields in the body
@@ -182,15 +183,16 @@ General parsing configuration
 +++++++++++++++++++++++++++++
 
 dtype : Type name or dict of column -> type, default ``None``
-  Data type for data or columns. E.g. ``{'a': np.float64, 'b': np.int32}``
-  (unsupported with ``engine='python'``). Use ``str`` or ``object`` together
-  with suitable ``na_values`` settings to preserve and
-  not interpret dtype.
+  Data type for data or columns. E.g. ``{'a': np.float64, 'b': np.int32, 'c': 'Int64'}``
+  Use ``str`` or ``object`` together with suitable ``na_values`` settings to preserve
+  and not interpret dtype. If converters are specified, they will be applied INSTEAD
+  of dtype conversion.
+
   .. versionadded:: 1.5.0
 
-    Support for defaultdict was added. Specify a defaultdict as input where
-    the default determines the dtype of the columns which are not explicitly
-    listed.
+     Support for defaultdict was added. Specify a defaultdict as input where
+     the default determines the dtype of the columns which are not explicitly
+     listed.
 engine : {``'c'``, ``'python'``, ``'pyarrow'``}
   Parser engine to use. The C and pyarrow engines are faster, while the python engine
   is currently more feature-complete. Multithreading is currently only supported by
@@ -283,7 +285,9 @@ parse_dates : boolean or list of ints or names or list of lists or dict, default
   * If ``[[1, 3]]`` -> combine columns 1 and 3 and parse as a single date
     column.
   * If ``{'foo': [1, 3]}`` -> parse columns 1, 3 as date and call result 'foo'.
-    A fast-path exists for iso8601-formatted dates.
+
+  .. note::
+     A fast-path exists for iso8601-formatted dates.
 infer_datetime_format : boolean, default ``False``
   If ``True`` and parse_dates is enabled for a column, attempt to infer the
   datetime format to speed up the processing.
@@ -1593,8 +1597,10 @@ of multi-columns indices.
 
    pd.read_csv("mi2.csv", header=[0, 1], index_col=0)
 
-Note: If an ``index_col`` is not specified (e.g. you don't have an index, or wrote it
-with ``df.to_csv(..., index=False)``, then any ``names`` on the columns index will be *lost*.
+.. note::
+   If an ``index_col`` is not specified (e.g. you don't have an index, or wrote it
+   with ``df.to_csv(..., index=False)``, then any ``names`` on the columns index will
+   be *lost*.
 
 .. ipython:: python
    :suppress:
@@ -3048,15 +3054,15 @@ Read in the content of the "books.xml" as instance of ``StringIO`` or
    df = pd.read_xml(bio)
    df
 
-Even read XML from AWS S3 buckets such as Python Software Foundation's IRS 990 Form:
+Even read XML from AWS S3 buckets such as NIH NCBI PMC Article Datasets providing
+Biomedical and Life Science Jorurnals:
 
 .. ipython:: python
    :okwarning:
 
    df = pd.read_xml(
-       "s3://irs-form-990/201923199349319487_public.xml",
-       xpath=".//irs:Form990PartVIISectionAGrp",
-       namespaces={"irs": "http://www.irs.gov/efile"}
+       "s3://pmc-oa-opendata/oa_comm/xml/all/PMC1236943.xml",
+       xpath=".//journal-meta",
    )
    df
 

doc/source/user_guide/sparse.rst

@@ -266,8 +266,8 @@ have no replacement.
 
 .. _sparse.scipysparse:
 
-Interaction with scipy.sparse
------------------------------
+Interaction with *scipy.sparse*
+-------------------------------
 
 Use :meth:`DataFrame.sparse.from_spmatrix` to create a :class:`DataFrame` with sparse values from a sparse matrix.