Merge remote-tracking branch 'upstream/master' into deprecate-week-of…

…-year
pandas-dev · May 13, 2020 · b81536b · b81536b
2 parents 762dc81 + 5724728
commit b81536b
Show file tree

Hide file tree

Showing 328 changed files with 7,494 additions and 4,619 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -14,6 +14,8 @@ cache:
 
 env:
   global:
+    # Variable for test workers
+    - PYTEST_WORKERS="auto"
     # create a github personal access token
     # cd pandas-dev/pandas
     # travis encrypt 'PANDAS_GH_TOKEN=personal_access_token' -r pandas-dev/pandas
@@ -27,12 +29,21 @@ matrix:
   fast_finish: true
 
   include:
+    # In allowed failures
+    - dist: bionic
+      python: 3.9-dev
+      env:
+        - JOB="3.9-dev" PATTERN="(not slow and not network and not clipboard)"
     - env:
         - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network and not clipboard)"
 
     - env:
         - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network and not clipboard)"
 
+    - arch: arm64
+      env:
+        - JOB="3.7, arm64" PYTEST_WORKERS=8 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)"
+
     - env:
         - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
       services:
@@ -53,6 +64,14 @@ matrix:
       services:
         - mysql
         - postgresql
+  allow_failures:
+    - arch: arm64
+      env:
+        - JOB="3.7, arm64" PYTEST_WORKERS=8 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)"
+    - dist: bionic
+      python: 3.9-dev
+      env:
+        - JOB="3.9-dev" PATTERN="(not slow and not network)"
 
 before_install:
   - echo "before_install"
@@ -83,7 +102,7 @@ install:
 script:
   - echo "script start"
   - echo "$JOB"
-  - source activate pandas-dev
+  - if [ "$JOB" != "3.9-dev" ]; then source activate pandas-dev; fi
   - ci/run_tests.sh
 
 after_script:

diff --git a/asv_bench/benchmarks/algorithms.py b/asv_bench/benchmarks/algorithms.py
@@ -34,7 +34,16 @@ class Factorize:
     params = [
         [True, False],
         [True, False],
-        ["int", "uint", "float", "string", "datetime64[ns]", "datetime64[ns, tz]"],
+        [
+            "int",
+            "uint",
+            "float",
+            "string",
+            "datetime64[ns]",
+            "datetime64[ns, tz]",
+            "Int64",
+            "boolean",
+        ],
     ]
     param_names = ["unique", "sort", "dtype"]
 
@@ -49,13 +58,15 @@ def setup(self, unique, sort, dtype):
             "datetime64[ns, tz]": pd.date_range(
                 "2011-01-01", freq="H", periods=N, tz="Asia/Tokyo"
             ),
+            "Int64": pd.array(np.arange(N), dtype="Int64"),
+            "boolean": pd.array(np.random.randint(0, 2, N), dtype="boolean"),
         }[dtype]
         if not unique:
             data = data.repeat(5)
-        self.idx = data
+        self.data = data
 
     def time_factorize(self, unique, sort, dtype):
-        self.idx.factorize(sort=sort)
+        pd.factorize(self.data, sort=sort)
 
 
 class Duplicated:

diff --git a/asv_bench/benchmarks/io/parsers.py b/asv_bench/benchmarks/io/parsers.py
@@ -2,7 +2,7 @@
 
 try:
     from pandas._libs.tslibs.parsing import (
-        _concat_date_cols,
+        concat_date_cols,
         _does_string_look_like_datetime,
     )
 except ImportError:
@@ -39,4 +39,4 @@ def setup(self, value, dim):
             )
 
     def time_check_concat(self, value, dim):
-        _concat_date_cols(self.object)
+        concat_date_cols(self.object)
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -5,6 +5,9 @@ trigger:
 pr:
 - master
 
+variables:
+  PYTEST_WORKERS: auto
+
 jobs:
 # Mac and Linux use the same template
 - template: ci/azure/posix.yml

diff --git a/ci/build39.sh b/ci/build39.sh
@@ -0,0 +1,21 @@
+#!/bin/bash -e
+# Special build for python3.9 until numpy puts its own wheels up
+
+sudo apt-get install build-essential gcc xvfb
+pip install --no-deps -U pip wheel setuptools
+pip install python-dateutil pytz pytest pytest-xdist hypothesis
+pip install cython --pre # https://github.com/cython/cython/issues/3395
+
+git clone https://github.com/numpy/numpy
+cd numpy
+python setup.py build_ext --inplace
+python setup.py install
+cd ..
+rm -rf numpy
+
+python setup.py build_ext -inplace
+python -m pip install --no-build-isolation -e .
+
+python -c "import sys; print(sys.version_info)"
+python -c "import pandas as pd"
+python -c "import hypothesis"
diff --git a/ci/code_checks.sh b/ci/code_checks.sh
@@ -353,8 +353,8 @@ fi
 ### DOCSTRINGS ###
 if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
-    MSG='Validate docstrings (GL03, GL04, GL05, GL06, GL07, GL09, GL10, SS04, SS05, PR03, PR04, PR05, PR10, EX04, RT01, RT04, RT05, SA02, SA03, SA05)' ; echo $MSG
-    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=GL03,GL04,GL05,GL06,GL07,GL09,GL10,SS04,SS05,PR03,PR04,PR05,PR10,EX04,RT01,RT04,RT05,SA02,SA03,SA05
+    MSG='Validate docstrings (GL03, GL04, GL05, GL06, GL07, GL09, GL10, SS04, SS05, PR03, PR04, PR05, PR10, EX04, RT01, RT04, RT05, SA02, SA03)' ; echo $MSG
+    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=GL03,GL04,GL05,GL06,GL07,GL09,GL10,SS04,SS05,PR03,PR04,PR05,PR10,EX04,RT01,RT04,RT05,SA02,SA03
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Validate correct capitalization among titles in documentation' ; echo $MSG

diff --git a/ci/deps/azure-36-minimum_versions.yaml b/ci/deps/azure-36-minimum_versions.yaml
@@ -1,6 +1,5 @@
 name: pandas-dev
 channels:
-  - defaults
   - conda-forge
 dependencies:
   - python=3.6.1
@@ -19,12 +18,12 @@ dependencies:
   - jinja2=2.8
   - numba=0.46.0
   - numexpr=2.6.2
-  - numpy=1.13.3
+  - numpy=1.15.4
   - openpyxl=2.5.7
   - pytables=3.4.3
   - python-dateutil=2.7.3
   - pytz=2017.2
-  - scipy=0.19.0
+  - scipy=1.2
   - xlrd=1.1.0
   - xlsxwriter=0.9.8
   - xlwt=1.2.0

diff --git a/ci/deps/azure-37-numpydev.yaml b/ci/deps/azure-37-numpydev.yaml
@@ -14,9 +14,9 @@ dependencies:
   - pytz
   - pip
   - pip:
-    - cython>=0.29.16
+    - cython==0.29.16 # GH#34014
     - "git+git://github.com/dateutil/dateutil.git"
-    - "-f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"
+    - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple"
     - "--pre"
     - "numpy"
     - "scipy"
diff --git a/ci/deps/azure-macos-36.yaml b/ci/deps/azure-macos-36.yaml
@@ -19,7 +19,7 @@ dependencies:
   - matplotlib=2.2.3
   - nomkl
   - numexpr
-  - numpy=1.14
+  - numpy=1.15.4
   - openpyxl
   - pyarrow>=0.13.0
   - pytables

diff --git a/ci/deps/travis-37-arm64.yaml b/ci/deps/travis-37-arm64.yaml
@@ -0,0 +1,21 @@
+name: pandas-dev
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - python=3.7.*
+
+  # tools
+  - cython>=0.29.13
+  - pytest>=5.0.1
+  - pytest-xdist>=1.21
+  - hypothesis>=3.58.0
+
+  # pandas dependencies
+  - botocore>=1.11
+  - numpy
+  - python-dateutil
+  - pytz
+  - pip
+  - pip:
+    - moto
diff --git a/ci/run_tests.sh b/ci/run_tests.sh
@@ -20,7 +20,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then
     XVFB="xvfb-run "
 fi
 
-PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"
+PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile -s --strict --durations=30 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"
 
 echo $PYTEST_CMD
 sh -c "$PYTEST_CMD"

diff --git a/ci/setup_env.sh b/ci/setup_env.sh
@@ -1,5 +1,10 @@
 #!/bin/bash -e
 
+if [ "$JOB" == "3.9-dev" ]; then
+    /bin/bash ci/build39.sh
+    exit 0
+fi
+
 # edit the locale file if needed
 if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then
     echo "Adding locale to the first line of pandas/__init__.py"
@@ -36,9 +41,17 @@ else
   exit 1
 fi
 
-wget -q "https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh" -O miniconda.sh
+if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then
+  sudo apt-get -y install xvfb
+  CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.2-1/Miniforge3-4.8.2-1-Linux-aarch64.sh"
+else
+  CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh"
+fi
+wget -q $CONDA_URL -O miniconda.sh
 chmod +x miniconda.sh
-./miniconda.sh -b
+
+# Installation path is required for ARM64 platform as miniforge script installs in path $HOME/miniforge3.
+./miniconda.sh -b -p $MINICONDA_DIR
 
 export PATH=$MINICONDA_DIR/bin:$PATH
 

diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml
@@ -20,12 +20,12 @@ requirements:
     - cython
     - numpy
     - setuptools >=3.3
-    - python-dateutil >=2.5.0
+    - python-dateutil >=2.7.3
     - pytz
   run:
     - python {{ python }}
     - {{ pin_compatible('numpy') }}
-    - python-dateutil >=2.5.0
+    - python-dateutil >=2.7.3
     - pytz
 
 test:

diff --git a/doc/source/conf.py b/doc/source/conf.py
@@ -410,7 +410,7 @@
     intersphinx_mapping = {
         "dateutil": ("https://dateutil.readthedocs.io/en/latest/", None),
         "matplotlib": ("https://matplotlib.org/", None),
-        "numpy": ("https://docs.scipy.org/doc/numpy/", None),
+        "numpy": ("https://numpy.org/doc/stable/", None),
         "pandas-gbq": ("https://pandas-gbq.readthedocs.io/en/latest/", None),
         "py": ("https://pylib.readthedocs.io/en/latest/", None),
         "python": ("https://docs.python.org/3/", None),

diff --git a/doc/source/development/contributing.rst b/doc/source/development/contributing.rst
@@ -110,7 +110,7 @@ version control to allow many people to work together on the project.
 Some great resources for learning Git:
 
 * the `GitHub help pages <https://help.github.com/>`_.
-* the `NumPy's documentation <https://docs.scipy.org/doc/numpy/dev/index.html>`_.
+* the `NumPy's documentation <https://numpy.org/doc/stable/dev/index.html>`_.
 * Matthew Brett's `Pydagogue <https://matthew-brett.github.com/pydagogue/>`_.
 
 Getting started with Git
@@ -974,7 +974,7 @@ it is worth getting in the habit of writing tests ahead of time so this is never
 Like many packages, pandas uses `pytest
 <https://docs.pytest.org/en/latest/>`_ and the convenient
 extensions in `numpy.testing
-<https://docs.scipy.org/doc/numpy/reference/routines.testing.html>`_.
+<https://numpy.org/doc/stable/reference/routines.testing.html>`_.
 
 .. note::
 

diff --git a/doc/source/development/extending.rst b/doc/source/development/extending.rst
@@ -219,7 +219,7 @@ and re-boxes it if necessary.
 
 If applicable, we highly recommend that you implement ``__array_ufunc__`` in your
 extension array to avoid coercion to an ndarray. See
-`the numpy documentation <https://docs.scipy.org/doc/numpy/reference/generated/numpy.lib.mixins.NDArrayOperatorsMixin.html>`__
+`the numpy documentation <https://numpy.org/doc/stable/reference/generated/numpy.lib.mixins.NDArrayOperatorsMixin.html>`__
 for an example.
 
 As part of your implementation, we require that you defer to pandas when a pandas

diff --git a/doc/source/getting_started/install.rst b/doc/source/getting_started/install.rst
@@ -220,7 +220,7 @@ Dependencies
 Package                                                          Minimum supported version
 ================================================================ ==========================
 `setuptools <https://setuptools.readthedocs.io/en/latest/>`__    24.2.0
-`NumPy <https://www.numpy.org>`__                                1.13.3
+`NumPy <https://www.numpy.org>`__                                1.15.4
 `python-dateutil <https://dateutil.readthedocs.io/en/stable/>`__ 2.7.3
 `pytz <https://pypi.org/project/pytz/>`__                        2017.2
 ================================================================ ==========================

diff --git a/doc/source/getting_started/intro_tutorials/02_read_write.rst b/doc/source/getting_started/intro_tutorials/02_read_write.rst
@@ -23,7 +23,7 @@
                     <div class="card-body">
                         <p class="card-text">
 
-This tutorial uses the titanic data set, stored as CSV. The data
+This tutorial uses the Titanic data set, stored as CSV. The data
 consists of the following data columns:
 
 -  PassengerId: Id of every passenger.
@@ -61,7 +61,7 @@ How do I read and write tabular data?
     <ul class="task-bullet">
         <li>
 
-I want to analyse the titanic passenger data, available as a CSV file.
+I want to analyze the Titanic passenger data, available as a CSV file.
 
 .. ipython:: python
 
@@ -134,7 +134,7 @@ strings (``object``).
     <ul class="task-bullet">
         <li>
 
-My colleague requested the titanic data as a spreadsheet.
+My colleague requested the Titanic data as a spreadsheet.
 
 .. ipython:: python
 

diff --git a/doc/source/getting_started/intro_tutorials/03_subset_data.rst b/doc/source/getting_started/intro_tutorials/03_subset_data.rst
@@ -330,7 +330,7 @@ When using the column names, row labels or a condition expression, use
 the ``loc`` operator in front of the selection brackets ``[]``. For both
 the part before and after the comma, you can use a single label, a list
 of labels, a slice of labels, a conditional expression or a colon. Using
-a colon specificies you want to select all rows or columns.
+a colon specifies you want to select all rows or columns.
 
 .. raw:: html
 

diff --git a/doc/source/getting_started/intro_tutorials/06_calculate_statistics.rst b/doc/source/getting_started/intro_tutorials/06_calculate_statistics.rst
@@ -23,7 +23,7 @@
                     <div class="card-body">
                         <p class="card-text">
 
-This tutorial uses the titanic data set, stored as CSV. The data
+This tutorial uses the Titanic data set, stored as CSV. The data
 consists of the following data columns:
 
 -  PassengerId: Id of every passenger.
@@ -72,7 +72,7 @@ Aggregating statistics
     <ul class="task-bullet">
         <li>
 
-What is the average age of the titanic passengers?
+What is the average age of the Titanic passengers?
 
 .. ipython:: python
 
@@ -95,7 +95,7 @@ across rows by default.
     <ul class="task-bullet">
         <li>
 
-What is the median age and ticket fare price of the titanic passengers?
+What is the median age and ticket fare price of the Titanic passengers?
 
 .. ipython:: python
 
@@ -148,7 +148,7 @@ Aggregating statistics grouped by category
     <ul class="task-bullet">
         <li>
 
-What is the average age for male versus female titanic passengers?
+What is the average age for male versus female Titanic passengers?
 
 .. ipython:: python