[CI]: Use conda-env for CI environments (#20767)

.travis.yml

@@ -34,55 +34,54 @@ matrix:
     - os: osx
       language: generic
       env:
-        - JOB="3.5_OSX" TEST_ARGS="--skip-slow --skip-network"
+        - JOB="3.5, OSX" ENV_FILE="ci/travis-35-osx.yaml" TEST_ARGS="--skip-slow --skip-network"
     - dist: trusty
       env:
-        - JOB="2.7_LOCALE" LOCALE_OVERRIDE="zh_CN.UTF-8" SLOW=true
+        - JOB="2.7, locale, slow, old NumPy" ENV_FILE="ci/travis-27-locale.yaml" LOCALE_OVERRIDE="zh_CN.UTF-8" SLOW=true
       addons:
         apt:
           packages:
           - language-pack-zh-hans
     - dist: trusty
       env:
-        - JOB="2.7" TEST_ARGS="--skip-slow" LINT=true
+        - JOB="2.7, lint" ENV_FILE="ci/travis-27.yaml" TEST_ARGS="--skip-slow" LINT=true
       addons:
         apt:
           packages:
           - python-gtk2
-    # In allow_failures
     - dist: trusty
       env:
-        - JOB="3.6" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" CONDA_FORGE=true COVERAGE=true
+        - JOB="3.6, coverage" ENV_FILE="ci/travis-36.yaml" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true
     # In allow_failures
     - dist: trusty
       env:
-        - JOB="3.6_SLOW" SLOW=true
+        - JOB="3.6, slow" ENV_FILE="ci/travis-36-slow.yaml" SLOW=true
     # In allow_failures
     - dist: trusty
       env:
-        - JOB="3.6_NUMPY_DEV" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate"
+        - JOB="3.6, NumPy dev" ENV_FILE="ci/travis-36-numpydev.yaml" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate"
       addons:
         apt:
           packages:
           - xsel
     # In allow_failures
     - dist: trusty
       env:
-        - JOB="3.6_DOC" DOC=true
+        - JOB="3.6, doc" ENV_FILE="ci/travis-36-doc.yaml" DOC=true
     allow_failures:
       - dist: trusty
         env:
-          - JOB="3.6_SLOW" SLOW=true
+          - JOB="3.6, slow" ENV_FILE="ci/travis-36-slow.yaml" SLOW=true
       - dist: trusty
         env:
-          - JOB="3.6_NUMPY_DEV" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate"
+          - JOB="3.6, NumPy dev" ENV_FILE="ci/travis-36-numpydev.yaml" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate"
         addons:
           apt:
             packages:
             - xsel
       - dist: trusty
         env:
-          - JOB="3.6_DOC" DOC=true
+          - JOB="3.6, doc" ENV_FILE="ci/travis-36-doc.yaml" DOC=true
 
 before_install:
   - echo "before_install"

appveyor.yml

@@ -73,19 +73,12 @@ install:
   - cmd: conda info -a
 
   # create our env
-  - cmd: conda create -n pandas python=%PYTHON_VERSION% cython pytest>=3.1.0 pytest-xdist
+  - cmd: conda env create -q -n pandas --file=ci\appveyor-%CONDA_PY%.yaml
   - cmd: activate pandas
-  - cmd: pip install moto
-  - SET REQ=ci\requirements-%PYTHON_VERSION%_WIN.run
-  - cmd: echo "installing requirements from %REQ%"
-  - cmd: conda install -n pandas --file=%REQ%
   - cmd: conda list -n pandas
-  - cmd: echo "installing requirements from %REQ% - done"
-
-  # add some pip only reqs to the env
-  - SET REQ=ci\requirements-%PYTHON_VERSION%_WIN.pip
-  - cmd: echo "installing requirements from %REQ%"
-  - cmd: pip install -Ur %REQ%
+  # uninstall pandas if it's present
+  - cmd: conda remove pandas -y --force & exit 0
+  - cmd: pip uninstall -y pandas & exit 0
 
   # build em using the local source checkout in the correct windows env
   - cmd: '%CMD_IN_ENV% python setup.py build_ext --inplace'

ci/appveyor-27.yaml

@@ -0,0 +1,29 @@
+name: pandas
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - beautifulsoup4
+  - bottleneck
+  - dateutil
+  - html5lib
+  - jinja2=2.8
+  - lxml
+  - matplotlib
+  - numexpr
+  - numpy=1.10*
+  - openpyxl
+  - pytables==3.2.2
+  - python=2.7.*
+  - pytz
+  - s3fs
+  - scipy
+  - sqlalchemy
+  - xlrd
+  - xlsxwriter
+  - xlwt
+  # universal
+  - cython
+  - pytest
+  - pytest-xdist
+  - moto

ci/appveyor-36.yaml

@@ -0,0 +1,27 @@
+name: pandas
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - blosc
+  - bottleneck
+  - fastparquet
+  - feather-format
+  - matplotlib
+  - numexpr
+  - numpy=1.13*
+  - openpyxl
+  - pyarrow
+  - pytables
+  - python-dateutil
+  - python=3.6.*
+  - pytz
+  - scipy
+  - thrift=0.10*
+  - xlrd
+  - xlsxwriter
+  - xlwt
+  # universal
+  - cython
+  - pytest
+  - pytest-xdist

ci/circle-27-compat.yaml

@@ -0,0 +1,28 @@
+name: pandas
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - bottleneck=1.0.0
+  - cython=0.24
+  - jinja2=2.8
+  - numexpr=2.4.4 # we test that we correctly don't use an unsupported numexpr
+  - numpy=1.9.2
+  - openpyxl
+  - psycopg2
+  - pytables=3.2.2
+  - python-dateutil=2.5.0
+  - python=2.7*
+  - pytz=2013b
+  - scipy=0.14.0
+  - sqlalchemy=0.7.8
+  - xlrd=0.9.2
+  - xlsxwriter=0.5.2
+  - xlwt=0.7.5
+  # universal
+  - pytest
+  - pytest-xdist
+  - pip:
+    - html5lib==1.0b2
+    - beautifulsoup4==4.2.1
+    - pymysql==0.6.0

ci/circle-35-ascii.yaml

@@ -0,0 +1,13 @@
+name: pandas
+channels:
+  - defaults
+dependencies:
+  - cython
+  - nomkl
+  - numpy
+  - python-dateutil
+  - python=3.5*
+  - pytz
+  # universal
+  - pytest
+  - pytest-xdist

ci/circle-36-locale.yaml

@@ -0,0 +1,33 @@
+name: pandas
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - beautifulsoup4
+  - cython
+  - html5lib
+  - ipython
+  - jinja2
+  - lxml
+  - matplotlib
+  - nomkl
+  - numexpr
+  - numpy
+  - openpyxl
+  - psycopg2
+  - pymysql
+  - pytables
+  - python-dateutil
+  - python=3.6*
+  - pytz
+  - s3fs
+  - scipy
+  - sqlalchemy
+  - xarray
+  - xlrd
+  - xlsxwriter
+  - xlwt
+  # universal
+  - pytest
+  - pytest-xdist
+  - moto

ci/circle-36-locale_slow.yaml

@@ -0,0 +1,33 @@
+name: pandas
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - beautifulsoup4
+  - cython
+  - html5lib
+  - ipython
+  - jinja2
+  - lxml
+  - matplotlib
+  - nomkl
+  - numexpr
+  - numpy
+  - openpyxl
+  - psycopg2
+  - pymysql
+  - pytables
+  - python-dateutil
+  - python=3.6*
+  - pytz
+  - s3fs
+  - scipy
+  - sqlalchemy
+  - xarray
+  - xlrd
+  - xlsxwriter
+  - xlwt
+  # universal
+  - pytest
+  - pytest-xdist
+  - moto

ci/install_circle.sh

@@ -46,10 +46,6 @@ echo "[environmental variable file]"
 cat $ENVS_FILE
 source $ENVS_FILE
 
-export REQ_BUILD=ci/requirements-${JOB}.build
-export REQ_RUN=ci/requirements-${JOB}.run
-export REQ_PIP=ci/requirements-${JOB}.pip
-
 # edit the locale override if needed
 if [ -n "$LOCALE_OVERRIDE" ]; then
     echo "[Adding locale to the first line of pandas/__init__.py]"
@@ -62,25 +58,23 @@ if [ -n "$LOCALE_OVERRIDE" ]; then
 fi
 
 # create envbuild deps
-echo "[create env: ${REQ_BUILD}]"
-time conda create -n pandas -q --file=${REQ_BUILD} || exit 1
-time conda install -n pandas pytest>=3.1.0 || exit 1
+echo "[create env]"
+time conda env create -q -n pandas --file="${ENV_FILE}" || exit 1
 
 source activate pandas
-time pip install moto || exit 1
+
+# remove any installed pandas package
+# w/o removing anything else
+echo
+echo "[removing installed pandas]"
+conda remove pandas -y --force
+pip uninstall -y pandas
 
 # build but don't install
 echo "[build em]"
 time python setup.py build_ext --inplace || exit 1
 
-# we may have run installations
-echo "[conda installs: ${REQ_RUN}]"
-if [ -e ${REQ_RUN} ]; then
-    time conda install -q --file=${REQ_RUN} || exit 1
-fi
+echo
+echo "[show environment]"
 
-# we may have additional pip installs
-echo "[pip installs: ${REQ_PIP}]"
-if [ -e ${REQ_PIP} ]; then
-   pip install -r $REQ_PIP
-fi
+conda list

ci/install_travis.sh

@@ -50,16 +50,6 @@ conda config --set ssl_verify false || exit 1
 conda config --set quiet true --set always_yes true --set changeps1 false || exit 1
 conda update -q conda
 
-echo
-echo "[add channels]"
-conda config --remove channels defaults || exit 1
-conda config --add channels defaults || exit 1
-
-if [ "$CONDA_FORGE" ]; then
-    # add conda-forge channel as priority
-    conda config --add channels conda-forge || exit 1
-fi
-
 # Useful for debugging any issues with conda
 conda info -a || exit 1
 
@@ -90,55 +80,10 @@ echo
 echo "[create env]"
 
 # create our environment
-REQ="ci/requirements-${JOB}.build"
-time conda create -n pandas --file=${REQ} || exit 1
+time conda env create -q -n pandas --file="${ENV_FILE}" || exit 1
 
 source activate pandas
 
-# may have addtl installation instructions for this build
-echo
-echo "[build addtl installs]"
-REQ="ci/requirements-${JOB}.build.sh"
-if [ -e ${REQ} ]; then
-    time bash $REQ || exit 1
-fi
-
-time conda install -n pandas pytest>=3.1.0
-time pip install -q pytest-xdist moto
-
-if [ "$LINT" ]; then
-   conda install flake8=3.4.1
-   pip install cpplint
-fi
-
-if [ "$COVERAGE" ]; then
-    pip install coverage pytest-cov
-fi
-
-# we may have run installations
-echo
-echo "[conda installs]"
-REQ="ci/requirements-${JOB}.run"
-if [ -e ${REQ} ]; then
-    time conda install -n pandas --file=${REQ} || exit 1
-fi
-
-# we may have additional pip installs
-echo
-echo "[pip installs]"
-REQ="ci/requirements-${JOB}.pip"
-if [ -e ${REQ} ]; then
-   pip install -r $REQ
-fi
-
-# may have addtl installation instructions for this build
-echo
-echo "[addtl installs]"
-REQ="ci/requirements-${JOB}.sh"
-if [ -e ${REQ} ]; then
-    time bash $REQ || exit 1
-fi
-
 # remove any installed pandas package
 # w/o removing anything else
 echo
@@ -156,8 +101,8 @@ echo "[running setup.py develop]"
 python setup.py develop  || exit 1
 
 echo
-echo "[show pandas]"
-conda list pandas
+echo "[show environment]"
+conda list
 
 echo
 echo "[done]"