Update CI (#352)

* Make dev install option and drop python 2 * Simplify CI * Update ci.yml * Separate workqueue test * Combine release and CI
CoffeaTeam · Sep 24, 2020 · d378994 · d378994
1 parent 5fce8fa
commit d378994
Show file tree

Hide file tree

Showing 6 changed files with 80 additions and 143 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -4,6 +4,8 @@ on:
   push:
     branches:
       - master
+    tags:
+      - v*.*.*
   pull_request:
     branches:
       - master
@@ -13,7 +15,6 @@ on:
 
 jobs:
   linter:
-
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -24,7 +25,7 @@ jobs:
     steps:
     - uses: actions/checkout@master
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
     - name: Lint with flake8
@@ -33,85 +34,117 @@ jobs:
         flake8 coffea
 
   test:
-
     runs-on: ${{ matrix.os }}
+    needs: linter
     strategy:
       matrix:
-        os: [ubuntu-latest, macOS-latest, windows-latest] #
+        os: [ubuntu-latest, macOS-latest, windows-latest]
         java-version: [1.8]
-        python-version: [3.6, 3.7, 3.8] #
-        awkward-version: ['awkward>=0.8.4']
+        python-version: [3.6, 3.7, 3.8]
     name: test coffea (${{ matrix.os }}) - python ${{ matrix.python-version }}, JDK${{ matrix.java-version }}
 
     steps:
     - uses: actions/checkout@master
-    - run: |
-        git fetch --no-tags --prune --depth=1 origin +refs/heads/*:refs/remotes/origin/*
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
     - name: Set up JDK ${{ matrix.java-version }}
+      if: matrix.os != 'windows-latest'
       uses: actions/setup-java@v1
       with:
         java-version: ${{ matrix.java-version }}
-    - name: Set up Conda (Linux/MacOS)
-      if: matrix.os != 'windows-latest'
-      uses: goanpeca/setup-miniconda@v1
-      with:
-        update-conda: true
-        python-version: ${{ matrix.python-version }}
-        miniconda-version: 'latest'
     - name: Install dependencies (Linux/MacOS)
       if: matrix.os != 'windows-latest'
       run: |
-        python -m pip install -q --upgrade pip setuptools setuptools_scm wheel six pytest codecov
-        python -m pip install -q ${{ matrix.awkward-version }}
-        python -m pip install -q https://github.com/Parsl/parsl/zipball/master
-        python -m pip install -q -e .[parsl,dask,spark]
+        python -m pip install --upgrade pip setuptools wheel
+        python -m pip install -q -e .[dev,parsl,dask,spark]
         python -m pip list
         java -version
     - name: Install dependencies (Windows)
       if: matrix.os == 'windows-latest'
       run: |
-        python -m pip install -q --upgrade pip setuptools setuptools_scm wheel six pytest codecov
-        python -m pip install -q ${{ matrix.awkward-version }}
-        python -m pip install -q -e .
+        python -m pip install --upgrade pip setuptools wheel
+        python -m pip install -q -e .[dev]
         python -m pip list
     - name: Test with pytest
       env:
         ARROW_PRE_0_15_IPC_FORMAT: 1
       run: |
-        coverage run --source=coffea/ -m pytest tests
-    - name: Test work_queue
-      if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.7
-      shell: bash -l {0}
-      run: |
-        conda create --name coffea-env-${{ matrix.python-version }} python=${{ matrix.python-version }}
-        conda activate coffea-env-${{ matrix.python-version }}
-        conda install -c conda-forge ndcctools dill six
-        python -m pip install -I .
-        cd tests
-        python wq.py
+        pytest --cov-report=xml --cov=coffea tests
     - name: Upload codecov
-      if: success() && matrix.os == 'ubuntu-latest' && matrix.python-version == 3.6      
+      if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.7
       run: |
+        python -m pip install codecov
         codecov
     - name: Install graphviz
-      if: success()
+      if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.7
       uses: kamiazya/setup-graphviz@v1 
     - name: Install pandoc
-      if: success() && matrix.os == 'ubuntu-latest' && matrix.python-version == 3.6
+      if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.7
       uses: r-lib/actions/setup-pandoc@v1
     - name: Build documentation
-      if: success() && matrix.os == 'ubuntu-latest' && matrix.python-version == 3.6
+      if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.7
       run: |
         source docs/build_docs.sh
     - name: Deploy documentation
-      if: success() && github.event_name != 'pull_request' && matrix.os == 'ubuntu-latest' && matrix.python-version == 3.6
+      if: github.event_name == 'push' && matrix.os == 'ubuntu-latest' && matrix.python-version == 3.7
       uses: crazy-max/ghaction-github-pages@v1.0.1
       with:
         target_branch: gh-pages
         build_dir: docs/build/html
       env:
         GITHUB_PAT: ${{ secrets.GITHUB_OAUTH }}
+
+  testwq:
+    runs-on: ubuntu-latest
+    needs: linter
+    name: test coffea-workqueue
+
+    steps:
+    - uses: actions/checkout@master
+    - name: Set up Conda
+      uses: conda-incubator/setup-miniconda@v1.7.0
+      with:
+        auto-update-conda: true
+        python-version: 3.7
+    - name: Test work_queue
+      shell: bash -l {0}
+      run: |
+        conda create --name coffea-env python=3.7
+        conda activate coffea-env
+        conda install -c conda-forge ndcctools dill
+        python -m pip install .
+        cd tests
+        python wq.py
+
+  release:
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    needs: [test, testwq]
+    strategy:
+      matrix:
+        python-version: [3.7]
+    name: deploy release
+
+    steps:
+    - uses: actions/checkout@master
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Create Release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_OAUTH }}
+      with:
+        tag_name: ${{ github.ref }}
+        release_name: Release ${{ github.ref }}
+    - name: Build package for PyPI
+      run: |
+        python setup.py sdist bdist_wheel --universal
+    - name: Publish package to PyPI
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_TOKEN }}
diff --git a/coffea/nanoaod/util.py b/coffea/nanoaod/util.py
@@ -1,4 +1,3 @@
-import six
 import awkward
 
 
@@ -10,6 +9,5 @@ def _mixin(methods, awkwardtype):
         raise ValueError("Can only mixin methods deriving from awkward Methods ABC")
     newtype = type(methods.__name__ + 'Array', (methods, awkwardtype), {})
     newtype.__dir__ = lambda self: dir(methods) + awkwardtype.__dir__(self)
-    if six.PY3:
-        newtype.__doc__ = methods.__doc__
+    newtype.__doc__ = methods.__doc__
     return newtype
diff --git a/coffea/processor/accumulator.py b/coffea/processor/accumulator.py
@@ -1,4 +1,3 @@
-from six import with_metaclass
 from abc import ABCMeta, abstractmethod
 from collections import defaultdict
 import numpy
@@ -9,7 +8,7 @@
     from collections import Set, Mapping
 
 
-class AccumulatorABC(with_metaclass(ABCMeta)):
+class AccumulatorABC(metaclass=ABCMeta):
     '''ABC for an accumulator
 
     Derived class must implement:

diff --git a/coffea/processor/processor.py b/coffea/processor/processor.py
@@ -1,8 +1,7 @@
-from six import with_metaclass
 from abc import ABCMeta, abstractmethod
 
 
-class ProcessorABC(with_metaclass(ABCMeta)):
+class ProcessorABC(metaclass=ABCMeta):
     '''ABC for a generalized processor
 
     The various data delivery mechanisms (spark, striped, uproot, uproot+futures, condor, ...)

diff --git a/setup.py b/setup.py
@@ -30,7 +30,6 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import os.path
-import six
 
 from setuptools import find_packages
 from setuptools import setup
@@ -61,28 +60,24 @@ def get_description():
                     'uproot>=3.11.0',
                     'awkward1>=0.3.0',
                     'uproot4>=0.0.17',
-                    'matplotlib<3' if six.PY2 else 'matplotlib>=3',
+                    'matplotlib>=3',
                     'numba>=0.50.0',
                     'numpy>=1.16.0',
                     'scipy>=1.1.0',
                     'futures; python_version == "2.7"',
                     'tqdm>=4.27.0',
                     'lz4',
-                    'six',
                     'cloudpickle>=1.2.3',
                     'mplhep>=0.1.18',
                     'packaging',
                     'ipywidgets',
                     'pandas',
                     ]
 EXTRAS_REQUIRE = {}
-if six.PY3:
-    templates = ['jinja2']
-    EXTRAS_REQUIRE['spark'] = ['pyspark>=2.4.1,<3.0.0', 'pyarrow>=0.10.0,!=0.14.0'] + templates
-    EXTRAS_REQUIRE['parsl'] = ['parsl>=0.7.2']
-    EXTRAS_REQUIRE['dask'] = ['dask>=2.6.0', 'distributed>=2.6.0', 'bokeh>=1.3.4', 'blosc']
-if six.PY2:
-    EXTRAS_REQUIRE['striped'] = []
+EXTRAS_REQUIRE['spark'] = ['pyspark>=2.4.1,<3.0.0', 'pyarrow>=0.10.0,!=0.14.0', 'jinja2']
+EXTRAS_REQUIRE['parsl'] = ['parsl>=0.7.2']
+EXTRAS_REQUIRE['dask'] = ['dask>=2.6.0', 'distributed>=2.6.0', 'bokeh>=1.3.4', 'blosc']
+EXTRAS_REQUIRE['dev'] = ['flake8', 'pytest', 'pytest-cov']
 
 setup(name="coffea",
       version=get_version(),
@@ -101,8 +96,6 @@ def get_description():
       test_suite="tests",
       install_requires=INSTALL_REQUIRES,
       extras_require=EXTRAS_REQUIRE,
-      setup_requires=["pytest-runner", "flake8"],
-      tests_require=["pytest"],
       classifiers=[
           "Development Status :: 4 - Beta",
           "Intended Audience :: Developers",