Merge branch 'master' into docs_core

.circleci/config.yml

@@ -16,7 +16,7 @@ docs_deploy: &docs
     - run:
         name: Install gh-pages tool
         command: |
-          npm install -g --silent gh-pages@2.0.1
+          npm install -g --silent gh-pages@3.0.0
     - checkout
     - run:
         name: Set git settings
@@ -30,7 +30,7 @@ docs_deploy: &docs
         command: touch docs/_build/html/.nojekyll
     - run:
         name: Deploy docs to gh-pages branch
-        command: gh-pages --dotfiles --message "doc(update) [skip ci]" --dist docs/_build/html
+        command: gh-pages --no-history --dotfiles --message "doc(update) [skip ci]" --dist docs/_build/html
 
 version: 2
 jobs:

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,6 @@
+## Acknowledgment
+- [ ] I acknowledge that this contribution will be available under the Apache 2 license.
+
 ## Types of changes
 <!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->
 - [ ] Bug fix (non-breaking change which fixes an issue)
@@ -16,6 +19,3 @@
 (we are using `black`: you can `pip install pre-commit`,
 run `pre-commit install` in the `pydra` directory
 and `black` will be run automatically with each commit)
-
-## Acknowledgment
-- [ ] I acknowledge that this contribution will be available under the Apache 2 license.

.github/workflows/publish.yml

@@ -0,0 +1,31 @@
+# This workflows will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+
+    - name: Build and publish
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+      run: |
+        python setup.py bdist_wheel
+        twine upload dist/*

.github/workflows/testpydra.yml

@@ -0,0 +1,58 @@
+name: Pydra
+
+on: [push, pull_request]
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [macos-latest, ubuntu-latest, windows-latest]
+        python-version: [3.7, 3.8, 3.9]
+        install: [install, develop, wheel]
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Disable etelemetry
+      run:  echo "NO_ET=TRUE" >> $GITHUB_ENV
+    - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Update build tools
+      run: python -m pip install --upgrade pip setuptools
+
+
+    - name: Install Pydra tests dependencies (develop or setup.py install)
+      if: matrix.install == 'develop' || matrix.install == 'install'
+      run: pip install ".[test]"
+
+
+    - name: Install dependencies (setup.py install)
+      if: matrix.install == 'install'
+      run: python setup.py install
+
+    - name: Install dependencies (wheel)
+      if: matrix.install == 'wheel'
+      run: |
+        python setup.py bdist_wheel
+        pip install dist/*.whl
+
+
+    - name: Install Pydra tests dependencies (wheel)
+      if: matrix.install == 'wheel'
+      run: pip install "$( ls dist/pydra*.whl )[test]"
+
+
+    - name: Pytest
+      run: pytest -vs -n auto --cov pydra  --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra
+
+
+    - name: Upload to codecov
+      run: codecov -f cov.xml -F unittests -e GITHUB_WORKFLOW

.github/workflows/testsingularity.yml

@@ -0,0 +1,66 @@
+name: Singularity
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+      fail-fast: False
+
+    steps:
+      - name: Set env
+        run: |
+          echo "RELEASE_VERSION=v3.7.1" >> $GITHUB_ENV
+          echo "NO_ET=TRUE" >> $GITHUB_ENV
+      - name: Setup Singularity
+        uses: actions/checkout@v2
+        with:
+          repository: hpcng/singularity
+          ref: 'v3.7.1'
+          path: 'singularity'
+      - name: Setup GO
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.13'
+      - name: Install OS deps
+        run: |
+          sudo apt-get update
+          sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.1 \
+          libarchive-dev libgpgme11-dev libseccomp-dev wget gcc make pkg-config -y
+      - name: Build
+        run: |
+          cd singularity
+          ./mconfig --without-suid -p /usr/local/
+          make -C ./builddir
+          sudo make -C ./builddir install
+          cd ..
+      - name: Echo singularity version
+        run: |
+          echo ${{ github.ref }}
+          singularity --version
+
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Update build tools
+        run: python -m pip install --upgrade pip setuptools
+
+
+      - name: Checkout Pydra repo
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ github.repository }}
+      - name: Install pydra (test)
+        run: pip install -e ".[test]"
+
+
+      - name: Pytest
+        run: pytest -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml pydra/engine/tests/test_singularity.py
+      - name: Upload to codecov
+        run: codecov -f cov.xml -F unittests -e GITHUB_WORKFLOW

.github/workflows/testslurm.yml

@@ -0,0 +1,43 @@
+name: SLURM
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_IMAGE: mgxd/slurm:19.05.1
+
+    steps:
+    - name: Disable etelemetry
+      run: echo "NO_ET=TRUE" >> $GITHUB_ENV
+    - uses: actions/checkout@v2
+    - name: Pull docker image
+      run: |
+        docker pull $DOCKER_IMAGE
+        # Have image running in background
+        docker run `bash <(curl -s https://codecov.io/env)` -itd -h ernie --name slurm -v `pwd`:/pydra -e NO_ET=$NO_ET $DOCKER_IMAGE
+    - name: Display previous jobs with sacct
+      run: |
+        echo "Allowing ports/daemons time to start" && sleep 10
+        docker exec slurm bash -c "sacctmgr -i add cluster name=linux \
+          && supervisorctl restart slurmdbd \
+          && supervisorctl restart slurmctld \
+          && sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'"
+        docker exec slurm bash -c "sacct && sinfo && squeue" 2&> /dev/null
+        if [ $? -ne 0 ]; then
+            echo "Slurm docker image error"
+            exit 1
+        fi
+    - name: Setup Python
+      run: |
+        docker exec slurm bash -c "echo $NO_ET"
+        docker exec slurm bash -c "ls -la && echo list top level dir"
+        docker exec slurm bash -c "ls -la /pydra && echo list pydra dir"
+        docker exec slurm bash -c "pip install -e /pydra[test] && python -c 'import pydra; print(pydra.__version__)'"
+    - name: Run pytest
+      run: docker exec slurm bash -c "pytest --color=yes -vs -n auto --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra"
+    - name: Upload to codecov
+      run: |
+        docker exec slurm bash -c "codecov --root /pydra -f /pydra/cov.xml -F unittests"
+        docker rm -f slurm

.github/workflows/teststyle.yml

@@ -0,0 +1,29 @@
+name: Style
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+      fail-fast: false
+    runs-on: ubuntu-latest
+
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }} on ubuntu-latest
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Update build tools
+      run: python -m pip install --upgrade pip setuptools
+
+    - name: Install Pydra
+      run: pip install ".[dev]"
+
+    - name: Check Style
+      run: |
+        pip install black==19.3b0 codecov
+        black --check pydra setup.py