Update ci_push_testing.yml #1522
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Push Testing | ||
| on: | ||
| pull_request_target: | ||
| branches: [ develop ] | ||
| jobs: | ||
| approve: | ||
| name: Approval | ||
| if: ${{github.event.pull_request.head.repo.full_name != 'ganga-devs/ganga'}} | ||
| runs-on: ubuntu-latest | ||
| environment: | ||
| name: Integrate Pull Request | ||
| steps: | ||
| - name: Approve | ||
| run: echo For security reasons, all pull requests need to be approved first before running any automated CI. | ||
| lint: | ||
| name: Linting | ||
| runs-on: ubuntu-latest | ||
| needs: [approve] | ||
| if: always() && !failure() && !cancelled() | ||
| continue-on-error: true | ||
| outputs: | ||
| files_to_lint: ${{ steps.flake8.outputs.files_to_lint }} | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| with: | ||
| ref: ${{ github.event.pull_request.head.sha }} | ||
| fetch-depth: 0 | ||
| - name: Set up Python 3.11 | ||
| uses: actions/setup-python@v1 | ||
| with: | ||
| python-version: 3.11 | ||
| - name: Get changed files | ||
| id: changed-files | ||
| uses: tj-actions/changed-files@v24 | ||
| - name: Install dependencies | ||
| run: | | ||
| python3 -m pip install --upgrade pip setuptools wheel | ||
| python3 -m pip install flake8 | ||
| - name: Setup flake8 annotations | ||
| if: startsWith(github.head_ref, 'autopep8-patches') | ||
| uses: rbialon/flake8-annotations@v1 | ||
| - name: Lint with flake8 | ||
| id: flake8 | ||
| run: | | ||
| files_to_lint="" | ||
| count=0 | ||
| for file in ${{ steps.changed-files.outputs.all_changed_files }}; do | ||
| if [[ $file == *.py ]]; then | ||
| linting_output=`flake8 --count --exit-zero $file` | ||
| count=`echo "$linting_output" | tail -1` | ||
| if [[ $count > 0 ]]; then | ||
| files_to_lint="$files_to_lint $file" | ||
| echo "$linting_output" | ||
| fi | ||
| fi | ||
| done | ||
| echo "::set-output name=files_to_lint::$files_to_lint"; | ||
| - name: Fail on linting violations | ||
| if: steps.flake8.outputs.files_to_lint != '' | ||
| run: exit 1 | ||
| autofix_pr: | ||
| name: Create autopep8 PR | ||
| if: startsWith(github.head_ref, 'autopep8-patches') == false && needs.lint.outputs.files_to_lint != '' | ||
| runs-on: ubuntu-latest | ||
| needs: lint | ||
| if: always() && !failure() && !cancelled() | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| with: | ||
| ref: ${{ github.event.pull_request.head.sha }} | ||
| - name: Run autopep8 | ||
| id: autopep8 | ||
| uses: peter-evans/autopep8@v1 | ||
| with: | ||
| args: --exit-code --recursive --in-place --aggressive --aggressive ${{ needs.lint.outputs.files_to_lint }} | ||
| - name: Set autopep8 branch name | ||
| id: vars | ||
| run: | | ||
| echo ::set-output name=branch-name::"autopep8-patches/${{ github.head_ref }}" | ||
| PR_NUMBER=`echo ${{ github.ref_name }} | cut -f 1 -d '/'` | ||
| echo ::set-output name=pr-number::"$PR_NUMBER" | ||
| - name: Create Pull Request | ||
| if: steps.autopep8.outputs.exit-code == 2 | ||
| uses: peter-evans/create-pull-request@v4 | ||
| with: | ||
| commit-message: autopep8 action fixes | ||
| title: "PEP8 fixes for PR #${{ steps.vars.outputs.pr-number }} (${{ github.head_ref }}) by autopep8" | ||
| body: This is an auto-generated PR with fixes by autopep8 for the ${{ steps.vars.outputs.branch-name }} branch. | ||
| labels: autopep8, automated pr | ||
| branch: ${{ steps.vars.outputs.branch-name }} | ||
| token: ${{ secrets.WORKFLOW }} | ||
| core-unit: | ||
| name: GangaCore Unit | ||
| needs: lint | ||
| if: always() && !failure() && !cancelled() | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| with: | ||
| ref: ${{ github.event.pull_request.head.sha }} | ||
| fetch-depth: 0 | ||
| - name: Set up Python 3.11 | ||
| uses: actions/setup-python@v1 | ||
| with: | ||
| python-version: 3.11 | ||
| - name: Install dependencies | ||
| run: | | ||
| python -m venv ~/venv | ||
| . ~/venv/bin/activate | ||
| python -m pip install --upgrade pip setuptools wheel | ||
| python -m pip install -e .[dev] | ||
| - name: Test with pytest | ||
| run: | | ||
| . ~/venv/bin/activate | ||
| python -m pytest --cov-report term-missing --cov ganga/GangaCore/Core --cov ganga/GangaCore/GPI --cov ganga/GangaCore/GPIDev --cov ganga/GangaCore/Lib --cov ganga/GangaCore/Runtime --cov ganga/GangaCore/PACKAGE.py --cov ganga/GangaCore/Utility --cov ganga/GangaCore/__init__.py ganga/GangaCore/test/Unit | ||
| core-integration: | ||
| name: GangaCore Integration | ||
| needs: core-unit | ||
| if: always() && !failure() && !cancelled() | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| with: | ||
| ref: ${{ github.event.pull_request.head.sha }} | ||
| fetch-depth: 0 | ||
| - name: Set up Python 3.11 | ||
| uses: actions/setup-python@v1 | ||
| with: | ||
| python-version: 3.11 | ||
| - name: Install dependencies | ||
| run: | | ||
| python -m pip install --upgrade pip setuptools wheel | ||
| python -m pip install -e .[dev] | ||
| - name: Test with pytest | ||
| run: python -m pytest --cov-report term-missing --cov ganga/GangaCore/Core --cov ganga/GangaCore/GPI --cov ganga/GangaCore/GPIDev --cov ganga/GangaCore/Lib --cov ganga/GangaCore/Runtime --cov ganga/GangaCore/PACKAGE.py --cov ganga/GangaCore/Utility --cov ganga/GangaCore/__init__.py ganga/GangaCore/test/GPI | ||
| gui: | ||
| name: GangaGUI tests | ||
| needs: lint | ||
| if: always() && !failure() && !cancelled() | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| with: | ||
| ref: ${{ github.event.pull_request.head.sha }} | ||
| fetch-depth: 0 | ||
| - name: Set up Python 3.11 | ||
| uses: actions/setup-python@v1 | ||
| with: | ||
| python-version: 3.11 | ||
| - name: Install dependencies | ||
| run: | | ||
| python -m pip install --upgrade pip setuptools wheel | ||
| python -m pip install -e .[dev] | ||
| - name: Test with pytest | ||
| run: python -m pytest --cov-report term-missing --cov ganga/GangaGUI ganga/GangaGUI/test | ||
| condor: | ||
| name: Condor | ||
| needs: lint | ||
| if: always() && !failure() && !cancelled() | ||
| runs-on: ubuntu-latest | ||
| container: htcondor/mini:el7 | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| with: | ||
| ref: ${{ github.event.pull_request.head.sha }} | ||
| - name: Set up Python 3 | ||
| run: | | ||
| yum install -y centos-release-scl-rh | ||
| yum install -y rh-python38-python rh-python38-python-pip | ||
| - name: Install dependencies | ||
| run: | | ||
| /opt/rh/rh-python38/root/usr/bin/python -m pip install --upgrade pip setuptools wheel | ||
| /opt/rh/rh-python38/root/usr/bin/python -m pip install -e .[dev] | ||
| - name: Start Condor | ||
| run: | | ||
| /start.sh > start.stdout & | ||
| sleep 30 | ||
| condor_status | ||
| - name: Test with pytest | ||
| run: | | ||
| pwd | ||
| find . -exec chown submituser:submituser {} \; | ||
| su submituser -s /opt/rh/rh-python38/root/usr/bin/python -- -m pytest --cov-report term-missing --cov ganga/GangaCore/Lib/Condor ganga/GangaCore/test/Condor | ||
| condor_status | ||
| condor_q | ||
| dirac-unit: | ||
| name: GangaDirac Unit | ||
| needs: lint | ||
| if: always() && !failure() && !cancelled() | ||
| runs-on: ubuntu-latest | ||
| container: centos:7 | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| with: | ||
| ref: ${{ github.event.pull_request.head.sha }} | ||
| - name: Set up Python 3 | ||
| run: | | ||
| yum install -y centos-release-scl-rh | ||
| yum install -y rh-python38-python rh-python38-python-pip | ||
| - name: Install dependencies | ||
| run: | | ||
| /opt/rh/rh-python38/root/usr/bin/python -m pip install --upgrade pip setuptools wheel | ||
| /opt/rh/rh-python38/root/usr/bin/python -m pip install -e .[dev] | ||
| - name: Install Robot certificate | ||
| env: # Or as an environment variable | ||
| ROBOT_CERT: ${{ secrets.GangaRobot_UserCert }} | ||
| ROBOT_KEY: ${{ secrets.GangaRobot_UserKey }} | ||
| run: | | ||
| mkdir ~/.globus | ||
| echo "$ROBOT_CERT" > ~/.globus/usercert.pem | ||
| echo "$ROBOT_KEY" > ~/.globus/userkey.pem | ||
| chmod 644 ~/.globus/usercert.pem | ||
| chmod 400 ~/.globus/userkey.pem | ||
| - name: Install DIRAC UI | ||
| run: | | ||
| yum install -y git wget which | ||
| mkdir ~/dirac_ui | ||
| cd ~/dirac_ui | ||
| wget -np -O dirac-install https://raw.githubusercontent.com/DIRACGrid/management/master/dirac-install.py | ||
| chmod u+x dirac-install | ||
| DIRAC_VERSION=`curl -s https://raw.githubusercontent.com/DIRACGrid/DIRAC/integration/release.notes | grep -oP "\\[\\Kv[7-8]r.*[^\\]]" | grep -m1 -v -e "pre"` | ||
| ./dirac-install -r $DIRAC_VERSION -i 27 -g v14r1 | ||
| source ~/dirac_ui/bashrc | ||
| mkdir ~/.proxy | ||
| DIRAC_PROXY_INIT=`which dirac-proxy-init` | ||
| export PATH=~/.proxy:"$PATH" | ||
| printf '#!/bin/sh\necho abcd | '"$DIRAC_PROXY_INIT"' -p $@ \n' > ~/.proxy/dirac-proxy-init | ||
| chmod +x ~/.proxy/dirac-proxy-init | ||
| dirac-proxy-init -x | ||
| dirac-configure -F -S GridPP -C dips://dirac01.grid.hep.ph.ic.ac.uk:9135/Configuration/Server -I | ||
| dirac-proxy-init -g gridpp_user -M | ||
| - name: Test with pytest | ||
| run: | | ||
| /opt/rh/rh-python38/root/usr/bin/python -m pytest --cov-report term-missing --cov ganga/GangaDirac ganga/GangaDirac/test/Unit | ||
| dirac-integration: | ||
| name: GangaDirac Integration | ||
| needs: dirac-unit | ||
| if: always() && !failure() && !cancelled() | ||
| runs-on: ubuntu-latest | ||
| container: centos:7 | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| with: | ||
| ref: ${{ github.event.pull_request.head.sha }} | ||
| - name: Set up Python 3 | ||
| run: | | ||
| yum install -y centos-release-scl-rh | ||
| yum install -y rh-python38-python rh-python38-python-pip | ||
| - name: Install dependencies | ||
| run: | | ||
| /opt/rh/rh-python38/root/usr/bin/python -m pip install --upgrade pip setuptools wheel | ||
| /opt/rh/rh-python38/root/usr/bin/python -m pip install -e .[dev] | ||
| - name: Install Robot certificate | ||
| env: # Or as an environment variable | ||
| ROBOT_CERT: ${{ secrets.GangaRobot_UserCert }} | ||
| ROBOT_KEY: ${{ secrets.GangaRobot_UserKey }} | ||
| run: | | ||
| mkdir ~/.globus | ||
| echo "$ROBOT_CERT" > ~/.globus/usercert.pem | ||
| echo "$ROBOT_KEY" > ~/.globus/userkey.pem | ||
| chmod 644 ~/.globus/usercert.pem | ||
| chmod 400 ~/.globus/userkey.pem | ||
| - name: Install DIRAC UI | ||
| run: | | ||
| yum install -y git wget which | ||
| mkdir ~/dirac_ui | ||
| cd ~/dirac_ui | ||
| wget -np -O dirac-install https://raw.githubusercontent.com/DIRACGrid/management/master/dirac-install.py | ||
| chmod u+x dirac-install | ||
| DIRAC_VERSION=`curl -s https://raw.githubusercontent.com/DIRACGrid/DIRAC/integration/release.notes | grep -oP "\\[\\Kv[7]r.*[^\\]]" | grep -m1 -v -e "pre"` | ||
| DIRAC_VERSION=v7r2p10 | ||
| ./dirac-install -r $DIRAC_VERSION -i 27 -g v14r1 | ||
| source ~/dirac_ui/bashrc | ||
| mkdir ~/.proxy | ||
| DIRAC_PROXY_INIT=`which dirac-proxy-init` | ||
| export PATH=~/.proxy:"$PATH" | ||
| printf '#!/bin/sh\necho abcd | '"$DIRAC_PROXY_INIT"' -p $@ \n' > ~/.proxy/dirac-proxy-init | ||
| chmod +x ~/.proxy/dirac-proxy-init | ||
| dirac-proxy-init -x | ||
| dirac-configure -F -S GridPP -C dips://dirac01.grid.hep.ph.ic.ac.uk:9135/Configuration/Server -I | ||
| dirac-proxy-init -g gridpp_user -M | ||
| - name: Install gangarc file | ||
| run: | | ||
| echo -e "[DIRAC]\nDiracEnvSource = ~/dirac_ui/bashrc" > ~/.gangarc | ||
| echo -e "[Configuration]\nRUNTIME_PATH=GangaDirac" >> ~/.gangarc | ||
| echo -e "[defaults_DiracProxy]\ngroup=gridpp_user\nencodeDefaultProxyFileName=False" >> ~/.gangarc | ||
| - name: Test with pytest | ||
| env: | ||
| GANGA_CONFIG_FILE: ~/.gangarc | ||
| GANGA_CONFIG_PATH: GangaDirac/Dirac.ini | ||
| DIRAC_DEPRECATED_FAIL: True | ||
| run: /opt/rh/rh-python38/root/usr/bin/python -m pytest --cov-report term-missing --cov ganga/GangaDirac ganga/GangaDirac/test/GPI | ||
| lhcb-unit: | ||
| name: GangaLHCb Unit | ||
| needs: lint | ||
| if: always() && !failure() && !cancelled() | ||
| runs-on: ubuntu-latest | ||
| container: | ||
| image: centos:7 | ||
| options: --privileged | ||
| steps: | ||
| - name: Install CVM-FS | ||
| run: | | ||
| yum install -y https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm | ||
| yum install -y cvmfs cvmfs-config-default | ||
| cvmfs_config setup | ||
| echo "CVMFS_REPOSITORIES=lhcb.cern.ch,sft.cern.ch" > /etc/cvmfs/default.local | ||
| echo "CVMFS_HTTP_PROXY=DIRECT" >> /etc/cvmfs/default.local | ||
| mkdir -p /cvmfs/lhcb.cern.ch | ||
| mkdir -p /cvmfs/sft.cern.ch | ||
| mount -t cvmfs lhcb.cern.ch /cvmfs/lhcb.cern.ch | ||
| mount -t cvmfs sft.cern.ch /cvmfs/sft.cern.ch | ||
| - uses: actions/checkout@v2 | ||
| with: | ||
| ref: ${{ github.event.pull_request.head.sha }} | ||
| - name: Install virtualenv and dependencies | ||
| run: | | ||
| /cvmfs/sft.cern.ch/lcg/releases/LCG_100/Python/3.8.6/x86_64-centos7-gcc9-opt/bin/python3 -m venv ~/venv | ||
| . ~/venv/bin/activate | ||
| python3 -m pip install --upgrade pip setuptools wheel | ||
| python3 -m pip install -e .[dev,LHCb] | ||
| - name: Test with pytest | ||
| run: | | ||
| . ~/venv/bin/activate | ||
| python3 -m pytest --cov-report term-missing --cov ganga/GangaLHCb ganga/GangaLHCb/test/Unit | ||
| lhcb-integration: | ||
| name: GangaLHCb Integration | ||
| needs: lhcb-unit | ||
| if: always() && !failure() && !cancelled() | ||
| runs-on: ubuntu-latest | ||
| container: | ||
| image: centos:7 | ||
| options: --privileged | ||
| steps: | ||
| - name: Install git | ||
| run: yum install -y git | ||
| - name: Install CVM-FS | ||
| run: | | ||
| yum install -y https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm | ||
| yum install -y cvmfs cvmfs-config-default | ||
| cvmfs_config setup | ||
| echo "CVMFS_REPOSITORIES=lhcb.cern.ch,sft.cern.ch" > /etc/cvmfs/default.local | ||
| echo "CVMFS_HTTP_PROXY=DIRECT" >> /etc/cvmfs/default.local | ||
| mkdir -p /cvmfs/lhcb.cern.ch | ||
| mkdir -p /cvmfs/sft.cern.ch | ||
| mount -t cvmfs lhcb.cern.ch /cvmfs/lhcb.cern.ch | ||
| mount -t cvmfs sft.cern.ch /cvmfs/sft.cern.ch | ||
| - name: Install Robot certificate | ||
| env: # Or as an environment variable | ||
| ROBOT_CERT: ${{ secrets.GangaRobot_UserCert }} | ||
| ROBOT_KEY: ${{ secrets.GangaRobot_UserKey }} | ||
| run: | | ||
| mkdir ~/.globus | ||
| echo "$ROBOT_CERT" > ~/.globus/usercert.pem | ||
| echo "$ROBOT_KEY" > ~/.globus/userkey.pem | ||
| chmod 644 ~/.globus/usercert.pem | ||
| chmod 400 ~/.globus/userkey.pem | ||
| - uses: actions/checkout@v2 | ||
| with: | ||
| ref: ${{ github.event.pull_request.head.sha }} | ||
| - name: Install virtualenv and dependencies | ||
| run: | | ||
| /cvmfs/sft.cern.ch/lcg/releases/LCG_100/Python/3.8.6/x86_64-centos7-gcc9-opt/bin/python3 -m venv ~/venv | ||
| . ~/venv/bin/activate | ||
| python3 -m pip install --upgrade pip setuptools wheel | ||
| python3 -m pip install -e .[dev,LHCb] | ||
| - name: Install gangarc file | ||
| run: | | ||
| echo -e "[Configuration]\nRUNTIME_PATH=GangaDirac:GangaGaudi:GangaLHCb" > ~/.gangarc | ||
| - name: Test with pytest | ||
| env: | ||
| GANGA_CONFIG_PATH: GangaLHCb/LHCb.ini | ||
| GANGA_CONFIG_FILE: ~/.gangarc | ||
| run: | | ||
| source /cvmfs/lhcb.cern.ch/lib/LbEnv 2>&1 | ||
| echo abcd | lhcb-proxy-init -p | ||
| . ~/venv/bin/activate | ||
| python3 -m pytest --cov-report term-missing --cov ganga/GangaLHCb ganga/GangaLHCb/test/GPI | ||
