fix intersecting bims #599
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - dev | |
| - main | |
| - ci | |
| pull_request: | |
| branches: | |
| - dev | |
| - main | |
| release: | |
| types: [published] | |
| env: | |
| SINGULARITY_VERSION: "3.8.3" | |
| NXF_ANSI_LOG: false | |
| CAPSULE_LOG: none | |
| NXF_SINGULARITY_CACHEDIR: "${{ github.workspace }}/singularity" | |
| jobs: | |
| preload_docker: | |
| name: Preload docker images | |
| if: ${{ github.event_name == 'push'}} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out pipeline code | |
| uses: actions/checkout@v3 | |
| - name: Cache docker | |
| id: cache-docker | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ runner.temp }}/docker | |
| key: ${{ runner.os }}-${{ github.sha }} | |
| - name: Pull and save docker | |
| if: matrix.profile == 'docker' && ${{ steps.cache-docker.outputs.cache-hit != 'true' }} | |
| run: | | |
| git grep 'ext.docker*' ${{ github.workspace }}/conf/modules.config | cut -f 2 -d '=' | xargs -L 2 echo | tr -d ' ' > ${{ runner.temp }}/images.txt | |
| cat ${{ runner.temp }}/images.txt | xargs -I {} sh -c 'docker pull --platform linux/amd64 "$1"' - {} | |
| mkdir -p ${{ runner.temp }}/docker/ | |
| cat ${{ runner.temp }}/images.txt | xargs -I {} sh -c 'docker save "$1" > ${{ runner.temp }}/docker/$(basename "$1").tar' - {} | |
| preload_singularity: | |
| name: Preload singularity images | |
| if: ${{ github.event_name == 'push'}} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out pipeline code | |
| uses: actions/checkout@v3 | |
| - name: Cache singularity setup | |
| id: cache-singularity-setup | |
| uses: actions/cache@v3 | |
| with: | |
| path: /opt/hostedtoolcache/singularity/${{ env.SINGULARITY_VERSION }}/x64 | |
| key: ${{ runner.os }}-singularity-${{ env.SINGULARITY_VERSION }} | |
| - name: Set up Singularity | |
| uses: eWaterCycle/setup-singularity@v7 | |
| if: ${{ steps.cache-singularity-setup.outputs.cache-hit != 'true' }} | |
| with: | |
| singularity-version: ${{ env.SINGULARITY_VERSION }} | |
| - name: Add singularity to path | |
| if: steps.cache-singularity-setup.outputs.cache-hit == 'true' | |
| run: | | |
| echo "/opt/hostedtoolcache/singularity/${{ env.SINGULARITY_VERSION }}/x64/bin" >> $GITHUB_PATH | |
| - name: Cache singularity images | |
| id: cache-singularity-pull | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ env.NXF_SINGULARITY_CACHEDIR }} | |
| key: ${{ runner.os }}-${{ github.sha }}--singularity | |
| - name: Pull and save singularity | |
| if: ${{ steps.cache-singularity-pull.outputs.cache-hit != 'true' }} | |
| run: | | |
| mkdir -p $NXF_SINGULARITY_CACHEDIR | |
| git grep 'ext.singularity*' conf/modules.config | cut -f 2 -d '=' | xargs -L 2 echo | tr -d ' ' > ${{ runner.temp }}/singularity_images.txt | |
| cat ${{ runner.temp }}/singularity_images.txt | sed 's/oras:\/\///;s/https:\/\///;s/\//-/g;s/$/.img/;s/:/-/' > ${{ runner.temp }}/singularity_image_paths.txt | |
| paste -d '\n' ${{ runner.temp }}/singularity_image_paths.txt ${{ runner.temp }}/singularity_images.txt | xargs -L 2 sh -c 'singularity pull --disable-cache --dir $NXF_SINGULARITY_CACHEDIR $0 $1' | |
| test: | |
| needs: [preload_docker, preload_singularity] | |
| name: Run standard workflow test | |
| if: ${{ github.event_name == 'push'}} | |
| runs-on: ubuntu-latest | |
| env: | |
| NXF_VER: ${{ matrix.nxf_ver }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_profile: ["test"] | |
| profile: ["docker", "singularity"] | |
| nxf_ver: ['22.10.0', ''] # minimum supported version, latest version | |
| steps: | |
| - name: Check out pipeline code | |
| uses: actions/checkout@v3 | |
| - name: Install Nextflow | |
| run: | | |
| wget -qO- get.nextflow.io | bash | |
| sudo mv nextflow /usr/local/bin/ | |
| - name: Restore docker cache | |
| id: restore-docker | |
| uses: actions/cache@v3 | |
| if: matrix.profile == 'docker' | |
| with: | |
| path: ${{ runner.temp }}/docker | |
| key: ${{ runner.os }}-${{ github.sha }} | |
| - name: Load docker images from cache | |
| if: steps.restore-docker.outputs.cache-hit == 'true' | |
| run: | | |
| find $HOME -name '*.tar' | |
| find ${{ runner.temp }}/docker/ -name '*.tar' -exec sh -c 'docker load < {}' \; | |
| - name: Cache singularity setup | |
| id: restore-singularity-setup | |
| if: matrix.profile == 'singularity' | |
| uses: actions/cache@v3 | |
| with: | |
| path: /opt/hostedtoolcache/singularity/${{ env.SINGULARITY_VERSION }}/x64 | |
| key: ${{ runner.os }}-singularity-${{ env.SINGULARITY_VERSION }} | |
| - name: Add singularity to path | |
| if: steps.restore-singularity-setup.outputs.cache-hit == 'true' | |
| run: | | |
| echo "/opt/hostedtoolcache/singularity/${{ env.SINGULARITY_VERSION }}/x64/bin" >> $GITHUB_PATH | |
| - name: Restore singularity cache | |
| id: restore-singularity | |
| uses: actions/cache@v3 | |
| if: matrix.profile == 'singularity' | |
| with: | |
| path: ${{ env.NXF_SINGULARITY_CACHEDIR }} | |
| key: ${{ runner.os }}-${{ github.sha }}--singularity | |
| - name: Run pipeline with test data | |
| continue-on-error: false | |
| run: | | |
| nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.test_profile}},${{ matrix.profile }} | |
| unit_test: | |
| needs: [preload_docker, preload_singularity] | |
| name: ${{ matrix.profile }} ${{ matrix.tags }} | |
| if: ${{ github.event_name == 'push'}} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| profile: ["docker", "singularity"] | |
| tags: | |
| - "test input check subworkflow" | |
| - "test input check subworkflow with PGS catalog API" | |
| - "test input check subworkflow with PGS catalog API and whitespace" | |
| - "test make compatible subworkflow with bfile" | |
| - "test make compatible subworkflow with vcf" | |
| - "test make compatible subworkflow with pfile" | |
| - "test input check subworkflow with liftover 38to37" | |
| - "test input check subworkflow with liftover 37to38" | |
| - "test apply score subworkflow" | |
| - "test perfect apply score" | |
| # TODO - "test score correlation" | |
| - "test combine scorefiles module" | |
| - "test match module" | |
| - "test match combine module" | |
| - "plink2 testrelabelpvar" | |
| - "plink2 testscore" | |
| - "plink2 testsmallscore" | |
| - "plink2 testmultiscore" | |
| - "plink2 testsmallmultiscore" | |
| - "plink2 testmultiscorefail" | |
| - "plink2 vcf" | |
| - "plink2 testrelabelbim" | |
| - "pgscatalog test --pgs_id" | |
| - "pgscatalog test --efo_trait --pgp_id and --pgs_id" | |
| - "pgscatalog test bad accession" | |
| - "pgscatalog test good and bad accessions GRCh38" | |
| steps: | |
| - name: Check out pipeline code | |
| uses: actions/checkout@v3 | |
| - name: Restore docker cache | |
| id: restore-docker | |
| uses: actions/cache@v3 | |
| if: matrix.profile == 'docker' | |
| with: | |
| path: ${{ runner.temp }}/docker | |
| key: ${{ runner.os }}-${{ github.sha }} | |
| - name: Load docker images from cache | |
| if: steps.restore-docker.outputs.cache-hit == 'true' | |
| run: | | |
| find $HOME -name '*.tar' | |
| find ${{ runner.temp }}/docker/ -name '*.tar' -exec sh -c 'docker load < {}' \; | |
| - name: Cache singularity setup | |
| id: cache-singularity | |
| if: matrix.profile == 'singularity' | |
| uses: actions/cache@v3 | |
| with: | |
| path: /opt/hostedtoolcache/singularity/${{ env.SINGULARITY_VERSION }}/x64 | |
| key: ${{ runner.os }}-singularity-${{ env.SINGULARITY_VERSION }} | |
| - name: Set up Singularity | |
| uses: eWaterCycle/setup-singularity@v7 | |
| if: steps.cache-singularity.outputs.cache-hit != 'true' && matrix.profile == 'singularity' | |
| - name: Add singularity to path | |
| if: steps.cache-singularity.outputs.cache-hit == 'true' && matrix.profile == 'singularity' | |
| run: | | |
| echo "/opt/hostedtoolcache/singularity/${{ env.SINGULARITY_VERSION }}/x64/bin" >> $GITHUB_PATH | |
| - name: Restore singularity cache | |
| id: restore-singularity | |
| uses: actions/cache@v3 | |
| if: matrix.profile == 'singularity' | |
| with: | |
| path: ${{ env.NXF_SINGULARITY_CACHEDIR }} | |
| key: ${{ runner.os }}-${{ github.sha }}--singularity | |
| - name: Set up Python | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: '3.10' | |
| cache: 'pip' | |
| - run: pip install -r ${{ github.workspace }}/tests/requirements.txt | |
| - name: Install Nextflow | |
| run: | | |
| wget -qO- get.nextflow.io | bash | |
| sudo mv nextflow /usr/local/bin/ | |
| - name: Run unit tests | |
| run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --kwdof --symlink --git-aware --wt 2 --tag "${{ matrix.tags }}" --ignore tests/bin | |
| - name: Output log on failure | |
| if: failure() | |
| run: | | |
| echo "======> log.out <=======" | |
| cat /home/runner/pytest_workflow_*/*/log.out | |
| echo | |
| echo | |
| echo "======> log.err <=======" | |
| cat /home/runner/pytest_workflow_*/*/log.err | |
| - name: Upload logs on failure | |
| if: failure() | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: logs-${{ matrix.profile }} | |
| path: | | |
| /home/runner/pytest_workflow_*/*/.nextflow.log | |
| /home/runner/pytest_workflow_*/*/log.out | |
| /home/runner/pytest_workflow_*/*/log.err | |
| /home/runner/pytest_workflow_*/*/work | |
| !/home/runner/pytest_workflow_*/*/work/conda | |
| !/home/runner/pytest_workflow_*/*/work/singularity |