Skip to content

Commit

Permalink
Merge pull request #150 from drpatelh/fixes
Browse files Browse the repository at this point in the history
Add multi-cloud CI for full-sized tests
  • Loading branch information
drpatelh committed May 8, 2023
2 parents 1dfe97b + 7d858a3 commit e90a023
Show file tree
Hide file tree
Showing 5 changed files with 163 additions and 65 deletions.
31 changes: 0 additions & 31 deletions .github/workflows/awsfulltest.yml

This file was deleted.

29 changes: 0 additions & 29 deletions .github/workflows/awstest.yml

This file was deleted.

81 changes: 81 additions & 0 deletions .github/workflows/cloud_tests_full.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
name: full-sized tests on cloud providers
run-name: Submitting workflow to all cloud providers using full sized data
on:
release:
types: [published]
workflow_dispatch:
inputs:
platform:
description: "Platform to run test"
required: true
default: "all"
type: choice
options:
- all
- aws
- azure
- gcp
jobs:
run-full-tests-on-aws:
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'aws' || !github.event.inputs }}
runs-on: ubuntu-latest
steps:
- uses: seqeralabs/action-tower-launch@v1
with:
workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
compute_env: ${{ secrets.TOWER_CE_AWS_CPU }}
workdir: "${{ secrets.TOWER_BUCKET_AWS }}/work/fetchngs/work-${{ github.sha }}"
run_name: "aws_fetchngs_full"
profiles: test_full
parameters: |
{
"hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
"outdir": "${{ secrets.TOWER_BUCKET_AWS }}/fetchngs/results-${{ github.sha }}"
}
- uses: actions/upload-artifact@v3
with:
name: Tower debug log file
path: tower_action_*.log
run-full-tests-on-gcp:
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'gcp' || !github.event.inputs }}
runs-on: ubuntu-latest
steps:
- uses: seqeralabs/action-tower-launch@v1
with:
workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
compute_env: ${{ secrets.TOWER_CE_GCP_CPU }}
workdir: "${{ secrets.TOWER_BUCKET_GCP }}/work/fetchngs/work-${{ github.sha }}"
run_name: "gcp_fetchngs_full"
profiles: test_full
parameters: |
{
"hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
"outdir": "${{ secrets.TOWER_BUCKET_GCP }}/fetchngs/results-${{ github.sha }}"
}
- uses: actions/upload-artifact@v3
with:
name: Tower debug log file
path: tower_action_*.log
run-full-tests-on-azure:
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'azure' || !github.event.inputs }}
runs-on: ubuntu-latest
steps:
- uses: seqeralabs/action-tower-launch@v1
with:
workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
compute_env: ${{ secrets.TOWER_CE_AZURE_CPU }}
workdir: "${{ secrets.TOWER_BUCKET_AZURE }}/work/fetchngs/work-${{ github.sha }}"
run_name: "azure_fetchngs_full"
profiles: test_full
parameters: |
{
"hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
"outdir": "${{ secrets.TOWER_BUCKET_AZURE }}/fetchngs/results-${{ github.sha }}"
}
- uses: actions/upload-artifact@v3
with:
name: Tower debug log file
path: tower_action_*.log
76 changes: 76 additions & 0 deletions .github/workflows/cloud_tests_small.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
name: small-sized tests on cloud providers
run-name: Submitting workflow to all cloud providers using small sized data
on:
workflow_dispatch:
inputs:
platform:
description: "Platform to run test"
required: true
default: "all"
type: choice
options:
- all
- aws
- azure
- gcp
jobs:
run-small-tests-on-aws:
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'aws' }}
runs-on: ubuntu-latest
steps:
- uses: seqeralabs/action-tower-launch@v1
with:
workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
compute_env: ${{ secrets.TOWER_CE_AWS_CPU }}
workdir: "${{ secrets.TOWER_BUCKET_AWS }}/work/fetchngs/work-${{ github.sha }}"
run_name: "aws_fetchngs_small"
profiles: test
parameters: |
{
"outdir": "${{ secrets.TOWER_BUCKET_AWS }}/fetchngs/results-test-${{ github.sha }}"
}
- uses: actions/upload-artifact@v3
with:
name: Tower debug log file
path: tower_action_*.log
run-small-tests-on-gcp:
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'gcp' }}
runs-on: ubuntu-latest
steps:
- uses: seqeralabs/action-tower-launch@v1
with:
workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
compute_env: ${{ secrets.TOWER_CE_GCP_CPU }}
workdir: "${{ secrets.TOWER_BUCKET_GCP }}/work/fetchngs/work-${{ github.sha }}"
run_name: "gcp_fetchngs_small"
profiles: test
parameters: |
{
"outdir": "${{ secrets.TOWER_BUCKET_GCP }}/fetchngs/results-test-${{ github.sha }}"
}
- uses: actions/upload-artifact@v3
with:
name: Tower debug log file
path: tower_action_*.log
run-small-tests-on-azure:
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'azure' }}
runs-on: ubuntu-latest
steps:
- uses: seqeralabs/action-tower-launch@v1
with:
workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
compute_env: ${{ secrets.TOWER_CE_AZURE_CPU }}
workdir: "${{ secrets.TOWER_BUCKET_AZURE }}/work/fetchngs/work-${{ github.sha }}"
run_name: "azure_fetchngs_small"
profiles: test
parameters: |
{
"outdir": "${{ secrets.TOWER_BUCKET_AZURE }}/fetchngs/results-test-${{ github.sha }}"
}
- uses: actions/upload-artifact@v3
with:
name: Tower debug log file
path: tower_action_*.log
11 changes: 6 additions & 5 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,13 @@ Thank you to everyone else that has contributed by reporting bugs, enhancements

### Enhancements & fixes

- [#85](https://github.com/nf-core/fetchngs/issues/85) - Not able to fetch metadata for ERR\* ids associated with ArrayExpress
- [#129](https://github.com/nf-core/fetchngs/issues/129) - Pipeline is working with SRA run IDs, but failing with corresponding Biosample IDs
- [#85](https://github.com/nf-core/fetchngs/issues/85) - Not able to fetch metadata for ERR ids associated with ArrayExpress
- [#129](https://github.com/nf-core/fetchngs/issues/129) - Pipeline is working with SRA run ids but failing with corresponding Biosample ids
- [#144](https://github.com/nf-core/fetchngs/issues/144) - Add support to download 10X Genomics data
- [#148](https://github.com/nf-core/fetchngs/pull/148) - Fix default metadata fields for ENA API v2.0
- Bumped modules version to allow for sratools download of sralite format files.
- Updated pipeline template to [nf-core/tools 2.8](https://github.com/nf-core/tools/releases/tag/2.8)
- [PR #140](https://github.com/nf-core/fetchngs/pull/140) - Bumped modules version to allow for sratools download of sralite format files
- [PR #147](https://github.com/nf-core/fetchngs/pull/147) - Updated pipeline template to [nf-core/tools 2.8](https://github.com/nf-core/tools/releases/tag/2.8)
- [PR #148](https://github.com/nf-core/fetchngs/pull/148) - Fix default metadata fields for ENA API v2.0
- [PR #150](https://github.com/nf-core/fetchngs/pull/150) - Add infrastructure and CI for multi-cloud full-sized tests run via Nextflow Tower

### Software dependencies

Expand Down

0 comments on commit e90a023

Please sign in to comment.