Merge pull request #909 from maxulysse/cache_test_data

Cache test data on GHA
nf-core · Jan 12, 2023 · 9457e12 · 9457e12
2 parents e675387 + 6337c08
commit 9457e12
Show file tree

Hide file tree

Showing 7 changed files with 68 additions and 2 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -30,10 +30,35 @@ jobs:
         profile: ["docker", "singularity", "conda"]
     env:
       NXF_ANSI_LOG: false
+      TEST_DATA_BASE: "${{ github.workspace }}/test-datasets"
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v3
 
+      - name: Cache test data
+        id: cache-testdata
+        uses: actions/cache@v3
+        with:
+          path: test-datasets/
+          key: sarek3-test-data
+
+      - name: Check out test data
+        if: steps.cache-testdata.outputs.cache-hit != 'true'
+        uses: actions/checkout@v3
+        with:
+          repository: nf-core/test-datasets
+          ref: sarek3
+          path: test-datasets/
+
+      - name: Replace remote paths in samplesheets
+        run: |
+          for f in tests/csv/3.0/*csv; do
+            sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/modules/=${{ github.workspace }}/test-datasets/=g" $f
+            echo "========== $f ============"
+            cat $f
+            echo "========================================"
+          done;
+
       - name: Set up Python
         uses: actions/setup-python@v2
         with:

diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml
@@ -33,6 +33,8 @@ jobs:
       matrix:
         tags: ["${{ fromJson(needs.changes.outputs.tags) }}"]
         profile: ["docker", "singularity", "conda"]
+        TEST_DATA_BASE:
+          - "test-datasets/data"
         NXF_VER:
           - "22.10.1"
           - "latest-everything"
@@ -55,10 +57,35 @@ jobs:
             tags: merge
     env:
       NXF_ANSI_LOG: false
+      TEST_DATA_BASE: "${{ github.workspace }}/test-datasets"
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v3
 
+      - name: Cache test data
+        id: cache-testdata
+        uses: actions/cache@v3
+        with:
+          path: test-datasets/
+          key: sarek3-test-data
+
+      - name: Check out test data
+        if: steps.cache-testdata.outputs.cache-hit != 'true'
+        uses: actions/checkout@v3
+        with:
+          repository: nf-core/test-datasets
+          ref: sarek3
+          path: test-datasets/
+
+      - name: Replace remote paths in samplesheets
+        run: |
+          for f in tests/csv/3.0/*csv; do
+            sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/modules/=${{ github.workspace }}/test-datasets/=g" $f
+            echo "========== $f ============"
+            cat $f
+            echo "========================================"
+          done;
+
       - name: Set up Python
         uses: actions/setup-python@v2
         with:

diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,4 @@ results/
 testing/
 testing*
 *.pyc
+test-datasets/
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#890](https://github.com/nf-core/sarek/pull/890) - Sync `TEMPLATE` with `tools` `2.7.1`
 - [#898](https://github.com/nf-core/sarek/pull/898) - Sync `TEMPLATE` with `tools` `2.7.2`
 - [#898](https://github.com/nf-core/sarek/pull/898) - Nextflow minimal version is now `22.10.1`
+- [#909](https://github.com/nf-core/sarek/pull/909) - Cache test data on GHA
 
 ### Fixed
 

diff --git a/conf/test/test.config b/conf/test/test.config
@@ -9,6 +9,12 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 */
 
+if (System.getenv('TEST_DATA_BASE')) {
+    if ("$TEST_DATA_BASE") {
+        params.test_data_base = "$TEST_DATA_BASE"
+    }
+}
+
 try {
     includeConfig "https://raw.githubusercontent.com/nf-core/modules/master/tests/config/test_data.config"
 } catch (Exception e) {

diff --git a/nextflow.config b/nextflow.config
@@ -108,15 +108,14 @@ params {
     show_hidden_params         = false
     schema_ignore_params       = 'genomes,snpeff_version,vep_version,cf_ploidy'
 
-
     // Config options
     custom_config_version      = 'master'
     custom_config_base         = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}"
     config_profile_description = null
     config_profile_contact     = null
     config_profile_url         = null
     config_profile_name        = null
-
+    test_data_base             = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek3'
 
     // Max resource options
     // Defaults only, expecting to be overwritten

diff --git a/nextflow_schema.json b/nextflow_schema.json
@@ -788,6 +788,13 @@
                     "hidden": true,
                     "fa_icon": "fas fa-users-cog"
                 },
+                "test_data_base": {
+                    "type": "string",
+                    "default": "https://raw.githubusercontent.com/nf-core/test-datasets/sarek3",
+                    "description": "Base path / URL for data used in the test profiles",
+                    "help_text": "Warning: The `-profile test` samplesheet file itself contains remote paths. Setting this parameter does not alter the contents of that file.",
+                    "hidden": true
+                },
                 "seq_center": {
                     "type": "string",
                     "fa_icon": "fas fa-university",