Refactor project structure (#5)

* move components

* fix config

* update readme

* fix dependencies after refactor

* add build ci script

* add test ci script

* add dependabot

* refactor generation script

* change metric output to anndata

* wip add unit test to components

* fix resources path

* also run method and metric in resource script

* remove unneeded scripts

* update paths

* fix metric

* add comment

* fix example path

* add file types to api files

* improve unit test

* finetune sync

* fix api file

* add changelog and PR template

* don't publish h5ad files

* rename zeros component

* fix script

* add benchmark wf

* fix workflows, api and scripts

* add download resources script

* update readme

* add missing metric info

* add missing dataset_summary

* remove trace before run

* fix missing dataset_id

* move tracing

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,17 @@
+## Describe your changes
+
+<!-- Describe your changes  -->
+
+## Checklist before requesting a review
+- [ ] I have performed a self-review of my code
+
+- Check the correct box. Does this PR contain:
+  - [ ] Breaking changes
+  - [ ] New functionality (new method, new metric, ...)
+  - [ ] Major changes
+  - [ ] Minor changes
+  - [ ] Bug fixes
+
+- [ ] Proposed changes are described in the CHANGELOG.md
+
+- [ ] CI Tests succeed and look good!

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/build.yml

@@ -0,0 +1,93 @@
+name: build
+
+on:
+  push:
+    branches: [ 'main' ]
+
+jobs:
+  # phase 1
+  list:
+    runs-on: ubuntu-latest
+
+    outputs:
+      component_matrix: ${{ steps.set_matrix.outputs.matrix }}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: viash-io/viash-actions/setup@v5
+
+    - name: Remove target folder from .gitignore
+      run: |
+        # allow publishing the target folder
+        sed -i '/^target.*/d' .gitignore
+
+    - uses: viash-io/viash-actions/ns-build@v5
+      with:
+        config_mod: .functionality.version := 'main_build'
+        parallel: true
+
+    - name: Deploy to target branch
+      uses: peaceiris/actions-gh-pages@v3
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        publish_dir: .
+        publish_branch: main_build
+
+    - id: ns_list
+      uses: viash-io/viash-actions/ns-list@v5
+      with:
+        platform: docker
+        src: src
+        format: json
+
+    - id: set_matrix
+      run: |
+        echo "matrix=$(jq -c '[ .[] | 
+          { 
+            "name": (.functionality.namespace + "/" + .functionality.name),
+            "dir": .info.config | capture("^(?<dir>.*\/)").dir
+          }
+        ]' ${{ steps.ns_list.outputs.output_file }} )" >> $GITHUB_OUTPUT
+
+  # phase 2
+  build:
+    needs: list
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        component: ${{ fromJson(needs.list.outputs.component_matrix) }}
+
+    steps:
+    # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
+    - uses: data-intuitive/reclaim-the-bytes@v2
+
+    - uses: actions/checkout@v4
+
+    - uses: viash-io/viash-actions/setup@v5
+
+    - name: Build container
+      uses: viash-io/viash-actions/ns-build@v5
+      with:
+        config_mod: .functionality.version := 'main_build'
+        platform: docker
+        src: ${{ matrix.component.dir }}
+        setup: build
+
+    - name: Login to container registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ secrets.GTHB_USER }}
+        password: ${{ secrets.GTHB_PAT }}
+
+    - name: Push container
+      uses: viash-io/viash-actions/ns-build@v5
+      with:
+        config_mod: .functionality.version := 'main_build'
+        platform: docker
+        src: ${{ matrix.component.dir }}
+        setup: push

.github/workflows/test.yml

@@ -0,0 +1,107 @@
+name: test
+
+on:
+  pull_request:
+  push:
+    branches: [ '**' ]
+
+jobs:
+  run_ci_check_job:
+    runs-on: ubuntu-latest
+    outputs:
+      run_ci: ${{ steps.github_cli.outputs.check }}
+    steps:
+      - name: 'Check if branch has an existing pull request and the trigger was a push'
+        id: github_cli
+        run: |
+          pull_request=$(gh pr list -R ${{ github.repository }} -H ${{ github.ref_name }} --json url --state open --limit 1 | jq '.[0].url')
+          # If the branch has a PR and this run was triggered by a push event, do not run
+          if [[ "$pull_request" != "null" && "$GITHUB_REF_NAME" != "main" && "${{ github.event_name == 'push' }}" == "true" && "${{ !contains(github.event.head_commit.message, 'ci force') }}" == "true" ]]; then
+            echo "check=false" >> $GITHUB_OUTPUT
+          else
+            echo "check=true" >> $GITHUB_OUTPUT
+          fi
+        env:
+          GITHUB_TOKEN: ${{ secrets.GTHB_PAT }}
+
+  # phase 1
+  list:
+    needs: run_ci_check_job
+    env:
+      s3_bucket: s3://openproblems-bio/public/neurips-2023-competition/workflow-resources/
+    runs-on: ubuntu-latest
+    if: ${{ needs.run_ci_check_job.outputs.run_ci == 'true' }}
+
+    outputs:
+      matrix: ${{ steps.set_matrix.outputs.matrix }}
+      cache_key: ${{ steps.cache.outputs.cache_key }}
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0 
+
+    - uses: viash-io/viash-actions/setup@v5
+
+    - uses: viash-io/viash-actions/project/sync-and-cache-s3@v5
+      id: cache
+      with:
+        s3_bucket: $s3_bucket
+        dest_path: resources
+        cache_key_prefix: resources__
+
+    - id: ns_list
+      uses: viash-io/viash-actions/ns-list@v5
+      with:
+        platform: docker
+        format: json
+
+    - id: ns_list_filtered
+      uses: viash-io/viash-actions/project/detect-changed-components@v5
+      with:
+        input_file: "${{ steps.ns_list.outputs.output_file }}"
+
+    - id: set_matrix
+      run: |
+        echo "matrix=$(jq -c '[ .[] | 
+          { 
+            "name": (.functionality.namespace + "/" + .functionality.name),
+            "config": .info.config
+          }
+        ]' ${{ steps.ns_list_filtered.outputs.output_file }} )" >> $GITHUB_OUTPUT
+
+  # phase 2
+  viash_test:
+    needs: list
+    if: ${{ needs.list.outputs.matrix != '[]' && needs.list.outputs.matrix != '' }}
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        component: ${{ fromJson(needs.list.outputs.matrix) }}
+
+    steps:
+    # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
+    - uses: data-intuitive/reclaim-the-bytes@v2
+
+    - uses: actions/checkout@v4
+
+    - uses: viash-io/viash-actions/setup@v5
+
+    # use cache
+    - name: Cache resources data
+      uses: actions/cache@v4
+      timeout-minutes: 10
+      with:
+        path: resources
+        key: ${{ needs.list.outputs.cache_key }}
+
+    - name: Run test
+      timeout-minutes: 30
+      run: |
+        VIASH_TEMP=$RUNNER_TEMP/viash viash test \
+          "${{ matrix.component.config }}" \
+          --cpus 2 \
+          --memory "5gb"
+

.gitignore

@@ -4,4 +4,6 @@ work
 target
 .idea
 .vscode
-.DS_Store
+.DS_Store
+output
+trace-*

CHANGELOG.md

@@ -0,0 +1,10 @@
+# task-dge-perturbation-prediction 0.1.0
+
+Initial release of the DGE Perturbation Prediction task. Initial components:
+
+* `src/task/process_dataset`: Compute the DGE data from the raw single-cell counts using Limma.
+* `src/task/control_methods`: Baseline control methods: sample, ground_truth, baseline_zero.
+* `src/task/methods`: DGE perturbation prediction methods: random_forest.
+* `src/task/metrics`: Evaluation metrics: mean_rowwise_rmse.
+
+