From 025e57bea8f98eeb1bdc0ac977a874cb1b05b77b Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 29 Sep 2022 16:29:34 +0200 Subject: [PATCH 01/29] make workflows reusable --- .github/workflows/ci.yaml | 198 ++------------- .github/workflows/job_deploy.yaml | 64 +++++ .github/workflows/job_lint.yaml | 216 ++++++++++++++++ .github/workflows/job_setup.yaml | 155 +++++++++++ .github/workflows/job_test.yaml | 181 +++++++++++++ .github/workflows/pr.yaml | 409 +++--------------------------- .github/workflows/slash.yaml | 2 +- 7 files changed, 677 insertions(+), 548 deletions(-) create mode 100644 .github/workflows/job_deploy.yaml create mode 100644 .github/workflows/job_lint.yaml create mode 100644 .github/workflows/job_setup.yaml create mode 100644 .github/workflows/job_test.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d3aedae834e..081d2b1bfbe 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,189 +5,27 @@ on: - cron: '0 0 * * 1' repository_dispatch: types: [run-all-tool-tests-command] -env: - GALAXY_FORK: galaxyproject - GALAXY_BRANCH: release_22.05 - MAX_CHUNKS: 40 jobs: setup: name: Setup cache and determine changed repositories - if: ${{ github.repository_owner == 'galaxyproject' }} - runs-on: ubuntu-latest - outputs: - galaxy-head-sha: ${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} - fork: ${{ steps.get-fork-branch.outputs.fork }} - branch: ${{ steps.get-fork-branch.outputs.branch }} - repository-list: ${{ steps.discover.outputs.repository-list }} - chunk-count: ${{ steps.discover.outputs.chunk-count }} - chunk-list: ${{ steps.discover.outputs.chunk-list }} - strategy: - matrix: - python-version: ['3.7'] - steps: - - name: Add reaction - if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }} - uses: peter-evans/create-or-update-comment@v1 - with: - token: ${{ secrets.PAT }} - repository: ${{ github.event.client_payload.github.payload.repository.full_name }} - comment-id: ${{ github.event.client_payload.github.payload.comment.id }} - reaction-type: hooray - - name: Set galaxy fork and branch - id: get-fork-branch - run: | - TMP="${{ github.event.client_payload.slash_command.args.named.fork }}" - echo "::set-output name=fork::${TMP:-$GALAXY_FORK}" - TMP="${{ github.event.client_payload.slash_command.args.named.branch }}" - echo "::set-output name=branch::${TMP:-$GALAXY_BRANCH}" - - name: Determine latest commit in the Galaxy repo - id: get-galaxy-sha - run: echo "::set-output name=galaxy-head-sha::$(git ls-remote https://github.com/${{ steps.get-fork-branch.outputs.fork }}/galaxy refs/heads/${{ steps.get-fork-branch.outputs.branch }} | cut -f1)" - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Cache .cache/pip - uses: actions/cache@v2 - id: cache-pip - with: - path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} - # Install the `wheel` package so that when installing other packages which - # are not available as wheels, pip will build a wheel for them, which can be cached. - - name: Install wheel - run: pip install wheel - - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks - uses: galaxyproject/planemo-ci-action@v1 - id: discover - with: - create-cache: ${{ steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' }} - galaxy-fork: ${{ steps.get-fork-branch.outputs.fork }} - galaxy-branch: ${{ steps.get-fork-branch.outputs.branch }} - max-chunks: ${{ env.MAX_CHUNKS }} - python-version: ${{ matrix.python-version }} - - name: Show repository list - run: echo '${{ steps.discover.outputs.repository-list }}' - - name: Show chunks - run: | - echo 'Using ${{ steps.discover.outputs.chunk-count }} chunks (${{ steps.discover.outputs.chunk-list }})' - + uses: bernt-matthias/tools-iuc/.github/workflows/job_setup.yaml@master + with: + # default-galaxy-fork: galaxyproject + # default-galaxy-branch: release_22.05 + max-chunks: 40 + secrets: + PAT: ${{ secrets.PAT }} test: name: Test tools - # This job runs on Linux - runs-on: ubuntu-latest needs: setup - if: ${{ needs.setup.outputs.repository-list != '' }} - strategy: - fail-fast: false - matrix: - chunk: ${{ fromJson(needs.setup.outputs.chunk-list) }} - python-version: ['3.7'] - services: - postgres: - image: postgres:11 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - ports: - - 5432:5432 - steps: - # checkout the repository - # and use it as the current working directory - - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Cache .cache/pip - uses: actions/cache@v2 - id: cache-pip - with: - path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v1 - id: cpu-cores - - name: Planemo test - uses: galaxyproject/planemo-ci-action@v1 - id: test - with: - mode: test - repository-list: ${{ needs.setup.outputs.repository-list }} - galaxy-fork: ${{ needs.setup.outputs.fork }} - galaxy-branch: ${{ needs.setup.outputs.branch }} - chunk: ${{ matrix.chunk }} - chunk-count: ${{ needs.setup.outputs.chunk-count }} - galaxy-slots: ${{ steps.cpu-cores.outputs.count }} - # Limit each test to 15 minutes - test_timeout: 900 - - uses: actions/upload-artifact@v2 - with: - name: 'Tool test output ${{ matrix.chunk }}' - path: upload - - # - combine the results of the test chunks (which will never fail due - # to `|| true`) and create a global test report as json and html which - # is provided as artifact - # - check if any tool test actually failed (by lookup in the combined json) - # and fail this step if this is the case - combine_outputs: - name: Combine chunked test results - needs: [setup, test] - strategy: - matrix: - python-version: ['3.7'] - # This job runs on Linux - runs-on: ubuntu-latest - steps: - - uses: actions/download-artifact@v2 - with: - path: artifacts - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Cache .cache/pip - uses: actions/cache@v2 - id: cache-pip - with: - path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - name: Combine outputs - uses: galaxyproject/planemo-ci-action@v1 - id: combine - with: - mode: combine - html-report: true - - uses: actions/upload-artifact@v2 - with: - name: 'All tool test results' - path: upload - - name: Create URL to the run output - if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }} - id: vars - run: echo "::set-output name=run-url::https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" - - - name: Create comment - if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }} - uses: peter-evans/create-or-update-comment@v1 - with: - token: ${{ secrets.PAT }} - repository: ${{ github.event.client_payload.github.payload.repository.full_name }} - issue-number: ${{ github.event.client_payload.github.payload.issue.number }} - body: | - Summary: - - ${{ steps.combine.outputs.statistics }} - - [Find all tool test results here][1] - - [1]: ${{ steps.vars.outputs.run-url }} - - name: Check outputs - uses: galaxyproject/planemo-ci-action@v1 - id: check - with: - mode: check + uses: bernt-matthias/tools-iuc/.github/workflows/job_test.yaml@master + with: + galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }} + galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }} + repository-list: ${{ needs.setup.outputs.repository-list }} + chunk-count: ${{ needs.setup.outputs.chunk-count }} + chunk-list: ${{ needs.setup.outputs.chunk-list }} + galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} + test-timeout: 900 + secrets: + PAT: ${{ secrets.PAT }} diff --git a/.github/workflows/job_deploy.yaml b/.github/workflows/job_deploy.yaml new file mode 100644 index 00000000000..afa0a1b681f --- /dev/null +++ b/.github/workflows/job_deploy.yaml @@ -0,0 +1,64 @@ +name: Deploy Tools +on: + workflow_call: + inputs: + galaxy-head-sha: + description: 'hash of the latest commit in the Galaxy repo' + required: true + type: string + repository-list: + description: 'list of repositories to deploy' + required: true + type: string + repository-owner: + description: 'owner, ie main fork, of the repository (deploy will only run for this fork)' + required: true + type: string + python-version-list: + description: 'Python versions (stringified JSON array)' + default: "[\"3.7\"]" + required: false + type: string + secrets: + TTS_API_KEY: + required: true + TS_API_KEY: + required: true + +jobs: + # deploy the tools to the toolsheds (first TTS for testing) + deploy: + name: Deploy + if: ${{ (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' ) && github.repository_owner == inputs.repository-owner }} + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ${{ fromJson(inputs.python-version-list) }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Cache .cache/pip + uses: actions/cache@v2 + id: cache-pip + with: + path: ~/.cache/pip + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} + - name: Deploy on testtoolshed + uses: galaxyproject/planemo-ci-action@v1 + with: + mode: deploy + repository-list: ${{ inputs.repository-list }} + shed-target: testtoolshed + shed-key: ${{ secrets.TTS_API_KEY }} + continue-on-error: true + - name: Deploy on toolshed + uses: galaxyproject/planemo-ci-action@v1 + with: + mode: deploy + repository-list: ${{ inputs.repository-list }} + shed-target: toolshed + shed-key: ${{ secrets.TS_API_KEY }} \ No newline at end of file diff --git a/.github/workflows/job_lint.yaml b/.github/workflows/job_lint.yaml new file mode 100644 index 00000000000..d8f051bf52e --- /dev/null +++ b/.github/workflows/job_lint.yaml @@ -0,0 +1,216 @@ +name: Lint +on: + workflow_call: + inputs: + repository-list: + description: 'list of repositories to lint' + default: '' + required: false + type: string + tool-list: + description: 'list of tools to lint' + default: '' + required: false + type: string + galaxy-head-sha: + description: 'hash of the latest commit in the Galaxy repo' + required: true + type: string + commit-range: + description: 'commit range to test' + default: '' + required: false + type: string + max-file-size: + description: 'maximum file size' + default: "1M" + required: false + type: string + python-version-list: + description: 'stringified JSON array of Python versions' + default: "[\"3.7\"]" + required: false + type: string + r-version-list: + description: 'stringified JSON array of R versions' + default: "[\"4.0.1\"]" + required: false + type: string + os-version-list: + description: 'stringified JSON array of OS versions' + default: "[\"ubuntu-20.04\"]" + required: false + type: string + outputs: + result: + description: "Result of the lint jobs" + value: ${{ jobs.lint-success.result }} + +jobs: + lint: + name: Tools + if: ${{ inputs.repository-list != '' || inputs.tool-list != '' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJson(inputs.python-version-list) }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Cache .cache/pip + uses: actions/cache@v2 + id: cache-pip + with: + path: ~/.cache/pip + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} + - name: Planemo lint + uses: galaxyproject/planemo-ci-action@v1 + id: lint + with: + mode: lint + repository-list: ${{ inputs.repository-list }} + tool-list: ${{ inputs.tool-list }} + - uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + name: 'Tool linting output' + path: lint_report.txt + + flake8: + name: Python scripts + if: ${{ inputs.repository-list != '' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJson(inputs.python-version-list) }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Cache .cache/pip + uses: actions/cache@v2 + id: cache-pip + with: + path: ~/.cache/pip + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} + - name: Install flake8 + run: pip install flake8 flake8-import-order + - name: Flake8 + run: echo '${{ inputs.repository-list }}' | xargs -d '\n' flake8 --output-file pylint_report.txt --tee + - uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + name: 'Python linting output' + path: pylint_report.txt + + lintr: + name: R scripts + if: ${{ inputs.repository-list != '' }} + strategy: + matrix: + os: ${{ fromJson(inputs.os-version-list) }} + r-version: ${{ fromJson(inputs.r-version-list) }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: r-lib/actions/setup-r@master + with: + r-version: ${{ matrix.r-version }} + - name: Cache R packages + uses: actions/cache@v2 + with: + path: ${{ env.R_LIBS_USER }} + key: r_cache_${{ matrix.os }}_${{ matrix.r-version }} + - name: Install non-R lintr dependencies + run: sudo apt-get install libcurl4-openssl-dev + - name: Install lintr + run: | + install.packages('remotes') + remotes::install_cran("lintr") + shell: Rscript {0} + - name: Save repositories to file + run: echo '${{ inputs.repository-list }}' > repository_list.txt + - name: lintr + run: | + library(lintr) + linters <- linters_with_defaults(line_length_linter = NULL, cyclocomp_linter = NULL, object_usage_linter = NULL) + con <- file("repository_list.txt", "r") + status <- 0 + while (TRUE) { + repo <- readLines(con, n = 1) + if (length(repo) == 0) { + break + } + lnt <- lint_dir(repo, relative_path=T, linters=linters) + if (length(lnt) > 0) { + status <- 1 + for (l in lnt) { + rel_path <- paste(repo, l$filename, sep="/") + write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message, paste("(", l$line, ")")), stderr()) + write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message, paste("(", l$line, ")")), "rlint_report.txt", append=TRUE) + } + } + } + quit(status = status) + shell: Rscript {0} + - uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + name: 'R linting output' + path: rlint_report.txt + + file_sizes: + name: Check file sizes + if: ${{ github.event_name == 'pull_request' && inputs.repository-list != '' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Check file sizes + run: | + touch file_size_report.txt + git diff --diff-filter=d --name-only ${{ inputs.commit-range }} > git.diff + while read line; do + find "$line" -type f -size +${{ inputs.max-file-size }} >> file_size_report.txt + done < git.diff + if [[ -s file_size_report.txt ]]; then + echo "Files larger than ${{ inputs.max-file-size }} found" + cat file_size_report.txt + exit 1 + fi + - uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + name: 'File size report' + path: file_size_report.txt + + lint-success: + name: Determine lint success + needs: [lint, flake8, lintr, file_sizes] + if: ${{ always() && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/main' }} + runs-on: ubuntu-latest + steps: + - name: Check tool lint status + if: ${{ needs.lint.result != 'success' && needs.flake8.result != 'skipped' }} + run: exit 1 + - name: Indicate Python script lint status + if: ${{ needs.flake8.result != 'success' && needs.flake8.result != 'skipped' }} + run: exit 1 + - name: Indicate R script lint status + if: ${{ needs.lintr.result != 'success' && needs.lintr.result != 'skipped' }} + run: exit 1 + - name: Indicate file size check status + if: ${{ needs.file_sizes.result != 'success' && needs.file_sizes.result != 'skipped' }} + run: exit 1 diff --git a/.github/workflows/job_setup.yaml b/.github/workflows/job_setup.yaml new file mode 100644 index 00000000000..395e311e7e7 --- /dev/null +++ b/.github/workflows/job_setup.yaml @@ -0,0 +1,155 @@ +name: Test tools +on: + workflow_call: + inputs: + default-galaxy-fork: + description: 'Default Galaxy fork to use' + default: 'galaxyproject' + required: false + type: string + default-galaxy-branch: + description: 'Default Galaxy branch to use' + default: 'release_22.05' + required: false + type: string + max-chunks: + description: 'maximum number of chunks to use' + required: true + type: number + fetch-depth: + description: 'Number of commits to fetch. 0 indicates all history for all branches and tags' + default: 1 + required: false + type: number + python-version-list: + description: 'Python versions (stringified JSON array)' + default: "[\"3.7\"]" + required: false + type: string + secrets: + PAT: + required: false + outputs: + galaxy-head-sha: + description: "hash of the latest commit in the Galaxy repo" + value: ${{ jobs.setup.outputs.galaxy-head-sha }} + repository-list: + description: "list of repositories to test" + value: ${{ jobs.setup.outputs.repository-list }} + tool-list: + description: "list of tools to test" + value: ${{ jobs.setup.outputs.tool-list }} + chunk-count: + description: "number of chunks" + value: ${{ jobs.setup.outputs.chunk-count }} + chunk-list: + description: "list of chunks (stringified JSON array)" + value: ${{ jobs.setup.outputs.chunk-list }} + commit-range: + description: "commit range to test" + value: ${{ jobs.setup.outputs.commit-range }} + galaxy-fork: + description: "The determined galaxy fork to use" + value: ${{ jobs.fork-branch.outputs.galaxy-fork }} + galaxy-branch: + description: "The determined galaxy branch to use" + value: ${{ jobs.fork-branch.outputs.galaxy-branch }} +jobs: + fork-branch: + name: Determine Galaxy fork and branch to use + runs-on: ubuntu-latest + outputs: + galaxy-fork: ${{ steps.get-fork-branch.outputs.fork }} + galaxy-branch: ${{ steps.get-fork-branch.outputs.branch }} + steps: + - name: Add reaction + if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }} + uses: peter-evans/create-or-update-comment@v1 + with: + token: ${{ secrets.PAT }} + repository: ${{ github.event.client_payload.github.payload.repository.full_name }} + comment-id: ${{ github.event.client_payload.github.payload.comment.id }} + reaction-type: hooray + - name: Set galaxy fork and branch + id: get-fork-branch + run: | + SLASH_FORK="${{ github.event.client_payload.slash_command.args.named.fork }}" + DEFAULT_FORK="${{ inputs.default-galaxy-fork }}" + echo "::set-output name=fork::${SLASH_FORK:-$DEFAULT_FORK}" + SLASH_BRANCH="${{ github.event.client_payload.slash_command.args.named.branch }}" + DEFAULT_BRANCH="${{ inputs.default-galaxy-branch }}" + echo "::set-output name=branch::${SLASH_BRANCH:-$DEFAULT_BRANCH}" + - name: Print used Galaxy for and branch + run: | + echo 'fork: ${{ steps.get-fork-branch.outputs.fork }}' + echo 'branch: ${{ steps.get-fork-branch.outputs.branch }}' + setup: + name: Setup cache and determine changed repositories + needs: fork-branch + runs-on: ubuntu-latest + outputs: + galaxy-head-sha: ${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} + repository-list: ${{ steps.discover.outputs.repository-list }} + tool-list: ${{ steps.discover.outputs.tool-list }} + chunk-count: ${{ steps.discover.outputs.chunk-count }} + chunk-list: ${{ steps.discover.outputs.chunk-list }} + commit-range: ${{ steps.discover.outputs.commit-range }} + strategy: + matrix: + python-version: ${{ fromJson(inputs.python-version-list) }} + steps: + - name: Print github context properties + run: | + echo 'event: ${{ github.event_name }}' + echo 'sha: ${{ github.sha }}' + echo 'ref: ${{ github.ref }}' + echo 'head_ref: ${{ github.head_ref }}' + echo 'base_ref: ${{ github.base_ref }}' + echo 'event.before: ${{ github.event.before }}' + echo 'event.after: ${{ github.event.after }}' + - name: Determine latest commit in the Galaxy repo + id: get-galaxy-sha + run: echo "::set-output name=galaxy-head-sha::$(git ls-remote https://github.com/${{ needs.fork-branch.outputs.galaxy-fork }}/galaxy refs/heads/${{ needs.fork-branch.outputs.galaxy-branch }} | cut -f1)" + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Cache .cache/pip + uses: actions/cache@v2 + id: cache-pip + with: + path: ~/.cache/pip + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} + - name: Cache .planemo + uses: actions/cache@v2 + id: cache-planemo + with: + path: ~/.planemo + key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} + # Install the `wheel` package so that when installing other packages which + # are not available as wheels, pip will build a wheel for them, which can be cached. + - name: Install wheel + run: pip install wheel + - name: Install flake8 + run: pip install flake8 flake8-import-order + - uses: actions/checkout@v2 + with: + # TODO make configurable CI uses 1 + fetch-depth: ${{ inputs.fetch-depth }} + - name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks + uses: galaxyproject/planemo-ci-action@v1 + id: discover + with: + create-cache: ${{ steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' }} + galaxy-fork: ${{ needs.fork-branch.outputs.galaxy-fork }} + galaxy-branch: ${{ needs.fork-branch.outputs.galaxy-branch }} + max-chunks: ${{ inputs.max-chunks }} + python-version: ${{ matrix.python-version }} + - name: Show commit range + run: echo '${{ steps.discover.outputs.commit-range }}' + - name: Show repository list + run: echo '${{ steps.discover.outputs.repository-list }}' + - name: Show tool list + run: echo '${{ steps.discover.outputs.tool-list }}' + - name: Show chunks + run: | + echo 'Using ${{ steps.discover.outputs.chunk-count }} chunks (${{ steps.discover.outputs.chunk-list }})' \ No newline at end of file diff --git a/.github/workflows/job_test.yaml b/.github/workflows/job_test.yaml new file mode 100644 index 00000000000..70ecd2b1e14 --- /dev/null +++ b/.github/workflows/job_test.yaml @@ -0,0 +1,181 @@ +name: Test tools +on: + workflow_call: + inputs: + python-version-list: + description: 'Python versions (stringified JSON array)' + default: "[\"3.7\"]" + required: false + type: string + chunk-count: + description: 'number of chunks' + required: true + type: string + chunk-list: + description: 'list of chunks (stringified JSON array)' + required: true + type: string + repository-list: + description: 'list of repositories to test' + required: true + type: string + galaxy-head-sha: + description: 'hash of the latest commit in the Galaxy repo' + required: true + type: string + galaxy-fork: + description: 'Galaxy fork to use' + required: true + type: string + galaxy-branch: + description: 'Galaxy branch to use' + required: true + type: string + cache-planemo: + description: 'cache the .planemo directory' + default: true + required: false + type: boolean + test-timeout: + description: 'Timeout in seconds per test' + default: 900 + required: false + type: number + secrets: + PAT: + required: false + outputs: + result: + description: "Result of the testing" + value: ${{ jobs.test-success.result }} + +jobs: + # Planemo test the changed repositories, each chunk creates an artifact + # containing HTML and JSON reports for the executed tests + test: + name: Test tools + if: ${{ inputs.repository-list != '' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + chunk: ${{ fromJson(inputs.chunk-list) }} + python-version: ${{ fromJson(inputs.python-version-list) }} + services: + postgres: + image: postgres:11 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Cache .cache/pip + uses: actions/cache@v2 + id: cache-pip + with: + path: ~/.cache/pip + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} + - name: Cache .planemo + if: ${{ inputs.cache-planemo }} + uses: actions/cache@v2 + id: cache-planemo + with: + path: ~/.planemo + key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v1 + id: cpu-cores + - name: Planemo test + uses: galaxyproject/planemo-ci-action@v1 + id: test + with: + mode: test + repository-list: ${{ inputs.repository-list }} + galaxy-fork: ${{ inputs.galaxy-fork }} + galaxy-branch: ${{ inputs.galaxy-branch }} + chunk: ${{ matrix.chunk }} + chunk-count: ${{ inputs.chunk-count }} + galaxy-slots: ${{ steps.cpu-cores.outputs.count }} + # Limit each test to 15 minutes + test_timeout: ${{ inputs.test-timeout }} + - uses: actions/upload-artifact@v2 + with: + name: 'Tool test output ${{ matrix.chunk }}' + path: upload + + # - combine the results of the test chunks (which will never fail due + # to `|| true`) and create a global test report as json and html which + # is provided as artifact + # - check if any tool test actually failed (by lookup in the combined json) + # and fail this step if this is the case + combine_outputs: + name: Combine chunked test results + needs: test + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ${{ fromJson(inputs.python-version-list) }} + steps: + - uses: actions/download-artifact@v2 + with: + path: artifacts + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Cache .cache/pip + uses: actions/cache@v2 + id: cache-pip + with: + path: ~/.cache/pip + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} + - name: Combine outputs + uses: galaxyproject/planemo-ci-action@v1 + id: combine + with: + mode: combine + html-report: true + - uses: actions/upload-artifact@v2 + with: + name: 'All tool test results' + path: upload + - name: Create comment + if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }} + uses: peter-evans/create-or-update-comment@v1 + with: + token: ${{ secrets.PAT }} + repository: ${{ github.event.client_payload.github.payload.repository.full_name }} + issue-number: ${{ github.event.client_payload.github.payload.issue.number }} + body: | + Summary: + + ${{ steps.combine.outputs.statistics }} + + [Find all tool test results here][1] + + [1]: ${{ steps.vars.outputs.run-url }} + - name: Print statistics + run: | + echo "${{ steps.combine.outputs.statistics }}" + - name: Check outputs + uses: galaxyproject/planemo-ci-action@v1 + id: check + with: + mode: check + + test-success: + name: Determine test success + needs: [combine_outputs] + if: ${{ always() && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/main' }} + runs-on: ubuntu-latest + steps: + - name: Check tool test status + if: ${{ needs.combine_outputs.result != 'success' && needs.combine_outputs.result != 'skipped' }} + run: exit 1 diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4ccb8b80ee0..53debbfb30c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -1,17 +1,13 @@ name: Galaxy Tool Linting and Tests for push and PR on: [push, pull_request] -env: - GALAXY_FORK: galaxyproject - GALAXY_BRANCH: release_22.05 - MAX_CHUNKS: 4 - MAX_FILE_SIZE: 1M concurrency: # Group runs by PR, but keep runs on the default branch separate # because we do not want to cancel ToolShed uploads group: pr-${{ (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main') && github.run_number || github.ref }} cancel-in-progress: true jobs: - # the setup job does two things: + # the setup job does: + # # 1. cache the pip cache and .planemo # 2. determine the list of changed repositories # it produces one artifact which contains @@ -20,387 +16,66 @@ jobs: # which are needed in subsequent steps. setup: name: Setup cache and determine changed repositories - runs-on: ubuntu-latest - outputs: - galaxy-head-sha: ${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} - repository-list: ${{ steps.discover.outputs.repository-list }} - tool-list: ${{ steps.discover.outputs.tool-list }} - chunk-count: ${{ steps.discover.outputs.chunk-count }} - chunk-list: ${{ steps.discover.outputs.chunk-list }} - commit-range: ${{ steps.discover.outputs.commit-range }} - strategy: - matrix: - python-version: ['3.7'] - steps: - - name: Print github context properties - run: | - echo 'event: ${{ github.event_name }}' - echo 'sha: ${{ github.sha }}' - echo 'ref: ${{ github.ref }}' - echo 'head_ref: ${{ github.head_ref }}' - echo 'base_ref: ${{ github.base_ref }}' - echo 'event.before: ${{ github.event.before }}' - echo 'event.after: ${{ github.event.after }}' - - name: Determine latest commit in the Galaxy repo - id: get-galaxy-sha - run: echo "::set-output name=galaxy-head-sha::$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Cache .cache/pip - uses: actions/cache@v2 - id: cache-pip - with: - path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} - - name: Cache .planemo - uses: actions/cache@v2 - id: cache-planemo - with: - path: ~/.planemo - key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} - # Install the `wheel` package so that when installing other packages which - # are not available as wheels, pip will build a wheel for them, which can be cached. - - name: Install wheel - run: pip install wheel - - name: Install flake8 - run: pip install flake8 flake8-import-order - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks - uses: galaxyproject/planemo-ci-action@v1 - id: discover - with: - create-cache: ${{ steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' }} - galaxy-fork: ${{ env.GALAXY_FORK }} - galaxy-branch: ${{ env.GALAXY_BRANCH }} - max-chunks: ${{ env.MAX_CHUNKS }} - python-version: ${{ matrix.python-version }} - - name: Show commit range - run: echo '${{ steps.discover.outputs.commit-range }}' - - name: Show repository list - run: echo '${{ steps.discover.outputs.repository-list }}' - - name: Show tool list - run: echo '${{ steps.discover.outputs.tool-list }}' - - name: Show chunks - run: | - echo 'Using ${{ steps.discover.outputs.chunk-count }} chunks (${{ steps.discover.outputs.chunk-list }})' + uses: bernt-matthias/tools-iuc/.github/workflows/job_setup.yaml@topic/reusable-workflows + with: + # set these variables if you want to update independent of the IUC repo + # default-galaxy-fork: galaxyproject + # default-galaxy-branch: release_22.05 + max-chunks: 4 + fetch-depth: 0 - # Planemo lint the changed repositories + # Lint changed repositories + # - Planemo lint + # - flake8 of Python scripts + # - lintr of R scripts lint: - name: Lint tool-list + name: Lint needs: setup - if: ${{ needs.setup.outputs.repository-list != '' || needs.setup.outputs.tool-list != '' }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.7'] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Cache .cache/pip - uses: actions/cache@v2 - id: cache-pip - with: - path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - name: Planemo lint - uses: galaxyproject/planemo-ci-action@v1 - id: lint - with: - mode: lint - repository-list: ${{ needs.setup.outputs.repository-list }} - tool-list: ${{ needs.setup.outputs.tool-list }} - - uses: actions/upload-artifact@v2 - if: ${{ failure() }} - with: - name: 'Tool linting output' - path: lint_report.txt - - # flake8 of Python scripts in the changed repositories - flake8: - name: Lint Python scripts - needs: setup - if: ${{ needs.setup.outputs.repository-list != '' }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.7'] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Cache .cache/pip - uses: actions/cache@v2 - id: cache-pip - with: - path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - name: Install flake8 - run: pip install flake8 flake8-import-order - - name: Flake8 - run: echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' flake8 --output-file pylint_report.txt --tee - - uses: actions/upload-artifact@v2 - if: ${{ failure() }} - with: - name: 'Python linting output' - path: pylint_report.txt - - lintr: - name: Lint R scripts - needs: setup - if: ${{ needs.setup.outputs.repository-list != '' }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-20.04] - r-version: ['4.0.1'] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - uses: r-lib/actions/setup-r@master - with: - r-version: ${{ matrix.r-version }} - - name: Cache R packages - uses: actions/cache@v2 - with: - path: ${{ env.R_LIBS_USER }} - key: r_cache_${{ matrix.os }}_${{ matrix.r-version }} - - name: Install non-R lintr dependencies - run: sudo apt-get install libcurl4-openssl-dev - - name: Install lintr - run: | - install.packages('remotes') - remotes::install_cran("lintr") - shell: Rscript {0} - - name: Save repositories to file - run: echo '${{ needs.setup.outputs.repository-list }}' > repository_list.txt - - name: lintr - run: | - library(lintr) - linters <- linters_with_defaults(line_length_linter = NULL, cyclocomp_linter = NULL, object_usage_linter = NULL) - con <- file("repository_list.txt", "r") - status <- 0 - while (TRUE) { - repo <- readLines(con, n = 1) - if (length(repo) == 0) { - break - } - lnt <- lint_dir(repo, relative_path=T, linters=linters) - if (length(lnt) > 0) { - status <- 1 - for (l in lnt) { - rel_path <- paste(repo, l$filename, sep="/") - write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message, paste("(", l$line, ")")), stderr()) - write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message, paste("(", l$line, ")")), "rlint_report.txt", append=TRUE) - } - } - } - quit(status = status) - shell: Rscript {0} - - uses: actions/upload-artifact@v2 - if: ${{ failure() }} - with: - name: 'R linting output' - path: rlint_report.txt - - file_sizes: - name: Check file sizes - needs: setup - if: ${{ github.event_name == 'pull_request' && needs.setup.outputs.repository-list != '' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Check file sizes - run: | - touch file_size_report.txt - git diff --diff-filter=d --name-only ${{ needs.setup.outputs.commit-range }} > git.diff - while read line; do - find "$line" -type f -size +${{ env.MAX_FILE_SIZE }} >> file_size_report.txt - done < git.diff - if [[ -s file_size_report.txt ]]; then - echo "Files larger than ${{ env.MAX_FILE_SIZE }} found" - cat file_size_report.txt - exit 1 - fi - - uses: actions/upload-artifact@v2 - if: ${{ failure() }} - with: - name: 'File size report' - path: file_size_report.txt + uses: bernt-matthias/tools-iuc/.github/workflows/job_lint.yaml@topic/reusable-workflows + with: + repository-list: ${{ needs.setup.outputs.repository-list }} + tool-list: ${{ needs.setup.outputs.tool-list }} + galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} + commit-range: ${{ needs.setup.outputs.commit-range }} + max-file-size: 1M # Planemo test the changed repositories, each chunk creates an artifact # containing HTML and JSON reports for the executed tests test: name: Test tools needs: setup - if: ${{ needs.setup.outputs.repository-list != '' }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - chunk: ${{ fromJson(needs.setup.outputs.chunk-list) }} - python-version: ['3.7'] - services: - postgres: - image: postgres:11 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - ports: - - 5432:5432 - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Cache .cache/pip - uses: actions/cache@v2 - id: cache-pip - with: - path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - name: Cache .planemo - uses: actions/cache@v2 - id: cache-planemo - with: - path: ~/.planemo - key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v1 - id: cpu-cores - - name: Planemo test - uses: galaxyproject/planemo-ci-action@v1 - id: test - with: - mode: test - repository-list: ${{ needs.setup.outputs.repository-list }} - galaxy-fork: ${{ env.GALAXY_FORK }} - galaxy-branch: ${{ env.GALAXY_BRANCH }} - chunk: ${{ matrix.chunk }} - chunk-count: ${{ needs.setup.outputs.chunk-count }} - galaxy-slots: ${{ steps.cpu-cores.outputs.count }} - # Limit each test to 15 minutes - test_timeout: 900 - - uses: actions/upload-artifact@v2 - with: - name: 'Tool test output ${{ matrix.chunk }}' - path: upload - - # - combine the results of the test chunks (which will never fail due - # to `|| true`) and create a global test report as json and html which - # is provided as artifact - # - check if any tool test actually failed (by lookup in the combined json) - # and fail this step if this is the case - combine_outputs: - name: Combine chunked test results - needs: [setup, test] - if: ${{ needs.setup.outputs.repository-list != '' }} - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.7'] - steps: - - uses: actions/download-artifact@v2 - with: - path: artifacts - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Cache .cache/pip - uses: actions/cache@v2 - id: cache-pip - with: - path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - name: Combine outputs - uses: galaxyproject/planemo-ci-action@v1 - id: combine - with: - mode: combine - html-report: true - - uses: actions/upload-artifact@v2 - with: - name: 'All tool test results' - path: upload - - name: Check outputs - uses: galaxyproject/planemo-ci-action@v1 - id: check - with: - mode: check + uses: bernt-matthias/tools-iuc/.github/workflows/job_test.yaml@topic/reusable-workflows + with: + galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }} + galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }} + repository-list: ${{ needs.setup.outputs.repository-list }} + chunk-count: ${{ needs.setup.outputs.chunk-count }} + chunk-list: ${{ needs.setup.outputs.chunk-list }} + galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} + test-timeout: 900 # deploy the tools to the toolsheds (first TTS for testing) deploy: name: Deploy - needs: [setup, lint, flake8, lintr, combine_outputs] - if: ${{ (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' ) && github.repository_owner == 'galaxyproject' }} - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.7'] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Cache .cache/pip - uses: actions/cache@v2 - id: cache-pip - with: - path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - name: Deploy on testtoolshed - uses: galaxyproject/planemo-ci-action@v1 - with: - mode: deploy - repository-list: ${{ needs.setup.outputs.repository-list }} - shed-target: testtoolshed - shed-key: ${{ secrets.TTS_API_KEY }} - continue-on-error: true - - name: Deploy on toolshed - uses: galaxyproject/planemo-ci-action@v1 - with: - mode: deploy - repository-list: ${{ needs.setup.outputs.repository-list }} - shed-target: toolshed - shed-key: ${{ secrets.TS_API_KEY }} + needs: [setup, lint, test] + uses: bernt-matthias/tools-iuc/.github/workflows/job_deploy.yaml@topic/reusable-workflows + with: + repository-list: ${{ needs.setup.outputs.repository-list }} + galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} + repository-owner: 'galaxyproject' + secrets: + TTS_API_KEY: ${{ secrets.TTS_API_KEY }} + TS_API_KEY: ${{ secrets.TS_API_KEY }} determine-success: name: Check workflow success - needs: [setup, lint, flake8, lintr, file_sizes, combine_outputs] + needs: [setup, lint, test] if: ${{ always() && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/main' }} runs-on: ubuntu-latest steps: - name: Check tool lint status - if: ${{ needs.lint.result != 'success' && needs.flake8.result != 'skipped' }} - run: exit 1 - - name: Indicate Python script lint status - if: ${{ needs.flake8.result != 'success' && needs.flake8.result != 'skipped' }} - run: exit 1 - - name: Indicate R script lint status - if: ${{ needs.lintr.result != 'success' && needs.lintr.result != 'skipped' }} - run: exit 1 - - name: Indicate file size check status - if: ${{ needs.file_sizes.result != 'success' && needs.file_sizes.result != 'skipped' }} + if: ${{ needs.lint.result != 'success' }} run: exit 1 - name: Check tool test status - if: ${{ needs.combine_outputs.result != 'success' && needs.combine_outputs.result != 'skipped' }} + if: ${{ needs.test.result != 'success' }} run: exit 1 diff --git a/.github/workflows/slash.yaml b/.github/workflows/slash.yaml index 664d53916cd..29724ac59a3 100644 --- a/.github/workflows/slash.yaml +++ b/.github/workflows/slash.yaml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Slash Command Dispatch - if: github.repository_owner == 'galaxyproject' + # if: github.repository_owner == 'galaxyproject' uses: peter-evans/slash-command-dispatch@v2 with: token: ${{ secrets.PAT }} From ed276d18568e7c880168d002954bf2d01d8a8614 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 29 Sep 2022 16:28:57 +0200 Subject: [PATCH 02/29] test bumps --- tools/extract_genomic_dna/extract_genomic_dna.xml | 2 +- tools/gwastools/manhattan_plot.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/extract_genomic_dna/extract_genomic_dna.xml b/tools/extract_genomic_dna/extract_genomic_dna.xml index 88ac3fb421c..e765bc382b3 100644 --- a/tools/extract_genomic_dna/extract_genomic_dna.xml +++ b/tools/extract_genomic_dna/extract_genomic_dna.xml @@ -1,4 +1,4 @@ - + using coordinates from assembled/unassembled genomes bx-python diff --git a/tools/gwastools/manhattan_plot.xml b/tools/gwastools/manhattan_plot.xml index 546de7f05ca..1553a77dff0 100644 --- a/tools/gwastools/manhattan_plot.xml +++ b/tools/gwastools/manhattan_plot.xml @@ -1,4 +1,4 @@ - + Generate plots for use with GWAS pipelines gwastools From f7697a558406205db55209712455a36b1de77373 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sat, 1 Oct 2022 13:12:03 +0200 Subject: [PATCH 03/29] bump tool failing testing --- tools/basil/basil.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/basil/basil.xml b/tools/basil/basil.xml index 5d22e1560f5..ab9968ad59a 100644 --- a/tools/basil/basil.xml +++ b/tools/basil/basil.xml @@ -1,4 +1,4 @@ - + Breakpoint detection, including large insertions anise_basil @@ -23,7 +23,7 @@ - + From 1fe2c4b3c5a49b87418ffbfcb574c75b4766cb55 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sat, 1 Oct 2022 15:30:13 +0200 Subject: [PATCH 04/29] Add linting report and error-level supersedes https://github.com/galaxyproject/galaxy-tool-repository-template/pull/8 --- .github/workflows/job_lint.yaml | 12 ++++++++++++ .github/workflows/pr.yaml | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/job_lint.yaml b/.github/workflows/job_lint.yaml index d8f051bf52e..805c5ae12ec 100644 --- a/.github/workflows/job_lint.yaml +++ b/.github/workflows/job_lint.yaml @@ -12,6 +12,16 @@ on: default: '' required: false type: string + report-level: + description: 'Tool lint report level' + default: 'all' + required: false + type: string + fail-level: + description: 'Tool lint fail level' + default: 'warn' + required: false + type: string galaxy-head-sha: description: 'hash of the latest commit in the Galaxy repo' required: true @@ -73,6 +83,8 @@ jobs: id: lint with: mode: lint + report-level: ${{ inputs.report-level }} + fail-level: ${{ inputs.report-level }} repository-list: ${{ inputs.repository-list }} tool-list: ${{ inputs.tool-list }} - uses: actions/upload-artifact@v2 diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 53debbfb30c..6f711b6d98f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -38,7 +38,8 @@ jobs: galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} commit-range: ${{ needs.setup.outputs.commit-range }} max-file-size: 1M - + report-level: all + fail-level: warn # Planemo test the changed repositories, each chunk creates an artifact # containing HTML and JSON reports for the executed tests test: From 984caeee8553b17c1bb2d1ad56a307e388b7da5b Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sat, 1 Oct 2022 15:32:17 +0200 Subject: [PATCH 05/29] rename workflow files and do not set defaults --- .github/workflows/ci.yaml | 4 ++-- .github/workflows/pr.yaml | 21 ++++++++++--------- .../{job_deploy.yaml => wf_deploy.yaml} | 0 .../workflows/{job_lint.yaml => wf_lint.yaml} | 0 .../{job_setup.yaml => wf_setup.yaml} | 0 .../workflows/{job_test.yaml => wf_test.yaml} | 0 6 files changed, 13 insertions(+), 12 deletions(-) rename .github/workflows/{job_deploy.yaml => wf_deploy.yaml} (100%) rename .github/workflows/{job_lint.yaml => wf_lint.yaml} (100%) rename .github/workflows/{job_setup.yaml => wf_setup.yaml} (100%) rename .github/workflows/{job_test.yaml => wf_test.yaml} (100%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 081d2b1bfbe..c5a9a89d874 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,7 +8,7 @@ on: jobs: setup: name: Setup cache and determine changed repositories - uses: bernt-matthias/tools-iuc/.github/workflows/job_setup.yaml@master + uses: bernt-matthias/tools-iuc/.github/workflows/wf_setup.yaml@master with: # default-galaxy-fork: galaxyproject # default-galaxy-branch: release_22.05 @@ -18,7 +18,7 @@ jobs: test: name: Test tools needs: setup - uses: bernt-matthias/tools-iuc/.github/workflows/job_test.yaml@master + uses: bernt-matthias/tools-iuc/.github/workflows/wf_test.yaml@master with: galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }} galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6f711b6d98f..7e730d7d5f2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -7,7 +7,6 @@ concurrency: cancel-in-progress: true jobs: # the setup job does: - # # 1. cache the pip cache and .planemo # 2. determine the list of changed repositories # it produces one artifact which contains @@ -16,13 +15,13 @@ jobs: # which are needed in subsequent steps. setup: name: Setup cache and determine changed repositories - uses: bernt-matthias/tools-iuc/.github/workflows/job_setup.yaml@topic/reusable-workflows + uses: bernt-matthias/tools-iuc/.github/workflows/wf_setup.yaml@topic/reusable-workflows with: + fetch-depth: 0 # set these variables if you want to update independent of the IUC repo # default-galaxy-fork: galaxyproject # default-galaxy-branch: release_22.05 max-chunks: 4 - fetch-depth: 0 # Lint changed repositories # - Planemo lint @@ -31,21 +30,22 @@ jobs: lint: name: Lint needs: setup - uses: bernt-matthias/tools-iuc/.github/workflows/job_lint.yaml@topic/reusable-workflows + uses: bernt-matthias/tools-iuc/.github/workflows/wf_lint.yaml@topic/reusable-workflows with: repository-list: ${{ needs.setup.outputs.repository-list }} tool-list: ${{ needs.setup.outputs.tool-list }} galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} commit-range: ${{ needs.setup.outputs.commit-range }} - max-file-size: 1M - report-level: all - fail-level: warn + # set these values if you want to use different values than the IUC defaults + # max-file-size: 1M + # report-level: all + # fail-level: warn # Planemo test the changed repositories, each chunk creates an artifact # containing HTML and JSON reports for the executed tests test: name: Test tools needs: setup - uses: bernt-matthias/tools-iuc/.github/workflows/job_test.yaml@topic/reusable-workflows + uses: bernt-matthias/tools-iuc/.github/workflows/wf_test.yaml@topic/reusable-workflows with: galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }} galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }} @@ -53,13 +53,14 @@ jobs: chunk-count: ${{ needs.setup.outputs.chunk-count }} chunk-list: ${{ needs.setup.outputs.chunk-list }} galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} - test-timeout: 900 + # set these values if you want to use different values than the IUC defaults + # test-timeout: 900 # deploy the tools to the toolsheds (first TTS for testing) deploy: name: Deploy needs: [setup, lint, test] - uses: bernt-matthias/tools-iuc/.github/workflows/job_deploy.yaml@topic/reusable-workflows + uses: bernt-matthias/tools-iuc/.github/workflows/wf_deploy.yaml@topic/reusable-workflows with: repository-list: ${{ needs.setup.outputs.repository-list }} galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} diff --git a/.github/workflows/job_deploy.yaml b/.github/workflows/wf_deploy.yaml similarity index 100% rename from .github/workflows/job_deploy.yaml rename to .github/workflows/wf_deploy.yaml diff --git a/.github/workflows/job_lint.yaml b/.github/workflows/wf_lint.yaml similarity index 100% rename from .github/workflows/job_lint.yaml rename to .github/workflows/wf_lint.yaml diff --git a/.github/workflows/job_setup.yaml b/.github/workflows/wf_setup.yaml similarity index 100% rename from .github/workflows/job_setup.yaml rename to .github/workflows/wf_setup.yaml diff --git a/.github/workflows/job_test.yaml b/.github/workflows/wf_test.yaml similarity index 100% rename from .github/workflows/job_test.yaml rename to .github/workflows/wf_test.yaml From ac2059a7da40e42a6108aa06f0a19bad4ee9c009 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sat, 1 Oct 2022 15:41:25 +0200 Subject: [PATCH 06/29] add autoupdate --- .autoupdate_skip | 3 + .github/workflows/autoupdate.yaml | 31 ++++++ .github/workflows/pr.yaml | 8 +- .github/workflows/slash.yaml | 3 +- .github/workflows/wf_autoupdate.yaml | 152 +++++++++++++++++++++++++++ autoupdate_pr_text.py | 47 +++++++++ 6 files changed, 239 insertions(+), 5 deletions(-) create mode 100644 .autoupdate_skip create mode 100644 .github/workflows/autoupdate.yaml create mode 100644 .github/workflows/wf_autoupdate.yaml create mode 100644 autoupdate_pr_text.py diff --git a/.autoupdate_skip b/.autoupdate_skip new file mode 100644 index 00000000000..50aa2067764 --- /dev/null +++ b/.autoupdate_skip @@ -0,0 +1,3 @@ +mitos2.xml +optitype.xml +interproscan.xml diff --git a/.github/workflows/autoupdate.yaml b/.github/workflows/autoupdate.yaml new file mode 100644 index 00000000000..e81b4754b30 --- /dev/null +++ b/.github/workflows/autoupdate.yaml @@ -0,0 +1,31 @@ +name: Weekly autoupdate of tools +on: + schedule: + # Run at midnight every monday + - cron: '0 0 * * 1' + repository_dispatch: + types: [run-autoupdate-command] +jobs: + setup: + name: Setup cache and determine changed repositories + uses: bernt-matthias/tools-iuc/.github/workflows/wf_setup.yaml@master + with: + # default-galaxy-fork: galaxyproject + # default-galaxy-branch: release_22.05 + max-chunks: 40 + secrets: + PAT: ${{ secrets.PAT }} + autoupdate: + name: Autoupdate + needs: setup + uses: bernt-matthias/tools-iuc/.github/workflows/wf_autoupdate.yaml@master + with: + # galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }} + # galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }} + repository-list: ${{ needs.setup.outputs.repository-list }} + # chunk-count: ${{ needs.setup.outputs.chunk-count }} + # chunk-list: ${{ needs.setup.outputs.chunk-list }} + galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} + # test-timeout: 900 + # secrets: + # PAT: ${{ secrets.PAT_AU }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7e730d7d5f2..750cd9614b5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,7 +15,7 @@ jobs: # which are needed in subsequent steps. setup: name: Setup cache and determine changed repositories - uses: bernt-matthias/tools-iuc/.github/workflows/wf_setup.yaml@topic/reusable-workflows + uses: bernt-matthias/tools-iuc/.github/workflows/wf_setup.yaml@master with: fetch-depth: 0 # set these variables if you want to update independent of the IUC repo @@ -30,7 +30,7 @@ jobs: lint: name: Lint needs: setup - uses: bernt-matthias/tools-iuc/.github/workflows/wf_lint.yaml@topic/reusable-workflows + uses: bernt-matthias/tools-iuc/.github/workflows/wf_lint.yaml@master with: repository-list: ${{ needs.setup.outputs.repository-list }} tool-list: ${{ needs.setup.outputs.tool-list }} @@ -45,7 +45,7 @@ jobs: test: name: Test tools needs: setup - uses: bernt-matthias/tools-iuc/.github/workflows/wf_test.yaml@topic/reusable-workflows + uses: bernt-matthias/tools-iuc/.github/workflows/wf_test.yaml@master with: galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }} galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }} @@ -60,7 +60,7 @@ jobs: deploy: name: Deploy needs: [setup, lint, test] - uses: bernt-matthias/tools-iuc/.github/workflows/wf_deploy.yaml@topic/reusable-workflows + uses: bernt-matthias/tools-iuc/.github/workflows/wf_deploy.yaml@master with: repository-list: ${{ needs.setup.outputs.repository-list }} galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} diff --git a/.github/workflows/slash.yaml b/.github/workflows/slash.yaml index 29724ac59a3..a48af5f22e8 100644 --- a/.github/workflows/slash.yaml +++ b/.github/workflows/slash.yaml @@ -7,9 +7,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Slash Command Dispatch - # if: github.repository_owner == 'galaxyproject' + # TODO if: github.repository_owner == 'galaxyproject' uses: peter-evans/slash-command-dispatch@v2 with: token: ${{ secrets.PAT }} commands: | run-all-tool-tests + run-autoupdate diff --git a/.github/workflows/wf_autoupdate.yaml b/.github/workflows/wf_autoupdate.yaml new file mode 100644 index 00000000000..b58b2d66087 --- /dev/null +++ b/.github/workflows/wf_autoupdate.yaml @@ -0,0 +1,152 @@ +name: tools-iuc +on: + workflow_call: + inputs: + # TODO using this will exclude tools from the skip list + repository-list: + description: 'list of repositories to test' + required: true + type: string + galaxy-head-sha: + description: 'hash of the latest commit in the Galaxy repo' + required: true + type: string + python-version-list: + description: 'Python versions (stringified JSON array)' + default: "[\"3.7\"]" + required: false + type: string + # secrets: + # PAT: + # required: false +permissions: + contents: write +jobs: + autoupdate: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJson(inputs.python-version-list) }} + steps: + - uses: actions/checkout@v2 + with: + token: ${{ github.token }} + fetch-depth: 0 + + # - uses: webfactory/ssh-agent@v0.4.1 + # with: + # ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Cache .cache/pip + uses: actions/cache@v2 + id: cache-pip + with: + path: ~/.cache/pip + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} + - name: Cache .planemo + uses: actions/cache@v2 + id: cache-planemo + with: + path: ~/.planemo + key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} + + # Runs a set of commands using the runners shell + - name: Run autoupdate and create or update PRs for changed repositories + run: | + pip install wheel planemo + + # git credentials + git config --global user.email "m.bernt@ufz.de" + git config --global user.name "planemo-autoupdate" + echo ${{ github.token }} > token.txt + gh auth login --with-token < token.txt + + BASEDIR=`pwd` + + echo "Pulling latest..." + git fetch --all + + REPOS=$(echo "${{ inputs.repository-list }}") + + for REPO in $REPOS; do + # branch name will be path name with slashes replaced by `__` + # because repos like tool-collections/samtools/samtools-XYZ + # can not be crated, if tool-collections/samtools already exists + REPO_BRANCH="autoupdate/$(echo $REPO | sed 's@/@__@g')" + echo "Processing $REPO on branch $REPO_BRANCH" + # checkout branch, create if it doesn't exist + # TODO use gh pr view .. like for PR_NUMBER + if [[ $(gh pr list --limit 10000 | grep $REPO_BRANCH.\s*OPEN) ]] + then + echo "PR exists, we will checkout the branch and add to it" + git checkout --track origin/$REPO_BRANCH + else + if [[ $(git branch -a --list origin/$REPO_BRANCH) ]] + then + echo "Branch exists without an open PR - deleting" + git push origin --delete $REPO_BRANCH + fi + echo "Creating branch and checking out" + git checkout -b $REPO_BRANCH master + fi + echo "Checked out branch $(git branch --show-current)" + echo "Running autoupdate command..." + planemo autoupdate --skiplist .autoupdate_skip $REPO > $BASEDIR/autoupdate.log + cat $BASEDIR/autoupdate.log >> autoupdate_report.txt + rm -f tool_test_output.* + + if [[ $(git diff) ]] + then + TITLE=$(python3 $BASEDIR/autoupdate_pr_text.py --repo $REPO --log $BASEDIR/autoupdate.log --shed $REPO/.shed.yml --out $BASEDIR/body.txt) + # first check if a closed PR for the same branch with the same title - if so, we don't continue + if [[ ! $(gh pr list -s closed --search "head:$REPO_BRANCH ^$TITLE\$ in:title" --limit 1000) ]] + then + + echo "Committing... git commit $REPO -m "$TITLE" --quiet" + git commit . -m "$TITLE" --quiet + + # check if the content of the current branch is equal to any other autoupdate/* branch + # because of repos with central macros file and separate .shed.yaml files in subdirs (tool-collections/bamtools/) + EQUAL="FALSE" + for branch in $(git branch | grep "autoupdate/" | grep -v "^\*"); do + if [[ ! $(git diff $branch) ]]; then + EQUAL="TRUE" + fi + done + if [[ "$EQUAL" == "TRUE" ]]; then + echo "Current branch is equal to $branch .. not commiting" + else + echo "Push branch to remote..." + git push --set-upstream origin $REPO_BRANCH --quiet + + # determine PR number (empty if no PR already exists) + PR_NUMBER=$(gh pr view $REPO_BRANCH --json number --jq '.[]' || true 2> /dev/null) + if [[ -n "$PR_NUMBER" ]] # just need to update PR title + then + echo "Need to update title" + gh pr edit $PR_NUMBER --title "$TITLE" --add-label + else # we need to create a PR + echo "Creating a PR..." + gh pr create --base master --title "$TITLE" --body-file $BASEDIR/body.txt --label "autoupdate" + break + fi + fi + fi + fi + # clean up for the next tool + git checkout -- $REPO + done + - uses: actions/upload-artifact@v2 + with: + name: 'Autoupdate report' + path: autoupdate_report.txt + - name: Create comment + if: ${{ github.event.client_payload.slash_command.command == 'run-autoupdate' }} + uses: peter-evans/create-or-update-comment@v1 + with: + token: ${{ github.token }} + repository: ${{ github.event.client_payload.github.payload.repository.full_name }} + issue-number: ${{ github.event.client_payload.github.payload.issue.number }} + body: | + Autoupdate finished \ No newline at end of file diff --git a/autoupdate_pr_text.py b/autoupdate_pr_text.py new file mode 100644 index 00000000000..cadc4ccb93d --- /dev/null +++ b/autoupdate_pr_text.py @@ -0,0 +1,47 @@ +""" +Output autoupdate PR text +""" + +import argparse +import yaml + + +parser = argparse.ArgumentParser() +parser.add_argument('--repo', required=True, help='Tool repo') +parser.add_argument('--log', required=True, help='Autoupdate log') +parser.add_argument('--shed', required=True, help='Location of .shed.yml file input.') +parser.add_argument('--out', required=True, help='Output file.') +args = parser.parse_args() + +with open(args.log) as f: + for n in f.readlines(): + if 'Updating' in n and 'from version' in n: + if n.split()[4] != n.split()[6]: + update = f"from version {n.split()[4]} to {n.split()[6]}" + break + else: + raise Error + +text = [] +text.append(f"Hello! This is an automated update of the following tool: **{args.repo}**. I created this PR because I think the tool's main dependency is out of date, i.e. there is a newer version available through conda.") +text.append(f"I have updated {args.repo} {update}.") + +with open(args.shed) as f: + y = yaml.load(f, Loader=yaml.SafeLoader) + +if y.get('homepage_url'): + url = y.get('homepage_url').strip('/') + if 'github.com' in url: + if len(url.split('github.com')[1].split('/')) > 1: + url += '/releases' + text.append(f'**Project home page:** {url}') + +if y.get('maintainers'): + text.append('**Maintainers:** ' + ', '.join([f'@{m}' for m in y.get('maintainers')])) + +text.append("For any comments, queries or criticism about the bot, not related to the tool being updated in this PR, please create an issue [here](https://github.com/galaxyproject/tools-iuc/new).") + +with open(args.out, 'w') as f: + f.write('\n\n'.join(text)) + +print(f'Updating {args.repo} {update}') From a7ebdec37f11c2ea01452172db454a84a57946df Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 4 Oct 2022 14:55:42 +0200 Subject: [PATCH 07/29] update all used actions --- .github/workflows/slash.yaml | 2 +- .github/workflows/wf_autoupdate.yaml | 8 ++++---- .github/workflows/wf_deploy.yaml | 6 +++--- .github/workflows/wf_lint.yaml | 26 +++++++++++++------------- .github/workflows/wf_setup.yaml | 9 ++++----- .github/workflows/wf_test.yaml | 20 ++++++++++---------- 6 files changed, 35 insertions(+), 36 deletions(-) diff --git a/.github/workflows/slash.yaml b/.github/workflows/slash.yaml index a48af5f22e8..0d6cc16ac75 100644 --- a/.github/workflows/slash.yaml +++ b/.github/workflows/slash.yaml @@ -8,7 +8,7 @@ jobs: steps: - name: Slash Command Dispatch # TODO if: github.repository_owner == 'galaxyproject' - uses: peter-evans/slash-command-dispatch@v2 + uses: peter-evans/slash-command-dispatch@v3 with: token: ${{ secrets.PAT }} commands: | diff --git a/.github/workflows/wf_autoupdate.yaml b/.github/workflows/wf_autoupdate.yaml index b58b2d66087..16ef6a0a3ba 100644 --- a/.github/workflows/wf_autoupdate.yaml +++ b/.github/workflows/wf_autoupdate.yaml @@ -29,7 +29,7 @@ jobs: matrix: python-version: ${{ fromJson(inputs.python-version-list) }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: token: ${{ github.token }} fetch-depth: 0 @@ -39,13 +39,13 @@ jobs: # ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-pip with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} - name: Cache .planemo - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-planemo with: path: ~/.planemo @@ -137,7 +137,7 @@ jobs: # clean up for the next tool git checkout -- $REPO done - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: 'Autoupdate report' path: autoupdate_report.txt diff --git a/.github/workflows/wf_deploy.yaml b/.github/workflows/wf_deploy.yaml index afa0a1b681f..f8a817dfa82 100644 --- a/.github/workflows/wf_deploy.yaml +++ b/.github/workflows/wf_deploy.yaml @@ -35,14 +35,14 @@ jobs: matrix: python-version: ${{ fromJson(inputs.python-version-list) }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-pip with: path: ~/.cache/pip diff --git a/.github/workflows/wf_lint.yaml b/.github/workflows/wf_lint.yaml index 805c5ae12ec..a8e8421fc9a 100644 --- a/.github/workflows/wf_lint.yaml +++ b/.github/workflows/wf_lint.yaml @@ -66,14 +66,14 @@ jobs: matrix: python-version: ${{ fromJson(inputs.python-version-list) }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-pip with: path: ~/.cache/pip @@ -87,7 +87,7 @@ jobs: fail-level: ${{ inputs.report-level }} repository-list: ${{ inputs.repository-list }} tool-list: ${{ inputs.tool-list }} - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: ${{ failure() }} with: name: 'Tool linting output' @@ -102,14 +102,14 @@ jobs: matrix: python-version: ${{ fromJson(inputs.python-version-list) }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-pip with: path: ~/.cache/pip @@ -118,7 +118,7 @@ jobs: run: pip install flake8 flake8-import-order - name: Flake8 run: echo '${{ inputs.repository-list }}' | xargs -d '\n' flake8 --output-file pylint_report.txt --tee - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: ${{ failure() }} with: name: 'Python linting output' @@ -133,14 +133,14 @@ jobs: r-version: ${{ fromJson(inputs.r-version-list) }} runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1 - uses: r-lib/actions/setup-r@master with: r-version: ${{ matrix.r-version }} - name: Cache R packages - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{ env.R_LIBS_USER }} key: r_cache_${{ matrix.os }}_${{ matrix.r-version }} @@ -176,7 +176,7 @@ jobs: } quit(status = status) shell: Rscript {0} - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: ${{ failure() }} with: name: 'R linting output' @@ -187,7 +187,7 @@ jobs: if: ${{ github.event_name == 'pull_request' && inputs.repository-list != '' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Check file sizes @@ -202,7 +202,7 @@ jobs: cat file_size_report.txt exit 1 fi - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: ${{ failure() }} with: name: 'File size report' diff --git a/.github/workflows/wf_setup.yaml b/.github/workflows/wf_setup.yaml index 395e311e7e7..fe48151e22d 100644 --- a/.github/workflows/wf_setup.yaml +++ b/.github/workflows/wf_setup.yaml @@ -110,17 +110,17 @@ jobs: - name: Determine latest commit in the Galaxy repo id: get-galaxy-sha run: echo "::set-output name=galaxy-head-sha::$(git ls-remote https://github.com/${{ needs.fork-branch.outputs.galaxy-fork }}/galaxy refs/heads/${{ needs.fork-branch.outputs.galaxy-branch }} | cut -f1)" - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-pip with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} - name: Cache .planemo - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-planemo with: path: ~/.planemo @@ -131,9 +131,8 @@ jobs: run: pip install wheel - name: Install flake8 run: pip install flake8 flake8-import-order - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: - # TODO make configurable CI uses 1 fetch-depth: ${{ inputs.fetch-depth }} - name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks uses: galaxyproject/planemo-ci-action@v1 diff --git a/.github/workflows/wf_test.yaml b/.github/workflows/wf_test.yaml index 70ecd2b1e14..3dd93fbaca7 100644 --- a/.github/workflows/wf_test.yaml +++ b/.github/workflows/wf_test.yaml @@ -71,21 +71,21 @@ jobs: ports: - 5432:5432 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-pip with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} - name: Cache .planemo if: ${{ inputs.cache-planemo }} - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-planemo with: path: ~/.planemo @@ -106,7 +106,7 @@ jobs: galaxy-slots: ${{ steps.cpu-cores.outputs.count }} # Limit each test to 15 minutes test_timeout: ${{ inputs.test-timeout }} - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: 'Tool test output ${{ matrix.chunk }}' path: upload @@ -124,14 +124,14 @@ jobs: matrix: python-version: ${{ fromJson(inputs.python-version-list) }} steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: path: artifacts - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-pip with: path: ~/.cache/pip @@ -142,13 +142,13 @@ jobs: with: mode: combine html-report: true - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: 'All tool test results' path: upload - name: Create comment if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }} - uses: peter-evans/create-or-update-comment@v1 + uses: peter-evans/create-or-update-comment@v2 with: token: ${{ secrets.PAT }} repository: ${{ github.event.client_payload.github.payload.repository.full_name }} From 736839ce4bba7880c662dfc5eeaaeaa010744200 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 4 Oct 2022 15:06:19 +0200 Subject: [PATCH 08/29] cleanup wf --- .github/workflows/ci.yaml | 6 ++++-- .github/workflows/pr.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c5a9a89d874..529abd16ff5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,7 +8,9 @@ on: jobs: setup: name: Setup cache and determine changed repositories - uses: bernt-matthias/tools-iuc/.github/workflows/wf_setup.yaml@master + # adapt the owner name + if: ${{ github.repository_owner == 'galaxyproject' }} + uses: galaxyproject/tools-iuc/.github/workflows/wf_setup.yaml@master with: # default-galaxy-fork: galaxyproject # default-galaxy-branch: release_22.05 @@ -18,7 +20,7 @@ jobs: test: name: Test tools needs: setup - uses: bernt-matthias/tools-iuc/.github/workflows/wf_test.yaml@master + uses: galaxyproject/tools-iuc/.github/workflows/wf_test.yaml@master with: galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }} galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 750cd9614b5..d1a4f75a27c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,7 +15,7 @@ jobs: # which are needed in subsequent steps. setup: name: Setup cache and determine changed repositories - uses: bernt-matthias/tools-iuc/.github/workflows/wf_setup.yaml@master + uses: galaxyproject/tools-iuc/.github/workflows/wf_setup.yaml@master with: fetch-depth: 0 # set these variables if you want to update independent of the IUC repo @@ -30,7 +30,7 @@ jobs: lint: name: Lint needs: setup - uses: bernt-matthias/tools-iuc/.github/workflows/wf_lint.yaml@master + uses: galaxyproject/tools-iuc/.github/workflows/wf_lint.yaml@master with: repository-list: ${{ needs.setup.outputs.repository-list }} tool-list: ${{ needs.setup.outputs.tool-list }} @@ -45,7 +45,7 @@ jobs: test: name: Test tools needs: setup - uses: bernt-matthias/tools-iuc/.github/workflows/wf_test.yaml@master + uses: galaxyproject/tools-iuc/.github/workflows/wf_test.yaml@master with: galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }} galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }} @@ -60,7 +60,7 @@ jobs: deploy: name: Deploy needs: [setup, lint, test] - uses: bernt-matthias/tools-iuc/.github/workflows/wf_deploy.yaml@master + uses: galaxyproject/tools-iuc/.github/workflows/wf_deploy.yaml@master with: repository-list: ${{ needs.setup.outputs.repository-list }} galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} From 065540e88798a9933ef317cee3fa0df53aed6a04 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 4 Oct 2022 15:06:41 +0200 Subject: [PATCH 09/29] cleanup autoupdate --- .github/workflows/autoupdate.yaml | 19 +++++-------------- .github/workflows/wf_autoupdate.yaml | 3 --- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/.github/workflows/autoupdate.yaml b/.github/workflows/autoupdate.yaml index e81b4754b30..75332e2dd11 100644 --- a/.github/workflows/autoupdate.yaml +++ b/.github/workflows/autoupdate.yaml @@ -8,24 +8,15 @@ on: jobs: setup: name: Setup cache and determine changed repositories - uses: bernt-matthias/tools-iuc/.github/workflows/wf_setup.yaml@master + # adapt the owner name + if: ${{ github.repository_owner == 'galaxyproject' }} + uses: galaxyproject/tools-iuc/.github/workflows/wf_setup.yaml@master with: - # default-galaxy-fork: galaxyproject - # default-galaxy-branch: release_22.05 - max-chunks: 40 - secrets: - PAT: ${{ secrets.PAT }} + max-chunks: 1 autoupdate: name: Autoupdate needs: setup - uses: bernt-matthias/tools-iuc/.github/workflows/wf_autoupdate.yaml@master + uses: galaxyproject/tools-iuc/.github/workflows/wf_autoupdate.yaml@master with: - # galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }} - # galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }} repository-list: ${{ needs.setup.outputs.repository-list }} - # chunk-count: ${{ needs.setup.outputs.chunk-count }} - # chunk-list: ${{ needs.setup.outputs.chunk-list }} galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} - # test-timeout: 900 - # secrets: - # PAT: ${{ secrets.PAT_AU }} diff --git a/.github/workflows/wf_autoupdate.yaml b/.github/workflows/wf_autoupdate.yaml index 16ef6a0a3ba..ae06375cfdb 100644 --- a/.github/workflows/wf_autoupdate.yaml +++ b/.github/workflows/wf_autoupdate.yaml @@ -16,9 +16,6 @@ on: default: "[\"3.7\"]" required: false type: string - # secrets: - # PAT: - # required: false permissions: contents: write jobs: From 022d11f5a37986f16028229b383989fe742f2e4b Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 4 Oct 2022 15:13:19 +0200 Subject: [PATCH 10/29] move deploy check for forked repos and changed for testing pr workflow --- .github/workflows/pr.yaml | 10 +++++----- .github/workflows/wf_deploy.yaml | 6 +----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d1a4f75a27c..7633fb98157 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,7 +15,7 @@ jobs: # which are needed in subsequent steps. setup: name: Setup cache and determine changed repositories - uses: galaxyproject/tools-iuc/.github/workflows/wf_setup.yaml@master + uses: bernt-matthias/tools-iuc/.github/workflows/wf_setup.yaml@topic/reusable-workflows with: fetch-depth: 0 # set these variables if you want to update independent of the IUC repo @@ -30,7 +30,7 @@ jobs: lint: name: Lint needs: setup - uses: galaxyproject/tools-iuc/.github/workflows/wf_lint.yaml@master + uses: bernt-matthias/tools-iuc/.github/workflows/wf_lint.yaml@topic/reusable-workflows with: repository-list: ${{ needs.setup.outputs.repository-list }} tool-list: ${{ needs.setup.outputs.tool-list }} @@ -45,7 +45,7 @@ jobs: test: name: Test tools needs: setup - uses: galaxyproject/tools-iuc/.github/workflows/wf_test.yaml@master + uses: bernt-matthias/tools-iuc/.github/workflows/wf_test.yaml@topic/reusable-workflows with: galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }} galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }} @@ -59,12 +59,12 @@ jobs: # deploy the tools to the toolsheds (first TTS for testing) deploy: name: Deploy + if: ${{ github.repository_owner == 'galaxyproject' }} needs: [setup, lint, test] - uses: galaxyproject/tools-iuc/.github/workflows/wf_deploy.yaml@master + uses: bernt-matthias/tools-iuc/.github/workflows/wf_deploy.yaml@topic/reusable-workflows with: repository-list: ${{ needs.setup.outputs.repository-list }} galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} - repository-owner: 'galaxyproject' secrets: TTS_API_KEY: ${{ secrets.TTS_API_KEY }} TS_API_KEY: ${{ secrets.TS_API_KEY }} diff --git a/.github/workflows/wf_deploy.yaml b/.github/workflows/wf_deploy.yaml index f8a817dfa82..7fd9ea18ee8 100644 --- a/.github/workflows/wf_deploy.yaml +++ b/.github/workflows/wf_deploy.yaml @@ -10,10 +10,6 @@ on: description: 'list of repositories to deploy' required: true type: string - repository-owner: - description: 'owner, ie main fork, of the repository (deploy will only run for this fork)' - required: true - type: string python-version-list: description: 'Python versions (stringified JSON array)' default: "[\"3.7\"]" @@ -29,7 +25,7 @@ jobs: # deploy the tools to the toolsheds (first TTS for testing) deploy: name: Deploy - if: ${{ (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' ) && github.repository_owner == inputs.repository-owner }} + if: ${{ (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' ) }} runs-on: ubuntu-latest strategy: matrix: From 426e1d0a51d6c7feb5424b3c64527cec1af74c0c Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 4 Oct 2022 15:51:36 +0200 Subject: [PATCH 11/29] fix fail level --- .github/workflows/wf_lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wf_lint.yaml b/.github/workflows/wf_lint.yaml index a8e8421fc9a..a1438328132 100644 --- a/.github/workflows/wf_lint.yaml +++ b/.github/workflows/wf_lint.yaml @@ -84,7 +84,7 @@ jobs: with: mode: lint report-level: ${{ inputs.report-level }} - fail-level: ${{ inputs.report-level }} + fail-level: ${{ inputs.fail-level }} repository-list: ${{ inputs.repository-list }} tool-list: ${{ inputs.tool-list }} - uses: actions/upload-artifact@v3 From 990d86166961eb5e93c39ab2da9743ca25fa0bda Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 5 Oct 2022 11:15:36 +0200 Subject: [PATCH 12/29] also don't run PR workflow on forks --- .github/workflows/pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7633fb98157..de681b47bea 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,6 +15,8 @@ jobs: # which are needed in subsequent steps. setup: name: Setup cache and determine changed repositories + # adapt the owner name + if: ${{ github.repository_owner == 'galaxyproject' }} uses: bernt-matthias/tools-iuc/.github/workflows/wf_setup.yaml@topic/reusable-workflows with: fetch-depth: 0 From 42c3ea5e0f142382beb781478c33cbc4c99f7812 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 6 Oct 2022 09:24:34 +0200 Subject: [PATCH 13/29] cont --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 529abd16ff5..4d110af8b53 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -28,6 +28,6 @@ jobs: chunk-count: ${{ needs.setup.outputs.chunk-count }} chunk-list: ${{ needs.setup.outputs.chunk-list }} galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} - test-timeout: 900 + # test-timeout: 900 secrets: PAT: ${{ secrets.PAT }} From 4cac4aebd415cf017381ebb959a41948e02bbde5 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 6 Oct 2022 09:34:30 +0200 Subject: [PATCH 14/29] remove autoupdate --- .autoupdate_skip | 3 - .github/workflows/autoupdate.yaml | 22 ---- .github/workflows/slash.yaml | 1 - .github/workflows/wf_autoupdate.yaml | 149 --------------------------- autoupdate_pr_text.py | 47 --------- 5 files changed, 222 deletions(-) delete mode 100644 .autoupdate_skip delete mode 100644 .github/workflows/autoupdate.yaml delete mode 100644 .github/workflows/wf_autoupdate.yaml delete mode 100644 autoupdate_pr_text.py diff --git a/.autoupdate_skip b/.autoupdate_skip deleted file mode 100644 index 50aa2067764..00000000000 --- a/.autoupdate_skip +++ /dev/null @@ -1,3 +0,0 @@ -mitos2.xml -optitype.xml -interproscan.xml diff --git a/.github/workflows/autoupdate.yaml b/.github/workflows/autoupdate.yaml deleted file mode 100644 index 75332e2dd11..00000000000 --- a/.github/workflows/autoupdate.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: Weekly autoupdate of tools -on: - schedule: - # Run at midnight every monday - - cron: '0 0 * * 1' - repository_dispatch: - types: [run-autoupdate-command] -jobs: - setup: - name: Setup cache and determine changed repositories - # adapt the owner name - if: ${{ github.repository_owner == 'galaxyproject' }} - uses: galaxyproject/tools-iuc/.github/workflows/wf_setup.yaml@master - with: - max-chunks: 1 - autoupdate: - name: Autoupdate - needs: setup - uses: galaxyproject/tools-iuc/.github/workflows/wf_autoupdate.yaml@master - with: - repository-list: ${{ needs.setup.outputs.repository-list }} - galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} diff --git a/.github/workflows/slash.yaml b/.github/workflows/slash.yaml index 0d6cc16ac75..aa7ed9d243f 100644 --- a/.github/workflows/slash.yaml +++ b/.github/workflows/slash.yaml @@ -13,4 +13,3 @@ jobs: token: ${{ secrets.PAT }} commands: | run-all-tool-tests - run-autoupdate diff --git a/.github/workflows/wf_autoupdate.yaml b/.github/workflows/wf_autoupdate.yaml deleted file mode 100644 index ae06375cfdb..00000000000 --- a/.github/workflows/wf_autoupdate.yaml +++ /dev/null @@ -1,149 +0,0 @@ -name: tools-iuc -on: - workflow_call: - inputs: - # TODO using this will exclude tools from the skip list - repository-list: - description: 'list of repositories to test' - required: true - type: string - galaxy-head-sha: - description: 'hash of the latest commit in the Galaxy repo' - required: true - type: string - python-version-list: - description: 'Python versions (stringified JSON array)' - default: "[\"3.7\"]" - required: false - type: string -permissions: - contents: write -jobs: - autoupdate: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ${{ fromJson(inputs.python-version-list) }} - steps: - - uses: actions/checkout@v3 - with: - token: ${{ github.token }} - fetch-depth: 0 - - # - uses: webfactory/ssh-agent@v0.4.1 - # with: - # ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} - - - name: Cache .cache/pip - uses: actions/cache@v3 - id: cache-pip - with: - path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} - - name: Cache .planemo - uses: actions/cache@v3 - id: cache-planemo - with: - path: ~/.planemo - key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} - - # Runs a set of commands using the runners shell - - name: Run autoupdate and create or update PRs for changed repositories - run: | - pip install wheel planemo - - # git credentials - git config --global user.email "m.bernt@ufz.de" - git config --global user.name "planemo-autoupdate" - echo ${{ github.token }} > token.txt - gh auth login --with-token < token.txt - - BASEDIR=`pwd` - - echo "Pulling latest..." - git fetch --all - - REPOS=$(echo "${{ inputs.repository-list }}") - - for REPO in $REPOS; do - # branch name will be path name with slashes replaced by `__` - # because repos like tool-collections/samtools/samtools-XYZ - # can not be crated, if tool-collections/samtools already exists - REPO_BRANCH="autoupdate/$(echo $REPO | sed 's@/@__@g')" - echo "Processing $REPO on branch $REPO_BRANCH" - # checkout branch, create if it doesn't exist - # TODO use gh pr view .. like for PR_NUMBER - if [[ $(gh pr list --limit 10000 | grep $REPO_BRANCH.\s*OPEN) ]] - then - echo "PR exists, we will checkout the branch and add to it" - git checkout --track origin/$REPO_BRANCH - else - if [[ $(git branch -a --list origin/$REPO_BRANCH) ]] - then - echo "Branch exists without an open PR - deleting" - git push origin --delete $REPO_BRANCH - fi - echo "Creating branch and checking out" - git checkout -b $REPO_BRANCH master - fi - echo "Checked out branch $(git branch --show-current)" - echo "Running autoupdate command..." - planemo autoupdate --skiplist .autoupdate_skip $REPO > $BASEDIR/autoupdate.log - cat $BASEDIR/autoupdate.log >> autoupdate_report.txt - rm -f tool_test_output.* - - if [[ $(git diff) ]] - then - TITLE=$(python3 $BASEDIR/autoupdate_pr_text.py --repo $REPO --log $BASEDIR/autoupdate.log --shed $REPO/.shed.yml --out $BASEDIR/body.txt) - # first check if a closed PR for the same branch with the same title - if so, we don't continue - if [[ ! $(gh pr list -s closed --search "head:$REPO_BRANCH ^$TITLE\$ in:title" --limit 1000) ]] - then - - echo "Committing... git commit $REPO -m "$TITLE" --quiet" - git commit . -m "$TITLE" --quiet - - # check if the content of the current branch is equal to any other autoupdate/* branch - # because of repos with central macros file and separate .shed.yaml files in subdirs (tool-collections/bamtools/) - EQUAL="FALSE" - for branch in $(git branch | grep "autoupdate/" | grep -v "^\*"); do - if [[ ! $(git diff $branch) ]]; then - EQUAL="TRUE" - fi - done - if [[ "$EQUAL" == "TRUE" ]]; then - echo "Current branch is equal to $branch .. not commiting" - else - echo "Push branch to remote..." - git push --set-upstream origin $REPO_BRANCH --quiet - - # determine PR number (empty if no PR already exists) - PR_NUMBER=$(gh pr view $REPO_BRANCH --json number --jq '.[]' || true 2> /dev/null) - if [[ -n "$PR_NUMBER" ]] # just need to update PR title - then - echo "Need to update title" - gh pr edit $PR_NUMBER --title "$TITLE" --add-label - else # we need to create a PR - echo "Creating a PR..." - gh pr create --base master --title "$TITLE" --body-file $BASEDIR/body.txt --label "autoupdate" - break - fi - fi - fi - fi - # clean up for the next tool - git checkout -- $REPO - done - - uses: actions/upload-artifact@v3 - with: - name: 'Autoupdate report' - path: autoupdate_report.txt - - name: Create comment - if: ${{ github.event.client_payload.slash_command.command == 'run-autoupdate' }} - uses: peter-evans/create-or-update-comment@v1 - with: - token: ${{ github.token }} - repository: ${{ github.event.client_payload.github.payload.repository.full_name }} - issue-number: ${{ github.event.client_payload.github.payload.issue.number }} - body: | - Autoupdate finished \ No newline at end of file diff --git a/autoupdate_pr_text.py b/autoupdate_pr_text.py deleted file mode 100644 index cadc4ccb93d..00000000000 --- a/autoupdate_pr_text.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Output autoupdate PR text -""" - -import argparse -import yaml - - -parser = argparse.ArgumentParser() -parser.add_argument('--repo', required=True, help='Tool repo') -parser.add_argument('--log', required=True, help='Autoupdate log') -parser.add_argument('--shed', required=True, help='Location of .shed.yml file input.') -parser.add_argument('--out', required=True, help='Output file.') -args = parser.parse_args() - -with open(args.log) as f: - for n in f.readlines(): - if 'Updating' in n and 'from version' in n: - if n.split()[4] != n.split()[6]: - update = f"from version {n.split()[4]} to {n.split()[6]}" - break - else: - raise Error - -text = [] -text.append(f"Hello! This is an automated update of the following tool: **{args.repo}**. I created this PR because I think the tool's main dependency is out of date, i.e. there is a newer version available through conda.") -text.append(f"I have updated {args.repo} {update}.") - -with open(args.shed) as f: - y = yaml.load(f, Loader=yaml.SafeLoader) - -if y.get('homepage_url'): - url = y.get('homepage_url').strip('/') - if 'github.com' in url: - if len(url.split('github.com')[1].split('/')) > 1: - url += '/releases' - text.append(f'**Project home page:** {url}') - -if y.get('maintainers'): - text.append('**Maintainers:** ' + ', '.join([f'@{m}' for m in y.get('maintainers')])) - -text.append("For any comments, queries or criticism about the bot, not related to the tool being updated in this PR, please create an issue [here](https://github.com/galaxyproject/tools-iuc/new).") - -with open(args.out, 'w') as f: - f.write('\n\n'.join(text)) - -print(f'Updating {args.repo} {update}') From d693232ea47ba2296699c2882d58b21dd7950d36 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 6 Oct 2022 10:04:28 +0200 Subject: [PATCH 15/29] add planemo-version to the slash command parameters --- .github/workflows/ci.yaml | 1 + .github/workflows/pr.yaml | 2 ++ .github/workflows/wf_setup.yaml | 13 +++++++++++++ 3 files changed, 16 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4d110af8b53..a7ffc4a6525 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,6 +24,7 @@ jobs: with: galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }} galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }} + planemo-version: ${{ needs.setup.outputs.planemo-version }} repository-list: ${{ needs.setup.outputs.repository-list }} chunk-count: ${{ needs.setup.outputs.chunk-count }} chunk-list: ${{ needs.setup.outputs.chunk-list }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index de681b47bea..949ed92573c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -38,6 +38,7 @@ jobs: tool-list: ${{ needs.setup.outputs.tool-list }} galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} commit-range: ${{ needs.setup.outputs.commit-range }} + planemo-version: ${{ needs.setup.outputs.planemo-version }} # set these values if you want to use different values than the IUC defaults # max-file-size: 1M # report-level: all @@ -51,6 +52,7 @@ jobs: with: galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }} galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }} + planemo-version: ${{ needs.setup.outputs.planemo-version }} repository-list: ${{ needs.setup.outputs.repository-list }} chunk-count: ${{ needs.setup.outputs.chunk-count }} chunk-list: ${{ needs.setup.outputs.chunk-list }} diff --git a/.github/workflows/wf_setup.yaml b/.github/workflows/wf_setup.yaml index fe48151e22d..415caaf26c4 100644 --- a/.github/workflows/wf_setup.yaml +++ b/.github/workflows/wf_setup.yaml @@ -12,6 +12,11 @@ on: default: 'release_22.05' required: false type: string + default-planemo-version: + description: 'Default planemo version to use' + default: 'planemo' + required: false + type: string max-chunks: description: 'maximum number of chunks to use' required: true @@ -54,6 +59,9 @@ on: galaxy-branch: description: "The determined galaxy branch to use" value: ${{ jobs.fork-branch.outputs.galaxy-branch }} + planemo-version: + description: "The determined planemo version to use" + value: ${{ jobs.fork-branch.outputs.planemo-version }} jobs: fork-branch: name: Determine Galaxy fork and branch to use @@ -61,6 +69,7 @@ jobs: outputs: galaxy-fork: ${{ steps.get-fork-branch.outputs.fork }} galaxy-branch: ${{ steps.get-fork-branch.outputs.branch }} + planemo-version: ${{ steps.get-fork-branch.outputs.planemo-version }} steps: - name: Add reaction if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }} @@ -79,6 +88,9 @@ jobs: SLASH_BRANCH="${{ github.event.client_payload.slash_command.args.named.branch }}" DEFAULT_BRANCH="${{ inputs.default-galaxy-branch }}" echo "::set-output name=branch::${SLASH_BRANCH:-$DEFAULT_BRANCH}" + SLASH_PLANEMO="${{ github.event.client_payload.slash_command.args.named.planemo-version }}" + DEFAULT_PLANEMO="${{ inputs.default-planemo-version }}" + echo "::set-output name=planemo-version::${SLASH_PLANEMO:-$DEFAULT_PLANEMO}" - name: Print used Galaxy for and branch run: | echo 'fork: ${{ steps.get-fork-branch.outputs.fork }}' @@ -143,6 +155,7 @@ jobs: galaxy-branch: ${{ needs.fork-branch.outputs.galaxy-branch }} max-chunks: ${{ inputs.max-chunks }} python-version: ${{ matrix.python-version }} + planemo-version: ${{ needs.fork-branch.outputs.planemo-version }} - name: Show commit range run: echo '${{ steps.discover.outputs.commit-range }}' - name: Show repository list From 4de59d7e380db69d8d97f5cd94581f4f8f86010c Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 17 Nov 2022 15:22:30 +0100 Subject: [PATCH 16/29] incorporate latest changes from - https://github.com/galaxyproject/tools-iuc/pull/4850 - https://github.com/galaxyproject/tools-iuc/pull/4892 and fix fork (use IUC master for called WFs, instead of my fork) --- .github/workflows/pr.yaml | 8 ++++---- .github/workflows/wf_lint.yaml | 4 ++-- .github/workflows/wf_setup.yaml | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 949ed92573c..bbf1e14cf21 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,7 +17,7 @@ jobs: name: Setup cache and determine changed repositories # adapt the owner name if: ${{ github.repository_owner == 'galaxyproject' }} - uses: bernt-matthias/tools-iuc/.github/workflows/wf_setup.yaml@topic/reusable-workflows + uses: galaxyproject/tools-iuc/.github/workflows/wf_setup.yaml@master with: fetch-depth: 0 # set these variables if you want to update independent of the IUC repo @@ -32,7 +32,7 @@ jobs: lint: name: Lint needs: setup - uses: bernt-matthias/tools-iuc/.github/workflows/wf_lint.yaml@topic/reusable-workflows + uses: galaxyproject/tools-iuc/.github/workflows/wf_lint.yaml@master with: repository-list: ${{ needs.setup.outputs.repository-list }} tool-list: ${{ needs.setup.outputs.tool-list }} @@ -48,7 +48,7 @@ jobs: test: name: Test tools needs: setup - uses: bernt-matthias/tools-iuc/.github/workflows/wf_test.yaml@topic/reusable-workflows + uses: galaxyproject/tools-iuc/.github/workflows/wf_test.yaml@master with: galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }} galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }} @@ -65,7 +65,7 @@ jobs: name: Deploy if: ${{ github.repository_owner == 'galaxyproject' }} needs: [setup, lint, test] - uses: bernt-matthias/tools-iuc/.github/workflows/wf_deploy.yaml@topic/reusable-workflows + uses: galaxyproject/tools-iuc/.github/workflows/wf_deploy.yaml@master with: repository-list: ${{ needs.setup.outputs.repository-list }} galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} diff --git a/.github/workflows/wf_lint.yaml b/.github/workflows/wf_lint.yaml index a1438328132..21b4d3694b1 100644 --- a/.github/workflows/wf_lint.yaml +++ b/.github/workflows/wf_lint.yaml @@ -43,7 +43,7 @@ on: type: string r-version-list: description: 'stringified JSON array of R versions' - default: "[\"4.0.1\"]" + default: "[\"release\"]" required: false type: string os-version-list: @@ -136,7 +136,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 1 - - uses: r-lib/actions/setup-r@master + - uses: r-lib/actions/setup-r@v2 with: r-version: ${{ matrix.r-version }} - name: Cache R packages diff --git a/.github/workflows/wf_setup.yaml b/.github/workflows/wf_setup.yaml index 415caaf26c4..b91fcbcd7cf 100644 --- a/.github/workflows/wf_setup.yaml +++ b/.github/workflows/wf_setup.yaml @@ -73,7 +73,7 @@ jobs: steps: - name: Add reaction if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }} - uses: peter-evans/create-or-update-comment@v1 + uses: peter-evans/create-or-update-comment@v2 with: token: ${{ secrets.PAT }} repository: ${{ github.event.client_payload.github.payload.repository.full_name }} @@ -84,13 +84,13 @@ jobs: run: | SLASH_FORK="${{ github.event.client_payload.slash_command.args.named.fork }}" DEFAULT_FORK="${{ inputs.default-galaxy-fork }}" - echo "::set-output name=fork::${SLASH_FORK:-$DEFAULT_FORK}" + echo "fork=${SLASH_FORK:-$DEFAULT_FORK}" >> $GITHUB_OUTPUT SLASH_BRANCH="${{ github.event.client_payload.slash_command.args.named.branch }}" DEFAULT_BRANCH="${{ inputs.default-galaxy-branch }}" - echo "::set-output name=branch::${SLASH_BRANCH:-$DEFAULT_BRANCH}" + echo "branch=${SLASH_BRANCH:-$DEFAULT_BRANCH}" >> $GITHUB_OUTPUT SLASH_PLANEMO="${{ github.event.client_payload.slash_command.args.named.planemo-version }}" DEFAULT_PLANEMO="${{ inputs.default-planemo-version }}" - echo "::set-output name=planemo-version::${SLASH_PLANEMO:-$DEFAULT_PLANEMO}" + echo "planemo-version=${SLASH_PLANEMO:-$DEFAULT_PLANEMO}" >> $GITHUB_OUTPUT - name: Print used Galaxy for and branch run: | echo 'fork: ${{ steps.get-fork-branch.outputs.fork }}' @@ -121,7 +121,7 @@ jobs: echo 'event.after: ${{ github.event.after }}' - name: Determine latest commit in the Galaxy repo id: get-galaxy-sha - run: echo "::set-output name=galaxy-head-sha::$(git ls-remote https://github.com/${{ needs.fork-branch.outputs.galaxy-fork }}/galaxy refs/heads/${{ needs.fork-branch.outputs.galaxy-branch }} | cut -f1)" + run: echo "galaxy-head-sha=$(git ls-remote https://github.com/${{ needs.fork-branch.outputs.galaxy-fork }}/galaxy refs/heads/${{ needs.fork-branch.outputs.galaxy-branch }} | cut -f1)" >> $GITHUB_OUTPUT - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 0a15fb89c064a6c745ec78da957f213e574f39d3 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sun, 27 Nov 2022 17:31:02 +0100 Subject: [PATCH 17/29] Add cleaning of dotnet folder analogous to 6cccc5681698f3057e4475d9c646f4812a97a483 this will also add it to the PR workflow (which cant hurt) --- .github/workflows/wf_test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/wf_test.yaml b/.github/workflows/wf_test.yaml index 3dd93fbaca7..9195b820a2b 100644 --- a/.github/workflows/wf_test.yaml +++ b/.github/workflows/wf_test.yaml @@ -93,6 +93,8 @@ jobs: - name: Get number of CPU cores uses: SimenB/github-actions-cpu-cores@v1 id: cpu-cores + - name: Clean dotnet folder for space + run: rm -Rf /usr/share/dotnet - name: Planemo test uses: galaxyproject/planemo-ci-action@v1 id: test From 81e2b8bbccf533a280a3ad06b1852350274d5c4a Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sun, 27 Nov 2022 17:48:21 +0100 Subject: [PATCH 18/29] TMP adjust to test in fork --- .github/workflows/ci.yaml | 6 +++--- .github/workflows/pr.yaml | 12 ++++++------ .github/workflows/slash.yaml | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6fbee545453..8eb35a1de75 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,8 +9,8 @@ jobs: setup: name: Setup cache and determine changed repositories # adapt the owner name - if: ${{ github.repository_owner == 'galaxyproject' }} - uses: galaxyproject/tools-iuc/.github/workflows/wf_setup.yaml@master + if: ${{ github.repository_owner == 'bernt-matthias' }} + uses: bernt-matthias/tools-iuc/.github/workflows/wf_setup.yaml@master with: # default-galaxy-fork: galaxyproject # default-galaxy-branch: release_22.05 @@ -21,7 +21,7 @@ jobs: test: name: Test tools needs: setup - uses: galaxyproject/tools-iuc/.github/workflows/wf_test.yaml@master + uses: bernt-matthias/tools-iuc/.github/workflows/wf_test.yaml@master with: galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }} galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6ad8be194af..0dec21775ea 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -16,8 +16,8 @@ jobs: setup: name: Setup cache and determine changed repositories # adapt the owner name - if: ${{ github.repository_owner == 'galaxyproject' }} - uses: galaxyproject/tools-iuc/.github/workflows/wf_setup.yaml@master + if: ${{ github.repository_owner == 'bernt-matthias' }} + uses: bernt-matthias/tools-iuc/.github/workflows/wf_setup.yaml@master with: fetch-depth: 0 # set these variables if you want to update independent of the IUC repo @@ -32,7 +32,7 @@ jobs: lint: name: Lint needs: setup - uses: galaxyproject/tools-iuc/.github/workflows/wf_lint.yaml@master + uses: bernt-matthias/tools-iuc/.github/workflows/wf_lint.yaml@master with: repository-list: ${{ needs.setup.outputs.repository-list }} tool-list: ${{ needs.setup.outputs.tool-list }} @@ -48,7 +48,7 @@ jobs: test: name: Test tools needs: setup - uses: galaxyproject/tools-iuc/.github/workflows/wf_test.yaml@master + uses: bernt-matthias/tools-iuc/.github/workflows/wf_test.yaml@master with: galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }} galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }} @@ -63,9 +63,9 @@ jobs: # deploy the tools to the toolsheds (first TTS for testing) deploy: name: Deploy - if: ${{ github.repository_owner == 'galaxyproject' }} + if: ${{ github.repository_owner == 'bernt-matthias' }} needs: [setup, lint, test] - uses: galaxyproject/tools-iuc/.github/workflows/wf_deploy.yaml@master + uses: bernt-matthias/tools-iuc/.github/workflows/wf_deploy.yaml@master with: repository-list: ${{ needs.setup.outputs.repository-list }} galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} diff --git a/.github/workflows/slash.yaml b/.github/workflows/slash.yaml index 838257cbb0c..f741da13d05 100644 --- a/.github/workflows/slash.yaml +++ b/.github/workflows/slash.yaml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Slash Command Dispatch - if: github.repository_owner == 'galaxyproject' + if: github.repository_owner == 'bernt-matthias' uses: peter-evans/slash-command-dispatch@v3 with: token: ${{ secrets.PAT }} From 1eafcceae239e1ff2aebfa304614d2bb812104e7 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sun, 27 Nov 2022 18:04:57 +0100 Subject: [PATCH 19/29] finish planemo-version --- .github/workflows/ci.yaml | 1 + .github/workflows/pr.yaml | 1 + .github/workflows/wf_deploy.yaml | 11 +++++++++-- .github/workflows/wf_lint.yaml | 10 ++++++++-- .github/workflows/wf_setup.yaml | 4 ++-- .github/workflows/wf_test.yaml | 13 ++++++++++--- 6 files changed, 31 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8eb35a1de75..f0dc018bd22 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,5 +31,6 @@ jobs: chunk-list: ${{ needs.setup.outputs.chunk-list }} galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} # test-timeout: 900 + planemo-version: ${{ needs.setup.outputs.planemo-version }} secrets: PAT: ${{ secrets.PAT }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0dec21775ea..a7c6230d1ea 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -69,6 +69,7 @@ jobs: with: repository-list: ${{ needs.setup.outputs.repository-list }} galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} + planemo-version: ${{ inputs.planemo-version }} secrets: TTS_API_KEY: ${{ secrets.TTS_API_KEY }} TS_API_KEY: ${{ secrets.TS_API_KEY }} diff --git a/.github/workflows/wf_deploy.yaml b/.github/workflows/wf_deploy.yaml index 7fd9ea18ee8..eeccce29c64 100644 --- a/.github/workflows/wf_deploy.yaml +++ b/.github/workflows/wf_deploy.yaml @@ -15,6 +15,11 @@ on: default: "[\"3.7\"]" required: false type: string + planemo-version: + description: 'Planemo version to use' + default: "planemo" + required: false + type: string secrets: TTS_API_KEY: required: true @@ -42,7 +47,7 @@ jobs: id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }}_planemo_${{ steps.get-galaxy-sha.outputs.planemo-version }} - name: Deploy on testtoolshed uses: galaxyproject/planemo-ci-action@v1 with: @@ -50,6 +55,7 @@ jobs: repository-list: ${{ inputs.repository-list }} shed-target: testtoolshed shed-key: ${{ secrets.TTS_API_KEY }} + planemo-version: ${{ inputs.planemo-version }} continue-on-error: true - name: Deploy on toolshed uses: galaxyproject/planemo-ci-action@v1 @@ -57,4 +63,5 @@ jobs: mode: deploy repository-list: ${{ inputs.repository-list }} shed-target: toolshed - shed-key: ${{ secrets.TS_API_KEY }} \ No newline at end of file + shed-key: ${{ secrets.TS_API_KEY }} + planemo-version: ${{ inputs.planemo-version }} \ No newline at end of file diff --git a/.github/workflows/wf_lint.yaml b/.github/workflows/wf_lint.yaml index 21b4d3694b1..f95e16fa182 100644 --- a/.github/workflows/wf_lint.yaml +++ b/.github/workflows/wf_lint.yaml @@ -51,6 +51,11 @@ on: default: "[\"ubuntu-20.04\"]" required: false type: string + planemo-version: + description: 'Planemo version to use' + default: "planemo" + required: false + type: string outputs: result: description: "Result of the lint jobs" @@ -77,7 +82,7 @@ jobs: id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }}_planemo_${{ steps.get-galaxy-sha.outputs.planemo-version }} - name: Planemo lint uses: galaxyproject/planemo-ci-action@v1 id: lint @@ -87,6 +92,7 @@ jobs: fail-level: ${{ inputs.fail-level }} repository-list: ${{ inputs.repository-list }} tool-list: ${{ inputs.tool-list }} + planemo-version: ${{ inputs.planemo-version }} - uses: actions/upload-artifact@v3 if: ${{ failure() }} with: @@ -113,7 +119,7 @@ jobs: id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }}_planemo_${{ steps.get-galaxy-sha.outputs.planemo-version }} - name: Install flake8 run: pip install flake8 flake8-import-order - name: Flake8 diff --git a/.github/workflows/wf_setup.yaml b/.github/workflows/wf_setup.yaml index b91fcbcd7cf..ce40264b1b7 100644 --- a/.github/workflows/wf_setup.yaml +++ b/.github/workflows/wf_setup.yaml @@ -130,13 +130,13 @@ jobs: id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}_planemo_${{ steps.get-galaxy-sha.outputs.planemo-version }} - name: Cache .planemo uses: actions/cache@v3 id: cache-planemo with: path: ~/.planemo - key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} + key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}_planemo_${{ steps.get-galaxy-sha.outputs.planemo-version }} # Install the `wheel` package so that when installing other packages which # are not available as wheels, pip will build a wheel for them, which can be cached. - name: Install wheel diff --git a/.github/workflows/wf_test.yaml b/.github/workflows/wf_test.yaml index 9195b820a2b..cf68bbc460c 100644 --- a/.github/workflows/wf_test.yaml +++ b/.github/workflows/wf_test.yaml @@ -41,6 +41,11 @@ on: default: 900 required: false type: number + planemo-version: + description: 'Planemo version to use' + default: "planemo" + required: false + type: string secrets: PAT: required: false @@ -82,14 +87,14 @@ jobs: id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }}_planemo_${{ steps.get-galaxy-sha.outputs.planemo-version }} - name: Cache .planemo if: ${{ inputs.cache-planemo }} uses: actions/cache@v3 id: cache-planemo with: path: ~/.planemo - key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} + key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }}_planemo_${{ steps.get-galaxy-sha.outputs.planemo-version }} - name: Get number of CPU cores uses: SimenB/github-actions-cpu-cores@v1 id: cpu-cores @@ -108,6 +113,7 @@ jobs: galaxy-slots: ${{ steps.cpu-cores.outputs.count }} # Limit each test to 15 minutes test_timeout: ${{ inputs.test-timeout }} + planemo-version: ${{ inputs.planemo-version }} - uses: actions/upload-artifact@v3 with: name: 'Tool test output ${{ matrix.chunk }}' @@ -137,12 +143,13 @@ jobs: id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }}_planemo_${{ steps.get-galaxy-sha.outputs.planemo-version }} - name: Combine outputs uses: galaxyproject/planemo-ci-action@v1 id: combine with: mode: combine + planemo-version: ${{ inputs.planemo-version }} html-report: true - uses: actions/upload-artifact@v3 with: From 48af3dbbc84cb0b867384f14848decbcf90c56ef Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sun, 27 Nov 2022 18:13:06 +0100 Subject: [PATCH 20/29] planemo version cbt --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f0dc018bd22..8eb35a1de75 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,6 +31,5 @@ jobs: chunk-list: ${{ needs.setup.outputs.chunk-list }} galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} # test-timeout: 900 - planemo-version: ${{ needs.setup.outputs.planemo-version }} secrets: PAT: ${{ secrets.PAT }} From bb0dc2509a3246683741f29ce68b5e1841a1d5d7 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 8 Dec 2022 10:12:02 +0100 Subject: [PATCH 21/29] restore vars step --- .github/workflows/wf_test.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/wf_test.yaml b/.github/workflows/wf_test.yaml index cf68bbc460c..eb7d2809039 100644 --- a/.github/workflows/wf_test.yaml +++ b/.github/workflows/wf_test.yaml @@ -155,6 +155,10 @@ jobs: with: name: 'All tool test results' path: upload + - name: Create URL to the run output + if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }} + id: vars + run: echo "run-url=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT - name: Create comment if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }} uses: peter-evans/create-or-update-comment@v2 From b247248f0366bbf788c7a16fc08eda35625aa59f Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 8 Dec 2022 10:15:18 +0100 Subject: [PATCH 22/29] add planemo-version to output --- .github/workflows/wf_setup.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wf_setup.yaml b/.github/workflows/wf_setup.yaml index ce40264b1b7..0e6b5206299 100644 --- a/.github/workflows/wf_setup.yaml +++ b/.github/workflows/wf_setup.yaml @@ -91,10 +91,11 @@ jobs: SLASH_PLANEMO="${{ github.event.client_payload.slash_command.args.named.planemo-version }}" DEFAULT_PLANEMO="${{ inputs.default-planemo-version }}" echo "planemo-version=${SLASH_PLANEMO:-$DEFAULT_PLANEMO}" >> $GITHUB_OUTPUT - - name: Print used Galaxy for and branch + - name: Print used Galaxy fork and branch and planemo version run: | echo 'fork: ${{ steps.get-fork-branch.outputs.fork }}' echo 'branch: ${{ steps.get-fork-branch.outputs.branch }}' + echo 'planemo-version: ${{ steps.get-fork-branch.outputs.planemo-version }}' setup: name: Setup cache and determine changed repositories needs: fork-branch From 888c7896132aff41c353fbaf8107a6de0fd6bca8 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sun, 18 Dec 2022 11:46:18 +0100 Subject: [PATCH 23/29] Revert "TMP adjust to test in fork" This reverts commit 81e2b8bbccf533a280a3ad06b1852350274d5c4a. --- .github/workflows/ci.yaml | 6 +++--- .github/workflows/pr.yaml | 12 ++++++------ .github/workflows/slash.yaml | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8eb35a1de75..6fbee545453 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,8 +9,8 @@ jobs: setup: name: Setup cache and determine changed repositories # adapt the owner name - if: ${{ github.repository_owner == 'bernt-matthias' }} - uses: bernt-matthias/tools-iuc/.github/workflows/wf_setup.yaml@master + if: ${{ github.repository_owner == 'galaxyproject' }} + uses: galaxyproject/tools-iuc/.github/workflows/wf_setup.yaml@master with: # default-galaxy-fork: galaxyproject # default-galaxy-branch: release_22.05 @@ -21,7 +21,7 @@ jobs: test: name: Test tools needs: setup - uses: bernt-matthias/tools-iuc/.github/workflows/wf_test.yaml@master + uses: galaxyproject/tools-iuc/.github/workflows/wf_test.yaml@master with: galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }} galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a7c6230d1ea..e072fd4a301 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -16,8 +16,8 @@ jobs: setup: name: Setup cache and determine changed repositories # adapt the owner name - if: ${{ github.repository_owner == 'bernt-matthias' }} - uses: bernt-matthias/tools-iuc/.github/workflows/wf_setup.yaml@master + if: ${{ github.repository_owner == 'galaxyproject' }} + uses: galaxyproject/tools-iuc/.github/workflows/wf_setup.yaml@master with: fetch-depth: 0 # set these variables if you want to update independent of the IUC repo @@ -32,7 +32,7 @@ jobs: lint: name: Lint needs: setup - uses: bernt-matthias/tools-iuc/.github/workflows/wf_lint.yaml@master + uses: galaxyproject/tools-iuc/.github/workflows/wf_lint.yaml@master with: repository-list: ${{ needs.setup.outputs.repository-list }} tool-list: ${{ needs.setup.outputs.tool-list }} @@ -48,7 +48,7 @@ jobs: test: name: Test tools needs: setup - uses: bernt-matthias/tools-iuc/.github/workflows/wf_test.yaml@master + uses: galaxyproject/tools-iuc/.github/workflows/wf_test.yaml@master with: galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }} galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }} @@ -63,9 +63,9 @@ jobs: # deploy the tools to the toolsheds (first TTS for testing) deploy: name: Deploy - if: ${{ github.repository_owner == 'bernt-matthias' }} + if: ${{ github.repository_owner == 'galaxyproject' }} needs: [setup, lint, test] - uses: bernt-matthias/tools-iuc/.github/workflows/wf_deploy.yaml@master + uses: galaxyproject/tools-iuc/.github/workflows/wf_deploy.yaml@master with: repository-list: ${{ needs.setup.outputs.repository-list }} galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }} diff --git a/.github/workflows/slash.yaml b/.github/workflows/slash.yaml index f741da13d05..838257cbb0c 100644 --- a/.github/workflows/slash.yaml +++ b/.github/workflows/slash.yaml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Slash Command Dispatch - if: github.repository_owner == 'bernt-matthias' + if: github.repository_owner == 'galaxyproject' uses: peter-evans/slash-command-dispatch@v3 with: token: ${{ secrets.PAT }} From c21ad94ec4a0da7aba39bb25c67dd0a465f2b1d7 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sun, 18 Dec 2022 11:50:11 +0100 Subject: [PATCH 24/29] make flake8 plugins configurable --- .flake8_requirements | 2 ++ .github/workflows/wf_lint.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 .flake8_requirements diff --git a/.flake8_requirements b/.flake8_requirements new file mode 100644 index 00000000000..90225d01e9d --- /dev/null +++ b/.flake8_requirements @@ -0,0 +1,2 @@ +flake8 +flake8-import-order diff --git a/.github/workflows/wf_lint.yaml b/.github/workflows/wf_lint.yaml index f95e16fa182..574ba16ec2e 100644 --- a/.github/workflows/wf_lint.yaml +++ b/.github/workflows/wf_lint.yaml @@ -121,7 +121,7 @@ jobs: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }}_planemo_${{ steps.get-galaxy-sha.outputs.planemo-version }} - name: Install flake8 - run: pip install flake8 flake8-import-order + run: pip install -r .flake8_requirements - name: Flake8 run: echo '${{ inputs.repository-list }}' | xargs -d '\n' flake8 --output-file pylint_report.txt --tee - uses: actions/upload-artifact@v3 From 2735e371e21d5e6d24e5cdf39643e24e2c9ff29b Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sun, 18 Dec 2022 11:51:47 +0100 Subject: [PATCH 25/29] only run slash workflow if PAT is defined --- .github/workflows/slash.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/slash.yaml b/.github/workflows/slash.yaml index 838257cbb0c..0b551866b46 100644 --- a/.github/workflows/slash.yaml +++ b/.github/workflows/slash.yaml @@ -7,7 +7,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Slash Command Dispatch - if: github.repository_owner == 'galaxyproject' + # workaround for checking availablity of secret https://github.com/actions/runner/issues/520 + env: + PAT: ${{ secrets.PAT }} + if: github.repository_owner == 'galaxyproject' and env.PAT != '' uses: peter-evans/slash-command-dispatch@v3 with: token: ${{ secrets.PAT }} From d99b6f4eca13caa2a1f703f35fb721bc8df77e20 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sat, 21 Jan 2023 12:33:15 +0100 Subject: [PATCH 26/29] add deploy report job as in https://github.com/galaxyproject/tools-iuc/pull/5047 https://github.com/galaxyproject/tools-iuc/pull/5070 --- .github/workflows/pr.yaml | 1 - .github/workflows/wf_deploy.yaml | 24 +++++++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e072fd4a301..3a1e62d4773 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -74,7 +74,6 @@ jobs: TTS_API_KEY: ${{ secrets.TTS_API_KEY }} TS_API_KEY: ${{ secrets.TS_API_KEY }} - determine-success: name: Check workflow success needs: [setup, lint, test] diff --git a/.github/workflows/wf_deploy.yaml b/.github/workflows/wf_deploy.yaml index eeccce29c64..c4bb0065bfe 100644 --- a/.github/workflows/wf_deploy.yaml +++ b/.github/workflows/wf_deploy.yaml @@ -64,4 +64,26 @@ jobs: repository-list: ${{ inputs.repository-list }} shed-target: toolshed shed-key: ${{ secrets.TS_API_KEY }} - planemo-version: ${{ inputs.planemo-version }} \ No newline at end of file + planemo-version: ${{ inputs.planemo-version }} + + deploy-report: + name: Report deploy status + needs: [deploy] + if: ${{ always() && needs.deploy.result != 'success' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' ) }} + runs-on: ubuntu-latest + steps: + # report to the PR if deployment failed + - name: Get PR object + uses: 8BitJonny/gh-get-current-pr@2.2.0 + id: getpr + with: + sha: ${{ github.event.after }} + - name: Create comment + uses: peter-evans/create-or-update-comment@v2 + with: + token: ${{ secrets.PAT }} + issue-number: ${{ steps.getpr.outputs.number }} + body: | + Attention: deployment ${{ needs.deploy.result }}! + + https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} \ No newline at end of file From 3a19c7b0e443612595a812f309525614fcdcd60c Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 27 Jan 2023 11:53:31 +0100 Subject: [PATCH 27/29] Revert "test bumps" This reverts commit ed276d18568e7c880168d002954bf2d01d8a8614. --- tools/extract_genomic_dna/extract_genomic_dna.xml | 2 +- tools/gwastools/manhattan_plot.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/extract_genomic_dna/extract_genomic_dna.xml b/tools/extract_genomic_dna/extract_genomic_dna.xml index e765bc382b3..88ac3fb421c 100644 --- a/tools/extract_genomic_dna/extract_genomic_dna.xml +++ b/tools/extract_genomic_dna/extract_genomic_dna.xml @@ -1,4 +1,4 @@ - + using coordinates from assembled/unassembled genomes bx-python diff --git a/tools/gwastools/manhattan_plot.xml b/tools/gwastools/manhattan_plot.xml index 1553a77dff0..546de7f05ca 100644 --- a/tools/gwastools/manhattan_plot.xml +++ b/tools/gwastools/manhattan_plot.xml @@ -1,4 +1,4 @@ - + Generate plots for use with GWAS pipelines gwastools From 69852ebd66efaedaf4c6be6158903d6ced774e6b Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 27 Jan 2023 11:53:48 +0100 Subject: [PATCH 28/29] Revert "bump tool failing testing" This reverts commit f7697a558406205db55209712455a36b1de77373. --- tools/basil/basil.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/basil/basil.xml b/tools/basil/basil.xml index ab9968ad59a..5d22e1560f5 100644 --- a/tools/basil/basil.xml +++ b/tools/basil/basil.xml @@ -1,4 +1,4 @@ - + Breakpoint detection, including large insertions anise_basil @@ -23,7 +23,7 @@ - + From ef6e04aaae01f169d569b713054bd63d1103a624 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 27 Jan 2023 11:56:18 +0100 Subject: [PATCH 29/29] fix merge mistake --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 04838d2f06a..d1f890942fb 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -30,6 +30,7 @@ jobs: # - flake8 of Python scripts # - lintr of R scripts lint: + name: Lint needs: setup uses: galaxyproject/tools-iuc/.github/workflows/wf_lint.yaml@master with: