diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 1643b32da320..b188c6614838 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -16,13 +16,9 @@ # name: Backend + on: - push: - pull_request: - branches: - - dev - paths-ignore: - - 'seatunnel-ui/**' + workflow_call: concurrency: group: backend-${{ github.event.pull_request.number || github.ref }} @@ -30,7 +26,6 @@ concurrency: jobs: license-header: - if: github.repository == '${{github.actor}}/seatunnel' name: License header runs-on: ubuntu-latest timeout-minutes: 10 @@ -42,7 +37,6 @@ jobs: uses: apache/skywalking-eyes@985866ce7e324454f61e22eb2db2e998db09d6f3 code-style: - if: github.repository == '${{github.actor}}/seatunnel' name: Code style runs-on: ubuntu-latest timeout-minutes: 10 @@ -54,7 +48,6 @@ jobs: run: ./mvnw --batch-mode --quiet --no-snapshot-updates clean spotless:check dead-link: - if: github.repository == '${{github.actor}}/seatunnel' name: Dead links runs-on: ubuntu-latest timeout-minutes: 30 @@ -67,7 +60,6 @@ jobs: done sanity-check: - if: github.repository == '${{github.actor}}/seatunnel' name: Sanity check results needs: [ license-header, code-style, dead-link ] runs-on: ubuntu-latest @@ -81,7 +73,6 @@ jobs: changes: runs-on: ubuntu-latest - if: github.repository == '${{github.actor}}/seatunnel' timeout-minutes: 10 outputs: api: ${{ steps.filter.outputs.api }} diff --git a/.github/workflows/build_main.yml b/.github/workflows/build_main.yml new file mode 100644 index 000000000000..f4816940a888 --- /dev/null +++ b/.github/workflows/build_main.yml @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: "Build" + +on: + push: + branches: + - '**' + +jobs: + call-build-and-test: + permissions: + packages: write + name: Run + uses: ./.github/workflows/backend.yml diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml deleted file mode 100644 index 032d1d82ec84..000000000000 --- a/.github/workflows/license.yml +++ /dev/null @@ -1,73 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: License - -on: - push: - pull_request: - branches: - - dev - paths-ignore: - - '**/*.md' - -concurrency: - group: lc-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - check-license: - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v2 - - name: Check License Header - uses: apache/skywalking-eyes/header@501a28d2fb4a9b962661987e50cf0219631b32ff - auto-license: - name: Auto License - runs-on: ubuntu-latest - timeout-minutes: 30 - # Have a buggy in https://github.com/apache/seatunnel/pull/1642, Can trigger when commit message contains - # keyword `[ci-auto-license]`. - if: "contains(toJSON(github.event.commits.*.message), '[ci-auto-license]')" - env: - MAVEN_OPTS: -Xmx2G -Xms2G - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v2 - with: - java-version: 8 - distribution: 'adopt' - - name: Set up Python 3 - uses: actions/setup-python@v2 - with: - python-version: '3.x' - architecture: 'x64' - - name: Cache local Maven repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - name: Generate THIRD-PARTY - run: | - ./mvnw clean license:aggregate-add-third-party -DskipTests -U - - name: Check LICENSE file - run: | - python3 tools/dependencies/license.py seatunnel-dist/target/THIRD-PARTY.txt seatunnel-dist/release-docs/LICENSE true diff --git a/.github/workflows/notify_test_workflow.yml b/.github/workflows/notify_test_workflow.yml new file mode 100644 index 000000000000..d0a920baabef --- /dev/null +++ b/.github/workflows/notify_test_workflow.yml @@ -0,0 +1,152 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Intentionally has a general name. +# because the test status check created in GitHub Actions +# currently randomly picks any associated workflow. +# So, the name was changed to make sense in that context too. +# See also https://github.community/t/specify-check-suite-when-creating-a-checkrun/118380/10 +name: On pull request update +on: + pull_request_target: + types: [opened, reopened, synchronize] + +jobs: + notify: + name: Notify test workflow + runs-on: ubuntu-20.04 + permissions: + actions: read + checks: write + steps: + - name: "Notify test workflow" + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const endpoint = 'GET /repos/:owner/:repo/actions/workflows/:id/runs?&branch=:branch' + const check_run_endpoint = 'GET /repos/:owner/:repo/commits/:ref/check-runs' + + // TODO: Should use pull_request.user and pull_request.user.repos_url? + // If a different person creates a commit to another forked repo, + // it wouldn't be able to detect. + const params = { + owner: context.payload.pull_request.head.repo.owner.login, + repo: context.payload.pull_request.head.repo.name, + id: 'build_main.yml', + branch: context.payload.pull_request.head.ref, + } + const check_run_params = { + owner: context.payload.pull_request.head.repo.owner.login, + repo: context.payload.pull_request.head.repo.name, + ref: context.payload.pull_request.head.ref, + } + + console.log('Ref: ' + context.payload.pull_request.head.ref) + console.log('SHA: ' + context.payload.pull_request.head.sha) + + // Wait 3 seconds to make sure the fork repository triggered a workflow. + await new Promise(r => setTimeout(r, 3000)) + + let runs + try { + runs = await github.request(endpoint, params) + } catch (error) { + console.error(error) + // Assume that runs were not found. + } + + const name = 'Build' + const head_sha = context.payload.pull_request.head.sha + let status = 'queued' + console.log('runs: ' + runs) + if (!runs || runs.data.workflow_runs.length === 0) { + status = 'completed' + const conclusion = 'action_required' + + github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + name: name, + head_sha: head_sha, + status: status, + conclusion: conclusion, + output: { + title: 'Workflow run detection failed', + summary: ` + Unable to detect the workflow run for testing the changes in your PR. + + 1. If you did not enable GitHub Actions in your forked repository, please enable it by clicking the button as shown in the image below. See also [Disabling or limiting GitHub Actions for a repository](https://docs.github.com/en/github/administering-a-repository/disabling-or-limiting-github-actions-for-a-repository) for more details. + 2. It is possible your branch is based on the old \`dev\` branch in Apache SeaTunnel, please sync your branch to the latest master branch. For example as below: + \`\`\`bash + git fetch upstream + git rebase upstream/master + git push origin YOUR_BRANCH --force + \`\`\``, + images: [ + { + alt: 'enabling workflows button', + image_url: 'https://raw.githubusercontent.com/apache/spark/master/.github/workflows/images/workflow-enable-button.png' + } + ] + } + }) + } else { + const run_id = runs.data.workflow_runs[0].id + + if (runs.data.workflow_runs[0].head_sha != context.payload.pull_request.head.sha) { + throw new Error('There was a new unsynced commit pushed. Please retrigger the workflow.'); + } + + // Here we get check run ID to provide Check run view instead of Actions view, see also SPARK-37879. + const check_runs = await github.request(check_run_endpoint, check_run_params) + const check_run_head = check_runs.data.check_runs.filter(r => r.name === "Run / License header")[0] + + if (check_run_head.head_sha != context.payload.pull_request.head.sha) { + throw new Error('There was a new unsynced commit pushed. Please retrigger the workflow.'); + } + + const check_run_url = 'https://github.com/' + + context.payload.pull_request.head.repo.full_name + + '/runs/' + + check_run_head.id + + const actions_url = 'https://github.com/' + + context.payload.pull_request.head.repo.full_name + + '/actions/runs/' + + run_id + + github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + name: name, + head_sha: head_sha, + status: status, + output: { + title: 'Test results', + summary: '[See test results](' + check_run_url + ')', + text: JSON.stringify({ + owner: context.payload.pull_request.head.repo.owner.login, + repo: context.payload.pull_request.head.repo.name, + run_id: run_id + }) + }, + details_url: actions_url, + }) + } diff --git a/.github/workflows/update_build_status.yml b/.github/workflows/update_build_status.yml new file mode 100644 index 000000000000..05cf4914a25c --- /dev/null +++ b/.github/workflows/update_build_status.yml @@ -0,0 +1,108 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Update build status workflow + +on: + schedule: + - cron: "*/15 * * * *" + +jobs: + update: + name: Update build status + runs-on: ubuntu-20.04 + permissions: + actions: read + checks: write + steps: + - name: "Update build status" + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const endpoint = 'GET /repos/:owner/:repo/pulls?state=:state' + const params = { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open' + } + + // See https://docs.github.com/en/graphql/reference/enums#mergestatestatus + const maybeReady = ['behind', 'clean', 'draft', 'has_hooks', 'unknown', 'unstable']; + + // Iterate open PRs + for await (const prs of github.paginate.iterator(endpoint,params)) { + // Each page + for await (const pr of prs.data) { + console.log('SHA: ' + pr.head.sha) + console.log(' Mergeable status: ' + pr.mergeable_state) + if (pr.mergeable_state == null || maybeReady.includes(pr.mergeable_state)) { + const checkRuns = await github.request('GET /repos/{owner}/{repo}/commits/{ref}/check-runs', { + owner: context.repo.owner, + repo: context.repo.repo, + ref: pr.head.sha + }) + + // Iterator GitHub Checks in the PR + for await (const cr of checkRuns.data.check_runs) { + if (cr.name == 'Build' && cr.conclusion != "action_required") { + // text contains parameters to make request in JSON. + const params = JSON.parse(cr.output.text) + + // Get the workflow run in the forked repository + let run + try { + run = await github.request('GET /repos/{owner}/{repo}/actions/runs/{run_id}', params) + } catch (error) { + console.error(error) + // Run not found. This can happen when the PR author removes GitHub Actions runs or + // disalbes GitHub Actions. + continue + } + + // Keep syncing the status of the checks + if (run.data.status == 'completed') { + console.log(' Run ' + cr.id + ': set status (' + run.data.status + ') and conclusion (' + run.data.conclusion + ')') + const response = await github.request('PATCH /repos/{owner}/{repo}/check-runs/{check_run_id}', { + owner: context.repo.owner, + repo: context.repo.repo, + check_run_id: cr.id, + output: cr.output, + status: run.data.status, + conclusion: run.data.conclusion, + details_url: run.data.details_url + }) + } else { + console.log(' Run ' + cr.id + ': set status (' + run.data.status + ')') + const response = await github.request('PATCH /repos/{owner}/{repo}/check-runs/{check_run_id}', { + owner: context.repo.owner, + repo: context.repo.repo, + check_run_id: cr.id, + output: cr.output, + status: run.data.status, + details_url: run.data.details_url + }) + } + + break + } + } + } + } + }