build.yaml #2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: | |
| pull_request: | |
| types: [ opened, edited ] | |
| push: | |
| workflow_dispatch: | |
| inputs: | |
| recreate_vm: | |
| required: true | |
| type: string | |
| description: "Whether to recreate the VM" | |
| default: "false" | |
| jobs: | |
| create-runner: | |
| uses: gitpod-io/gce-github-runner/.github/workflows/create-vm.yml@main | |
| secrets: | |
| runner_token: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_TOKEN }} | |
| gcp_credentials: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_GCP_CREDENTIALS }} | |
| concurrency: | |
| group: ${{ github.ref == 'refs/heads/main' && github.run_id || github.sha }}-create-runner | |
| cancel-in-progress: false | |
| configuration: | |
| name: Configure job parameters | |
| runs-on: ${{ needs.create-runner.outputs.label }} | |
| needs: [ create-runner ] | |
| concurrency: | |
| group: ${{ github.ref == 'refs/heads/main' && github.run_id || github.sha }}-configuration | |
| cancel-in-progress: true | |
| outputs: | |
| is_main_branch: ${{ (github.head_ref || github.ref) == 'refs/heads/main' }} | |
| version: ${{ steps.branches.outputs.sanitized-branch-name }}-gha.${{github.run_number}} | |
| preview_enable: ${{ contains( steps.pr-details.outputs.pr_body, '[x] /werft with-preview') || (steps.output.outputs.with_integration_tests != '') }} | |
| preview_name: ${{ github.head_ref || github.ref_name }} | |
| preview_infra_provider: ${{ contains( steps.pr-details.outputs.pr_body, '[x] /werft with-gce-vm') && 'gce' || 'harvester' }} | |
| build_no_cache: ${{ contains( steps.pr-details.outputs.pr_body, '[x] leeway-no-cache') }} | |
| build_no_test: ${{ contains( steps.pr-details.outputs.pr_body, '[x] /werft no-test') }} | |
| with_large_vm: ${{ contains( steps.pr-details.outputs.pr_body, '[X] /werft with-large-vm') || (steps.output.outputs.with_integration_tests != '') }} | |
| publish_to_npm: ${{ contains( steps.pr-details.outputs.pr_body, '[X] /werft publish-to-npm') }} | |
| publish_to_jbmp: ${{ contains( steps.pr-details.outputs.pr_body, '[X] /werft publish-to-jb-marketplace') }} | |
| with_preemptible: ${{ contains( steps.pr-details.outputs.pr_body, '[X] /werft preemptible') }} | |
| with_dedicated_emulation: ${{ contains( steps.pr-details.outputs.pr_body, '[X] with-dedicated-emulation') }} | |
| analytics: ${{ steps.output.outputs.analytics }} | |
| workspace_feature_flags: ${{ steps.output.outputs.workspace_feature_flags }} | |
| pr_no_diff_skip: ${{ steps.pr-diff.outputs.pr_no_diff_skip }} | |
| with_werft: ${{ steps.output.outputs.with-werft }} | |
| with_integration_tests: ${{ steps.output.outputs.with_integration_tests }} | |
| with_monitoring: ${{ contains( steps.pr-details.outputs.pr_body, '[x] with-monitoring') }} | |
| latest_ide_version: ${{ contains( steps.pr-details.outputs.pr_body, '[x] latest-ide-version=true') }} | |
| leeway_cache_bucket: ${{ steps.output.outputs.leeway_cache_bucket }} | |
| pr_number: ${{ steps.pr-details.outputs.number }} | |
| pr_body: ${{ steps.pr-details.outputs.pr_body }} | |
| steps: | |
| - name: "Determine Branch" | |
| id: branches | |
| uses: transferwise/sanitize-branch-name@v1 | |
| # Since we trigger this worklow on other event types, besides pull_request | |
| # We use this action to help us get the pr body, as it's not included in push/workflow_dispatch events | |
| - uses: 8BitJonny/gh-get-current-pr@2.2.0 | |
| id: pr-details | |
| with: | |
| sha: ${{ github.event.pull_request.head.sha }} | |
| # If the PR got edited, but no checkbox changes occurred, we don't want to run the whole build | |
| # Therefore we set a flag and use it to skip the next jobs | |
| - name: "Check PR diff" | |
| id: pr-diff | |
| if: (github.event_name == 'pull_request' && github.event.action == 'edited') | |
| env: | |
| PR_DESC: "${{ steps.pr-details.outputs.pr_body }}" | |
| OLD_BODY: "${{ github.event.changes.body.from }}" | |
| run: | | |
| if ! diff <(echo "$OLD_BODY") <(echo "$PR_DESC") | grep -e '\[x\]' -e '\[X\]'; then | |
| echo "pr_no_diff_skip=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: "Set outputs" | |
| id: output | |
| env: | |
| PR_DESC: "${{ steps.pr-details.outputs.pr_body }}" | |
| MAIN_BRANCH: ${{ (github.head_ref || github.ref) == 'refs/heads/main' }} | |
| shell: bash | |
| run: | | |
| { | |
| echo "workspace_feature_flags=$(echo "$PR_DESC" | grep -oiP '(?<=\[x\] workspace-feature-flags=).*?(?=\s*$)')" | |
| echo "with_integration_tests=$(echo "$PR_DESC" | grep -oiP '(?<=\[x\] with-integration-tests=).*?(?=\s*$)')" | |
| echo "analytics=$(echo "$PR_DESC" | grep -oiP '(?<=\[x\] analytics=).*?(?=\s*$)')" | |
| echo "leeway_cache_bucket=$([[ "$MAIN_BRANCH" = "true" ]] && echo "gitpod-core-leeway-cache-main" || echo "gitpod-core-leeway-cache-branch")" | |
| } >> $GITHUB_OUTPUT | |
| build-previewctl: | |
| name: Build previewctl | |
| if: | | |
| (needs.configuration.outputs.pr_no_diff_skip != 'true') && | |
| (needs.configuration.outputs.preview_enable == 'true') | |
| needs: [ configuration, create-runner ] | |
| concurrency: | |
| group: ${{ github.ref == 'refs/heads/main' && github.run_id || github.sha }}-build-previewctl | |
| cancel-in-progress: ${{ needs.configuration.outputs.is_main_branch == 'false' }} | |
| runs-on: ${{ needs.create-runner.outputs.label }} | |
| container: | |
| image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:at-user-getauthenticated-dashboard-gha.20954 | |
| outputs: | |
| previewctl_hash: ${{ steps.build.outputs.previewctl_hash }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-environment | |
| with: | |
| sa_key: ${{ secrets.GCP_CREDENTIALS }} | |
| leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }} | |
| - name: Build previewctl | |
| id: build | |
| shell: bash | |
| run: | | |
| leeway build dev/preview/previewctl:docker -Dversion="${{needs.configuration.outputs.version}}" | |
| echo "previewctl_hash=$(leeway describe dev/preview/previewctl:docker -Dversion="${{needs.configuration.outputs.version}}" -t '{{ .Metadata.Version }}')" >> $GITHUB_OUTPUT | |
| infrastructure: | |
| needs: [ configuration, build-previewctl, create-runner ] | |
| if: | | |
| (needs.configuration.outputs.pr_no_diff_skip != 'true') && | |
| (needs.configuration.outputs.preview_enable == 'true') && | |
| (needs.configuration.outputs.is_main_branch != 'true') | |
| runs-on: ${{ needs.create-runner.outputs.label }} | |
| concurrency: | |
| group: ${{ github.ref == 'refs/heads/main' && github.run_id || github.sha }}-infrastructure | |
| cancel-in-progress: true | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Create preview environment infrastructure | |
| id: create | |
| uses: ./.github/actions/preview-create | |
| with: | |
| name: ${{ needs.configuration.outputs.preview_name }} | |
| sa_key: ${{ secrets.GCP_CREDENTIALS }} | |
| infrastructure_provider: ${{ needs.configuration.outputs.preview_infra_provider }} | |
| previewctl_hash: ${{ needs.build-previewctl.outputs.previewctl_hash }} | |
| large_vm: ${{ needs.configuration.outputs.with_large_vm }} | |
| preemptible: ${{ needs.configuration.outputs.with_preemptible }} | |
| recreate_vm: ${{ inputs.recreate_vm }} | |
| build-gitpod: | |
| name: Build Gitpod | |
| needs: [ configuration, create-runner ] | |
| runs-on: ${{ needs.create-runner.outputs.label }} | |
| concurrency: | |
| group: ${{ github.ref == 'refs/heads/main' && github.run_id || github.sha }}-build-gitpod | |
| cancel-in-progress: ${{ needs.configuration.outputs.is_main_branch == 'false' }} | |
| services: | |
| mysql: | |
| image: bitnami/mysql:8.0.33-debian-11-r24 | |
| env: | |
| MYSQL_ROOT_PASSWORD: test | |
| #MYSQL_TCP_PORT: 23306 bitnami/mysql does not honor this, but has it's own: | |
| MYSQL_PORT_NUMBER: 23306 | |
| MYSQL_AUTHENTICATION_PLUGIN: mysql_native_password | |
| ports: | |
| - 23306:23306 | |
| redis: | |
| image: redis | |
| ports: | |
| - 6379:6379 | |
| container: | |
| image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:at-user-getauthenticated-dashboard-gha.20954 | |
| env: | |
| DB_HOST: "mysql" | |
| DB_PORT: "23306" | |
| REDIS_HOST: "redis" | |
| # GitHub action + MySQL 8.0 need longer to initialize | |
| DB_RETRIES: 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-environment | |
| with: | |
| sa_key: ${{ secrets.GCP_CREDENTIALS }} | |
| leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }} | |
| - name: Leeway Vet | |
| shell: bash | |
| run: | | |
| leeway vet --ignore-warnings | |
| - name: Pre-Commit Checks | |
| shell: bash | |
| run: | | |
| RESULT=0 | |
| pre-commit run --show-diff-on-failure || RESULT=$? | |
| if [ $RESULT -ne 0 ]; then | |
| echo "::error title=The Pre-Commit Checks Failed.::Please run 'pre-commit run --show-diff-on-failure'" | |
| fi | |
| exit "$RESULT" | |
| - name: Get Secrets from GCP | |
| id: "secrets" | |
| uses: "google-github-actions/get-secretmanager-secrets@v1" | |
| with: | |
| secrets: |- | |
| segment-io-token:gitpod-core-dev/segment-io-token | |
| npm-auth-token:gitpod-core-dev/npm-auth-token | |
| jb-marketplace-publish-token:gitpod-core-dev/jb-marketplace-publish-token | |
| codecov-token:gitpod-core-dev/codecov | |
| - name: Dev Build | |
| id: dev-build | |
| env: | |
| JAVA_HOME: /home/gitpod/.sdkman/candidates/java/current | |
| VERSION: ${{needs.configuration.outputs.version}} | |
| shell: bash | |
| run: | | |
| RESULT=0 | |
| set -x | |
| leeway build dev:all \ | |
| --docker-build-options network=host \ | |
| --cache remote \ | |
| -Dversion=$VERSION \ | |
| -DlocalAppVersion=$VERSION \ | |
| -DimageRepoBase=eu.gcr.io/gitpod-core-dev/dev \ | |
| --report large-report.html || RESULT=$? | |
| set +x | |
| # capture logs for the dev/image build, they're too big for GITHUB_STEP_SUMMARY | |
| pup -p -f large-report.html '[id="dev/image:docker"] + details + details' > remove.txt | |
| # remove logs for the dev/image build output from the report file | |
| grep -vxFf remove.txt large-report.html > report.html | |
| cat report.html >> $GITHUB_STEP_SUMMARY | |
| exit $RESULT | |
| - name: Leeway Build | |
| id: leeway | |
| shell: bash | |
| env: | |
| NODE_OPTIONS: "--max_old_space_size=4096" | |
| JAVA_HOME: /home/gitpod/.sdkman/candidates/java/current | |
| VERSION: ${{needs.configuration.outputs.version}} | |
| SEGMENT_IO_TOKEN: "${{ steps.secrets.outputs.segment-io-token }}" | |
| PR_NO_CACHE: ${{needs.configuration.outputs.build_no_cache}} | |
| PR_NO_TEST: ${{needs.configuration.outputs.build_no_test}} | |
| NPM_AUTH_TOKEN: "${{ steps.secrets.outputs.npm-auth-token }}" | |
| PUBLISH_TO_NPM: ${{ needs.configuration.outputs.publish_to_npm == 'true' || needs.configuration.outputs.is_main_branch == 'true' }} | |
| JB_MARKETPLACE_PUBLISH_TOKEN: "${{ steps.secrets.outputs.jb-marketplace-publish-token }}" | |
| PUBLISH_TO_JBPM: ${{ needs.configuration.outputs.publish_to_jbmp == 'true' || needs.configuration.outputs.is_main_branch == 'true' }} | |
| CODECOV_TOKEN: "${{ steps.secrets.outputs.codecov-token }}" | |
| run: | | |
| [[ "$PR_NO_CACHE" = "true" ]] && CACHE="none" || CACHE="remote" | |
| [[ "$PR_NO_TEST" = "true" ]] && TEST="--dont-test" || TEST="" | |
| [[ "${PUBLISH_TO_NPM}" = 'true' ]] && NPM_PUBLISH_TRIGGER=$(date +%s%3N) || NPM_PUBLISH_TRIGGER="false" | |
| RESULT=0 | |
| set -x | |
| # CI=true is a var set by GHA. Unsetting it for the build, as yarn builds treat warnings as errors if that var is set to true | |
| CI= leeway build \ | |
| --cache $CACHE \ | |
| $TEST \ | |
| -Dversion=$VERSION \ | |
| --docker-build-options network=host \ | |
| --max-concurrent-tasks 1 \ | |
| -DlocalAppVersion=$VERSION \ | |
| -DSEGMENT_IO_TOKEN=$SEGMENT_IO_TOKEN \ | |
| -DpublishToNPM="${PUBLISH_TO_NPM}" \ | |
| -DnpmPublishTrigger="${NPM_PUBLISH_TRIGGER}" \ | |
| -DpublishToJBMarketplace="${PUBLISH_TO_JBPM}" \ | |
| -DimageRepoBase=eu.gcr.io/gitpod-core-dev/build \ | |
| --report report.html || RESULT=$? | |
| set +x | |
| cat report.html >> $GITHUB_STEP_SUMMARY | |
| SUMMARY_SIZE=$(wc -c < $GITHUB_STEP_SUMMARY) | |
| if [[ $SUMMARY_SIZE -gt 1000000 ]]; then | |
| head -c 1000000 $GITHUB_STEP_SUMMARY > $GITHUB_STEP_SUMMARY.tmp | |
| mv $GITHUB_STEP_SUMMARY.tmp $GITHUB_STEP_SUMMARY | |
| ( | |
| echo '.....' | |
| echo '' | |
| echo ':x: **WARNING: Summary is too large and has been truncated.**' | |
| echo '' | |
| ) >> $GITHUB_STEP_SUMMARY | |
| fi | |
| exit $RESULT | |
| - name: Tag the release | |
| if: github.ref == 'refs/heads/main' | |
| run: | | |
| git config --global user.name $GITHUB_USER | |
| git config --global user.email $GITHUB_EMAIL | |
| git tag -a ${VERSION} -m "Release ${VERSION}" | |
| git push origin ${VERSION} | |
| env: | |
| GITHUB_USER: roboquat | |
| GITHUB_EMAIL: roboquat@gitpod.io | |
| VERSION: ${{ needs.configuration.outputs.version }} | |
| install-app: | |
| runs-on: ${{ needs.create-runner.outputs.label }} | |
| needs: [ configuration, build-gitpod, create-runner ] | |
| if: ${{ needs.configuration.outputs.is_main_branch == 'true' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| env: [ staging, production ] | |
| steps: | |
| - uses: gitpod-io/gh-app-auth@v0.1 | |
| id: auth | |
| with: | |
| private-key: ${{ secrets.ACTIONS_APP_PKEY }} | |
| app-id: 308947 | |
| installation-id: 35574470 | |
| - name: trigger installation | |
| uses: actions/github-script@v6 | |
| with: | |
| github-token: ${{ steps.auth.outputs.token }} | |
| script: | | |
| const result = await github.rest.actions.createWorkflowDispatch({ | |
| owner: context.repo.owner, | |
| repo: 'gitpod-dedicated', | |
| workflow_id: 'install-app.yaml', | |
| ref: 'main', | |
| inputs: { | |
| "version": '${{ needs.configuration.outputs.version }}', | |
| "repo": "https://github.com/gitpod-io/gitpod", | |
| "commit": "${{github.sha}}", | |
| "release": "${{ matrix.env }}", | |
| "env": "${{ matrix.env }}" | |
| } | |
| }) | |
| install: | |
| name: "Install Gitpod" | |
| needs: | |
| - configuration | |
| - build-previewctl | |
| - build-gitpod | |
| - infrastructure | |
| - create-runner | |
| runs-on: ${{ needs.create-runner.outputs.label }} | |
| concurrency: | |
| group: ${{ github.ref == 'refs/heads/main' && github.run_id || github.sha }}-install | |
| cancel-in-progress: ${{ needs.configuration.outputs.is_main_branch == 'false' }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Deploy Gitpod to the preview environment | |
| id: deploy-gitpod | |
| uses: ./.github/actions/deploy-gitpod | |
| with: | |
| name: ${{ needs.configuration.outputs.preview_name }} | |
| version: ${{needs.configuration.outputs.version}} | |
| sa_key: ${{ secrets.GCP_CREDENTIALS }} | |
| previewctl_hash: ${{ needs.build-previewctl.outputs.previewctl_hash }} | |
| with_dedicated_emu: ${{needs.configuration.outputs.with_dedicated_emulation}} | |
| analytics: ${{needs.configuration.outputs.analytics}} | |
| workspace_feature_flags: ${{needs.configuration.outputs.workspace_feature_flags}} | |
| - uses: actions/github-script@v6 | |
| if: needs.configuration.outputs.pr_number != '' && contains(needs.configuration.outputs.pr_body, 'gitpod:summary') | |
| with: | |
| script: | | |
| const prNumber = ${{ needs.configuration.outputs.pr_number }}; | |
| const summaryEncoded = `${{ steps.deploy-gitpod.outputs.report }}`; | |
| const summary = Buffer.from(summaryEncoded, 'base64').toString('utf8'); | |
| const { data: pr } = await github.rest.pulls.get({ | |
| ...context.repo, | |
| pull_number: prNumber, | |
| }); | |
| const prBody = pr.body; | |
| const newBody = prBody.replace('gitpod:summary', summary); | |
| await github.rest.pulls.update({ | |
| ...context.repo, | |
| pull_number: prNumber, | |
| body: newBody, | |
| }); | |
| monitoring: | |
| name: "Install Monitoring Satellite" | |
| needs: [ infrastructure, build-previewctl, create-runner ] | |
| runs-on: ${{ needs.create-runner.outputs.label }} | |
| if: needs.configuration.outputs.with_monitoring == 'true' | |
| concurrency: | |
| group: ${{ github.ref == 'refs/heads/main' && github.run_id || github.sha }}-monitoring | |
| cancel-in-progress: true | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Deploy monitoring satellite to the preview environment | |
| id: deploy-monitoring-satellite | |
| uses: ./.github/actions/deploy-monitoring-satellite | |
| with: | |
| sa_key: ${{ secrets.GCP_CREDENTIALS }} | |
| previewctl_hash: ${{ needs.build-previewctl.outputs.previewctl_hash }} | |
| integration-test: | |
| name: "Run integration test" | |
| needs: | |
| - configuration | |
| - build-previewctl | |
| - build-gitpod | |
| - infrastructure | |
| - install | |
| - create-runner | |
| runs-on: ${{ needs.create-runner.outputs.label }} | |
| container: | |
| image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:at-user-getauthenticated-dashboard-gha.20954 | |
| if: needs.configuration.outputs.with_integration_tests != '' | |
| concurrency: | |
| group: ${{ needs.configuration.outputs.preview_name }}-integration-test | |
| cancel-in-progress: true | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup-environment | |
| with: | |
| sa_key: ${{ secrets.GCP_CREDENTIALS }} | |
| leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }} | |
| - name: Run integration test | |
| id: integration-test | |
| uses: ./.github/actions/integration-tests | |
| with: | |
| preview_name: ${{ needs.configuration.outputs.preview_name }} | |
| test_suite: ${{ needs.configuration.outputs.with_integration_tests }} | |
| notify_slack_webhook: '' | |
| sa_key: ${{ secrets.GCP_CREDENTIALS }} | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| latest_ide_version: ${{ needs.configuration.outputs.latest_ide_version }} | |
| test_build_id: ${{ github.run_id }} | |
| test_build_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| test_build_ref: ${{ github.head_ref || github.ref }} | |
| workspace-integration-tests-main: | |
| name: "Run workspace integration tests on main branch" | |
| needs: | |
| - configuration | |
| - build-gitpod | |
| - create-runner | |
| if: needs.configuration.outputs.is_main_branch == 'true' | |
| uses: ./.github/workflows/workspace-integration-tests.yml | |
| with: | |
| version: ${{ needs.configuration.outputs.version }} | |
| secrets: inherit | |
| delete-runner: | |
| if: always() | |
| needs: | |
| - create-runner | |
| - configuration | |
| - build-previewctl | |
| - infrastructure | |
| - build-gitpod | |
| - install-app | |
| - install | |
| - monitoring | |
| - integration-test | |
| uses: gitpod-io/gce-github-runner/.github/workflows/delete-vm.yml@main | |
| secrets: | |
| gcp_credentials: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_GCP_CREDENTIALS }} | |
| with: | |
| runner-label: ${{ needs.create-runner.outputs.label }} | |
| machine-zone: ${{ needs.create-runner.outputs.machine-zone }} |