Streaming improvements #1 (#19137) #964
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| # CI code for building release artifacts. | |
| name: Build | |
| on: | |
| push: # Master branch checks only validate the build and generate artifacts for testing. | |
| branches: | |
| - master | |
| pull_request: null # PR checks only validate the build and generate artifacts for testing. | |
| workflow_dispatch: # Dispatch runs build and does limited validation, then pushes to the appropriate storage location. | |
| inputs: | |
| type: | |
| description: Build Type | |
| default: nightly | |
| required: true | |
| version: | |
| description: Version Tag | |
| default: nightly | |
| required: true | |
| concurrency: # This keeps multiple instances of the job from running concurrently for the same ref and event type. | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} | |
| cancel-in-progress: true | |
| jobs: | |
| file-check: # Check what files changed if we’re being run in a PR or on a push. | |
| name: Check Modified Files | |
| runs-on: ubuntu-latest | |
| outputs: | |
| run: ${{ steps.check-run.outputs.run }} | |
| skip-go: ${{ steps.check-go.outputs.skip-go }} | |
| steps: | |
| - name: Checkout | |
| id: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - name: Check source files | |
| id: check-source-files | |
| uses: tj-actions/changed-files@v45 | |
| with: | |
| since_last_remote_commit: ${{ github.event_name != 'pull_request' }} | |
| files: | | |
| **/*.c | |
| **/*.cc | |
| **/*.h | |
| **/*.hh | |
| **/*.in | |
| **/*.patch | |
| src/aclk/aclk-schemas/ | |
| src/ml/dlib/ | |
| src/fluent-bit/ | |
| src/web/server/h2o/libh2o/ | |
| files_ignore: | | |
| netdata.spec.in | |
| **/*.md | |
| - name: Check build files | |
| id: check-build-files | |
| uses: tj-actions/changed-files@v45 | |
| with: | |
| since_last_remote_commit: ${{ github.event_name != 'pull_request' }} | |
| files: | | |
| **/*.cmake | |
| CMakeLists.txt | |
| netdata-installer.sh | |
| .github/data/distros.yml | |
| .github/workflows/build.yml | |
| .github/scripts/build-static.sh | |
| .github/scripts/get-static-cache-key.sh | |
| .github/scripts/gen-matrix-build.py | |
| .github/scripts/run-updater-check.sh | |
| packaging/cmake/ | |
| packaging/makeself/ | |
| packaging/installer/ | |
| packaging/windows/ | |
| packaging/*.sh | |
| packaging/*.version | |
| packaging/*.checksums | |
| files_ignore: | | |
| **/*.md | |
| packaging/repoconfig/ | |
| - name: List all changed files in pattern | |
| continue-on-error: true | |
| if: github.event_name != 'workflow_dispatch' | |
| env: | |
| CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }} | |
| CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }} | |
| run: | | |
| for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do | |
| echo "$file was changed" | |
| done | |
| - name: Check Run | |
| id: check-run | |
| run: | | |
| if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then | |
| echo 'run=true' >> "${GITHUB_OUTPUT}" | |
| else | |
| echo 'run=false' >> "${GITHUB_OUTPUT}" | |
| fi | |
| - name: Check Go | |
| id: check-go | |
| run: | | |
| if [ '${{ github.event_name }}' == 'pull_request' ]; then | |
| if echo "${{ steps.check-source-files.outputs.other_changed_files }}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then | |
| echo 'skip-go=' >> "${GITHUB_OUTPUT}" | |
| else | |
| echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}" | |
| fi | |
| else | |
| echo 'skip-go=' >> "${GITHUB_OUTPUT}" | |
| fi | |
| build-dist: # Build the distribution tarball and store it as an artifact. | |
| name: Build Distribution Tarball | |
| runs-on: ubuntu-latest | |
| needs: | |
| - file-check | |
| outputs: | |
| distfile: ${{ steps.build.outputs.distfile }} | |
| steps: | |
| - name: Skip Check | |
| id: skip | |
| if: needs.file-check.outputs.run != 'true' | |
| run: echo "SKIPPED" | |
| - name: Checkout | |
| id: checkout | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - name: Fix tags | |
| id: fix-tags | |
| if: github.event_name != 'push' && needs.file-check.outputs.run == 'true' | |
| run: | | |
| git fetch --tags --force | |
| - name: Mark Stable | |
| id: channel | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly' && needs.file-check.outputs.run == 'true' | |
| run: | | |
| sed -i 's/^RELEASE_CHANNEL="nightly"/RELEASE_CHANNEL="stable"/' netdata-installer.sh | |
| - name: Build | |
| id: build | |
| if: needs.file-check.outputs.run == 'true' | |
| run: | | |
| mkdir -p artifacts/ | |
| tar --create --file "artifacts/netdata-$(git describe).tar.gz" \ | |
| --sort=name --posix --auto-compress --exclude=artifacts/ --exclude=.git \ | |
| --exclude=.gitignore --exclude=.gitattributes --exclude=.gitmodules \ | |
| --transform "s/^\\.\\//netdata-$(git describe)\\//" --verbose . | |
| cd artifacts/ | |
| echo "distfile=$(find . -name 'netdata-*.tar.gz')" >> "${GITHUB_OUTPUT}" | |
| - name: Store | |
| id: store | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: actions/upload-artifact@v4.4.2 | |
| with: | |
| name: dist-tarball | |
| path: artifacts/*.tar.gz | |
| retention-days: 30 | |
| - name: Failure Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| env: | |
| SLACK_COLOR: 'danger' | |
| SLACK_FOOTER: '' | |
| SLACK_ICON_EMOJI: ':github-actions:' | |
| SLACK_TITLE: 'Distribution tarball creation failed:' | |
| SLACK_USERNAME: 'GitHub Actions' | |
| SLACK_MESSAGE: |- | |
| ${{ github.repository }}: Failed to create source tarball for distribution. | |
| Checkout: ${{ steps.checkout.outcome }} | |
| Fix Tags: ${{ steps.fix-tags.outcome }} | |
| Mark stable: ${{ steps.channel.outcome }} | |
| Build: ${{ steps.build.outcome }} | |
| Store: ${{ steps.store.outcome }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| if: >- | |
| ${{ | |
| failure() | |
| && startsWith(github.ref, 'refs/heads/master') | |
| && github.event_name != 'pull_request' | |
| && github.repository == 'netdata/netdata' | |
| && needs.file-check.outputs.run == 'true' | |
| }} | |
| build-static: # Build the static binary archives, and store them as artifacts. | |
| name: Build Static | |
| runs-on: ubuntu-latest | |
| needs: | |
| - file-check | |
| strategy: | |
| matrix: | |
| arch: | |
| - x86_64 | |
| - armv6l | |
| - armv7l | |
| - aarch64 | |
| - ppc64le | |
| steps: | |
| - name: Skip Check | |
| id: skip | |
| if: needs.file-check.outputs.run != 'true' | |
| run: echo "SKIPPED" | |
| - name: Checkout | |
| id: checkout | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - name: Fix tags | |
| id: fix-tags | |
| if: github.event_name != 'push' && needs.file-check.outputs.run == 'true' | |
| run: | | |
| git fetch --tags --force | |
| - name: Mark Stable | |
| id: channel | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly' && needs.file-check.outputs.run == 'true' | |
| run: | | |
| sed -i 's/^RELEASE_CHANNEL="nightly"/RELEASE_CHANNEL="stable"/' netdata-installer.sh packaging/makeself/install-or-update.sh | |
| - name: Get Cache Key | |
| if: (github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'run-ci/no-cache')) && needs.file-check.outputs.run == 'true' | |
| id: cache-key | |
| run: .github/scripts/get-static-cache-key.sh ${{ matrix.arch }} "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/no-cache') }}" | |
| - name: Cache | |
| if: (github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'run-ci/no-cache')) && needs.file-check.outputs.run == 'true' | |
| id: cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: artifacts/cache | |
| key: ${{ steps.cache-key.outputs.key }} | |
| - name: Build | |
| if: github.event_name != 'workflow_dispatch' && needs.file-check.outputs.run == 'true' # Don’t use retries on PRs. | |
| run: | | |
| export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }} | |
| .github/scripts/build-static.sh ${{ matrix.arch }} | |
| - name: Build | |
| if: github.event_name == 'workflow_dispatch' && needs.file-check.outputs.run == 'true' | |
| id: build | |
| uses: nick-fields/retry@v3 | |
| with: | |
| timeout_minutes: 180 | |
| max_attempts: 3 | |
| command: | | |
| export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }} | |
| .github/scripts/build-static.sh ${{ matrix.arch }} | |
| - name: Store | |
| id: store | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: actions/upload-artifact@v4.4.2 | |
| with: | |
| name: dist-static-${{ matrix.arch }} | |
| path: artifacts/*.gz.run | |
| retention-days: 30 | |
| - name: Failure Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| env: | |
| SLACK_COLOR: 'danger' | |
| SLACK_FOOTER: '' | |
| SLACK_ICON_EMOJI: ':github-actions:' | |
| SLACK_TITLE: 'Static build failed:' | |
| SLACK_USERNAME: 'GitHub Actions' | |
| SLACK_MESSAGE: |- | |
| ${{ github.repository }}: Failed to create static installer archive for ${{ matrix.arch }}. | |
| Checkout: ${{ steps.checkout.outcome }} | |
| Fix Tags: ${{ steps.fix-tags.outcome }} | |
| Mark stable: ${{ steps.channel.outcome }} | |
| Build: ${{ steps.build.outcome }} | |
| Store: ${{ steps.store.outcome }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| if: >- | |
| ${{ | |
| failure() | |
| && startsWith(github.ref, 'refs/heads/master') | |
| && github.event_name != 'pull_request' | |
| && github.repository == 'netdata/netdata' | |
| && needs.file-check.outputs.run == 'true' | |
| }} | |
| windows-build: # Test building on Windows | |
| name: Test building on Windows | |
| runs-on: windows-latest | |
| needs: | |
| - file-check | |
| steps: | |
| - name: Skip Check | |
| id: skip | |
| if: needs.file-check.outputs.run != 'true' | |
| run: Write-Output "SKIPPED" | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| id: checkout | |
| if: needs.file-check.outputs.run == 'true' | |
| with: | |
| submodules: recursive | |
| lfs: true | |
| - name: Set Up Go | |
| id: golang | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: "^1.23" | |
| - name: Set Up Dependencies | |
| id: deps | |
| if: needs.file-check.outputs.run == 'true' | |
| run: ./packaging/windows/install-dependencies.ps1 | |
| - name: Build Netdata | |
| id: build | |
| if: needs.file-check.outputs.run == 'true' | |
| env: | |
| BUILD_DIR: ${{ github.workspace }}\build | |
| run: ./packaging/windows/build.ps1 | |
| - name: Sign Agent Code | |
| id: sign-agent | |
| if: needs.file-check.outputs.run == 'true' && github.event_name != 'pull_request' | |
| uses: azure/trusted-signing-action@v0.5.0 | |
| with: | |
| azure-tenant-id: ${{ secrets.CODE_SIGNING_TENNANT_ID }} | |
| azure-client-id: ${{ secrets.CODE_SIGNING_CLIENT_ID }} | |
| azure-client-secret: ${{ secrets.CODE_SIGNING_CLIENT_SECRET }} | |
| endpoint: "https://eus.codesigning.azure.net/" | |
| trusted-signing-account-name: Netdata | |
| certificate-profile-name: Netdata | |
| files-folder: ${{ github.workspace }}\build | |
| files-folder-filter: exe,dll | |
| files-folder-recurse: true | |
| file-digest: SHA256 | |
| timestamp-rfc3161: "http://timestamp.acs.microsoft.com" | |
| timestamp-digest: SHA256 | |
| - name: Package Netdata | |
| id: package | |
| if: needs.file-check.outputs.run == 'true' | |
| env: | |
| BUILD_DIR: ${{ github.workspace }}\build | |
| run: ./packaging/windows/package.ps1 | |
| - name: Sign Installer | |
| id: sign-installer | |
| if: needs.file-check.outputs.run == 'true' && github.event_name != 'pull_request' | |
| uses: azure/trusted-signing-action@v0.5.0 | |
| with: | |
| azure-tenant-id: ${{ secrets.CODE_SIGNING_TENNANT_ID }} | |
| azure-client-id: ${{ secrets.CODE_SIGNING_CLIENT_ID }} | |
| azure-client-secret: ${{ secrets.CODE_SIGNING_CLIENT_SECRET }} | |
| endpoint: "https://eus.codesigning.azure.net/" | |
| trusted-signing-account-name: Netdata | |
| certificate-profile-name: Netdata | |
| files-folder: ${{ github.workspace }}\packaging\windows | |
| files-folder-filter: msi | |
| file-digest: SHA256 | |
| timestamp-rfc3161: "http://timestamp.acs.microsoft.com" | |
| timestamp-digest: SHA256 | |
| - name: Upload Installer | |
| id: upload | |
| uses: actions/upload-artifact@v4.4.2 | |
| with: | |
| name: windows-x86_64-installer | |
| path: packaging\windows\netdata*.msi | |
| retention-days: 30 | |
| - name: Failure Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| env: | |
| SLACK_COLOR: 'danger' | |
| SLACK_FOOTER: '' | |
| SLACK_ICON_EMOJI: ':github-actions:' | |
| SLACK_TITLE: 'Windows build failed:' | |
| SLACK_USERNAME: 'GitHub Actions' | |
| SLACK_MESSAGE: |- | |
| ${{ github.repository }}: Updater checks for ${{ matrix.distro }} failed. | |
| Checkout: ${{ steps.checkout.outcome }} | |
| Set Up Dependencies: ${{ steps.deps.outcome }} | |
| Build Netdata: ${{ steps.build.outcome }} | |
| Sign Agent Code: ${{ steps.sign-agent.outcome }} | |
| Package Netdata: ${{ steps.package.outcome }} | |
| Sign Installer: ${{ steps.sign-installer.outcome }} | |
| Upload Installer: ${{ steps.upload.outcome }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| if: >- | |
| ${{ | |
| failure() | |
| && startsWith(github.ref, 'refs/heads/master') | |
| && github.event_name != 'pull_request' | |
| && github.repository == 'netdata/netdata' | |
| && needs.file-check.outputs.run == 'true' | |
| }} | |
| prepare-upload: # Consolidate the artifacts for uploading or releasing. | |
| name: Prepare Artifacts | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-dist | |
| - build-static | |
| - windows-build | |
| - file-check | |
| steps: | |
| - name: Skip Check | |
| id: skip | |
| if: needs.file-check.outputs.run != 'true' | |
| run: echo "SKIPPED" | |
| - name: Checkout | |
| id: checkout | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: actions/checkout@v4 | |
| - name: Prepare Environment | |
| id: prepare | |
| if: needs.file-check.outputs.run == 'true' | |
| run: mkdir -p artifacts | |
| - name: Retrieve Build Artifacts | |
| id: fetch-dist | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: Wandalen/wretry.action@v3 | |
| with: | |
| action: actions/download-artifact@v4 | |
| with: | | |
| pattern: dist-* | |
| path: dist-artifacts | |
| merge-multiple: true | |
| attempt_limit: 3 | |
| attempt_delay: 2000 | |
| - name: Retrieve Windows Artifacts | |
| id: fetch-windows | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: Wandalen/wretry.action@v3 | |
| with: | |
| action: actions/download-artifact@v4 | |
| with: | | |
| pattern: windows-*-installer | |
| path: dist-artifacts | |
| merge-multiple: true | |
| attempt_limit: 3 | |
| attempt_delay: 2000 | |
| - name: Prepare Artifacts | |
| id: consolidate | |
| if: needs.file-check.outputs.run == 'true' | |
| working-directory: ./artifacts/ | |
| run: | | |
| mv ../dist-artifacts/* . || exit 1 | |
| ln -s ${{ needs.build-dist.outputs.distfile }} netdata-latest.tar.gz || exit 1 | |
| cp ../packaging/version ./latest-version.txt || exit 1 | |
| sha256sum -b ./* > sha256sums.txt || exit 1 | |
| cat sha256sums.txt | |
| - name: Store Artifacts | |
| id: store | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: actions/upload-artifact@v4.4.2 | |
| with: | |
| name: final-artifacts | |
| path: artifacts/* | |
| retention-days: 30 | |
| - name: Failure Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| env: | |
| SLACK_COLOR: 'danger' | |
| SLACK_FOOTER: '' | |
| SLACK_ICON_EMOJI: ':github-actions:' | |
| SLACK_TITLE: 'Failed to prepare release artifacts for upload:' | |
| SLACK_USERNAME: 'GitHub Actions' | |
| SLACK_MESSAGE: |- | |
| ${{ github.repository }}: Failed to prepare release artifacts for upload. | |
| Checkout: ${{ steps.checkout.outcome }} | |
| Prepare environment: ${{ steps.prepare.outcome }} | |
| Fetch dist artifacts: ${{ steps.fetch-dist.outcome }} | |
| Fetch Windows installers: ${{ steps.fetch-windows.outcome }} | |
| Consolidate artifacts: ${{ steps.consolidate.outcome }} | |
| Store: ${{ steps.store.outcome }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| if: >- | |
| ${{ | |
| failure() | |
| && startsWith(github.ref, 'refs/heads/master') | |
| && github.event_name != 'pull_request' | |
| && github.repository == 'netdata/netdata' | |
| && needs.file-check.outputs.run == 'true' | |
| }} | |
| artifact-verification-dist: # Verify the regular installer works with the consolidated artifacts. | |
| name: Test Consolidated Artifacts (Source) | |
| runs-on: ubuntu-latest | |
| needs: | |
| - prepare-upload | |
| - file-check | |
| services: | |
| apache: # This gets used to serve the dist tarball for the updater script. | |
| image: httpd:2.4 | |
| ports: | |
| - 8080:80 | |
| volumes: | |
| - ${{ github.workspace }}:/usr/local/apache2/htdocs/ | |
| steps: | |
| - name: Skip Check | |
| id: skip | |
| if: needs.file-check.outputs.run != 'true' | |
| run: echo "SKIPPED" | |
| - name: Checkout | |
| id: checkout | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: actions/checkout@v4 | |
| - name: Fetch artifacts | |
| id: fetch | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: Wandalen/wretry.action@v3 | |
| with: | |
| action: actions/download-artifact@v4 | |
| with: | | |
| name: final-artifacts | |
| path: artifacts | |
| attempt_limit: 3 | |
| attempt_delay: 2000 | |
| - name: Prepare artifacts directory | |
| id: prepare | |
| if: needs.file-check.outputs.run == 'true' | |
| run: | | |
| mkdir -p download/latest | |
| mv artifacts/* download/latest | |
| ls -al download/latest | |
| - name: Verify that artifacts work with installer | |
| id: verify | |
| if: needs.file-check.outputs.run == 'true' | |
| env: | |
| NETDATA_TARBALL_BASEURL: http://localhost:8080/ | |
| run: sh -x packaging/installer/kickstart.sh --build-only --dont-start-it --disable-telemetry --dont-wait | |
| - name: Failure Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| env: | |
| SLACK_COLOR: 'danger' | |
| SLACK_FOOTER: '' | |
| SLACK_ICON_EMOJI: ':github-actions:' | |
| SLACK_TITLE: 'Artifact verification for source tarball failed.' | |
| SLACK_USERNAME: 'GitHub Actions' | |
| SLACK_MESSAGE: |- | |
| ${{ github.repository }}: Artifact verification for source tarball failed. | |
| Checkout: ${{ steps.checkout.outcome }} | |
| Fetch artifacts: ${{ steps.fetch.outcome }} | |
| Verify artifacts: ${{ steps.verify.outcome }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| if: >- | |
| ${{ | |
| failure() | |
| && startsWith(github.ref, 'refs/heads/master') | |
| && github.event_name != 'pull_request' | |
| && github.repository == 'netdata/netdata' | |
| && needs.file-check.outputs.run == 'true' | |
| }} | |
| artifact-verification-static: # Verify the static installer works with the consolidated artifacts. | |
| name: Test Consolidated Artifacts (Static) | |
| runs-on: ubuntu-latest | |
| needs: | |
| - prepare-upload | |
| - file-check | |
| services: | |
| apache: # This gets used to serve the static archives. | |
| image: httpd:2.4 | |
| ports: | |
| - 8080:80 | |
| volumes: | |
| - ${{ github.workspace }}:/usr/local/apache2/htdocs/ | |
| steps: | |
| - name: Skip Check | |
| id: skip | |
| if: needs.file-check.outputs.run != 'true' | |
| run: echo "SKIPPED" | |
| - name: Checkout | |
| id: checkout | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: actions/checkout@v4 | |
| - name: Fetch artifacts | |
| id: fetch-artifacts | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: Wandalen/wretry.action@v3 | |
| with: | |
| action: actions/download-artifact@v4 | |
| with: | | |
| name: final-artifacts | |
| path: artifacts | |
| attempt_limit: 3 | |
| attempt_delay: 2000 | |
| - name: Prepare artifacts directory | |
| id: prepare | |
| if: needs.file-check.outputs.run == 'true' | |
| run: | | |
| mkdir -p download/latest | |
| mv artifacts/* download/latest | |
| ls -al download/latest | |
| - name: Verify that artifacts work with installer | |
| id: verify | |
| if: needs.file-check.outputs.run == 'true' | |
| env: | |
| NETDATA_TARBALL_BASEURL: http://localhost:8080/ | |
| run: sh -x packaging/installer/kickstart.sh --static-only --dont-start-it --disable-telemetry | |
| - name: Failure Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| env: | |
| SLACK_COLOR: 'danger' | |
| SLACK_FOOTER: '' | |
| SLACK_ICON_EMOJI: ':github-actions:' | |
| SLACK_TITLE: 'Artifact verification for static build failed.' | |
| SLACK_USERNAME: 'GitHub Actions' | |
| SLACK_MESSAGE: |- | |
| ${{ github.repository }}: Artifact verification for static build failed. | |
| Checkout: ${{ steps.checkout.outcome }} | |
| Fetch artifacts: ${{ steps.fetch-artifacts.outcome }} | |
| Verify artifacts: ${{ steps.verify.outcome }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| if: >- | |
| ${{ | |
| failure() | |
| && startsWith(github.ref, 'refs/heads/master') | |
| && github.event_name != 'pull_request' | |
| && github.repository == 'netdata/netdata' | |
| && needs.file-check.outputs.run == 'true' | |
| }} | |
| artifact-verification-updater: # Test the generated dist archive using the updater code. | |
| name: Test Consolidated Artifacts (Updater) | |
| runs-on: ubuntu-latest | |
| needs: | |
| - prepare-upload | |
| - file-check | |
| services: | |
| apache: # This gets used to serve the dist tarball for the updater script. | |
| image: httpd:2.4 | |
| ports: | |
| - 8080:80 | |
| volumes: | |
| - ${{ github.workspace }}:/usr/local/apache2/htdocs/ | |
| steps: | |
| - name: Skip Check | |
| id: skip | |
| if: needs.file-check.outputs.run != 'true' | |
| run: echo "SKIPPED" | |
| - name: Checkout | |
| id: checkout | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: actions/checkout@v4 | |
| - name: Fetch artifacts | |
| id: fetch-artifacts | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: Wandalen/wretry.action@v3 | |
| with: | |
| action: actions/download-artifact@v4 | |
| with: | | |
| name: final-artifacts | |
| path: artifacts | |
| attempt_limit: 3 | |
| attempt_delay: 2000 | |
| - name: Prepare artifacts directory | |
| id: prepare | |
| if: needs.file-check.outputs.run == 'true' | |
| run: | | |
| mkdir -p download/latest | |
| mv artifacts/* download/latest | |
| ls -al download/latest | |
| - name: Run Updater Check | |
| id: check | |
| if: needs.file-check.outputs.run == 'true' | |
| run: | | |
| docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 --network host \ | |
| -v $PWD:/netdata -w /netdata \ | |
| ubuntu:latest /bin/sh -x /netdata/.github/scripts/run-updater-check.sh | |
| - name: Failure Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| env: | |
| SLACK_COLOR: 'danger' | |
| SLACK_FOOTER: '' | |
| SLACK_ICON_EMOJI: ':github-actions:' | |
| SLACK_TITLE: 'Updater checks for ${{ matrix.distro }} failed:' | |
| SLACK_USERNAME: 'GitHub Actions' | |
| SLACK_MESSAGE: |- | |
| ${{ github.repository }}: Updater checks for ${{ matrix.distro }} failed. | |
| Checkout: ${{ steps.checkout.outcome }} | |
| Fetch artifacts: ${{ steps.fetch-artifacts.outcome }} | |
| Prepare artifact directory: ${{ steps.prepare.outcome }} | |
| Updater check: ${{ steps.check.outcome }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| if: >- | |
| ${{ | |
| failure() | |
| && startsWith(github.ref, 'refs/heads/master') | |
| && github.event_name != 'pull_request' | |
| && github.repository == 'netdata/netdata' | |
| && needs.file-check.outputs.run == 'true' | |
| }} | |
| create-nightly: # Create a nightly build release in netdata/netdata-nightlies | |
| name: Create Nightly Release | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'nightly' && github.repository == 'netdata/netdata' | |
| needs: | |
| - prepare-upload | |
| - artifact-verification-dist | |
| - artifact-verification-static | |
| steps: | |
| - name: Checkout Main Repo | |
| id: checkout-main | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| - name: Checkout Nightly Repo | |
| id: checkout-nightly | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: netdata/netdata-nightlies | |
| path: nightlies | |
| token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} | |
| - name: Retrieve Artifacts | |
| id: fetch | |
| uses: Wandalen/wretry.action@v3 | |
| with: | |
| action: actions/download-artifact@v4 | |
| with: | | |
| name: final-artifacts | |
| path: final-artifacts | |
| attempt_limit: 3 | |
| attempt_delay: 2000 | |
| - name: Prepare version info | |
| id: version | |
| run: | | |
| # shellcheck disable=SC2129 | |
| echo "version=$(cat main/packaging/version)" >> "${GITHUB_OUTPUT}" | |
| echo "commit=$(cd nightlies && git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" | |
| echo "date=$(date +%F)" >> "${GITHUB_OUTPUT}" | |
| - name: Create Release | |
| id: create-release | |
| uses: ncipollo/release-action@v1 | |
| with: | |
| allowUpdates: false | |
| artifactErrorsFailBuild: true | |
| artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run,final-artifacts/netdata-*.msi' | |
| owner: netdata | |
| repo: netdata-nightlies | |
| body: Netdata nightly build for ${{ steps.version.outputs.date }}. | |
| commit: ${{ steps.version.outputs.commit }} | |
| makeLatest: true | |
| tag: ${{ steps.version.outputs.version }} | |
| token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} | |
| - name: Checkout netdata main Repo # Checkout back to netdata/netdata repo to the update latest packaged versions | |
| id: checkout-netdata | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} | |
| - name: Init python environment for publish release metadata | |
| uses: actions/setup-python@v5 | |
| id: init-python | |
| with: | |
| python-version: "3.12" | |
| - name: Setup python environment | |
| id: setup-python | |
| run: | | |
| pip install -r .github/scripts/modules/requirements.txt | |
| - name: Check if the version is latest and published | |
| id: check-latest-version | |
| run: | | |
| python .github/scripts/check_latest_versions.py ${{ steps.version.outputs.version }} | |
| - name: SSH setup | |
| id: ssh-setup | |
| if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true' | |
| uses: shimataro/ssh-key-action@v2 | |
| with: | |
| key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }} | |
| name: id_ecdsa | |
| known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }} | |
| - name: Sync release info to packages.netdata.cloud | |
| id: sync-releases | |
| continue-on-error: true | |
| if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true' | |
| run: | | |
| .github/scripts/upload-new-version-tags.sh packages.netdata.cloud | |
| - name: Sync release info to packages2.netdata.cloud | |
| id: sync-releases2 | |
| if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true' | |
| run: | | |
| .github/scripts/upload-new-version-tags.sh packages.netdata.cloud | |
| - name: Failure Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| env: | |
| SLACK_COLOR: 'danger' | |
| SLACK_FOOTER: '' | |
| SLACK_ICON_EMOJI: ':github-actions:' | |
| SLACK_TITLE: 'Failed to draft release:' | |
| SLACK_USERNAME: 'GitHub Actions' | |
| SLACK_MESSAGE: |- | |
| ${{ github.repository }}: Failed to create nightly release or attach artifacts. | |
| Checkout netdata/netdata: ${{ steps.checkout-main.outcome }} | |
| Checkout netdata/netdata-nightlies: ${{ steps.checkout-nightly.outcome }} | |
| Fetch artifacts: ${{ steps.fetch.outcome }} | |
| Prepare version info: ${{ steps.version.outcome }} | |
| Create release: ${{ steps.create-release.outcome }} | |
| Checkout back netdata/netdata: ${{ steps.checkout-netdata.outcome }} | |
| Init python environment: ${{ steps.init-python.outcome }} | |
| Setup python environment: ${{ steps.setup-python.outcome }} | |
| Check the nearly published release against the advertised: ${{ steps.check-latest-version.outcome }} | |
| Setup ssh: ${{ steps.ssh-setup.outcome }} | |
| Sync release info to packages.netdata.cloud: ${{ steps.sync-releases.outcome }} | |
| Sync release info to packages2.netdata.cloud: ${{ steps.sync-releases2.outcome }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| if: >- | |
| ${{ | |
| failure() | |
| && github.event_name == 'workflow_dispatch' | |
| }} | |
| normalize-tag: # Fix the release tag if needed | |
| name: Normalize Release Tag | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'release' | |
| outputs: | |
| tag: ${{ steps.tag.outputs.tag }} | |
| steps: | |
| - name: Normalize Tag | |
| id: tag | |
| run: | | |
| if echo ${{ github.event.inputs.version }} | grep -qE '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$'; then | |
| echo "tag=v${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}" | |
| else | |
| echo "tag=${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}" | |
| fi | |
| upload-release: # Create the draft release and upload the build artifacts. | |
| name: Create Release Draft | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'release' && github.repository == 'netdata/netdata' | |
| needs: | |
| - artifact-verification-dist | |
| - artifact-verification-static | |
| - normalize-tag | |
| steps: | |
| - name: Checkout | |
| id: checkout | |
| uses: actions/checkout@v4 | |
| - name: Retrieve Artifacts | |
| id: fetch | |
| uses: Wandalen/wretry.action@v3 | |
| with: | |
| action: actions/download-artifact@v4 | |
| with: | | |
| name: final-artifacts | |
| path: final-artifacts | |
| attempt_limit: 3 | |
| attempt_delay: 2000 | |
| - name: Create Release | |
| id: create-release | |
| uses: ncipollo/release-action@v1 | |
| with: | |
| allowUpdates: false | |
| artifactErrorsFailBuild: true | |
| artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run,final-artifacts/netdata-*.msi' | |
| draft: true | |
| tag: ${{ needs.normalize-tag.outputs.tag }} | |
| token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} | |
| - name: Failure Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| env: | |
| SLACK_COLOR: 'danger' | |
| SLACK_FOOTER: '' | |
| SLACK_ICON_EMOJI: ':github-actions:' | |
| SLACK_TITLE: 'Failed to draft release:' | |
| SLACK_USERNAME: 'GitHub Actions' | |
| SLACK_MESSAGE: |- | |
| ${{ github.repository }}: Failed to create draft release or attach artifacts. | |
| Checkout: ${{ steps.checkout.outcome }} | |
| Fetch artifacts: ${{ steps.fetch.outcome }} | |
| Create draft release: ${{ steps.create-release.outcome }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| if: >- | |
| ${{ | |
| failure() | |
| && github.event_name == 'workflow_dispatch' | |
| }} | |
| - name: Success Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| env: | |
| SLACK_COLOR: 'good' | |
| SLACK_FOOTER: '' | |
| SLACK_ICON_EMOJI: ':github-actions:' | |
| SLACK_TITLE: 'Created agent draft release:' | |
| SLACK_USERNAME: 'GitHub Actions' | |
| SLACK_MESSAGE: "${{ github.repository }}: ${{ steps.create-release.outputs.html_url }}" | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| if: >- | |
| ${{ | |
| success() | |
| && github.event_name == 'workflow_dispatch' | |
| }} | |
| # Remaining jobs are only used for CI checks, and not as part of the release process | |
| matrix: # Generate the shared build matrix for our Linux build tests. | |
| name: Prepare Build Matrix | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'workflow_dispatch' | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout | |
| id: checkout | |
| uses: actions/checkout@v4 | |
| - name: Prepare tools | |
| id: prepare | |
| run: | | |
| sudo apt-get update || true | |
| sudo apt-get install -y python3-ruamel.yaml | |
| - name: Read build matrix | |
| id: set-matrix | |
| run: | | |
| matrix="$(.github/scripts/gen-matrix-build.py)" | |
| echo "Generated matrix: ${matrix}" | |
| echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}" | |
| - name: Failure Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| env: | |
| SLACK_COLOR: 'danger' | |
| SLACK_FOOTER: '' | |
| SLACK_ICON_EMOJI: ':github-actions:' | |
| SLACK_TITLE: 'Build matrix preparation failed:' | |
| SLACK_USERNAME: 'GitHub Actions' | |
| SLACK_MESSAGE: |- | |
| ${{ github.repository }}: Failed to prepare build matrix for build checks. | |
| Checkout: ${{ steps.checkout.outcome }} | |
| Prepare tools: ${{ steps.prepare.outcome }} | |
| Read build matrix: ${{ steps.set-matrix.outcome }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| if: >- | |
| ${{ | |
| failure() | |
| && startsWith(github.ref, 'refs/heads/master') | |
| && github.event_name != 'pull_request' | |
| && github.repository == 'netdata/netdata' | |
| }} | |
| source-build: # Test various source build arrangements. | |
| name: Test Source Build | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'workflow_dispatch' | |
| needs: | |
| - matrix | |
| - file-check | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 8 | |
| matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} | |
| steps: | |
| - name: Skip Check | |
| id: skip | |
| if: needs.file-check.outputs.run != 'true' | |
| run: echo "SKIPPED" | |
| - name: Checkout | |
| id: checkout | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Setup Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build test environment | |
| id: build | |
| uses: Wandalen/wretry.action@v3 | |
| with: | |
| action: docker/build-push-action@v6 | |
| with: | | |
| push: false | |
| load: true | |
| file: .github/dockerfiles/Dockerfile.build_test | |
| build-args: | | |
| BASE=${{ matrix.distro }} | |
| PRE=${{ matrix.env_prep }} | |
| RMJSONC=${{ matrix.jsonc_removal }} | |
| tags: test:${{ matrix.artifact_key }} | |
| attempt_limit: 3 | |
| attempt_delay: 15000 | |
| - name: netdata-installer on ${{ matrix.distro }} | |
| id: build-cloud | |
| if: needs.file-check.outputs.run == 'true' | |
| run: | | |
| docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \ | |
| /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --one-time-build ${{ needs.file-check.outputs.skip-go }}' | |
| - name: Failure Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| env: | |
| SLACK_COLOR: 'danger' | |
| SLACK_FOOTER: '' | |
| SLACK_ICON_EMOJI: ':github-actions:' | |
| SLACK_TITLE: 'Build tests for ${{ matrix.distro }} failed:' | |
| SLACK_USERNAME: 'GitHub Actions' | |
| SLACK_MESSAGE: |- | |
| ${{ github.repository }}: Build tests for ${{ matrix.distro }} failed. | |
| Checkout: ${{ steps.checkout.outcome }} | |
| Set up Buildx: ${{ steps.buildx.outcome }} | |
| Build test environment: ${{ steps.build1.outcome }} | |
| netdata-installer: ${{ steps.build-cloud.outcome }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| if: >- | |
| ${{ | |
| failure() | |
| && startsWith(github.ref, 'refs/heads/master') | |
| && github.event_name != 'pull_request' | |
| && github.repository == 'netdata/netdata' | |
| && needs.file-check.outputs.run == 'true' | |
| }} | |
| macos-build: # Test building on macOS | |
| name: Test building on macOS | |
| runs-on: ${{ matrix.runner }} | |
| if: github.event_name != 'workflow_dispatch' | |
| needs: | |
| - file-check | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 8 | |
| matrix: | |
| include: | |
| - name: macos-13 | |
| runner: macos-13 | |
| - name: macos-14-M1 | |
| runner: macos-14 | |
| - name: macos-15-M1 | |
| runner: macos-15 | |
| steps: | |
| - name: Skip Check | |
| id: skip | |
| if: needs.file-check.outputs.run != 'true' | |
| run: echo "SKIPPED" | |
| - uses: actions/checkout@v4 | |
| id: checkout | |
| if: needs.file-check.outputs.run == 'true' | |
| with: | |
| submodules: recursive | |
| - name: Install latest bash | |
| id: install-bash | |
| if: needs.file-check.outputs.run == 'true' | |
| run: | | |
| brew install bash | |
| - name: Install netdata dependencies | |
| id: install-nd-dep | |
| if: needs.file-check.outputs.run == 'true' | |
| run: | | |
| bash ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all | |
| - name: Build from source | |
| id: build-source | |
| if: needs.file-check.outputs.run == 'true' | |
| run: | | |
| sudo bash ./netdata-installer.sh --install-no-prefix /usr/local/netdata --dont-wait --dont-start-it --one-time-build | |
| - name: Test Agent start up | |
| id: test-agent | |
| if: needs.file-check.outputs.run == 'true' | |
| run: | | |
| /usr/local/netdata/usr/sbin/netdata -D > ./netdata.log 2>&1 & | |
| ./packaging/runtime-check.sh | |
| - name: Failure Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| env: | |
| SLACK_COLOR: 'danger' | |
| SLACK_FOOTER: '' | |
| SLACK_ICON_EMOJI: ':github-actions:' | |
| SLACK_TITLE: 'Build & test from source macOS failed:' | |
| SLACK_USERNAME: 'GitHub Actions' | |
| SLACK_MESSAGE: |- | |
| ${{ github.repository }}: macOS Build and test. | |
| Checkout: ${{ steps.checkout.outcome }} | |
| Setup runner: ${{ steps.install-bash.outcome }} | |
| Install netdata required packages: ${{ steps.install-nd-dep.outcome }} | |
| Build from source: ${{ steps.build-source.outcome }} | |
| Test Agent runtime: ${{ steps.test-agent.outcome }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| if: >- | |
| ${{ | |
| failure() | |
| && startsWith(github.ref, 'refs/heads/master') | |
| && github.event_name != 'pull_request' | |
| && github.repository == 'netdata/netdata' | |
| }} |