From 1067dc3430be9f7cafd8c250b6dcd15fae8891dc Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 30 Oct 2023 15:44:05 +0100 Subject: [PATCH] [CI] Pull requests migration to buildkite (#3573) * buildkite test on Ubuntu * Fixed step ids * fix image family * Use aws from arm64 * try windows post-checkout hook * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * Windows pre-command hook and unit tests ps * set GOPATH * set GOPATH * set GOPATH * set GOPATH * set GOPATH * Installed GCC * Installed GCC * Installed GCC * Windows 2016 * Propagate exit code on windows * Added GOTMPDIR * Added GOTMPDIR * Merge coverage reports step * Merge coverage reports step * Merge coverage reports step * changed installation path * changed installation path * changed installation path * changed installation path * changed installation path * changed installation path * test * test * merge.sh * Added coverage.out to simplify artifacts download * Fixed artifacts copy * buildite orka integration * fix os name * Add err message to upgrade test * Fix autocrlf for windows * Craig's patch * Complete test pipeline * Complete test pipeline * Remove macos GHA workflow * Fix Processing test results step * fix k8s jenkins step * Removed m1 stage * Enabled race detector for unit tests * Enables race detector for windows (cherry picked from commit bf78cc1e1498d19651857da3e69930f6db185903) # Conflicts: # .buildkite/pipeline.yml # .buildkite/scripts/common.sh # .buildkite/scripts/steps/unit-tests.sh # .ci/Jenkinsfile --- .buildkite/hooks/post-checkout.ps1 | 60 +++++++ .buildkite/hooks/pre-command.ps1 | 25 +++ .buildkite/pipeline.yml | 157 ++++++++++++++++++ .buildkite/scripts/common.sh | 116 +++++++++++++ .buildkite/scripts/steps/merge.sh | 23 +++ .buildkite/scripts/steps/unit-tests.ps1 | 28 ++++ .buildkite/scripts/steps/unit-tests.sh | 9 + .ci/Jenkinsfile | 130 +-------------- .github/workflows/macos.yml | 23 --- .../agent/application/upgrade/upgrade_test.go | 2 +- 10 files changed, 427 insertions(+), 146 deletions(-) create mode 100644 .buildkite/hooks/post-checkout.ps1 create mode 100644 .buildkite/hooks/pre-command.ps1 create mode 100644 .buildkite/scripts/common.sh create mode 100755 .buildkite/scripts/steps/merge.sh create mode 100644 .buildkite/scripts/steps/unit-tests.ps1 create mode 100644 .buildkite/scripts/steps/unit-tests.sh delete mode 100644 .github/workflows/macos.yml diff --git a/.buildkite/hooks/post-checkout.ps1 b/.buildkite/hooks/post-checkout.ps1 new file mode 100644 index 00000000000..f2c590378fa --- /dev/null +++ b/.buildkite/hooks/post-checkout.ps1 @@ -0,0 +1,60 @@ +# Set error handling +$ErrorActionPreference = "Stop" + +# Define a function to checkout and merge +function Checkout-Merge { + param ( + [string]$targetBranch, + [string]$prCommit, + [string]$mergeBranch + ) + + if (-not $targetBranch) { + Write-Host "No pull request target branch" + exit 1 + } + + git fetch -v origin $targetBranch + git checkout FETCH_HEAD + Write-Host "Current branch: $(git rev-parse --abbrev-ref HEAD)" + + # Create a temporary branch to merge the PR with the target branch + git checkout -b $mergeBranch + Write-Host "New branch created: $(git rev-parse --abbrev-ref HEAD)" + + # Set author identity so it can be used for git merge + git config user.name "github-merged-pr-post-checkout" + git config user.email "auto-merge@buildkite" + + git merge --no-edit $prCommit + + if ($LASTEXITCODE -ne 0) { + $mergeResult = $LASTEXITCODE + Write-Host "Merge failed: $mergeResult" + git merge --abort + exit $mergeResult + } +} + +$pullRequest = $env:BUILDKITE_PULL_REQUEST + +if ($pullRequest -eq "false") { + Write-Host "Not a pull request, skipping" + exit 0 +} + +$targetBranch = $env:BUILDKITE_PULL_REQUEST_BASE_BRANCH +$prCommit = $env:BUILDKITE_COMMIT +$prId = $env:BUILDKITE_PULL_REQUEST +$mergeBranch = "pr_merge_$prId" + +Checkout-Merge $targetBranch $prCommit $mergeBranch + +Write-Host "Commit information" +git --no-pager log --format=%B -n 1 + +Write-Host "Fixing CRLF in git checkout --" +git config core.autocrlf true + +# Ensure Buildkite groups are rendered +Write-Host "" diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1 new file mode 100644 index 00000000000..ca5b34bc920 --- /dev/null +++ b/.buildkite/hooks/pre-command.ps1 @@ -0,0 +1,25 @@ +# Install gcc TODO: Move to the VM image +choco install mingw +Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 +refreshenv + +# Install gvm and go +# TODO: Move GVM download to the base VM image +$env:GvmVersion = "0.5.2" +[Net.ServicePointManager]::SecurityProtocol = "tls12" +$env:GoVersion = Get-Content -Path .go-version +Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe +gvm --format=powershell $env:GoVersion | Invoke-Expression +go version + +$GOPATH = $(go env GOPATH) +$env:Path = "$GOPATH\bin;" + $env:Path +[Environment]::SetEnvironmentVariable("GOPATH", "$GOPATH", [EnvironmentVariableTarget]::Machine) +[Environment]::SetEnvironmentVariable("Path", "$GOPATH\bin;$env:Path", [EnvironmentVariableTarget]::Machine) + +# Install tools +go install github.com/magefile/mage +go install github.com/elastic/go-licenser +go install golang.org/x/tools/cmd/goimports +go install github.com/jstemmer/go-junit-report +go install gotest.tools/gotestsum diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index edd8b7ff301..ae5c8239fbd 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -4,6 +4,151 @@ env: VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp" DOCKER_REGISTRY: "docker.elastic.co" steps: +<<<<<<< HEAD +======= + - label: "Unit tests - Ubuntu 22.04" + key: "unit-tests-2204" + command: ".buildkite/scripts/steps/unit-tests.sh" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "gcp" + image: "family/core-ubuntu-2204" + retry: + manual: + allowed: true + + - label: "Unit tests - Ubuntu 22.04 ARM64" + key: "unit-tests-2204-arm64" + command: ".buildkite/scripts/steps/unit-tests.sh" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "aws" + imagePrefix: "core-ubuntu-2204-aarch64" + diskSizeGb: 200 + instanceType: "m6g.4xlarge" + retry: + manual: + allowed: true + + - label: "Unit tests - Windows 2022" + key: "unit-tests-win2022" + command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "gcp" + image: "family/core-windows-2022" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + retry: + manual: + allowed: true + + - label: "Unit tests - Windows 2016" + key: "unit-tests-win2016" + command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "gcp" + image: "family/core-windows-2016" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + retry: + manual: + allowed: true + + - label: "Unit tests - MacOS 13 ARM" + key: "unit-tests-macos-13-arm" + command: ".buildkite/scripts/steps/unit-tests.sh" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: orka + imagePrefix: generic-13-ventura-arm + retry: + manual: + allowed: true + + - label: "Unit tests - MacOS 13" + key: "unit-tests-macos-13" + command: ".buildkite/scripts/steps/unit-tests.sh" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: orka + imagePrefix: generic-13-ventura-x64 + retry: + manual: + allowed: true + + - label: "Merge coverage reports" + key: "merge-coverage" + env: + BUILDKITE_REPO: "" + command: " + .buildkite/scripts/steps/merge.sh + unit-tests-2204 + unit-tests-2204-arm64 + unit-tests-win2016 + unit-tests-win2022 + unit-tests-macos-13 + unit-tests-macos-13-arm + " + artifact_paths: + - "build/TEST-**" + agents: + image: "golang:1.20.10" + depends_on: + - unit-tests-2204 + - unit-tests-2204-arm64 + - unit-tests-win2022 + - unit-tests-win2016 + - unit-tests-macos-13 + - unit-tests-macos-13-arm + allow_dependency_failure: true + + - label: ":sonarqube: Continuous Code Inspection" + env: + VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token" + agents: + image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest" + command: + - "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ." + - "/scan-source-code.sh" + depends_on: + - "merge-coverage" + retry: + manual: + allowed: true + + - label: "Serverless integration test" + key: "serverless-integration-tests" + command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestMonitoringLogsShipped" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + agents: + provider: "gcp" + machineType: "n1-standard-8" + +>>>>>>> bf78cc1e14 ([CI] Pull requests migration to buildkite (#3573)) - label: "Integration tests" key: "integration-tests" command: ".buildkite/scripts/integration_tests.sh" @@ -13,3 +158,15 @@ steps: agents: provider: "gcp" machineType: "n1-standard-8" +<<<<<<< HEAD +======= + + - wait: ~ + continue_on_failure: true + - label: "Processing test results" + agents: + provider: "gcp" + plugins: + - junit-annotate#v2.4.1: + artifacts: build/TEST-go-integration*.xml +>>>>>>> bf78cc1e14 ([CI] Pull requests migration to buildkite (#3573)) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh new file mode 100644 index 00000000000..16d32fb77ab --- /dev/null +++ b/.buildkite/scripts/common.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +set -euo pipefail + +if [[ -z "${WORKSPACE-""}" ]]; then + WORKSPACE=$(git rev-parse --show-toplevel) + export WORKSPACE +fi +PIPELINE="${WORKSPACE}/.buildkite/pipeline.elastic-agent-package.yml" +if [[ -z "${SETUP_MAGE_VERSION-""}" ]]; then + SETUP_MAGE_VERSION=$(grep -oe "SETUP_MAGE_VERSION\: [\"'].*[\"']" "$PIPELINE" | awk '{print $2}' | sed "s/'//g" ) +fi +if [[ -z "${SETUP_GVM_VERSION-""}" ]]; then + SETUP_GVM_VERSION=$(grep -oe "SETUP_GVM_VERSION\: [\"'].*[\"']" "$PIPELINE" | awk '{print $2}' | sed "s/'//g" ) +fi +BEAT_VERSION=$(grep -oE '[0-9]+\.[0-9]+\.[0-9]+(\-[a-zA-Z]+[0-9]+)?' "${WORKSPACE}/version/version.go") +export BEAT_VERSION + +getOSOptions() { + case $(uname | tr '[:upper:]' '[:lower:]') in + linux*) + export AGENT_OS_NAME=linux + ;; + darwin*) + export AGENT_OS_NAME=darwin + ;; + msys*) + export AGENT_OS_NAME=windows + ;; + *) + export AGENT_OS_NAME=notset + ;; + esac + case $(uname -m | tr '[:upper:]' '[:lower:]') in + aarch64*) + export AGENT_OS_ARCH=arm64 + ;; + arm64*) + export AGENT_OS_ARCH=arm64 + ;; + amd64*) + export AGENT_OS_ARCH=amd64 + ;; + x86_64*) + export AGENT_OS_ARCH=amd64 + ;; + *) + export AGENT_OS_ARCH=notset + ;; + esac +} + +# Wrapper function for executing mage +mage() { + go version + if ! [ -x "$(type -p mage | sed 's/mage is //g')" ]; + then + echo "installing mage ${SETUP_MAGE_VERSION}" + make mage + fi + pushd "$WORKSPACE" + command "mage" "$@" + ACTUAL_EXIT_CODE=$? + popd + return $ACTUAL_EXIT_CODE +} + +# Wrapper function for executing go +go(){ + # Search for the go in the Path + if ! [ -x "$(type -p go | sed 's/go is //g')" ]; + then + getOSOptions + echo "installing golang "${GO_VERSION}" for "${AGENT_OS_NAME}/${AGENT_OS_ARCH}" " + local _bin="${WORKSPACE}/bin" + mkdir -p "${_bin}" + retry 5 curl -sL -o "${_bin}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${AGENT_OS_NAME}-${AGENT_OS_ARCH}" + chmod +x "${_bin}/gvm" + eval "$(command "${_bin}/gvm" "${GO_VERSION}" )" + export GOPATH=$(command go env GOPATH) + export PATH="${PATH}:${GOPATH}/bin" + fi + pushd "$WORKSPACE" + command go "$@" + ACTUAL_EXIT_CODE=$? + popd + return $ACTUAL_EXIT_CODE +} + +google_cloud_auth() { + local keyFile=$1 + + gcloud auth activate-service-account --key-file ${keyFile} 2> /dev/null + + export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation} +} + +retry() { + local retries=$1 + shift + + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} diff --git a/.buildkite/scripts/steps/merge.sh b/.buildkite/scripts/steps/merge.sh new file mode 100755 index 00000000000..4eda3e2fd3b --- /dev/null +++ b/.buildkite/scripts/steps/merge.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Downloads and merges coverage files from multiple steps into a single file (build/TEST-go-unit.cov). +# Usage: merge.sh ... Where is the id of the step that contains the coverage artifact.# + +set -exuo pipefail + +COV_ARTIFACT="coverage.out" +MERGED_COV_FILE="build/TEST-go-unit.cov" +# Space separated list of paths to coverage files +COV_PATHS="" + +go install github.com/wadey/gocovmerge@latest +mkdir -p build + +for STEP_ID in "$@"; do + mkdir -p $STEP_ID + buildkite-agent artifact download --step $STEP_ID $COV_ARTIFACT $STEP_ID + COV_PATHS="${COV_PATHS} $STEP_ID/$COV_ARTIFACT" +done + +gocovmerge $COV_PATHS > $MERGED_COV_FILE +echo "Merged coverage file: $MERGED_COV_FILE. See artifacts" \ No newline at end of file diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1 new file mode 100644 index 00000000000..e1645554532 --- /dev/null +++ b/.buildkite/scripts/steps/unit-tests.ps1 @@ -0,0 +1,28 @@ +$ErrorActionPreference = "Stop" + +Write-Host "-- Fixing CRLF in git checkout --" +git config core.autocrlf input +git rm --quiet --cached -r . +git reset --quiet --hard + +$env:GOTMPDIR = "$env:BUILDKITE_BUILD_CHECKOUT_PATH" + +Write-Host "--- Build" +mage build + +if ($LASTEXITCODE -ne 0) { + exit 1 +} + +Write-Host "--- Unit tests" +$env:TEST_COVERAGE = $true +$env:RACE_DETECTOR = $true +mage unitTest +# Copy coverage file to build directory so it can be downloaded as an artifact +cp .\build\TEST-go-unit.cov coverage.out + +if ($LASTEXITCODE -ne 0) { + exit 1 +} + + diff --git a/.buildkite/scripts/steps/unit-tests.sh b/.buildkite/scripts/steps/unit-tests.sh new file mode 100644 index 00000000000..6df75a26a08 --- /dev/null +++ b/.buildkite/scripts/steps/unit-tests.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euxo pipefail + +source .buildkite/scripts/common.sh + +echo "--- Unit tests" +RACE_DETECTOR=true TEST_COVERAGE=true mage unitTest +# Copy coverage file to build directory so it can be downloaded as an artifact +cp build/TEST-go-unit.cov coverage.out \ No newline at end of file diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 0b7d459834e..07bf19d7bf1 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -95,57 +95,7 @@ pipeline { values 'ubuntu-22 && immutable', 'aws && aarch64 && gobld/diskSizeGb:200', 'windows-2016 && windows-immutable', 'windows-2022 && windows-immutable' //, 'macos12 && x86_64' } } - stages { - stage('build'){ - steps { - withGithubNotify(context: "Build-${PLATFORM}") { - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - withMageEnv(){ - dir("${BASE_DIR}"){ - cmd(label: 'Go build', script: 'mage build') - } - } - } - } - } - stage('Test') { - steps { - withGithubNotify(context: "Test-${PLATFORM}") { - withMageEnv(){ - dir("${BASE_DIR}"){ - withEnv(["RACE_DETECTOR=true", "TEST_COVERAGE=${isCodeCoverageEnabled()}"]) { - cmd(label: 'Go unitTest', script: 'mage unitTest') - } - } - } - } - } - post { - always { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/TEST-*.xml") - whenTrue(isCodeCoverageEnabled()) { - coverageReport(baseDir: "**/build", reportFiles: 'TEST-go-unit.html', coverageFiles: 'TEST-go-unit-cov.xml') - } - } - } - } - stage('Integration tests') { - when { - // Always when running builds on branches/tags - // Enable if integration test related changes. - beforeAgent true - anyOf { - not { changeRequest() } - expression { return params.integration_tests_ci } - expression { return env.GITHUB_COMMENT?.contains('integration tests') } - expression { matchesPrLabel(label: 'ci:integration') } - } - } - steps { - echo 'TBD' - } - } + stages { stage('K8s') { when { beforeAgent true @@ -160,7 +110,13 @@ pipeline { } } steps { +<<<<<<< HEAD runK8s(k8sVersion: 'v1.26.0', kindVersion: 'v0.17.0', context: "K8s-${PLATFORM}") +======= + deleteDir() + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + runK8s(k8sVersion: 'v1.27.3', kindVersion: 'v0.20.0', context: "K8s-${PLATFORM}") +>>>>>>> bf78cc1e14 ([CI] Pull requests migration to buildkite (#3573)) } } stage('Package') { @@ -348,77 +304,7 @@ pipeline { } } } - } - stage('m1') { - agent { label 'orka && darwin && aarch64' } - options { skipDefaultCheckout() } - when { - // Always when running builds on branches/tags - // Enable if extended M1 support related changes. - beforeAgent true - anyOf { - not { changeRequest() } - expression { return isExtendedM1Enabled() && env.ONLY_DOCS == "false"} - } - } - stages { - stage('build'){ - steps { - withGithubNotify(context: "Build-darwin-aarch64") { - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - withMageEnv(){ - dir("${BASE_DIR}"){ - cmd(label: 'Go build', script: 'mage build') - } - } - } - } - } - stage('Test') { - steps { - withGithubNotify(context: "Test-darwin-aarch64") { - withMageEnv(){ - dir("${BASE_DIR}"){ - withEnv(["RACE_DETECTOR=true", "TEST_COVERAGE=${isCodeCoverageEnabled()}"]) { - cmd(label: 'Go unitTest', script: 'mage unitTest') - } - } - } - } - } - post { - always { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/TEST-*.xml") - whenTrue(isCodeCoverageEnabled()) { - coverageReport(baseDir: "**/build", reportFiles: 'TEST-go-unit.html', coverageFiles: 'TEST-go-unit-cov.xml') - } - } - } - } - } - } - stage('e2e tests') { - when { - // Disable until we fix the e2e test. - // This should be running when there is a comment or the e2e label - beforeAgent true - anyOf { - // package artifacts are not generated if ONLY_DOCS, therefore e2e should not run if ONLY_DOCS - expression { return isE2eEnabled() && env.ONLY_DOCS == "false"} - } - } - steps { - runE2E(testMatrixFile: '.ci/.e2e-tests-for-elastic-agent.yaml', - beatVersion: "${env.BEAT_VERSION}-SNAPSHOT", - elasticAgentVersion: "${env.BEAT_VERSION}-SNAPSHOT", - gitHubCheckName: "e2e-tests", - gitHubCheckRepo: env.REPO, - gitHubCheckSha1: env.GIT_BASE_COMMIT, - propagate: true, - wait: true) - } - } + } } post { cleanup { diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml deleted file mode 100644 index 23a7c6a4da8..00000000000 --- a/.github/workflows/macos.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: macos - -on: - pull_request: - push: - branches: - - main - - 8.* - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Install dependencies - run: go install github.com/magefile/mage - - name: Run build - run: mage build - - name: Run test - run: mage unitTest diff --git a/internal/pkg/agent/application/upgrade/upgrade_test.go b/internal/pkg/agent/application/upgrade/upgrade_test.go index 84a2977e84b..456a115b4db 100644 --- a/internal/pkg/agent/application/upgrade/upgrade_test.go +++ b/internal/pkg/agent/application/upgrade/upgrade_test.go @@ -93,7 +93,7 @@ func Test_CopyFile(t *testing.T) { } err := copyDir(l, tc.From, tc.To, tc.IgnoreErr) - require.Equal(t, tc.ExpectedErr, err != nil) + require.Equal(t, tc.ExpectedErr, err != nil, err) }) } }