From 909b0fc29af5f76d8c9d3253382c997d4f604b74 Mon Sep 17 00:00:00 2001 From: Colin Hutchinson Date: Wed, 23 Nov 2022 12:38:27 +0000 Subject: [PATCH 1/2] chore(ci): migrate some CI from Jenkins to github actions (#9795) * chore(Make): add the convenience make tasks * chore(gha): migrate test package and unofficial packaging from Jenkins to GHA * chore(dev): can the rockspec be renamed so we don't overwrite unofficial tagged version? * chore(ci): revert the rockspec rename * fix(ci): we don't need to docker login for CE test packaging * chore(ci): we need to keep building alpine unofficial in Jenkins (cherry picked from commit b1a1f07fe9ebc58bd4d202345e733c945c50b398) --- .github/workflows/package.yml | 126 ++++++++++++++++++++++++++++++++++ Jenkinsfile | 93 +++++++++++++++---------- Makefile | 53 +++++++++++++- 3 files changed, 235 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/package.yml diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml new file mode 100644 index 00000000000..a7400595d8a --- /dev/null +++ b/.github/workflows/package.yml @@ -0,0 +1,126 @@ +name: Package & Smoke Test + +on: # yamllint disable-line rule:truthy + pull_request: + push: + branches: + - master + - next/* + - release/* + +env: + DOCKER_REPOSITORY: kong/kong-build-tools + +jobs: + package-and-test: + if: github.event_name == 'pull_request' + name: Build & Smoke Test Packages + runs-on: ubuntu-22.04 + + steps: + - name: Swap git with https + run: git config --global url."https://github".insteadOf git://github + + - name: Setup some environment variables + run: | + echo "KONG_SOURCE_LOCATION=$GITHUB_WORKSPACE/kong-src" >> $GITHUB_ENV + echo "KONG_BUILD_TOOLS_LOCATION=$GITHUB_WORKSPACE/kong-build-tools" >> $GITHUB_ENV + + - name: Checkout Kong source code + uses: actions/checkout@v3 + with: + path: ${{ env.KONG_SOURCE_LOCATION }} + submodules: recursive + token: ${{ secrets.GHA_KONG_BOT_READ_TOKEN }} + + - name: Setup kong-build-tools + run: | + pushd ${{ env.KONG_SOURCE_LOCATION }} + make setup-kong-build-tools + + - name: Setup package naming environment variables + run: | + grep -v '^#' ${{ env.KONG_SOURCE_LOCATION}}/.requirements >> $GITHUB_ENV + + - name: Package & Test + env: + GITHUB_TOKEN: ${{ secrets.GHA_KONG_BOT_READ_TOKEN }} + run: | + pushd ${{ env.KONG_SOURCE_LOCATION }} + make package/test/deb + + package-test-and-unofficial-release: + if: github.event_name == 'push' + name: Build & Smoke & Unofficial Release Packages + runs-on: ubuntu-22.04 + strategy: + matrix: + package_type: [deb, rpm, apk] + + steps: + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.GHA_DOCKERHUB_PUSH_USER }} + password: ${{ secrets.GHA_KONG_ORG_DOCKERHUB_PUSH_TOKEN }} + + - name: Swap git with https + run: git config --global url."https://github".insteadOf git://github + + - name: Setup directory environment variables + run: | + echo "KONG_SOURCE_LOCATION=$GITHUB_WORKSPACE/kong-src" >> $GITHUB_ENV + echo "KONG_BUILD_TOOLS_LOCATION=$GITHUB_WORKSPACE/kong-build-tools" >> $GITHUB_ENV + + - name: Checkout Kong source code + uses: actions/checkout@v3 + with: + path: ${{ env.KONG_SOURCE_LOCATION }} + submodules: recursive + token: ${{ secrets.GHA_KONG_BOT_READ_TOKEN }} + + - name: Setup kong-build-tools + run: | + pushd ${{ env.KONG_SOURCE_LOCATION }} + make setup-kong-build-tools + + - name: Setup package naming environment variables + run: | + grep -v '^#' ${{ env.KONG_SOURCE_LOCATION}}/.requirements >> $GITHUB_ENV + echo "DOCKER_RELEASE_REPOSITORY=kong/kong" >> $GITHUB_ENV + echo "KONG_TEST_CONTAINER_TAG=$GITHUB_REF_NAME-${{ matrix.package_type }}" >> $GITHUB_ENV + if [[ ${{matrix.package_type }} == "apk" ]]; then + echo "ADDITIONAL_TAG_LIST=$GITHUB_REF_NAME-alpine" >> $GITHUB_ENV + fi + if [[ ${{matrix.package_type }} == "deb" ]]; then + echo "ADDITIONAL_TAG_LIST=$GITHUB_REF_NAME-debian $GITHUB_REF_NAME $GITHUB_SHA" >> $GITHUB_ENV + fi + + - name: Package & Test + env: + GITHUB_TOKEN: ${{ secrets.GHA_KONG_BOT_READ_TOKEN }} + run: | + pushd ${{ env.KONG_SOURCE_LOCATION }} + make package/test/${{ matrix.package_type }} + + - name: Push Docker Image + env: + SKIP_TESTS: true + run: | + pushd ${{ env.KONG_SOURCE_LOCATION }} + make release/docker/${{ matrix.package_type }} + + - name: Store the package artifacts + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.package_type }} + path: ${{ env.KONG_BUILD_TOOLS_LOCATION }}/output/* + + - name: Comment on commit + continue-on-error: true + uses: peter-evans/commit-comment@v2 + with: + token: ${{ secrets.GHA_COMMENT_TOKEN }} + body: | + Docker image avaialble ${{ env.DOCKER_RELEASE_REPOSITORY }}:${{ env.KONG_TEST_CONTAINER_TAG }} + Artifacts availabe https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/Jenkinsfile b/Jenkinsfile index e3092e8a0d9..ed3904323f4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,24 +2,23 @@ pipeline { agent none options { retry(1) - timeout(time: 2, unit: 'HOURS') + timeout(time: 3, unit: 'HOURS') } environment { UPDATE_CACHE = "true" DOCKER_CREDENTIALS = credentials('dockerhub') DOCKER_USERNAME = "${env.DOCKER_CREDENTIALS_USR}" DOCKER_PASSWORD = "${env.DOCKER_CREDENTIALS_PSW}" - KONG_PACKAGE_NAME = "kong" DOCKER_CLI_EXPERIMENTAL = "enabled" - PULP_HOST_PROD = "https://api.download.konghq.com" + // PULP_PROD and PULP_STAGE are used to do releases + PULP_HOST_PROD = "https://api.pulp.konnect-prod.konghq.com" PULP_PROD = credentials('PULP') - PULP_HOST_STAGE = "https://api.download-dev.konghq.com" + PULP_HOST_STAGE = "https://api.pulp.konnect-stage.konghq.com" PULP_STAGE = credentials('PULP_STAGE') - GITHUB_TOKEN = credentials('github_bot_access_token') DEBUG = 0 } stages { - stage('Release Per Commit') { + stage('Release -- Release Branch Release to Unofficial Asset Stores') { when { beforeAgent true anyOf { @@ -27,30 +26,33 @@ pipeline { branch 'release/*'; } } - agent { - node { - label 'bionic' + parallel { + stage('Alpine') { + agent { + node { + label 'bionic' + } + } + environment { + KONG_SOURCE_LOCATION = "${env.WORKSPACE}" + KONG_BUILD_TOOLS_LOCATION = "${env.WORKSPACE}/../kong-build-tools" + AWS_ACCESS_KEY = "instanceprofile" + PACKAGE_TYPE = "apk" + GITHUB_SSH_KEY = credentials('github_bot_ssh_key') + } + options { + retry(2) + timeout(time: 2, unit: 'HOURS') + } + steps { + sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true' + sh 'make setup-kong-build-tools' + sh 'make RESTY_IMAGE_BASE=alpine RESTY_IMAGE_TAG=3 KONG_TEST_CONTAINER_TAG="${GIT_BRANCH##*/}-alpine" DOCKER_MACHINE_ARM64_NAME="kong-"`cat /proc/sys/kernel/random/uuid` release-docker-images' + } } } - environment { - KONG_PACKAGE_NAME = "kong" - KONG_SOURCE_LOCATION = "${env.WORKSPACE}" - KONG_BUILD_TOOLS_LOCATION = "${env.WORKSPACE}/../kong-build-tools" - AWS_ACCESS_KEY = "instanceprofile" - CACHE = "false" - UPDATE_CACHE = "true" - RELEASE_DOCKER_ONLY="true" - PACKAGE_TYPE="apk" - RESTY_IMAGE_BASE="alpine" - RESTY_IMAGE_TAG="latest" - } - steps { - sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true' - sh 'make setup-kong-build-tools' - sh 'KONG_VERSION=`git rev-parse --short HEAD` DOCKER_MACHINE_ARM64_NAME="jenkins-kong-"`cat /proc/sys/kernel/random/uuid` make release' - } } - stage('Release') { + stage('Release -- Tag Release to Official Asset Stores') { when { beforeAgent true allOf { @@ -73,14 +75,19 @@ pipeline { PRIVATE_KEY_PASSPHRASE = credentials('kong.private.gpg-key.asc.password') GITHUB_SSH_KEY = credentials('github_bot_ssh_key') } + options { + retry(2) + timeout(time: 2, unit: 'HOURS') + } steps { sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true' sh 'make setup-kong-build-tools' sh 'cp $PRIVATE_KEY_FILE ../kong-build-tools/kong.private.gpg-key.asc' sh 'make RESTY_IMAGE_BASE=amazonlinux RESTY_IMAGE_TAG=2 release' + sh 'make RESTY_IMAGE_BASE=amazonlinux RESTY_IMAGE_TAG=2022 release' sh 'make RESTY_IMAGE_BASE=centos RESTY_IMAGE_TAG=7 release' - sh 'make RESTY_IMAGE_BASE=rhel RESTY_IMAGE_TAG=7 release' - sh 'make RESTY_IMAGE_BASE=rhel RESTY_IMAGE_TAG=8 RELEASE_DOCKER=true release' + sh 'make RESTY_IMAGE_BASE=rhel RESTY_IMAGE_TAG=7.9 release' + sh 'make RESTY_IMAGE_BASE=rhel RESTY_IMAGE_TAG=8.6 RELEASE_DOCKER=true release' } } stage('DEB') { @@ -95,13 +102,18 @@ pipeline { PACKAGE_TYPE = "deb" GITHUB_SSH_KEY = credentials('github_bot_ssh_key') } + options { + retry(2) + timeout(time: 2, unit: 'HOURS') + } steps { sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true' sh 'make setup-kong-build-tools' sh 'make RESTY_IMAGE_BASE=debian RESTY_IMAGE_TAG=10 release' sh 'make RESTY_IMAGE_BASE=debian RESTY_IMAGE_TAG=11 RELEASE_DOCKER=true release' - sh 'make RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=18.04 AWS_ACCESS_KEY=instanceprofile CACHE=false DOCKER_MACHINE_ARM64_NAME="jenkins-kong-"`cat /proc/sys/kernel/random/uuid` release' + sh 'make RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=18.04 release' sh 'make RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=20.04 RELEASE_DOCKER=true release' + sh 'make RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=22.04 RELEASE_DOCKER=true release' } } stage('SRC & Alpine') { @@ -117,16 +129,20 @@ pipeline { GITHUB_SSH_KEY = credentials('github_bot_ssh_key') AWS_ACCESS_KEY = "instanceprofile" } + options { + retry(2) + timeout(time: 2, unit: 'HOURS') + } steps { sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true' sh 'make setup-kong-build-tools' sh 'make RESTY_IMAGE_BASE=src RESTY_IMAGE_TAG=src PACKAGE_TYPE=src release' - sh 'make RESTY_IMAGE_BASE=alpine RESTY_IMAGE_TAG=3.10 PACKAGE_TYPE=apk DOCKER_MACHINE_ARM64_NAME="kong-"`cat /proc/sys/kernel/random/uuid` RELEASE_DOCKER=true release' + sh 'make RESTY_IMAGE_BASE=alpine RESTY_IMAGE_TAG=3 PACKAGE_TYPE=apk DOCKER_MACHINE_ARM64_NAME="kong-"`cat /proc/sys/kernel/random/uuid` RELEASE_DOCKER=true release' } } } } - stage('Post Packaging Steps') { + stage('Post Release Steps') { when { beforeAgent true allOf { @@ -143,13 +159,14 @@ pipeline { } } environment { + GITHUB_TOKEN = credentials('github_bot_access_token') GITHUB_SSH_KEY = credentials('github_bot_ssh_key') SLACK_WEBHOOK = credentials('core_team_slack_webhook') GITHUB_USER = "mashapedeployment" } steps { sh './scripts/setup-ci.sh' - sh 'echo "y" | ./scripts/make-patch-release $TAG_NAME update_docker' + sh 'echo "y" | ./scripts/make-release $TAG_NAME update_docker' } post { failure { @@ -171,13 +188,14 @@ pipeline { } } environment { + GITHUB_TOKEN = credentials('github_bot_access_token') GITHUB_SSH_KEY = credentials('github_bot_ssh_key') SLACK_WEBHOOK = credentials('core_team_slack_webhook') GITHUB_USER = "mashapedeployment" } steps { sh './scripts/setup-ci.sh' - sh 'echo "y" | ./scripts/make-patch-release $TAG_NAME homebrew' + sh 'echo "y" | ./scripts/make-release $TAG_NAME homebrew' } post { failure { @@ -199,13 +217,14 @@ pipeline { } } environment { + GITHUB_TOKEN = credentials('github_bot_access_token') GITHUB_SSH_KEY = credentials('github_bot_ssh_key') SLACK_WEBHOOK = credentials('core_team_slack_webhook') GITHUB_USER = "mashapedeployment" } steps { sh './scripts/setup-ci.sh' - sh 'echo "y" | ./scripts/make-patch-release $TAG_NAME vagrant' + sh 'echo "y" | ./scripts/make-release $TAG_NAME vagrant' } post { failure { @@ -227,13 +246,14 @@ pipeline { } } environment { + GITHUB_TOKEN = credentials('github_bot_access_token') GITHUB_SSH_KEY = credentials('github_bot_ssh_key') SLACK_WEBHOOK = credentials('core_team_slack_webhook') GITHUB_USER = "mashapedeployment" } steps { sh './scripts/setup-ci.sh' - sh 'echo "y" | ./scripts/make-patch-release $TAG_NAME pongo' + sh 'echo "y" | ./scripts/make-release $TAG_NAME pongo' } post { always { @@ -247,3 +267,4 @@ pipeline { } } } + diff --git a/Makefile b/Makefile index 0ebee957602..4a4709438fe 100644 --- a/Makefile +++ b/Makefile @@ -101,9 +101,60 @@ setup-ci: KONG_NGINX_MODULE_BRANCH=$(KONG_NGINX_MODULE_BRANCH) \ .ci/setup_env.sh +package/deb: setup-kong-build-tools + cd $(KONG_BUILD_TOOLS_LOCATION); \ + PACKAGE_TYPE=deb RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=22.04 $(MAKE) package-kong && \ + cp $(KONG_BUILD_TOOLS_LOCATION)/output/*.deb . + +package/apk: setup-kong-build-tools + cd $(KONG_BUILD_TOOLS_LOCATION); \ + PACKAGE_TYPE=apk RESTY_IMAGE_BASE=alpine RESTY_IMAGE_TAG=3 $(MAKE) package-kong && \ + cp $(KONG_BUILD_TOOLS_LOCATION)/output/*.apk.* . + +package/rpm: setup-kong-build-tools + cd $(KONG_BUILD_TOOLS_LOCATION); \ + PACKAGE_TYPE=rpm RESTY_IMAGE_BASE=rhel RESTY_IMAGE_TAG=8.6 $(MAKE) package-kong && \ + cp $(KONG_BUILD_TOOLS_LOCATION)/output/*.rpm . + +package/test/deb: package/deb + cd $(KONG_BUILD_TOOLS_LOCATION); \ + PACKAGE_TYPE=deb RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=22.04 $(MAKE) test + +package/test/apk: package/apk + cd $(KONG_BUILD_TOOLS_LOCATION); \ + PACKAGE_TYPE=apk RESTY_IMAGE_BASE=alpine RESTY_IMAGE_TAG=3 $(MAKE) test + +package/test/rpm: package/rpm + cd $(KONG_BUILD_TOOLS_LOCATION); \ + PACKAGE_TYPE=rpm RESTY_IMAGE_BASE=rhel RESTY_IMAGE_TAG=8.6 $(MAKE) test + +package/docker/deb: package/deb + cd $(KONG_BUILD_TOOLS_LOCATION); \ + PACKAGE_TYPE=deb RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=22.04 $(MAKE) build-test-container + +package/docker/apk: package/apk + cd $(KONG_BUILD_TOOLS_LOCATION); \ + PACKAGE_TYPE=apk RESTY_IMAGE_BASE=alpine RESTY_IMAGE_TAG=3 $(MAKE) build-test-container + +package/docker/rpm: package/rpm + cd $(KONG_BUILD_TOOLS_LOCATION); \ + PACKAGE_TYPE=rpm RESTY_IMAGE_BASE=rhel RESTY_IMAGE_TAG=8.6 $(MAKE) build-test-container + +release/docker/deb: package/docker/deb + cd $(KONG_BUILD_TOOLS_LOCATION); \ + PACKAGE_TYPE=deb RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=22.04 $(MAKE) release-kong-docker-images + +release/docker/apk: package/docker/apk + cd $(KONG_BUILD_TOOLS_LOCATION); \ + PACKAGE_TYPE=apk RESTY_IMAGE_BASE=alpine RESTY_IMAGE_TAG=3 $(MAKE) release-kong-docker-images + +release/docker/rpm: package/docker/rpm + cd $(KONG_BUILD_TOOLS_LOCATION); \ + PACKAGE_TYPE=rpm RESTY_IMAGE_BASE=rhel RESTY_IMAGE_TAG=8.6 $(MAKE) release-kong-docker-images + setup-kong-build-tools: -rm -rf $(KONG_BUILD_TOOLS_LOCATION) - -git clone https://github.com/Kong/kong-build-tools.git $(KONG_BUILD_TOOLS_LOCATION) + -git clone https://github.com/Kong/kong-build-tools.git --recursive $(KONG_BUILD_TOOLS_LOCATION) cd $(KONG_BUILD_TOOLS_LOCATION); \ git reset --hard && git checkout $(KONG_BUILD_TOOLS); \ From b4c26322b908f38c460d00f8a98d775aaf2ac7ac Mon Sep 17 00:00:00 2001 From: Colin Hutchinson Date: Wed, 23 Nov 2022 13:53:50 +0000 Subject: [PATCH 2/2] fix(submodule): removing a submodule I believe was accidentally committed --- kong-dp-spec | 1 - 1 file changed, 1 deletion(-) delete mode 160000 kong-dp-spec diff --git a/kong-dp-spec b/kong-dp-spec deleted file mode 160000 index f9432f8c80b..00000000000 --- a/kong-dp-spec +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f9432f8c80b419892b3479178e8c3779b3b1f258