fix(arm64): use qemu to do the packaging step for arm64 images refs: #ENGEN-715

.ci/setup_ci.sh

@@ -8,6 +8,11 @@ if [ "$RESTY_IMAGE_TAG" != "bionic" ] && [ "$RESTY_IMAGE_TAG" != "18.04" ] && [
     exit 0
 fi
 
+sudo apt-get install -y \
+    qemu \
+    binfmt-support \
+    qemu-user-static
+
 docker version
 RESULT=$?
 if [ "$RESULT" != "0" ]; then
@@ -36,6 +41,7 @@ if ! [ -x "$(command -v docker-machine)" ]; then
     sudo install docker-machine /usr/local/bin/docker-machine
 fi
 
+set -e
 echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true
 echo "$REDHAT_PASSWORD" | docker login -u "$REDHAT_USERNAME" registry.access.redhat.com --password-stdin || true
 docker-machine version

Jenkinsfile

@@ -135,7 +135,6 @@ pipeline {
                         }
                     }
                     environment {
-                        AWS_ACCESS_KEY = "instance-profile"
                         PATH = "/home/ubuntu/bin/:${env.PATH}"
                         PACKAGE_TYPE = "rpm"
                     }
@@ -161,7 +160,7 @@ pipeline {
                         sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true'
                         sh 'git clone --single-branch --branch ${KONG_SOURCE} https://github.com/Kong/kong.git ${KONG_SOURCE_LOCATION}'
                         sh 'make RESTY_IMAGE_BASE=src    RESTY_IMAGE_TAG=src  PACKAGE_TYPE=src package-kong test cleanup'
-                        sh 'make RESTY_IMAGE_BASE=alpine RESTY_IMAGE_TAG=3.10 PACKAGE_TYPE=apk CACHE=false UPDATE_CACHE=true DOCKER_MACHINE_ARM64_NAME="jenkins-kong-"`cat /proc/sys/kernel/random/uuid` package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=alpine RESTY_IMAGE_TAG=3 PACKAGE_TYPE=apk CACHE=false UPDATE_CACHE=true DOCKER_MACHINE_ARM64_NAME="jenkins-kong-"`cat /proc/sys/kernel/random/uuid` package-kong test cleanup'
                     }
                 }
                 stage('Kong OSS DEB') {

Makefile

@@ -43,11 +43,12 @@ OFFICIAL_RELEASE ?= true
 PACKAGE_CONFLICTS ?= `grep PACKAGE_CONFLICTS $(KONG_SOURCE_LOCATION)/.requirements | awk -F"=" '{print $$2}'`
 PACKAGE_PROVIDES ?= `grep PACKAGE_PROVIDES $(KONG_SOURCE_LOCATION)/.requirements | awk -F"=" '{print $$2}'`
 PACKAGE_REPLACES ?= `grep PACKAGE_REPLACES $(KONG_SOURCE_LOCATION)/.requirements | awk -F"=" '{print $$2}'`
-DOCKER_RELEASE_REPOSITORY ?= `grep DOCKER_RELEASE_REPOSITORY $(KONG_SOURCE_LOCATION)/.requirements | awk -F"=" '{print $$2}'`
+DOCKER_RELEASE_REPOSITORY?="kong/kong"
 
 KONG_TEST_CONTAINER_NAME=kong-tests
-KONG_TEST_CONTAINER_TAG?=5000/kong-$(RESTY_IMAGE_BASE)-$(RESTY_IMAGE_TAG)
-KONG_TEST_IMAGE_NAME?=localhost:$(KONG_TEST_CONTAINER_TAG)
+KONG_TEST_CONTAINER_TAG?=$(KONG_VERSION)-$(RESTY_IMAGE_BASE)-$(RESTY_IMAGE_TAG)
+ADDITIONAL_TAG_LIST?=
+KONG_TEST_IMAGE_NAME?=$(DOCKER_RELEASE_REPOSITORY):$(KONG_TEST_CONTAINER_TAG)
 
 # This logic should mirror the kong-build-tools equivalent
 KONG_VERSION?=`$(KONG_SOURCE_LOCATION)/distribution/grep-kong-version.sh`
@@ -99,7 +100,7 @@ BUILDX_INFO ?= $(shell docker buildx 2>&1 >/dev/null; echo $?)
 DOCKER_LABELS?=--label org.opencontainers.image.version=$(KONG_VERSION) --label org.opencontainers.image.created=`date -u +'%Y-%m-%dT%H:%M:%SZ'` --label org.opencontainers.image.revision=$(KONG_SHA)
 
 ifeq ($(BUILDX),false)
-	DOCKER_COMMAND?=docker build --progress=$(DOCKER_BUILD_PROGRESS) $(KONG_EE_PORTS_FLAG) --build-arg BUILDPLATFORM=x/amd64 $(DOCKER_LABELS)
+	DOCKER_COMMAND?=docker build --build-arg TARGETPLATFORM=linux/amd64 --progress=$(DOCKER_BUILD_PROGRESS) $(KONG_EE_PORTS_FLAG) $(DOCKER_LABELS)
 else
 	DOCKER_COMMAND?=docker buildx build --progress=$(DOCKER_BUILD_PROGRESS) $(KONG_EE_PORTS_FLAG) --push --platform="linux/amd64,linux/arm64" $(DOCKER_LABELS)
 endif
@@ -328,6 +329,25 @@ test-kong: kong-test-container
 	bash -c 'healthy=$$(docker-compose ps | grep healthy | wc -l); while [[ "$$(( $$healthy ))" != "3" ]]; do docker-compose ps && sleep 5; done'
 	docker exec kong /kong/.ci/run_tests.sh && make update-cache-images
 
+release-kong-docker-images: test
+ifeq ($(BUILDX),false)
+	docker push $(KONG_TEST_IMAGE_NAME)
+else
+	docker push $(DOCKER_RELEASE_REPOSITORY):amd64-$(KONG_TEST_CONTAINER_TAG)
+	docker push $(DOCKER_RELEASE_REPOSITORY):arm64-$(KONG_TEST_CONTAINER_TAG)
+	docker manifest create $(KONG_TEST_IMAGE_NAME) -a \
+		$(DOCKER_RELEASE_REPOSITORY):amd64-$(KONG_TEST_CONTAINER_TAG) \
+		$(DOCKER_RELEASE_REPOSITORY):arm64-$(KONG_TEST_CONTAINER_TAG)
+	docker manifest push $(KONG_TEST_IMAGE_NAME)
+endif
+	for ADDITIONAL_TAG in $(ADDITIONAL_TAG_LIST); do \
+		docker run -t --rm \
+			-v ~/.docker/config.json:/tmp/auth.json \
+			quay.io/skopeo/stable:latest \
+			copy --all docker://docker.io/$(KONG_TEST_IMAGE_NAME) \
+			docker://docker.io/$(DOCKER_RELEASE_REPOSITORY):$$ADDITIONAL_TAG ; \
+	done
+
 release-kong: test
 	ARCHITECTURE=amd64 \
 	RESTY_IMAGE_BASE=$(RESTY_IMAGE_BASE) \
@@ -410,34 +430,39 @@ ifneq ($(RESTY_IMAGE_BASE),src)
 endif
 
 build-test-container:
+ifneq ($(RESTY_IMAGE_BASE),src)
 	touch test/kong_license.private
 	ARCHITECTURE=amd64 \
+	KONG_PACKAGE_NAME=$(KONG_PACKAGE_NAME) \
 	PACKAGE_TYPE=$(PACKAGE_TYPE) \
 	RESTY_IMAGE_BASE=$(RESTY_IMAGE_BASE) \
 	RESTY_IMAGE_TAG=$(RESTY_IMAGE_TAG) \
 	KONG_VERSION=$(KONG_VERSION) \
-	KONG_PACKAGE_NAME=$(KONG_PACKAGE_NAME) \
-	KONG_TEST_IMAGE_NAME=$(KONG_TEST_IMAGE_NAME) \
+	DOCKER_RELEASE_REPOSITORY=$(DOCKER_RELEASE_REPOSITORY) \
+	KONG_TEST_CONTAINER_TAG=$(KONG_TEST_CONTAINER_TAG) \
 	DOCKER_KONG_VERSION=$(DOCKER_KONG_VERSION) \
 	DOCKER_BUILD_PROGRESS=$(DOCKER_BUILD_PROGRESS) \
 	DOCKER_LABELS="$(DOCKER_LABELS)" \
 	test/build_container.sh
+	docker tag $(DOCKER_RELEASE_REPOSITORY):amd64-$(KONG_TEST_CONTAINER_TAG) \
+		$(DOCKER_RELEASE_REPOSITORY):$(KONG_TEST_CONTAINER_TAG)
+	docker tag $(DOCKER_RELEASE_REPOSITORY):amd64-$(KONG_TEST_CONTAINER_TAG) \
+		$(KONG_TEST_IMAGE_NAME)
 ifeq ($(BUILDX),true)
-	DOCKER_MACHINE_NAME=$(shell docker-machine env $(DOCKER_MACHINE_ARM64_NAME) | grep 'DOCKER_MACHINE_NAME=".*"' | cut -d\" -f2) \
-	DOCKER_TLS_VERIFY=1 \
-	DOCKER_HOST=$(shell docker-machine env $(DOCKER_MACHINE_ARM64_NAME) | grep 'DOCKER_HOST=".*"' | cut -d\" -f2) \
-	DOCKER_CERT_PATH=$(shell docker-machine env $(DOCKER_MACHINE_ARM64_NAME) | grep 'DOCKER_CERT_PATH=".*"' | cut -d\" -f2) \
+	docker run --rm --privileged multiarch/qemu-user-static --reset -p yes && \
 	ARCHITECTURE=arm64 \
+	KONG_PACKAGE_NAME=$(KONG_PACKAGE_NAME) \
 	PACKAGE_TYPE=$(PACKAGE_TYPE) \
 	RESTY_IMAGE_BASE=$(RESTY_IMAGE_BASE) \
 	RESTY_IMAGE_TAG=$(RESTY_IMAGE_TAG) \
 	KONG_VERSION=$(KONG_VERSION) \
-	KONG_PACKAGE_NAME=$(KONG_PACKAGE_NAME) \
-	KONG_TEST_IMAGE_NAME=$(KONG_TEST_IMAGE_NAME) \
+	DOCKER_RELEASE_REPOSITORY=$(DOCKER_RELEASE_REPOSITORY) \
+	KONG_TEST_CONTAINER_TAG=$(KONG_TEST_CONTAINER_TAG) \
 	DOCKER_KONG_VERSION=$(DOCKER_KONG_VERSION) \
 	DOCKER_LABELS="$(DOCKER_LABELS)" \
 	test/build_container.sh
 endif
+endif
 
 setup-tests: cleanup-tests
 ifneq ($(RESTY_IMAGE_BASE),src)

README.md

@@ -22,28 +22,19 @@ Packaging arm64 architectures additionally requires:
 
 - [Docker-machine](https://github.com/docker/machine)
 - [Buildx Docker plugin](https://github.com/docker/buildx)
-- AWS Credentials
+- AWS Credentials (or access via an instance profile)
 
-## Packaging a Kong Distribution
+Packaging kong-ee additionally requires:
 
-The default build task builds an Ubuntu bionic package of Kong where the Kong source is assumed to be
-in a sibling directory to where this repository is cloned
+- A `GITHUB_TOKEN` environment variable with access to Kong's private github repositories
+
+## Building a Kong Package
 
 ```
+export PACKAGE_TYPE=deb RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=20.04 # defaults if not set
 make package-kong
 ls output/
-kong-0.0.0.bionic.all.deb
-```
-
-**Environment variables:**
-
-You can find all available environment variables at the top of the [Makefile](https://github.com/Kong/kong-build-tools/blob/master/Makefile).
-The most common ones are the following:
-
-```
-RESTY_IMAGE_BASE=ubuntu|centos|rhel|debian|alpine|amazonlinux
-RESTY_IMAGE_TAG=18.04|20.04|7|8|9|10|11|latest|latest
-PACKAGE_TYPE=deb|rpm|apk
+kong-x.y.z.20.04.all.deb
 ```
 
 ### Details
@@ -55,7 +46,23 @@ The Docker files in the dockerfiles directory build on each other in the followi
 - `Dockerfile.package` builds on top of the result of `Dockerfile.kong` to package Kong using `fpm-entrypoint.sh`
 - `Dockerfile.kong` builds on top of the result of `Dockerfile.openresty` to build Kong using `build-kong.sh`
 - `Dockerfile.openresty` builds on top of the result of `Dockerfile.(deb|apk|rpm)` to build the Kong prerequisites using `openresty-build-tools/kong-ngx-build`
-- `Dockerfile.(deb|apk|rpm)` builds the compilation / building prerequisites
+- [github://kong/kong-build-tools-base-images](https://github.com/Kong/kong-build-tools-base-images) builds the compilation / building prerequisites
+
+## Building a Kong Docker Image
+
+Prerequisite: you did the packaging step
+```
+export KONG_TEST_CONTAINER_NAME=kong/kong:x.y.z-ubuntu-20.04 #default if not set
+make build-test-container
+```
+
+## Releasing Docker Images
+
+Prerequisite: you did the packaging step and you're logged into docker with the necessary push permissions
+```
+export DOCKER_RELEASE_REPOSITORY=kong/kong KONG_TEST_CONTAINER_TAG=x.y.z-ubuntu-20.04 #default if not set
+make release-kong-docker-images
+```
 
 ## Running Kong Tests
 
@@ -82,29 +89,7 @@ TEST_SUITE = "dbless|plugins|unit|integration"
 - `Dockerfile.openresty` builds on top of the result of `Dockerfile.(deb|apk|rpm)` to build the Kong prerequisites using `openresty-build-tools/kong-ngx-build`
 - `Dockerfile.(deb|apk|rpm)` builds the compilation / building prerequisites
 
-### Debugging Tests
-
-If you want to mirror a failed test from CI pull the test image the CI built and retag it:
-
-```
-docker pull mashape/kong-build-tools:test-33c8ceb-e2bb1fd54f8d5c12f989a801a44979b610-14
-docker tag mashape/kong-build-tools:test-33c8ceb-e2bb1fd54f8d5c12f989a801a44979b610-14 mashape/kong-build-tools:test
-```
-
-If you're trying to test local Kong source code build the test image:
-
-```
-make kong-test-container
-```
-
-Now spin up the containers using docker-compose and jump into the Kong image
-```
-docker-compose up -d
-docker-compose exec kong /bin/bash
-./ci/run_tests.sh
-```
-
-## Running Functional Tests
+## Running Packaging / Smoke Tests
 
 The Kong Build Tools functional tests suite run a tests on a Kong package which we then integrate
 into our official docker build image dockerfile.

dockerfiles/Dockerfile.package

@@ -22,7 +22,10 @@ ARG RESTY_IMAGE_TAG="18.04"
 ARG PACKAGE_TYPE
 ARG KONG_VERSION
 ARG KONG_PACKAGE_NAME
-ARG BUILDPLATFORM
+ARG TARGETPLATFORM
+ENV TARGETPLATFORM=${TARGETPLATFORM}
+ARG TARGETPLATFORM
+ENV TARGETPLATFORM=${TARGETPLATFORM}
 
 ARG PACKAGE_CONFLICTS="kong-enterprise-edition"
 ENV PACKAGE_CONFLICTS $PACKAGE_CONFLICTS
@@ -40,4 +43,4 @@ COPY kong.logrotate /tmp/build/etc/kong/kong.logrotate
 RUN /fpm-entrypoint.sh
 
 FROM alpine
-COPY --from=FPM /output /output
+COPY --from=FPM /output /output

fpm-entrypoint.sh

@@ -43,8 +43,7 @@ elif [ "$PACKAGE_TYPE" == "rpm" ]; then
     OUTPUT_FILE_SUFFIX=".el${RESTY_IMAGE_TAG}"
   fi
 fi
-OUTPUT_FILE_SUFFIX="${OUTPUT_FILE_SUFFIX}."$(echo ${BUILDPLATFORM} | awk -F "/" '{ print $2}')
-
+OUTPUT_FILE_SUFFIX="${OUTPUT_FILE_SUFFIX}."$(echo ${TARGETPLATFORM} | awk -F "/" '{ print $2}')
 ROCKSPEC_VERSION=`basename /tmp/build/build/usr/local/lib/luarocks/rocks/kong/*`
 
 if [ "$PACKAGE_TYPE" == "apk" ]; then

test/build_container.sh

@@ -5,6 +5,8 @@ set -e
 
 DOCKER_BUILD_ARGS=()
 
+KONG_TEST_IMAGE_NAME=$DOCKER_RELEASE_REPOSITORY:$ARCHITECTURE-$KONG_TEST_CONTAINER_TAG
+
 image_id=$(docker image inspect -f '{{.ID}}' "$KONG_TEST_IMAGE_NAME" || true)
 if [ -n "$image_id" ]; then
   msg_test "Tests image Name: $KONG_TEST_IMAGE_NAME"
@@ -19,7 +21,7 @@ chmod -R 755 docker-kong/*.sh
 if [ "$RESTY_IMAGE_BASE" == "src" ]; then
   exit 0
 elif [ "$RESTY_IMAGE_BASE" == "alpine" ]; then
-  cp output/*${ARCHITECTURE}*.apk.tar.gz docker-kong/kong.apk.tar.gz
+  cp output/${KONG_PACKAGE_NAME}-${KONG_VERSION}.${ARCHITECTURE}.apk.tar.gz docker-kong/kong.apk.tar.gz
 elif [ "$PACKAGE_TYPE" == "deb" ]; then
   cp output/*${ARCHITECTURE}*.deb docker-kong/kong.deb
 else
@@ -45,14 +47,18 @@ pushd ./docker-kong
     DOCKER_BUILD_ARGS+=(--label "org.opencontainers.image.revision=$DOCKER_LABEL_REVISION")
   fi
 
+  DOCKER_BUILD_ARGS+=(--platform linux/${ARCHITECTURE})
   DOCKER_BUILD_ARGS+=(--no-cache)
+  DOCKER_BUILD_ARGS+=(--pull)
   DOCKER_BUILD_ARGS+=(--build-arg ASSET=local .)
-
-  docker build --progress=${DOCKER_BUILD_PROGRESS:-auto} -t $KONG_TEST_IMAGE_NAME -f Dockerfile.$PACKAGE_TYPE \
+
+  docker build \
+    --progress=${DOCKER_BUILD_PROGRESS:-auto} \
+    -t $KONG_TEST_IMAGE_NAME \
+    -f Dockerfile.$PACKAGE_TYPE \
     ${DOCKER_LABELS} \
     "${DOCKER_BUILD_ARGS[@]}"
 
-  docker run -t $KONG_TEST_IMAGE_NAME kong version
   msg_test "Tests image Name: $KONG_TEST_IMAGE_NAME"
 popd
 

test/tests/01-package/run.sh

@@ -8,6 +8,7 @@ then
   major="${RESTY_IMAGE_TAG%%.*}"
   docker run -d --name user-validation-tests --rm -e KONG_DATABASE=off -v $PWD:/src registry.access.redhat.com/ubi${major}/ubi tail -f /dev/null
 else
+  docker rmi ${RESTY_IMAGE_BASE}:${RESTY_IMAGE_TAG} --force
   docker run -d --name user-validation-tests --rm -e KONG_DATABASE=off -v $PWD:/src ${RESTY_IMAGE_BASE}:${RESTY_IMAGE_TAG} tail -f /dev/null
 fi