From c69e47a27aace0f9b9f21bb9f2d960ec786fcb77 Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Fri, 10 Dec 2021 13:18:31 +0100 Subject: [PATCH] Iniital multi arch build --- Makefile | 52 +++++++++++++++++++++---------- docs/advanced/developer-guide.md | 28 +++++++++++++++++ hack/init-buildx.sh | 46 +++++++++++++++++++++++++++ scripts/test-infra/build-image.sh | 4 +++ scripts/test-infra/push-image.sh | 3 +- 5 files changed, 115 insertions(+), 18 deletions(-) create mode 100755 hack/init-buildx.sh diff --git a/Makefile b/Makefile index b2388a952c..9f38abcb3d 100644 --- a/Makefile +++ b/Makefile @@ -59,6 +59,30 @@ E2E_TEST_CONFIG ?= LDFLAGS = -ldflags "-s -w -X sigs.k8s.io/node-feature-discovery/pkg/version.version=$(VERSION) -X sigs.k8s.io/node-feature-discovery/source.pathPrefix=$(HOSTMOUNT_PREFIX)" +# multi-arch build with buildx +IMAGE_ALL_PLATFORMS ?= linux/amd64,linux/arm64 + +# enable buildx +ensure-buildx: + ./hack/init-buildx.sh + +IMAGE_BUILDX_CMD ?= DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --platform=${IMAGE_ALL_PLATFORMS} --progress=auto --pull + +IMAGE_BUILD_ARGS = --build-arg VERSION=$(VERSION) \ + --build-arg HOSTMOUNT_PREFIX=$(CONTAINER_HOSTMOUNT_PREFIX) \ + --build-arg BASE_IMAGE_FULL=$(BASE_IMAGE_FULL) \ + --build-arg BASE_IMAGE_MINIMAL=$(BASE_IMAGE_MINIMAL) + +IMAGE_BUILD_ARGS_FULL = --target full \ + -t $(IMAGE_TAG) \ + $(foreach tag,$(IMAGE_EXTRA_TAGS),-t $(tag)) \ + $(IMAGE_BUILD_EXTRA_OPTS) ./ + +IMAGE_BUILD_ARGS_MINIMAL = --target minimal \ + -t $(IMAGE_TAG)-minimal \ + $(foreach tag,$(IMAGE_EXTRA_TAGS),-t $(tag)-minimal) \ + $(IMAGE_BUILD_EXTRA_OPTS) ./ + all: image build: @@ -69,22 +93,14 @@ install: $(GO_CMD) install -v $(LDFLAGS) ./cmd/... image: yamls - $(IMAGE_BUILD_CMD) --build-arg VERSION=$(VERSION) \ - --target full \ - --build-arg HOSTMOUNT_PREFIX=$(CONTAINER_HOSTMOUNT_PREFIX) \ - --build-arg BASE_IMAGE_FULL=$(BASE_IMAGE_FULL) \ - --build-arg BASE_IMAGE_MINIMAL=$(BASE_IMAGE_MINIMAL) \ - -t $(IMAGE_TAG) \ - $(foreach tag,$(IMAGE_EXTRA_TAGS),-t $(tag)) \ - $(IMAGE_BUILD_EXTRA_OPTS) ./ - $(IMAGE_BUILD_CMD) --build-arg VERSION=$(VERSION) \ - --target minimal \ - --build-arg HOSTMOUNT_PREFIX=$(CONTAINER_HOSTMOUNT_PREFIX) \ - --build-arg BASE_IMAGE_FULL=$(BASE_IMAGE_FULL) \ - --build-arg BASE_IMAGE_MINIMAL=$(BASE_IMAGE_MINIMAL) \ - -t $(IMAGE_TAG)-minimal \ - $(foreach tag,$(IMAGE_EXTRA_TAGS),-t $(tag)-minimal) \ - $(IMAGE_BUILD_EXTRA_OPTS) ./ + $(IMAGE_BUILD_CMD) $(IMAGE_BUILD_ARGS) $(IMAGE_BUILD_ARGS_FULL) + $(IMAGE_BUILD_CMD) $(IMAGE_BUILD_ARGS) $(IMAGE_BUILD_ARGS_MINIMAL) + +image-all: ensure-buildx yamls +# --load : not implemented yet, see: https://github.com/docker/buildx/issues/59 + $(IMAGE_BUILDX_CMD) $(IMAGE_BUILD_ARGS) $(IMAGE_BUILD_ARGS_FULL) + $(IMAGE_BUILDX_CMD) $(IMAGE_BUILD_ARGS) $(IMAGE_BUILD_ARGS_MINIMAL) + # clean NFD labels on all nodes # devel only @@ -166,6 +182,10 @@ push: $(IMAGE_PUSH_CMD) $(IMAGE_TAG)-minimal for tag in $(IMAGE_EXTRA_TAGS); do $(IMAGE_PUSH_CMD) $$tag; $(IMAGE_PUSH_CMD) $$tag-minimal; done +push-all: + $(IMAGE_BUILDX_CMD) --push $(IMAGE_BUILD_ARGS) $(IMAGE_BUILD_ARGS_FULL) + $(IMAGE_BUILDX_CMD) --push $(IMAGE_BUILD_ARGS) $(IMAGE_BUILD_ARGS_MINIMAL) + poll-images: set -e; \ tags="$(foreach tag,$(IMAGE_TAG_NAME) $(IMAGE_EXTRA_TAG_NAMES),$(tag) $(tag)-minimal)" \ diff --git a/docs/advanced/developer-guide.md b/docs/advanced/developer-guide.md index 756ef679a8..38b2e2657a 100644 --- a/docs/advanced/developer-guide.md +++ b/docs/advanced/developer-guide.md @@ -43,6 +43,32 @@ Optional, this example with Docker. docker push ``` +### Docker multi-arch builds with buildx + +The default set of architectures enabled for mulit-arch builds are `linux/amd64` +and `linux/arm64`. If more architectures are needed one can override the +`IMAGE_ALL_PLATFORMS` variable with a comma separated list of `OS/ARCH` tuples. + +#### Build the manifest-list with a container image per arch + +```bash +make image-all +``` + +Currently `docker` does not support loading of manifest-lists meaning the images +are not shown when executing `docker images`, see: +[buildx issue #59](https://github.com/docker/buildx/issues/59). + +#### Push the manifest-list with container image per arch + +```bash +make push-all +``` + +The resulting container image can be used in the same way on each arch by pulling +e.g. `node-feature-discovery:v0.10.0` without specifying the architechture. The +manifest-list will take care of providing the right architecture image. + #### Change the job spec to use your custom image (optional) To use your published image from the step above instead of the @@ -88,6 +114,8 @@ makefile overrides. | HOSTMOUNT_PREFIX | Prefix of system directories for feature discovery (local builds) | / (*local builds*) /host- (*container builds*) | IMAGE_BUILD_CMD | Command to build the image | docker build | IMAGE_BUILD_EXTRA_OPTS | Extra options to pass to build command | *empty* +| IMAGE_BUILDX_CMD | Command to build and push multi-arch images with buildx | DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --platform=${IMAGE_ALL_PLATFORMS} --progress=auto --pull +| IMAGE_ALL_PLATFORMS | Comma seperated list of OS/ARCH tuples for mulit-arch builds | linux/amd64,linux/arm64 | IMAGE_PUSH_CMD | Command to push the image to remote registry | docker push | IMAGE_REGISTRY | Container image registry to use | k8s.gcr.io/nfd | IMAGE_TAG_NAME | Container image tag name | <nfd version> diff --git a/hack/init-buildx.sh b/hack/init-buildx.sh new file mode 100755 index 0000000000..ba41d60f5f --- /dev/null +++ b/hack/init-buildx.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Copyright 2022 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit -o nounset -o pipefail + +export DOCKER_CLI_EXPERIMENTAL=enabled + +# We can skip setup if the current builder already has multi-arch +# AND if it isn't the docker driver, which doesn't work +current_builder="$(docker buildx inspect)" +# linux/amd64, linux/arm64, linux/riscv64, linux/ppc64le, linux/s390x, linux/386, linux/arm/v7, linux/arm/v6 +if ! grep -q "^Driver: docker$" <<<"${current_builder}" && \ + grep -q "linux/amd64" <<<"${current_builder}" && \ + grep -q "linux/arm64" <<<"${current_builder}"; then + exit 0 +fi + +# Ensure qemu is in binfmt_misc +# Docker desktop already has these in versions recent enough to have buildx +# We only need to do this setup on linux hosts +if [ "$(uname)" == 'Linux' ]; then + # NOTE: this is pinned to a digest for a reason! + docker run --rm --privileged tonistiigi/binfmt:qemu-v6.1.0@sha256:11128304bc582dc7dbaa35947ff3e52e2610d23cecb410ddfa381a6ce74fa763 --install all +fi + +# Ensure we use a builder that can leverage it (the default on linux will not) +docker buildx rm nfd-builder || true +docker buildx create --use --name=nfd-builder \ + ${http_proxy:+--driver-opt env.http_proxy="$http_proxy"} \ + ${HTTP_PROXY:+--driver-opt env.HTTP_PROXY="$HTTP_PROXY"} \ + ${https_proxy:+--driver-opt env.https_proxy="$https_proxy"} \ + ${HTTPS_PROXY:+--driver-opt env.HTTPS_PROXY="$HTTPS_PROXY"} \ + ${no_proxy:+--driver-opt '"env.no_proxy='$no_proxy'"'} \ + ${NO_PROXY:+--driver-opt '"env.NO_PROXY='$NO_PROXY'"'} diff --git a/scripts/test-infra/build-image.sh b/scripts/test-infra/build-image.sh index 88381a1343..a1d9bf187e 100755 --- a/scripts/test-infra/build-image.sh +++ b/scripts/test-infra/build-image.sh @@ -1,3 +1,7 @@ #!/bin/bash -e +# local build make image + +# cross build +make image-all diff --git a/scripts/test-infra/push-image.sh b/scripts/test-infra/push-image.sh index adefba64a1..9c6e55e700 100755 --- a/scripts/test-infra/push-image.sh +++ b/scripts/test-infra/push-image.sh @@ -5,5 +5,4 @@ # container image tag VERSION_OVERRIDE=${_GIT_TAG+VERSION=${_GIT_TAG:10}} -make image $VERSION_OVERRIDE -make push $VERSION_OVERRIDE +make push-all $VERSION_OVERRIDE