From a426fe5d05f4852a35e0c7b096369779960a8e47 Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Mon, 13 Dec 2021 11:48:42 +0100 Subject: [PATCH] Added additional Makefile for mulit arch builds --- Makefile | 71 +++++++++++++++---------------- docs/advanced/developer-guide.md | 28 ++++++++++++ hack/init-buildx.sh | 11 +++-- scripts/test-infra/build-image.sh | 4 ++ scripts/test-infra/push-image.sh | 3 +- 5 files changed, 75 insertions(+), 42 deletions(-) diff --git a/Makefile b/Makefile index b9541eec49..9f38abcb3d 100644 --- a/Makefile +++ b/Makefile @@ -3,12 +3,10 @@ GO_CMD ?= go GO_FMT ?= gofmt -# The default build cmd will build multi arch images, use -# IMAGE_BUILD_CMD="docker build" for local/default arch builds -IMAGE_BUILD_CMD ?= docker buildx build --platform=${PLATFORMS} $(OUTPUT) --progress=$(PROGRESS) --pull + +IMAGE_BUILD_CMD ?= docker build IMAGE_BUILD_EXTRA_OPTS ?= IMAGE_PUSH_CMD ?= docker push - CONTAINER_RUN_CMD ?= docker run BASE_IMAGE_FULL ?= debian:buster-slim BASE_IMAGE_MINIMAL ?= gcr.io/distroless/base @@ -61,22 +59,29 @@ E2E_TEST_CONFIG ?= LDFLAGS = -ldflags "-s -w -X sigs.k8s.io/node-feature-discovery/pkg/version.version=$(VERSION) -X sigs.k8s.io/node-feature-discovery/source.pathPrefix=$(HOSTMOUNT_PREFIX)" -## multi-arch build -# required to enable buildx -export DOCKER_CLI_EXPERIMENTAL=enabled - -# build with buildx -PLATFORMS ?= linux/amd64,linux/arm64 -PROGRESS = auto -# buildx keeps the build in the cache no default output -# use --push as OUTPUT to push to registry -# or --load to have it available as an image -OUTPUT= +# multi-arch build with buildx +IMAGE_ALL_PLATFORMS ?= linux/amd64,linux/arm64 # enable buildx ensure-buildx: ./hack/init-buildx.sh +IMAGE_BUILDX_CMD ?= DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --platform=${IMAGE_ALL_PLATFORMS} --progress=auto --pull + +IMAGE_BUILD_ARGS = --build-arg VERSION=$(VERSION) \ + --build-arg HOSTMOUNT_PREFIX=$(CONTAINER_HOSTMOUNT_PREFIX) \ + --build-arg BASE_IMAGE_FULL=$(BASE_IMAGE_FULL) \ + --build-arg BASE_IMAGE_MINIMAL=$(BASE_IMAGE_MINIMAL) + +IMAGE_BUILD_ARGS_FULL = --target full \ + -t $(IMAGE_TAG) \ + $(foreach tag,$(IMAGE_EXTRA_TAGS),-t $(tag)) \ + $(IMAGE_BUILD_EXTRA_OPTS) ./ + +IMAGE_BUILD_ARGS_MINIMAL = --target minimal \ + -t $(IMAGE_TAG)-minimal \ + $(foreach tag,$(IMAGE_EXTRA_TAGS),-t $(tag)-minimal) \ + $(IMAGE_BUILD_EXTRA_OPTS) ./ all: image @@ -87,26 +92,14 @@ build: install: $(GO_CMD) install -v $(LDFLAGS) ./cmd/... -image: yamls ensure-buildx - $(IMAGE_BUILD_CMD) --build-arg VERSION=$(VERSION) \ - --target full \ - --build-arg HOSTMOUNT_PREFIX=$(CONTAINER_HOSTMOUNT_PREFIX) \ - --build-arg BASE_IMAGE_FULL=$(BASE_IMAGE_FULL) \ - --build-arg BASE_IMAGE_MINIMAL=$(BASE_IMAGE_MINIMAL) \ - -t $(IMAGE_TAG) \ - $(foreach tag,$(IMAGE_EXTRA_TAGS),-t $(tag)) \ - $(IMAGE_BUILD_EXTRA_OPTS) ./ - $(IMAGE_BUILD_CMD) --build-arg VERSION=$(VERSION) \ - --target minimal \ - --build-arg HOSTMOUNT_PREFIX=$(CONTAINER_HOSTMOUNT_PREFIX) \ - --build-arg BASE_IMAGE_FULL=$(BASE_IMAGE_FULL) \ - --build-arg BASE_IMAGE_MINIMAL=$(BASE_IMAGE_MINIMAL) \ - -t $(IMAGE_TAG)-minimal \ - $(foreach tag,$(IMAGE_EXTRA_TAGS),-t $(tag)-minimal) \ - $(IMAGE_BUILD_EXTRA_OPTS) ./ - -push: OUTPUT=--push -push: image +image: yamls + $(IMAGE_BUILD_CMD) $(IMAGE_BUILD_ARGS) $(IMAGE_BUILD_ARGS_FULL) + $(IMAGE_BUILD_CMD) $(IMAGE_BUILD_ARGS) $(IMAGE_BUILD_ARGS_MINIMAL) + +image-all: ensure-buildx yamls +# --load : not implemented yet, see: https://github.com/docker/buildx/issues/59 + $(IMAGE_BUILDX_CMD) $(IMAGE_BUILD_ARGS) $(IMAGE_BUILD_ARGS_FULL) + $(IMAGE_BUILDX_CMD) $(IMAGE_BUILD_ARGS) $(IMAGE_BUILD_ARGS_MINIMAL) # clean NFD labels on all nodes @@ -184,11 +177,15 @@ e2e-test: -kubeconfig=$(KUBECONFIG) -nfd.e2e-config=$(E2E_TEST_CONFIG) -ginkgo.focus="\[kubernetes-sigs\]" \ $(if $(OPENSHIFT),-nfd.openshift,) -push-old: - $(IMAGE_PUSH_CMD) $(IMAGE_TAG) . +push: + $(IMAGE_PUSH_CMD) $(IMAGE_TAG) $(IMAGE_PUSH_CMD) $(IMAGE_TAG)-minimal for tag in $(IMAGE_EXTRA_TAGS); do $(IMAGE_PUSH_CMD) $$tag; $(IMAGE_PUSH_CMD) $$tag-minimal; done +push-all: + $(IMAGE_BUILDX_CMD) --push $(IMAGE_BUILD_ARGS) $(IMAGE_BUILD_ARGS_FULL) + $(IMAGE_BUILDX_CMD) --push $(IMAGE_BUILD_ARGS) $(IMAGE_BUILD_ARGS_MINIMAL) + poll-images: set -e; \ tags="$(foreach tag,$(IMAGE_TAG_NAME) $(IMAGE_EXTRA_TAG_NAMES),$(tag) $(tag)-minimal)" \ diff --git a/docs/advanced/developer-guide.md b/docs/advanced/developer-guide.md index 756ef679a8..38b2e2657a 100644 --- a/docs/advanced/developer-guide.md +++ b/docs/advanced/developer-guide.md @@ -43,6 +43,32 @@ Optional, this example with Docker. docker push ``` +### Docker multi-arch builds with buildx + +The default set of architectures enabled for mulit-arch builds are `linux/amd64` +and `linux/arm64`. If more architectures are needed one can override the +`IMAGE_ALL_PLATFORMS` variable with a comma separated list of `OS/ARCH` tuples. + +#### Build the manifest-list with a container image per arch + +```bash +make image-all +``` + +Currently `docker` does not support loading of manifest-lists meaning the images +are not shown when executing `docker images`, see: +[buildx issue #59](https://github.com/docker/buildx/issues/59). + +#### Push the manifest-list with container image per arch + +```bash +make push-all +``` + +The resulting container image can be used in the same way on each arch by pulling +e.g. `node-feature-discovery:v0.10.0` without specifying the architechture. The +manifest-list will take care of providing the right architecture image. + #### Change the job spec to use your custom image (optional) To use your published image from the step above instead of the @@ -88,6 +114,8 @@ makefile overrides. | HOSTMOUNT_PREFIX | Prefix of system directories for feature discovery (local builds) | / (*local builds*) /host- (*container builds*) | IMAGE_BUILD_CMD | Command to build the image | docker build | IMAGE_BUILD_EXTRA_OPTS | Extra options to pass to build command | *empty* +| IMAGE_BUILDX_CMD | Command to build and push multi-arch images with buildx | DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --platform=${IMAGE_ALL_PLATFORMS} --progress=auto --pull +| IMAGE_ALL_PLATFORMS | Comma seperated list of OS/ARCH tuples for mulit-arch builds | linux/amd64,linux/arm64 | IMAGE_PUSH_CMD | Command to push the image to remote registry | docker push | IMAGE_REGISTRY | Container image registry to use | k8s.gcr.io/nfd | IMAGE_TAG_NAME | Container image tag name | <nfd version> diff --git a/hack/init-buildx.sh b/hack/init-buildx.sh index 17fedaf823..ba41d60f5f 100755 --- a/hack/init-buildx.sh +++ b/hack/init-buildx.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright 2020 The Kubernetes Authors. +# Copyright 2022 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -37,5 +37,10 @@ fi # Ensure we use a builder that can leverage it (the default on linux will not) docker buildx rm nfd-builder || true -docker buildx create --use --name=nfd-builder - +docker buildx create --use --name=nfd-builder \ + ${http_proxy:+--driver-opt env.http_proxy="$http_proxy"} \ + ${HTTP_PROXY:+--driver-opt env.HTTP_PROXY="$HTTP_PROXY"} \ + ${https_proxy:+--driver-opt env.https_proxy="$https_proxy"} \ + ${HTTPS_PROXY:+--driver-opt env.HTTPS_PROXY="$HTTPS_PROXY"} \ + ${no_proxy:+--driver-opt '"env.no_proxy='$no_proxy'"'} \ + ${NO_PROXY:+--driver-opt '"env.NO_PROXY='$NO_PROXY'"'} diff --git a/scripts/test-infra/build-image.sh b/scripts/test-infra/build-image.sh index 88381a1343..a1d9bf187e 100755 --- a/scripts/test-infra/build-image.sh +++ b/scripts/test-infra/build-image.sh @@ -1,3 +1,7 @@ #!/bin/bash -e +# local build make image + +# cross build +make image-all diff --git a/scripts/test-infra/push-image.sh b/scripts/test-infra/push-image.sh index adefba64a1..9c6e55e700 100755 --- a/scripts/test-infra/push-image.sh +++ b/scripts/test-infra/push-image.sh @@ -5,5 +5,4 @@ # container image tag VERSION_OVERRIDE=${_GIT_TAG+VERSION=${_GIT_TAG:10}} -make image $VERSION_OVERRIDE -make push $VERSION_OVERRIDE +make push-all $VERSION_OVERRIDE