Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add parameter base_image and addon_image to BUILD_PLATFORMS #150

Merged
merged 1 commit into from
May 17, 2021
Merged

Add parameter base_image and addon_image to BUILD_PLATFORMS #150

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 38 additions & 11 deletions build.make
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# force the usage of /bin/bash instead of /bin/sh
SHELL := /bin/bash

.PHONY: build-% build container-% container push-% push clean test

# A space-separated list of all commands in the repository, must be
Expand Down Expand Up @@ -63,26 +66,35 @@ endif
# Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables
# to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below.

# BUILD_PLATFORMS contains a set of <os> <arch> <suffix> triplets,
# BUILD_PLATFORMS contains a set of tuples [os arch suffix base_image addon_image]
# separated by semicolon. An empty variable or empty entry (= just a
# semicolon) builds for the default platform of the current Go
# toolchain.
BUILD_PLATFORMS =

# Add go ldflags using LDFLAGS at the time of compilation.
IMPORTPATH_LDFLAGS = -X main.version=$(REV)
IMPORTPATH_LDFLAGS = -X main.version=$(REV)
EXT_LDFLAGS = -extldflags "-static"
LDFLAGS =
LDFLAGS =
FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS)
# This builds each command (= the sub-directories of ./cmd) for the target platform(s)
# defined by BUILD_PLATFORMS.
$(CMDS:%=build-%): build-%: check-go-version-go
mkdir -p bin
echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \
# os_arch_seen captures all of the $$os-$$arch seen for the current binary
# that we want to build, if we've seen an $$os-$$arch before it means that
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could you give me an example about this logic?

Copy link
Member Author

@mauriciopoppe mauriciopoppe May 14, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, so if you're iterating over this array of tuples: linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2

go build will be called like this:

+ CGO_ENABLED=0
+ GOOS=linux
+ GOARCH=amd64
+ go build -a -ldflags ' -X main.version=v12345 -extldflags "-static"' -o ./bin/csi-node-driver-registrar ./cmd/csi-node-driver-registrar
+ CGO_ENABLED=0
+ GOOS=linux
+ GOARCH=ppc64le
+ go build -a -ldflags ' -X main.version=v12345 -extldflags "-static"' -o ./bin/csi-node-driver-registrar-ppc64le ./cmd/csi-node-driver-registrar
+ CGO_ENABLED=0
+ GOOS=linux
+ GOARCH=s390x
+ go build -a -ldflags ' -X main.version=v12345 -extldflags "-static"' -o ./bin/csi-node-driver-registrar-s390x ./cmd/csi-node-driver-registrar
+ CGO_ENABLED=0
+ GOOS=linux
+ GOARCH=arm64
+ go build -a -ldflags ' -X main.version=v12345 -extldflags "-static"' -o ./bin/csi-node-driver-registrar-arm64 ./cmd/csi-node-driver-registrar
+ CGO_ENABLED=0
+ GOOS=windows
+ GOARCH=amd64
+ go build -a -ldflags ' -X main.version=v12345 -extldflags "-static"' -o ./bin/csi-node-driver-registrar.exe ./cmd/csi-node-driver-registrar
+ CGO_ENABLED=0
+ GOOS=windows
+ GOARCH=amd64
+ go build -a -ldflags ' -X main.version=v12345 -extldflags "-static"' -o ./bin/csi-node-driver-registrar.exe ./cmd/csi-node-driver-registrar
+ CGO_ENABLED=0
+ GOOS=windows
+ GOARCH=amd64
+ go build -a -ldflags ' -X main.version=v12345 -extldflags "-static"' -o ./bin/csi-node-driver-registrar.exe ./cmd/csi-node-driver-registrar
+ CGO_ENABLED=0
+ GOOS=windows
+ GOARCH=amd64
+ go build -a -ldflags ' -X main.version=v12345 -extldflags "-static"' -o ./bin/csi-node-driver-registrar.exe ./cmd/csi-node-driver-registrar

As you can see the windows binary is built 4 times for the same os,arch combination, with this code I'm making sure that we only build it once per os,arch combination

+ CGO_ENABLED=0
+ GOOS=linux
+ GOARCH=amd64
+ go build -a -ldflags ' -X main.version=v12345 -extldflags "-static"' -o ./bin/csi-node-driver-registrar ./cmd/csi-node-driver-registrar
+ CGO_ENABLED=0
+ GOOS=linux
+ GOARCH=ppc64le
+ go build -a -ldflags ' -X main.version=v12345 -extldflags "-static"' -o ./bin/csi-node-driver-registrar-ppc64le ./cmd/csi-node-driver-registrar
+ CGO_ENABLED=0
+ GOOS=linux
+ GOARCH=s390x
+ go build -a -ldflags ' -X main.version=v12345 -extldflags "-static"' -o ./bin/csi-node-driver-registrar-s390x ./cmd/csi-node-driver-registrar
+ CGO_ENABLED=0
+ GOOS=linux
+ GOARCH=arm64
+ go build -a -ldflags ' -X main.version=v12345 -extldflags "-static"' -o ./bin/csi-node-driver-registrar-arm64 ./cmd/csi-node-driver-registrar
+ CGO_ENABLED=0
+ GOOS=windows
+ GOARCH=amd64
+ go build -a -ldflags ' -X main.version=v12345 -extldflags "-static"' -o ./bin/csi-node-driver-registrar.exe ./cmd/csi-node-driver-registrar

# we don't need to build it again, this is done to avoid building
# the windows binary multiple times (see the default value of $$BUILD_PLATFORMS)
export os_arch_seen="" && echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
os_arch_seen_pre=$${os_arch_seen%%$$os-$$arch*}; \
if ! [ $${#os_arch_seen_pre} = $${#os_arch_seen} ]; then \
continue; \
fi; \
if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \
echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \
exit 1; \
fi; \
os_arch_seen+=";$$os-$$arch"; \
done

$(CMDS:%=container-%): container-%: build-%
Expand Down Expand Up @@ -131,30 +143,46 @@ DOCKER_BUILDX_CREATE_ARGS ?=
# the tag for the resulting multiarch image.
$(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
set -ex; \
DOCKER_CLI_EXPERIMENTAL=enabled; \
export DOCKER_CLI_EXPERIMENTAL; \
export DOCKER_CLI_EXPERIMENTAL=enabled; \
docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \
trap "docker buildx rm multiarchimage-buildertest" EXIT; \
dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \
dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \
if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \
if ! [ -f "$$dockerfile_windows" ]; then \
build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \
build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$//')"; \
fi; \
pushMultiArch () { \
tag=$$1; \
echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \
echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
escaped_base_image=$${base_image/:/-}; \
if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
docker buildx build --push \
--tag $(IMAGE_NAME):$$arch-$$os-$$tag \
--tag $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag \
--platform=$$os/$$arch \
--file $$(eval echo \$${dockerfile_$$os}) \
--build-arg binary=./bin/$*$$suffix \
--build-arg ARCH=$$arch \
--build-arg BASE_IMAGE=$$base_image \
--build-arg ADDON_IMAGE=$$addon_image \
--label revision=$(REV) \
.; \
done; \
images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \
images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
escaped_base_image=$${base_image/:/-}; \
if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
echo $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \
done); \
docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \
echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
if [ $$os = "windows" ]; then \
escaped_base_image=$${base_image/:/-}; \
if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
image=$(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \
os_version=$$(docker manifest inspect mcr.microsoft.com/windows/$${base_image} | grep "os.version" | head -n 1 | awk '{print $$2}' | sed -e 's/"//g') || true; \
docker manifest annotate --os-version $$os_version $(IMAGE_NAME):$$tag $$image; \
fi; \
done; \
docker manifest push -p $(IMAGE_NAME):$$tag; \
}; \
if [ $(PULL_BASE_REF) = "master" ]; then \
Expand Down Expand Up @@ -288,4 +316,3 @@ test-spelling:
test-boilerplate:
@ echo; echo "### $@:"
@ ./release-tools/verify-boilerplate.sh "$(pwd)"

2 changes: 1 addition & 1 deletion cloudbuild.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ steps:
# The image must contain bash and curl. Ideally it should also contain
# the desired version of Go (currently defined in release-tools/prow.sh),
# but that just speeds up the build and is not required.
- name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8'
- name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20210331-c732583'
entrypoint: ./.cloudbuild.sh
env:
- GIT_TAG=${_GIT_TAG}
Expand Down
17 changes: 10 additions & 7 deletions prow.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,10 @@ version_to_git () {
esac
}

configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
# the list of windows versions was matched from:
# - https://hub.docker.com/_/microsoft-windows-nanoserver
# - https://hub.docker.com/_/microsoft-windows-servercore
configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"

# If we have a vendor directory, then use it. We must be careful to only
# use this for "make" invocations inside the project's repo itself because
Expand Down Expand Up @@ -723,7 +726,7 @@ install_csi_driver () {
fi
}

# Installs all nessesary snapshotter CRDs
# Installs all nessesary snapshotter CRDs
install_snapshot_crds() {
# Wait until volumesnapshot CRDs are in place.
CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd"
Expand Down Expand Up @@ -770,7 +773,7 @@ install_snapshot_controller() {
fi
echo "$(date +%H:%M:%S)" "waiting for snapshot RBAC setup complete, attempt #$cnt"
cnt=$((cnt + 1))
sleep 10
sleep 10
done

SNAPSHOT_CONTROLLER_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml"
Expand Down Expand Up @@ -839,7 +842,7 @@ install_snapshot_controller() {
fi
echo "$(date +%H:%M:%S)" "waiting for snapshot controller deployment to complete, attempt #$cnt"
cnt=$((cnt + 1))
sleep 10
sleep 10
done
}

Expand Down Expand Up @@ -979,7 +982,7 @@ run_sanity () (
kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- mkdir "\$@" && echo "\$@"
EOF
# Using "rm -rf" as fallback for "rmdir" is a workaround for:
# Node Service
# Node Service
# should work
# /nvme/gopath.tmp/src/github.com/kubernetes-csi/csi-test/pkg/sanity/node.go:624
# STEP: reusing connection to CSI driver at dns:///172.17.0.2:30896
Expand Down Expand Up @@ -1143,7 +1146,7 @@ make_test_to_junit () {
# version_gt 1.3.1 v1.2.0 (returns true)
# version_gt 1.1.1 release-1.2.0 (returns false)
# version_gt 1.2.0 1.2.2 (returns false)
function version_gt() {
function version_gt() {
versions=$(for ver in "$@"; do ver=${ver#release-}; ver=${ver#kubernetes-}; echo "${ver#v}"; done)
greaterVersion=${1#"release-"};
greaterVersion=${greaterVersion#"kubernetes-"};
Expand Down Expand Up @@ -1206,7 +1209,7 @@ main () {
if [ "$rbac_file_path" == "" ]; then
rbac_file_path="$(pwd)/deploy/kubernetes/rbac.yaml"
fi

if [ -e "$rbac_file_path" ]; then
# This is one of those components which has its own RBAC rules (like external-provisioner).
# We are testing a locally built image and also want to test with the the current,
Expand Down