diff --git a/release-tools/SIDECAR_RELEASE_PROCESS.md b/release-tools/SIDECAR_RELEASE_PROCESS.md index 91a1e1b8..6960607e 100644 --- a/release-tools/SIDECAR_RELEASE_PROCESS.md +++ b/release-tools/SIDECAR_RELEASE_PROCESS.md @@ -50,18 +50,22 @@ naming convention `-on-`. ## Release Process 1. Identify all issues and ongoing PRs that should go into the release, and drive them to resolution. -1. Download [K8s release notes +1. Download v2.8+ [K8s release notes generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes) 1. Generate release notes for the release. Replace arguments with the relevant information. - ``` - GITHUB_TOKEN= ./release-notes --start-sha=0ed6978fd199e3ca10326b82b4b8b8e916211c9b --end-sha=3cb3d2f18ed8cb40371c6d8886edcabd1f27e7b9 \ - --github-org=kubernetes-csi --github-repo=external-attacher -branch=master -output out.md - ``` - * `--start-sha` should point to the last release from the same branch. For - example: - * `1.X-1.0` tag when releasing `1.X.0` - * `1.X.Y-1` tag when releasing `1.X.Y` + * For new minor releases on master: + ``` + GITHUB_TOKEN= release-notes --discover=mergebase-to-latest + --github-org=kubernetes-csi --github-repo=external-provisioner + --required-author="" --output out.md + ``` + * For new patch releases on a release branch: + ``` + GITHUB_TOKEN= release-notes --discover=patch-to-latest --branch=release-1.1 + --github-org=kubernetes-csi --github-repo=external-provisioner + --required-author="" --output out.md + ``` 1. Compare the generated output to the new commits for the release to check if any notable change missed a release note. 1. Reword release notes as needed. Make sure to check notes for breaking diff --git a/release-tools/build.make b/release-tools/build.make index a9b9d25d..c17a1c19 100644 --- a/release-tools/build.make +++ b/release-tools/build.make @@ -60,23 +60,30 @@ else TESTARGS = endif -ARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH)) - # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below. -build-%: check-go-version-go +# BUILD_PLATFORMS contains a set of triplets, +# separated by semicolon. An empty variable or empty entry (= just a +# semicolon) builds for the default platform of the current Go +# toolchain. +BUILD_PLATFORMS = + +# This builds each command (= the sub-directories of ./cmd) for the target platform(s) +# defined by BUILD_PLATFORMS. +$(CMDS:%=build-%): build-%: check-go-version-go mkdir -p bin - CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* - if [ "$$ARCH" = "amd64" ]; then \ - CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \ - CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* ; \ - fi + echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \ + if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o "./bin/$*$$suffix" ./cmd/$*); then \ + echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \ + exit 1; \ + fi; \ + done -container-%: build-% +$(CMDS:%=container-%): container-%: build-% docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) . -push-%: container-% +$(CMDS:%=push-%): push-%: container-% set -ex; \ push_image () { \ docker tag $*:latest $(IMAGE_NAME):$$tag; \ @@ -98,6 +105,77 @@ build: $(CMDS:%=build-%) container: $(CMDS:%=container-%) push: $(CMDS:%=push-%) +# Additional parameters are needed when pushing to a local registry, +# see https://github.com/docker/buildx/issues/94. +# However, that then runs into https://github.com/docker/cli/issues/2396. +# +# What works for local testing is: +# make push-multiarch PULL_BASE_REF=master REGISTRY_NAME= BUILD_PLATFORMS="linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x" +DOCKER_BUILDX_CREATE_ARGS ?= + +# This target builds a multiarch image for one command using Moby BuildKit builder toolkit. +# Docker Buildx is included in Docker 19.03. +# +# ./cmd//Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows]. +# It is currently optional: if no such file exists, Windows images are not included, +# even when Windows is listed in BUILD_PLATFORMS. That way, projects can test that +# Windows binaries can be built before adding a Dockerfile for it. +# +# BUILD_PLATFORMS determines which individual images are included in the multiarch image. +# PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines +# the tag for the resulting multiarch image. +$(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% + set -ex; \ + DOCKER_CLI_EXPERIMENTAL=enabled; \ + export DOCKER_CLI_EXPERIMENTAL; \ + docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \ + trap "docker buildx rm multiarchimage-buildertest" EXIT; \ + dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \ + dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ + if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \ + if ! [ -f "$$dockerfile_windows" ]; then \ + build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \ + fi; \ + pushMultiArch () { \ + tag=$$1; \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \ + docker buildx build --push \ + --tag $(IMAGE_NAME):$$arch-$$os-$$tag \ + --platform=$$os/$$arch \ + --file $$(eval echo \$${dockerfile_$$os}) \ + --build-arg binary=./bin/$*$$suffix \ + --label revision=$(REV) \ + .; \ + done; \ + images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \ + docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \ + docker manifest push -p $(IMAGE_NAME):$$tag; \ + }; \ + if [ $(PULL_BASE_REF) = "master" ]; then \ + : "creating or overwriting canary image"; \ + pushMultiArch canary; \ + elif echo $(PULL_BASE_REF) | grep -q -e 'release-*' ; then \ + : "creating or overwriting canary image for release branch"; \ + release_canary_tag=$$(echo $(PULL_BASE_REF) | cut -f2 -d '-')-canary; \ + pushMultiArch $$release_canary_tag; \ + elif docker pull $(IMAGE_NAME):$(PULL_BASE_REF) 2>&1 | tee /dev/stderr | grep -q "manifest for $(IMAGE_NAME):$(PULL_BASE_REF) not found"; then \ + : "creating release image"; \ + pushMultiArch $(PULL_BASE_REF); \ + else \ + : "ERROR: release image $(IMAGE_NAME):$(PULL_BASE_REF) already exists: a new tag is required!"; \ + exit 1; \ + fi + +.PHONY: check-pull-base-ref +check-pull-base-ref: + if ! [ "$(PULL_BASE_REF)" ]; then \ + echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name."; \ + exit 1; \ + fi + +.PHONY: push-multiarch +push-multiarch: $(CMDS:%=push-multiarch-%) + clean: -rm -rf bin @@ -113,7 +191,7 @@ test-go: test: test-vet test-vet: @ echo; echo "### $@:" - go test $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v vendor $(TEST_VET_FILTER_CMD)` + go vet $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v vendor $(TEST_VET_FILTER_CMD)` .PHONY: test-fmt test: test-fmt diff --git a/release-tools/cloudbuild.sh b/release-tools/cloudbuild.sh new file mode 100755 index 00000000..3ba11eca --- /dev/null +++ b/release-tools/cloudbuild.sh @@ -0,0 +1,6 @@ +#! /bin/bash + +# shellcheck disable=SC1091 +. release-tools/prow.sh + +gcr_cloud_build diff --git a/release-tools/cloudbuild.yaml b/release-tools/cloudbuild.yaml new file mode 100644 index 00000000..8f678924 --- /dev/null +++ b/release-tools/cloudbuild.yaml @@ -0,0 +1,46 @@ +# A configuration file for multi-arch image building with the Google cloud build service. +# +# Repos using this file must: +# - import csi-release-tools +# - add a symlink cloudbuild.yaml -> release-tools/cloudbuild.yaml +# - add a .cloudbuild.sh which can be a custom file or a symlink +# to release-tools/cloudbuild.sh +# - accept "binary" as build argument in their Dockerfile(s) (see +# https://github.com/pohly/node-driver-registrar/blob/3018101987b0bb6da2a2657de607174d6e3728f7/Dockerfile#L4-L6) +# because binaries will get built for different architectures and then +# get copied from the built host into the container image +# +# See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md +# for more details on image pushing process in Kubernetes. +# +# To promote release images, see https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage. + +# This must be specified in seconds. If omitted, defaults to 600s (10 mins). +timeout: 1800s +# This prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF, +# or any new substitutions added in the future. +options: + substitution_option: ALLOW_LOOSE +steps: + # The image must contain bash and curl. Ideally it should also contain + # the desired version of Go (currently defined in release-tools/travis.yml), + # but that just speeds up the build and is not required. + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8' + entrypoint: ./.cloudbuild.sh + env: + - GIT_TAG=${_GIT_TAG} + - PULL_BASE_REF=${_PULL_BASE_REF} + - REGISTRY_NAME=gcr.io/${_STAGING_PROJECT} + - HOME=/root +substitutions: + # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and + # can be used as a substitution. + _GIT_TAG: '12345' + # _PULL_BASE_REF will contain the ref that was pushed to trigger this build - + # a branch like 'master' or 'release-0.2', or a tag like 'v0.2'. + _PULL_BASE_REF: 'master' + # The default gcr.io staging project for Kubernetes-CSI + # (=> https://console.cloud.google.com/gcr/images/k8s-staging-sig-storage/GLOBAL). + # Might be overridden in the Prow build job for a repo which wants + # images elsewhere. + _STAGING_PROJECT: 'k8s-staging-sig-storage' diff --git a/release-tools/filter-junit.go b/release-tools/filter-junit.go index 2f51be00..cf1cb4ab 100644 --- a/release-tools/filter-junit.go +++ b/release-tools/filter-junit.go @@ -15,10 +15,10 @@ limitations under the License. */ /* - * This command filters a JUnit file such that only tests with a name - * matching a regular expression are passed through. By concatenating - * multiple input files it is possible to merge them into a single file. - */ +This command filters a JUnit file such that only tests with a name +matching a regular expression are passed through. By concatenating +multiple input files it is possible to merge them into a single file. +*/ package main import ( diff --git a/release-tools/prow.sh b/release-tools/prow.sh index 118c5bd1..9778635f 100755 --- a/release-tools/prow.sh +++ b/release-tools/prow.sh @@ -85,6 +85,8 @@ get_versioned_variable () { echo "$value" } +configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" + # If we have a vendor directory, then use it. We must be careful to only # use this for "make" invocations inside the project's repo itself because # setting it globally can break other go usages (like "go get " @@ -157,7 +159,9 @@ csi_prow_kubernetes_version_suffix="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | tr # the caller. configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory" -# The hostpath deployment script is searched for in several places. +# By default, this script tests sidecars with the CSI hostpath driver, +# using the install_csi_driver function. That function depends on +# a deployment script that it searches for in several places: # # - The "deploy" directory in the current repository: this is useful # for the situation that a component becomes incompatible with the @@ -165,11 +169,11 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # own example until the shared one can be updated; it's also how # csi-driver-host-path itself provides the example. # -# - CSI_PROW_HOSTPATH_VERSION of the CSI_PROW_HOSTPATH_REPO is checked +# - CSI_PROW_DRIVER_VERSION of the CSI_PROW_DRIVER_REPO is checked # out: this allows other repos to reference a version of the example # that is known to be compatible. # -# - The csi-driver-host-path/deploy directory has multiple sub-directories, +# - The /deploy directory can have multiple sub-directories, # each with different deployments (stable set of images for Kubernetes 1.13, # stable set of images for Kubernetes 1.14, canary for latest Kubernetes, etc.). # This is necessary because there may be incompatible changes in the @@ -186,16 +190,26 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # "none" disables the deployment of the hostpath driver. # # When no deploy script is found (nothing in `deploy` directory, -# CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed. -configvar CSI_PROW_HOSTPATH_VERSION "v1.3.0-rc3" "hostpath driver" -configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" +# CSI_PROW_DRIVER_REPO=none), nothing gets deployed. +# +# If the deployment script is called with CSI_PROW_TEST_DRIVER= as +# environment variable, then it must write a suitable test driver configuration +# into that file in addition to installing the driver. +configvar CSI_PROW_DRIVER_VERSION "v1.3.0" "CSI driver version" +configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" -configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name" -# If CSI_PROW_HOSTPATH_CANARY is set (typically to "canary", but also -# "1.0-canary"), then all image versions are replaced with that -# version tag. -configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image" +# The install_csi_driver function may work also for other CSI drivers, +# as long as they follow the conventions of the CSI hostpath driver. +# If they don't, then a different install function can be provided in +# a .prow.sh file and this config variable can be overridden. +configvar CSI_PROW_DRIVER_INSTALL "install_csi_driver" "name of the shell function which installs the CSI driver" + +# If CSI_PROW_DRIVER_CANARY is set (typically to "canary", but also +# version tag. Usually empty. CSI_PROW_HOSTPATH_CANARY is +# accepted as alternative name because some test-infra jobs +# still use that name. +configvar CSI_PROW_DRIVER_CANARY "${CSI_PROW_HOSTPATH_CANARY}" "driver image override for canary images" # The E2E testing can come from an arbitrary repo. The expectation is that # the repo supports "go test ./test/e2e -args --storage.testdriver" (https://github.com/kubernetes/kubernetes/pull/72836) @@ -328,7 +342,7 @@ configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kub configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment -configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0' "external-snapshotter version tag" +configvar CSI_SNAPSHOTTER_VERSION 'v2.0.1' "external-snapshotter version tag" # Some tests are known to be unusable in a KinD cluster. For example, # stopping kubelet with "ssh systemctl stop kubelet" simply @@ -568,6 +582,13 @@ kubeadmConfigPatches: nodeRegistration: kubeletExtraArgs: "feature-gates": "$gates" +- | + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + metadata: + name: config + featureGates: +$(list_gates "$gates") - | apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration @@ -613,7 +634,7 @@ find_deployment () { # Fixed deployment name? Use it if it exists, otherwise fail. if [ "${CSI_PROW_DEPLOYMENT}" ]; then - file="$dir/${CSI_PROW_DEPLOYMENT}/deploy-hostpath.sh" + file="$dir/${CSI_PROW_DEPLOYMENT}/deploy.sh" if ! [ -e "$file" ]; then return 1 fi @@ -623,9 +644,9 @@ find_deployment () { # Ignore: See if you can use ${variable//search/replace} instead. # shellcheck disable=SC2001 - file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy-hostpath.sh" + file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy.sh" if ! [ -e "$file" ]; then - file="$dir/kubernetes-latest/deploy-hostpath.sh" + file="$dir/kubernetes-latest/deploy.sh" if ! [ -e "$file" ]; then return 1 fi @@ -633,12 +654,11 @@ find_deployment () { echo "$file" } -# This installs the hostpath driver example. CSI_PROW_HOSTPATH_CANARY overrides all -# image versions with that canary version. The parameters of install_hostpath can be -# used to override registry and/or tag of individual images (CSI_PROVISIONER_REGISTRY=localhost:9000 -# CSI_PROVISIONER_TAG=latest). -install_hostpath () { - local images deploy_hostpath +# This installs the CSI driver. It's called with a list of env variables +# that override the default images. CSI_PROW_DRIVER_CANARY overrides all +# image versions with that canary version. +install_csi_driver () { + local images deploy_driver images="$*" if [ "${CSI_PROW_DEPLOYMENT}" = "none" ]; then @@ -654,31 +674,31 @@ install_hostpath () { done fi - if deploy_hostpath="$(find_deployment "$(pwd)/deploy")"; then + if deploy_driver="$(find_deployment "$(pwd)/deploy")"; then : - elif [ "${CSI_PROW_HOSTPATH_REPO}" = "none" ]; then + elif [ "${CSI_PROW_DRIVER_REPO}" = "none" ]; then return 1 else - git_checkout "${CSI_PROW_HOSTPATH_REPO}" "${CSI_PROW_WORK}/hostpath" "${CSI_PROW_HOSTPATH_VERSION}" --depth=1 || die "checking out hostpath repo failed" - if deploy_hostpath="$(find_deployment "${CSI_PROW_WORK}/hostpath/deploy")"; then + git_checkout "${CSI_PROW_DRIVER_REPO}" "${CSI_PROW_WORK}/csi-driver" "${CSI_PROW_DRIVER_VERSION}" --depth=1 || die "checking out CSI driver repo failed" + if deploy_driver="$(find_deployment "${CSI_PROW_WORK}/csi-driver/deploy")"; then : else - die "deploy-hostpath.sh not found in ${CSI_PROW_HOSTPATH_REPO} ${CSI_PROW_HOSTPATH_VERSION}. To disable E2E testing, set CSI_PROW_HOSTPATH_REPO=none" + die "deploy.sh not found in ${CSI_PROW_DRIVER_REPO} ${CSI_PROW_DRIVER_VERSION}. To disable E2E testing, set CSI_PROW_DRIVER_REPO=none" fi fi - if [ "${CSI_PROW_HOSTPATH_CANARY}" != "stable" ]; then - images="$images IMAGE_TAG=${CSI_PROW_HOSTPATH_CANARY}" + if [ "${CSI_PROW_DRIVER_CANARY}" != "stable" ]; then + images="$images IMAGE_TAG=${CSI_PROW_DRIVER_CANARY}" fi # Ignore: Double quote to prevent globbing and word splitting. # It's intentional here for $images. # shellcheck disable=SC2086 - if ! run env $images "${deploy_hostpath}"; then + if ! run env "CSI_PROW_TEST_DRIVER=${CSI_PROW_WORK}/test-driver.yaml" $images "${deploy_driver}"; then # Collect information about failed deployment before failing. collect_cluster_info (start_loggers >/dev/null; wait) info "For container output see job artifacts." - die "deploying the hostpath driver with ${deploy_hostpath} failed" + die "deploying the CSI driver with ${deploy_driver} failed" fi } @@ -804,33 +824,6 @@ install_sanity () ( run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go test -c -o "${CSI_PROW_WORK}/csi-sanity" "${CSI_PROW_SANITY_IMPORT_PATH}/cmd/csi-sanity" || die "building csi-sanity failed" ) -# The default implementation of this function generates a external -# driver test configuration for the hostpath driver. -# -# The content depends on both what the E2E suite expects and what the -# installed hostpath driver supports. Generating it here seems prone -# to breakage, but it is uncertain where a better place might be. -generate_test_driver () { - cat <"${CSI_PROW_WORK}/test-driver.yaml" || die "generating test-driver.yaml failed" - # Rename, merge and filter JUnit files. Necessary in case that we run the E2E suite again # and to avoid the large number of "skipped" tests that we get from using # the full Kubernetes E2E testsuite while only running a few tests. @@ -1037,7 +1028,7 @@ main () { images= if ${CSI_PROW_BUILD_JOB}; then # A successful build is required for testing. - run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make all' failed" + run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" "BUILD_PLATFORMS=${CSI_PROW_BUILD_PLATFORMS}" || die "'make all' failed" # We don't want test failures to prevent E2E testing below, because the failure # might have been minor or unavoidable, for example when experimenting with # changes in "release-tools" in a PR (that fails the "is release-tools unmodified" @@ -1063,7 +1054,7 @@ main () { cmds="$(grep '^\s*CMDS\s*=' Makefile | sed -e 's/\s*CMDS\s*=//')" # Get the image that was just built (if any) from the # top-level Makefile CMDS variable and set the - # deploy-hostpath.sh env variables for it. We also need to + # deploy.sh env variables for it. We also need to # side-load those images into the cluster. for i in $cmds; do e=$(echo "$i" | tr '[:lower:]' '[:upper:]' | tr - _) @@ -1073,18 +1064,24 @@ main () { # always pulling the image # (https://github.com/kubernetes-sigs/kind/issues/328). docker tag "$i:latest" "$i:csiprow" || die "tagging the locally built container image for $i failed" - done - if [ -e deploy/kubernetes/rbac.yaml ]; then - # This is one of those components which has its own RBAC rules (like external-provisioner). - # We are testing a locally built image and also want to test with the the current, - # potentially modified RBAC rules. - if [ "$(echo "$cmds" | wc -w)" != 1 ]; then - die "ambiguous deploy/kubernetes/rbac.yaml: need exactly one command, got: $cmds" + # For components with multiple cmds, the RBAC file should be in the following format: + # rbac-$cmd.yaml + # If this file cannot be found, we can default to the standard location: + # deploy/kubernetes/rbac.yaml + rbac_file_path=$(find . -type f -name "rbac-$i.yaml") + if [ "$rbac_file_path" == "" ]; then + rbac_file_path="$(pwd)/deploy/kubernetes/rbac.yaml" fi - e=$(echo "$cmds" | tr '[:lower:]' '[:upper:]' | tr - _) - images="$images ${e}_RBAC=$(pwd)/deploy/kubernetes/rbac.yaml" - fi + + if [ -e "$rbac_file_path" ]; then + # This is one of those components which has its own RBAC rules (like external-provisioner). + # We are testing a locally built image and also want to test with the the current, + # potentially modified RBAC rules. + e=$(echo "$i" | tr '[:lower:]' '[:upper:]' | tr - _) + images="$images ${e}_RBAC=$rbac_file_path" + fi + done fi if tests_need_non_alpha_cluster; then @@ -1101,7 +1098,7 @@ main () { fi # Installing the driver might be disabled. - if install_hostpath "$images"; then + if ${CSI_PROW_DRIVER_INSTALL} "$images"; then collect_cluster_info if sanity_enabled; then @@ -1158,7 +1155,7 @@ main () { fi # Installing the driver might be disabled. - if install_hostpath "$images"; then + if ${CSI_PROW_DRIVER_INSTALL} "$images"; then collect_cluster_info if tests_enabled "parallel-alpha"; then @@ -1192,3 +1189,23 @@ main () { return "$ret" } + +# This function can be called by a repo's top-level cloudbuild.sh: +# it handles environment set up in the GCR cloud build and then +# invokes "make push-multiarch" to do the actual image building. +gcr_cloud_build () { + # Register gcloud as a Docker credential helper. + # Required for "docker buildx build --push". + gcloud auth configure-docker + + if find . -name Dockerfile | grep -v ^./vendor | xargs --no-run-if-empty cat | grep -q ^RUN; then + # Needed for "RUN" steps on non-linux/amd64 platforms. + # See https://github.com/multiarch/qemu-user-static#getting-started + (set -x; docker run --rm --privileged multiarch/qemu-user-static --reset -p yes) + fi + + # Extract tag-n-hash value from GIT_TAG (form vYYYYMMDD-tag-n-hash) for REV value. + REV=v$(echo "$GIT_TAG" | cut -f3- -d 'v') + + run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make push-multiarch REV="${REV}" REGISTRY_NAME="${REGISTRY_NAME}" BUILD_PLATFORMS="${CSI_PROW_BUILD_PLATFORMS}" +}