diff --git a/incubator/hnc/Makefile b/incubator/hnc/Makefile index 39db49889..9e784ac71 100644 --- a/incubator/hnc/Makefile +++ b/incubator/hnc/Makefile @@ -4,34 +4,49 @@ # If CONFIG is `kind`, various defaults will be optimized for deploying locally to Kind CONFIG ?= "default" -PROJECT_ID=$(shell gcloud config get-value project) +# The GCP project ID useful to have when performing operations that require one +# (e.g. release). If you don't have gcloud, all other operations in this +# makefile should still succeed. +# +# We use simple expansion to prevent this from being called every time we need +# the project ID. gcloud prints some noise to stderr, hence the redirect. +PROJECT_ID := ${shell gcloud config get-value project 2>/dev/null} + +# The registry is the location of the image to be built, if any. This is used +# both to create the manifests and to actually build the image. +# +# By default, we use the Container Registry in the current GCP project, but you +# can set it to anything, UNLESS you are calling 'make release' since the image +# needs to be built in the same project as the GCB instance that builds it. +HNC_REGISTRY ?= gcr.io/${PROJECT_ID} -# Set default version tag for krew build and Docker image (unless version is set) +# The image name is the *base* name - excluding the registry and the tag. HNC_IMG_NAME ?= hnc-manager -HNC_IMG_TAG ?= latest -# Image URL to use all building/pushing image targets -ifeq ($(CONFIG),kind) -# The tag is `kind-local` since K8s always attempst to re-pull an image with -# the `latest` tag, and this doesn't work when we're testing locally (we rely -# on the docker-push target, below, to push the image into Kind). -HNC_IMG ?= controller:kind-local +# By default, the image tag is "latest" but you should override this when +# creating a release. If you're using Kind, the tag is 'kind-local' since K8s +# always attempts to re-pull an image the 'latest' tag, and this doesn't work +# when we're testing locally (we rely on the docker-push target, below, to push +# the image into Kind). +ifneq ($(CONFIG),kind) + HNC_IMG_TAG ?= latest else -HNC_IMG ?= "gcr.io/${PROJECT_ID}/${HNC_IMG_NAME}:${HNC_IMG_TAG}" -# We don't want to overwrite an existing docker image and tag so we query the repository -# to see if the image already exists. This can be overridden by setting FORCE_RELEASE=true -ifeq ($(FORCE_RELEASE), true) - IMG_READ_ERROR=1 -else - IMG_READ_ERROR=$(shell gcloud container images describe $(HNC_IMG) > /dev/null 2>&1; echo $$?) -endif + HNC_IMG_TAG ?= kind-local endif -# Override this to release from a different repo -HNC_REPO_USER ?= "kubernetes-sigs" -HNC_RELEASED_IMG ?= "gcr.io/k8s-staging-multitenancy/${HNC_IMG_NAME}:${HNC_IMG_TAG}" +# HNC_IMG is the full image name. It can't be overridden in its entirety since +# parts of the Makefile and Cloud Build target make certain assumptions about +# its components. +# +# Note that if you're using Kind, this image will never actually be pushed to +# the registry. +HNC_IMG = ${HNC_REGISTRY}/${HNC_IMG_NAME}:${HNC_IMG_TAG} -CONTROLLER_GEN ?= "./bin/controller-gen" +# `make` must be called from the HNC root, or all kinds of things will break +# (starting with this). +CURDIR = $(shell pwd) +KUSTOMIZE ?= ${CURDIR}/hack/kustomize-3.8.1 +CONTROLLER_GEN ?= ${CURDIR}/bin/controller-gen # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true" @@ -108,15 +123,15 @@ manifests: controller-gen mkdir manifests cd manifests && \ touch kustomization.yaml && \ - kustomize edit add resource ../config/default && \ - kustomize edit set image controller=${HNC_IMG} - kustomize build manifests/ -o manifests/${HNC_IMG_NAME}.yaml + ${KUSTOMIZE} edit add resource ../config/default && \ + ${KUSTOMIZE} edit set image controller=${HNC_IMG} + ${KUSTOMIZE} build manifests/ -o manifests/${HNC_IMG_NAME}.yaml @echo "Building CRD-only manifest" rm manifests/kustomization.yaml cd manifests && \ touch kustomization.yaml && \ - kustomize edit add resource ../config/crd - kustomize build manifests/ -o manifests/hnc-crds.yaml + ${KUSTOMIZE} edit add resource ../config/crd + ${KUSTOMIZE} build manifests/ -o manifests/hnc-crds.yaml # Run go fmt against code fmt: @@ -239,27 +254,70 @@ test-conversion: docker-push kubectl manifests ###################### RELEASE ACTIONS ######################### # Build the container image by Cloud Build and build YAMLs locally -# -# ALL COMMANDS THAT INCLUDE THE PERSONAL ACCESS TOKEN ARE HIDDEN -release: check-release-env test +HNC_RELEASE_REGISTRY ?= gcr.io/k8s-staging-multitenancy +HNC_RELEASE_IMG = ${HNC_RELEASE_REGISTRY}/${HNC_IMG_NAME}:${HNC_IMG_TAG} +# Override this to release from a different Git repo +HNC_RELEASE_REPO_OWNER ?= kubernetes-sigs + +HNC_GCB_SUBS := _HNC_REGISTRY=${HNC_RELEASE_REGISTRY} +HNC_GCB_SUBS := ${HNC_GCB_SUBS},_HNC_IMG_NAME=${HNC_IMG_NAME} +HNC_GCB_SUBS := ${HNC_GCB_SUBS},_HNC_IMG_TAG=${HNC_IMG_TAG} +HNC_GCB_SUBS := ${HNC_GCB_SUBS},_HNC_USER=${HNC_USER} +HNC_GCB_SUBS := ${HNC_GCB_SUBS},_HNC_PERSONAL_ACCESS_TOKEN=${HNC_PAT} +HNC_GCB_SUBS := ${HNC_GCB_SUBS},_HNC_RELEASE_ID=${HNC_RELEASE_ID} +HNC_GCB_SUBS := ${HNC_GCB_SUBS},_HNC_REPO_OWNER=${HNC_RELEASE_REPO_OWNER} +release: check-release-env + @echo "*********************************************" + @echo "*********************************************" + @echo "Releasing ${HNC_RELEASE_IMG}" + @echo "GCP project: ${PROJECT_ID}" + @echo "Temporary build image (must be in same project): ${HNC_IMG}" + @echo "Any existing images with the same tag will be overwritten!" + @echo "" +ifeq (${HNC_FORCE_RELEASE}, true) + @echo "HNC_FORCE_RELEASE IS ENABLED. YOU WILL PROBABLY BE OVERWRITING" + @echo "AN EXISTING RELEASE. ARE YOU REALLY SURE ABOUT THIS????" + @sleep 1 + @echo "" +endif + @echo "YOU HAVE FIVE SECONDS TO CANCEL" + @echo "*********************************************" + @echo "*********************************************" + @sleep 5 + @echo + @echo "*********************************************" + @echo "*********************************************" + @echo "Starting build." @echo "*********************************************" @echo "*********************************************" - @echo "Invoking Cloud Build" + gcloud builds submit --config cloudbuild.yaml --no-source --substitutions=${HNC_GCB_SUBS} + @echo "*********************************************" + @echo "*********************************************" + @echo "Pushing ${HNC_IMG} to ${HNC_RELEASE_IMG}" @echo "*********************************************" @echo "*********************************************" - @gcloud builds submit --config cloudbuild.yaml --no-source --substitutions=_HNC_REPO_USER=${HNC_REPO_USER},_HNC_IMG_NAME=${HNC_IMG_NAME},_HNC_IMG_TAG=${HNC_IMG_TAG},_HNC_USER=${HNC_USER},_HNC_PERSONAL_ACCESS_TOKEN=${HNC_PAT},_HNC_RELEASE_ID=${HNC_RELEASE_ID},_HNC_RELEASED_IMG=${HNC_RELEASED_IMG} - @echo Pushing image to K8s staging @docker pull ${HNC_IMG} - @docker tag ${HNC_IMG} ${HNC_RELEASED_IMG} - @docker push ${HNC_RELEASED_IMG} + @docker tag ${HNC_IMG} ${HNC_RELEASE_IMG} + @docker push ${HNC_RELEASE_IMG} +# Set up error checking variables ERR_MSG=Ensure that HNC_IMG_TAG (eg v0.1.0), HNC_USER (your Github username), HNC_PAT (Github personal access token) and HNC_RELEASE_ID (Github numeric ID) are set +# During a release, We don't want to overwrite an existing docker image and tag +# so we query the repository to see if the image already exists. This can be +# overridden by setting HNC_FORCE_RELEASE=true +ifeq ($(HNC_FORCE_RELEASE), true) + COULDNT_READ_RELEASE_IMG=1 +else + COULDNT_READ_RELEASE_IMG=$(shell gcloud container images describe $(HNC_RELEASE_IMG) > /dev/null 2>&1; echo $$?) +endif + +# Actual error checking: check-release-env: ifndef HNC_IMG_TAG $(error HNC_IMG_TAG is undefined; ${ERR_MSG}) endif ifndef HNC_USER - $(error HNC_USER is undefinied; ${ERR_MSG}) + $(error HNC_USER is undefined; ${ERR_MSG}) endif ifndef HNC_PAT $(error HNC_PAT is undefined; ${ERR_MSG}) @@ -267,11 +325,6 @@ endif ifndef HNC_RELEASE_ID $(error HNC_RELEASE_ID is undefined; ${ERR_MSG}) endif -ifeq ($(IMG_READ_ERROR), 0) - $(error The image ${HNC_IMG} already exists. Force and overwrite this image by using FORCE_RELEASE=true) +ifeq ($(COULDNT_READ_RELEASE_IMG), 0) + $(error The image ${HNC_RELEASE_IMG} already exists. Force and overwrite this image by using HNC_FORCE_RELEASE=true) endif - @echo "*********************************************" - @echo "*********************************************" - @echo "Releasing ${HNC_RELEASED_IMG} using ${PROJECT_ID}" - @echo "*********************************************" - @echo "*********************************************" diff --git a/incubator/hnc/cloudbuild.yaml b/incubator/hnc/cloudbuild.yaml index cc6302d19..19d852fc9 100644 --- a/incubator/hnc/cloudbuild.yaml +++ b/incubator/hnc/cloudbuild.yaml @@ -6,31 +6,25 @@ steps: args: - '-c' - | - git clone https://github.com/$_HNC_REPO_USER/multi-tenancy + set -e + echo Cloning from https://github.com/$_HNC_REPO_OWNER/multi-tenancy + git clone https://github.com/$_HNC_REPO_OWNER/multi-tenancy cd multi-tenancy + echo Checking out hnc-$_HNC_IMG_TAG git checkout hnc-$_HNC_IMG_TAG # Build the manifests and the kubectl plugin -- name: mirror.gcr.io/library/golang +- name: golang:1.14 entrypoint: 'bash' args: - '-c' - | - # Get kustomize - export PATH=$$(go env GOPATH)/bin:$$PATH - mkdir -p $$(go env GOPATH)/bin - GO111MODULE=on go get sigs.k8s.io/kustomize/kustomize/v3@v3.5.4 - - # Build manifests + set -e cd multi-tenancy/incubator/hnc - mkdir out - cd out - touch kustomization.yaml - kustomize edit add resource ../config/default - kustomize edit set image controller=$_HNC_RELEASED_IMG - kustomize build . -o ./hnc-manager.yaml - - # Build plugin - go build -o kubectl-hns ../cmd/kubectl/main.go + export HNC_REGISTRY=$_HNC_REGISTRY + export HNC_IMG_NAME=$_HNC_IMG_NAME + export HNC_IMG_TAG=$_HNC_IMG_TAG + # This makes manifests as a side effect + make kubectl # Upload manifest - name: gcr.io/cloud-builders/curl args: @@ -39,7 +33,7 @@ steps: - '-H' - 'Content-Type: application/x-application' - '--data-binary' - - '@multi-tenancy/incubator/hnc/out/hnc-manager.yaml' + - '@multi-tenancy/incubator/hnc/manifests/hnc-manager.yaml' - '-u' - '$_HNC_USER:$_HNC_PERSONAL_ACCESS_TOKEN' - 'https://uploads.github.com/repos/kubernetes-sigs/multi-tenancy/releases/$_HNC_RELEASE_ID/assets?name=hnc-manager.yaml' @@ -51,7 +45,7 @@ steps: - '-H' - 'Content-Type: application/x-application' - '--data-binary' - - '@multi-tenancy/incubator/hnc/out/kubectl-hns' + - '@multi-tenancy/incubator/hnc/bin/kubectl/kubectl-hns' - '-u' - '$_HNC_USER:$_HNC_PERSONAL_ACCESS_TOKEN' - 'https://uploads.github.com/repos/kubernetes-sigs/multi-tenancy/releases/$_HNC_RELEASE_ID/assets?name=kubectl-hns' diff --git a/incubator/hnc/docs/releasing.md b/incubator/hnc/docs/releasing.md index 3b69ea2c3..24ebe215b 100644 --- a/incubator/hnc/docs/releasing.md +++ b/incubator/hnc/docs/releasing.md @@ -25,11 +25,14 @@ You must have permission to write to this repo, and create a [personal access token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) that includes that permission. -You also need the ability to push to `gcr.io/k8s-staging-multitenancy`. +You also need the ability to push to `gcr.io/k8s-staging-multitenancy`. You can +get this by joining the k8s-infra-staging-multitenancy@kubernetes.io Google +Group, which also gives you access to the `k8s-staging-multitenancy` GCP project +(this is a standalone project and isn't in a GCP Organization). Finally, you must have a GCP project with Cloud Build enabled, and `gcloud` must be configured with this as your default project. _TODO: create a central project -that anyone can use._ +that anyone can use, but without leaking personal access tokens._ ## Document new/changed features diff --git a/incubator/hnc/hack/README.md b/incubator/hnc/hack/README.md new file mode 100644 index 000000000..d5c0f506a --- /dev/null +++ b/incubator/hnc/hack/README.md @@ -0,0 +1,31 @@ +# Hacks + +Gotta love this directory name, eh? Let's try to cover what's in here. + +## HNC Build tools + +Whenever possible, we've tried to include build tools like controller-gen in the +/vendors directory. In order to to force the Go tools _not_ to remove these +runtime dependencies from the go.mod file, this directory contains a fake +`tools.go` whose only purpose is to import these packages. Before you ask any +more questions about that, check out +https://stackoverflow.com/questions/52428230/how-do-go-modules-work-with-installable-commands +and hopefully it will answer them. + +However, I wasn't able to get kustomize to fit in there, since it seems to have +dependencies which are not compatible with the rest of HNC. So I've just checked +in the Linux binary directly here. The exact version probably doesn't matter too +much; I just used whatever was most current and it worked. + +## Templates + +`boilerplate.go.txt` includes the Apache header. Kubebuilder put it here and it +seems like as good a place as any. + +`krew-hierarchical-namespaces.yaml` is a template for the Krew `kubectl-hns` +plugin. + +## CI + +Other projects seem to put their presubmits, postsubmits etc here, so we did +too. See ../README.md for where these are configured in Prow. diff --git a/incubator/hnc/hack/ci-test.sh b/incubator/hnc/hack/ci-test.sh index ba9bb7f4e..cf0d5aacc 100755 --- a/incubator/hnc/hack/ci-test.sh +++ b/incubator/hnc/hack/ci-test.sh @@ -27,15 +27,8 @@ set -e export PATH=$(go env GOPATH)/bin:$PATH mkdir -p $(go env GOPATH)/bin -echo "Installing 'kubebuilder'" -wget https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.0.0-alpha.1/kubebuilder_2.0.0-alpha.1_linux_amd64.tar.gz -tar -zxvf kubebuilder_2.0.0-alpha.1_linux_amd64.tar.gz -mv kubebuilder_2.0.0-alpha.1_linux_amd64 /usr/local/kubebuilder - -echo "Installing 'kustomize'" -GO111MODULE=on go get sigs.k8s.io/kustomize/kustomize/v3@v3.5.4 - hack_dir=$(dirname ${BASH_SOURCE}) echo "Running 'make tests'" -make test -C "$hack_dir/.." +cd "$hack_dir/.." +make test diff --git a/incubator/hnc/hack/kustomize-3.8.1 b/incubator/hnc/hack/kustomize-3.8.1 new file mode 100755 index 000000000..0f364da29 Binary files /dev/null and b/incubator/hnc/hack/kustomize-3.8.1 differ