Hermetic, unified builds

This commit makes two changes:

* Includes the kustomize tool so that we don't need to either fetch it
  from the internet or rely on one being installed locally.
* Replaces portions of the Cloud Build process with the equivalent steps
  from the Makefile so they can't get out of sync.

As a side effect, this required a bit of refactoring in the makefile,
and I've moved some lines around to be more closely located with the
relevant recipes.

Finally, this includes a small docs change that was mistakenly left out
of the last PR.

Tested: made all targets locally; pushed a test tag to my repo and used
the HNC_RELEASE_REPO_OWNER flag to build a fake release from that tag.

incubator/hnc/Makefile

@@ -4,34 +4,49 @@
 # If CONFIG is `kind`, various defaults will be optimized for deploying locally to Kind
 CONFIG ?= "default"
 
-PROJECT_ID=$(shell gcloud config get-value project)
+# The GCP project ID useful to have when performing operations that require one
+# (e.g. release). If you don't have gcloud, all other operations in this
+# makefile should still succeed.
+#
+# We use simple expansion to prevent this from being called every time we need
+# the project ID. gcloud prints some noise to stderr, hence the redirect.
+PROJECT_ID := ${shell gcloud config get-value project 2>/dev/null}
+
+# The registry is the location of the image to be built, if any. This is used
+# both to create the manifests and to actually build the image.
+#
+# By default, we use the Container Registry in the current GCP project, but you
+# can set it to anything, UNLESS you are calling 'make release' since the image
+# needs to be built in the same project as the GCB instance that builds it.
+HNC_REGISTRY ?= gcr.io/${PROJECT_ID}
 
-# Set default version tag for krew build and Docker image (unless version is set)
+# The image name is the *base* name - excluding the registry and the tag.
 HNC_IMG_NAME ?= hnc-manager
-HNC_IMG_TAG ?= latest
 
-# Image URL to use all building/pushing image targets
-ifeq ($(CONFIG),kind)
-# The tag is `kind-local` since K8s always attempst to re-pull an image with
-# the `latest` tag, and this doesn't work when we're testing locally (we rely
-# on the docker-push target, below, to push the image into Kind).
-HNC_IMG ?= controller:kind-local
+# By default, the image tag is "latest" but you should override this when
+# creating a release. If you're using Kind, the tag is 'kind-local' since K8s
+# always attempts to re-pull an image the 'latest' tag, and this doesn't work
+# when we're testing locally (we rely on the docker-push target, below, to push
+# the image into Kind).
+ifneq ($(CONFIG),kind)
+  HNC_IMG_TAG ?= latest
 else
-HNC_IMG ?= "gcr.io/${PROJECT_ID}/${HNC_IMG_NAME}:${HNC_IMG_TAG}"
-# We don't want to overwrite an existing docker image and tag so we query the repository
-# to see if the image already exists. This can be overridden by setting FORCE_RELEASE=true
-ifeq ($(FORCE_RELEASE), true)
-	IMG_READ_ERROR=1
-else
-	IMG_READ_ERROR=$(shell gcloud container images describe $(HNC_IMG) > /dev/null 2>&1; echo $$?)
-endif
+  HNC_IMG_TAG ?= kind-local
 endif
 
-# Override this to release from a different repo
-HNC_REPO_USER ?= "kubernetes-sigs"
-HNC_RELEASED_IMG ?= "gcr.io/k8s-staging-multitenancy/${HNC_IMG_NAME}:${HNC_IMG_TAG}"
+# HNC_IMG is the full image name. It can't be overridden in its entirety since
+# parts of the Makefile and Cloud Build target make certain assumptions about
+# its components.
+#
+# Note that if you're using Kind, this image will never actually be pushed to
+# the registry.
+HNC_IMG = ${HNC_REGISTRY}/${HNC_IMG_NAME}:${HNC_IMG_TAG}
 
-CONTROLLER_GEN ?= "./bin/controller-gen"
+# `make` must be called from the HNC root, or all kinds of things will break
+# (starting with this).
+CURDIR = $(shell pwd)
+KUSTOMIZE ?= ${CURDIR}/hack/kustomize-3.8.1
+CONTROLLER_GEN ?= ${CURDIR}/bin/controller-gen
 
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true"
@@ -108,15 +123,15 @@ manifests: controller-gen
 	mkdir manifests
 	cd manifests && \
 		touch kustomization.yaml && \
-		kustomize edit add resource ../config/default && \
-		kustomize edit set image controller=${HNC_IMG}
-	kustomize build manifests/ -o manifests/${HNC_IMG_NAME}.yaml
+		${KUSTOMIZE} edit add resource ../config/default && \
+		${KUSTOMIZE} edit set image controller=${HNC_IMG}
+	${KUSTOMIZE} build manifests/ -o manifests/${HNC_IMG_NAME}.yaml
 	@echo "Building CRD-only manifest"
 	rm manifests/kustomization.yaml
 	cd manifests && \
 		touch kustomization.yaml && \
-		kustomize edit add resource ../config/crd
-	kustomize build manifests/ -o manifests/hnc-crds.yaml
+		${KUSTOMIZE} edit add resource ../config/crd
+	${KUSTOMIZE} build manifests/ -o manifests/hnc-crds.yaml
 
 # Run go fmt against code
 fmt:
@@ -239,39 +254,77 @@ test-conversion: docker-push kubectl manifests
 
 ###################### RELEASE ACTIONS #########################
 # Build the container image by Cloud Build and build YAMLs locally
-#
-# ALL COMMANDS THAT INCLUDE THE PERSONAL ACCESS TOKEN ARE HIDDEN
-release: check-release-env test
+HNC_RELEASE_REGISTRY ?= gcr.io/k8s-staging-multitenancy
+HNC_RELEASE_IMG = ${HNC_RELEASE_REGISTRY}/${HNC_IMG_NAME}:${HNC_IMG_TAG}
+# Override this to release from a different Git repo
+HNC_RELEASE_REPO_OWNER ?= kubernetes-sigs
+
+HNC_GCB_SUBS := _HNC_REGISTRY=${HNC_RELEASE_REGISTRY}
+HNC_GCB_SUBS := ${HNC_GCB_SUBS},_HNC_IMG_NAME=${HNC_IMG_NAME}
+HNC_GCB_SUBS := ${HNC_GCB_SUBS},_HNC_IMG_TAG=${HNC_IMG_TAG}
+HNC_GCB_SUBS := ${HNC_GCB_SUBS},_HNC_USER=${HNC_USER}
+HNC_GCB_SUBS := ${HNC_GCB_SUBS},_HNC_PERSONAL_ACCESS_TOKEN=${HNC_PAT}
+HNC_GCB_SUBS := ${HNC_GCB_SUBS},_HNC_RELEASE_ID=${HNC_RELEASE_ID}
+HNC_GCB_SUBS := ${HNC_GCB_SUBS},_HNC_REPO_OWNER=${HNC_RELEASE_REPO_OWNER}
+release: check-release-env
+	@echo "*********************************************"
+	@echo "*********************************************"
+	@echo "Releasing ${HNC_RELEASE_IMG}"
+	@echo "GCP project: ${PROJECT_ID}"
+	@echo "Temporary build image (must be in same project): ${HNC_IMG}"
+	@echo "Any existing images with the same tag will be overwritten!"
+	@echo ""
+ifeq (${HNC_FORCE_RELEASE}, true)
+	@echo "HNC_FORCE_RELEASE IS ENABLED. YOU WILL PROBABLY BE OVERWRITING"
+	@echo "AN EXISTING RELEASE. ARE YOU REALLY SURE ABOUT THIS????"
+	@sleep 1
+	@echo ""
+endif
+	@echo "YOU HAVE FIVE SECONDS TO CANCEL"
+	@echo "*********************************************"
+	@echo "*********************************************"
+	@sleep 5
+	@echo
+	@echo "*********************************************"
+	@echo "*********************************************"
+	@echo "Starting build."
 	@echo "*********************************************"
 	@echo "*********************************************"
-	@echo "Invoking Cloud Build"
+	gcloud builds submit --config cloudbuild.yaml --no-source --substitutions=${HNC_GCB_SUBS}
+	@echo "*********************************************"
+	@echo "*********************************************"
+	@echo "Pushing ${HNC_IMG} to ${HNC_RELEASE_IMG}"
 	@echo "*********************************************"
 	@echo "*********************************************"
-	@gcloud builds submit --config cloudbuild.yaml --no-source --substitutions=_HNC_REPO_USER=${HNC_REPO_USER},_HNC_IMG_NAME=${HNC_IMG_NAME},_HNC_IMG_TAG=${HNC_IMG_TAG},_HNC_USER=${HNC_USER},_HNC_PERSONAL_ACCESS_TOKEN=${HNC_PAT},_HNC_RELEASE_ID=${HNC_RELEASE_ID},_HNC_RELEASED_IMG=${HNC_RELEASED_IMG}
-	@echo Pushing image to K8s staging
 	@docker pull ${HNC_IMG}
-	@docker tag ${HNC_IMG} ${HNC_RELEASED_IMG}
-	@docker push ${HNC_RELEASED_IMG}
+	@docker tag ${HNC_IMG} ${HNC_RELEASE_IMG}
+	@docker push ${HNC_RELEASE_IMG}
 
+# Set up error checking variables
 ERR_MSG=Ensure that HNC_IMG_TAG (eg v0.1.0), HNC_USER (your Github username), HNC_PAT (Github personal access token) and HNC_RELEASE_ID (Github numeric ID) are set
+# During a release, We don't want to overwrite an existing docker image and tag
+# so we query the repository to see if the image already exists. This can be
+# overridden by setting HNC_FORCE_RELEASE=true
+ifeq ($(HNC_FORCE_RELEASE), true)
+	COULDNT_READ_RELEASE_IMG=1
+else
+	COULDNT_READ_RELEASE_IMG=$(shell gcloud container images describe $(HNC_RELEASE_IMG) > /dev/null 2>&1; echo $$?)
+endif
+
+# Actual error checking:
 check-release-env:
 ifndef HNC_IMG_TAG
 	$(error HNC_IMG_TAG is undefined; ${ERR_MSG})
 endif
 ifndef HNC_USER
-	$(error HNC_USER is undefinied; ${ERR_MSG})
+	$(error HNC_USER is undefined; ${ERR_MSG})
 endif
 ifndef HNC_PAT
 	$(error HNC_PAT is undefined; ${ERR_MSG})
 endif
 ifndef HNC_RELEASE_ID
 	$(error HNC_RELEASE_ID is undefined; ${ERR_MSG})
 endif
-ifeq ($(IMG_READ_ERROR), 0)
-	$(error The image ${HNC_IMG} already exists. Force and overwrite this image by using FORCE_RELEASE=true)
+ifeq ($(COULDNT_READ_RELEASE_IMG), 0)
+	$(error The image ${HNC_RELEASE_IMG} already exists. Force and overwrite this image by using HNC_FORCE_RELEASE=true)
 endif
-	@echo "*********************************************"
-	@echo "*********************************************"
-	@echo "Releasing ${HNC_RELEASED_IMG} using ${PROJECT_ID}"
-	@echo "*********************************************"
-	@echo "*********************************************"

incubator/hnc/cloudbuild.yaml

@@ -6,31 +6,25 @@ steps:
   args:
   - '-c'
   - |
-    git clone https://github.com/$_HNC_REPO_USER/multi-tenancy
+    set -e
+    echo Cloning from https://github.com/$_HNC_REPO_OWNER/multi-tenancy
+    git clone https://github.com/$_HNC_REPO_OWNER/multi-tenancy
     cd multi-tenancy
+    echo Checking out hnc-$_HNC_IMG_TAG
     git checkout hnc-$_HNC_IMG_TAG
 # Build the manifests and the kubectl plugin
-- name: mirror.gcr.io/library/golang
+- name: golang:1.14
   entrypoint: 'bash'
   args:
   - '-c'
   - |
-    # Get kustomize
-    export PATH=$$(go env GOPATH)/bin:$$PATH
-    mkdir -p $$(go env GOPATH)/bin
-    GO111MODULE=on go get sigs.k8s.io/kustomize/kustomize/v3@v3.5.4
-
-    # Build manifests
+    set -e
     cd multi-tenancy/incubator/hnc
-    mkdir out
-    cd out
-    touch kustomization.yaml
-    kustomize edit add resource ../config/default
-    kustomize edit set image controller=$_HNC_RELEASED_IMG
-    kustomize build . -o ./hnc-manager.yaml
-
-    # Build plugin
-    go build -o kubectl-hns ../cmd/kubectl/main.go
+    export HNC_REGISTRY=$_HNC_REGISTRY
+    export HNC_IMG_NAME=$_HNC_IMG_NAME
+    export HNC_IMG_TAG=$_HNC_IMG_TAG
+    # This makes manifests as a side effect
+    make kubectl
 # Upload manifest
 - name: gcr.io/cloud-builders/curl
   args:
@@ -39,7 +33,7 @@ steps:
   - '-H'
   - 'Content-Type: application/x-application'
   - '--data-binary'
-  - '@multi-tenancy/incubator/hnc/out/hnc-manager.yaml'
+  - '@multi-tenancy/incubator/hnc/manifests/hnc-manager.yaml'
   - '-u'
   - '$_HNC_USER:$_HNC_PERSONAL_ACCESS_TOKEN'
   - 'https://uploads.github.com/repos/kubernetes-sigs/multi-tenancy/releases/$_HNC_RELEASE_ID/assets?name=hnc-manager.yaml'
@@ -51,7 +45,7 @@ steps:
   - '-H'
   - 'Content-Type: application/x-application'
   - '--data-binary'
-  - '@multi-tenancy/incubator/hnc/out/kubectl-hns'
+  - '@multi-tenancy/incubator/hnc/bin/kubectl/kubectl-hns'
   - '-u'
   - '$_HNC_USER:$_HNC_PERSONAL_ACCESS_TOKEN'
   - 'https://uploads.github.com/repos/kubernetes-sigs/multi-tenancy/releases/$_HNC_RELEASE_ID/assets?name=kubectl-hns'

incubator/hnc/docs/releasing.md

@@ -25,11 +25,14 @@ You must have permission to write to this repo, and create a [personal access
 token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token)
 that includes that permission.
 
-You also need the ability to push to `gcr.io/k8s-staging-multitenancy`.
+You also need the ability to push to `gcr.io/k8s-staging-multitenancy`. You can
+get this by joining the k8s-infra-staging-multitenancy@kubernetes.io Google
+Group, which also gives you access to the `k8s-staging-multitenancy` GCP project
+(this is a standalone project and isn't in a GCP Organization).
 
 Finally, you must have a GCP project with Cloud Build enabled, and `gcloud` must
 be configured with this as your default project. _TODO: create a central project
-that anyone can use._
+that anyone can use, but without leaking personal access tokens._
 
 ## Document new/changed features
 

incubator/hnc/hack/README.md

@@ -0,0 +1,31 @@
+# Hacks
+
+Gotta love this directory name, eh? Let's try to cover what's in here.
+
+## HNC Build tools
+
+Whenever possible, we've tried to include build tools like controller-gen in the
+/vendors directory. In order to to force the Go tools _not_ to remove these
+runtime dependencies from the go.mod file, this directory contains a fake
+`tools.go` whose only purpose is to import these packages. Before you ask any
+more questions about that, check out
+https://stackoverflow.com/questions/52428230/how-do-go-modules-work-with-installable-commands
+and hopefully it will answer them.
+
+However, I wasn't able to get kustomize to fit in there, since it seems to have
+dependencies which are not compatible with the rest of HNC. So I've just checked
+in the Linux binary directly here. The exact version probably doesn't matter too
+much; I just used whatever was most current and it worked.
+
+## Templates
+
+`boilerplate.go.txt` includes the Apache header. Kubebuilder put it here and it
+seems like as good a place as any.
+
+`krew-hierarchical-namespaces.yaml` is a template for the Krew `kubectl-hns`
+plugin.
+
+## CI
+
+Other projects seem to put their presubmits, postsubmits etc here, so we did
+too. See ../README.md for where these are configured in Prow.

incubator/hnc/hack/ci-test.sh

@@ -27,15 +27,8 @@ set -e
 export PATH=$(go env GOPATH)/bin:$PATH
 mkdir -p $(go env GOPATH)/bin
 
-echo "Installing 'kubebuilder'"
-wget https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.0.0-alpha.1/kubebuilder_2.0.0-alpha.1_linux_amd64.tar.gz
-tar -zxvf kubebuilder_2.0.0-alpha.1_linux_amd64.tar.gz
-mv kubebuilder_2.0.0-alpha.1_linux_amd64 /usr/local/kubebuilder
-
-echo "Installing 'kustomize'"
-GO111MODULE=on go get sigs.k8s.io/kustomize/kustomize/v3@v3.5.4
-
 hack_dir=$(dirname ${BASH_SOURCE})
 
 echo "Running 'make tests'"
-make test -C "$hack_dir/.."
+cd "$hack_dir/.."
+make test