From cad662f72a3340241618190ee17c5a2e6ef89842 Mon Sep 17 00:00:00 2001 From: Mark Mandel Date: Thu, 19 Nov 2020 16:10:32 -0800 Subject: [PATCH] Update minikube dev tooling Needed to update minikube to Kubernetes 1.17.x and I figured I would also go through the minikube dev experience and update it. This includes: * Switch to default to the Docker driver, since everyone should have Docker installed. * Removing the Windows hacks, because they were awful and I feel bad I even wrote them in the first place. * Migrate tooling to use new minikube functionality * Update minikube commands to up to date release conformity. * Updated the documentation Work on #1824 --- build/README.md | 47 ++++++++++++-------------------------- build/includes/linux.mk | 12 ---------- build/includes/macos.mk | 10 -------- build/includes/minikube.mk | 47 +++++++++++++------------------------- build/includes/windows.mk | 26 --------------------- 5 files changed, 31 insertions(+), 111 deletions(-) diff --git a/build/README.md b/build/README.md index 7936eae11a..d479f8dd9f 100644 --- a/build/README.md +++ b/build/README.md @@ -102,8 +102,6 @@ Table of Contents ### Linux - Install Make, either via `apt install make` or `yum install make` depending on platform. - [Install Docker](https://docs.docker.com/engine/installation/) for your Linux platform. -- (optional) Minikube will require [VirtualBox](https://www.virtualbox.org) and will need to be installed if you wish - to develop on Minikube ### Windows Building and developing Agones requires you to use the @@ -121,16 +119,12 @@ as this makes it easy to create a (relatively) cross platform development and bu - Agones will need to be cloned somewhere on your `/c` (or drive of your choice) path, as that is what Docker will support mounts from - All interaction with Agones must be on the `/c` (or drive of your choice) path, otherwise Docker mounts will not work - Now the `make` commands can all be run from within your WSL shell -- (optional) Minikube is supported via the [HyperV](https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/index) - driver - the same virtualisation platform as the Docker installation. -- **Note**: If you want to dev and test with Minikube, you **must** run WSL as Administrator, otherwise Minikube can't control HyperV. +- The Minikube setup on Windows has not been tested. Pull Requests would be appreciated! ### macOS - Install Make, `brew install make`, if it's not installed already - Install [Docker for Mac](https://docs.docker.com/docker-for-mac/install/) -- (optional) Minikube will require [VirtualBox](https://www.virtualbox.org) and will need to be installed if you wish - to develop on Minikube ## GOPATH @@ -251,13 +245,10 @@ When your are finished, you can run `make clean-gcloud-test-cluster` to tear dow ### Running a Test Minikube cluster This will setup a [Minikube](https://github.com/kubernetes/minikube) cluster, running on an `agones` profile, -Because Minikube runs on a virtualisation layer on the host, some of the standard build and development Make targets -need to be replaced by Minikube specific targets. +Because Minikube runs on a virtualisation layer on the host (usually Docker), some of the standard build and development + Make targets need to be replaced by Minikube specific targets. -First, [install Minikube](https://github.com/kubernetes/minikube#installation), which may also require you to install -a virtualisation solution, such as [VirtualBox](https://www.virtualbox.org) as well. -Check the [Building on Different Platforms](#building-on-different-platforms) above for details on what virtualisation -solution to use. +First, [install Minikube](https://github.com/kubernetes/minikube#installation). Next we will create the Agones Minikube cluster. Run `make minikube-test-cluster` to create the `agones` profile, and a Kubernetes cluster of the supported version under this profile. @@ -287,14 +278,8 @@ It's worth noting that Minikube does let you [reuse its Docker daemon](https://g and build directly on Minikube, but in this case this approach is far simpler, and makes cross-platform support for the build system much easier. -If you find you also want to push your own images into Minikube, -the convenience make target `make minikube-transfer-image` can be run with the `TAG` argument specifying -the tag of the Docker image you wish to transfer into Minikube. - -For example: -```bash -$ make minikube-transfer-image TAG=myimage:0.1 -``` +To push your own images into the cluster, take a look at Minikube's +[Pushing Images](https://minikube.sigs.k8s.io/docs/handbook/pushing/) guide. Running end-to-end tests on Minikube is done via the `make minikube-test-e2e` target. This target use the same `make test-e2e` but also setup some prerequisites for use with a Minikube cluster. @@ -653,20 +638,22 @@ Since Minikube runs locally, there are some targets that need to be used instead #### `make minikube-test-cluster` Switches to an "agones" profile, and starts a kubernetes cluster -of the right version. +of the right version. Uses "docker" as the default driver. -Use MINIKUBE_DRIVER variable to change the VM driver -(defaults virtualbox for Linux and macOS, hyperv for windows) if you so desire. - -#### `make minikube-push` -Push the local Agones Docker images that have already been built -via `make build` or `make build-images` into the "agones" minikube instance. +If needed, use MINIKUBE_DRIVER variable to change the VM driver. #### `make minikube-install` + Installs the current development version of Agones into the Kubernetes cluster. Use this instead of `make install`, as it disables PullAlways on the install.yaml +#### `make minikube-push` + +Push the local Agones Docker images that have already been built +via `make build` or `make build-images` into the "agones" minikube instance with `minikube cache add` + #### `make minikube-setup-prometheus` + Installs prometheus metric backend into the Kubernetes cluster. Use this instead of `make setup-prometheus`, as it disables Persistent Volume Claim. @@ -693,10 +680,6 @@ These tests validate Agones flow from start to finish. Connecting to Minikube requires so enhanced permissions, so use this target instead of `make shell` to start an interactive shell for development on Minikube. -#### `make minikube-transfer-image` -Convenience target for transferring images into minikube. -Use TAG to specify the image to transfer into minikube - #### `make minikube-controller-portforward` The minikube version of [`make controller-portforward`](#make-controller-portforward) to setup port forwarding to the controller deployment. diff --git a/build/includes/linux.mk b/build/includes/linux.mk index ba76b460fb..e486f4978d 100644 --- a/build/includes/linux.mk +++ b/build/includes/linux.mk @@ -31,13 +31,6 @@ sha_dir = $(shell sha256sum $(1) | cut -d' ' -f1 | sha256sum | head -c 10 ) # Minikube executable MINIKUBE ?= minikube -# Default minikube driver -MINIKUBE_DRIVER ?= virtualbox -# set docker env for minikube -MINIKUBE_DOCKER_ENV ?= eval $$($(MINIKUBE) docker-env) - -# minikube shell mount for certificates -minikube_cert_mount := ~/.minikube:$(HOME)/.minikube # _____ _ # |_ _|_ _ _ __ __ _ ___| |_ ___ @@ -46,11 +39,6 @@ minikube_cert_mount := ~/.minikube:$(HOME)/.minikube # |_|\__,_|_| \__, |\___|\__|___/ # |___/ -# Does not do anything -minikube-post-start: -# kubectl > 1.11 may have --address flag, but for the time being, -# we will use --network=host, as port-forward binds to localhost - # port forward the agones controller. # useful for pprof and stats viewing, etc controller-portforward: PORT ?= 8080 diff --git a/build/includes/macos.mk b/build/includes/macos.mk index 743128b655..55fbbaf64b 100644 --- a/build/includes/macos.mk +++ b/build/includes/macos.mk @@ -31,13 +31,6 @@ sha_dir = $(shell shasum -a 256 $(1) | cut -d' ' -f1 | shasum -a 256 | head -c 1 # Minikube executable MINIKUBE ?= minikube -# Default minikube driver -MINIKUBE_DRIVER ?= virtualbox -# set docker env for minikube -MINIKUBE_DOCKER_ENV ?= eval $$($(MINIKUBE) docker-env) - -# minikube shell mount for certificates -minikube_cert_mount := ~/.minikube:$(HOME)/.minikube # _____ _ # |_ _|_ _ _ __ __ _ ___| |_ ___ @@ -46,9 +39,6 @@ minikube_cert_mount := ~/.minikube:$(HOME)/.minikube # |_|\__,_|_| \__, |\___|\__|___/ # |___/ -# Does nothing -minikube-post-start: - # port forward the agones controller. # useful for pprof and stats viewing, etc controller-portforward: PORT ?= 8080 diff --git a/build/includes/minikube.mk b/build/includes/minikube.mk index 63a6d0f957..fc54d2a503 100644 --- a/build/includes/minikube.mk +++ b/build/includes/minikube.mk @@ -12,6 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +MINIKUBE_DRIVER ?= docker + +# minikube shell mount for certificates +minikube_cert_mount := ~/.minikube:$(HOME)/.minikube # __ __ _ _ _ _ # | \/ (_)_ __ (_) | ___ _| |__ ___ @@ -22,58 +26,39 @@ # Switches to an "agones" profile, and starts a kubernetes cluster # of the right version. -# -# Use MINIKUBE_DRIVER variable to change the VM driver -# (defaults virtualbox for Linux and macOS, hyperv for windows) if you so desire. minikube-test-cluster: DOCKER_RUN_ARGS+=--network=host -v $(minikube_cert_mount) -minikube-test-cluster: $(ensure-build-image) minikube-agones-profile - $(MINIKUBE) start --kubernetes-version v1.16.13 --vm-driver $(MINIKUBE_DRIVER) - # wait until the master is up - until docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) kubectl cluster-info; \ - do \ - echo "Waiting for cluster to start..."; \ - sleep 1; \ - done - $(MAKE) minikube-post-start - -# switch to the agones cluster -minikube-agones-profile: - $(MINIKUBE) profile $(MINIKUBE_PROFILE) +minikube-test-cluster: $(ensure-build-image) + $(MINIKUBE) start --kubernetes-version v1.17.13 -p $(MINIKUBE_PROFILE) --vm-driver $(MINIKUBE_DRIVER) # Connecting to minikube requires so enhanced permissions, so use this target # instead of `make shell` to start an interactive shell for development on minikube. -minikube-shell: $(ensure-build-image) minikube-agones-profile +minikube-shell: $(ensure-build-image) $(MAKE) shell DOCKER_RUN_ARGS="--network=host -v $(minikube_cert_mount) $(DOCKER_RUN_ARGS)" # Push the local Agones Docker images that have already been built # via `make build` or `make build-images` into the "agones" minikube instance. -minikube-push: minikube-agones-profile - $(MAKE) minikube-transfer-image TAG=$(sidecar_tag) - $(MAKE) minikube-transfer-image TAG=$(controller_tag) - $(MAKE) minikube-transfer-image TAG=$(ping_tag) - $(MAKE) minikube-transfer-image TAG=$(allocator_tag) +minikube-push: + $(MINIKUBE) cache add $(sidecar_tag) + $(MINIKUBE) cache add $(controller_tag) + $(MINIKUBE) cache add $(ping_tag) + $(MINIKUBE) cache add $(allocator_tag) # Installs the current development version of Agones into the Kubernetes cluster. # Use this instead of `make install`, as it disables PullAlways on the install.yaml -minikube-install: minikube-agones-profile +minikube-install: $(MAKE) install DOCKER_RUN_ARGS="--network=host -v $(minikube_cert_mount)" ALWAYS_PULL_SIDECAR=false \ IMAGE_PULL_POLICY=IfNotPresent PING_SERVICE_TYPE=NodePort ALLOCATOR_SERVICE_TYPE=NodePort -minikube-uninstall: $(ensure-build-image) minikube-agones-profile +minikube-uninstall: $(ensure-build-image) $(MAKE) uninstall DOCKER_RUN_ARGS="--network=host -v $(minikube_cert_mount)" -# Convenience target for transferring images into minikube. -# Use TAG to specify the image to transfer into minikube -minikube-transfer-image: - docker save $(TAG) | ($(MINIKUBE_DOCKER_ENV) && docker load) - # Runs e2e tests against our minikube minikube-test-e2e: DOCKER_RUN_ARGS=--network=host -v $(minikube_cert_mount) -minikube-test-e2e: minikube-agones-profile test-e2e +minikube-test-e2e: test-e2e # Runs stress tests against our minikube minikube-stress-test-e2e: DOCKER_RUN_ARGS=--network=host -v $(minikube_cert_mount) -minikube-stress-test-e2e: minikube-agones-profile stress-test-e2e +minikube-stress-test-e2e: stress-test-e2e # prometheus on minkube # we have to disable PVC as it's not supported on minkube. diff --git a/build/includes/windows.mk b/build/includes/windows.mk index 283d9767d7..3b6d21f760 100644 --- a/build/includes/windows.mk +++ b/build/includes/windows.mk @@ -31,20 +31,6 @@ sha_dir = $(shell sha256sum $(1) | cut -d' ' -f1 | sha256sum | head -c 10 ) # Minikube executable MINIKUBE ?= minikube.exe -# Default minikube driver -MINIKUBE_DRIVER ?= hyperv -# set docker env for minikube -MINIKUBE_DOCKER_ENV ?= eval $$($(MINIKUBE) docker-env --shell=bash) && \ - export DOCKER_CERT_PATH=$$(echo $$DOCKER_CERT_PATH | $(win_to_wsl_path)) - -# minikube shell mount for certificates -minikube_cert_mount = $(cert_path):$(cert_path) - -# transform the path from windows to WSL -win_to_wsl_path := sed -e 's|\([A-Z]\):|/\L\1|' -e 's|\\|/|g' - -# find the cert path -cert_path = $(realpath $(shell $(MINIKUBE) docker-env --shell bash | grep DOCKER_CERT_PATH | awk -F "=" '{ print $$2 }' | sed 's/"//g' | $(win_to_wsl_path))/..) # _____ _ # |_ _|_ _ _ __ __ _ ___| |_ ___ @@ -53,18 +39,6 @@ cert_path = $(realpath $(shell $(MINIKUBE) docker-env --shell bash | grep DOCKER # |_|\__,_|_| \__, |\___|\__|___/ # |___/ -# Sets minikube credentials -minikube-post-start: - echo "Creating minikube credentials" - export CERT_PATH=$(cert_path) && \ - docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) kubectl config set-cluster $(MINIKUBE_PROFILE) \ - --certificate-authority=$$CERT_PATH/ca.crt --server=https://$$($(MINIKUBE) ip):8443 && \ - docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) kubectl config set-credentials $(MINIKUBE_PROFILE) \ - --client-certificate=$$CERT_PATH/client.crt --client-key=$$CERT_PATH/client.key - docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) kubectl config set-context $(MINIKUBE_PROFILE) \ - --cluster=$(MINIKUBE_PROFILE) --user=$(MINIKUBE_PROFILE) - docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) kubectl config use-context $(MINIKUBE_PROFILE) - # port forward the agones controller. # useful for pprof and stats viewing, etc controller-portforward: PORT ?= 8080