Skip to content

Commit

Permalink
adds the first e2e test
Browse files Browse the repository at this point in the history
This test creates a gameserver and connect to it

Adds the gcloud builder step to test it in a special e2e cluster
Updates build documentations with e2e instructions
Adds minikube e2e target
  • Loading branch information
Cyril TOVENA committed Aug 15, 2018
1 parent 2af9c22 commit dc0e9dd
Show file tree
Hide file tree
Showing 16 changed files with 547 additions and 37 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,4 @@ tmp
build/local-includes/*
!build/local-includes/README.md
/release
debug.test
87 changes: 62 additions & 25 deletions build/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ GCP_CLUSTER_ZONE ?= us-west1-c
# the profile to use when developing on minikube
MINIKUBE_PROFILE ?= agones

# Game Server image to use while doing end-to-end tests
GS_TEST_IMAGE ?= gcr.io/agones-images/udp-server:0.1

# Directory that this Makefile is in.
mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
build_path := $(dir $(mkfile_path))
Expand All @@ -63,7 +66,7 @@ controller_tag = $(REGISTRY)/agones-controller:$(VERSION)
sidecar_tag = $(REGISTRY)/agones-sdk:$(VERSION)

go_version_flags = -ldflags "-X agones.dev/agones/pkg.Version=$(VERSION)"

DOCKER_RUN ?= docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag)
# ___ ____ ___ _ _
# / _ \/ ___| |_ _|_ __ ___| |_ _ __| | ___
# | | | \___ \ | || '_ \ / __| | | | |/ _` |/ _ \
Expand All @@ -85,6 +88,10 @@ include ./includes/$(osinclude)
# personal includes, excluded from the git repository
-include ./local-includes/*.mk

ifdef DOCKER_RUN
ensure-build-image += ensure-build-image
endif

# _____ _
# |_ _|_ _ _ __ __ _ ___| |_ ___
# | |/ _` | '__/ _` |/ _ \ __/ __|
Expand All @@ -102,11 +109,19 @@ build-images: build-controller-image build-agones-sdk-image
build-sdks: build-sdk-cpp

# Run all tests
test: ensure-build-image test-go test-install-yaml
test: $(ensure-build-image) test-go test-install-yaml

# Run go tests
test-go:
docker run --rm $(common_mounts) $(build_tag) go test -race $(agones_package)/...
docker run --rm $(common_mounts) $(build_tag) go test -race $(agones_package)/pkg/... \
$(agones_package)/sdks/...

# Runs end-to-end tests on the current configured cluster
# For minikube user the minikube-test-e2e targets
test-e2e:
$(DOCKER_RUN) go test -v $(agones_package)/test/e2e/... \
--kubeconfig /root/.kube/config \
--gameserver-image=$(GS_TEST_IMAGE)

# Run test on install yaml - make sure there is no change
# mostly this is for CI
Expand All @@ -124,34 +139,34 @@ push: push-controller-image push-agones-sdk-image
# Installs the current development version of Agones into the Kubernetes cluster
install: ALWAYS_PULL_SIDECAR := true
install: IMAGE_PULL_POLICY := "Always"
install: ensure-build-image
docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) \
helm upgrade --install --namespace=agones-system \
install: $(ensure-build-image)
$(DOCKER_RUN) \
helm upgrade --install --recreate-pods --wait --namespace=agones-system \
--set agones.image.tag=$(VERSION),agones.image.registry=$(REGISTRY),agones.image.controller.pullPolicy=$(IMAGE_PULL_POLICY),agones.image.sdk.alwaysPull=$(ALWAYS_PULL_SIDECAR) \
agones $(mount_path)/install/helm/agones/

# Build a static binary for the gameserver controller
build-controller-binary: ensure-build-image
build-controller-binary: $(ensure-build-image)
docker run --rm -e "CGO_ENABLED=0" $(common_mounts) $(build_tag) go build \
-o $(mount_path)/cmd/controller/bin/controller -a $(go_version_flags) -installsuffix cgo $(agones_package)/cmd/controller

# Lint the go source code.
# use LINT_TIMEOUT to manipulate the linter timeout
lint: LINT_TIMEOUT ?= 15m
lint: ensure-build-image
lint: $(ensure-build-image)
docker run --rm $(common_mounts) -w $(mount_path) $(DOCKER_RUN_ARGS) $(build_tag) bash -c \
"/root/gen-lint-exclude.sh && gometalinter --config .exclude.gometalinter.json --deadline=$(LINT_TIMEOUT) -t --skip vendor ./..."

# Build the image for the gameserver controller
build-controller-image: ensure-build-image build-controller-binary
build-controller-image: $(ensure-build-image) build-controller-binary
docker build $(agones_path)/cmd/controller/ --tag=$(controller_tag) $(DOCKER_BUILD_ARGS)

# push the gameservers controller image
push-controller-image: ensure-build-image
push-controller-image: $(ensure-build-image)
docker push $(controller_tag)

# build the static binary for the gamesever sidecar
build-agones-sdk-binary: ensure-build-image
build-agones-sdk-binary: $(ensure-build-image)
docker run --rm -e "CGO_ENABLED=0" $(common_mounts) $(build_tag) go build \
-o $(mount_path)/cmd/sdk-server/bin/sdk-server.linux.amd64 -a $(go_version_flags) -installsuffix cgo $(agones_package)/cmd/sdk-server
docker run --rm -e "GOOS=darwin" -e "GOARCH=amd64" $(common_mounts) $(build_tag) go build \
Expand All @@ -162,38 +177,38 @@ build-agones-sdk-binary: ensure-build-image
agonessdk-server-$(VERSION).zip sdk-server.darwin.amd64 sdk-server.linux.amd64 sdk-server.windows.amd64.exe

# Build the image for the gameserver sidecar
build-agones-sdk-image: ensure-build-image build-agones-sdk-binary
build-agones-sdk-image: $(ensure-build-image) build-agones-sdk-binary
docker build $(agones_path)/cmd/sdk-server/ --tag=$(sidecar_tag) $(DOCKER_BUILD_ARGS)

# Build the cpp sdk linux archive
build-sdk-cpp: ensure-build-image
build-sdk-cpp: $(ensure-build-image)
docker run --rm $(common_mounts) -w $(mount_path)/sdks/cpp $(build_tag) make build install archive VERSION=$(VERSION)

# push the gameservers sidecar image
push-agones-sdk-image: ensure-build-image
push-agones-sdk-image: $(ensure-build-image)
docker push $(sidecar_tag)

# Generate the static install script
gen-install: ensure-build-image
gen-install: $(ensure-build-image)
docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) bash -c \
'helm template --name=agones-manual --namespace agones-system $(mount_path)/install/helm/agones \
--set agones.controller.generateTLS=false \
> $(mount_path)/install/yaml/install.yaml'

# Generate the SDK gRPC server and client code
gen-gameservers-sdk-grpc: ensure-build-image
gen-gameservers-sdk-grpc: $(ensure-build-image)
docker run --rm $(common_mounts) -w $(mount_path) $(build_tag) /root/gen-grpc-go.sh
docker run --rm $(common_mounts) -w $(mount_path) $(build_tag) /root/gen-grpc-cpp.sh
docker run --rm $(common_mounts) -w $(mount_path) $(build_tag) /root/gen-grpc-rust.sh

# Generate the client for our CustomResourceDefinition
gen-crd-client: ensure-build-image
gen-crd-client: $(ensure-build-image)
docker run --rm $(common_mounts) -w $(mount_path) $(build_tag) /root/gen-crd-client.sh
docker run --rm $(common_mounts) -w $(mount_path)/pkg $(build_tag) goimports -w .

# Run a bash shell with the developer tools in it. (Creates the image if it doesn't exist)
# Can use DOCKER_RUN_ARGS for extra arguments.
shell: ensure-build-image
shell: $(ensure-build-image)
docker run -it --rm \
$(common_mounts) \
-w $(mount_path) \
Expand Down Expand Up @@ -286,7 +301,7 @@ gcloud-init: ensure-build-config
gcloud-test-cluster: GCP_CLUSTER_LEGACYABAC ?= false
gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4
gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= n1-standard-4
gcloud-test-cluster: ensure-build-image
gcloud-test-cluster: $(ensure-build-image)
docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) gcloud \
deployment-manager deployments create $(GCP_CLUSTER_NAME) \
--properties cluster.zone:$(GCP_CLUSTER_ZONE),cluster.name:$(GCP_CLUSTER_NAME),cluster.nodePool.initialNodeCount:$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT),cluster.nodePool.machineType:$(GCP_CLUSTER_NODEPOOL_MACHINETYPE),cluster.legacyAbac:$(GCP_CLUSTER_LEGACYABAC)\
Expand All @@ -295,21 +310,39 @@ gcloud-test-cluster: ensure-build-image
docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) kubectl apply -f $(mount_path)/build/helm.yaml
docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) helm init --service-account helm

clean-gcloud-test-cluster: ensure-build-image
clean-gcloud-test-cluster: $(ensure-build-image)
docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) gcloud \
deployment-manager deployments delete $(GCP_CLUSTER_NAME)


# Creates a gcloud cluster for end-to-end
# it installs also a consul cluster to handle build system concurrency using a distributed lock
gcloud-e2e-test-cluster: $(ensure-build-image)
docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) gcloud \
deployment-manager deployments create e2e-test-cluster \
--config=$(mount_path)/build/gke-test-cluster/cluster-e2e.yml
GCP_CLUSTER_NAME=e2e-test-cluster GCP_CLUSTER_ZONE=us-west1-c $(MAKE) gcloud-auth-cluster
docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) \
kubectl apply -f $(mount_path)/build/helm.yaml
docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) helm init --service-account helm --wait && \
helm install --wait --set Replicas=1,uiService.type=ClusterIP --name consul stable/consul

# Deletes the gcloud e2e cluster and cleanup any left pvc volumes
clean-gcloud-e2e-test-cluster: $(ensure-build-image)
docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) \
helm delete --purge consul && kubectl delete pvc -l component=consul-consul
GCP_CLUSTER_NAME=e2e-test-cluster $(MAKE) clean-gcloud-test-cluster

# Pulls down authentication information for kubectl against a cluster, name can be specified through GCP_CLUSTER_NAME
# (defaults to 'test-cluster')
gcloud-auth-cluster: ensure-build-image
gcloud-auth-cluster: $(ensure-build-image)
docker run --rm $(common_mounts) $(build_tag) gcloud config set container/cluster $(GCP_CLUSTER_NAME)
docker run --rm $(common_mounts) $(build_tag) gcloud config set compute/zone $(GCP_CLUSTER_ZONE)
docker run --rm $(common_mounts) $(build_tag) gcloud container clusters get-credentials $(GCP_CLUSTER_NAME)
-docker run --rm $(common_mounts) $(build_tag) bash -c 'kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user $$(gcloud config get-value account)'

# authenticate our docker configuration so that you can do a docker push directly
# to the gcr.io repository
gcloud-auth-docker: ensure-build-image
gcloud-auth-docker: $(ensure-build-image)
docker run --rm $(common_mounts) $(build_tag) gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://gcr.io

# Clean the gcloud configuration
Expand All @@ -328,7 +361,7 @@ clean-gcloud-config:
#
# Use MINIKUBE_DRIVER variable to change the VM driver
# (defaults virtualbox for Linux and macOS, hyperv for windows) if you so desire.
minikube-test-cluster: ensure-build-image minikube-agones-profile
minikube-test-cluster: $(ensure-build-image) minikube-agones-profile
# localkube bootstrapper fixes issues with profiles
$(MINIKUBE) start --kubernetes-version v1.10.0 --vm-driver $(MINIKUBE_DRIVER) \
--bootstrapper=localkube \
Expand All @@ -355,7 +388,7 @@ minikube-agones-profile:

# Connecting to minikube requires so enhanced permissions, so use this target
# instead of `make shell` to start an interactive shell for development on minikube.
minikube-shell: ensure-build-image minikube-agones-profile
minikube-shell: $(ensure-build-image) minikube-agones-profile
$(MAKE) shell DOCKER_RUN_ARGS="--network=host -v $(minikube_cert_mount) $(DOCKER_RUN_ARGS)"

# Push the local Agones Docker images that have already been built
Expand All @@ -373,3 +406,7 @@ minikube-install: minikube-agones-profile
# Use TAG to specify the image to transfer into minikube
minikube-transfer-image:
docker save $(TAG) | ($(MINIKUBE_DOCKER_ENV) && docker load)

# Runs e2e tests against our minikube
minikube-test-e2e: DOCKER_RUN_ARGS=--network=host -v $(minikube_cert_mount)
minikube-test-e2e: minikube-agones-profile test-e2e
15 changes: 15 additions & 0 deletions build/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,8 @@ Now that the images are pushed, to install the development version (with all ima
run `make install` and Agones will install the image that you just built and pushed on the test cluster you
created at the beginning of this section. (if you want to see the resulting installation yaml, you can find it in `build/.install.yaml`)

Finally to run end-to-end tests against your development version previously installed in your test cluster run `make test-e2e`, this will validate the whole application flow (from start to finish). If you're curious about how they work head to [tests/e2e](../test/e2e/)

### Running a Test Minikube cluster
This will setup a [Minikube](https://github.com/kubernetes/minikube) cluster, running on an `agones` profile,

Expand Down Expand Up @@ -226,6 +228,8 @@ For example:
$ make minikube-transfer-image TAG=myimage:0.1
```

Running end-to-end tests on minikube is done via the `make minikube-test-e2e` target. This target use the same `make test-e2e` but also setup some prerequisites for use with a minikube cluster.

### Next Steps

Have a look in the [examples](../examples) folder to see examples of running Game Servers on Agones.
Expand Down Expand Up @@ -274,6 +278,13 @@ Pushes all built images up to the `$(REGISTRY)`
#### `make install`
Installs the current development version of Agones into the Kubernetes cluster

### `make test-e2e`
Runs end-to-end tests on the previously installed version of Agones.
These tests validate Agones flow from start to finish.

It uses the kube config (located by default in `~/.kube`) to target a Kubernetes cluster.
See [`make minikube-test-e2e`](#make-minikube-test-e2e) to run end-to-end tests on minikube.

#### `make shell`
Run a bash shell with the developer tools (go tooling, kubectl, etc) and source code in it.

Expand Down Expand Up @@ -349,6 +360,10 @@ via `make build` or `make build-images` into the "agones" minikube instance.
Installs the current development version of Agones into the Kubernetes cluster.
Use this instead of `make install`, as it disables PullAlways on the install.yaml

### `make minikube-test-e2e`
Runs end-to-end tests on the previously installed version of Agones.
These tests validate Agones flow from start to finish.

#### `make minikube-shell`
Connecting to Minikube requires so enhanced permissions, so use this target
instead of `make shell` to start an interactive shell for development on Minikube.
Expand Down
20 changes: 13 additions & 7 deletions build/build-image/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# ForceUpdate 5 -- change here if you need to force a rebuild
# ForceUpdate 6 -- change here if you need to force a rebuild

# compiling proto + grpc takes an exceptionally long time
# so we'll use a base from `base` - which is manually built using the below tag.
FROM gcr.io/agones-images/grpc-cxx:1.12

RUN apt-get update && \
apt-get install -y wget rsync make python bash-completion zip nano jq && \
apt-get install -y wget psmisc rsync make python bash-completion zip nano jq && \
apt-get clean

# install go
Expand All @@ -39,17 +39,23 @@ RUN wget -q https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.zip && un
ENV PATH /usr/local/go/bin:/go/bin:/opt/google-cloud-sdk/bin:$PATH

# RUN gcloud components update
RUN gcloud components update && gcloud components install kubectl
RUN gcloud components update

# install kubectl without gcloud as we want the last version
ENV KUBECTL_VER 1.11.0
RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VER}/bin/linux/amd64/kubectl && \
chmod go+rx ./kubectl && \
mv ./kubectl /usr/local/bin/kubectl
RUN echo "source <(kubectl completion bash)" >> /root/.bashrc

# install Helm package manager
ENV HELM_VER 2.9.1
ENV HELM_URL https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VER}-linux-amd64.tar.gz
RUN curl -L ${HELM_URL} > /tmp/helm.tar.gz \
&& tar -zxvf /tmp/helm.tar.gz -C /tmp \
&& mv /tmp/linux-amd64/helm /usr/local/bin/helm \
&& chmod go+rx /usr/local/bin/helm \
&& rm /tmp/helm.tar.gz && rm -rf /tmp/linux-amd64
&& tar -zxvf /tmp/helm.tar.gz -C /tmp \
&& mv /tmp/linux-amd64/helm /usr/local/bin/helm \
&& chmod go+rx /usr/local/bin/helm \
&& rm /tmp/helm.tar.gz && rm -rf /tmp/linux-amd64
RUN echo "source <(helm completion bash)" >> /root/.bashrc

# install go-proto-gen 1.1
Expand Down
52 changes: 52 additions & 0 deletions build/e2e-image/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
FROM gcr.io/cloud-builders/gcloud-slim

RUN apt-get update && \
apt-get install -y wget psmisc make python jq zip && \
apt-get clean

# install go
WORKDIR /usr/local
ENV GO_VERSION=1.10.3
ENV GOPATH /go
RUN wget -q https://redirector.gvt1.com/edgedl/go/go${GO_VERSION}.linux-amd64.tar.gz && \
tar -xzf go${GO_VERSION}.linux-amd64.tar.gz && rm go${GO_VERSION}.linux-amd64.tar.gz && mkdir ${GOPATH}

ENV PATH /usr/local/go/bin:/go/bin:$PATH

# install kubectl without gcloud as we want the last version
ENV KUBECTL_VER 1.11.0
RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VER}/bin/linux/amd64/kubectl && \
chmod go+rx ./kubectl && \
mv ./kubectl /usr/local/bin/kubectl

# install Helm package manager
ENV HELM_VER 2.9.1
ENV HELM_URL https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VER}-linux-amd64.tar.gz
RUN curl -L ${HELM_URL} > /tmp/helm.tar.gz \
&& tar -zxvf /tmp/helm.tar.gz -C /tmp \
&& mv /tmp/linux-amd64/helm /usr/local/bin/helm \
&& chmod go+rx /usr/local/bin/helm \
&& rm /tmp/helm.tar.gz && rm -rf /tmp/linux-amd64

# set up Consul.
ENV CONSUL_VERSION=1.2.1
ENV HASHICORP_RELEASES=https://releases.hashicorp.com
RUN mkdir -p /tmp/build && \
wget ${HASHICORP_RELEASES}/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip \
-P /tmp/build/ && \
unzip -d /usr/local/bin/ /tmp/build/consul_${CONSUL_VERSION}_linux_amd64.zip && \
cd /tmp && \
rm -rf /tmp/build && \
chmod go+rx /usr/local/bin/consul && \
# tiny smoke test to ensure the binary we downloaded runs
consul version

# make sure we keep the path to go
RUN echo "export PATH=/usr/local/go/bin:/go/bin/:\$PATH" >> /root/.bashrc
# scripts
COPY *.sh /root/
RUN chmod +x /root/*.sh

WORKDIR /go

ENTRYPOINT [ "/root/entrypoint.sh" ]
19 changes: 19 additions & 0 deletions build/e2e-image/e2e.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/usr/bin/env bash

# Copyright 2018 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
echo "installing current release"
DOCKER_RUN= make install
echo "starting e2e test"
DOCKER_RUN= make test-e2e
Loading

0 comments on commit dc0e9dd

Please sign in to comment.