Skip to content

Commit

Permalink
adds the first e2e test which create a gameserver and connect to it
Browse files Browse the repository at this point in the history
this also add the gcloud builder step to test it in a special e2e cluster
  • Loading branch information
Cyril TOVENA committed Aug 1, 2018
1 parent 7009bea commit 9e1598b
Show file tree
Hide file tree
Showing 15 changed files with 528 additions and 37 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,4 @@ tmp
build/local-includes/*
!build/local-includes/README.md
/release
debug.test
79 changes: 54 additions & 25 deletions build/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ GCP_CLUSTER_ZONE ?= us-west1-c
# the profile to use when developing on minikube
MINIKUBE_PROFILE ?= agones

# Game Server image to use while doing end-to-end tests
GS_TEST_IMAGE ?= gcr.io/agones-images/cpp-simple-server:0.2

# Directory that this Makefile is in.
mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
build_path := $(dir $(mkfile_path))
Expand All @@ -63,7 +66,7 @@ controller_tag = $(REGISTRY)/agones-controller:$(VERSION)
sidecar_tag = $(REGISTRY)/agones-sdk:$(VERSION)

go_version_flags = -ldflags "-X agones.dev/agones/pkg.Version=$(VERSION)"

DOCKER_RUN ?= docker run --rm $(common_mounts) $(build_tag)
# ___ ____ ___ _ _
# / _ \/ ___| |_ _|_ __ ___| |_ _ __| | ___
# | | | \___ \ | || '_ \ / __| | | | |/ _` |/ _ \
Expand All @@ -85,6 +88,10 @@ include ./includes/$(osinclude)
# personal includes, excluded from the git repository
-include ./local-includes/*.mk

ifdef DOCKER_RUN
ensure-build-image += ensure-build-image
endif

# _____ _
# |_ _|_ _ _ __ __ _ ___| |_ ___
# | |/ _` | '__/ _` |/ _ \ __/ __|
Expand All @@ -102,11 +109,18 @@ build-images: build-controller-image build-agones-sdk-image
build-sdks: build-sdk-cpp

# Run all tests
test: ensure-build-image test-go test-install-yaml
test: $(ensure-build-image) test-go test-install-yaml

# Run go tests
test-go:
docker run --rm $(common_mounts) $(build_tag) go test -race $(agones_package)/...
docker run --rm $(common_mounts) $(build_tag) go test -race $(agones_package)/pkg/... \
$(agones_package)/sdks/...

# Runs end-to-end tests on the current configured cluster
test-e2e:
$(DOCKER_RUN) go test -v $(agones_package)/test/e2e/... \
--kubeconfig /root/.kube/config \
--gameserver-image=$(GS_TEST_IMAGE)

# Run test on install yaml - make sure there is no change
# mostly this is for CI
Expand All @@ -124,34 +138,34 @@ push: push-controller-image push-agones-sdk-image
# Installs the current development version of Agones into the Kubernetes cluster
install: ALWAYS_PULL_SIDECAR := true
install: IMAGE_PULL_POLICY := "Always"
install: ensure-build-image
docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) \
helm upgrade --install --namespace=agones-system \
install: $(ensure-build-image)
$(DOCKER_RUN) \
helm upgrade --install --recreate-pods --wait --namespace=agones-system \
--set agones.image.tag=$(VERSION),agones.image.registry=$(REGISTRY),agones.image.controller.pullPolicy=$(IMAGE_PULL_POLICY),agones.image.sdk.alwaysPull=$(ALWAYS_PULL_SIDECAR) \
agones $(mount_path)/install/helm/agones/

# Build a static binary for the gameserver controller
build-controller-binary: ensure-build-image
build-controller-binary: $(ensure-build-image)
docker run --rm -e "CGO_ENABLED=0" $(common_mounts) $(build_tag) go build \
-o $(mount_path)/cmd/controller/bin/controller -a $(go_version_flags) -installsuffix cgo $(agones_package)/cmd/controller

# Lint the go source code.
# use LINT_TIMEOUT to manipulate the linter timeout
lint: LINT_TIMEOUT ?= 15m
lint: ensure-build-image
lint: $(ensure-build-image)
docker run --rm $(common_mounts) -w $(mount_path) $(DOCKER_RUN_ARGS) $(build_tag) bash -c \
"/root/gen-lint-exclude.sh && gometalinter --config .exclude.gometalinter.json --deadline=$(LINT_TIMEOUT) -t --skip vendor ./..."

# Build the image for the gameserver controller
build-controller-image: ensure-build-image build-controller-binary
build-controller-image: $(ensure-build-image) build-controller-binary
docker build $(agones_path)/cmd/controller/ --tag=$(controller_tag) $(DOCKER_BUILD_ARGS)

# push the gameservers controller image
push-controller-image: ensure-build-image
push-controller-image: $(ensure-build-image)
docker push $(controller_tag)

# build the static binary for the gamesever sidecar
build-agones-sdk-binary: ensure-build-image
build-agones-sdk-binary: $(ensure-build-image)
docker run --rm -e "CGO_ENABLED=0" $(common_mounts) $(build_tag) go build \
-o $(mount_path)/cmd/sdk-server/bin/sdk-server.linux.amd64 -a $(go_version_flags) -installsuffix cgo $(agones_package)/cmd/sdk-server
docker run --rm -e "GOOS=darwin" -e "GOARCH=amd64" $(common_mounts) $(build_tag) go build \
Expand All @@ -162,38 +176,38 @@ build-agones-sdk-binary: ensure-build-image
agonessdk-server-$(VERSION).zip sdk-server.darwin.amd64 sdk-server.linux.amd64 sdk-server.windows.amd64.exe

# Build the image for the gameserver sidecar
build-agones-sdk-image: ensure-build-image build-agones-sdk-binary
build-agones-sdk-image: $(ensure-build-image) build-agones-sdk-binary
docker build $(agones_path)/cmd/sdk-server/ --tag=$(sidecar_tag) $(DOCKER_BUILD_ARGS)

# Build the cpp sdk linux archive
build-sdk-cpp: ensure-build-image
build-sdk-cpp: $(ensure-build-image)
docker run --rm $(common_mounts) -w $(mount_path)/sdks/cpp $(build_tag) make build install archive VERSION=$(VERSION)

# push the gameservers sidecar image
push-agones-sdk-image: ensure-build-image
push-agones-sdk-image: $(ensure-build-image)
docker push $(sidecar_tag)

# Generate the static install script
gen-install: ensure-build-image
gen-install: $(ensure-build-image)
docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) bash -c \
'helm template --name=agones-manual --namespace agones-system $(mount_path)/install/helm/agones \
--set agones.controller.generateTLS=false \
> $(mount_path)/install/yaml/install.yaml'

# Generate the SDK gRPC server and client code
gen-gameservers-sdk-grpc: ensure-build-image
gen-gameservers-sdk-grpc: $(ensure-build-image)
docker run --rm $(common_mounts) -w $(mount_path) $(build_tag) /root/gen-grpc-go.sh
docker run --rm $(common_mounts) -w $(mount_path) $(build_tag) /root/gen-grpc-cpp.sh
docker run --rm $(common_mounts) -w $(mount_path) $(build_tag) /root/gen-grpc-rust.sh

# Generate the client for our CustomResourceDefinition
gen-crd-client: ensure-build-image
gen-crd-client: $(ensure-build-image)
docker run --rm $(common_mounts) -w $(mount_path) $(build_tag) /root/gen-crd-client.sh
docker run --rm $(common_mounts) -w $(mount_path)/pkg $(build_tag) goimports -w .

# Run a bash shell with the developer tools in it. (Creates the image if it doesn't exist)
# Can use DOCKER_RUN_ARGS for extra arguments.
shell: ensure-build-image
shell: $(ensure-build-image)
docker run -it --rm \
$(common_mounts) \
-w $(mount_path) \
Expand Down Expand Up @@ -286,7 +300,7 @@ gcloud-init: ensure-build-config
gcloud-test-cluster: GCP_CLUSTER_LEGACYABAC ?= false
gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4
gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= n1-standard-4
gcloud-test-cluster: ensure-build-image
gcloud-test-cluster: $(ensure-build-image)
docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) gcloud \
deployment-manager deployments create $(GCP_CLUSTER_NAME) \
--properties cluster.zone:$(GCP_CLUSTER_ZONE),cluster.name:$(GCP_CLUSTER_NAME),cluster.nodePool.initialNodeCount:$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT),cluster.nodePool.machineType:$(GCP_CLUSTER_NODEPOOL_MACHINETYPE),cluster.legacyAbac:$(GCP_CLUSTER_LEGACYABAC)\
Expand All @@ -295,21 +309,36 @@ gcloud-test-cluster: ensure-build-image
docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) kubectl apply -f $(mount_path)/build/helm.yaml
docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) helm init --service-account helm

clean-gcloud-test-cluster: ensure-build-image
clean-gcloud-test-cluster: $(ensure-build-image)
docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) gcloud \
deployment-manager deployments delete $(GCP_CLUSTER_NAME)


gcloud-e2e-test-cluster: $(ensure-build-image)
docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) gcloud \
deployment-manager deployments create e2e-test-cluster \
--config=$(mount_path)/build/gke-test-cluster/cluster-e2e.yml
GCP_CLUSTER_NAME=e2e-test-cluster GCP_CLUSTER_ZONE=us-west1-c $(MAKE) gcloud-auth-cluster
docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) \
kubectl apply -f $(mount_path)/build/helm.yaml
docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) helm init --service-account helm --wait && \
helm install --wait --set Replicas=1,uiService.type=ClusterIP --name consul stable/consul

clean-gcloud-e2e-test-cluster: $(ensure-build-image)
docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) \
helm delete --purge consul && kubectl delete pvc -l component=consul-consul
GCP_CLUSTER_NAME=e2e-test-cluster $(MAKE) clean-gcloud-test-cluster

# Pulls down authentication information for kubectl against a cluster, name can be specified through GCP_CLUSTER_NAME
# (defaults to 'test-cluster')
gcloud-auth-cluster: ensure-build-image
gcloud-auth-cluster: $(ensure-build-image)
docker run --rm $(common_mounts) $(build_tag) gcloud config set container/cluster $(GCP_CLUSTER_NAME)
docker run --rm $(common_mounts) $(build_tag) gcloud config set compute/zone $(GCP_CLUSTER_ZONE)
docker run --rm $(common_mounts) $(build_tag) gcloud container clusters get-credentials $(GCP_CLUSTER_NAME)
-docker run --rm $(common_mounts) $(build_tag) bash -c 'kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user $$(gcloud config get-value account)'

# authenticate our docker configuration so that you can do a docker push directly
# to the gcr.io repository
gcloud-auth-docker: ensure-build-image
gcloud-auth-docker: $(ensure-build-image)
docker run --rm $(common_mounts) $(build_tag) gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://gcr.io

# Clean the gcloud configuration
Expand All @@ -328,7 +357,7 @@ clean-gcloud-config:
#
# Use MINIKUBE_DRIVER variable to change the VM driver
# (defaults virtualbox for Linux and macOS, hyperv for windows) if you so desire.
minikube-test-cluster: ensure-build-image minikube-agones-profile
minikube-test-cluster: $(ensure-build-image) minikube-agones-profile
# localkube bootstrapper fixes issues with profiles
$(MINIKUBE) start --kubernetes-version v1.10.0 --vm-driver $(MINIKUBE_DRIVER) \
--bootstrapper=localkube \
Expand All @@ -355,7 +384,7 @@ minikube-agones-profile:

# Connecting to minikube requires so enhanced permissions, so use this target
# instead of `make shell` to start an interactive shell for development on minikube.
minikube-shell: ensure-build-image minikube-agones-profile
minikube-shell: $(ensure-build-image) minikube-agones-profile
$(MAKE) shell DOCKER_RUN_ARGS="--network=host -v $(minikube_cert_mount) $(DOCKER_RUN_ARGS)"

# Push the local Agones Docker images that have already been built
Expand Down
32 changes: 25 additions & 7 deletions build/build-image/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# ForceUpdate 5 -- change here if you need to force a rebuild
# ForceUpdate 6 -- change here if you need to force a rebuild

# compiling proto + grpc takes an exceptionally long time
# so we'll use a base from `base` - which is manually built using the below tag.
FROM gcr.io/agones-images/grpc-cxx:1.12

RUN apt-get update && \
apt-get install -y wget rsync make python bash-completion zip nano jq && \
apt-get install -y wget psmisc rsync make python bash-completion zip nano jq && \
apt-get clean

# install go
Expand All @@ -39,17 +39,22 @@ RUN wget -q https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.zip && un
ENV PATH /usr/local/go/bin:/go/bin:/opt/google-cloud-sdk/bin:$PATH

# RUN gcloud components update
RUN gcloud components update && gcloud components install kubectl
RUN gcloud components update
# install kubectl without gcloud as we want the last version
ENV KUBECTL_VER 1.11.0
RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VER}/bin/linux/amd64/kubectl && \
chmod go+rx ./kubectl && \
mv ./kubectl /usr/local/bin/kubectl
RUN echo "source <(kubectl completion bash)" >> /root/.bashrc

# install Helm package manager
ENV HELM_VER 2.9.1
ENV HELM_URL https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VER}-linux-amd64.tar.gz
RUN curl -L ${HELM_URL} > /tmp/helm.tar.gz \
&& tar -zxvf /tmp/helm.tar.gz -C /tmp \
&& mv /tmp/linux-amd64/helm /usr/local/bin/helm \
&& chmod go+rx /usr/local/bin/helm \
&& rm /tmp/helm.tar.gz && rm -rf /tmp/linux-amd64
&& tar -zxvf /tmp/helm.tar.gz -C /tmp \
&& mv /tmp/linux-amd64/helm /usr/local/bin/helm \
&& chmod go+rx /usr/local/bin/helm \
&& rm /tmp/helm.tar.gz && rm -rf /tmp/linux-amd64
RUN echo "source <(helm completion bash)" >> /root/.bashrc

# install go-proto-gen 1.1
Expand Down Expand Up @@ -87,6 +92,19 @@ RUN wget -q https://static.rust-lang.org/rustup/archive/1.11.0/${RUST_ARCH}/rust
RUN cargo install protobuf-codegen --vers 2.0.2
RUN cargo install grpcio-compiler --vers 0.3.0

# set up Consul.
ENV CONSUL_VERSION=1.2.1
ENV HASHICORP_RELEASES=https://releases.hashicorp.com
RUN mkdir -p /tmp/build && \
wget ${HASHICORP_RELEASES}/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip \
-P /tmp/build/ && \
unzip -d /usr/local/bin/ /tmp/build/consul_${CONSUL_VERSION}_linux_amd64.zip && \
cd /tmp && \
rm -rf /tmp/build && \
chmod go+rx /usr/local/bin/consul && \
# tiny smoke test to ensure the binary we downloaded runs
consul version

# make sure we keep the path to go
RUN echo "export PATH=/usr/local/go/bin:/go/bin/:\$PATH" >> /root/.bashrc
# make nano the editor
Expand Down
52 changes: 52 additions & 0 deletions build/e2e-image/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
FROM gcr.io/cloud-builders/gcloud-slim

RUN apt-get update && \
apt-get install -y wget psmisc make python jq zip && \
apt-get clean

# install go
WORKDIR /usr/local
ENV GO_VERSION=1.10.3
ENV GOPATH /go
RUN wget -q https://redirector.gvt1.com/edgedl/go/go${GO_VERSION}.linux-amd64.tar.gz && \
tar -xzf go${GO_VERSION}.linux-amd64.tar.gz && rm go${GO_VERSION}.linux-amd64.tar.gz && mkdir ${GOPATH}

ENV PATH /usr/local/go/bin:/go/bin:$PATH

# install kubectl without gcloud as we want the last version
ENV KUBECTL_VER 1.11.0
RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VER}/bin/linux/amd64/kubectl && \
chmod go+rx ./kubectl && \
mv ./kubectl /usr/local/bin/kubectl

# install Helm package manager
ENV HELM_VER 2.9.1
ENV HELM_URL https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VER}-linux-amd64.tar.gz
RUN curl -L ${HELM_URL} > /tmp/helm.tar.gz \
&& tar -zxvf /tmp/helm.tar.gz -C /tmp \
&& mv /tmp/linux-amd64/helm /usr/local/bin/helm \
&& chmod go+rx /usr/local/bin/helm \
&& rm /tmp/helm.tar.gz && rm -rf /tmp/linux-amd64

# set up Consul.
ENV CONSUL_VERSION=1.2.1
ENV HASHICORP_RELEASES=https://releases.hashicorp.com
RUN mkdir -p /tmp/build && \
wget ${HASHICORP_RELEASES}/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip \
-P /tmp/build/ && \
unzip -d /usr/local/bin/ /tmp/build/consul_${CONSUL_VERSION}_linux_amd64.zip && \
cd /tmp && \
rm -rf /tmp/build && \
chmod go+rx /usr/local/bin/consul && \
# tiny smoke test to ensure the binary we downloaded runs
consul version

# make sure we keep the path to go
RUN echo "export PATH=/usr/local/go/bin:/go/bin/:\$PATH" >> /root/.bashrc
# scripts
COPY *.sh /root/
RUN chmod +x /root/*.sh

WORKDIR /go

ENTRYPOINT [ "/root/entrypoint.sh" ]
19 changes: 19 additions & 0 deletions build/e2e-image/e2e.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/usr/bin/env bash

# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
echo "installing current release"
DOCKER_RUN= make install
echo "starting e2e test"
DOCKER_RUN= make test-e2e
35 changes: 35 additions & 0 deletions build/e2e-image/entrypoint.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
#!/usr/bin/env bash

# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e

export SHELL="/bin/bash"
export KUBECONFIG="/root/.kube/config"
mkdir -p /go/src/agones.dev/agones/ /root/.kube/
cp -r /workspace/. /go/src/agones.dev/agones/
cd /go/src/agones.dev/agones/build
if [ "$1" = 'local' ]
then
gcloud auth login
fi
gcloud container clusters get-credentials e2e-test-cluster \
--zone=us-west1-c --project=agones-images
kubectl port-forward statefulset/consul-consul 8500:8500 &
echo "Waiting consul port-forward to launch on 8500..."
timeout 60 bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' 127.0.0.1 8500
echo "consul port-forward launched. Starting e2e tests..."
consul lock -child-exit-code=true -timeout 5m -try 5m -verbose LockE2E /root/e2e.sh
killall -q kubectl

Loading

0 comments on commit 9e1598b

Please sign in to comment.