Implement GameServerAlocation as API Extension

This moves the implementation of GameServerAllocation (GSA) to a [Kubernetes API Extension](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/apiserver-aggregation/) instead of using CRDs. This was essentially done for performance reasons, but to break it down: 1. GSA is now create only. Since we had no need for GSA storage, and don't want the performance hit. 1. This removes the mutation and validation webhooks, which have 30s timeout and are run in serial 1. API Server still does cut off a response after 60s, but the api can continue processing (60s gives us enough time, I think for a SDK.Ack() on Allocate, which I don't think we had before) 1. Validation now happens in the request. 1. We can now do batching of requests for higher throughput (#536), since we control the entire http request. The breaking changes are: 1. GameServerAllocation's group is now `allocation.agones.dev` rather than `stable.agones.dev`, because a CRD group can't overlap with a api server. 1. Since there is only the `create` verb for GSA, there is no get/list/watch options for GameServerAllocations - so no informers/listers either. But this could be added at a later date, if needed.
googleforgames · Mar 31, 2019 · ebabf1c · ebabf1c
1 parent 2165b38
commit ebabf1c
Show file tree

Hide file tree

Showing 63 changed files with 1,331 additions and 1,520 deletions.
diff --git a/build/build-image/Dockerfile b/build/build-image/Dockerfile
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# ForceUpdate 6 -- change here if you need to force a rebuild
+# ForceUpdate 7 -- change here if you need to force a rebuild
 
 # compiling proto + grpc takes an exceptionally long time
 # so we'll use a base from `base` - which is manually built using the below tag.
@@ -140,11 +140,12 @@ RUN echo "export PATH=/usr/local/go/bin:/go/bin/:\$PATH" >> /root/.bashrc
 # make nano the editor
 RUN echo "export EDITOR=nano" >> /root/.bashrc
 
+# update CMake
+RUN wget https://github.com/Kitware/CMake/releases/download/v3.14.0/cmake-3.14.0-Linux-x86_64.sh && mkdir /opt/cmake && \
+    sh ./cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/opt/cmake && export PATH=$PATH:/opt/cmake/bin
+
 # code generation scripts
 COPY *.sh /root/
 RUN chmod +x /root/*.sh
 
-RUN wget https://github.com/Kitware/CMake/releases/download/v3.14.0/cmake-3.14.0-Linux-x86_64.sh && mkdir /opt/cmake && \
-    sh ./cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/opt/cmake && export PATH=$PATH:/opt/cmake/bin
-
 WORKDIR /go
diff --git a/build/build-image/gen-crd-client.sh b/build/build-image/gen-crd-client.sh
@@ -14,9 +14,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -x
+
+# not sure why the modules break things, but they do.
+export GO111MODULE=off
+rm -r $GOPATH/src/agones.dev/agones/pkg/client
 rsync -r /go/src/agones.dev/agones/vendor/k8s.io/ /go/src/k8s.io/
 cd /go/src/k8s.io/code-generator
 ./generate-groups.sh "all" \
     agones.dev/agones/pkg/client \
-    agones.dev/agones/pkg/apis stable:v1alpha1 \
+    agones.dev/agones/pkg/apis "allocation:v1alpha1 stable:v1alpha1" \
     --go-header-file=/go/src/agones.dev/agones/build/boilerplate.go.txt
diff --git a/cmd/controller/main.go b/cmd/controller/main.go
@@ -142,9 +142,7 @@ func main() {
 	// https server and the items that share the Mux for routing
 	httpsServer := https.NewServer(ctlConf.CertFile, ctlConf.KeyFile)
 	wh := webhooks.NewWebHook(httpsServer.Mux)
-	// will register openapi endpoint, which is currently not used
-	// but gets the code ready for usage in a later PR.
-	_ = apiserver.NewAPIServer(httpsServer.Mux)
+	api := apiserver.NewAPIServer(httpsServer.Mux)
 
 	agonesInformerFactory := externalversions.NewSharedInformerFactory(agonesClient, defaultResync)
 	kubeInformerFactory := informers.NewSharedInformerFactory(kubeClient, defaultResync)
@@ -201,8 +199,8 @@ func main() {
 	fleetController := fleets.NewController(wh, health, kubeClient, extClient, agonesClient, agonesInformerFactory)
 	faController := fleetallocation.NewController(wh, allocationMutex,
 		kubeClient, extClient, agonesClient, agonesInformerFactory)
-	gasController := gameserverallocations.NewController(wh, health, gsCounter, topNGSForAllocation,
-		kubeClient, extClient, agonesClient, agonesInformerFactory)
+	gasController := gameserverallocations.NewController(api, health, gsCounter, topNGSForAllocation,
+		kubeClient, agonesClient, agonesInformerFactory)
 	fasController := fleetautoscalers.NewController(wh, health,
 		kubeClient, extClient, agonesClient, agonesInformerFactory)
 

diff --git a/examples/gameserverallocation.yaml b/examples/gameserverallocation.yaml
@@ -20,12 +20,8 @@
 # multiple Fleets, or a self managed group of GameServers.
 #
 
-apiVersion: "stable.agones.dev/v1alpha1"
+apiVersion: "allocation.agones.dev/v1alpha1"
 kind: GameServerAllocation
-metadata:
-  # We recommend using the following to generate a unique name when creating Allocations
-  # This will need to be created with `kubectl create` if using the command line tooling
-  generateName: simple-udp-
 spec:
   # GameServer selector from which to choose GameServers from.
   # GameServers still have the hard requirement to be `Ready` to be allocated from

diff --git a/examples/simple-udp/gameserverallocation.yaml b/examples/simple-udp/gameserverallocation.yaml
@@ -12,14 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+#
+# A GameServerAllocation against a Fleet named 'simple-udp'
 # (GameServerAllocation is currently experimental, and likely to change in upcoming releases)
+#
 
-apiVersion: "stable.agones.dev/v1alpha1"
+apiVersion: "allocation.agones.dev/v1alpha1"
 kind: GameServerAllocation
-metadata:
-  # We recommend using the following to generate a unique name when creating Allocations
-  # This will need to be created with `kubectl create` if using the command line tooling
-  generateName: simple-udp-
 spec:
   # GameServer selector from which to choose GameServers from.
   # GameServers still have the hard requirement to be `Ready` to be allocated from

diff --git a/examples/xonotic/gameserverallocation.yaml b/examples/xonotic/gameserverallocation.yaml
@@ -14,12 +14,8 @@
 
 # (GameServerAllocation is currently experimental, and likely to change in upcoming releases)
 
-apiVersion: "stable.agones.dev/v1alpha1"
+apiVersion: "allocation.agones.dev/v1alpha1"
 kind: GameServerAllocation
-metadata:
-  # We recommend using the following to generate a unique name when creating Allocations
-  # This will need to be created with `kubectl create` if using the command line tooling
-  generateName: xonotic-
 spec:
   # GameServer selector from which to choose GameServers from.
   # GameServers still have the hard requirement to be `Ready` to be allocated from

diff --git a/install/helm/agones/scripts/delete_agones_resources.sh b/install/helm/agones/scripts/delete_agones_resources.sh
@@ -32,7 +32,6 @@ for ns in $namespaces; do
   kubectl -n $ns delete gameserversets --all
   kubectl -n $ns delete gameservers --all
   kubectl -n $ns delete fleetallocations --all
-  kubectl -n $ns delete gameserverallocations --all
 
   # Since we don't have the nifty kubectl wait yet, hack one in the meantime
   for p in $pods; do

diff --git a/install/helm/agones/templates/crds/_selector.yaml b/install/helm/agones/templates/crds/_selector.yaml
diff --git a/install/helm/agones/templates/crds/gameserverallocation.yaml b/install/helm/agones/templates/crds/gameserverallocation.yaml
diff --git a/...ones/templates/admissionregistration.yaml → ...all/helm/agones/templates/extensions.yaml b/...ones/templates/admissionregistration.yaml → ...all/helm/agones/templates/extensions.yaml
@@ -16,12 +16,44 @@
 {{- $altName1 := printf "agones-controller-service.%s"  .Release.Namespace }}
 {{- $altName2 := printf "agones-controller-service.%s.svc" .Release.Namespace }}
 {{- $cert := genSignedCert $cn nil (list $altName1 $altName2) 3650 $ca }}
+---
+{{- if .Values.agones.registerApiService }}
+apiVersion: apiregistration.k8s.io/v1beta1
+kind: APIService
+metadata:
+  name: v1alpha1.allocation.agones.dev
+  labels:
+    component: controller
+    app: {{ template "agones.name" . }}
+    chart: {{ template "agones.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  group: allocation.agones.dev
+  groupPriorityMinimum: 1000
+  versionPriority: 15
+  service:
+    name: agones-controller-service
+    namespace: {{ .Release.Namespace }}
+        {{- if .Values.agones.controller.generateTLS }}
+  caBundle: {{ b64enc $ca.Cert }}
+        {{- else }}
+  caBundle: {{ .Files.Get "certs/server.crt" | b64enc }}
+        {{- end }}
+  version: v1alpha1
+{{- end}}
 {{- if .Values.agones.registerWebhooks }}
 ---
 apiVersion: admissionregistration.k8s.io/v1beta1
 kind: ValidatingWebhookConfiguration
 metadata:
   name: agones-validation-webhook
+  labels:
+    component: controller
+    app: {{ template "agones.name" . }}
+    chart: {{ template "agones.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
 webhooks:
   - name: validations.stable.agones.dev
     failurePolicy: Fail
@@ -55,7 +87,6 @@ webhooks:
           - "gameserversets"
           - "fleetallocations"
           - "fleetautoscalers"
-          - "gameserverallocations"
         apiVersions:
           - "v1alpha1"
         operations:
@@ -91,7 +122,6 @@ webhooks:
           - "gameservers"
           - "fleets"
           - "fleetallocations"
-          - "gameserverallocations"
         apiVersions:
           - "v1alpha1"
         operations:

diff --git a/install/helm/agones/templates/hooks/sa.yaml b/install/helm/agones/templates/hooks/sa.yaml
@@ -45,7 +45,7 @@ metadata:
     "helm.sh/hook-delete-policy": before-hook-creation
 rules:
 - apiGroups: ["stable.agones.dev"]
-  resources: ["fleets", "fleetallocations", "fleetautoscalers", "gameservers", "gameserversets", "gameserverallocations"]
+  resources: ["fleets", "fleetallocations", "fleetautoscalers", "gameservers", "gameserversets"]
   verbs: ["delete", "list" ]
 - apiGroups: [""]
   resources: ["pods"]

diff --git a/install/helm/agones/templates/serviceaccounts/controller.yaml b/install/helm/agones/templates/serviceaccounts/controller.yaml
@@ -64,9 +64,6 @@ rules:
 - apiGroups: ["stable.agones.dev"]
   resources: ["fleets/status"]
   verbs: ["update"]
-- apiGroups: ["stable.agones.dev"]
-  resources: ["gameserverallocations"]
-  verbs: ["list", "watch",  "delete"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -87,4 +84,34 @@ roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
   name: {{ .Values.agones.serviceaccount.controller }}
+---
+#
+# RBACs for APIService
+#
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Values.agones.serviceaccount.controller }}:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+  - kind: ServiceAccount
+    name: apiserver
+    namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Values.agones.serviceaccount.controller }}-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: apiserver
+    namespace: {{ .Release.Namespace }}
 {{- end }}
diff --git a/install/helm/agones/values.yaml b/install/helm/agones/values.yaml
@@ -23,6 +23,7 @@ agones:
   rbacEnabled: true
   registerServiceAccounts: true
   registerWebhooks: true
+  registerApiService: true
   crds:
     install: true
     cleanupOnDelete: true