CRD OpenAPI Spec for ObjectMeta & PodTemplateSpec (#1956)

* CRD OpenAPI Spec for ObjectMeta & PodTemplateSpec

This PR provides the mechanisms to pull core Kubernetes resource OpenAPI
schemas out of the API, and customise and convert them into a format
that is embeddable in our own CRD specifications.

This has been used to extract OpenAPI schemas for `ObjectMeta`, and
`PodTemplateSpec`, and then embedded in `GameServer` as well as parent
CRDs, creating a layer of validation for Pods definitions that are
implemented via a GameServer.

Also included in a new helm config option of
`gameservers.podPreserveUnknownFields` in case a user needs to escape
the field pruning on a Pod, or if PR causes a bug of any kind.

Also worth noting, this is not comprehensive validation, and therefore I
have not closed the the referenced issue.

Work on #1298

* Generated yaml for Schema improvements

* Upgrade Helm to 3.5.0

Needed upgrade to fix bug wth upgrade.

* Updates based on review

- Fix for typo in license.
- Fix comment in test to be more clear.
- Rebase against master and regen index.yaml

build/Makefile

@@ -468,6 +468,10 @@ endif
 push-agones-sdk-linux-image: $(ensure-build-image)
 	docker push $(sidecar_linux_amd64_tag)
 
+# Generate the Embedded CRD openapi
+gen-embedded-openapi:
+	docker run --rm $(common_mounts) -w $(workdir_path)/build $(build_tag) ./export-openapi.sh
+
 # Generate the static install script
 gen-install: $(ensure-build-image)
 	docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) bash -c \

build/README.md

@@ -57,6 +57,7 @@ Table of Contents
         * [make build-controller-image](#make-build-controller-image)
         * [make build-agones-sdk-image](#make-build-agones-sdk-image)
         * [make gen-install](#make-gen-install)
+        * [make gen-embedded-openapi](#make-gen-embedded-openapi)  
         * [make gen-crd-client](#make-gen-crd-client)
         * [make gen-sdk-grpc](#make-gen-sdk-grpc)
      * [Build Image Targets](#build-image-targets)
@@ -568,6 +569,9 @@ Compile the ping binary and then build the docker image
 #### `make gen-install`
 Generate the `/install/yaml/install.yaml` from the Helm template
 
+#### `gen-embedded-openapi`
+Generate the embedded OpenAPI specs for existing Kubernetes Objects, such as `PodTemplateSpec` and `ObjectMeta`
+
 #### `make gen-crd-client`
 Generate the Custom Resource Definition client(s)
 

build/boilerplate.yaml.txt

@@ -0,0 +1,16 @@
+# Copyright 2021 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This code was autogenerated. Do not edit directly.
+

build/build-image/Dockerfile

@@ -58,7 +58,7 @@ RUN mkdir -p /go/src/k8s.io && cd /go/src/k8s.io && \
     git clone -b kubernetes-${KUBERNETES_VER} --depth=3 https://github.com/kubernetes/code-generator.git
 
 # install Helm package manager
-ENV HELM_VER 3.2.3
+ENV HELM_VER 3.5.0
 ENV HELM_URL https://get.helm.sh/helm-v${HELM_VER}-linux-amd64.tar.gz
 RUN curl -L  ${HELM_URL} > /tmp/helm.tar.gz \
     && tar -zxvf /tmp/helm.tar.gz -C /tmp \

build/e2e-image/Dockerfile

@@ -21,7 +21,7 @@ RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v${KUBECT
     mv ./kubectl /usr/local/bin/kubectl
 
 # install Helm package manager
-ENV HELM_VER 3.2.3
+ENV HELM_VER 3.5.0
 ENV HELM_URL https://get.helm.sh/helm-v${HELM_VER}-linux-amd64.tar.gz
 RUN curl -L  ${HELM_URL} > /tmp/helm.tar.gz \
     && tar -zxvf /tmp/helm.tar.gz -C /tmp \

build/export-openapi.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+#
+# Copyright 2021 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+do_expand() {
+  echo "Processing $1"
+  jq '.definitions."'"$1"'"' ./openapi.json >"$1.json"
+  children=$(jq -r '..|objects|.["$ref"]|select (.!=null)' "$1.json" | sed 's!#/definitions/!!')
+  if [ -n "$children" ]; then
+    while IFS= read -r line; do
+      do_expand "$line"
+    done <<<"$children"
+  fi
+}
+
+json_2_helm_yaml() {
+  yaml="_$1.yaml"
+  echo "Converting to YAML: $1"
+  ./yq eval -P "$1.json" >"$yaml"
+
+  echo "{{- define \"$1\" }}" | cat ../boilerplate.yaml.txt - "$yaml" | sponge "$yaml"
+  echo "{{- end }}" >>"$yaml"
+  mv "$yaml" ../../install/helm/agones/templates/crds/k8s/
+}
+
+rm -r ./tmp || true
+mkdir tmp
+cd tmp
+kubectl proxy &
+
+# install deps while waiting for kubectl proxy to startup
+wget https://github.com/mikefarah/yq/releases/download/v4.3.2/yq_linux_amd64 -O yq
+apt install -y moreutils
+chmod +x ./yq
+
+# cleanup and format
+curl http://127.0.0.1:8001/openapi/v2 | jq 'del(.. | .["x-kubernetes-patch-strategy"]?, .["x-kubernetes-patch-merge-key"]?, .["x-kubernetes-list-type"]?)' |
+  jq 'del(.. |  .["x-kubernetes-group-version-kind"]?, .["x-kubernetes-list-map-keys"]? )' >openapi.json
+do_expand "io.k8s.api.core.v1.PodTemplateSpec"
+
+# do any editing you need to do to any types here, before they are expanded.
+# creationTimestamp is defaulted to it's zero value, which gets serialised as "null", so we have to set it as nullable.
+jq '.properties.creationTimestamp.nullable = true' io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta.json | sponge io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta.json
+# Make IntOrString type have `x-kubernetes-int-or-string` member
+jq '.["x-kubernetes-int-or-string"] = true' io.k8s.apimachinery.pkg.util.intstr.IntOrString.json | jq 'del(.type)' | sponge io.k8s.apimachinery.pkg.util.intstr.IntOrString.json
+
+# easier debugging if something goes wrong, still have the original
+mkdir orig
+cp *.json orig
+rm openapi.json
+
+for f in *.json; do
+  echo "Expanding $f"
+  # remove description, because there is usually another description when at its replacement point
+  # replace \" with \\" as it get re-unescaped somewhere in the pipe
+  # any "foo\nbar" values need their \n escaped
+  # remove top and bottom line to get rid of first { and last } (we know all are formatted because jq)
+  # then format for multiline - replace real multilines breaks with \n
+  contents=$(cat "$f" | jq 'del(.description)' | sed 's/\\n/\\\\n/g' | sed '$d' | sed '1d' | sed ':a;N;$!ba;s/\n/\\n/g' | sed 's/\$/\\$/g' | sed 's@\"@\\"@g')
+  ref=$(basename "$f" .json)
+
+  find -maxdepth 1 -name '*.json' | xargs sed -i 's@"$ref": "#/definitions/'"$ref"'"@'"$contents"'@g'
+done
+
+# convert the ones you want to include via helm to yaml
+json_2_helm_yaml "io.k8s.api.core.v1.PodTemplateSpec"
+json_2_helm_yaml "io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"

install/helm/agones/templates/crds/_gameserverspecschema.yaml

@@ -20,51 +20,18 @@ required:
 properties:
   {{- if .metadata | default false }}
   metadata:
-    type: object
-    properties:
-      labels:
-        type: object
-        additionalProperties:
-          type: string
-      annotations:
-        type: object
-        additionalProperties:
-          type: string
+    {{- include "io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta" . | indent 4 }}
   {{- end}}
   spec:
     type: object
     required:
     - template
     properties:
       template:
-        type: object
+        {{- if .podPreserveUnknownFields }}
         x-kubernetes-preserve-unknown-fields: true
-        required:
-        - spec
-        properties:
-          spec:
-            type: object
-            x-kubernetes-preserve-unknown-fields: true
-            required:
-            - containers
-            properties:
-              containers:
-                type: array
-                minItems: 1
-                items:
-                  type: object
-                  x-kubernetes-preserve-unknown-fields: true
-                  required:
-                  - image
-                  properties:
-                    name:
-                      type: string
-                      minLength: 0
-                      maxLength: 63
-                      pattern: "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$"
-                    image:
-                      type: string
-                      minLength: 1
+        {{- end }}
+        {{- include "io.k8s.api.core.v1.PodTemplateSpec" . | indent 8 }}
       container:
         title: The container name running the gameserver
         description: if there is more than one container, specify which one is the game server

install/helm/agones/templates/crds/fleet.yaml

@@ -96,7 +96,7 @@ spec:
                             - type: integer
                             - type: string
                 template:
-                  {{- $data := dict "metadata" true}}
+                  {{- $data := dict "metadata" true "podPreserveUnknownFields" .Values.gameservers.podPreserveUnknownFields }}
                   {{- include "gameserver.schema" $data | indent 17 }}
             status:
               type: object

install/helm/agones/templates/crds/gameserver.yaml

@@ -55,6 +55,7 @@ spec:
           type: date
       schema:
         openAPIV3Schema:
+          {{- $data := dict "podPreserveUnknownFields" .Values.gameservers.podPreserveUnknownFields }}
           {{- include "gameserver.schema" . | indent 9 }}
           {{- include "gameserver.status" . | indent 11 }} # in an include, as it's easier to align
 {{- end }}

install/helm/agones/templates/crds/gameserverset.yaml

@@ -76,7 +76,7 @@ spec:
                     - Packed
                     - Distributed
                 template:
-                  {{- $data := dict "metadata" true}}
+                  {{- $data := dict "metadata" true "podPreserveUnknownFields" .Values.gameservers.podPreserveUnknownFields }}
                   {{- include "gameserver.schema" $data | indent 18 }}
             status:
               type: object