Add Cloudbuild step to run performance test by using the scenario tes…

…t framework. (#3429)

* Cloud build to run performance test

* Add performance test fleet autoscaler template files

* fix lint

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* Increase timeout

* fix a typo

build/includes/build-image.mk

@@ -51,6 +51,9 @@ endif
 pull-build-image:
 	$(MAKE) pull-remote-build-image REMOTE_TAG=$(build_remote_tag) LOCAL_TAG=$(build_tag)
 
+pull-build-image-with-tag:
+	$(MAKE) pull-remote-build-image REMOTE_TAG=$(build_remote_tag) LOCAL_TAG=$(CUSTOM_LOCAL_TAG)
+
 # push the local build image up to your repository
 push-build-image:
 	$(MAKE) push-remote-build-image REMOTE_TAG=$(build_remote_tag) LOCAL_TAG=$(build_tag)

build/performance-test.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+# Copyright 2023 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -e
+CLUSTER_NAME=$1
+CLUSTER_LOCATION=$2
+REGISTRY=$3
+PROJECT=$4
+REPLICAS=$5
+AUTO_SHUTDOWN_DELAY=$6
+BUFFER_SIZE=$7
+MAX_REPLICAS=$8
+DURATION=$9
+CLIENTS=${10}
+INTERVAL=${11}
+
+export SHELL="/bin/bash"
+mkdir -p /go/src/agones.dev/
+ln -s /workspace /go/src/agones.dev/agones
+cd /go/src/agones.dev/agones/build
+
+gcloud config set project $PROJECT
+gcloud container clusters get-credentials $CLUSTER_NAME \
+        --zone=$CLUSTER_LOCATION --project=$PROJECT
+
+DOCKER_RUN= make install REGISTRY='"'$REGISTRY'"' 
+
+cd /go/src/agones.dev/agones/test/load/allocation
+
+# use the input values to populate the yaml files for fleet and autoscaler, and then apply them
+cp performance-test-fleet-template.yaml performance-test-fleet.yaml
+cp performance-test-autoscaler-template.yaml performance-test-autoscaler.yaml
+cp performance-test-variable-template.txt performance-test-variable.txt
+
+sed -i 's/REPLICAS_REPLACEMENT/'$REPLICAS'/g' performance-test-fleet.yaml
+sed -i 's/AUTOMATIC_SHUTDOWN_DELAY_SEC_REPLACEMENT/'$AUTO_SHUTDOWN_DELAY'/g' performance-test-fleet.yaml
+
+sed -i 's/BUFFER_SIZE_REPLACEMENT/'$BUFFER_SIZE'/g' performance-test-autoscaler.yaml
+sed -i 's/MIN_REPLICAS_REPLACEMENT/'$REPLICAS'/g' performance-test-autoscaler.yaml
+sed -i 's/MAX_REPLICAS_REPLACEMENT/'$MAX_REPLICAS'/g' performance-test-autoscaler.yaml
+
+sed -i 's/DURATION_REPLACEMENT/'$DURATION'/g' performance-test-variable.txt
+sed -i 's/CLIENTS_REPLACEMENT/'$CLIENTS'/g' performance-test-variable.txt
+sed -i 's/INTERVAL_REPLACEMENT/'$INTERVAL'/g' performance-test-variable.txt
+
+kubectl apply -f performance-test-fleet.yaml
+kubectl apply -f performance-test-autoscaler.yaml
+
+# wait for the fleet to be ready
+while [ $(kubectl get -f performance-test-fleet.yaml -o=jsonpath='{.spec.replicas}') != $(kubectl get -f performance-test-fleet.yaml -o=jsonpath='{.status.readyReplicas}') ]
+do
+    sleep 1
+done
+
+cat performance-test-fleet.yaml
+cat performance-test-autoscaler.yaml
+cat performance-test-variable.txt
+
+printf "\nStart testing."
+./runScenario.sh performance-test-variable.txt
+printf "\nFinish testing."
+
+rm performance-test-fleet.yaml performance-test-autoscaler.yaml performance-test-variable.txt

ci/perf-test-cloudbuild.yaml

@@ -0,0 +1,82 @@
+---
+# Copyright 2023 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Google Cloud Builder CI configuration
+#
+steps:
+  #
+  # Creates the initial make + docker build platform
+  #
+  - name: ubuntu
+    args:
+      - bash
+      - -c
+      - "echo 'FROM gcr.io/cloud-builders/docker\nRUN apt-get install make\nENTRYPOINT\
+        \ [\"/usr/bin/make\"]' > Dockerfile.build"
+    waitFor: ['-']
+
+  - name: gcr.io/cloud-builders/docker
+    id: build-make-docker
+    args: [build, -f, Dockerfile.build, -t, make-docker, .]  # we need docker and make to run everything.
+
+  #
+  # pull the main build image if it exists
+  #
+  - name: make-docker
+    id: pull-build-image
+    dir: build
+    env: ['REGISTRY=${_REGISTRY}', 'CUSTOM_LOCAL_TAG=performance-test']
+    args: [pull-build-image-with-tag]
+
+  #
+  # Run the performance tests with default feature gates setting
+  #
+  - name: performance-test
+    dir: build
+    args:
+      - ./performance-test.sh
+      - ${_TEST_CLUSTER_NAME}
+      - ${_TEST_CLUSTER_LOCATION}
+      - ${_REGISTRY}
+      - ${_TEST_PROJECT_ID}
+      - ${_TEST_FLEET_REPLICAS}
+      - ${_TEST_AUTOMATIC_SHUTDOWN_DELAY_SEC}
+      - ${_TEST_BUFFER_SIZE}
+      - ${_TEST_MAX_REPLICAS}
+      - ${_TEST_DURATION}
+      - ${_TEST_CLIENTS}
+      - ${_TEST_INTERVAL}
+    id: performance-test
+    waitFor:
+      - pull-build-image
+
+substitutions:
+  _TEST_CLUSTER_NAME: standard-performance-test-cluster-1-27
+  _TEST_CLUSTER_LOCATION: us-central1
+  _REGISTRY: us-docker.pkg.dev/agones-images/ci
+  _TEST_PROJECT_ID: agones-images
+  _TEST_FLEET_REPLICAS: "10000"
+  _TEST_AUTOMATIC_SHUTDOWN_DELAY_SEC: "60"
+  _TEST_BUFFER_SIZE: "9900"
+  _TEST_MAX_REPLICAS: "20000"
+  _TEST_DURATION: "10m"
+  _TEST_CLIENTS: "50"
+  _TEST_INTERVAL: "1000"
+tags: [ci, 'commit-${COMMIT_SHA}']
+timeout: 7200s  # 2h
+queueTtl: 10800s  # 3h
+options:
+  machineType: E2_HIGHCPU_32
+  dynamic_substitutions: true

test/load/allocation/.gitignore

@@ -2,4 +2,5 @@
 *.crt
 *.txt
 !fixed.txt
-!variable.txt
+!variable.txt
+!performance-test-variable-template.txt

test/load/allocation/performance-test-autoscaler-template.yaml

@@ -0,0 +1,26 @@
+---
+# Copyright 2023 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: autoscaling.agones.dev/v1
+kind: FleetAutoscaler
+metadata:
+  name: performance-test-fleet-autoscaler
+spec:
+  fleetName: performance-test-fleet
+  policy:
+    type: Buffer
+    buffer:
+      bufferSize: BUFFER_SIZE_REPLACEMENT
+      minReplicas: MIN_REPLICAS_REPLACEMENT
+      maxReplicas: MAX_REPLICAS_REPLACEMENT

test/load/allocation/performance-test-fleet-template.yaml

@@ -0,0 +1,44 @@
+---
+# Copyright 2020 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: agones.dev/v1
+kind: Fleet
+metadata:
+  name: performance-test-fleet
+spec:
+  # the number of GameServers to keep Ready or Allocated in this Fleet
+  replicas: REPLICAS_REPLACEMENT
+  template:
+    metadata:
+      labels:
+        foo: bar
+    # GameServer specification
+    spec:
+      ports:
+        - containerPort: 7654
+          name: default
+      # The GameServer's Pod template
+      template:
+        spec:
+          containers:
+            - args: [-automaticShutdownDelaySec=AUTOMATIC_SHUTDOWN_DELAY_SEC_REPLACEMENT]
+              image: us-docker.pkg.dev/agones-images/examples/simple-game-server:0.18
+              name: simple-game-server
+              resources:
+                limits:
+                  cpu: 20m
+                  memory: 32Mi
+                requests:
+                  cpu: 20m
+                  memory: 32Mi

test/load/allocation/performance-test-variable-template.txt

@@ -0,0 +1 @@
+DURATION_REPLACEMENT,CLIENTS_REPLACEMENT,INTERVAL_REPLACEMENT

test/load/allocation/runscenario/runscenario.go

@@ -156,7 +156,7 @@ func main() {
 				logger.Printf("Count: %v\t\tError: %v", v, k)
 			}
 		}
-		logger.Printf("\n\n%v\nnScenario Failure Count: %v, Allocation Count: %v, Failure rate: %v, allocation rate: %v", time.Now(), scnFailureCnt, scnAllocCnt, float64(scnFailureCnt)/float64(scnAllocCnt), float64(scnAllocCnt-scnFailureCnt)/sc.duration.Seconds())
+		logger.Printf("\n\n%v\nScenario Failure Count: %v, Allocation Count: %v, Failure rate: %v, allocation rate: %v", time.Now(), scnFailureCnt, scnAllocCnt, float64(scnFailureCnt)/float64(scnAllocCnt), float64(scnAllocCnt-scnFailureCnt)/sc.duration.Seconds())
 	}
 
 	logger.Print("\nFinal Error Totals\n")