Refactor prerequisites to use resource manager

.ci/integration

@@ -71,13 +71,12 @@ crt="$(cat $SOURCE_PATH/assets/tls.crt)"
 key="$(cat $SOURCE_PATH/assets/tls.key)"
 
 github_config=$($SOURCE_PATH/.ci/scripts/gh_config.py | base64 -w 0)
-pull_secrets_value_file_path="$SOURCE_PATH/.ci/pull_secrets.yaml"
-$SOURCE_PATH/.ci/scripts/pull_secrets.py > $pull_secrets_value_file_path
 
-prerequisites_chart="$(helm template --namespace ${NAMESPACE} -f $pull_secrets_value_file_path \
-  --set "objectStorage.minio.tag=RELEASE.2019-04-18T21-44-59Z" $SOURCE_PATH/charts/bootstrap_tm_prerequisites)"
 controller_chart="$(helm template --namespace ${NAMESPACE} \
-  --set "secrets.github.data=${github_config},controller.verbosity=5,controller.tag=${VERSION},controller.tls.caBundle=${ca_bundle},controller.tls.crt=${crt},controller.tls.key=${key},controller.testDefPath=test/.test-defs" $SOURCE_PATH/charts/testmachinery)"
+  --set "testmachinery.github.credentials=${github_config}" \
+  --set "controller.verbosity=5,controller.tag=${VERSION}" \
+  --set "controller.tls.caBundle=${ca_bundle},controller.tls.crt=${crt},controller.tls.key=${key}" \
+  -f $SOURCE_PATH/.ci/integrationtest-config.yaml $SOURCE_PATH/charts/testmachinery)"
 
 
 ################################
@@ -99,15 +98,20 @@ for ns in $(kubectl --kubeconfig=$TM_KUBECONFIG get ns --no-headers | awk '{ pri
     kubectl --kubeconfig=$TM_KUBECONFIG delete ns $ns
   fi
 done
+
+echo "> Delete leaked managed resrources"
+kubectl --kubeconfig=$TM_KUBECONFIG -n $NAMESPACE scale --replicas=0 deploy testmachinery-controller
+kubectl --kubeconfig=$TM_KUBECONFIG -n $NAMESPACE delete managedresources --all
+kubectl --kubeconfig=$TM_KUBECONFIG -n $NAMESPACE wait --for=delete managedresources --all
+echo "> Delete leaked volumes"
+kubectl --kubeconfig=$TM_KUBECONFIG -n $NAMESPACE delete pvc --all
+echo "> Delete previous tm installation"
 echo "$controller_chart" | kubectl --kubeconfig=$TM_KUBECONFIG delete -f -
-echo "$prerequisites_chart" | kubectl --kubeconfig=$TM_KUBECONFIG delete -f -
 set -e
 
 ############################################################################
 # installation of prerequisites, controller, validation webhooks and roles #
 ############################################################################
-# install prerequisites
-echo "$prerequisites_chart" | kubectl --kubeconfig=$TM_KUBECONFIG create -f -
 # install controller, validation webhook and roles
 echo "$controller_chart" | kubectl --kubeconfig=$TM_KUBECONFIG create -f -
 
@@ -122,5 +126,11 @@ ginkgo -mod=vendor --slowSpecThreshold=600 -p --nodes=5 ./test/testrunner/... --
 #######################
 # wait for controller to clean up all testruns.
 sleep 30
-echo "$controller_chart" | kubectl --kubeconfig=$TM_KUBECONFIG delete -f -
-echo "$prerequisites_chart" | kubectl --kubeconfig=$TM_KUBECONFIG delete -f -
+
+echo "> Delete leaked managed resources"
+kubectl --kubeconfig=$TM_KUBECONFIG -n $NAMESPACE scale --replicas=0 deploy testmachinery-controller
+kubectl --kubeconfig=$TM_KUBECONFIG -n $NAMESPACE delete managedresources --all
+echo "> Delete leaked volumes"
+kubectl --kubeconfig=$TM_KUBECONFIG -n $NAMESPACE delete pvc --all
+echo "> Delete previous tm installation"
+echo "$controller_chart" | kubectl --kubeconfig=$TM_KUBECONFIG delete -f -

.ci/integrationtest-config.yaml

@@ -0,0 +1,73 @@
+# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+controller:
+  enableLeaderElection: false
+  maxConcurrentSyncs: 5
+  webhook:
+    port: 443
+    certDir: ""
+
+testmachinery:
+  testdefPath: test/.test-defs
+  local: false
+  insecure: false
+  disableCollector: true
+  cleanWorkflowPods: false
+
+  github:
+    cache:
+      cacheDir: /tmp/tm/cache
+      cacheDiskSizeGB: 5
+      maxAgeSeconds: 60
+  argo:
+    argoUI:
+      ingress:
+        enabled: false
+        host: argoui.example.com
+
+  s3Configuration:
+    server:
+      minio:
+        distributed: false
+        ingress:
+          enabled: true
+          host: minio.ingress.tm-it.core.shoot.canary.k8s-hana.ondemand.com
+      ssl: false
+    bucketName: testmachinery
+    accessKey: kjdpasnvpornv
+    secretKey: askldmackmdpacm
+
+#  esConfiguration:
+#    endpoint: https:...:9200
+#    username: user
+#    password: my-password
+
+# imageVectorOverwrite: |
+#  - name: argo-ui
+#    sourceRepository: github.com/argoproj/argo
+#    repository: argoproj/argocli
+#    tag: "v2.6.3"
+#  - name: argo-workflow-controller
+#    sourceRepository: github.com/argoproj/argo
+#    repository: argoproj/workflow-controller
+#    tag: "v2.6.3"
+#  - name: argo-executor
+#    sourceRepository: github.com/argoproj/argo
+#    repository: argoproj/argoexec
+#    tag: "v2.6.3"
+#  - name: minio
+#    sourceRepository: github.com/minio/minio
+#    repository: minio/minio
+#    tag: "RELEASE.2020-04-04T05-39-31Z"

.gitignore

@@ -2,7 +2,7 @@
 .idea
 .DS_Store
 temp/
-tmp/
+tmp
 venv
 
 # Generated secrets

Dockerfile

@@ -18,20 +18,13 @@ FROM golang:1.14.1 AS builder
 WORKDIR /go/src/github.com/gardener/test-infra
 COPY . .
 
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go install -mod=vendor \
-   -ldflags "-X github.com/gardener/test-infra/pkg/version.gitVersion=$(cat VERSION) \
-               -X github.com/gardener/test-infra/pkg/version.gitTreeState=$([ -z git status --porcelain 2>/dev/null ] && echo clean || echo dirty) \
-               -X github.com/gardener/test-infra/pkg/version.gitCommit=$(git rev-parse --verify HEAD) \
-               -X github.com/gardener/test-infra/pkg/version.buildDate=$(date --rfc-3339=seconds | sed 's/ /T/')" \
-  ./cmd/...
+RUN make install-requirements && make all
 
 ############# tm-controller #############
 FROM alpine:3.10 AS tm-controller
 
-RUN apk add --update bash curl
-
+COPY charts /charts
 COPY --from=builder /go/bin/testmachinery-controller /testmachinery-controller
-COPY ./.env /
 
 WORKDIR /
 

Makefile

@@ -13,52 +13,64 @@
 # limitations under the License.
 
 SHELL = /bin/sh
+REPO_ROOT   := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
 current_dir := $(shell dirname $(mkfile_path))
 current_sha := $(shell GIT_DIR=${current_dir}/.git git rev-parse @)
 
 REGISTRY            := eu.gcr.io/gardener-project/gardener/testmachinery
 
 TM_CONTROLLER_IMAGE := $(REGISTRY)/testmachinery-controller
-VERSION             := $(shell cat VERSION)
+VERSION             := $(shell cat ${REPO_ROOT}/VERSION)
 IMAGE_TAG           := ${VERSION}
 
 TELEMETRY_CONTROLLER_IMAGE := $(REGISTRY)/telemetry-controller
-TM_RUN_IMAGE := $(REGISTRY)/testmachinery-run
-TM_BOT_IMAGE := $(REGISTRY)/bot
-PREPARESTEP_IMAGE := $(REGISTRY)/testmachinery-prepare
+TM_RUN_IMAGE               := $(REGISTRY)/testmachinery-run
+TM_BOT_IMAGE               := $(REGISTRY)/bot
+PREPARESTEP_IMAGE          := $(REGISTRY)/testmachinery-prepare
 
 NS ?= default
 KUBECONFIG ?= "~/.kube/config"
 TESTRUN ?= "examples/int-testrun.yaml"
 LD_FLAGS := $(shell ./hack/get-build-ld-flags)
 
+#####################
+# Utils             #
+#####################
 
-################################
-# Prerequisistes, Installation #
-################################
 
-.PHONY: install
-install: create-ns install-prerequisites
+.PHONY: revendor
+revendor:
+	@GO111MODULE=on go mod vendor
+	@GO111MODULE=on go mod tidy
+
+.PHONY: code-gen
+code-gen:
+	@./hack/generate-code
 
-.PHONY: clean
-clean: remove-prerequisites delete-ns
+.PHONY: mock-gen
+mock-gen:
+	@./hack/generate-mocks
 
-.PHONY: install-controller
-install-controller:
-	helm template --namespace ${NS} -f ./charts/testmachinery/local-values.yaml --set "controller.tag=${VERSION}" ./charts/testmachinery | kubectl create -f -
+.PHONY: generate
+generate:
+	@$(REPO_ROOT)/vendor/github.com/gardener/gardener-extensions/hack/generate.sh ./...
 
-.PHONY: remove-controller
-remove-controller:
-	helm template --namespace ${NS} -f ./charts/testmachinery/local-values.yaml ./charts/testmachinery | kubectl delete -f -
+.PHONY: format
+format:
+	@$(REPO_ROOT)/vendor/github.com/gardener/gardener-extensions/hack/format.sh ./cmd ./pkg
+
+.PHONY: install
+install:
+	@./hack/install
 
-.PHONY: install-prerequisites
-install-prerequisites:
-	helm template --namespace ${NS} ./charts/bootstrap_tm_prerequisites | kubectl apply -f -
+.PHONY: all
+all: format generate install
 
-.PHONY: remove-prerequisites
-remove-prerequisites:
-	helm template --namespace ${NS} ./charts/bootstrap_tm_prerequisites | kubectl delete -f -
+.PHONY: install-requirements
+install-requirements:
+	@go install -mod=vendor $(REPO_ROOT)/vendor/github.com/gobuffalo/packr/v2/packr2
+	@go install -mod=vendor $(REPO_ROOT)/vendor/github.com/golang/mock/mockgen
 
 .PHONY: gen-certs
 gen-certs:
@@ -73,17 +85,19 @@ gen-certs:
 		-subj "/C=DE/O=SAP SE/OU=testmachinery/CN=testmachinery-controller.default.svc"
 	@openssl x509 -req -sha256 -days 365 -in assets/tls.csr -CA assets/ca.crt -CAkey assets/ca.key -CAcreateserial -out assets/tls.crt
 
-.PHONY: create-ns
-create-ns:
-	@ if [ ${NS} != "default" ] && [ ! kubectl get ns ${NS} &> /dev/null ]; then \
-		kubectl create ns ${NS}; \
-	fi
 
-.PHONY: delete-ns
-delete-ns:
-	@ if [ ${NS} != "default" ] && [ kubectl get ns ${NS} ]; then \
-		kubectl delete ns ${NS}; \
-	fi
+################################
+# Prerequisistes, Installation #
+################################
+
+.PHONY: deploy-controller
+deploy-controller:
+	helm template --namespace ${NS} -f ./charts/testmachinery/local-values.yaml --set "controller.tag=${VERSION}" ./charts/testmachinery | kubectl apply -f -
+
+.PHONY: remove-controller
+remove-controller:
+	helm template --namespace ${NS} -f ./charts/testmachinery/local-values.yaml ./charts/testmachinery | kubectl delete -f -
+
 
 #####################
 # Local development #
@@ -111,14 +125,6 @@ run-it-tests:
 	GIT_COMMIT_SHA=${current_sha} ginkgo ./test/... -v -progress -- \
 		--kubecfg=${KUBECONFIG} --tm-namespace=${NS} --namespace="" --git-commit-sha=master --s3-endpoint=""
 
-.PHONY: code-gen
-code-gen:
-	@./hack/generate-code
-
-.PHONY: mock-gen
-mock-gen:
-	@./hack/generate-mocks
-
 .PHONY: validate
 validate:
 	@go run cmd/local-validator/main.go -testrun=${TESTRUN}
@@ -128,11 +134,6 @@ validate:
 # Binary build and docker image  #
 ##################################
 
-.PHONY: revendor
-revendor:
-	@GO111MODULE=on go mod vendor
-	@GO111MODULE=on go mod tidy
-
 .PHONY: testrunner
 testrunner:
 	@go install -v \

charts/images.yaml

@@ -0,0 +1,17 @@
+images:
+- name: argo-ui
+  sourceRepository: github.com/argoproj/argo
+  repository: argoproj/argocli
+  tag: "v2.6.3"
+- name: argo-workflow-controller
+  sourceRepository: github.com/argoproj/argo
+  repository: argoproj/workflow-controller
+  tag: "v2.6.3"
+- name: argo-executor
+  sourceRepository: github.com/argoproj/argo
+  repository: argoproj/argoexec
+  tag: "v2.6.3"
+- name: minio
+  sourceRepository: github.com/minio/minio
+  repository: minio/minio
+  tag: "RELEASE.2020-04-04T05-39-31Z"

charts/bootstrap_tm_prerequisites/Chart.yaml → charts/internal/argo/Chart.yaml

@@ -14,6 +14,6 @@
 
 apiVersion: v1
 appVersion: "1.0"
-description: A Helm chart for boostrapping the prerequisites of the testmachinery. This includes the argo engine and a minio object store.
-name: bootstrap_tm_prerequisites
+description: A Helm chart for boostrapping the argo engine.
+name: argo
 version: 0.1.0