hack: implement prowjob-generator

prowjob-generator allows to generate the periodic and presubmit configuration files in test-infra
from a configuration file which simplifies and automates introducing and chaning tests for branches.

hack/tools/go.mod

@@ -8,6 +8,7 @@ replace sigs.k8s.io/cluster-api/test => ../../test
 
 require (
 	cloud.google.com/go/storage v1.36.0
+	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/blang/semver/v4 v4.0.0
 	github.com/onsi/gomega v1.30.0
 	github.com/pkg/errors v0.9.1
@@ -37,6 +38,7 @@ require (
 	cloud.google.com/go/iam v1.1.5 // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/semver/v3 v3.2.0 // indirect
 	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
 	github.com/Microsoft/go-winio v0.5.2 // indirect

hack/tools/go.sum

@@ -16,8 +16,12 @@ github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ
 github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g=
 github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
+github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60=
+github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
 github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
 github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
 github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=

hack/tools/prowjob-generator/config.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+type ProwIgnoredConfig struct {
+	ProwIgnored Config `json:"prow_ignored"`
+}
+
+// Config is the configuration file struct.
+type Config struct {
+	Branches map[string]BranchConfig  `json:"branches"`
+	Versions map[string]VersionMapper `json:"versions"`
+}
+
+// BranchConfig is the branch-based configuration struct.
+type BranchConfig struct {
+	Interval                            string    `json:"interval"`
+	KubekinsImage                       string    `json:"kubekinsImage"`
+	KubernetesVersionManagement         string    `json:"kubernetesVersionManagement"`
+	KubebuilderEnvtestKubernetesVersion string    `json:"kubebuilderEnvtestKubernetesVersion"`
+	Upgrades                            []Upgrade `json:"upgrades"`
+}
+
+// Upgrade describes a kubernetes upgrade.
+type Upgrade struct {
+	From string `json:"from"`
+	To   string `json:"to"`
+}
+
+// VersionMapper is a key value map strings to versions.
+type VersionMapper map[string]string

hack/tools/prowjob-generator/main.go

@@ -0,0 +1,175 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// main is the main package for prowjob-generator.
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+	"text/template"
+
+	"github.com/Masterminds/sprig"
+	"github.com/pkg/errors"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/yaml"
+)
+
+var (
+	configFile     = flag.String("config", "", "Path to the config file")
+	outputDir      = flag.String("output-dir", "", "Path to the directory to create the files in")
+	fileNamePrefix = flag.String("file-name-prefix", "cluster-api-", "File name prefix for the created prowjob files")
+)
+
+func main() {
+	flag.Parse()
+
+	if *outputDir == "" {
+		klog.Fatal("Expected flag \"output-dir\" to be set")
+	}
+
+	if *configFile == "" {
+		klog.Fatal("Expected flag \"config\" to be set")
+	}
+
+	rawConfig, err := os.ReadFile(*configFile)
+	if err != nil {
+		klog.Fatalf("Failed to read config file %q: %v", *configFile, err)
+	}
+
+	prowIgnoredConfig := ProwIgnoredConfig{}
+	if err := yaml.Unmarshal(rawConfig, &prowIgnoredConfig); err != nil {
+		klog.Fatalf("Failed to parse config file %q: %v", *configFile, err)
+	}
+
+	g, err := newGenerator(prowIgnoredConfig.ProwIgnored, *outputDir)
+	if err != nil {
+		klog.Fatalf("Failed to initialize generator: %v", err)
+	}
+
+	if err := g.generate(); err != nil {
+		klog.Fatalf("Failed to generate prowjobs: %v", err)
+	}
+}
+
+func k8sShortVersionString(s string) string {
+	if strings.HasPrefix(s, "ci/latest-") {
+		return "latest"
+	}
+	return strings.ReplaceAll(strings.TrimPrefix(s, "stable-"), ".", "-")
+}
+
+func (g *generator) k8sVersionLookup(version, component string) string {
+	v, ok := g.config.Versions[version]
+	if !ok {
+		klog.Fatalf("Failed to lookup version (%q) in config", version)
+	}
+	c, ok := v[component]
+	if !ok {
+		klog.Fatalf("Failed to lookup component version (%q) for version %q in config", component, version)
+	}
+	return c
+}
+
+func (g *generator) lastUpgradeVersionFrom(branch string) string {
+	upgrades := g.config.Branches[branch].Upgrades
+	return upgrades[len(upgrades)-1].From
+}
+func (g *generator) lastUpgradeVersionTo(branch string) string {
+	upgrades := g.config.Branches[branch].Upgrades
+	return upgrades[len(upgrades)-1].To
+}
+
+func newGenerator(config Config, outputDir string) (*generator, error) {
+	g := &generator{
+		config:    config,
+		outputDir: outputDir,
+	}
+	funcs := sprig.HermeticTxtFuncMap()
+	funcs["k8sShortVersionString"] = k8sShortVersionString
+	funcs["k8sVersionLookup"] = g.k8sVersionLookup
+	funcs["lastUpgradeVersionFrom"] = g.lastUpgradeVersionFrom
+	funcs["lastUpgradeVersionTo"] = g.lastUpgradeVersionTo
+
+	var err error
+	g.templates, err = template.New("").Funcs(funcs).ParseFS(templatesFS, "templates/*.yaml")
+	if err != nil {
+		return nil, err
+	}
+
+	return g, err
+}
+
+type generator struct {
+	templates *template.Template
+	config    Config
+	outputDir string
+}
+
+var jobTypes = [][]string{
+	{"periodics"},
+	{"periodics", "upgrades"},
+	{"presubmits"},
+}
+
+func (g *generator) generate() error {
+	for _, jobType := range jobTypes {
+		for branch := range g.config.Branches {
+			if err := g.generateProwjobs(branch, jobType...); err != nil {
+				return errors.Wrapf(err, "Generating prowjobs for type %s", strings.Join(jobType, "-"))
+			}
+		}
+	}
+	return nil
+}
+
+func (g *generator) generateProwjobs(branch string, jobType ...string) error {
+	fileName := g.generateFilename(branch, jobType...)
+	templateName := strings.Join(jobType, "-") + ".yaml"
+
+	klog.Infof("Generating prowjobs to %q using template %q", fileName, templateName)
+
+	data := map[string]interface{}{
+		"branch": branch,
+		"config": g.config.Branches[branch],
+	}
+
+	var out bytes.Buffer
+	if err := g.templates.ExecuteTemplate(&out, templateName, data); err != nil {
+		return errors.Wrapf(err, "Executing template %q for branch %q", templateName, branch)
+	}
+
+	filePath := path.Join(g.outputDir, fileName)
+	if err := os.WriteFile(filePath, out.Bytes(), 0644); err != nil { //nolint:gosec
+		return errors.Wrapf(err, "Writing prowjob to %q", filePath)
+	}
+
+	return nil
+}
+
+func (g *generator) generateFilename(branch string, jobType ...string) string {
+	fileFriendlyBranchName := strings.ReplaceAll(branch, ".", "-")
+
+	n := fmt.Sprintf("%s%s-%s", *fileNamePrefix, jobType[0], fileFriendlyBranchName)
+	if len(jobType) == 2 {
+		n = fmt.Sprintf("%s-%s", n, jobType[1])
+	}
+	return n + ".yaml"
+}

hack/tools/prowjob-generator/main_test.go

@@ -0,0 +1,30 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// main is the main package for prowjob-generator.
+package main
+
+import (
+	"testing"
+)
+
+func Test_newGenerator(t *testing.T) {
+	_, err := newGenerator(Config{}, "")
+	if err != nil {
+		t.Errorf("newGenerator() error = %v", err)
+		return
+	}
+}

hack/tools/prowjob-generator/template.go

@@ -0,0 +1,26 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"embed"
+)
+
+var (
+	//go:embed templates/*.yaml
+	templatesFS embed.FS
+)

hack/tools/prowjob-generator/template_test.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	_ "embed"
+	"testing"
+	"text/template"
+)
+
+func Test_Templates(t *testing.T) {
+	if _, err := template.ParseFS(templatesFS, "templates/*.yaml"); err != nil {
+		t.Errorf("Error parsing embedded templates: %v", err)
+	}
+}

hack/tools/prowjob-generator/templates/periodics-upgrades.yaml

@@ -0,0 +1,57 @@
+periodics:
+{{- range $_, $upgrade := .config.Upgrades }}
+- name: periodic-cluster-api-e2e-workload-upgrade-{{ $upgrade.From | k8sShortVersionString }}-{{ $upgrade.To | k8sShortVersionString }}-{{ $.branch | replace "." "-" }}
+  cluster: eks-prow-build-cluster
+  interval: 24h
+  decorate: true
+  rerun_auth_config:
+    github_team_slugs:
+    - org: kubernetes-sigs
+      slug: cluster-api-maintainers
+  labels:
+    preset-dind-enabled: "true"
+    preset-kind-volume-mounts: "true"
+  extra_refs:
+  - org: kubernetes-sigs
+    repo: cluster-api
+    base_ref: {{ $.branch }}
+    path_alias: sigs.k8s.io/cluster-api
+  - org: kubernetes
+    repo: kubernetes
+    base_ref: master
+    path_alias: k8s.io/kubernetes
+  spec:
+    containers:
+    - image: {{ $.config.KubekinsImage }}
+      args:
+      - runner.sh
+      - "./scripts/ci-e2e.sh"
+      env:
+      - name: ALWAYS_BUILD_KIND_IMAGES
+        value: "true"
+      - name: KUBERNETES_VERSION_UPGRADE_FROM
+        value: "{{ k8sVersionLookup $upgrade.From "k8sRelease" }}"
+      - name: KUBERNETES_VERSION_UPGRADE_TO
+        value: "{{ k8sVersionLookup $upgrade.To "k8sRelease" }}"
+      - name: ETCD_VERSION_UPGRADE_TO
+        value: "{{ k8sVersionLookup $upgrade.From "etcd" }}"
+      - name: COREDNS_VERSION_UPGRADE_TO
+        value: "{{ k8sVersionLookup $upgrade.From "coreDNS" }}"
+      - name: GINKGO_FOCUS
+        value: "\\[K8s-Upgrade\\]"
+      # we need privileged mode in order to do docker in docker
+      securityContext:
+        privileged: true
+      resources:
+        requests:
+          cpu: 7300m
+          memory: 32Gi
+        limits:
+          cpu: 7300m
+          memory: 32Gi
+  annotations:
+    testgrid-dashboards: sig-cluster-lifecycle-cluster-api{{ if eq $.branch "main" | not -}}{{ $.branch | trimPrefix "release" }}{{- end }}
+    testgrid-tab-name: capi-e2e-{{ $.branch | replace "." "-" }}-{{ $upgrade.From | k8sShortVersionString }}-{{ $upgrade.To | k8sShortVersionString }}
+    testgrid-alert-email: sig-cluster-lifecycle-cluster-api-alerts@kubernetes.io
+    testgrid-num-failures-to-alert: "4"
+{{ end -}}