diff --git a/.github/ISSUE_TEMPLATE/kubernetes_bump.md b/.github/ISSUE_TEMPLATE/kubernetes_bump.md index 9b65aec8d365..a6ea6b646253 100644 --- a/.github/ISSUE_TEMPLATE/kubernetes_bump.md +++ b/.github/ISSUE_TEMPLATE/kubernetes_bump.md @@ -34,21 +34,25 @@ changes should be cherry-picked to all release series that will support the new * Note: Only bump for Cluster API versions that will support the new Kubernetes release. * Prior art: #9160 * [ ] Ensure the jobs are adjusted to provide test coverage according to our [support policy](https://cluster-api.sigs.k8s.io/reference/versions.html#supported-kubernetes-versions): - * For the main branch: - * periodics: - * Drop the oldest upgrade job as the oldest Kubernetes minor version is now out of support. - * Add new upgrade job which upgrades from the previous to the new Kubernetes version. - * periodics & presubmits: - * Bump `KUBERNETES_VERSION_MANAGEMENT` of the `e2e-mink8s` job to the new minimum supported management cluster version. - * Bump `KUBEBUILDER_ENVTEST_KUBERNETES_VERSION` of the `test-mink8s` jobs to the new minimum supported management cluster version. - * Adjust the `-latest` upgrade job to upgrade from the new Kubernetes to the next Kubernetes version. - * For the release branch of the latest supported Cluster API minor release: - * periodics & presubmits: - * Adust the `-latest` upgrade jobs to upgrade to the new Kubernetes version instead of latest. - * Note: Also check if `ETCD_VERSION_UPGRADE_TO` or `COREDNS_VERSION_UPGRADE_TO` needs to change for the upgrades jobs to the new or next Kubernetes version. - * For etcd, see the `DefaultEtcdVersion` kubeadm constant: [e.g. for v1.28.0](https://github.com/kubernetes/kubernetes/blob/v1.28.0/cmd/kubeadm/app/constants/constants.go#L308) - * For coredns, see the `CoreDNSVersion` kubeadm constant:[e.g. for v1.28.0](https://github.com/kubernetes/kubernetes/blob/v1.28.0/cmd/kubeadm/app/constants/constants.go#L344) - * Prior art: https://github.com/kubernetes/test-infra/pull/30347 https://github.com/kubernetes/test-infra/pull/30406 https://github.com/kubernetes/test-infra/pull/30407 + + * At the `.versions` section in the `cluster-api-prowjob-gen.yaml` file in [test-infra](https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-sigs/cluster-api/): + * Add a new entry for the new Kubernetes version + * Adjust the released kKubernetes's version entry to refer `stable-1.` instead of `ci/latest-1.` + * Check and update the versions for the keys `etcd` and `coreDNS` if necessary: + * For etcd, see the `DefaultEtcdVersion` kubeadm constant: [e.g. for v1.28.0](https://github.com/kubernetes/kubernetes/blob/v1.28.0/cmd/kubeadm/app/constants/constants.go#L308) + * For coredns, see the `CoreDNSVersion` kubeadm constant:[e.g. for v1.28.0](https://github.com/kubernetes/kubernetes/blob/v1.28.0/cmd/kubeadm/app/constants/constants.go#L344) + * For the `.branches.main` section in the `cluster-api-prowjob-gen.yaml` file in [test-infra](https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-sigs/cluster-api/): + * For the `.upgrades` section: + * Drop the oldest upgrade + * Add a new upgrade entry from the previous to the new Kubernetes version + * Bump the version set at `.kubernetesVersionManagement` to the new minimum supported management cluster version (This is the image version available as kind image). + * Bump the version set at `.kubebuilderEnvtestKubernetesVersion` to the new minimum supported management cluster version. + * Run `make generate-test-infra-prowjobs` to generate the resulting prowjob configuration: + + ```sh + TEST_INFRA_DIR=../../k8s.io/test-infra make generate-test-infra-prowjobs + ``` + * [ ] Update book: * Update supported versions in `versions.md` * Update job documentation in `jobs.md` @@ -65,7 +69,7 @@ need them in older releases as they are not necessary to manage workload cluster run the Cluster API controllers on the new Kubernetes version. * [ ] Ensure there is a new controller-runtime minor release which uses the new Kubernetes Go dependencies. -* [ ] Update our Prow jobs for the `main` branch to use the correct `kubekins-e2e` image +* [ ] Update our Prow jobs for the `main` branch to use the correct `kubekins-e2e` image via the configuration file and by running `make generate-test-infra-prowjobs`. * It is recommended to have one PR for presubmit and one for periodic jobs to reduce the risk of breaking the periodic jobs. * Prior art: presubmit jobs: https://github.com/kubernetes/test-infra/pull/27311 * Prior art: periodic jobs: https://github.com/kubernetes/test-infra/pull/27326 diff --git a/Makefile b/Makefile index cb0425f3e350..e830ea893d69 100644 --- a/Makefile +++ b/Makefile @@ -190,6 +190,9 @@ OPENAPI_GEN_BIN := openapi-gen OPENAPI_GEN := $(abspath $(TOOLS_BIN_DIR)/$(OPENAPI_GEN_BIN)) OPENAPI_GEN_PKG := k8s.io/kube-openapi/cmd/openapi-gen +PROWJOB_GEN_BIN := prowjob-gen +PROWJOB_GEN := $(abspath $(TOOLS_BIN_DIR)/$(PROWJOB_GEN_BIN)) + RUNTIME_OPENAPI_GEN_BIN := runtime-openapi-gen RUNTIME_OPENAPI_GEN := $(abspath $(TOOLS_BIN_DIR)/$(RUNTIME_OPENAPI_GEN_BIN)) @@ -600,6 +603,13 @@ generate-diagrams-book: ## Generate diagrams for *.plantuml files in book generate-diagrams-proposals: ## Generate diagrams for *.plantuml files in proposals docker run -v $(ROOT_DIR)/$(DOCS_DIR):/$(DOCS_DIR)$(DOCKER_VOL_OPTS) plantuml/plantuml:$(PLANTUML_VER) /$(DOCS_DIR)/proposals/**/*.plantuml +.PHONY: generate-test-infra-prowjobs +generate-test-infra-prowjobs: $(PROWJOB_GEN) ## Generates the prowjob configurations in test-infra + @if [ -z "${TEST_INFRA_DIR}" ]; then echo "TEST_INFRA_DIR is not set"; exit 1; fi + $(PROWJOB_GEN) \ + -config "$(TEST_INFRA_DIR)/config/jobs/kubernetes-sigs/cluster-api/cluster-api-prowjob-gen.yaml" \ + -templates-dir "$(TEST_INFRA_DIR)/config/jobs/kubernetes-sigs/cluster-api/templates" \ + -output-dir "$(TEST_INFRA_DIR)/config/jobs/kubernetes-sigs/cluster-api" ## -------------------------------------- ## Lint / Verify @@ -1307,6 +1317,9 @@ $(OPENAPI_GEN_BIN): $(OPENAPI_GEN) ## Build a local copy of openapi-gen. .PHONY: $(RUNTIME_OPENAPI_GEN_BIN) $(RUNTIME_OPENAPI_GEN_BIN): $(RUNTIME_OPENAPI_GEN) ## Build a local copy of runtime-openapi-gen. +.PHONY: $(PROWJOB_GEN_BIN) +$(PROWJOB_GEN_BIN): $(PROWJOB_GEN) ## Build a local copy of prowjob-gen. + .PHONY: $(CONVERSION_VERIFIER_BIN) $(CONVERSION_VERIFIER_BIN): $(CONVERSION_VERIFIER) ## Build a local copy of conversion-verifier. @@ -1367,6 +1380,10 @@ $(OPENAPI_GEN): # Build openapi-gen from tools folder. $(RUNTIME_OPENAPI_GEN): $(TOOLS_DIR)/go.mod # Build openapi-gen from tools folder. cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/$(RUNTIME_OPENAPI_GEN_BIN) sigs.k8s.io/cluster-api/hack/tools/runtime-openapi-gen +.PHONY: $(PROWJOB_GEN) +$(PROWJOB_GEN): $(TOOLS_DIR)/go.mod # Build prowjob-gen from tools folder. + cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/$(PROWJOB_GEN_BIN) sigs.k8s.io/cluster-api/hack/tools/prowjob-gen + $(GOTESTSUM): # Build gotestsum from tools folder. GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(GOTESTSUM_PKG) $(GOTESTSUM_BIN) $(GOTESTSUM_VER) diff --git a/hack/boilerplate/boilerplate.py b/hack/boilerplate/boilerplate.py index 830fcdd3e842..7a9fb6155645 100755 --- a/hack/boilerplate/boilerplate.py +++ b/hack/boilerplate/boilerplate.py @@ -143,6 +143,7 @@ def file_passes(filename, refs, regexs): for line in difflib.unified_diff(ref, data, 'reference', filename, lineterm=''): print(line, file=verbose_out) print(file=verbose_out) + return False return True @@ -154,7 +155,10 @@ def file_extension(filename): # list all the files contain 'DO NOT EDIT', but are not generated skipped_ungenerated_files = [ - 'hack/lib/swagger.sh', 'hack/boilerplate/boilerplate.py'] + 'hack/lib/swagger.sh', + 'hack/boilerplate/boilerplate.py', + '/hack/tools/prowjob-gen/generator.go', + ] def normalize_files(files): newfiles = [] diff --git a/hack/tools/prowjob-gen/README.md b/hack/tools/prowjob-gen/README.md new file mode 100644 index 000000000000..5333aa130616 --- /dev/null +++ b/hack/tools/prowjob-gen/README.md @@ -0,0 +1,68 @@ +# prowjob-gen + +Prowjob-gen is a tool which helps generating prowjob configuration. + +## Usage + +Flags: + +```txt + -config string + Path to the config file + -output-dir string + Path to the directory to create the files in + -templates-dir string + Path to the directory containing the template files referenced inside the config file +``` + +When running prowjob-gen, all flags need to be provided. +The tool then will iterate over all templates defined in the config file and execute them per configured branch. + +The configuration file is supposed to be in yaml format and to be stored inside the [test-infra](https://github.com/kubernetes/test-infra) +repository, we have to make sure it is not getting parsed as configuration for prow. +Because of that the top-level key for the configuration file is `prow-ignored:`. + +A sample configuration looks as follows: + +```yaml +prow_ignored: + branches: + main: # values below the branch here are available in the template + kubekinsImage: "gcr.io/k8s-staging-test-infra/kubekins-e2e:v20231208-8b9fd88e88-1.29" + interval: "2h" + kubernetesVersionManagement: "v1.26.6@sha256:6e2d8b28a5b601defe327b98bd1c2d1930b49e5d8c512e1895099e4504007adb" + kubebuilderEnvtestKubernetesVersion: "1.26.1" + upgrades: + - from: "1.29" + to: "1.30" + + templates: + - name: "cluster-api-periodics.yaml.tpl" + format: "cluster-api-periodics-%s.yaml" + + versions: + "1.29": + etcd: "3.5.10-0" + coreDNS: "v1.11.1" + k8sRelease: "stable-1.29" + "1.30": + etcd: "3.5.10-0" + coreDNS: "v1.11.1" + k8sRelease: "ci/latest-1.30" +``` + +With this configuration, the template `cluster-api-periodics.yaml.tpl` would get executed for each branch. +In this example we only configure the `main` branch which results in the output file `cluster-api-periodics-main.yaml`. + +When executing a template, the following functions are available as addition to the standard functions in go templates: + +- `TrimPrefix`: [strings.TrimPrefix](https://pkg.go.dev/strings#TrimPrefix) +- `TrimSuffix`: [strings.TrimSuffix](https://pkg.go.dev/strings#TrimSuffix) +- `ReplaceAll`: [strings.ReplaceAll](https://pkg.go.dev/strings#ReplaceAll) +- `last`: `func(any) any`: returns the last element of an array or slice. + +When executing a template, the following variables are available: + +- `branch`: The branch name the file get's templated for (The key in `.prow_ignored.branches`). +- `config`: The branch's configuration from `.prow_ignored.branches.`. +- `versions`: The versions mapper from `.prow_ignored.versions`. diff --git a/hack/tools/prowjob-gen/config.go b/hack/tools/prowjob-gen/config.go new file mode 100644 index 000000000000..6a083f425262 --- /dev/null +++ b/hack/tools/prowjob-gen/config.go @@ -0,0 +1,56 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +// ProwIgnoredConfig is the top-level configuration struct. Because we want to +// store the configuration in test-infra as yaml file, we have to prevent prow +// from trying to parse our configuration as prow configuration. Prow provides +// the well-known `prow_ignored` key which is not parsed further by Prow. +type ProwIgnoredConfig struct { + ProwIgnored Config `json:"prow_ignored"` +} + +// Config is the configuration file struct. +type Config struct { + Branches map[string]BranchConfig `json:"branches"` + Templates []Template `json:"templates"` + VersionsMapper VersionsMapper `json:"versions"` +} + +// BranchConfig is the branch-based configuration struct. +type BranchConfig struct { + Interval string `json:"interval"` + KubekinsImage string `json:"kubekinsImage"` + KubernetesVersionManagement string `json:"kubernetesVersionManagement"` + KubebuilderEnvtestKubernetesVersion string `json:"kubebuilderEnvtestKubernetesVersion"` + Upgrades []*Upgrade `json:"upgrades"` +} + +// Template refers a template file and defines the target file name format. +type Template struct { + Format string `json:"format"` + Name string `json:"name"` +} + +// Upgrade describes a kubernetes upgrade. +type Upgrade struct { + From string `json:"from"` + To string `json:"to"` +} + +// VersionsMapper provides key value pairs for a parent key. +type VersionsMapper map[string]map[string]string diff --git a/hack/tools/prowjob-gen/generator.go b/hack/tools/prowjob-gen/generator.go new file mode 100644 index 000000000000..ed2666db06ed --- /dev/null +++ b/hack/tools/prowjob-gen/generator.go @@ -0,0 +1,164 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "fmt" + "os" + "path" + "path/filepath" + "reflect" + "strings" + "text/template" + + "github.com/pkg/errors" + "k8s.io/klog/v2" +) + +const generatedFileHeader = "# Code generated by cluster-api's prowjob-gen. DO NOT EDIT.\n" + +// newGenerator initializes a generator which includes parsing the configured templates. +func newGenerator(config Config, templatesDir, outputDir string) (*generator, error) { + g := &generator{ + config: config, + outputDir: outputDir, + createdFiles: map[string]bool{}, + } + + var err error + g.templates, err = template.New(""). + Funcs(g.templateFunctions()). + ParseGlob(templatesDir + "/*.yaml.tpl") + if err != nil { + return nil, err + } + + return g, err +} + +type generator struct { + templates *template.Template + config Config + outputDir string + createdFiles map[string]bool +} + +// generate executes every template for every branch and writes the result to a +// file in outputDir. +func (g *generator) generate() error { + for _, tpl := range g.config.Templates { + for branch := range g.config.Branches { + out, err := g.executeTemplate(branch, tpl.Name) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return errors.Wrapf(err, "Generating prowjobs for template %s", tpl.Name) + } + + fileName := fmt.Sprintf(tpl.Format, strings.ReplaceAll(branch, ".", "-")) + filePath := filepath.Clean(path.Join(g.outputDir, fileName)) + if err := os.WriteFile(filePath, out.Bytes(), 0644); err != nil { //nolint:gosec + return errors.Wrapf(err, "Writing prowjob to %q", filePath) + } + + g.createdFiles[fileName] = true + } + } + return nil +} + +// cleanup deletes files which have the generatedFileHeader and had not been updated +// during generate. +func (g *generator) cleanup() error { + entries, err := os.ReadDir(g.outputDir) + if err != nil { + return err + } + + for _, entry := range entries { + if _, ok := g.createdFiles[entry.Name()]; ok { + continue + } + + if entry.IsDir() { + continue + } + + path := filepath.Clean(path.Join(g.outputDir, entry.Name())) + data, err := os.ReadFile(path) + if err != nil { + return err + } + + if strings.HasPrefix(string(data), generatedFileHeader) { + klog.Infof("Deleting file %s", entry.Name()) + if err := os.Remove(path); err != nil { + return err + } + } + } + + return nil +} + +// executeTemplate executes a previously parsed template with the data for a specific branch. +func (g *generator) executeTemplate(branch, templateName string) (*bytes.Buffer, error) { + klog.Infof("executing template %q for branch %q", templateName, branch) + + data := map[string]interface{}{ + "branch": branch, + "config": g.config.Branches[branch], + "versions": g.config.VersionsMapper, + } + + var out bytes.Buffer + + // Write yaml comment as header to indicate this file got generated. + out.WriteString(generatedFileHeader) + + if err := g.templates.ExecuteTemplate(&out, templateName, data); err != nil { + return nil, errors.Wrapf(err, "Executing template %q for branch %q", templateName, branch) + } + + return &out, nil +} + +// templateFunctions returns the functions available inside of templates. +func (g *generator) templateFunctions() template.FuncMap { + funcs := map[string]any{} + funcs["TrimPrefix"] = strings.TrimPrefix + funcs["TrimSuffix"] = strings.TrimSuffix + funcs["ReplaceAll"] = strings.ReplaceAll + funcs["last"] = last + return funcs +} + +func last(list any) any { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + if l == 0 { + return nil + } + + return l2.Index(l - 1).Interface() + default: + panic(fmt.Sprintf("Cannot find last on type %s", tp)) + } +} diff --git a/hack/tools/prowjob-gen/main.go b/hack/tools/prowjob-gen/main.go new file mode 100644 index 000000000000..8f631538d27c --- /dev/null +++ b/hack/tools/prowjob-gen/main.go @@ -0,0 +1,72 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// main is the main package for prowjob-gen. +package main + +import ( + "flag" + "os" + + "k8s.io/klog/v2" + "sigs.k8s.io/yaml" +) + +var ( + configFile = flag.String("config", "", "Path to the config file") + outputDir = flag.String("output-dir", "", "Path to the directory to create the files in") + templatesDir = flag.String("templates-dir", "", "Path to the directory containing the template files referenced inside the config file") +) + +func main() { + // Parse flags and validate input. + flag.Parse() + if *configFile == "" { + klog.Fatal("Expected flag \"config\" to be set") + } + if *outputDir == "" { + klog.Fatal("Expected flag \"output-dir\" to be set") + } + if *templatesDir == "" { + klog.Fatal("Expected flag \"templates-dir\" to be set") + } + + // Read and Unmarshal the configuration file. + rawConfig, err := os.ReadFile(*configFile) + if err != nil { + klog.Fatalf("Failed to read config file %q: %v", *configFile, err) + } + prowIgnoredConfig := ProwIgnoredConfig{} + if err := yaml.Unmarshal(rawConfig, &prowIgnoredConfig); err != nil { + klog.Fatalf("Failed to parse config file %q: %v", *configFile, err) + } + + // Initialize a generator using the config data. + g, err := newGenerator(prowIgnoredConfig.ProwIgnored, *templatesDir, *outputDir) + if err != nil { + klog.Fatalf("Failed to initialize generator: %v", err) + } + + // Generate new files. + if err := g.generate(); err != nil { + klog.Fatalf("Failed to generate prowjobs: %v", err) + } + + // Cleanup old files which did not get updated. + if err := g.cleanup(); err != nil { + klog.Fatalf("Failed to cleanup old generated files: %v", err) + } +} diff --git a/hack/tools/prowjob-gen/main_test.go b/hack/tools/prowjob-gen/main_test.go new file mode 100644 index 000000000000..a1caaa0be601 --- /dev/null +++ b/hack/tools/prowjob-gen/main_test.go @@ -0,0 +1,30 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// main is the main package for prowjob-gen. +package main + +import ( + "testing" +) + +func Test_newGenerator(t *testing.T) { + _, err := newGenerator(Config{}, "test", "") + if err != nil { + t.Errorf("newGenerator() error = %v", err) + return + } +} diff --git a/hack/tools/prowjob-gen/test/test.yaml.tpl b/hack/tools/prowjob-gen/test/test.yaml.tpl new file mode 100644 index 000000000000..e69de29bb2d1