From bb53d0bfa44f445488bddc5ad472d88597bd7058 Mon Sep 17 00:00:00 2001 From: Joe Betz Date: Mon, 9 Nov 2020 14:57:48 -0800 Subject: [PATCH] Stub out generator --- cmd/controller-gen/main.go | 2 + pkg/applyconfigurations/doc.go | 18 ++ pkg/applyconfigurations/gen.go | 270 ++++++++++++++++++ pkg/applyconfigurations/jsontagutil.go | 97 +++++++ pkg/applyconfigurations/traverse.go | 257 +++++++++++++++++ .../zz_generated.markerhelp.go | 45 +++ 6 files changed, 689 insertions(+) create mode 100644 pkg/applyconfigurations/doc.go create mode 100644 pkg/applyconfigurations/gen.go create mode 100644 pkg/applyconfigurations/jsontagutil.go create mode 100644 pkg/applyconfigurations/traverse.go create mode 100644 pkg/applyconfigurations/zz_generated.markerhelp.go diff --git a/cmd/controller-gen/main.go b/cmd/controller-gen/main.go index 692b26c8d..7e009ec25 100644 --- a/cmd/controller-gen/main.go +++ b/cmd/controller-gen/main.go @@ -24,6 +24,7 @@ import ( "github.com/spf13/cobra" + "sigs.k8s.io/controller-tools/pkg/applyconfigurations" "sigs.k8s.io/controller-tools/pkg/crd" "sigs.k8s.io/controller-tools/pkg/deepcopy" "sigs.k8s.io/controller-tools/pkg/genall" @@ -51,6 +52,7 @@ var ( "crd": crd.Generator{}, "rbac": rbac.Generator{}, "object": deepcopy.Generator{}, + "apply": applyconfigurations.Generator{}, "webhook": webhook.Generator{}, "schemapatch": schemapatcher.Generator{}, } diff --git a/pkg/applyconfigurations/doc.go b/pkg/applyconfigurations/doc.go new file mode 100644 index 000000000..82c2c10f5 --- /dev/null +++ b/pkg/applyconfigurations/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package applyconfigurations generates types for constructing declarative apply configurations. +package applyconfigurations diff --git a/pkg/applyconfigurations/gen.go b/pkg/applyconfigurations/gen.go new file mode 100644 index 000000000..70e3c2119 --- /dev/null +++ b/pkg/applyconfigurations/gen.go @@ -0,0 +1,270 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package applyconfigurations + +import ( + "bytes" + "fmt" + "go/ast" + "go/format" + "io" + "sort" + "strings" + + "sigs.k8s.io/controller-tools/pkg/genall" + "sigs.k8s.io/controller-tools/pkg/loader" + "sigs.k8s.io/controller-tools/pkg/markers" +) + +// Based on deepcopy gen but with legacy marker support removed. + +var ( + enablePkgMarker = markers.Must(markers.MakeDefinition("kubebuilder:object:generate", markers.DescribesPackage, false)) + enableTypeMarker = markers.Must(markers.MakeDefinition("kubebuilder:object:generate", markers.DescribesType, false)) + isObjectMarker = markers.Must(markers.MakeDefinition("kubebuilder:object:root", markers.DescribesType, false)) +) + +// +controllertools:marker:generateHelp + +// Generator generates code containing apply configuration type implementations. +type Generator struct { + // HeaderFile specifies the header text (e.g. license) to prepend to generated files. + HeaderFile string `marker:",optional"` + // Year specifies the year to substitute for " YEAR" in the header file. + Year string `marker:",optional"` +} + +func (Generator) CheckFilter() loader.NodeFilter { + return func(node ast.Node) bool { + // ignore interfaces + _, isIface := node.(*ast.InterfaceType) + return !isIface + } +} + +func (Generator) RegisterMarkers(into *markers.Registry) error { + if err := markers.RegisterAll(into, + enablePkgMarker, enableTypeMarker, isObjectMarker); err != nil { + return err + } + into.AddHelp(enablePkgMarker, + markers.SimpleHelp("apply", "enables or disables object apply configuration generation for this package")) + into.AddHelp( + enableTypeMarker, markers.SimpleHelp("apply", "overrides enabling or disabling apply configuration generation for this type")) + into.AddHelp(isObjectMarker, + markers.SimpleHelp("apply", "enables apply configuration generation for this type")) + return nil +} + +func enabledOnPackage(col *markers.Collector, pkg *loader.Package) (bool, error) { + pkgMarkers, err := markers.PackageMarkers(col, pkg) + if err != nil { + return false, err + } + pkgMarker := pkgMarkers.Get(enablePkgMarker.Name) + if pkgMarker != nil { + return pkgMarker.(bool), nil + } + + return false, nil +} + +func enabledOnType(allTypes bool, info *markers.TypeInfo) bool { + if typeMarker := info.Markers.Get(enableTypeMarker.Name); typeMarker != nil { + return typeMarker.(bool) + } + return allTypes || genObjectInterface(info) +} + +func genObjectInterface(info *markers.TypeInfo) bool { + objectEnabled := info.Markers.Get(isObjectMarker.Name) + if objectEnabled != nil { + return objectEnabled.(bool) + } + return false +} + +func (d Generator) Generate(ctx *genall.GenerationContext) error { + var headerText string + + if d.HeaderFile != "" { + headerBytes, err := ctx.ReadFile(d.HeaderFile) + if err != nil { + return err + } + headerText = string(headerBytes) + } + headerText = strings.ReplaceAll(headerText, " YEAR", " "+d.Year) + + objGenCtx := ObjectGenCtx{ + Collector: ctx.Collector, + Checker: ctx.Checker, + HeaderText: headerText, + } + + for _, root := range ctx.Roots { + outContents := objGenCtx.generateForPackage(root) + if outContents == nil { + continue + } + + writeOut(ctx, root, outContents) + } + + return nil +} + +// ObjectGenCtx contains the common info for generating apply configuration implementations. +// It mostly exists so that generating for a package can be easily tested without +// requiring a full set of output rules, etc. +type ObjectGenCtx struct { + Collector *markers.Collector + Checker *loader.TypeChecker + HeaderText string +} + +// writeHeader writes out the build tag, package declaration, and imports +func writeHeader(pkg *loader.Package, out io.Writer, packageName string, imports *importsList, headerText string) { + // NB(directxman12): blank line after build tags to distinguish them from comments + _, err := fmt.Fprintf(out, `// +build !ignore_autogenerated + +%[3]s + +// Code generated by controller-gen. DO NOT EDIT. + +package %[1]s + +import ( +%[2]s +) + +`, packageName, strings.Join(imports.ImportSpecs(), "\n"), headerText) + if err != nil { + pkg.AddError(err) + } + +} + +// generateForPackage generates apply configuration implementations for +// types in the given package, writing the formatted result to given writer. +// May return nil if source could not be generated. +func (ctx *ObjectGenCtx) generateForPackage(root *loader.Package) []byte { + allTypes, err := enabledOnPackage(ctx.Collector, root) + if err != nil { + root.AddError(err) + return nil + } + + ctx.Checker.Check(root) + + root.NeedTypesInfo() + + byType := make(map[string][]byte) + imports := &importsList{ + byPath: make(map[string]string), + byAlias: make(map[string]string), + pkg: root, + } + // avoid confusing aliases by "reserving" the root package's name as an alias + imports.byAlias[root.Name] = "" + + if err := markers.EachType(ctx.Collector, root, func(info *markers.TypeInfo) { + outContent := new(bytes.Buffer) + + if !enabledOnType(allTypes, info) { + //root.AddError(fmt.Errorf("skipping type: %v", info.Name)) // TODO(jpbetz): Remove + return + } + + // not all types required a generate apply configuration. For example, no apply configuration + // type is needed for Quantity, IntOrString, RawExtension or Unknown. + if !shouldBeApplyConfiguration(root, info) { + //root.AddError(fmt.Errorf("skipping type: %v", info.Name)) // TODO(jpbetz): Remove + return + } + + copyCtx := &applyConfigurationMaker{ + pkg: root, + importsList: imports, + codeWriter: &codeWriter{out: outContent}, + } + + copyCtx.GenerateTypesFor(root, info) + + outBytes := outContent.Bytes() + if len(outBytes) > 0 { + byType[info.Name] = outBytes + } + }); err != nil { + root.AddError(err) + return nil + } + + if len(byType) == 0 { + return nil + } + + outContent := new(bytes.Buffer) + writeHeader(root, outContent, root.Name, imports, ctx.HeaderText) + writeTypes(root, outContent, byType) + + outBytes := outContent.Bytes() + formattedBytes, err := format.Source(outBytes) + if err != nil { + root.AddError(err) + // we still write the invalid source to disk to figure out what went wrong + } else { + outBytes = formattedBytes + } + + return outBytes +} + +// writeTypes writes each method to the file, sorted by type name. +func writeTypes(pkg *loader.Package, out io.Writer, byType map[string][]byte) { + sortedNames := make([]string, 0, len(byType)) + for name := range byType { + sortedNames = append(sortedNames, name) + } + sort.Strings(sortedNames) + + for _, name := range sortedNames { + _, err := out.Write(byType[name]) + if err != nil { + pkg.AddError(err) + } + } +} + +// writeFormatted outputs the given code, after gofmt-ing it. If we couldn't gofmt, +// we write the unformatted code for debugging purposes. +func writeOut(ctx *genall.GenerationContext, root *loader.Package, outBytes []byte) { + outputFile, err := ctx.Open(root, "zz_generated.applyconfigurations.go") + if err != nil { + root.AddError(err) + return + } + defer outputFile.Close() + n, err := outputFile.Write(outBytes) + if err != nil { + root.AddError(err) + return + } + if n < len(outBytes) { + root.AddError(io.ErrShortWrite) + } +} diff --git a/pkg/applyconfigurations/jsontagutil.go b/pkg/applyconfigurations/jsontagutil.go new file mode 100644 index 000000000..01df470b2 --- /dev/null +++ b/pkg/applyconfigurations/jsontagutil.go @@ -0,0 +1,97 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package applyconfigurations + +import ( + "strings" + + "sigs.k8s.io/controller-tools/pkg/markers" +) + +// TODO: This implements the same functionality as https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/runtime/converter.go#L236 +// but is based on the more efficient approach from https://golang.org/src/encoding/json/encode.go + +type jsonTags struct { + name string + omit bool + inline bool + omitempty bool +} + +func (t jsonTags) String() string { + var tag string + if !t.inline { + tag += t.name + } + if t.omitempty { + tag += ",omitempty" + } + if t.inline { + tag += ",inline" + } + return tag +} + +func lookupJsonTags(field markers.FieldInfo) (jsonTags, bool) { + tag := field.Tag.Get("json") + if tag == "" || tag == "-" { + return jsonTags{}, false + } + name, opts := parseTag(tag) + if name == "" { + name = field.Name + } + return jsonTags{ + name: name, + omit: false, + inline: opts.Contains("inline"), + omitempty: opts.Contains("omitempty"), + }, true +} + +type tagOptions string + +// parseTag splits a struct field's json tag into its name and +// comma-separated options. +func parseTag(tag string) (string, tagOptions) { + if idx := strings.Index(tag, ","); idx != -1 { + return tag[:idx], tagOptions(tag[idx+1:]) + } + return tag, "" +} + +// Contains reports whether a comma-separated listAlias of options +// contains a particular substr flag. substr must be surrounded by a +// string boundary or commas. +func (o tagOptions) Contains(optionName string) bool { + if len(o) == 0 { + return false + } + s := string(o) + for s != "" { + var next string + i := strings.Index(s, ",") + if i >= 0 { + s, next = s[:i], s[i+1:] + } + if s == optionName { + return true + } + s = next + } + return false +} diff --git a/pkg/applyconfigurations/traverse.go b/pkg/applyconfigurations/traverse.go new file mode 100644 index 000000000..3c8bb1bb9 --- /dev/null +++ b/pkg/applyconfigurations/traverse.go @@ -0,0 +1,257 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package applyconfigurations + +import ( + "fmt" + "go/ast" + "go/types" + "io" + "path" + "strings" + "unicode" + "unicode/utf8" + + "sigs.k8s.io/controller-tools/pkg/loader" + "sigs.k8s.io/controller-tools/pkg/markers" +) + +// TODO(jpbetz): This was copied from deepcopy-gen + +// codeWriter assists in writing out Go code lines and blocks to a writer. +type codeWriter struct { + out io.Writer +} + +// Line writes a single line. +func (c *codeWriter) Line(line string) { + if _, err := fmt.Fprintln(c.out, line); err != nil { + panic(err) + } +} + +// Linef writes a single line with formatting (as per fmt.Sprintf). +func (c *codeWriter) Linef(line string, args ...interface{}) { + if _, err := fmt.Fprintf(c.out, line+"\n", args...); err != nil { + panic(err) + } +} + +// importsList keeps track of required imports, automatically assigning aliases +// to import statement. +type importsList struct { + byPath map[string]string + byAlias map[string]string + + pkg *loader.Package +} + +// NeedImport marks that the given package is needed in the list of imports, +// returning the ident (import alias) that should be used to reference the package. +func (l *importsList) NeedImport(importPath string) string { + // we get an actual path from Package, which might include venddored + // packages if running on a package in vendor. + if ind := strings.LastIndex(importPath, "/vendor/"); ind != -1 { + importPath = importPath[ind+8: /* len("/vendor/") */] + } + + // check to see if we've already assigned an alias, and just return that. + alias, exists := l.byPath[importPath] + if exists { + return alias + } + + // otherwise, calculate an import alias by joining path parts till we get something unique + restPath, nextWord := path.Split(importPath) + + for otherPath, exists := "", true; exists && otherPath != importPath; otherPath, exists = l.byAlias[alias] { + if restPath == "" { + // do something else to disambiguate if we're run out of parts and + // still have duplicates, somehow + alias += "x" + } + + // can't have a first digit, per Go identifier rules, so just skip them + for firstRune, runeLen := utf8.DecodeRuneInString(nextWord); unicode.IsDigit(firstRune); firstRune, runeLen = utf8.DecodeRuneInString(nextWord) { + nextWord = nextWord[runeLen:] + } + + // make a valid identifier by replacing "bad" characters with underscores + nextWord = strings.Map(func(r rune) rune { + if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' { + return r + } + return '_' + }, nextWord) + + alias = nextWord + alias + if len(restPath) > 0 { + restPath, nextWord = path.Split(restPath[:len(restPath)-1] /* chop off final slash */) + } + } + + l.byPath[importPath] = alias + l.byAlias[alias] = importPath + return alias +} + +// ImportSpecs returns a string form of each import spec +// (i.e. `alias "path/to/import"). Aliases are only present +// when they don't match the package name. +func (l *importsList) ImportSpecs() []string { + res := make([]string, 0, len(l.byPath)) + for importPath, alias := range l.byPath { + pkg := l.pkg.Imports()[importPath] + if pkg != nil && pkg.Name == alias { + // don't print if alias is the same as package name + // (we've already taken care of duplicates). + res = append(res, fmt.Sprintf("%q", importPath)) + } else { + res = append(res, fmt.Sprintf("%s %q", alias, importPath)) + } + } + return res +} + +// namingInfo holds package and syntax for referencing a field, type, +// etc. It's used to allow lazily marking import usage. +// You should generally retrieve the syntax using Syntax. +type namingInfo struct { + // typeInfo is the type being named. + typeInfo types.Type + nameOverride string +} + +// Syntax calculates the code representation of the given type or name, +// and marks that is used (potentially marking an import as used). +func (n *namingInfo) Syntax(basePkg *loader.Package, imports *importsList) string { + if n.nameOverride != "" { + return n.nameOverride + } + + // NB(directxman12): typeInfo.String gets us most of the way there, + // but fails (for us) on named imports, since it uses the full package path. + switch typeInfo := n.typeInfo.(type) { + case *types.Named: + // register that we need an import for this type, + // so we can get the appropriate alias to use. + typeName := typeInfo.Obj() + otherPkg := typeName.Pkg() + if otherPkg == basePkg.Types { + // local import + return typeName.Name() + } + alias := imports.NeedImport(loader.NonVendorPath(otherPkg.Path())) + return alias + "." + typeName.Name() + case *types.Basic: + return typeInfo.String() + case *types.Pointer: + return "*" + (&namingInfo{typeInfo: typeInfo.Elem()}).Syntax(basePkg, imports) + case *types.Slice: + return "[]" + (&namingInfo{typeInfo: typeInfo.Elem()}).Syntax(basePkg, imports) + case *types.Map: + return fmt.Sprintf( + "map[%s]%s", + (&namingInfo{typeInfo: typeInfo.Key()}).Syntax(basePkg, imports), + (&namingInfo{typeInfo: typeInfo.Elem()}).Syntax(basePkg, imports)) + default: + basePkg.AddError(fmt.Errorf("name requested for invalid type: %s", typeInfo)) + return typeInfo.String() + } +} + +// copyMethodMakers makes apply configurations for Go types, +// writing them to its codeWriter. +type applyConfigurationMaker struct { + pkg *loader.Package + *importsList + *codeWriter +} + +// GenerateTypesFor makes makes apply configuration types for the given type, when appropriate +func (c *applyConfigurationMaker) GenerateTypesFor(root *loader.Package, info *markers.TypeInfo) { + typeInfo := root.TypesInfo.TypeOf(info.RawSpec.Name) + if typeInfo == types.Typ[types.Invalid] { + root.AddError(loader.ErrFromNode(fmt.Errorf("unknown type: %s", info.Name), info.RawSpec)) + } + + // TODO(jpbetz): Generate output here + + c.Linef("type %sApplyConfiguration struct {}", info.Name) +} + +// shouldBeApplyConfiguration checks if we're supposed to make apply configurations for the given type. +// +// TODO(jpbetz): Copy over logic for inclusion from requiresApplyConfiguration +func shouldBeApplyConfiguration(pkg *loader.Package, info *markers.TypeInfo) bool { + if !ast.IsExported(info.Name) { + return false + } + + typeInfo := pkg.TypesInfo.TypeOf(info.RawSpec.Name) + if typeInfo == types.Typ[types.Invalid] { + pkg.AddError(loader.ErrFromNode(fmt.Errorf("unknown type: %s", info.Name), info.RawSpec)) + return false + } + + // according to gengo, everything named is an alias, except for an alias to a pointer, + // which is just a pointer, afaict. Just roll with it. + if asPtr, isPtr := typeInfo.(*types.Named).Underlying().(*types.Pointer); isPtr { + typeInfo = asPtr + } + + lastType := typeInfo + if _, isNamed := typeInfo.(*types.Named); isNamed { + for underlyingType := typeInfo.Underlying(); underlyingType != lastType; lastType, underlyingType = underlyingType, underlyingType.Underlying() { + // aliases to other things besides basics need copy methods + // (basics can be straight-up shallow-copied) + if _, isBasic := underlyingType.(*types.Basic); !isBasic { + return true + } + } + } + + // structs are the only thing that's not a basic that apply configurations are generated for + _, isStruct := lastType.(*types.Struct) + if !isStruct { + return false + } + if _, ok := excludeTypes[info.Name]; ok { // TODO(jpbetz): What to do here? + return false + } + var hasJsonTaggedMembers bool + for _, field := range info.Fields { + if _, ok := lookupJsonTags(field); ok { + hasJsonTaggedMembers = true + } + } + return hasJsonTaggedMembers +} + +var ( + rawExtension = "k8s.io/apimachinery/pkg/runtime/RawExtension" + unknown = "k8s.io/apimachinery/pkg/runtime/Unknown" +) +// excludeTypes contains well known types that we do not generate apply configurations for. +// Hard coding because we only have two, very specific types that serve a special purpose +// in the type system here. +var excludeTypes = map[string]struct{}{ + rawExtension: {}, + unknown: {}, + // DO NOT ADD TO THIS LIST. If we need to exclude other types, we should consider allowing the + // go type declarations to be annotated as excluded from this generator. +} diff --git a/pkg/applyconfigurations/zz_generated.markerhelp.go b/pkg/applyconfigurations/zz_generated.markerhelp.go new file mode 100644 index 000000000..84ef16fc2 --- /dev/null +++ b/pkg/applyconfigurations/zz_generated.markerhelp.go @@ -0,0 +1,45 @@ +// +build !ignore_autogenerated + +/* +Copyright2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by helpgen. DO NOT EDIT. + +package applyconfigurations + +import ( + "sigs.k8s.io/controller-tools/pkg/markers" +) + +func (Generator) Help() *markers.DefinitionHelp { + return &markers.DefinitionHelp{ + Category: "", + DetailedHelp: markers.DetailedHelp{ + Summary: "generates code containing apply configuration type implementations.", + Details: "", + }, + FieldHelp: map[string]markers.DetailedHelp{ + "HeaderFile": markers.DetailedHelp{ + Summary: "specifies the header text (e.g. license) to prepend to generated files.", + Details: "", + }, + "Year": markers.DetailedHelp{ + Summary: "specifies the year to substitute for \" YEAR\" in the header file.", + Details: "", + }, + }, + } +}