From d71b091db12dc5d5272ee0c1b77ead216eef9912 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Wed, 12 Apr 2023 17:14:03 +0300 Subject: [PATCH 1/3] Generate provider subpackages Signed-off-by: Alper Rifat Ulucinar --- pkg/config/provider.go | 10 +++++++ pkg/pipeline/run.go | 15 +++++++--- pkg/pipeline/setup.go | 44 ++++++++++++++++++++++++++-- pkg/pipeline/templates/setup.go.tmpl | 4 +-- 4 files changed, 65 insertions(+), 8 deletions(-) diff --git a/pkg/config/provider.go b/pkg/config/provider.go index acf5ee41..73e7c941 100644 --- a/pkg/config/provider.go +++ b/pkg/config/provider.go @@ -89,6 +89,10 @@ type Provider struct { // can add "aws_waf.*" to the list. SkipList []string + // MainTemplate is the template string to be used to render the + // provider subpackage main program. + MainTemplate string + // skippedResourceNames is a list of Terraform resource names // available in the Terraform provider schema, but // not in the include list or in the skip list, meaning that @@ -183,6 +187,12 @@ func WithFeaturesPackage(s string) ProviderOption { } } +func WithMainTemplate(template string) ProviderOption { + return func(p *Provider) { + p.MainTemplate = template + } +} + // NewProvider builds and returns a new Provider from provider // tfjson schema, that is generated using Terraform CLI with: // `terraform providers schema --json` diff --git a/pkg/pipeline/run.go b/pkg/pipeline/run.go index 22ff1290..0d8f5c72 100644 --- a/pkg/pipeline/run.go +++ b/pkg/pipeline/run.go @@ -56,9 +56,14 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo apiVersionPkgList = append(apiVersionPkgList, filepath.Join(pc.ModulePath, p)) } // Add ProviderConfig controller package to the list of controller packages. - controllerPkgList := make([]string, 0) + controllerPkgMap := make(map[string][]string) for _, p := range pc.BasePackages.Controller { - controllerPkgList = append(controllerPkgList, filepath.Join(pc.ModulePath, p)) + tokens := strings.Split(p, "/") + group := tokens[len(tokens)-1] + if group == "providerconfig" { + group = pc.ShortName + } + controllerPkgMap[group] = append(controllerPkgMap[group], filepath.Join(pc.ModulePath, p)) } count := 0 for group, versions := range resourcesGroups { @@ -87,7 +92,8 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo if err != nil { panic(errors.Wrapf(err, "cannot generate controller for resource %s", name)) } - controllerPkgList = append(controllerPkgList, ctrlPkgPath) + sGroup := strings.Split(group, ".")[0] + controllerPkgMap[sGroup] = append(controllerPkgMap[sGroup], ctrlPkgPath) if err := exampleGen.Generate(group, version, resources[name]); err != nil { panic(errors.Wrapf(err, "cannot generate example manifest for resource %s", name)) } @@ -112,7 +118,8 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo if err := NewRegisterGenerator(rootDir, pc.ModulePath).Generate(apiVersionPkgList); err != nil { panic(errors.Wrap(err, "cannot generate register file")) } - if err := NewSetupGenerator(rootDir, pc.ModulePath).Generate(controllerPkgList); err != nil { + // TODO: make a separate generator for the provider main program + if err := NewSetupGenerator(rootDir, pc.ModulePath).Generate(controllerPkgMap, pc.MainTemplate); err != nil { panic(errors.Wrap(err, "cannot generate setup file")) } diff --git a/pkg/pipeline/setup.go b/pkg/pipeline/setup.go index 14eb9f43..96d179b3 100644 --- a/pkg/pipeline/setup.go +++ b/pkg/pipeline/setup.go @@ -5,9 +5,11 @@ Copyright 2021 Upbound Inc. package pipeline import ( + "fmt" "os" "path/filepath" "sort" + "text/template" "github.com/muvaf/typewriter/pkg/wrapper" "github.com/pkg/errors" @@ -18,6 +20,7 @@ import ( // NewSetupGenerator returns a new SetupGenerator. func NewSetupGenerator(rootDir, modulePath string) *SetupGenerator { return &SetupGenerator{ + ProviderPath: filepath.Join(rootDir, "cmd", "provider"), LocalDirectoryPath: filepath.Join(rootDir, "internal", "controller"), LicenseHeaderPath: filepath.Join(rootDir, "hack", "boilerplate.go.txt"), ModulePath: modulePath, @@ -26,6 +29,7 @@ func NewSetupGenerator(rootDir, modulePath string) *SetupGenerator { // SetupGenerator generates controller setup file. type SetupGenerator struct { + ProviderPath string LocalDirectoryPath string LicenseHeaderPath string ModulePath string @@ -33,7 +37,42 @@ type SetupGenerator struct { // Generate writes the setup file with the content produced using given // list of version packages. -func (sg *SetupGenerator) Generate(versionPkgList []string) error { + +func (sg *SetupGenerator) Generate(versionPkgMap map[string][]string, mainTemplate string) error { + t, err := template.New("main").Parse(mainTemplate) + if err != nil { + return errors.Wrap(err, "failed to parse provider main program template") + } + for g, versionPkgList := range versionPkgMap { + if err := sg.generate(g, versionPkgList); err != nil { + return errors.Wrapf(err, "failed to generate controller setup file for group: %s", g) + } + if err := writeMainProgram(sg.ProviderPath, g, t); err != nil { + return errors.Wrapf(err, "failed to write main program for group: %s", g) + } + } + return nil +} + +func writeMainProgram(providerPath, group string, t *template.Template) error { + f := filepath.Join(providerPath, group) + if err := os.MkdirAll(f, 0755); err != nil { + return errors.Wrapf(err, "failed to mkdir provider main program path: %s", f) + } + m, err := os.OpenFile(filepath.Join(f, "main.go"), os.O_WRONLY|os.O_CREATE, 0755) + if err != nil { + return errors.Wrap(err, "failed to open provider main program file") + } + defer m.Close() + if err := t.Execute(m, map[string]any{ + "Group": group, + }); err != nil { + return errors.Wrap(err, "failed to execute provider main program template") + } + return nil +} + +func (sg *SetupGenerator) generate(group string, versionPkgList []string) error { setupFile := wrapper.NewFile(filepath.Join(sg.ModulePath, "apis"), "apis", templates.SetupTemplate, wrapper.WithGenStatement(GenStatement), wrapper.WithHeaderPath(sg.LicenseHeaderPath), @@ -45,7 +84,8 @@ func (sg *SetupGenerator) Generate(versionPkgList []string) error { } vars := map[string]any{ "Aliases": aliases, + "Group": group, } - filePath := filepath.Join(sg.LocalDirectoryPath, "zz_setup.go") + filePath := filepath.Join(sg.LocalDirectoryPath, fmt.Sprintf("zz_%s_setup.go", group)) return errors.Wrap(setupFile.Write(filePath, vars, os.ModePerm), "cannot write setup file") } diff --git a/pkg/pipeline/templates/setup.go.tmpl b/pkg/pipeline/templates/setup.go.tmpl index 9cc1ff68..4a49ebc4 100644 --- a/pkg/pipeline/templates/setup.go.tmpl +++ b/pkg/pipeline/templates/setup.go.tmpl @@ -12,9 +12,9 @@ import ( {{ .Imports }} ) -// Setup creates all controllers with the supplied logger and adds them to +// Setup_{{ .Group }} creates all controllers with the supplied logger and adds them to // the supplied manager. -func Setup(mgr ctrl.Manager, o controller.Options) error { +func Setup_{{ .Group }}(mgr ctrl.Manager, o controller.Options) error { for _, setup := range []func(ctrl.Manager, controller.Options) error{ {{- range $alias := .Aliases }} {{ $alias }}Setup, From 66268861d0cd9988fb6ae4254c853975f2ff9a1a Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Mon, 24 Apr 2023 21:53:51 +0300 Subject: [PATCH 2/3] Rename base package as config Signed-off-by: Alper Rifat Ulucinar --- pkg/config/provider.go | 5 ++- pkg/pipeline/run.go | 18 +++++++---- pkg/pipeline/setup.go | 48 ++++++++++++++++++++-------- pkg/pipeline/templates/setup.go.tmpl | 4 +-- 4 files changed, 52 insertions(+), 23 deletions(-) diff --git a/pkg/config/provider.go b/pkg/config/provider.go index 73e7c941..1384167e 100644 --- a/pkg/config/provider.go +++ b/pkg/config/provider.go @@ -90,7 +90,10 @@ type Provider struct { SkipList []string // MainTemplate is the template string to be used to render the - // provider subpackage main program. + // provider subpackage main program. If this is set, the generated provider + // is broken up into subpackage families partitioned across the API groups. + // A monolithic provider is also generated to + // ensure backwards-compatibility. MainTemplate string // skippedResourceNames is a list of Terraform resource names diff --git a/pkg/pipeline/run.go b/pkg/pipeline/run.go index 0d8f5c72..e4d2ac4b 100644 --- a/pkg/pipeline/run.go +++ b/pkg/pipeline/run.go @@ -22,6 +22,14 @@ type terraformedInput struct { ParametersTypeName string } +const ( + // TODO: we should be careful that there may also exist short groups with + // these names. We can consider making these configurable by the provider + // maintainer. + configPackageName = "config" + monolithPackageName = "monolith" +) + // Run runs the Upjet code generation pipelines. func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo // Note(turkenh): nolint reasoning - this is the main function of the code @@ -58,12 +66,9 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo // Add ProviderConfig controller package to the list of controller packages. controllerPkgMap := make(map[string][]string) for _, p := range pc.BasePackages.Controller { - tokens := strings.Split(p, "/") - group := tokens[len(tokens)-1] - if group == "providerconfig" { - group = pc.ShortName - } - controllerPkgMap[group] = append(controllerPkgMap[group], filepath.Join(pc.ModulePath, p)) + path := filepath.Join(pc.ModulePath, p) + controllerPkgMap[configPackageName] = append(controllerPkgMap[configPackageName], path) + controllerPkgMap[monolithPackageName] = append(controllerPkgMap[monolithPackageName], path) } count := 0 for group, versions := range resourcesGroups { @@ -94,6 +99,7 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo } sGroup := strings.Split(group, ".")[0] controllerPkgMap[sGroup] = append(controllerPkgMap[sGroup], ctrlPkgPath) + controllerPkgMap[monolithPackageName] = append(controllerPkgMap[monolithPackageName], ctrlPkgPath) if err := exampleGen.Generate(group, version, resources[name]); err != nil { panic(errors.Wrapf(err, "cannot generate example manifest for resource %s", name)) } diff --git a/pkg/pipeline/setup.go b/pkg/pipeline/setup.go index 96d179b3..55ef131d 100644 --- a/pkg/pipeline/setup.go +++ b/pkg/pipeline/setup.go @@ -6,6 +6,7 @@ package pipeline import ( "fmt" + "log" "os" "path/filepath" "sort" @@ -35,35 +36,45 @@ type SetupGenerator struct { ModulePath string } -// Generate writes the setup file with the content produced using given -// list of version packages. - +// Generate writes the setup file and the corresponding provider main file +// using the given list of version packages. func (sg *SetupGenerator) Generate(versionPkgMap map[string][]string, mainTemplate string) error { - t, err := template.New("main").Parse(mainTemplate) - if err != nil { - return errors.Wrap(err, "failed to parse provider main program template") + var t *template.Template + if len(mainTemplate) != 0 { + tmpl, err := template.New("main").Parse(mainTemplate) + if err != nil { + return errors.Wrap(err, "failed to parse the provider main program template") + } + t = tmpl + } + if t == nil { + return errors.Wrap(sg.generate("", versionPkgMap[monolithPackageName]), "failed to generate the controller setup file") } for g, versionPkgList := range versionPkgMap { if err := sg.generate(g, versionPkgList); err != nil { - return errors.Wrapf(err, "failed to generate controller setup file for group: %s", g) + return errors.Wrapf(err, "failed to generate the controller setup file for group: %s", g) } - if err := writeMainProgram(sg.ProviderPath, g, t); err != nil { + if err := generateProviderMain(sg.ProviderPath, g, t); err != nil { return errors.Wrapf(err, "failed to write main program for group: %s", g) } } return nil } -func writeMainProgram(providerPath, group string, t *template.Template) error { +func generateProviderMain(providerPath, group string, t *template.Template) error { f := filepath.Join(providerPath, group) - if err := os.MkdirAll(f, 0755); err != nil { + if err := os.MkdirAll(f, 0750); err != nil { return errors.Wrapf(err, "failed to mkdir provider main program path: %s", f) } - m, err := os.OpenFile(filepath.Join(f, "main.go"), os.O_WRONLY|os.O_CREATE, 0755) + m, err := os.OpenFile(filepath.Join(filepath.Clean(f), "zz_main.go"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return errors.Wrap(err, "failed to open provider main program file") } - defer m.Close() + defer func() { + if err := m.Close(); err != nil { + log.Fatalf("Failed to close the templated main %q: %s", f, err.Error()) + } + }() if err := t.Execute(m, map[string]any{ "Group": group, }); err != nil { @@ -82,10 +93,19 @@ func (sg *SetupGenerator) generate(group string, versionPkgList []string) error for i, pkgPath := range versionPkgList { aliases[i] = setupFile.Imports.UsePackage(pkgPath) } + g := "" + if len(group) != 0 { + g = "_" + group + } vars := map[string]any{ "Aliases": aliases, - "Group": group, + "Group": g, + } + filePath := "" + if len(group) == 0 { + filePath = filepath.Join(sg.LocalDirectoryPath, "zz_setup.go") + } else { + filePath = filepath.Join(sg.LocalDirectoryPath, fmt.Sprintf("zz_%s_setup.go", group)) } - filePath := filepath.Join(sg.LocalDirectoryPath, fmt.Sprintf("zz_%s_setup.go", group)) return errors.Wrap(setupFile.Write(filePath, vars, os.ModePerm), "cannot write setup file") } diff --git a/pkg/pipeline/templates/setup.go.tmpl b/pkg/pipeline/templates/setup.go.tmpl index 4a49ebc4..72b9f5a4 100644 --- a/pkg/pipeline/templates/setup.go.tmpl +++ b/pkg/pipeline/templates/setup.go.tmpl @@ -12,9 +12,9 @@ import ( {{ .Imports }} ) -// Setup_{{ .Group }} creates all controllers with the supplied logger and adds them to +// Setup{{ .Group }} creates all controllers with the supplied logger and adds them to // the supplied manager. -func Setup_{{ .Group }}(mgr ctrl.Manager, o controller.Options) error { +func Setup{{ .Group }}(mgr ctrl.Manager, o controller.Options) error { for _, setup := range []func(ctrl.Manager, controller.Options) error{ {{- range $alias := .Aliases }} {{ $alias }}Setup, From 37babb8f78092b9da428314fde9f8871ea148082 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Thu, 27 Apr 2023 16:15:47 +0300 Subject: [PATCH 3/3] Rename pipeline.SetupGenerator as ProviderGenerator Signed-off-by: Alper Rifat Ulucinar --- pkg/pipeline/run.go | 5 +++-- pkg/pipeline/setup.go | 14 +++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/pkg/pipeline/run.go b/pkg/pipeline/run.go index e4d2ac4b..dc74f61d 100644 --- a/pkg/pipeline/run.go +++ b/pkg/pipeline/run.go @@ -124,8 +124,9 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo if err := NewRegisterGenerator(rootDir, pc.ModulePath).Generate(apiVersionPkgList); err != nil { panic(errors.Wrap(err, "cannot generate register file")) } - // TODO: make a separate generator for the provider main program - if err := NewSetupGenerator(rootDir, pc.ModulePath).Generate(controllerPkgMap, pc.MainTemplate); err != nil { + // Generate the provider, + // i.e. the setup function and optionally the provider's main program. + if err := NewProviderGenerator(rootDir, pc.ModulePath).Generate(controllerPkgMap, pc.MainTemplate); err != nil { panic(errors.Wrap(err, "cannot generate setup file")) } diff --git a/pkg/pipeline/setup.go b/pkg/pipeline/setup.go index 55ef131d..f239a666 100644 --- a/pkg/pipeline/setup.go +++ b/pkg/pipeline/setup.go @@ -18,9 +18,9 @@ import ( "github.com/upbound/upjet/pkg/pipeline/templates" ) -// NewSetupGenerator returns a new SetupGenerator. -func NewSetupGenerator(rootDir, modulePath string) *SetupGenerator { - return &SetupGenerator{ +// NewProviderGenerator returns a new ProviderGenerator. +func NewProviderGenerator(rootDir, modulePath string) *ProviderGenerator { + return &ProviderGenerator{ ProviderPath: filepath.Join(rootDir, "cmd", "provider"), LocalDirectoryPath: filepath.Join(rootDir, "internal", "controller"), LicenseHeaderPath: filepath.Join(rootDir, "hack", "boilerplate.go.txt"), @@ -28,8 +28,8 @@ func NewSetupGenerator(rootDir, modulePath string) *SetupGenerator { } } -// SetupGenerator generates controller setup file. -type SetupGenerator struct { +// ProviderGenerator generates controller setup file. +type ProviderGenerator struct { ProviderPath string LocalDirectoryPath string LicenseHeaderPath string @@ -38,7 +38,7 @@ type SetupGenerator struct { // Generate writes the setup file and the corresponding provider main file // using the given list of version packages. -func (sg *SetupGenerator) Generate(versionPkgMap map[string][]string, mainTemplate string) error { +func (sg *ProviderGenerator) Generate(versionPkgMap map[string][]string, mainTemplate string) error { var t *template.Template if len(mainTemplate) != 0 { tmpl, err := template.New("main").Parse(mainTemplate) @@ -83,7 +83,7 @@ func generateProviderMain(providerPath, group string, t *template.Template) erro return nil } -func (sg *SetupGenerator) generate(group string, versionPkgList []string) error { +func (sg *ProviderGenerator) generate(group string, versionPkgList []string) error { setupFile := wrapper.NewFile(filepath.Join(sg.ModulePath, "apis"), "apis", templates.SetupTemplate, wrapper.WithGenStatement(GenStatement), wrapper.WithHeaderPath(sg.LicenseHeaderPath),