-
Notifications
You must be signed in to change notification settings - Fork 23
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #113 from ulucinar/lint-smaller-providers
Add provider family linter
- Loading branch information
Showing
4 changed files
with
365 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,328 @@ | ||
| // Copyright 2023 Upbound Inc. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| // main package for the ssop-linter tool, | ||
| // a linter for checking the packages of a provider family. | ||
| package main | ||
|
|
||
| import ( | ||
| "archive/tar" | ||
| "context" | ||
| "fmt" | ||
| "io" | ||
| "log" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
|
|
||
| "github.com/google/go-containerregistry/pkg/name" | ||
| v1 "github.com/google/go-containerregistry/pkg/v1" | ||
| "github.com/google/go-containerregistry/pkg/v1/remote" | ||
| "github.com/pkg/errors" | ||
| "gopkg.in/alecthomas/kingpin.v2" | ||
| admv1 "k8s.io/api/admissionregistration/v1" | ||
| extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" | ||
| extv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" | ||
| "k8s.io/apimachinery/pkg/runtime" | ||
| "k8s.io/apimachinery/pkg/runtime/serializer/json" | ||
|
|
||
| xpkgparser "github.com/crossplane/crossplane-runtime/pkg/parser" | ||
| xpv1 "github.com/crossplane/crossplane/apis/apiextensions/v1" | ||
| pkgmetav1 "github.com/crossplane/crossplane/apis/pkg/meta/v1" | ||
| pkgmetav1alpha1 "github.com/crossplane/crossplane/apis/pkg/meta/v1alpha1" | ||
| ) | ||
|
|
||
| const ( | ||
| streamFile = "package.yaml" | ||
| labelFamily = "pkg.crossplane.io/provider-family" | ||
| annotationAuthConfig = "auth.upbound.io/config" | ||
| ) | ||
|
|
||
| var ( | ||
| metaScheme, objScheme *runtime.Scheme | ||
| ) | ||
|
|
||
| type ssopLinterConfig struct { | ||
| crdDir *string | ||
| providerName *string | ||
| providerVersion *string | ||
| packageRepoOrg *string | ||
| checkAuthAnnotation *bool | ||
| } | ||
|
|
||
| func main() { | ||
| app := kingpin.New("ssop-linter", "Linter for the official provider families").DefaultEnvars() | ||
| // family command | ||
| familyCmd := app.Command("family", "Checks whether all CRDs generated for a provider family are packaged in the corresponding service-scoped provider and checks the provider metadata.") | ||
|
|
||
| config := &ssopLinterConfig{} | ||
| config.crdDir = familyCmd.Flag("crd-dir", "Directory containing all the generated CRDs for the provider family.").Envar("CRD_DIR").Default("./package/crds").ExistingDir() | ||
| config.providerName = familyCmd.Flag("provider-name", `Provider name such as "aws".`).Envar("PROVIDER_NAME").Required().String() | ||
| config.providerVersion = familyCmd.Flag("provider-version", `Provider family tag to check such as "v0.37.0".`).Envar("PROVIDER_NAME").Required().String() | ||
| config.packageRepoOrg = familyCmd.Flag("package-repo-org", `Package repo organization with the registry host for the provider family.`).Envar("PACKAGE_REPO_ORG").Default("xpkg.upbound.io/upbound").String() | ||
| config.checkAuthAnnotation = familyCmd.Flag("check-auth-annotation", `Check Upbound authentication annotation on the ProviderConfig CRD in the provider family's config package.'`).Envar("CHECK_AUTH_ANNOTATION").Default("false").Bool() | ||
|
|
||
| cmd := kingpin.MustParse(app.Parse(os.Args[1:])) | ||
| if cmd == familyCmd.FullCommand() { | ||
| kingpin.FatalIfError(lint(config), "Linter error reported: ") | ||
| } | ||
| } | ||
|
|
||
| func lint(config *ssopLinterConfig) error { //nolint:gocyclo // sequential flow easier to follow | ||
| packageURLFormat := *config.packageRepoOrg + "/%s" | ||
| packageURLFormatTagged := packageURLFormat + ":%s" | ||
| familyConfigPackageName := fmt.Sprintf("provider-family-%s", *config.providerName) | ||
| familyConfigPackageRef := fmt.Sprintf(packageURLFormat, familyConfigPackageName) | ||
| providerConfigCRDName := fmt.Sprintf("providerconfigs.%s.upbound.io", *config.providerName) | ||
|
|
||
| entries, err := os.ReadDir(*config.crdDir) | ||
| if err != nil { | ||
| return errors.Wrapf(err, "failed to list CRDs from directory: %s", *config.crdDir) | ||
| } | ||
|
|
||
| metaMap := make(map[string]*xpkgparser.Package) | ||
| for _, e := range entries { | ||
| info, err := e.Info() | ||
| if err != nil { | ||
| return errors.Wrap(err, "failed to get directory entry info") | ||
| } | ||
| if info.IsDir() { | ||
| continue | ||
| } | ||
|
|
||
| crd, err := loadCRD(filepath.Join(*config.crdDir, e.Name())) | ||
| if err != nil { | ||
| return errors.Wrapf(err, "failed to load CRD from path: %s", e.Name()) | ||
| } | ||
| log.Println("Checking CRD: ", crd.Name) | ||
| group := strings.Split(crd.Spec.Group, ".")[0] | ||
| repoName := fmt.Sprintf("provider-%s-%s", *config.providerName, group) | ||
| if group == *config.providerName { | ||
| repoName = familyConfigPackageName | ||
| } | ||
|
|
||
| if _, ok := metaMap[group]; !ok { | ||
| packageURL := fmt.Sprintf(packageURLFormatTagged, repoName, *config.providerVersion) | ||
| xpkg, err := getPackageMetadata(context.TODO(), packageURL) | ||
| if err != nil { | ||
| return errors.Wrapf(err, "failed to get package metadata for provider package: %s", packageURL) | ||
| } | ||
| metaMap[group] = xpkg | ||
| } | ||
|
|
||
| // check if the provider contains the CRD | ||
| found := false | ||
| for _, o := range metaMap[group].GetObjects() { | ||
| pCRD := o.(*extv1.CustomResourceDefinition) | ||
| if pCRD.Name == crd.Name { | ||
| found = true | ||
| break | ||
| } | ||
| } | ||
| if !found { | ||
| log.Fatalln("CRD not found: ", e.Name()) | ||
| } | ||
|
|
||
| // if `--check-auth-annotation` command-line option has been specified | ||
| // and the CRD is the ProviderConfig CRD for the provider, then | ||
| // check if the ProviderConfig CRD in the package metadata contains | ||
| // the Upbound authentication annotation | ||
| if *config.checkAuthAnnotation && crd.Name == providerConfigCRDName { | ||
| if err := checkAuthAnnotation(metaMap[group], providerConfigCRDName); err != nil { | ||
| log.Fatalln("Upbound authentication annotation check failed: ", err.Error()) | ||
| } | ||
| } | ||
|
|
||
| // check if the Provider.pkg has a family label | ||
| foundMeta := false | ||
| for _, o := range metaMap[group].GetMeta() { | ||
| m, ok := o.(*pkgmetav1alpha1.Provider) | ||
| if !ok { | ||
| continue | ||
| } | ||
| foundMeta = true | ||
|
|
||
| // check family label | ||
| foundLabel := false | ||
| for k, v := range m.Labels { | ||
| if k == labelFamily && v == familyConfigPackageName { | ||
| foundLabel = true | ||
| break | ||
| } | ||
| } | ||
| if foundLabel && repoName == familyConfigPackageName { | ||
| log.Fatalln("Family label found on family config package: ", familyConfigPackageName) | ||
| } | ||
| if !foundLabel && repoName != familyConfigPackageName { | ||
| log.Fatalln("Family label not found: ", e.Name()) | ||
| } | ||
|
|
||
| // check dependency to family config package | ||
| if group != *config.providerName && (len(m.Spec.DependsOn) != 1 || m.Spec.DependsOn[0].Provider == nil || *m.Spec.DependsOn[0].Provider != familyConfigPackageRef) { | ||
| log.Fatalln("Missing dependency to family config package: ", e.Name()) | ||
| } | ||
| break | ||
| } | ||
| if !foundMeta { | ||
| log.Fatalln("Provider package metadata not found: ", e.Name()) | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func checkAuthAnnotation(xpkg *xpkgparser.Package, providerConfigCRDName string) error { | ||
| for _, o := range xpkg.GetObjects() { | ||
| switch crd := o.(type) { | ||
| case *extv1.CustomResourceDefinition: | ||
| if crd.Name != providerConfigCRDName { | ||
| continue | ||
| } | ||
| if crd.Annotations == nil { | ||
| return errors.New("no annotations on the ProviderConfig CRD") | ||
| } | ||
| if crd.Annotations[annotationAuthConfig] == "" { | ||
| return errors.Errorf("ProviderConfig CRD does not have the %q annotation", annotationAuthConfig) | ||
| } | ||
| return nil | ||
| default: | ||
| continue | ||
| } | ||
| } | ||
| return errors.New("no ProviderConfig CRD in package metadata") | ||
| } | ||
|
|
||
| func loadCRD(f string) (*extv1.CustomResourceDefinition, error) { | ||
| do := json.NewSerializerWithOptions(json.DefaultMetaFactory, objScheme, objScheme, json.SerializerOptions{Yaml: true}) | ||
| buff, err := os.ReadFile(filepath.Clean(f)) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| o, _, err := do.Decode(buff, nil, nil) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return o.(*extv1.CustomResourceDefinition), nil | ||
| } | ||
|
|
||
| type readCloser struct { | ||
| reader io.Reader | ||
| } | ||
|
|
||
| func (r *readCloser) Close() error { | ||
| return nil | ||
| } | ||
|
|
||
| func (r *readCloser) Read(p []byte) (n int, err error) { | ||
| return r.reader.Read(p) | ||
| } | ||
|
|
||
| func getPackageMetadata(ctx context.Context, packageURL string) (*xpkgparser.Package, error) { | ||
| ref, err := name.ParseReference(packageURL) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| img, err := remote.Image(ref) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| cfgFile, err := img.ConfigFile() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| digest := "" | ||
| for k, v := range cfgFile.Config.Labels { | ||
| if strings.Contains(v, "base") { | ||
| digest = strings.Join(strings.Split(k, ":")[1:], ":") | ||
| } | ||
| } | ||
| layer, err := getBaseLayer(img, digest) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return extractPackageMetadata(ctx, layer) | ||
| } | ||
|
|
||
| func getBaseLayer(img v1.Image, digest string) (v1.Layer, error) { | ||
| layers, err := img.Layers() | ||
| if err != nil { | ||
| return nil, errors.Wrap(err, "cannot get image layers") | ||
| } | ||
| var layer v1.Layer | ||
| for _, l := range layers { | ||
| d, err := l.Digest() | ||
| if err != nil { | ||
| return nil, errors.Wrap(err, "cannot compute image layer's digest") | ||
| } | ||
| if d.String() == digest { | ||
| layer = l | ||
| break | ||
| } | ||
| } | ||
| if layer == nil { | ||
| return nil, errors.New("cannot find the base layer in the image") | ||
| } | ||
| return layer, nil | ||
| } | ||
|
|
||
| func extractPackageMetadata(ctx context.Context, layer v1.Layer) (*xpkgparser.Package, error) { | ||
| rc, err := layer.Uncompressed() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| defer func() { _ = rc.Close() }() | ||
| tr := tar.NewReader(rc) | ||
| for { | ||
| hdr, err := tr.Next() | ||
| if errors.Is(err, io.EOF) { | ||
| // End of tar archive | ||
| break | ||
| } | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if hdr.Name != streamFile { | ||
| continue | ||
| } | ||
| parser := xpkgparser.New(metaScheme, objScheme) | ||
| xpkg, err := parser.Parse(ctx, &readCloser{ | ||
| reader: tr, | ||
| }) | ||
| return xpkg, errors.Wrap(err, "cannot parse package metadata") | ||
| } | ||
| return nil, errors.Errorf("%s not found in the base layer", streamFile) | ||
| } | ||
|
|
||
| func init() { | ||
| metaScheme = runtime.NewScheme() | ||
| if err := pkgmetav1alpha1.SchemeBuilder.AddToScheme(metaScheme); err != nil { | ||
| kingpin.FatalIfError(err, "Failed to add package metadata v1alpha1 APIs to the runtime scheme: ") | ||
| } | ||
| if err := pkgmetav1.SchemeBuilder.AddToScheme(metaScheme); err != nil { | ||
| kingpin.FatalIfError(err, "Failed to add package metadata v1 APIs to the runtime scheme: ") | ||
| } | ||
|
|
||
| objScheme = runtime.NewScheme() | ||
| if err := xpv1.AddToScheme(objScheme); err != nil { | ||
| kingpin.FatalIfError(err, "Failed to add Crossplane extension v1 APIs to the runtime scheme: ") | ||
| } | ||
| if err := extv1beta1.AddToScheme(objScheme); err != nil { | ||
| kingpin.FatalIfError(err, "Failed to add Crossplane extension v1beta1 APIs to the runtime scheme: ") | ||
| } | ||
| if err := extv1.AddToScheme(objScheme); err != nil { | ||
| kingpin.FatalIfError(err, "Failed to add Kubernetes API Server extension v1 APIs to the runtime scheme: ") | ||
| } | ||
| if err := admv1.AddToScheme(objScheme); err != nil { | ||
| kingpin.FatalIfError(err, "Failed to add Kubernetes admission v1 APIs to the runtime scheme: ") | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.