From 1ab84dfa79754eaeb82451cb230d0efe025bd409 Mon Sep 17 00:00:00 2001 From: Joe Lanford Date: Tue, 10 Aug 2021 13:41:56 -0400 Subject: [PATCH] render: support rendering DC from packagemanifest and bundle directories --- internal/action/render.go | 136 ++++++++++++++-- internal/action/render_test.go | 153 +++++++++++++++++- .../0.1.0/foo.v0.1.0.csv.yaml | 19 +++ .../0.1.0/foos.test.foo.crd.yaml | 12 ++ .../0.2.0/foo.v0.2.0.csv.yaml | 23 +++ .../0.2.0/foos.test.foo.crd.yaml | 12 ++ .../foo.package.yaml | 7 + 7 files changed, 342 insertions(+), 20 deletions(-) create mode 100644 internal/action/testdata/foo-index-v0.2.0-package-manifest/0.1.0/foo.v0.1.0.csv.yaml create mode 100644 internal/action/testdata/foo-index-v0.2.0-package-manifest/0.1.0/foos.test.foo.crd.yaml create mode 100644 internal/action/testdata/foo-index-v0.2.0-package-manifest/0.2.0/foo.v0.2.0.csv.yaml create mode 100644 internal/action/testdata/foo-index-v0.2.0-package-manifest/0.2.0/foos.test.foo.crd.yaml create mode 100644 internal/action/testdata/foo-index-v0.2.0-package-manifest/foo.package.yaml diff --git a/internal/action/render.go b/internal/action/render.go index 2c9c9b3da..15254b8fb 100644 --- a/internal/action/render.go +++ b/internal/action/render.go @@ -13,6 +13,7 @@ import ( "github.com/h2non/filetype" "github.com/h2non/filetype/matchers" + "github.com/operator-framework/api/pkg/operators/v1alpha1" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/sets" @@ -30,10 +31,12 @@ type RefType uint const ( RefBundleImage RefType = 1 << iota + RefBundleDir RefSqliteImage RefSqliteFile RefDCImage RefDCDir + RefPackageManifestDir RefAll = 0 ) @@ -100,25 +103,48 @@ func (r Render) createRegistry() (*containerdregistry.Registry, error) { } func (r Render) renderReference(ctx context.Context, ref string) (*declcfg.DeclarativeConfig, error) { - if stat, serr := os.Stat(ref); serr == nil { - if stat.IsDir() { - if !r.AllowedRefMask.Allowed(RefDCDir) { - return nil, fmt.Errorf("cannot render declarative config directory: %w", ErrNotAllowed) + stat, err := os.Stat(ref) + if err != nil { + return r.imageToDeclcfg(ctx, ref) + } + if stat.IsDir() { + dirEntries, err := os.ReadDir(ref) + if err != nil { + return nil, err + } + if isBundle(dirEntries) { + // Looks like a bundle directory + if !r.AllowedRefMask.Allowed(RefBundleDir) { + return nil, fmt.Errorf("cannot render bundle directory: %w", ErrNotAllowed) } - return declcfg.LoadFS(os.DirFS(ref)) - } else { - // The only supported file type is an sqlite DB file, - // since declarative configs will be in a directory. - if err := checkDBFile(ref); err != nil { + img, err := registry.NewImageInput(image.SimpleReference(""), ref) + if err != nil { return nil, err } - if !r.AllowedRefMask.Allowed(RefSqliteFile) { - return nil, fmt.Errorf("cannot render sqlite file: %w", ErrNotAllowed) + return bundleToDeclcfg(img.Bundle, true) + } else if isPackageManifest(dirEntries) { + // Looks like a package manifest directory + if !r.AllowedRefMask.Allowed(RefPackageManifestDir) { + return nil, fmt.Errorf("cannot render package manifest directory: %w", ErrNotAllowed) } - return sqliteToDeclcfg(ctx, ref) + return renderPackageManifest(ctx, ref) } + + // Otherwise, assume it is a declarative config root directory. + if !r.AllowedRefMask.Allowed(RefDCDir) { + return nil, fmt.Errorf("cannot render declarative config directory: %w", ErrNotAllowed) + } + return declcfg.LoadFS(os.DirFS(ref)) + } + // The only supported file type is an sqlite DB file, + // since declarative configs will be in a directory. + if err := checkDBFile(ref); err != nil { + return nil, err + } + if !r.AllowedRefMask.Allowed(RefSqliteFile) { + return nil, fmt.Errorf("cannot render sqlite file: %w", ErrNotAllowed) } - return r.imageToDeclcfg(ctx, ref) + return sqliteToDeclcfg(ctx, ref) } func (r Render) imageToDeclcfg(ctx context.Context, imageRef string) (*declcfg.DeclarativeConfig, error) { @@ -165,7 +191,7 @@ func (r Render) imageToDeclcfg(ctx context.Context, imageRef string) (*declcfg.D return nil, err } - cfg, err = bundleToDeclcfg(img.Bundle) + cfg, err = bundleToDeclcfg(img.Bundle, false) if err != nil { return nil, err } @@ -273,7 +299,7 @@ func populateDBRelatedImages(ctx context.Context, cfg *declcfg.DeclarativeConfig return nil } -func bundleToDeclcfg(bundle *registry.Bundle) (*declcfg.DeclarativeConfig, error) { +func bundleToDeclcfg(bundle *registry.Bundle, inlineManifests bool) (*declcfg.DeclarativeConfig, error) { bundleProperties, err := registry.PropertiesFromBundle(bundle) if err != nil { return nil, fmt.Errorf("get properties for bundle %q: %v", bundle.Name, err) @@ -283,6 +309,24 @@ func bundleToDeclcfg(bundle *registry.Bundle) (*declcfg.DeclarativeConfig, error return nil, fmt.Errorf("get related images for bundle %q: %v", bundle.Name, err) } + var ( + objs []string + csvJSON string + ) + if inlineManifests { + for _, obj := range bundle.Objects { + objJSON, err := json.Marshal(obj) + if err != nil { + return nil, fmt.Errorf("marshal bundle object %q (kind: %q)", obj.GetName(), obj.GetKind()) + } + bundleProperties = append(bundleProperties, property.MustBuildBundleObjectData(objJSON)) + objs = append(objs, string(objJSON)) + if obj.GetKind() == v1alpha1.ClusterServiceVersionKind && csvJSON == "" { + csvJSON = string(objJSON) + } + } + } + dBundle := declcfg.Bundle{ Schema: "olm.bundle", Name: bundle.Name, @@ -290,6 +334,8 @@ func bundleToDeclcfg(bundle *registry.Bundle) (*declcfg.DeclarativeConfig, error Image: bundle.BundleImage, Properties: bundleProperties, RelatedImages: relatedImages, + Objects: objs, + CsvJSON: csvJSON, } return &declcfg.DeclarativeConfig{Bundles: []declcfg.Bundle{dBundle}}, nil @@ -343,3 +389,63 @@ func combineConfigs(cfgs []declcfg.DeclarativeConfig) *declcfg.DeclarativeConfig } return out } + +func isBundle(entries []os.DirEntry) bool { + foundManifests := false + foundMetadata := false + for _, e := range entries { + if e.IsDir() { + switch e.Name() { + case "manifests": + foundManifests = true + case "metadata": + foundMetadata = true + } + } + if foundMetadata && foundManifests { + return true + } + } + return false +} + +func isPackageManifest(entries []os.DirEntry) bool { + for _, e := range entries { + if strings.HasSuffix(e.Name(), ".package.yaml") || strings.HasSuffix(e.Name(), ".package.yml") { + return true + } + } + return false +} + +func renderPackageManifest(ctx context.Context, ref string) (*declcfg.DeclarativeConfig, error) { + tmpDB, err := os.CreateTemp("", "opm-render-pm-") + if err != nil { + return nil, err + } + if err := tmpDB.Close(); err != nil { + return nil, err + } + + db, err := sqlite.Open(tmpDB.Name()) + if err != nil { + return nil, err + } + defer db.Close() + defer os.RemoveAll(tmpDB.Name()) + + dbLoader, err := sqlite.NewSQLLiteLoader(db) + if err != nil { + return nil, err + } + if err := dbLoader.Migrate(context.TODO()); err != nil { + return nil, err + } + + loader := sqlite.NewSQLLoaderForDirectory(dbLoader, ref) + if err := loader.Populate(); err != nil { + return nil, fmt.Errorf("error loading manifests from directory: %s", err) + } + + return sqliteToDeclcfg(ctx, tmpDB.Name()) +} diff --git a/internal/action/render_test.go b/internal/action/render_test.go index a9017f4f4..1f03f1440 100644 --- a/internal/action/render_test.go +++ b/internal/action/render_test.go @@ -402,6 +402,109 @@ func TestRender(t *testing.T) { }, assertion: require.NoError, }, + { + name: "Success/BundleDirectory", + render: action.Render{ + Refs: []string{"testdata/foo-bundle-v0.2.0"}, + Registry: reg, + }, + expectCfg: &declcfg.DeclarativeConfig{ + Bundles: []declcfg.Bundle{ + { + Schema: "olm.bundle", + Name: "foo.v0.2.0", + Package: "foo", + Properties: []property.Property{ + property.MustBuildChannel("beta", "foo.v0.1.0"), + property.MustBuildChannel("stable", "foo.v0.1.0"), + property.MustBuildGVK("test.foo", "v1", "Foo"), + property.MustBuildGVKRequired("test.bar", "v1alpha1", "Bar"), + property.MustBuildPackage("foo", "0.2.0"), + property.MustBuildPackageRequired("bar", "<0.1.0"), + property.MustBuildSkipRange("<0.2.0"), + property.MustBuildSkips("foo.v0.1.1"), + property.MustBuildSkips("foo.v0.1.2"), + property.MustBuildBundleObjectData(foov2csv), + property.MustBuildBundleObjectData(foov2crd), + }, + RelatedImages: []declcfg.RelatedImage{ + { + Name: "operator", + Image: "test.registry/foo-operator/foo:v0.2.0", + }, + }, + Objects: []string{string(foov2csv), string(foov2crd)}, + CsvJSON: string(foov2csv), + }, + }, + }, + assertion: require.NoError, + }, + { + name: "Success/PackageManifestDirectory", + render: action.Render{ + Refs: []string{"testdata/foo-index-v0.2.0-package-manifest"}, + Registry: reg, + }, + expectCfg: &declcfg.DeclarativeConfig{ + Packages: []declcfg.Package{ + { + Schema: "olm.package", + Name: "foo", + DefaultChannel: "stable", + }, + }, + Bundles: []declcfg.Bundle{ + { + Schema: "olm.bundle", + Name: "foo.v0.1.0", + Package: "foo", + Properties: []property.Property{ + property.MustBuildChannel("beta", ""), + property.MustBuildChannel("stable", ""), + property.MustBuildGVK("test.foo", "v1", "Foo"), + property.MustBuildPackage("foo", "0.1.0"), + property.MustBuildSkipRange("<0.1.0"), + property.MustBuildBundleObjectData(foov1csv), + property.MustBuildBundleObjectData(foov1crd), + }, + RelatedImages: []declcfg.RelatedImage{ + { + Name: "operator", + Image: "test.registry/foo-operator/foo:v0.1.0", + }, + }, + CsvJSON: string(foov1csv), + Objects: []string{string(foov1csv), string(foov1crd)}, + }, + { + Schema: "olm.bundle", + Name: "foo.v0.2.0", + Package: "foo", + Properties: []property.Property{ + property.MustBuildChannel("beta", "foo.v0.1.0"), + property.MustBuildChannel("stable", "foo.v0.1.0"), + property.MustBuildGVK("test.foo", "v1", "Foo"), + property.MustBuildPackage("foo", "0.2.0"), + property.MustBuildSkipRange("<0.2.0"), + property.MustBuildSkips("foo.v0.1.1"), + property.MustBuildSkips("foo.v0.1.2"), + property.MustBuildBundleObjectData(foov2csv), + property.MustBuildBundleObjectData(foov2crd), + }, + RelatedImages: []declcfg.RelatedImage{ + { + Name: "operator", + Image: "test.registry/foo-operator/foo:v0.2.0", + }, + }, + CsvJSON: string(foov2csv), + Objects: []string{string(foov2csv), string(foov2crd)}, + }, + }, + }, + assertion: require.NoError, + }, } for _, s := range specs { @@ -446,7 +549,7 @@ func TestAllowRefMask(t *testing.T) { render: action.Render{ Refs: []string{"test.registry/foo-operator/foo-index-sqlite:v0.2.0"}, Registry: reg, - AllowedRefMask: action.RefDCImage | action.RefDCDir | action.RefSqliteFile | action.RefBundleImage, + AllowedRefMask: action.RefDCImage | action.RefDCDir | action.RefSqliteFile | action.RefBundleImage | action.RefBundleDir | action.RefPackageManifestDir, }, expectErr: action.ErrNotAllowed, }, @@ -464,7 +567,7 @@ func TestAllowRefMask(t *testing.T) { render: action.Render{ Refs: []string{dbFile}, Registry: reg, - AllowedRefMask: action.RefDCImage | action.RefDCDir | action.RefSqliteImage | action.RefBundleImage, + AllowedRefMask: action.RefDCImage | action.RefDCDir | action.RefSqliteImage | action.RefBundleImage | action.RefBundleDir | action.RefPackageManifestDir, }, expectErr: action.ErrNotAllowed, }, @@ -482,7 +585,7 @@ func TestAllowRefMask(t *testing.T) { render: action.Render{ Refs: []string{"test.registry/foo-operator/foo-index-declcfg:v0.2.0"}, Registry: reg, - AllowedRefMask: action.RefDCDir | action.RefSqliteImage | action.RefSqliteFile | action.RefBundleImage, + AllowedRefMask: action.RefDCDir | action.RefSqliteImage | action.RefSqliteFile | action.RefBundleImage | action.RefBundleDir | action.RefPackageManifestDir, }, expectErr: action.ErrNotAllowed, }, @@ -500,7 +603,7 @@ func TestAllowRefMask(t *testing.T) { render: action.Render{ Refs: []string{"testdata/foo-index-v0.2.0-declcfg"}, Registry: reg, - AllowedRefMask: action.RefDCImage | action.RefSqliteImage | action.RefSqliteFile | action.RefBundleImage, + AllowedRefMask: action.RefDCImage | action.RefSqliteImage | action.RefSqliteFile | action.RefBundleImage | action.RefBundleDir | action.RefPackageManifestDir, }, expectErr: action.ErrNotAllowed, }, @@ -518,7 +621,43 @@ func TestAllowRefMask(t *testing.T) { render: action.Render{ Refs: []string{"test.registry/foo-operator/foo-bundle:v0.2.0"}, Registry: reg, - AllowedRefMask: action.RefDCImage | action.RefDCDir | action.RefSqliteImage | action.RefSqliteFile, + AllowedRefMask: action.RefDCImage | action.RefDCDir | action.RefSqliteImage | action.RefSqliteFile | action.RefBundleDir | action.RefPackageManifestDir, + }, + expectErr: action.ErrNotAllowed, + }, + { + name: "BundleDir/Allowed", + render: action.Render{ + Refs: []string{"testdata/foo-bundle-v0.2.0"}, + Registry: reg, + AllowedRefMask: action.RefBundleDir, + }, + expectErr: nil, + }, + { + name: "BundleDir/NotAllowed", + render: action.Render{ + Refs: []string{"testdata/foo-bundle-v0.2.0"}, + Registry: reg, + AllowedRefMask: action.RefDCImage | action.RefDCDir | action.RefSqliteImage | action.RefSqliteFile | action.RefBundleImage | action.RefPackageManifestDir, + }, + expectErr: action.ErrNotAllowed, + }, + { + name: "PackageManifestDir/Allowed", + render: action.Render{ + Refs: []string{"testdata/foo-index-v0.2.0-package-manifest"}, + Registry: reg, + AllowedRefMask: action.RefPackageManifestDir, + }, + expectErr: nil, + }, + { + name: "PackageManifestDir/NotAllowed", + render: action.Render{ + Refs: []string{"testdata/foo-index-v0.2.0-package-manifest"}, + Registry: reg, + AllowedRefMask: action.RefDCImage | action.RefDCDir | action.RefSqliteImage | action.RefSqliteFile | action.RefBundleImage | action.RefBundleDir, }, expectErr: action.ErrNotAllowed, }, @@ -531,6 +670,8 @@ func TestAllowRefMask(t *testing.T) { "test.registry/foo-operator/foo-index-declcfg:v0.2.0", "testdata/foo-index-v0.2.0-declcfg", "test.registry/foo-operator/foo-bundle:v0.2.0", + "testdata/foo-bundle-v0.2.0", + "testdata/foo-index-v0.2.0-package-manifest", }, Registry: reg, }, @@ -564,6 +705,8 @@ func TestAllowRefMaskAllowed(t *testing.T) { action.RefSqliteImage, action.RefSqliteFile, action.RefBundleImage, + action.RefBundleDir, + action.RefPackageManifestDir, }, fail: []action.RefType{}, }, diff --git a/internal/action/testdata/foo-index-v0.2.0-package-manifest/0.1.0/foo.v0.1.0.csv.yaml b/internal/action/testdata/foo-index-v0.2.0-package-manifest/0.1.0/foo.v0.1.0.csv.yaml new file mode 100644 index 000000000..189b6849e --- /dev/null +++ b/internal/action/testdata/foo-index-v0.2.0-package-manifest/0.1.0/foo.v0.1.0.csv.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + name: foo.v0.1.0 + annotations: + olm.skipRange: <0.1.0 +spec: + displayName: "Foo Operator" + customresourcedefinitions: + owned: + - group: test.foo + version: v1 + kind: Foo + name: foos.test.foo + version: 0.1.0 + relatedImages: + - name: operator + image: test.registry/foo-operator/foo:v0.1.0 diff --git a/internal/action/testdata/foo-index-v0.2.0-package-manifest/0.1.0/foos.test.foo.crd.yaml b/internal/action/testdata/foo-index-v0.2.0-package-manifest/0.1.0/foos.test.foo.crd.yaml new file mode 100644 index 000000000..39b762b50 --- /dev/null +++ b/internal/action/testdata/foo-index-v0.2.0-package-manifest/0.1.0/foos.test.foo.crd.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: foos.test.foo +spec: + group: test.foo + names: + kind: Foo + plural: foos + versions: + - name: v1 diff --git a/internal/action/testdata/foo-index-v0.2.0-package-manifest/0.2.0/foo.v0.2.0.csv.yaml b/internal/action/testdata/foo-index-v0.2.0-package-manifest/0.2.0/foo.v0.2.0.csv.yaml new file mode 100644 index 000000000..d7e30bee0 --- /dev/null +++ b/internal/action/testdata/foo-index-v0.2.0-package-manifest/0.2.0/foo.v0.2.0.csv.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + name: foo.v0.2.0 + annotations: + olm.skipRange: <0.2.0 +spec: + displayName: "Foo Operator" + customresourcedefinitions: + owned: + - group: test.foo + version: v1 + kind: Foo + name: foos.test.foo + version: 0.2.0 + replaces: foo.v0.1.0 + skips: + - foo.v0.1.1 + - foo.v0.1.2 + relatedImages: + - name: operator + image: test.registry/foo-operator/foo:v0.2.0 diff --git a/internal/action/testdata/foo-index-v0.2.0-package-manifest/0.2.0/foos.test.foo.crd.yaml b/internal/action/testdata/foo-index-v0.2.0-package-manifest/0.2.0/foos.test.foo.crd.yaml new file mode 100644 index 000000000..39b762b50 --- /dev/null +++ b/internal/action/testdata/foo-index-v0.2.0-package-manifest/0.2.0/foos.test.foo.crd.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: foos.test.foo +spec: + group: test.foo + names: + kind: Foo + plural: foos + versions: + - name: v1 diff --git a/internal/action/testdata/foo-index-v0.2.0-package-manifest/foo.package.yaml b/internal/action/testdata/foo-index-v0.2.0-package-manifest/foo.package.yaml new file mode 100644 index 000000000..0db3312b8 --- /dev/null +++ b/internal/action/testdata/foo-index-v0.2.0-package-manifest/foo.package.yaml @@ -0,0 +1,7 @@ +packageName: foo +defaultChannel: stable +channels: +- name: stable + currentCSV: foo.v0.2.0 +- name: beta + currentCSV: foo.v0.2.0