From 2f2e8c43be7b0c1b3fbe5106f6d57d12955292bc Mon Sep 17 00:00:00 2001 From: Joe Lanford Date: Tue, 10 Aug 2021 13:41:56 -0400 Subject: [PATCH] render: support rendering DC from packagemanifest and bundle directories Signed-off-by: Joe Lanford --- alpha/action/render.go | 125 ++++++++++--- alpha/action/render_test.go | 176 +++++++++++++++++- .../0.1.0/foo.v0.1.0.csv.yaml | 19 ++ .../0.1.0/foos.test.foo.crd.yaml | 12 ++ .../0.2.0/foo.v0.2.0.csv.yaml | 45 +++++ .../0.2.0/foos.test.foo.crd.yaml | 12 ++ .../foo.package.yaml | 7 + 7 files changed, 369 insertions(+), 27 deletions(-) create mode 100644 alpha/action/testdata/foo-index-v0.2.0-package-manifest/0.1.0/foo.v0.1.0.csv.yaml create mode 100644 alpha/action/testdata/foo-index-v0.2.0-package-manifest/0.1.0/foos.test.foo.crd.yaml create mode 100644 alpha/action/testdata/foo-index-v0.2.0-package-manifest/0.2.0/foo.v0.2.0.csv.yaml create mode 100644 alpha/action/testdata/foo-index-v0.2.0-package-manifest/0.2.0/foos.test.foo.crd.yaml create mode 100644 alpha/action/testdata/foo-index-v0.2.0-package-manifest/foo.package.yaml diff --git a/alpha/action/render.go b/alpha/action/render.go index 3f38390a0e..2eafba9404 100644 --- a/alpha/action/render.go +++ b/alpha/action/render.go @@ -34,10 +34,12 @@ type RefType uint const ( RefBundleImage RefType = 1 << iota + RefBundleDir RefSqliteImage RefSqliteFile RefDCImage RefDCDir + RefPackageManifestDir RefAll = 0 ) @@ -117,25 +119,54 @@ func (r Render) createRegistry() (*containerdregistry.Registry, error) { } func (r Render) renderReference(ctx context.Context, ref string) (*declcfg.DeclarativeConfig, error) { - if stat, serr := os.Stat(ref); serr == nil { - if stat.IsDir() { - if !r.AllowedRefMask.Allowed(RefDCDir) { - return nil, fmt.Errorf("cannot render declarative config directory: %w", ErrNotAllowed) + stat, err := os.Stat(ref) + if err != nil { + return r.imageToDeclcfg(ctx, ref) + } + if stat.IsDir() { + dirEntries, err := os.ReadDir(ref) + if err != nil { + return nil, err + } + if isBundle(dirEntries) { + // Looks like a bundle directory + if !r.AllowedRefMask.Allowed(RefBundleDir) { + return nil, fmt.Errorf("cannot render bundle directory: %w", ErrNotAllowed) } - return declcfg.LoadFS(os.DirFS(ref)) - } else { - // The only supported file type is an sqlite DB file, - // since declarative configs will be in a directory. - if err := checkDBFile(ref); err != nil { + img, err := registry.NewImageInput(image.SimpleReference(""), ref) + if err != nil { return nil, err } - if !r.AllowedRefMask.Allowed(RefSqliteFile) { - return nil, fmt.Errorf("cannot render sqlite file: %w", ErrNotAllowed) + return bundleToDeclcfg(img.Bundle) + } else if isPackageManifest(dirEntries) { + // Looks like a package manifest directory + if !r.AllowedRefMask.Allowed(RefPackageManifestDir) { + return nil, fmt.Errorf("cannot render package manifest directory: %w", ErrNotAllowed) } - return sqliteToDeclcfg(ctx, ref) + return renderPackageManifest(ctx, ref) } + + // Otherwise, assume it is a declarative config root directory. + if !r.AllowedRefMask.Allowed(RefDCDir) { + return nil, fmt.Errorf("cannot render declarative config directory: %w", ErrNotAllowed) + } + return declcfg.LoadFS(os.DirFS(ref)) + } + // The only supported file type is an sqlite DB file, + // since declarative configs will be in a directory. + if err := checkDBFile(ref); err != nil { + return nil, err + } + if !r.AllowedRefMask.Allowed(RefSqliteFile) { + return nil, fmt.Errorf("cannot render sqlite file: %w", ErrNotAllowed) } - return r.imageToDeclcfg(ctx, ref) + + db, err := sqlite.Open(ref) + if err != nil { + return nil, err + } + defer db.Close() + return sqliteToDeclcfg(ctx, db) } func (r Render) imageToDeclcfg(ctx context.Context, imageRef string) (*declcfg.DeclarativeConfig, error) { @@ -161,7 +192,12 @@ func (r Render) imageToDeclcfg(ctx context.Context, imageRef string) (*declcfg.D if !r.AllowedRefMask.Allowed(RefSqliteImage) { return nil, fmt.Errorf("cannot render sqlite image: %w", ErrNotAllowed) } - cfg, err = sqliteToDeclcfg(ctx, filepath.Join(tmpDir, dbFile)) + db, err := sqlite.Open(filepath.Join(tmpDir, dbFile)) + if err != nil { + return nil, err + } + defer db.Close() + cfg, err = sqliteToDeclcfg(ctx, db) if err != nil { return nil, err } @@ -213,17 +249,11 @@ func checkDBFile(ref string) error { return nil } -func sqliteToDeclcfg(ctx context.Context, dbFile string) (*declcfg.DeclarativeConfig, error) { +func sqliteToDeclcfg(ctx context.Context, db *sql.DB) (*declcfg.DeclarativeConfig, error) { logDeprecationMessage.Do(func() { sqlite.LogSqliteDeprecation() }) - db, err := sqlite.Open(dbFile) - if err != nil { - return nil, err - } - defer db.Close() - migrator, err := sqlite.NewSQLLiteMigrator(db) if err != nil { return nil, err @@ -354,7 +384,7 @@ func getRelatedImages(b *registry.Bundle) ([]declcfg.RelatedImage, error) { allImages = allImages.Insert(ri.Image) } - if !allImages.Has(b.BundleImage) { + if b.BundleImage != "" && !allImages.Has(b.BundleImage) { relatedImages = append(relatedImages, declcfg.RelatedImage{ Image: b.BundleImage, }) @@ -402,3 +432,54 @@ func combineConfigs(cfgs []declcfg.DeclarativeConfig) *declcfg.DeclarativeConfig } return out } + +func isBundle(entries []os.DirEntry) bool { + foundManifests := false + foundMetadata := false + for _, e := range entries { + if e.IsDir() { + switch e.Name() { + case "manifests": + foundManifests = true + case "metadata": + foundMetadata = true + } + } + if foundMetadata && foundManifests { + return true + } + } + return false +} + +func isPackageManifest(entries []os.DirEntry) bool { + for _, e := range entries { + if strings.HasSuffix(e.Name(), ".package.yaml") || strings.HasSuffix(e.Name(), ".package.yml") { + return true + } + } + return false +} + +func renderPackageManifest(ctx context.Context, ref string) (*declcfg.DeclarativeConfig, error) { + db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?mode=memory&cache=shared&_foreign_keys=on", ref)) + if err != nil { + return nil, err + } + defer db.Close() + + dbLoader, err := sqlite.NewSQLLiteLoader(db) + if err != nil { + return nil, err + } + if err := dbLoader.Migrate(ctx); err != nil { + return nil, err + } + + loader := sqlite.NewSQLLoaderForDirectory(dbLoader, ref) + if err := loader.Populate(); err != nil { + return nil, fmt.Errorf("error loading manifests from directory: %s", err) + } + + return sqliteToDeclcfg(ctx, db) +} diff --git a/alpha/action/render_test.go b/alpha/action/render_test.go index b355233ece..c5ef3fe4b8 100644 --- a/alpha/action/render_test.go +++ b/alpha/action/render_test.go @@ -530,6 +530,132 @@ func TestRender(t *testing.T) { }, assertion: require.NoError, }, + { + name: "Success/BundleDirectory", + render: action.Render{ + Refs: []string{"testdata/foo-bundle-v0.2.0"}, + Registry: reg, + }, + expectCfg: &declcfg.DeclarativeConfig{ + Bundles: []declcfg.Bundle{ + { + Schema: "olm.bundle", + Name: "foo.v0.2.0", + Package: "foo", + Properties: []property.Property{ + property.MustBuildGVK("test.foo", "v1", "Foo"), + property.MustBuildGVKRequired("test.bar", "v1alpha1", "Bar"), + property.MustBuildPackage("foo", "0.2.0"), + property.MustBuildPackageRequired("bar", "<0.1.0"), + property.MustBuildBundleObjectData(foov2csv), + property.MustBuildBundleObjectData(foov2crd), + }, + Objects: []string{string(foov2csv), string(foov2crd)}, + CsvJSON: string(foov2csv), + RelatedImages: []declcfg.RelatedImage{ + { + Image: "test.registry/foo-operator/foo-2:v0.2.0", + }, + { + Image: "test.registry/foo-operator/foo-init-2:v0.2.0", + }, + { + Image: "test.registry/foo-operator/foo-init:v0.2.0", + }, + { + Name: "other", + Image: "test.registry/foo-operator/foo-other:v0.2.0", + }, + { + Name: "operator", + Image: "test.registry/foo-operator/foo:v0.2.0", + }, + }, + }, + }, + }, + assertion: require.NoError, + }, + { + name: "Success/PackageManifestDirectory", + render: action.Render{ + Refs: []string{"testdata/foo-index-v0.2.0-package-manifest"}, + Registry: reg, + }, + expectCfg: &declcfg.DeclarativeConfig{ + Packages: []declcfg.Package{ + { + Schema: "olm.package", + Name: "foo", + DefaultChannel: "stable", + }, + }, + Channels: []declcfg.Channel{ + {Schema: "olm.channel", Package: "foo", Name: "beta", Entries: []declcfg.ChannelEntry{ + {Name: "foo.v0.1.0", SkipRange: "<0.1.0"}, + {Name: "foo.v0.2.0", Replaces: "foo.v0.1.0", SkipRange: "<0.2.0", Skips: []string{"foo.v0.1.1", "foo.v0.1.2"}}, + }}, + {Schema: "olm.channel", Package: "foo", Name: "stable", Entries: []declcfg.ChannelEntry{ + {Name: "foo.v0.1.0", SkipRange: "<0.1.0"}, + {Name: "foo.v0.2.0", Replaces: "foo.v0.1.0", SkipRange: "<0.2.0", Skips: []string{"foo.v0.1.1", "foo.v0.1.2"}}, + }}, + }, + Bundles: []declcfg.Bundle{ + { + Schema: "olm.bundle", + Name: "foo.v0.1.0", + Package: "foo", + Properties: []property.Property{ + property.MustBuildGVK("test.foo", "v1", "Foo"), + property.MustBuildPackage("foo", "0.1.0"), + property.MustBuildBundleObjectData(foov1csv), + property.MustBuildBundleObjectData(foov1crd), + }, + Objects: []string{string(foov1csv), string(foov1crd)}, + CsvJSON: string(foov1csv), + RelatedImages: []declcfg.RelatedImage{ + { + Name: "operator", + Image: "test.registry/foo-operator/foo:v0.1.0", + }, + }, + }, + { + Schema: "olm.bundle", + Name: "foo.v0.2.0", + Package: "foo", + Properties: []property.Property{ + property.MustBuildGVK("test.foo", "v1", "Foo"), + property.MustBuildPackage("foo", "0.2.0"), + property.MustBuildBundleObjectData(foov2csv), + property.MustBuildBundleObjectData(foov2crd), + }, + Objects: []string{string(foov2csv), string(foov2crd)}, + CsvJSON: string(foov2csv), + RelatedImages: []declcfg.RelatedImage{ + { + Image: "test.registry/foo-operator/foo-2:v0.2.0", + }, + { + Image: "test.registry/foo-operator/foo-init-2:v0.2.0", + }, + { + Image: "test.registry/foo-operator/foo-init:v0.2.0", + }, + { + Name: "other", + Image: "test.registry/foo-operator/foo-other:v0.2.0", + }, + { + Name: "operator", + Image: "test.registry/foo-operator/foo:v0.2.0", + }, + }, + }, + }, + }, + assertion: require.NoError, + }, } for _, s := range specs { @@ -574,7 +700,7 @@ func TestAllowRefMask(t *testing.T) { render: action.Render{ Refs: []string{"test.registry/foo-operator/foo-index-sqlite:v0.2.0"}, Registry: reg, - AllowedRefMask: action.RefDCImage | action.RefDCDir | action.RefSqliteFile | action.RefBundleImage, + AllowedRefMask: action.RefDCImage | action.RefDCDir | action.RefSqliteFile | action.RefBundleImage | action.RefBundleDir | action.RefPackageManifestDir, }, expectErr: action.ErrNotAllowed, }, @@ -592,7 +718,7 @@ func TestAllowRefMask(t *testing.T) { render: action.Render{ Refs: []string{dbFile}, Registry: reg, - AllowedRefMask: action.RefDCImage | action.RefDCDir | action.RefSqliteImage | action.RefBundleImage, + AllowedRefMask: action.RefDCImage | action.RefDCDir | action.RefSqliteImage | action.RefBundleImage | action.RefBundleDir | action.RefPackageManifestDir, }, expectErr: action.ErrNotAllowed, }, @@ -610,7 +736,7 @@ func TestAllowRefMask(t *testing.T) { render: action.Render{ Refs: []string{"test.registry/foo-operator/foo-index-declcfg:v0.2.0"}, Registry: reg, - AllowedRefMask: action.RefDCDir | action.RefSqliteImage | action.RefSqliteFile | action.RefBundleImage, + AllowedRefMask: action.RefDCDir | action.RefSqliteImage | action.RefSqliteFile | action.RefBundleImage | action.RefBundleDir | action.RefPackageManifestDir, }, expectErr: action.ErrNotAllowed, }, @@ -628,7 +754,7 @@ func TestAllowRefMask(t *testing.T) { render: action.Render{ Refs: []string{"testdata/foo-index-v0.2.0-declcfg"}, Registry: reg, - AllowedRefMask: action.RefDCImage | action.RefSqliteImage | action.RefSqliteFile | action.RefBundleImage, + AllowedRefMask: action.RefDCImage | action.RefSqliteImage | action.RefSqliteFile | action.RefBundleImage | action.RefBundleDir | action.RefPackageManifestDir, }, expectErr: action.ErrNotAllowed, }, @@ -646,7 +772,43 @@ func TestAllowRefMask(t *testing.T) { render: action.Render{ Refs: []string{"test.registry/foo-operator/foo-bundle:v0.2.0"}, Registry: reg, - AllowedRefMask: action.RefDCImage | action.RefDCDir | action.RefSqliteImage | action.RefSqliteFile, + AllowedRefMask: action.RefDCImage | action.RefDCDir | action.RefSqliteImage | action.RefSqliteFile | action.RefBundleDir | action.RefPackageManifestDir, + }, + expectErr: action.ErrNotAllowed, + }, + { + name: "BundleDir/Allowed", + render: action.Render{ + Refs: []string{"testdata/foo-bundle-v0.2.0"}, + Registry: reg, + AllowedRefMask: action.RefBundleDir, + }, + expectErr: nil, + }, + { + name: "BundleDir/NotAllowed", + render: action.Render{ + Refs: []string{"testdata/foo-bundle-v0.2.0"}, + Registry: reg, + AllowedRefMask: action.RefDCImage | action.RefDCDir | action.RefSqliteImage | action.RefSqliteFile | action.RefBundleImage | action.RefPackageManifestDir, + }, + expectErr: action.ErrNotAllowed, + }, + { + name: "PackageManifestDir/Allowed", + render: action.Render{ + Refs: []string{"testdata/foo-index-v0.2.0-package-manifest"}, + Registry: reg, + AllowedRefMask: action.RefPackageManifestDir, + }, + expectErr: nil, + }, + { + name: "PackageManifestDir/NotAllowed", + render: action.Render{ + Refs: []string{"testdata/foo-index-v0.2.0-package-manifest"}, + Registry: reg, + AllowedRefMask: action.RefDCImage | action.RefDCDir | action.RefSqliteImage | action.RefSqliteFile | action.RefBundleImage | action.RefBundleDir, }, expectErr: action.ErrNotAllowed, }, @@ -659,6 +821,8 @@ func TestAllowRefMask(t *testing.T) { "test.registry/foo-operator/foo-index-declcfg:v0.2.0", "testdata/foo-index-v0.2.0-declcfg", "test.registry/foo-operator/foo-bundle:v0.2.0", + "testdata/foo-bundle-v0.2.0", + "testdata/foo-index-v0.2.0-package-manifest", }, Registry: reg, }, @@ -692,6 +856,8 @@ func TestAllowRefMaskAllowed(t *testing.T) { action.RefSqliteImage, action.RefSqliteFile, action.RefBundleImage, + action.RefBundleDir, + action.RefPackageManifestDir, }, fail: []action.RefType{}, }, diff --git a/alpha/action/testdata/foo-index-v0.2.0-package-manifest/0.1.0/foo.v0.1.0.csv.yaml b/alpha/action/testdata/foo-index-v0.2.0-package-manifest/0.1.0/foo.v0.1.0.csv.yaml new file mode 100644 index 0000000000..189b6849eb --- /dev/null +++ b/alpha/action/testdata/foo-index-v0.2.0-package-manifest/0.1.0/foo.v0.1.0.csv.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + name: foo.v0.1.0 + annotations: + olm.skipRange: <0.1.0 +spec: + displayName: "Foo Operator" + customresourcedefinitions: + owned: + - group: test.foo + version: v1 + kind: Foo + name: foos.test.foo + version: 0.1.0 + relatedImages: + - name: operator + image: test.registry/foo-operator/foo:v0.1.0 diff --git a/alpha/action/testdata/foo-index-v0.2.0-package-manifest/0.1.0/foos.test.foo.crd.yaml b/alpha/action/testdata/foo-index-v0.2.0-package-manifest/0.1.0/foos.test.foo.crd.yaml new file mode 100644 index 0000000000..39b762b505 --- /dev/null +++ b/alpha/action/testdata/foo-index-v0.2.0-package-manifest/0.1.0/foos.test.foo.crd.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: foos.test.foo +spec: + group: test.foo + names: + kind: Foo + plural: foos + versions: + - name: v1 diff --git a/alpha/action/testdata/foo-index-v0.2.0-package-manifest/0.2.0/foo.v0.2.0.csv.yaml b/alpha/action/testdata/foo-index-v0.2.0-package-manifest/0.2.0/foo.v0.2.0.csv.yaml new file mode 100644 index 0000000000..823f91c0fd --- /dev/null +++ b/alpha/action/testdata/foo-index-v0.2.0-package-manifest/0.2.0/foo.v0.2.0.csv.yaml @@ -0,0 +1,45 @@ +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + name: foo.v0.2.0 + annotations: + olm.skipRange: <0.2.0 +spec: + displayName: "Foo Operator" + customresourcedefinitions: + owned: + - group: test.foo + version: v1 + kind: Foo + name: foos.test.foo + version: 0.2.0 + replaces: foo.v0.1.0 + skips: + - foo.v0.1.1 + - foo.v0.1.2 + install: + strategy: deployment + spec: + deployments: + - name: foo-operator + spec: + template: + spec: + initContainers: + - image: test.registry/foo-operator/foo-init:v0.2.0 + containers: + - image: test.registry/foo-operator/foo:v0.2.0 + - name: foo-operator-2 + spec: + template: + spec: + initContainers: + - image: test.registry/foo-operator/foo-init-2:v0.2.0 + containers: + - image: test.registry/foo-operator/foo-2:v0.2.0 + relatedImages: + - name: operator + image: test.registry/foo-operator/foo:v0.2.0 + - name: other + image: test.registry/foo-operator/foo-other:v0.2.0 diff --git a/alpha/action/testdata/foo-index-v0.2.0-package-manifest/0.2.0/foos.test.foo.crd.yaml b/alpha/action/testdata/foo-index-v0.2.0-package-manifest/0.2.0/foos.test.foo.crd.yaml new file mode 100644 index 0000000000..39b762b505 --- /dev/null +++ b/alpha/action/testdata/foo-index-v0.2.0-package-manifest/0.2.0/foos.test.foo.crd.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: foos.test.foo +spec: + group: test.foo + names: + kind: Foo + plural: foos + versions: + - name: v1 diff --git a/alpha/action/testdata/foo-index-v0.2.0-package-manifest/foo.package.yaml b/alpha/action/testdata/foo-index-v0.2.0-package-manifest/foo.package.yaml new file mode 100644 index 0000000000..0db3312b8a --- /dev/null +++ b/alpha/action/testdata/foo-index-v0.2.0-package-manifest/foo.package.yaml @@ -0,0 +1,7 @@ +packageName: foo +defaultChannel: stable +channels: +- name: stable + currentCSV: foo.v0.2.0 +- name: beta + currentCSV: foo.v0.2.0