Allow disabling of migrations for bundles and sqlite DBs

Signed-off-by: Joe Lanford <joe.lanford@gmail.com>
grokspawn · Jul 18, 2024 · a1bdc8f · a1bdc8f
1 parent 3be1d97
commit a1bdc8f
Show file tree

Hide file tree

Showing 12 changed files with 70 additions and 52 deletions.
diff --git a/alpha/action/migrate.go b/alpha/action/migrate.go
@@ -12,6 +12,7 @@ import (
 type Migrate struct {
 	CatalogRef string
 	OutputDir  string
+	Stages     int
 
 	WriteFunc declcfg.WriteFunc
 	FileExt   string
@@ -28,8 +29,8 @@ func (m Migrate) Run(ctx context.Context) error {
 	}
 
 	r := Render{
-		Refs:    []string{m.CatalogRef},
-		Migrate: true,
+		Refs:          []string{m.CatalogRef},
+		MigrateStages: m.Stages,
 
 		// Only allow catalogs to be migrated.
 		AllowedRefMask: RefSqliteImage | RefSqliteFile | RefDCImage | RefDCDir,

diff --git a/alpha/action/render.go b/alpha/action/render.go
@@ -15,9 +15,10 @@ import (
 
 	"github.com/h2non/filetype"
 	"github.com/h2non/filetype/matchers"
-	"github.com/operator-framework/api/pkg/operators/v1alpha1"
 	"k8s.io/apimachinery/pkg/util/sets"
 
+	"github.com/operator-framework/api/pkg/operators/v1alpha1"
+
 	"github.com/operator-framework/operator-registry/alpha/declcfg"
 	"github.com/operator-framework/operator-registry/alpha/property"
 	"github.com/operator-framework/operator-registry/pkg/containertools"
@@ -54,7 +55,7 @@ type Render struct {
 	Refs             []string
 	Registry         image.Registry
 	AllowedRefMask   RefType
-	Migrate          bool
+	MigrateStages    int
 	ImageRefTemplate *template.Template
 
 	skipSqliteDeprecationLog bool
@@ -88,10 +89,8 @@ func (r Render) Run(ctx context.Context) (*declcfg.DeclarativeConfig, error) {
 			})
 		}
 
-		if r.Migrate {
-			if err := migrate(cfg); err != nil {
-				return nil, fmt.Errorf("migrate: %v", err)
-			}
+		if err := migrate(cfg, r.MigrateStages); err != nil {
+			return nil, fmt.Errorf("migrate: %v", err)
 		}
 
 		cfgs = append(cfgs, *cfg)
@@ -416,12 +415,26 @@ func moveBundleObjectsToEndOfPropertySlices(cfg *declcfg.DeclarativeConfig) {
 	}
 }
 
-func migrate(cfg *declcfg.DeclarativeConfig) error {
+func migrate(cfg *declcfg.DeclarativeConfig, migrateStages int) error {
+	// Do not delete or change the order of the below migrations.
+	// The --migrate-stage flag is an API that depends on the presence
+	// and order of these migrations.
 	migrations := []func(*declcfg.DeclarativeConfig) error{
 		convertObjectsToCSVMetadata,
 	}
 
-	for _, m := range migrations {
+	if migrateStages > len(migrations) {
+		return fmt.Errorf("number of requested migration stages to run (%d) exceeds number of available migrations (%d)", migrateStages, len(migrations))
+	}
+
+	// migrateStages <0 means all migrations
+	// migrateStages 0 means no migrations
+	// migrateStages 1 means only the first migration
+	// etc...
+	for i, m := range migrations {
+		if i == migrateStages {
+			break
+		}
 		if err := m(cfg); err != nil {
 			return err
 		}

diff --git a/alpha/template/basic/basic.go b/alpha/template/basic/basic.go
@@ -6,16 +6,18 @@ import (
 	"fmt"
 	"io"
 
+	"k8s.io/apimachinery/pkg/util/yaml"
+
 	"github.com/operator-framework/operator-registry/alpha/action"
 	"github.com/operator-framework/operator-registry/alpha/declcfg"
 	"github.com/operator-framework/operator-registry/pkg/image"
-	"k8s.io/apimachinery/pkg/util/yaml"
 )
 
 const schema string = "olm.template.basic"
 
 type Template struct {
-	Registry image.Registry
+	Registry      image.Registry
+	MigrateStages int
 }
 
 type BasicTemplate struct {
@@ -58,6 +60,7 @@ func (t Template) Render(ctx context.Context, reader io.Reader) (*declcfg.Declar
 	r := action.Render{
 		Registry:       t.Registry,
 		AllowedRefMask: action.RefBundleImage,
+		MigrateStages:  t.MigrateStages,
 	}
 
 	for _, b := range cfg.Bundles {

diff --git a/alpha/template/semver/semver.go b/alpha/template/semver/semver.go
@@ -6,13 +6,13 @@ import (
 	"io"
 	"sort"
 
-	"github.com/operator-framework/operator-registry/alpha/action"
-	"github.com/operator-framework/operator-registry/alpha/declcfg"
-	"github.com/operator-framework/operator-registry/alpha/property"
-
 	"github.com/blang/semver/v4"
 	"k8s.io/apimachinery/pkg/util/errors"
 	"sigs.k8s.io/yaml"
+
+	"github.com/operator-framework/operator-registry/alpha/action"
+	"github.com/operator-framework/operator-registry/alpha/declcfg"
+	"github.com/operator-framework/operator-registry/alpha/property"
 )
 
 func (t Template) Render(ctx context.Context) (*declcfg.DeclarativeConfig, error) {
@@ -35,6 +35,7 @@ func (t Template) Render(ctx context.Context) (*declcfg.DeclarativeConfig, error
 			AllowedRefMask: action.RefBundleImage,
 			Refs:           []string{b},
 			Registry:       t.Registry,
+			MigrateStages:  t.MigrateStages,
 		}
 		c, err := r.Run(ctx)
 		if err != nil {

diff --git a/alpha/template/semver/types.go b/alpha/template/semver/types.go
@@ -4,13 +4,15 @@ import (
 	"io"
 
 	"github.com/blang/semver/v4"
+
 	"github.com/operator-framework/operator-registry/pkg/image"
 )
 
 // data passed into this module externally
 type Template struct {
-	Data     io.Reader
-	Registry image.Registry
+	Data          io.Reader
+	Registry      image.Registry
+	MigrateStages int
 }
 
 // IO structs -- BEGIN

diff --git a/cmd/opm/alpha/render-graph/cmd.go b/cmd/opm/alpha/render-graph/cmd.go
@@ -5,11 +5,12 @@ import (
 	"log"
 	"os"
 
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
 	"github.com/operator-framework/operator-registry/alpha/action"
 	"github.com/operator-framework/operator-registry/alpha/declcfg"
 	"github.com/operator-framework/operator-registry/cmd/opm/internal/util"
-	"github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
 )
 
 func NewCmd() *cobra.Command {
@@ -61,6 +62,9 @@ $ opm alpha render-graph quay.io/operatorhubio/catalog:latest | \
 			render.AllowedRefMask = action.RefDCImage | action.RefDCDir | action.RefSqliteImage | action.RefSqliteFile
 			render.Registry = registry
 
+			// Run all migrations
+			render.MigrateStages = -1
+
 			cfg, err := render.Run(cmd.Context())
 			if err != nil {
 				log.Fatal(err)

diff --git a/cmd/opm/alpha/template/basic.go b/cmd/opm/alpha/template/basic.go
@@ -73,5 +73,8 @@ When FILE is '-' or not provided, the template is read from standard input`,
 			}
 		},
 	}
+
+	cmd.Flags().IntVar(&template.MigrateStages, "migrate-stages", 0, "Number of migration stages to run; use -1 for all available stages")
+
 	return cmd
 }
diff --git a/cmd/opm/alpha/template/semver.go b/cmd/opm/alpha/template/semver.go
@@ -7,14 +7,18 @@ import (
 	"os"
 
 	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
 
 	"github.com/operator-framework/operator-registry/alpha/declcfg"
 	"github.com/operator-framework/operator-registry/alpha/template/semver"
 	"github.com/operator-framework/operator-registry/cmd/opm/internal/util"
-	"github.com/spf13/cobra"
 )
 
 func newSemverTemplateCmd() *cobra.Command {
+	var (
+		migrateStages int
+	)
+
 	cmd := &cobra.Command{
 		Use: "semver [FILE]",
 		Short: `Generate a file-based catalog from a single 'semver template' file
@@ -82,5 +86,7 @@ When FILE is '-' or not provided, the template is read from standard input`,
 		},
 	}
 
+	cmd.Flags().IntVar(&migrateStages, "migrate-stages", 0, "Number of migration stages to run; use -1 for all available stages")
+
 	return cmd
 }
diff --git a/cmd/opm/migrate/cmd.go b/cmd/opm/migrate/cmd.go
@@ -51,5 +51,6 @@ parsers that assume that a file contains exactly one valid JSON object.
 		},
 	}
 	cmd.Flags().StringVarP(&output, "output", "o", "json", "Output format (json|yaml)")
+	cmd.Flags().IntVar(&migrate.Stages, "stages", -1, "Number of migration stages to run; use -1 for all available stages")
 	return cmd
 }
diff --git a/cmd/opm/render/cmd.go b/cmd/opm/render/cmd.go
@@ -20,6 +20,8 @@ func NewCmd(showAlphaHelp bool) *cobra.Command {
 		render           action.Render
 		output           string
 		imageRefTemplate string
+
+		deprecatedMigrateFlag bool
 	)
 	cmd := &cobra.Command{
 		Use:   "render [catalog-image | catalog-directory | bundle-image | bundle-directory | sqlite-file]...",
@@ -63,6 +65,11 @@ database files.
 				render.ImageRefTemplate = tmpl
 			}
 
+			if deprecatedMigrateFlag {
+				// If the deprecated --migrate flag is set, run all migration stages.
+				render.MigrateStages = -1
+			}
+
 			cfg, err := render.Run(cmd.Context())
 			if err != nil {
 				log.Fatal(err)
@@ -74,7 +81,11 @@ database files.
 		},
 	}
 	cmd.Flags().StringVarP(&output, "output", "o", "json", "Output format of the streamed file-based catalog objects (json|yaml)")
-	cmd.Flags().BoolVar(&render.Migrate, "migrate", false, "Perform migrations on the rendered FBC")
+
+	cmd.Flags().IntVar(&render.MigrateStages, "migrate-stages", 0, "Number of migration stages to run; use -1 for all available stages")
+	cmd.Flags().BoolVar(&deprecatedMigrateFlag, "migrate", false, "Perform migrations on the rendered FBC")
+	cmd.Flags().MarkDeprecated("migrate", "use --migrate-stages instead")
+	cmd.MarkFlagsMutuallyExclusive("migrate", "migrate-stages")
 
 	// Alpha flags
 	cmd.Flags().StringVar(&imageRefTemplate, "alpha-image-ref-template", "", "When bundle image reference information is unavailable, populate it with this template")

diff --git a/pkg/api/api_to_model.go b/pkg/api/api_to_model.go
@@ -5,8 +5,6 @@ import (
 	"fmt"
 	"sort"
 
-	"github.com/operator-framework/api/pkg/operators/v1alpha1"
-
 	"github.com/operator-framework/operator-registry/alpha/model"
 	"github.com/operator-framework/operator-registry/alpha/property"
 )
@@ -108,17 +106,8 @@ func convertAPIBundleToModelProperties(b *Bundle) ([]property.Property, error) {
 		out = append(out, property.MustBuildGVKRequired(p.Group, p.Version, p.Kind))
 	}
 
-	// If there is a bundle image reference and a valid CSV, create an
-	// olm.csv.metadata property. Otherwise, create a bundle object property for
-	// each object in the bundle.
-	var csv v1alpha1.ClusterServiceVersion
-	csvErr := json.Unmarshal([]byte(b.CsvJson), &csv)
-	if csvErr == nil && b.BundlePath != "" {
-		out = append(out, property.MustBuildCSVMetadata(csv))
-	} else {
-		for _, obj := range b.Object {
-			out = append(out, property.MustBuildBundleObject([]byte(obj)))
-		}
+	for _, obj := range b.Object {
+		out = append(out, property.MustBuildBundleObject([]byte(obj)))
 	}
 
 	sort.Slice(out, func(i, j int) bool {

diff --git a/pkg/registry/registry_to_model.go b/pkg/registry/registry_to_model.go
@@ -5,9 +5,6 @@ import (
 	"fmt"
 	"sort"
 
-	"github.com/operator-framework/api/pkg/operators"
-	"github.com/operator-framework/api/pkg/operators/v1alpha1"
-
 	"github.com/operator-framework/operator-registry/alpha/property"
 )
 
@@ -102,21 +99,8 @@ func ObjectsAndPropertiesFromBundle(b *Bundle) ([]string, []property.Property, e
 		if err != nil {
 			return nil, nil, fmt.Errorf("marshal object %s/%s (%s) to json: %v", obj.GetName(), obj.GetNamespace(), obj.GroupVersionKind(), err)
 		}
+		props = append(props, property.MustBuildBundleObject(objData))
 		objects = append(objects, string(objData))
-
-		// Make an olm.bundle.object property if there is no bundle image set.
-		// Otherwise, make a olm.csv.metadata property if the object is a CSV
-		// (and fallback to olm.bundle.object if parsing the CSV fails).
-		if b.BundleImage == "" {
-			props = append(props, property.MustBuildBundleObject(objData))
-		} else if obj.GetKind() == operators.ClusterServiceVersionKind {
-			var csv v1alpha1.ClusterServiceVersion
-			if err := json.Unmarshal(objData, &csv); err != nil {
-				props = append(props, property.MustBuildBundleObject(objData))
-			} else {
-				props = append(props, property.MustBuildCSVMetadata(csv))
-			}
-		}
 	}
 
 	if packageProvidedProperty == nil {