Add support for storing and retrieving related images

cmd/configmap-server/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"net"
 
@@ -98,11 +99,19 @@ func runCmdFunc(cmd *cobra.Command, args []string) error {
 		logger.Fatalf("error getting configmap: %s", err)
 	}
 
-	sqlLoader, err := sqlite.NewSQLLiteLoader(sqlite.WithDBName(dbName))
+	db, err := sql.Open("sqlite3", dbName)
 	if err != nil {
 		return err
 	}
 
+	sqlLoader, err := sqlite.NewSQLLiteLoader(db)
+	if err != nil {
+		return err
+	}
+	if err := sqlLoader.Migrate(context.TODO()); err != nil {
+		return err
+	}
+
 	configMapPopulator := sqlite.NewSQLLoaderForConfigMap(sqlLoader, *configMap)
 	if err := configMapPopulator.Populate(); err != nil {
 		err = fmt.Errorf("error loading manifests from configmap: %s", err)

cmd/initializer/main.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"context"
+	"database/sql"
 	"fmt"
 
 	"github.com/sirupsen/logrus"
@@ -53,11 +55,19 @@ func runCmdFunc(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	dbLoader, err := sqlite.NewSQLLiteLoader(sqlite.WithDBName(outFilename))
+	db, err := sql.Open("sqlite3", outFilename)
 	if err != nil {
 		return err
 	}
-	defer dbLoader.Close()
+	defer db.Close()
+
+	dbLoader, err := sqlite.NewSQLLiteLoader(db)
+	if err != nil {
+		return err
+	}
+	if err := dbLoader.Migrate(context.TODO()); err != nil {
+		return err
+	}
 
 	loader := sqlite.NewSQLLoaderForDirectory(dbLoader, manifestDir)
 	if err := loader.Populate(); err != nil {

cmd/opm/registry/serve.go

@@ -2,6 +2,8 @@ package registry
 
 import (
 	"context"
+	"database/sql"
+	"fmt"
 	"net"
 
 	"github.com/sirupsen/logrus"
@@ -12,7 +14,6 @@ import (
 	"github.com/operator-framework/operator-registry/pkg/api"
 	health "github.com/operator-framework/operator-registry/pkg/api/grpc_health_v1"
 	"github.com/operator-framework/operator-registry/pkg/lib/log"
-	"github.com/operator-framework/operator-registry/pkg/registry"
 	"github.com/operator-framework/operator-registry/pkg/server"
 	"github.com/operator-framework/operator-registry/pkg/sqlite"
 )
@@ -37,7 +38,7 @@ func newRegistryServeCmd() *cobra.Command {
 	rootCmd.Flags().StringP("database", "d", "bundles.db", "relative path to sqlite db")
 	rootCmd.Flags().StringP("port", "p", "50051", "port number to serve on")
 	rootCmd.Flags().StringP("termination-log", "t", "/dev/termination-log", "path to a container termination log file")
-
+	rootCmd.Flags().Bool("skip-migrate", false, "do  not attempt to migrate to the latest db revision when starting")
 	return rootCmd
 
 }
@@ -64,15 +65,18 @@ func runRegistryServeCmdFunc(cmd *cobra.Command, args []string) error {
 
 	logger := logrus.WithFields(logrus.Fields{"database": dbName, "port": port})
 
-	var store registry.Query
-	store, err = sqlite.NewSQLLiteQuerier(dbName)
+	db, err := sql.Open("sqlite3", dbName)
 	if err != nil {
-		logger.WithError(err).Warnf("failed to load db")
+		return err
 	}
-	if store == nil {
-		store = registry.NewEmptyQuerier()
+
+	// migrate to the latest version
+	if err := migrate(cmd, db); err != nil {
+		logger.WithError(err).Warnf("couldn't migrate db")
 	}
 
+	store := sqlite.NewSQLLiteQuerierFromDb(db)
+
 	// sanity check that the db is available
 	tables, err := store.ListTables(context.TODO())
 	if err != nil {
@@ -98,3 +102,23 @@ func runRegistryServeCmdFunc(cmd *cobra.Command, args []string) error {
 
 	return nil
 }
+
+func migrate(cmd *cobra.Command, db *sql.DB) error {
+	shouldSkipMigrate, err := cmd.Flags().GetBool("skip-migrate")
+	if err != nil {
+		return err
+	}
+	if shouldSkipMigrate {
+		return nil
+	}
+
+	migrator, err := sqlite.NewSQLLiteMigrator(db)
+	if err != nil {
+		return err
+	}
+	if migrator == nil {
+		return fmt.Errorf("failed to load migrator")
+	}
+
+	return migrator.Migrate(context.TODO())
+}

cmd/registry-server/main.go

@@ -2,17 +2,19 @@ package main
 
 import (
 	"context"
+	"database/sql"
+	"fmt"
 	"net"
 
-	"github.com/operator-framework/operator-registry/pkg/lib/log"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/reflection"
 
+	"github.com/operator-framework/operator-registry/pkg/lib/log"
+
 	"github.com/operator-framework/operator-registry/pkg/api"
 	health "github.com/operator-framework/operator-registry/pkg/api/grpc_health_v1"
-	"github.com/operator-framework/operator-registry/pkg/registry"
 	"github.com/operator-framework/operator-registry/pkg/server"
 	"github.com/operator-framework/operator-registry/pkg/sqlite"
 )
@@ -36,6 +38,7 @@ func init() {
 	rootCmd.Flags().StringP("database", "d", "bundles.db", "relative path to sqlite db")
 	rootCmd.Flags().StringP("port", "p", "50051", "port number to serve on")
 	rootCmd.Flags().StringP("termination-log", "t", "/dev/termination-log", "path to a container termination log file")
+	rootCmd.Flags().Bool("skip-migrate", false, "do  not attempt to migrate to the latest db revision when starting")
 	if err := rootCmd.Flags().MarkHidden("debug"); err != nil {
 		logrus.Panic(err.Error())
 	}
@@ -69,15 +72,18 @@ func runCmdFunc(cmd *cobra.Command, args []string) error {
 
 	logger := logrus.WithFields(logrus.Fields{"database": dbName, "port": port})
 
-	var store registry.Query
-	store, err = sqlite.NewSQLLiteQuerier(dbName)
+	db, err := sql.Open("sqlite3", dbName)
 	if err != nil {
-		logger.WithError(err).Warnf("failed to load db")
+		return err
 	}
-	if store == nil {
-		store = registry.NewEmptyQuerier()
+
+	// migrate to the latest version
+	if err := migrate(cmd, db); err != nil {
+		logger.WithError(err).Warnf("couldn't migrate db")
 	}
 
+	store := sqlite.NewSQLLiteQuerierFromDb(db)
+
 	// sanity check that the db is available
 	tables, err := store.ListTables(context.TODO())
 	if err != nil {
@@ -103,3 +109,23 @@ func runCmdFunc(cmd *cobra.Command, args []string) error {
 
 	return nil
 }
+
+func migrate(cmd *cobra.Command, db *sql.DB) error {
+	shouldSkipMigrate, err := cmd.Flags().GetBool("skip-migrate")
+	if err != nil {
+		return err
+	}
+	if shouldSkipMigrate {
+		return nil
+	}
+
+	migrator, err := sqlite.NewSQLLiteMigrator(db)
+	if err != nil {
+		return err
+	}
+	if migrator == nil {
+		return fmt.Errorf("failed to load migrator")
+	}
+
+	return migrator.Migrate(context.TODO())
+}

configmap.example.yaml

@@ -5710,6 +5710,11 @@ data:
         annotations:
           tectonic-visibility: ocs
       spec:
+        relatedImages:
+        - name: etcd-v3.4.0
+          image: quay.io/coreos/etcd@sha256:3816b6daf9b66d6ced6f0f966314e2d4f894982c6b1493061502f8c2bf86ac84
+        - name: etcd-3.4.1
+          image: quay.io/coreos/etcd@sha256:49d3d4a81e0d030d3f689e7167f23e120abf955f7d08dbedf3ea246485acee9f
         displayName: etcd
         description: |
           etcd is a distributed key value store that provides a reliable way to store data across a cluster of machines. It’s open-source and available on GitHub. etcd gracefully handles leader elections during network partitions and will tolerate machine failure, including the leader. Your applications can read and write data into etcd.

docs/contributors/add_migration.md

@@ -0,0 +1,45 @@
+# Add a new migration
+
+Migrations live in `pkg/sqlite/migrations`.
+
+Create a new file (and tests!) that increments the migration number:
+
+```sh
+touch pkg/sqlite/migrations/002_migration_description.go
+touch pkg/sqlite/migrations/002_migration_description_test.go
+```
+
+Create the migration instance and register it:
+
+```go
+package migrations
+
+import (
+	"context"
+	"database/sql"
+)
+
+// This should increment the value from the previous migration
+const MyMigrationKey = 2
+
+
+var myNewMigration = &Migration{
+    // The id for this migration
+	Id: MyMigrationKey,
+	Up: func(ctx context.Context, tx *sql.Tx) error {
+	    // the up version of this migration
+		return nil
+	},
+	Down: func(ctx context.Context, tx *sql.Tx) error {
+	    // the down version of this migration
+		return nil
+	},
+}
+
+// Register this migration 
+func init() {
+	migrations[MyMigrationKey] = myNewMigration
+}
+```
+
+See the existing migrations in the `pkg/sqlite/migrations` for examples of migrations and tests.

manifests/etcd/0.9.2/etcdoperator.v0.9.2.clusterserviceversion.yaml

@@ -8,6 +8,11 @@ metadata:
     tectonic-visibility: ocs
     alm-examples: '[{"apiVersion":"etcd.database.coreos.com/v1beta2","kind":"EtcdCluster","metadata":{"name":"example","namespace":"default"},"spec":{"size":3,"version":"3.2.13"}},{"apiVersion":"etcd.database.coreos.com/v1beta2","kind":"EtcdRestore","metadata":{"name":"example-etcd-cluster"},"spec":{"etcdCluster":{"name":"example-etcd-cluster"},"backupStorageType":"S3","s3":{"path":"<full-s3-path>","awsSecret":"<aws-secret>"}}},{"apiVersion":"etcd.database.coreos.com/v1beta2","kind":"EtcdBackup","metadata":{"name":"example-etcd-cluster-backup"},"spec":{"etcdEndpoints":["<etcd-cluster-endpoints>"],"storageType":"S3","s3":{"path":"<full-s3-path>","awsSecret":"<aws-secret>"}}}]'
 spec:
+  relatedImages:
+    - name: etcd-v3.4.0
+      image: quay.io/coreos/etcd@sha256:3816b6daf9b66d6ced6f0f966314e2d4f894982c6b1493061502f8c2bf86ac84
+    - name: etcd-3.4.1
+      image: quay.io/coreos/etcd@sha256:49d3d4a81e0d030d3f689e7167f23e120abf955f7d08dbedf3ea246485acee9f
   displayName: etcd
   description: |
     etcd is a distributed key value store that provides a reliable way to store data across a cluster of machines. It’s open-source and available on GitHub. etcd gracefully handles leader elections during network partitions and will tolerate machine failure, including the leader. Your applications can read and write data into etcd.

pkg/appregistry/appregistry.go

@@ -100,10 +100,7 @@ func (a *AppregistryLoader) Load(csvSources []string, csvPackages string) (regis
 		errs = append(errs, fmt.Errorf("error loading operator manifests: %s", err))
 	}
 
-	store, err := a.loader.GetStore()
-	if err != nil {
-		errs = append(errs, err)
-	}
+	store := a.loader.GetStore()
 
 	return store, utilerrors.NewAggregate(errs)
 }

pkg/appregistry/dbloader.go

@@ -2,6 +2,7 @@ package appregistry
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"os"
 
@@ -11,32 +12,35 @@ import (
 )
 
 func NewDbLoader(dbName string, logger *logrus.Entry) (*dbLoader, error) {
-	sqlLoader, err := sqlite.NewSQLLiteLoader(sqlite.WithDBName(dbName))
+	db, err := sql.Open("sqlite3", dbName)
 	if err != nil {
 		return nil, err
 	}
 
+	sqlLoader, err := sqlite.NewSQLLiteLoader(db)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := sqlLoader.Migrate(context.TODO()); err != nil {
+		return nil, err
+	}
+
 	return &dbLoader{
 		loader: sqlLoader,
 		logger: logger,
-		dbName: dbName,
+		db:     db,
 	}, nil
 }
 
 type dbLoader struct {
-	dbName string
-
+	db     *sql.DB
 	loader *sqlite.SQLLoader
 	logger *logrus.Entry
 }
 
-func (l *dbLoader) GetStore() (registry.Query, error) {
-	s, err := sqlite.NewSQLLiteQuerier(l.dbName)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load db: %v", err)
-	}
-
-	return s, nil
+func (l *dbLoader) GetStore() registry.Query {
+	return sqlite.NewSQLLiteQuerierFromDb(l.db)
 }
 
 // LoadDataToSQLite uses configMaploader to load the downloaded operator
@@ -55,10 +59,7 @@ func (l *dbLoader) LoadFlattenedToSQLite(manifest *RawOperatorManifestData) erro
 		return err
 	}
 
-	s, err := sqlite.NewSQLLiteQuerier(l.dbName)
-	if err != nil {
-		return fmt.Errorf("failed to load db: %v", err)
-	}
+	s := sqlite.NewSQLLiteQuerierFromDb(l.db)
 
 	// sanity check that the db is available.
 	tables, err := s.ListTables(context.TODO())
@@ -87,8 +88,9 @@ func (l *dbLoader) LoadBundleDirectoryToSQLite(directory string) error {
 	return nil
 }
 
-func (l *dbLoader) Close() {
-	if l.loader != nil {
-		l.loader.Close()
+func (l *dbLoader) Close() error {
+	if l.db != nil {
+		return l.db.Close()
 	}
+	return nil
 }

pkg/appregistry/downloader.go

@@ -27,8 +27,8 @@ func (d *downloadItem) String() string {
 }
 
 type downloader struct {
-	logger            *logrus.Entry
-	kubeClient        kubernetes.Clientset
+	logger     *logrus.Entry
+	kubeClient kubernetes.Clientset
 }
 
 // Download downloads manifest(s) associated with the specified package(s) from

pkg/appregistry/parser.go

@@ -186,7 +186,6 @@ func (*manifestYAMLParser) Marshal(bundle *StructuredOperatorManifestData) (*Raw
 	return data, nil
 }
 
-
 func (*manifestYAMLParser) MarshalCSV(csv *ClusterServiceVersion) (string, error) {
 	csvRaw, err := yaml.Marshal(csv)
 	if err != nil {