Database migration initialization

Makefile

@@ -3,7 +3,7 @@ CMDS  := $(addprefix bin/, $(shell go list $(MOD_FLAGS) ./cmd/... | xargs -I{} b
 
 .PHONY: build test vendor clean
 
-all: clean test build
+all: clean install-go-bindata test build
 
 $(CMDS):
 	go build $(MOD_FLAGS) $(extra_flags) -o $@ ./cmd/$(shell basename $@)
@@ -25,6 +25,12 @@ image-upstream:
 vendor:
 	go mod vendor
 
+install-go-bindata:
+	go get -u github.com/go-bindata/go-bindata/...
+
+generate-migration-bundle:
+	go-bindata -pkg sqlite -o ./pkg/sqlite/migrations.go ./pkg/sqlite/db_migrations/
+
 codegen:
 	protoc -I pkg/api/ --go_out=plugins=grpc:pkg/api pkg/api/*.proto
 	protoc -I pkg/api/grpc_health_v1 --go_out=plugins=grpc:pkg/api/grpc_health_v1 pkg/api/grpc_health_v1/*.proto

cmd/configmap-server/main.go

@@ -98,7 +98,7 @@ func runCmdFunc(cmd *cobra.Command, args []string) error {
 		logger.Fatalf("error getting configmap: %s", err)
 	}
 
-	sqlLoader, err := sqlite.NewSQLLiteLoader(dbName)
+	sqlLoader, err := sqlite.NewSQLLiteLoader(sqlite.WithDBName(dbName))
 	if err != nil {
 		return err
 	}

cmd/initializer/main.go

@@ -53,7 +53,7 @@ func runCmdFunc(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	dbLoader, err := sqlite.NewSQLLiteLoader(outFilename)
+	dbLoader, err := sqlite.NewSQLLiteLoader(sqlite.WithDBName(outFilename))
 	if err != nil {
 		return err
 	}

go.mod

@@ -3,6 +3,7 @@ module github.com/operator-framework/operator-registry
 require (
 	github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6
 	github.com/ghodss/yaml v1.0.0
+	github.com/golang-migrate/migrate/v4 v4.6.2
 	github.com/golang/mock v1.2.0
 	github.com/golang/protobuf v1.3.1
 	github.com/googleapis/gnostic v0.2.0 // indirect
@@ -16,12 +17,12 @@ require (
 	github.com/stretchr/testify v1.3.0
 	golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
-	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
 	google.golang.org/grpc v1.23.0
-	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.2.2
 	k8s.io/api v0.0.0-20190918155943-95b840bb6a1f
 	k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783
 	k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655
 	k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90
 )
+
+go 1.13

pkg/appregistry/dbloader.go

@@ -11,7 +11,7 @@ import (
 )
 
 func NewDbLoader(dbName string, logger *logrus.Entry) (*dbLoader, error) {
-	sqlLoader, err := sqlite.NewSQLLiteLoader(dbName)
+	sqlLoader, err := sqlite.NewSQLLiteLoader(sqlite.WithDBName(dbName))
 	if err != nil {
 		return nil, err
 	}

pkg/server/server_test.go

@@ -29,7 +29,7 @@ func server() {
 	}
 	s := grpc.NewServer()
 
-	load, err := sqlite.NewSQLLiteLoader(dbName)
+	load, err := sqlite.NewSQLLiteLoader(sqlite.WithDBName(dbName))
 	if err != nil {
 		logrus.Fatal(err)
 	}

pkg/sqlite/configmap_test.go

@@ -4,19 +4,19 @@ import (
 	"context"
 	"os"
 	"testing"
-	
+
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/yaml"
-	
+
 	"github.com/operator-framework/operator-registry/pkg/registry"
 )
 
 func TestConfigMapLoader(t *testing.T) {
 	logrus.SetLevel(logrus.DebugLevel)
 
-	store, err := NewSQLLiteLoader("test.db")
+	store, err := NewSQLLiteLoader(WithDBName("test.db"))
 	require.NoError(t, err)
 	defer os.Remove("test.db")
 
@@ -34,7 +34,7 @@ func TestConfigMapLoader(t *testing.T) {
 }
 
 func TestQuerierForConfigmap(t *testing.T) {
-	load, err := NewSQLLiteLoader("test.db")
+	load, err := NewSQLLiteLoader(WithDBName("test.db"))
 	require.NoError(t, err)
 	defer os.Remove("test.db")
 

pkg/sqlite/db_migrations/201909251211_initialize_version.down.sql

@@ -0,0 +1 @@
+/* Initialize the database version */

pkg/sqlite/db_migrations/201909251211_initialize_version.up.sql

@@ -0,0 +1 @@
+/* Initialize the database version */

pkg/sqlite/db_options.go

@@ -0,0 +1,23 @@
+package sqlite
+
+type DbOptions struct {
+	// OutFileName is used to define the database file name that is generated from the loader
+	OutFileName string
+
+	// Migrator refers to the SQL migrator used to initialize the database with
+	MigrationsPath string
+}
+
+type DbOption func(*DbOptions)
+
+func WithDBName(name string) DbOption {
+	return func(o *DbOptions) {
+        o.OutFileName = name
+    }
+}
+
+func WithMigrationsPath(path string) DbOption {
+	return func(o *DbOptions) {
+        o.MigrationsPath = path
+    }
+}

pkg/sqlite/directory_test.go

@@ -18,7 +18,7 @@ import (
 func TestDirectoryLoader(t *testing.T) {
 	logrus.SetLevel(logrus.DebugLevel)
 
-	store, err := NewSQLLiteLoader("test.db")
+	store, err := NewSQLLiteLoader(WithDBName("test.db"))
 	require.NoError(t, err)
 	defer func() {
 		if err := os.Remove("test.db"); err != nil {
@@ -33,7 +33,7 @@ func TestDirectoryLoader(t *testing.T) {
 func TestDirectoryLoaderWithBadManifests(t *testing.T) {
 	logrus.SetLevel(logrus.DebugLevel)
 
-	store, err := NewSQLLiteLoader("test.db")
+	store, err := NewSQLLiteLoader(WithDBName("test.db"))
 	require.NoError(t, err)
 	defer func() {
 		if err := os.Remove("test.db"); err != nil {
@@ -72,7 +72,7 @@ func TestDirectoryLoaderWithBadManifests(t *testing.T) {
 }
 
 func TestQuerierForDirectory(t *testing.T) {
-	load, err := NewSQLLiteLoader("test.db")
+	load, err := NewSQLLiteLoader(WithDBName("test.db"))
 	require.NoError(t, err)
 	defer func() {
 		if err := os.Remove("test.db"); err != nil {

pkg/sqlite/load.go

@@ -20,8 +20,13 @@ type SQLLoader struct {
 
 var _ registry.Load = &SQLLoader{}
 
-func NewSQLLiteLoader(outFilename string) (*SQLLoader, error) {
-	db, err := sql.Open("sqlite3", outFilename) // TODO: ?immutable=true
+func NewSQLLiteLoader(opts ...DbOption) (*SQLLoader, error) {
+	options := DbOptions{}
+    for _, o := range opts {
+        o(&options)
+    }
+
+	db, err := sql.Open("sqlite3", options.OutFileName) // TODO: ?immutable=true
 	if err != nil {
 		return nil, err
 	}
@@ -77,6 +82,19 @@ func NewSQLLiteLoader(outFilename string) (*SQLLoader, error) {
 	if _, err = db.Exec(createTable); err != nil {
 		return nil, err
 	}
+
+	// Apply the current latest database version to keep net new databases in sync with upgradeable ones
+	migrator, err := NewSQLLiteMigrator(db, options.MigrationsPath)
+	if err != nil {
+		return nil, err
+	}
+	defer migrator.CleanUpMigrator()
+
+	err = migrator.InitMigrationVersion()
+	if err != nil {
+		return nil, err
+	}
+
 	return &SQLLoader{db}, nil
 }
 

pkg/sqlite/load_test.go

@@ -131,7 +131,7 @@ func TestAddPackageChannels(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.description, func(t *testing.T) {
 			db := fmt.Sprintf("%d.db", rand.Int())
-			store, err := NewSQLLiteLoader(db)
+			store, err := NewSQLLiteLoader(WithDBName(db))
 			require.NoError(t, err)
 			defer func() {
 				if err := os.Remove(db); err != nil {