feat(cmd): import and export namespace flag (#1452)

* refactor(export): promote namespace to NewExporter argument

* refactor(export): use GRPC service interface as exporter dependency

* feat(export): support --export-from-address remote flipt address flag

* refactor(cmd): share config as argument not global state

* refactor(cmd/export): move command boostrapping from main.go to export.go

* refactor(cmd): move signal trapping into main

* refactor(cmd/import): move command boostrapping from main.go to import.go

* refactor(ext): expose namespace as argument to NewImporter

* feat(cmd/import): support remote import via --import-to-address

* refactor(hack/build): restructure testing packages

* refactor(hack/build): create import and export integration tests

* feat(hack/build): more import integration test cases

* feat(cmd/export): support --namespace flag

* chore: go mod tidy

* fix(cmd): do not close db early in import/export

* refactor(cmd): rename import/export flags to --address and --token

* refactor(cmd/import): move server config into server.go

cmd/flipt/export.go

@@ -5,73 +5,100 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"os/signal"
-	"syscall"
 	"time"
 
+	"github.com/spf13/cobra"
 	"go.flipt.io/flipt/internal/ext"
-	"go.flipt.io/flipt/internal/storage"
-	"go.flipt.io/flipt/internal/storage/sql"
-	"go.flipt.io/flipt/internal/storage/sql/mysql"
-	"go.flipt.io/flipt/internal/storage/sql/postgres"
-	"go.flipt.io/flipt/internal/storage/sql/sqlite"
 	"go.uber.org/zap"
 )
 
-var exportFilename string
-
-func runExport(ctx context.Context, logger *zap.Logger) error {
-	ctx, cancel := context.WithCancel(ctx)
-
-	defer cancel()
-
-	interrupt := make(chan os.Signal, 1)
-	signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM)
+type exportCommand struct {
+	filename  string
+	address   string
+	token     string
+	namespace string
+}
 
-	go func() {
-		<-interrupt
-		cancel()
-	}()
+func newExportCommand() *cobra.Command {
+	export := &exportCommand{}
 
-	db, driver, err := sql.Open(*cfg)
-	if err != nil {
-		return fmt.Errorf("opening db: %w", err)
+	cmd := &cobra.Command{
+		Use:   "export",
+		Short: "Export flags/segments/rules to file/stdout",
+		RunE:  export.run,
 	}
 
-	defer db.Close()
-
-	var store storage.Store
-
-	switch driver {
-	case sql.SQLite:
-		store = sqlite.NewStore(db, logger)
-	case sql.Postgres, sql.CockroachDB:
-		store = postgres.NewStore(db, logger)
-	case sql.MySQL:
-		store = mysql.NewStore(db, logger)
-	}
+	cmd.Flags().StringVarP(
+		&export.filename,
+		"output", "o",
+		"",
+		"export to filename (default STDOUT)",
+	)
+
+	cmd.Flags().StringVarP(
+		&export.address,
+		"address", "a",
+		"",
+		"address of remote Flipt instance to export from (defaults to direct DB export if not supplied)",
+	)
+
+	cmd.Flags().StringVarP(
+		&export.token,
+		"token", "t",
+		"",
+		"client token used to authenticate access to remote Flipt instance when exporting.",
+	)
+
+	cmd.Flags().StringVarP(
+		&export.namespace,
+		"namespace", "n",
+		"default",
+		"source namespace for exported resources.",
+	)
+
+	return cmd
+}
 
-	// default to stdout
-	var out io.WriteCloser = os.Stdout
+func (c *exportCommand) run(cmd *cobra.Command, _ []string) error {
+	var (
+		// default to stdout
+		out    io.Writer = os.Stdout
+		logger           = zap.Must(zap.NewDevelopment())
+	)
 
 	// export to file
-	if exportFilename != "" {
-		logger.Debug("exporting", zap.String("destination_path", exportFilename))
+	if c.filename != "" {
+		logger.Debug("exporting", zap.String("destination_path", c.filename))
 
-		out, err = os.Create(exportFilename)
+		fi, err := os.Create(c.filename)
 		if err != nil {
 			return fmt.Errorf("creating output file: %w", err)
 		}
 
-		fmt.Fprintf(out, "# exported by Flipt (%s) on %s\n\n", version, time.Now().UTC().Format(time.RFC3339))
+		defer fi.Close()
+
+		fmt.Fprintf(fi, "# exported by Flipt (%s) on %s\n\n", version, time.Now().UTC().Format(time.RFC3339))
+
+		out = fi
 	}
 
-	defer out.Close()
+	// Use client when remote address is configured.
+	if c.address != "" {
+		return c.export(cmd.Context(), out, fliptClient(logger, c.address, c.token))
+	}
 
-	exporter := ext.NewExporter(store)
-	if err := exporter.Export(ctx, out); err != nil {
-		return fmt.Errorf("exporting: %w", err)
+	// Otherwise, go direct to the DB using Flipt configuration file.
+	logger, cfg := buildConfig()
+	server, cleanup, err := fliptServer(logger, cfg)
+	if err != nil {
+		return err
 	}
 
-	return nil
+	defer cleanup()
+
+	return c.export(cmd.Context(), out, server)
+}
+
+func (c *exportCommand) export(ctx context.Context, dst io.Writer, lister ext.Lister) error {
+	return ext.NewExporter(lister, c.namespace).Export(ctx, dst)
 }

cmd/flipt/import.go

@@ -1,63 +1,88 @@
 package main
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"io"
 	"os"
-	"os/signal"
 	"path/filepath"
-	"syscall"
 
+	"github.com/spf13/cobra"
 	"go.flipt.io/flipt/internal/ext"
-	"go.flipt.io/flipt/internal/storage"
 	"go.flipt.io/flipt/internal/storage/sql"
-	"go.flipt.io/flipt/internal/storage/sql/mysql"
-	"go.flipt.io/flipt/internal/storage/sql/postgres"
-	"go.flipt.io/flipt/internal/storage/sql/sqlite"
 	"go.uber.org/zap"
 )
 
-var (
+type importCommand struct {
 	dropBeforeImport bool
 	importStdin      bool
-)
-
-func runImport(ctx context.Context, logger *zap.Logger, args []string) error {
-	ctx, cancel := context.WithCancel(ctx)
-
-	defer cancel()
-
-	interrupt := make(chan os.Signal, 1)
-	signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM)
+	address          string
+	token            string
+	namespace        string
+	createNamespace  bool
+}
 
-	go func() {
-		<-interrupt
-		cancel()
-	}()
+func newImportCommand() *cobra.Command {
+	importCmd := &importCommand{}
 
-	db, driver, err := sql.Open(*cfg)
-	if err != nil {
-		return fmt.Errorf("opening db: %w", err)
+	cmd := &cobra.Command{
+		Use:   "import",
+		Short: "Import flags/segments/rules from file",
+		RunE:  importCmd.run,
 	}
 
-	defer db.Close()
-
-	var store storage.Store
-
-	switch driver {
-	case sql.SQLite:
-		store = sqlite.NewStore(db, logger)
-	case sql.Postgres, sql.CockroachDB:
-		store = postgres.NewStore(db, logger)
-	case sql.MySQL:
-		store = mysql.NewStore(db, logger)
-	}
+	cmd.Flags().BoolVar(
+		&importCmd.dropBeforeImport,
+		"drop",
+		false,
+		"drop database before import",
+	)
+
+	cmd.Flags().BoolVar(
+		&importCmd.importStdin,
+		"stdin",
+		false,
+		"import from STDIN",
+	)
+
+	cmd.Flags().StringVarP(
+		&importCmd.address,
+		"address", "a",
+		"",
+		"address of remote Flipt instance to import into (defaults to direct DB import if not supplied)",
+	)
+
+	cmd.Flags().StringVarP(
+		&importCmd.token,
+		"token", "t",
+		"",
+		"client token used to authenticate access to remote Flipt instance when importing.",
+	)
+
+	cmd.Flags().StringVarP(
+		&importCmd.namespace,
+		"namespace", "n",
+		"default",
+		"destination namespace for imported resources.",
+	)
+
+	cmd.Flags().BoolVar(
+		&importCmd.createNamespace,
+		"create-namespace",
+		false,
+		"create the namespace if it does not exist.",
+	)
+
+	return cmd
+}
 
-	var in io.ReadCloser = os.Stdin
+func (c *importCommand) run(cmd *cobra.Command, args []string) error {
+	var (
+		in     io.Reader = os.Stdin
+		logger           = zap.Must(zap.NewDevelopment())
+	)
 
-	if !importStdin {
+	if !c.importStdin {
 		importFilename := args[0]
 		if importFilename == "" {
 			return errors.New("import filename required")
@@ -67,37 +92,43 @@ func runImport(ctx context.Context, logger *zap.Logger, args []string) error {
 
 		logger.Debug("importing", zap.String("source_path", f))
 
-		in, err = os.Open(f)
+		fi, err := os.Open(f)
 		if err != nil {
 			return fmt.Errorf("opening import file: %w", err)
 		}
-	}
-
-	defer in.Close()
 
-	// drop tables if specified
-	if dropBeforeImport {
-		logger.Debug("dropping tables before import")
-		migrator, err := sql.NewMigrator(*cfg, logger)
-		if err != nil {
-			return err
-		}
+		defer fi.Close()
 
-		if err := migrator.Drop(); err != nil {
-			return fmt.Errorf("attempting to drop during import: %w", err)
-		}
+		in = fi
+	}
 
-		if _, err := migrator.Close(); err != nil {
-			return fmt.Errorf("closing migrator: %w", err)
-		}
+	// Use client when remote address is configured.
+	if c.address != "" {
+		return ext.NewImporter(
+			fliptClient(logger, c.address, c.token),
+			c.namespace,
+			c.createNamespace,
+		).Import(cmd.Context(), in)
 	}
 
+	logger, cfg := buildConfig()
+
 	migrator, err := sql.NewMigrator(*cfg, logger)
 	if err != nil {
 		return err
 	}
 
 	defer migrator.Close()
+
+	// drop tables if specified
+	if c.dropBeforeImport {
+		logger.Debug("dropping tables")
+
+		if err := migrator.Drop(); err != nil {
+			return fmt.Errorf("attempting to drop: %w", err)
+		}
+	}
+
 	if err := migrator.Up(forceMigrate); err != nil {
 		return err
 	}
@@ -106,10 +137,17 @@ func runImport(ctx context.Context, logger *zap.Logger, args []string) error {
 		return fmt.Errorf("closing migrator: %w", err)
 	}
 
-	importer := ext.NewImporter(store)
-	if err := importer.Import(ctx, in); err != nil {
-		return fmt.Errorf("importing: %w", err)
+	// Otherwise, go direct to the DB using Flipt configuration file.
+	server, cleanup, err := fliptServer(logger, cfg)
+	if err != nil {
+		return err
 	}
 
-	return nil
+	defer cleanup()
+
+	return ext.NewImporter(
+		server,
+		c.namespace,
+		c.createNamespace,
+	).Import(cmd.Context(), in)
 }