Namespaces (#1459)

* feat: (sqlite) support namespace at storage level, mostly for flags (#1368) * feat: (sqlite) wip support namespace at storage level, mostly for flags * chore: rename down namespace file correctly * chore: fix server/flag_test.go * chore: redo migration to use temp_tables for composite keys * chore: just use TODO for now for down migration * chore: comments * chore: wip namespace protos * chore: better err messages * Update internal/storage/sql/common/flag.go Co-authored-by: George <me@georgemac.com> * chore: fix tests * chore: split into seperate migrations * chore: more error messages * chore: rm println statment --------- Co-authored-by: George <me@georgemac.com> * feat: Namespaces segments storage (#1369) * feat: (sqlite) add namespace support for rules storage (#1371) * feat: (sqlite) eval storage namespace support (#1372) * Namespaces storage (#1383) * feat: wip namespace storage * chore: add tests; storage impl * chore: appease the linter * chore: less sleepy * Namespaces mysql (#1386) * feat: wip namespace storage * chore: add tests; storage impl * chore: appease the linter * feat: (wip) add namespace mysql migrations * chore: rename migrations * chore: rm empty migrations for the moment * chore: less sleepy * chore: 6 bytes for timestamp * chore: add rules migration * chore: add migration comments, revert local config * chore: reset mysql example * Namespaces postgres (#1388) * feat: (wip) postgres ddl * chore: fix tests * chore: disable container logging unless verbose env var passed * Namespaces cockroach (#1390) * feat: cockroachdb migrations * chore: fix migrations for cockroach * chore: namespaces down migrations (#1396) * Namespaces: flags storage tests (#1406) * Namespaces: segments storage tests (#1408) * chore: add segment namespace storage tests * chore: regen protos * chore: ignore sdk dir when fmting * chore: add remaining storage layer tests (#1412) * feat: add namespaces server mappings (#1415) * chore: add remaining storage layer tests * feat: add namespaces server mappings * chore: add back default namespace to otel metrics * chore: fix middleware test * Namespaces rpc (#1421) * feat: check for protected or flags existing when deleting a namespace (#1422) * feat: add namespaces server impl * chore: spacing * chore: rm ability to set protected * feat: check for protected or flags existing when deleting a namespace * chore: fix test * chore: add test for non-existing delete * feat(hack/build): add cases for namespace scoped integration tests (#1436) * chore: regenerate protobuf * feat(rpc/flipt): configure gateway routes for namespaces * feat(hack/build): add cases for namespace scoped integration tests * chore(rpc/flipt): remove TODO * chore(hack/build): use random string for namespace key * test(hack/build): ensure default namespace cannot be deleted * test(hack/build): ensure default namespace is protected but updateable * test(hack/build): ensure namespace with flags cannot be deleted * feat: add check for rule namespaced errors * fix: namespace segment REST API routes were not correct (#1445) * feat: add distribution test for cross-namespace entities * feat: leave namespace check to distribution * feat: generate namespaced scoped routes for distributions * chore: address changes in regards to error messaging * chore: abstract out the two types of DistributionRequests * fix(migrations/sqlite): reorder to ensure we copy all data before drop (#1446) * fix(migrations/sqlite): reorder to ensure we copy all data before drop * fix(migrations/sqlite): copy distributions into temp table before dropping everything * chore: do joins on the query and make error messages match * chore: merge main into namespaces (#1448) * chore(deps): bump go.opentelemetry.io/otel/exporters/zipkin (#1440) Bumps [go.opentelemetry.io/otel/exporters/zipkin](https://github.com/open-telemetry/opentelemetry-go) from 1.13.0 to 1.14.0. - [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md) - [Commits](open-telemetry/opentelemetry-go@v1.13.0...v1.14.0) --- updated-dependencies: - dependency-name: go.opentelemetry.io/otel/exporters/zipkin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat(github): dagger based integrations tests github workflow (#1428) * feat(github): install dagger based ITs as action workflow * fix(github): add workdir to mage action * chore(hack/build): bump dagger to 0.5.1 * chore: enable daggers experimental gha cache * chore(hack/build): reword test log lines * fix(github): move env var from legacy to experimental tests * chore(hack/build): arbitrary change to trigger rebuild * chore(github): remove experimental dagger cache env var * feat(hack/build): direct integration test flipt logs into directory * chore: empty commit to trigger CI * chore(hack/build): log error when failing to copy flipt logs * chore(hack/build): adding log line to integration test start * fix(github): stop caching entire hack build directory * refactor(hack/build): organize integration cases into test case struct * refactor(github): remove legacy API workflows and replace with dagger mage task * chore(hack/build): use protocolPorts map for test case iteration --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore: add missing namespace metrics (#1450) * chore: allow testing of namespaces ui branch for build * chore: rm down migrations (#1449) * chore: wip rm down migrations * chore: fix tests * chore: fix tests; cleanup * chore: use new migrator on import if drop specified * chore: fix drop before import for sqlite * chore: move mysql fk disable into migrator down method * chore: codecov's last chance * feat(cmd): import and export namespace flag (#1452) * refactor(export): promote namespace to NewExporter argument * refactor(export): use GRPC service interface as exporter dependency * feat(export): support --export-from-address remote flipt address flag * refactor(cmd): share config as argument not global state * refactor(cmd/export): move command boostrapping from main.go to export.go * refactor(cmd): move signal trapping into main * refactor(cmd/import): move command boostrapping from main.go to import.go * refactor(ext): expose namespace as argument to NewImporter * feat(cmd/import): support remote import via --import-to-address * refactor(hack/build): restructure testing packages * refactor(hack/build): create import and export integration tests * feat(hack/build): more import integration test cases * feat(cmd/export): support --namespace flag * chore: go mod tidy * fix(cmd): do not close db early in import/export * refactor(cmd): rename import/export flags to --address and --token * refactor(cmd/import): move server config into server.go * fix(cmd/import): pass namespace key as name on create * fix(hack/build): assert returned namespace name is as expected * chore: validation logic for namespace requests (#1457) * chore: validation logic for namespace requests * chore: fix import command to use the namespace key as the name as well * chore: rm TODO comments in exporter * chore: mage proto * chore: update changelog for future namespace release; add mage proto lint check (#1460) * chore: trigger lint build * chore: rm mage proto check for now as its not working --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: George <me@georgemac.com> Co-authored-by: Yoofi Quansah <ybquansah@gmail.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
flipt-io · Apr 5, 2023 · 1eb2750 · 1eb2750
1 parent f04ddb1
commit 1eb2750
Show file tree

Hide file tree

Showing 100 changed files with 13,668 additions and 3,308 deletions.
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -76,4 +76,4 @@ jobs:
         run: go mod tidy
 
       - name: Ensure clean git state.
-        run: git diff-index --quiet HEAD -- || (echo "Please run go mod tidy." && exit 1)
+        run: git diff-index --quiet HEAD -- || (echo "Please run 'go mod tidy' and commit changes." && exit 1)
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,18 @@
 This format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Added
+
+- Support for 'namespacing' / multi-environments. All types can now belong to a namespace allowing you to seperate your flags/segments/etc.
+
+### Changed
+
+- All existing objects have been moved to the 'default' namespace to be fully backward compatible.
+- Import/Export have been updated to be 'namespace-aware'
+- Dependency updates
+
 ## [v1.19.3](https://github.com/flipt-io/flipt/releases/tag/v1.19.3) - 2023-03-22
 
 ### Changed

diff --git a/cmd/flipt/export.go b/cmd/flipt/export.go
@@ -5,73 +5,100 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"os/signal"
-	"syscall"
 	"time"
 
+	"github.com/spf13/cobra"
 	"go.flipt.io/flipt/internal/ext"
-	"go.flipt.io/flipt/internal/storage"
-	"go.flipt.io/flipt/internal/storage/sql"
-	"go.flipt.io/flipt/internal/storage/sql/mysql"
-	"go.flipt.io/flipt/internal/storage/sql/postgres"
-	"go.flipt.io/flipt/internal/storage/sql/sqlite"
 	"go.uber.org/zap"
 )
 
-var exportFilename string
-
-func runExport(ctx context.Context, logger *zap.Logger) error {
-	ctx, cancel := context.WithCancel(ctx)
-
-	defer cancel()
-
-	interrupt := make(chan os.Signal, 1)
-	signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM)
+type exportCommand struct {
+	filename  string
+	address   string
+	token     string
+	namespace string
+}
 
-	go func() {
-		<-interrupt
-		cancel()
-	}()
+func newExportCommand() *cobra.Command {
+	export := &exportCommand{}
 
-	db, driver, err := sql.Open(*cfg)
-	if err != nil {
-		return fmt.Errorf("opening db: %w", err)
+	cmd := &cobra.Command{
+		Use:   "export",
+		Short: "Export flags/segments/rules to file/stdout",
+		RunE:  export.run,
 	}
 
-	defer db.Close()
-
-	var store storage.Store
-
-	switch driver {
-	case sql.SQLite:
-		store = sqlite.NewStore(db, logger)
-	case sql.Postgres, sql.CockroachDB:
-		store = postgres.NewStore(db, logger)
-	case sql.MySQL:
-		store = mysql.NewStore(db, logger)
-	}
+	cmd.Flags().StringVarP(
+		&export.filename,
+		"output", "o",
+		"",
+		"export to filename (default STDOUT)",
+	)
+
+	cmd.Flags().StringVarP(
+		&export.address,
+		"address", "a",
+		"",
+		"address of remote Flipt instance to export from (defaults to direct DB export if not supplied)",
+	)
+
+	cmd.Flags().StringVarP(
+		&export.token,
+		"token", "t",
+		"",
+		"client token used to authenticate access to remote Flipt instance when exporting.",
+	)
+
+	cmd.Flags().StringVarP(
+		&export.namespace,
+		"namespace", "n",
+		"default",
+		"source namespace for exported resources.",
+	)
+
+	return cmd
+}
 
-	// default to stdout
-	var out io.WriteCloser = os.Stdout
+func (c *exportCommand) run(cmd *cobra.Command, _ []string) error {
+	var (
+		// default to stdout
+		out    io.Writer = os.Stdout
+		logger           = zap.Must(zap.NewDevelopment())
+	)
 
 	// export to file
-	if exportFilename != "" {
-		logger.Debug("exporting", zap.String("destination_path", exportFilename))
+	if c.filename != "" {
+		logger.Debug("exporting", zap.String("destination_path", c.filename))
 
-		out, err = os.Create(exportFilename)
+		fi, err := os.Create(c.filename)
 		if err != nil {
 			return fmt.Errorf("creating output file: %w", err)
 		}
 
-		fmt.Fprintf(out, "# exported by Flipt (%s) on %s\n\n", version, time.Now().UTC().Format(time.RFC3339))
+		defer fi.Close()
+
+		fmt.Fprintf(fi, "# exported by Flipt (%s) on %s\n\n", version, time.Now().UTC().Format(time.RFC3339))
+
+		out = fi
 	}
 
-	defer out.Close()
+	// Use client when remote address is configured.
+	if c.address != "" {
+		return c.export(cmd.Context(), out, fliptClient(logger, c.address, c.token))
+	}
 
-	exporter := ext.NewExporter(store)
-	if err := exporter.Export(ctx, out); err != nil {
-		return fmt.Errorf("exporting: %w", err)
+	// Otherwise, go direct to the DB using Flipt configuration file.
+	logger, cfg := buildConfig()
+	server, cleanup, err := fliptServer(logger, cfg)
+	if err != nil {
+		return err
 	}
 
-	return nil
+	defer cleanup()
+
+	return c.export(cmd.Context(), out, server)
+}
+
+func (c *exportCommand) export(ctx context.Context, dst io.Writer, lister ext.Lister) error {
+	return ext.NewExporter(lister, c.namespace).Export(ctx, dst)
 }
diff --git a/cmd/flipt/import.go b/cmd/flipt/import.go
@@ -1,63 +1,88 @@
 package main
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"io"
 	"os"
-	"os/signal"
 	"path/filepath"
-	"syscall"
 
+	"github.com/spf13/cobra"
 	"go.flipt.io/flipt/internal/ext"
-	"go.flipt.io/flipt/internal/storage"
 	"go.flipt.io/flipt/internal/storage/sql"
-	"go.flipt.io/flipt/internal/storage/sql/mysql"
-	"go.flipt.io/flipt/internal/storage/sql/postgres"
-	"go.flipt.io/flipt/internal/storage/sql/sqlite"
 	"go.uber.org/zap"
 )
 
-var (
+type importCommand struct {
 	dropBeforeImport bool
 	importStdin      bool
-)
-
-func runImport(ctx context.Context, logger *zap.Logger, args []string) error {
-	ctx, cancel := context.WithCancel(ctx)
-
-	defer cancel()
-
-	interrupt := make(chan os.Signal, 1)
-	signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM)
+	address          string
+	token            string
+	namespace        string
+	createNamespace  bool
+}
 
-	go func() {
-		<-interrupt
-		cancel()
-	}()
+func newImportCommand() *cobra.Command {
+	importCmd := &importCommand{}
 
-	db, driver, err := sql.Open(*cfg)
-	if err != nil {
-		return fmt.Errorf("opening db: %w", err)
+	cmd := &cobra.Command{
+		Use:   "import",
+		Short: "Import flags/segments/rules from file",
+		RunE:  importCmd.run,
 	}
 
-	defer db.Close()
-
-	var store storage.Store
-
-	switch driver {
-	case sql.SQLite:
-		store = sqlite.NewStore(db, logger)
-	case sql.Postgres, sql.CockroachDB:
-		store = postgres.NewStore(db, logger)
-	case sql.MySQL:
-		store = mysql.NewStore(db, logger)
-	}
+	cmd.Flags().BoolVar(
+		&importCmd.dropBeforeImport,
+		"drop",
+		false,
+		"drop database before import",
+	)
+
+	cmd.Flags().BoolVar(
+		&importCmd.importStdin,
+		"stdin",
+		false,
+		"import from STDIN",
+	)
+
+	cmd.Flags().StringVarP(
+		&importCmd.address,
+		"address", "a",
+		"",
+		"address of remote Flipt instance to import into (defaults to direct DB import if not supplied)",
+	)
+
+	cmd.Flags().StringVarP(
+		&importCmd.token,
+		"token", "t",
+		"",
+		"client token used to authenticate access to remote Flipt instance when importing.",
+	)
+
+	cmd.Flags().StringVarP(
+		&importCmd.namespace,
+		"namespace", "n",
+		"default",
+		"destination namespace for imported resources.",
+	)
+
+	cmd.Flags().BoolVar(
+		&importCmd.createNamespace,
+		"create-namespace",
+		false,
+		"create the namespace if it does not exist.",
+	)
+
+	return cmd
+}
 
-	var in io.ReadCloser = os.Stdin
+func (c *importCommand) run(cmd *cobra.Command, args []string) error {
+	var (
+		in     io.Reader = os.Stdin
+		logger           = zap.Must(zap.NewDevelopment())
+	)
 
-	if !importStdin {
+	if !c.importStdin {
 		importFilename := args[0]
 		if importFilename == "" {
 			return errors.New("import filename required")
@@ -67,13 +92,26 @@ func runImport(ctx context.Context, logger *zap.Logger, args []string) error {
 
 		logger.Debug("importing", zap.String("source_path", f))
 
-		in, err = os.Open(f)
+		fi, err := os.Open(f)
 		if err != nil {
 			return fmt.Errorf("opening import file: %w", err)
 		}
+
+		defer fi.Close()
+
+		in = fi
+	}
+
+	// Use client when remote address is configured.
+	if c.address != "" {
+		return ext.NewImporter(
+			fliptClient(logger, c.address, c.token),
+			c.namespace,
+			c.createNamespace,
+		).Import(cmd.Context(), in)
 	}
 
-	defer in.Close()
+	logger, cfg := buildConfig()
 
 	migrator, err := sql.NewMigrator(*cfg, logger)
 	if err != nil {
@@ -83,11 +121,11 @@ func runImport(ctx context.Context, logger *zap.Logger, args []string) error {
 	defer migrator.Close()
 
 	// drop tables if specified
-	if dropBeforeImport {
-		logger.Debug("dropping tables before import")
+	if c.dropBeforeImport {
+		logger.Debug("dropping tables")
 
-		if err := migrator.Down(); err != nil {
-			return fmt.Errorf("attempting to drop during import: %w", err)
+		if err := migrator.Drop(); err != nil {
+			return fmt.Errorf("attempting to drop: %w", err)
 		}
 	}
 
@@ -99,10 +137,17 @@ func runImport(ctx context.Context, logger *zap.Logger, args []string) error {
 		return fmt.Errorf("closing migrator: %w", err)
 	}
 
-	importer := ext.NewImporter(store)
-	if err := importer.Import(ctx, in); err != nil {
-		return fmt.Errorf("importing: %w", err)
+	// Otherwise, go direct to the DB using Flipt configuration file.
+	server, cleanup, err := fliptServer(logger, cfg)
+	if err != nil {
+		return err
 	}
 
-	return nil
+	defer cleanup()
+
+	return ext.NewImporter(
+		server,
+		c.namespace,
+		c.createNamespace,
+	).Import(cmd.Context(), in)
 }