Import/export (#238)

Add import/export functionality

.github/workflows/database-test.yml

@@ -0,0 +1,45 @@
+name: Database Tests
+on:
+  push:
+    paths-ignore:
+      - '*.md'
+      - '.all-contributorsrc'
+
+jobs:
+  ## Postgres Tests
+  postgres:
+    name: Postgres
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres@sha256:c132d7802dcc127486a403fb9e9a52d9df2e3ab84037c5de8395ed6ba2743e20
+        ports:
+        # will assign a random free host port
+        - 5432/tcp
+        # needed because the postgres container does not provide a healthcheck
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+        env:
+          POSTGRES_DB: flipt_test
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: ''
+
+    steps:
+    - name: Setup Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13.7
+      id: go
+
+    - name: Checkout
+      uses: actions/checkout@v1
+
+    - name: Restore Cache
+      uses: actions/cache@preview
+      id: cache
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-${{ hashFiles('**/go.sum') }}
+
+    - name: Unit Test (Postgres)
+      run: DB_URL="postgres://postgres@localhost:${{ job.services.postgres.ports['5432'] }}/flipt_test?sslmode=disable" go test -count=1 -v ./...

.github/workflows/integration-test.yml

@@ -9,9 +9,6 @@ on:
   pull_request:
     branches:
       - master
-    paths-ignore:
-      - '*.md'
-      - '.all-contributorsrc'
 
 jobs:
 
@@ -40,12 +37,12 @@ jobs:
     - name: Build binary
       run: go build -o ./bin/flipt ./cmd/flipt/.
 
-    - name: Test CLI
+    - name: Test API
       uses: ./.github/actions/api-test
       with:
-        args: ./script/test/cli
+        args: ./script/test/api
 
-    - name: Test API
+    - name: Test CLI
       uses: ./.github/actions/api-test
       with:
-        args: ./script/test/api
+        args: ./script/test/cli

.github/workflows/test.yml

@@ -40,19 +40,6 @@ jobs:
     name: Test
     runs-on: ubuntu-latest
 
-    services:
-      postgres:
-        image: postgres@sha256:c132d7802dcc127486a403fb9e9a52d9df2e3ab84037c5de8395ed6ba2743e20
-        ports:
-        # will assign a random free host port
-        - 5432/tcp
-        # needed because the postgres container does not provide a healthcheck
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
-        env:
-          POSTGRES_DB: flipt_test
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: ''
-
     steps:
     - name: Setup Go
       uses: actions/setup-go@v1
@@ -79,6 +66,3 @@ jobs:
         CI_BRANCH: ${{ github.ref }}
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       run: bash <(curl -s https://codecov.io/bash)
-
-    - name: Unit Test (Postgres)
-      run: DB_URL="postgres://postgres@localhost:${{ job.services.postgres.ports['5432'] }}/flipt_test?sslmode=disable" go test -count=1 -v ./...

CHANGELOG.md

@@ -3,6 +3,20 @@
 This format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Added
+
+* `export` and `import` commands to export and import data [https://github.com/markphelps/flipt/issues/225](https://github.com/markphelps/flipt/issues/225)
+
+### Changed
+
+* List calls are no longer cached because of pagination
+
+### Fixed
+
+* Issue where `GetRule` would not return proper error if rule did not exist
+
 ## [v0.12.1](https://github.com/markphelps/flipt/releases/tag/v0.12.1) - 2020-02-18
 
 ### Fixed

Makefile

@@ -113,5 +113,5 @@ clients: ## Generate GRPC clients
 help:
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-.DEFAULT_GOAL := help
-default: help
+.DEFAULT_GOAL := build
+default: build

README.md

@@ -38,8 +38,8 @@
         <a href="#features">Features</a> |
         <a href="#values">Values</a> |
         <a href="#usecases">Usecases</a> |
-        <a href="#examples">Examples</a> | 
-	<a href="#enterprise">Enterprise</a>
+        <a href="#examples">Examples</a> |
+        <a href="#enterprise">Enterprise</a>
     </h4>
 </div>
 
@@ -56,6 +56,7 @@ Flipt can be deployed within your existing infrastructure so that you don't have
 * Simple REST API
 * Modern UI and debug console
 * Support for multiple databases
+* Data import and export to allow storing your flags as code
 
 ## Values
 

cmd/flipt/export.go

@@ -0,0 +1,222 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	sq "github.com/Masterminds/squirrel"
+	"github.com/markphelps/flipt/storage"
+	"github.com/markphelps/flipt/storage/db"
+	"gopkg.in/yaml.v2"
+)
+
+type Document struct {
+	Flags    []*Flag    `yaml:"flags,omitempty"`
+	Segments []*Segment `yaml:"segments,omitempty"`
+}
+
+type Flag struct {
+	Key         string     `yaml:"key,omitempty"`
+	Name        string     `yaml:"name,omitempty"`
+	Description string     `yaml:"description,omitempty"`
+	Enabled     bool       `yaml:"enabled"`
+	Variants    []*Variant `yaml:"variants,omitempty"`
+	Rules       []*Rule    `yaml:"rules,omitempty"`
+}
+
+type Variant struct {
+	Key         string `yaml:"key,omitempty"`
+	Name        string `yaml:"name,omitempty"`
+	Description string `yaml:"description,omitempty"`
+}
+
+type Rule struct {
+	SegmentKey    string          `yaml:"segment,omitempty"`
+	Rank          uint            `yaml:"rank,omitempty"`
+	Distributions []*Distribution `yaml:"distributions,omitempty"`
+}
+
+type Distribution struct {
+	VariantKey string  `yaml:"variant,omitempty"`
+	Rollout    float32 `yaml:"rollout,omitempty"`
+}
+
+type Segment struct {
+	Key         string        `yaml:"key,omitempty"`
+	Name        string        `yaml:"name,omitempty"`
+	Description string        `yaml:"description,omitempty"`
+	Constraints []*Constraint `yaml:"constraints,omitempty"`
+}
+
+type Constraint struct {
+	Type     string `yaml:"type,omitempty"`
+	Property string `yaml:"property,omitempty"`
+	Operator string `yaml:"operator,omitempty"`
+	Value    string `yaml:"value,omitempty"`
+}
+
+const batchSize = 25
+
+var exportFilename = ""
+
+func runExport(_ []string) error {
+	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
+
+	defer cancel()
+
+	interrupt := make(chan os.Signal, 1)
+	signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM)
+
+	go func() {
+		<-interrupt
+		cancel()
+	}()
+
+	sql, driver, err := db.Open(cfg.Database.URL)
+	if err != nil {
+		return fmt.Errorf("opening db: %w", err)
+	}
+
+	defer sql.Close()
+
+	var (
+		builder    sq.StatementBuilderType
+		stmtCacher = sq.NewStmtCacher(sql)
+	)
+
+	switch driver {
+	case db.SQLite:
+		builder = sq.StatementBuilder.RunWith(stmtCacher)
+	case db.Postgres:
+		builder = sq.StatementBuilder.PlaceholderFormat(sq.Dollar).RunWith(stmtCacher)
+	}
+
+	// default to stdout
+	var out io.WriteCloser = os.Stdout
+
+	// export to file
+	if exportFilename != "" {
+		logger.Debugf("exporting to %q", exportFilename)
+
+		out, err = os.Create(exportFilename)
+		if err != nil {
+			return fmt.Errorf("creating output file: %w", err)
+		}
+
+		fmt.Fprintf(out, "# exported by Flipt (%s) on %s\n\n", version, time.Now().UTC().Format(time.RFC3339))
+	}
+
+	defer out.Close()
+
+	var (
+		flagStore    = db.NewFlagStore(builder)
+		segmentStore = db.NewSegmentStore(builder)
+		ruleStore    = db.NewRuleStore(builder, sql)
+
+		enc = yaml.NewEncoder(out)
+		doc = new(Document)
+	)
+
+	defer enc.Close()
+
+	var remaining = true
+
+	// export flags/variants in batches
+	for batch := uint64(0); remaining; batch++ {
+		flags, err := flagStore.ListFlags(ctx, storage.WithOffset(batch*batchSize), storage.WithLimit(batchSize))
+		if err != nil {
+			return fmt.Errorf("getting flags: %w", err)
+		}
+
+		remaining = len(flags) == batchSize
+
+		for _, f := range flags {
+			flag := &Flag{
+				Key:         f.Key,
+				Name:        f.Name,
+				Description: f.Description,
+				Enabled:     f.Enabled,
+			}
+
+			// map variant id => variant key
+			variantKeys := make(map[string]string)
+
+			for _, v := range f.Variants {
+				flag.Variants = append(flag.Variants, &Variant{
+					Key:         v.Key,
+					Name:        v.Name,
+					Description: v.Description,
+				})
+
+				variantKeys[v.Id] = v.Key
+			}
+
+			// export rules for flag
+			rules, err := ruleStore.ListRules(ctx, flag.Key)
+			if err != nil {
+				return fmt.Errorf("getting rules for flag %q: %w", flag.Key, err)
+			}
+
+			for _, r := range rules {
+				rule := &Rule{
+					SegmentKey: r.SegmentKey,
+					Rank:       uint(r.Rank),
+				}
+
+				for _, d := range r.Distributions {
+					rule.Distributions = append(rule.Distributions, &Distribution{
+						VariantKey: variantKeys[d.VariantId],
+						Rollout:    d.Rollout,
+					})
+				}
+
+				flag.Rules = append(flag.Rules, rule)
+			}
+
+			doc.Flags = append(doc.Flags, flag)
+		}
+	}
+
+	remaining = true
+
+	// export segments/constraints in batches
+	for batch := uint64(0); remaining; batch++ {
+		segments, err := segmentStore.ListSegments(ctx, storage.WithOffset(batch*batchSize), storage.WithLimit(batchSize))
+		if err != nil {
+			return fmt.Errorf("getting segments: %w", err)
+		}
+
+		remaining = len(segments) == batchSize
+
+		for _, s := range segments {
+			segment := &Segment{
+				Key:         s.Key,
+				Name:        s.Name,
+				Description: s.Description,
+			}
+
+			for _, c := range s.Constraints {
+				segment.Constraints = append(segment.Constraints, &Constraint{
+					Type:     c.Type.String(),
+					Property: c.Property,
+					Operator: c.Operator,
+					Value:    c.Value,
+				})
+			}
+
+			doc.Segments = append(doc.Segments, segment)
+		}
+	}
+
+	if err := enc.Encode(doc); err != nil {
+		return fmt.Errorf("exporting: %w", err)
+	}
+
+	return nil
+}