Merge pull request #28 from bank-vaults/feat/update-api

README.md

@@ -1,71 +1,135 @@
 ## Secret Sync
 
-Enables secret synchronization between two secret store services (e.g. between Vault and AWS) in a configurable manner.
+Enables secret synchronization between two secret store services (e.g. between Hashicorp Vault and AWS) in a configurable and explicit manner.
 
 > [!WARNING]  
 > This is an early alpha version and there will be changes made to the API. You can support us with your feedback.
 
 ### Supported secret stores
-- Vault
-- FileDir (regular system directory)
+- Hashicorp Vault
+- FileDir (store is a folder, secrets are plain unencrypted files)
 
-### Quick usage
-Synchronize secrets every hour from Vault-A to Vault-B instance.
+### Examples
 
-#### Define stores and sync job strategy
+<details>
+<summary>Synchronize specific secrets every hour between two Hashicorp Vault instance</summary>
+
+#### Define stores
 ```yaml
 ### Vault-A - Source
 ### SecretStore: path/to/vault-source.yaml
-permissions: Read
-provider:
-  vault:
+vault:
     address: "http://0.0.0.0:8200"
-    unseal-keys-path: "secret"
+    storePath: "secret"
     role: ""
-    auth-path: "userpass"
-    token-path: ""
+    authPath: "userpass"
+    tokenPath: ""
     token: "root"
 ```
 ```yaml
-### Vault-B - Dest
-### SecretStore: path/to/vault-dest.yaml
-permissions: Write
-provider:
-  vault:
+### Vault-B - Target
+### SecretStore: path/to/vault-target.yaml
+vault:
     address: "http://0.0.0.0:8201"
-    unseal-keys-path: "secret"
+    storePath: "secret"
     role: ""
-    auth-path: "userpass"
-    token-path: ""
+    authPath: "userpass"
+    tokenPath: ""
     token: "root"
 ```
+
+#### Define sync strategy
 ```yaml
 ### SyncJob: path/to/sync-job.yaml
 schedule: "@every 1h"
-plan:
-  - secret:
-      key: "a"
-  - secret:
-      key: "b/b"
-  - secret:
-      key: "c/c/c"
-  - query:
-      path: "d/d/d"
+## Defines how the secrets will be synced
+sync:
+  ## 1. Usage: Sync key from ref
+  - secretRef:
+      key: /source/credentials/username
+    target: # If not specified, will be synced under the same key
+      key: /target/example-1
+
+  ## 2. Usage: Sync all keys from query
+  - secretQuery:
+      path: /source/credentials
+      key:
+        regexp: .*
+    target: # If not specified, all keys will be synced under the same path
+      keyPrefix: /target/example-2/
+
+  ## 3. Usage: Sync key from ref with templating
+  - secretRef:
+      key: /source/credentials/password
+    target:
+      key: /target/example-3
+
+    # Template defines how the secret will be synced to target store.
+    # Either "rawData" or "data" should be specified, not both.
+    template:
+      rawData: '{{ .Data }}'   # Save as raw (accepts multiline string)
+      data:                    # Save as map (accepts nested values)
+        example: '{{ .Data }}'
+
+  ## 4. Usage: Sync all keys from query with templating
+  - secretQuery:
+      path: /source/credentials
       key:
-        regexp: ".*"
-    key-transform:
-      - regexp:
-          source: "d/d/d/(.*)"
-          target: "d/d/d/$1-final"
+        regexp: .*
+    target:
+      keyPrefix: /target/example-4/
+    template:
+      rawData: 'SECRET-PREFIX-{{ .Data }}'
+
+  ## 5. Usage: Sync single key from query with templating
+  - secretQuery:
+      path: /source/credentials/query-data/
+      key:
+        regexp: (username|password)
+    flatten: true
+    target:
+      key: /target/example-5
+
+    template:
+      data:
+        user: '{{ .Data.username }}'
+        pass: '{{ .Data.password }}'
+
+  ## 6. Usage: Sync single key from multiple sources with templating
+  - secretSources:
+      - name: username # Username mapping, available as ".Data.username"
+        secretRef:
+          key: /source/credentials/username
+
+      - name: password # Password mapping, available as ".Data.password"
+        secretRef:
+          key: /source/credentials/password
+
+      - name: dynamic_query # Query mapping, available as "Data.dynamic_query.<key>"
+        secretQuery:
+          path: /source/credentials
+          key:
+            regexp: .*
+
+    target:
+      key: /target/example-6
+
+    template:
+      data:
+        username: '{{ .Data.username }}'
+        password: '{{ .Data.password }}'
+        userpass: '{{ .Data.dynamic_query.username }}/{{ .Data.dynamic_query.password }}'
 ```
 
 #### Perform sync
 ```bash
 secret-sync --source path/to/vault-source.yaml \
-            --dest path/to/vault-dest.yaml \
+            --target path/to/vault-target.yaml \
             --sync path/to/sync-job.yaml
 # Use --schedule "@every 1m" to override sync job file config.
 ```
 
+</details>
+
 ### Docs
-Check documentation and example usage at [PROPOSAL](docs/proposal.md).
+Check documentation and example usage at [DOCS](docs/).

cmd/sync.go

@@ -17,7 +17,6 @@ package cmd
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"os"
 	"os/signal"
 
@@ -36,7 +35,7 @@ func NewSyncCmd() *cobra.Command {
 	cmd := &syncCmd{}
 	cobraCmd := &cobra.Command{
 		Use:   "sync",
-		Short: "Synchronizes a key-value destination store from source store",
+		Short: "Synchronizes secrets from a source to a target store based on sync strategy.",
 		RunE: func(_ *cobra.Command, _ []string) error {
 			if err := cmd.init(); err != nil {
 				return err
@@ -46,15 +45,20 @@ func NewSyncCmd() *cobra.Command {
 	}
 
 	// Register cmd flags
-	cobraCmd.Flags().StringVar(&cmd.flgSrcFile, "source", "", "Source store config file")
+	cobraCmd.Flags().StringVar(&cmd.flgSrcFile, "source", "", "Source store config file. "+
+		"This is the store where the data will be fetched from.")
 	_ = cobraCmd.MarkFlagRequired("source")
-	cobraCmd.Flags().StringVar(&cmd.flagDstFile, "dest", "", "Destination store config file")
-	_ = cobraCmd.MarkFlagRequired("dest")
-	cobraCmd.Flags().StringVar(&cmd.flagSyncFile, "sync", "", "Sync job config file")
+	cobraCmd.Flags().StringVar(&cmd.flagDstFile, "target", "", "Target store config file. "+
+		"This is the store where the data will be synced to.")
+	_ = cobraCmd.MarkFlagRequired("target")
+	cobraCmd.Flags().StringVar(&cmd.flagSyncFile, "sync", "", "Sync job config file. "+
+		"This is the strategy sync template.")
 	_ = cobraCmd.MarkFlagRequired("sync")
 
-	cobraCmd.Flags().StringVar(&cmd.flagSchedule, "schedule", v1alpha1.DefaultSyncJobSchedule, "Synchronization CRON schedule. Overrides --sync params")
-	cobraCmd.Flags().BoolVar(&cmd.flagOnce, "once", false, "Synchronize once and exit. Overrides --sync params")
+	cobraCmd.Flags().StringVar(&cmd.flagSchedule, "schedule", v1alpha1.DefaultSyncJobSchedule,
+		"Sync on CRON schedule. Either --schedule or --once should be specified.")
+	cobraCmd.Flags().BoolVar(&cmd.flagOnce, "once", false,
+		"Synchronize once and exit. Either --schedule or --once should be specified.")
 
 	return cobraCmd
 }
@@ -67,8 +71,8 @@ type syncCmd struct {
 	flagOnce     bool
 
 	source v1alpha1.StoreReader
-	dest   v1alpha1.StoreWriter
-	sync   *v1alpha1.SyncJobSpec
+	target v1alpha1.StoreWriter
+	sync   *v1alpha1.SyncJob
 }
 
 func (cmd *syncCmd) init() error {
@@ -79,29 +83,23 @@ func (cmd *syncCmd) init() error {
 	if err != nil {
 		return err
 	}
-	if !srcStore.GetPermissions().CanPerform(v1alpha1.SecretStorePermissionsRead) {
-		return fmt.Errorf("source does not have Read permissions")
-	}
-	cmd.source, err = provider.NewClient(context.Background(), &srcStore.Provider)
+	cmd.source, err = provider.NewClient(context.Background(), srcStore)
 	if err != nil {
 		return err
 	}
 
-	// Init dest
-	destStore, err := loadStore(cmd.flagDstFile)
+	// Init target
+	targetStore, err := loadStore(cmd.flagDstFile)
 	if err != nil {
 		return err
 	}
-	if !destStore.GetPermissions().CanPerform(v1alpha1.SecretStorePermissionsWrite) {
-		return fmt.Errorf("dest does not have Write permissions")
-	}
-	cmd.dest, err = provider.NewClient(context.Background(), &destStore.Provider)
+	cmd.target, err = provider.NewClient(context.Background(), targetStore)
 	if err != nil {
 		return err
 	}
 
 	// Init sync request by loading from file and overriding from cli
-	cmd.sync, err = loadRequest(cmd.flagSyncFile)
+	cmd.sync, err = loadStrategy(cmd.flagSyncFile)
 	if err != nil {
 		return err
 	}
@@ -115,10 +113,10 @@ func (cmd *syncCmd) init() error {
 	return nil
 }
 
-func (cmd *syncCmd) run(syncReq *v1alpha1.SyncJobSpec) error {
+func (cmd *syncCmd) run(syncReq *v1alpha1.SyncJob) error {
 	// Run once
 	if syncReq.RunOnce {
-		resp, err := storesync.Sync(context.Background(), cmd.source, cmd.dest, syncReq.Plan)
+		resp, err := storesync.Sync(context.Background(), cmd.source, cmd.target, syncReq.Sync)
 		if err != nil {
 			return err
 		}
@@ -138,7 +136,7 @@ func (cmd *syncCmd) run(syncReq *v1alpha1.SyncJobSpec) error {
 		select {
 		case <-cronTicker.C:
 			logrus.Info("Handling a new sync request...")
-			resp, err := storesync.Sync(context.Background(), cmd.source, cmd.dest, syncReq.Plan)
+			resp, err := storesync.Sync(context.Background(), cmd.source, cmd.target, syncReq.Sync)
 			if err != nil {
 				return err
 			}
@@ -150,16 +148,15 @@ func (cmd *syncCmd) run(syncReq *v1alpha1.SyncJobSpec) error {
 	}
 }
 
-// loadRequest loads apis.SyncJobSpec data from a YAML file.
-func loadRequest(path string) (*v1alpha1.SyncJobSpec, error) {
+func loadStrategy(path string) (*v1alpha1.SyncJob, error) {
 	// Load file
 	yamlBytes, err := os.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
 
 	// Unmarshal (convert YAML to JSON)
-	var ruleCfg v1alpha1.SyncJobSpec
+	var ruleCfg v1alpha1.SyncJob
 	jsonBytes, err := yaml.YAMLToJSON(yamlBytes)
 	if err != nil {
 		return nil, err
@@ -170,16 +167,15 @@ func loadRequest(path string) (*v1alpha1.SyncJobSpec, error) {
 	return &ruleCfg, nil
 }
 
-// loadStore loads apis.SecretStoreSpec from a YAML file.
-func loadStore(path string) (*v1alpha1.SecretStoreSpec, error) {
+func loadStore(path string) (*v1alpha1.ProviderBackend, error) {
 	// Load file
 	yamlBytes, err := os.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
 
 	// Unmarshal (convert YAML to JSON)
-	var spec v1alpha1.SecretStoreSpec
+	var spec v1alpha1.ProviderBackend
 	jsonBytes, err := yaml.YAMLToJSON(yamlBytes)
 	if err != nil {
 		return nil, err

cmd/sync_test.go

@@ -23,11 +23,13 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+// TODO: Expand tests
+
 func TestSync(t *testing.T) {
 	syncCmd := NewSyncCmd()
 	syncCmd.SetArgs([]string{
-		"--source", storeFile(t, "testdata/source"),
-		"--dest", storeFile(t, filepath.Join(os.TempDir(), "dest")),
+		"--source", storeFile(t, "testdata"),
+		"--target", storeFile(t, filepath.Join(os.TempDir(), "target")),
 		"--sync", "testdata/syncjob.yaml",
 		"--once",
 	})
@@ -49,10 +51,8 @@ func storeFile(t *testing.T, dirPath string) string {
 
 	// Write
 	_, err = tmpFile.Write([]byte(fmt.Sprintf(`
-permissions: ReadWrite
-provider:
-  file:
-    dir-path: %q
+file:
+  dirPath: %q
 `, path)))
 	assert.Nil(t, err)
 

cmd/testdata/source/credentials/password

@@ -0,0 +1 @@
+this-is-password

cmd/testdata/source/credentials/username

@@ -0,0 +1 @@
+this-is-username

cmd/testdata/store-file-dest.yaml

@@ -1,4 +1,2 @@
-permissions: ReadWrite
-provider:
-  file:
-    dir-path: "/tmp/dest"
+file:
+  dirPath: "/tmp/target"

cmd/testdata/store-file-source.yaml

@@ -1,4 +1,2 @@
-permissions: ReadWrite
-provider:
-  file:
-    dir-path: "/tmp/source"
+file:
+  dirPath: "/tmp/source"