Skip to content

Commit

Permalink
feat: resolve formatting and docs
Browse files Browse the repository at this point in the history
Signed-off-by: Ramiz Polic <ramiz.polic@hotmail.com>
  • Loading branch information
ramizpolic committed Sep 14, 2023
1 parent 73f5f3a commit b48b3f7
Show file tree
Hide file tree
Showing 13 changed files with 368 additions and 197 deletions.
135 changes: 99 additions & 36 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,71 +1,134 @@
## Secret Sync

Enables secret synchronization between two secret store services (e.g. between Vault and AWS) in a configurable manner.
Enables secret synchronization between two secret store services (e.g. between Hashicorp Vault and AWS) in a configurable and explicit manner.

> [!WARNING]
> This is an early alpha version and there will be changes made to the API. You can support us with your feedback.
### Supported secret stores
- Vault
- FileDir (regular system directory)
- Hashicorp Vault
- FileDir (store is a folder, secrets are plain unencrypted files)

### Quick usage
Synchronize secrets every hour from Vault-A to Vault-B instance.
### Examples

#### Define stores and sync job strategy
<details>
<summary>Synchronize specific secrets every hour between two Hashicorp Vault instance</summary>

#### Define stores
```yaml
### Vault-A - Source
### SecretStore: path/to/vault-source.yaml
permissions: Read
provider:
vault:
vault:
address: "http://0.0.0.0:8200"
unseal-keys-path: "secret"
storePath: "secret"
role: ""
auth-path: "userpass"
token-path: ""
authPath: "userpass"
tokenPath: ""
token: "root"
```
```yaml
### Vault-B - Dest
### SecretStore: path/to/vault-dest.yaml
permissions: Write
provider:
vault:
### Vault-B - Target
### SecretStore: path/to/vault-target.yaml
vault:
address: "http://0.0.0.0:8201"
unseal-keys-path: "secret"
storePath: "secret"
role: ""
auth-path: "userpass"
token-path: ""
authPath: "userpass"
tokenPath: ""
token: "root"
```
#### Define sync strategy
```yaml
### SyncJob: path/to/sync-job.yaml
schedule: "@every 1h"
plan:
- secret:
key: "a"
- secret:
key: "b/b"
- secret:
key: "c/c/c"
- query:
path: "d/d/d"
## Defines how the secrets will be synced
sync:
## 1. Usage: Sync key from ref
- secretRef:
key: /source/credentials/username
target: # If not specified, will be synced under the same key
key: /target/example-1

## 2. Usage: Sync all keys from query
- secretQuery:
path: /source/credentials
key:
regexp: .*
target: # If not specified, all keys will be synced under the same path
keyPrefix: /target/example-2/

## 3. Usage: Sync key from ref with templating
- secretRef:
key: /source/credentials/password
target:
key: /target/example-3

# Template defines how the secret will be synced to target store.
# Either "rawData" or "data" should be specified, not both.
template:
rawData: '{{ .Data }}' # Save as raw (accepts multiline string)
data: # Save as map (accepts nested values)
example: '{{ .Data }}'

## 4. Usage: Sync all keys from query with templating
- secretQuery:
path: /source/credentials
key:
regexp: .*
target:
keyPrefix: /target/example-4/
template:
rawData: 'SECRET-PREFIX-{{ .Data }}'

## 5. Usage: Sync single key from query with templating
- secretQuery:
path: /source/credentials/query-data/
key:
regexp: ".*"
key-transform:
- regexp:
source: "d/d/d/(.*)"
target: "d/d/d/$1-final"
regexp: (username|password)
target:
key: /target/example-5

template:
data:
user: '{{ .Data.username }}'
pass: '{{ .Data.password }}'

## 6. Usage: Sync single key from multiple sources with templating
- secretSources:
- name: username # Username mapping, available as ".Data.username"
secretRef:
key: /source/credentials/username

- name: password # Password mapping, available as ".Data.password"
secretRef:
key: /source/credentials/password

- name: dynamic_query # Query mapping, available as "Data.dynamic_query.<key>"
secretQuery:
path: /source/credentials
key:
regexp: .*

target:
key: /target/example-6

template:
data:
username: '{{ .Data.username }}'
password: '{{ .Data.password }}'
userpass: '{{ .Data.dynamic_query.username }}/{{ .Data.dynamic_query.password }}'
```
#### Perform sync
```bash
secret-sync --source path/to/vault-source.yaml \
--dest path/to/vault-dest.yaml \
--target path/to/vault-target.yaml \
--sync path/to/sync-job.yaml
# Use --schedule "@every 1m" to override sync job file config.
```

</details>

### Docs
Check documentation and example usage at [PROPOSAL](docs/proposal.md).
Check documentation and example usage at [DOCS](docs/).
32 changes: 16 additions & 16 deletions cmd/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,27 +51,27 @@ func NewSyncCmd() *cobra.Command {
cobraCmd.Flags().StringVar(&cmd.flagDstFile, "target", "", "Target store config file. "+
"This is the store where the data will be synced to.")
_ = cobraCmd.MarkFlagRequired("target")
cobraCmd.Flags().StringVar(&cmd.flagPlanFile, "plan", "", "Sync job config file. "+
cobraCmd.Flags().StringVar(&cmd.flagSyncFile, "sync", "", "Sync job config file. "+
"This is the strategy sync template.")
_ = cobraCmd.MarkFlagRequired("plan")
_ = cobraCmd.MarkFlagRequired("sync")

cobraCmd.Flags().StringVar(&cmd.flagSync, "sync", v1alpha1.DefaultSyncJobSchedule,
"Synchronization CRON schedule. Either --sync or --once should be specified.")
cobraCmd.Flags().StringVar(&cmd.flagSchedule, "schedule", v1alpha1.DefaultSyncJobSchedule,
"Sync on CRON schedule. Either --schedule or --once should be specified.")
cobraCmd.Flags().BoolVar(&cmd.flagOnce, "once", false,
"Synchronize once and exit. Either --sync or --once should be specified.")
"Synchronize once and exit. Either --schedule or --once should be specified.")

return cobraCmd
}

type syncCmd struct {
flgSrcFile string
flagDstFile string
flagPlanFile string
flagSync string
flagSyncFile string
flagSchedule string
flagOnce bool

source v1alpha1.StoreReader
dest v1alpha1.StoreWriter
target v1alpha1.StoreWriter
sync *v1alpha1.SyncJob
}

Expand All @@ -88,26 +88,26 @@ func (cmd *syncCmd) init() error {
return err
}

// Init dest
destStore, err := loadStore(cmd.flagDstFile)
// Init target
targetStore, err := loadStore(cmd.flagDstFile)
if err != nil {
return err
}
cmd.dest, err = provider.NewClient(context.Background(), destStore)
cmd.target, err = provider.NewClient(context.Background(), targetStore)
if err != nil {
return err
}

// Init sync request by loading from file and overriding from cli
cmd.sync, err = loadStrategy(cmd.flagPlanFile)
cmd.sync, err = loadStrategy(cmd.flagSyncFile)
if err != nil {
return err
}
if cmd.flagOnce {
cmd.sync.RunOnce = cmd.flagOnce
}
if cmd.flagSync != "" {
cmd.sync.Schedule = cmd.flagSync
if cmd.flagSchedule != "" {
cmd.sync.Schedule = cmd.flagSchedule
}

return nil
Expand All @@ -116,7 +116,7 @@ func (cmd *syncCmd) init() error {
func (cmd *syncCmd) run(syncReq *v1alpha1.SyncJob) error {
// Run once
if syncReq.RunOnce {
resp, err := storesync.Sync(context.Background(), cmd.source, cmd.dest, syncReq.Sync)
resp, err := storesync.Sync(context.Background(), cmd.source, cmd.target, syncReq.Sync)
if err != nil {
return err
}
Expand All @@ -136,7 +136,7 @@ func (cmd *syncCmd) run(syncReq *v1alpha1.SyncJob) error {
select {
case <-cronTicker.C:
logrus.Info("Handling a new sync request...")
resp, err := storesync.Sync(context.Background(), cmd.source, cmd.dest, syncReq.Sync)
resp, err := storesync.Sync(context.Background(), cmd.source, cmd.target, syncReq.Sync)
if err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/sync_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ func TestSync(t *testing.T) {
syncCmd := NewSyncCmd()
syncCmd.SetArgs([]string{
"--source", storeFile(t, "testdata"),
"--target", storeFile(t, filepath.Join(os.TempDir(), "dest")),
"--plan", "testdata/syncjob.yaml",
"--target", storeFile(t, filepath.Join(os.TempDir(), "target")),
"--sync", "testdata/syncjob.yaml",
"--once",
})
err := syncCmd.Execute()
Expand Down
1 change: 0 additions & 1 deletion cmd/testdata/source/credentials/example

This file was deleted.

2 changes: 1 addition & 1 deletion cmd/testdata/store-file-dest.yaml
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
file:
dirPath: "/tmp/dest"
dirPath: "/tmp/target"
2 changes: 1 addition & 1 deletion cmd/testdata/store-vault.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
vault:
address: http://0.0.0.0:8200
unsealKeysPath: secret
storePath: secret
role: ''
authPath: userpass
tokenPath: ''
Expand Down
Loading

0 comments on commit b48b3f7

Please sign in to comment.