From b48b3f76b7b7567e9a3874be1832763ec0b0bd8e Mon Sep 17 00:00:00 2001 From: Ramiz Polic Date: Thu, 14 Sep 2023 15:28:08 +0200 Subject: [PATCH] feat: resolve formatting and docs Signed-off-by: Ramiz Polic --- README.md | 135 ++++++++---- cmd/sync.go | 32 +-- cmd/sync_test.go | 4 +- cmd/testdata/source/credentials/example | 1 - cmd/testdata/store-file-dest.yaml | 2 +- cmd/testdata/store-vault.yaml | 2 +- cmd/testdata/syncjob.yaml | 239 ++++++++++++++-------- pkg/apis/v1alpha1/provider_vault_types.go | 12 +- pkg/provider/vault/client.go | 2 +- pkg/provider/vault/provider.go | 6 +- pkg/storesync/processor.go | 17 +- pkg/storesync/storesync.go | 111 +++++----- pkg/storesync/storesync_test.go | 2 +- 13 files changed, 368 insertions(+), 197 deletions(-) delete mode 100644 cmd/testdata/source/credentials/example diff --git a/README.md b/README.md index c5da104..28f67da 100644 --- a/README.md +++ b/README.md @@ -1,71 +1,134 @@ ## Secret Sync -Enables secret synchronization between two secret store services (e.g. between Vault and AWS) in a configurable manner. +Enables secret synchronization between two secret store services (e.g. between Hashicorp Vault and AWS) in a configurable and explicit manner. > [!WARNING] > This is an early alpha version and there will be changes made to the API. You can support us with your feedback. ### Supported secret stores -- Vault -- FileDir (regular system directory) +- Hashicorp Vault +- FileDir (store is a folder, secrets are plain unencrypted files) -### Quick usage -Synchronize secrets every hour from Vault-A to Vault-B instance. +### Examples -#### Define stores and sync job strategy +
+Synchronize specific secrets every hour between two Hashicorp Vault instance + +#### Define stores ```yaml ### Vault-A - Source ### SecretStore: path/to/vault-source.yaml -permissions: Read -provider: - vault: +vault: address: "http://0.0.0.0:8200" - unseal-keys-path: "secret" + storePath: "secret" role: "" - auth-path: "userpass" - token-path: "" + authPath: "userpass" + tokenPath: "" token: "root" ``` ```yaml -### Vault-B - Dest -### SecretStore: path/to/vault-dest.yaml -permissions: Write -provider: - vault: +### Vault-B - Target +### SecretStore: path/to/vault-target.yaml +vault: address: "http://0.0.0.0:8201" - unseal-keys-path: "secret" + storePath: "secret" role: "" - auth-path: "userpass" - token-path: "" + authPath: "userpass" + tokenPath: "" token: "root" ``` + +#### Define sync strategy ```yaml ### SyncJob: path/to/sync-job.yaml schedule: "@every 1h" -plan: - - secret: - key: "a" - - secret: - key: "b/b" - - secret: - key: "c/c/c" - - query: - path: "d/d/d" +## Defines how the secrets will be synced +sync: + ## 1. Usage: Sync key from ref + - secretRef: + key: /source/credentials/username + target: # If not specified, will be synced under the same key + key: /target/example-1 + + ## 2. Usage: Sync all keys from query + - secretQuery: + path: /source/credentials + key: + regexp: .* + target: # If not specified, all keys will be synced under the same path + keyPrefix: /target/example-2/ + + ## 3. Usage: Sync key from ref with templating + - secretRef: + key: /source/credentials/password + target: + key: /target/example-3 + + # Template defines how the secret will be synced to target store. + # Either "rawData" or "data" should be specified, not both. + template: + rawData: '{{ .Data }}' # Save as raw (accepts multiline string) + data: # Save as map (accepts nested values) + example: '{{ .Data }}' + + ## 4. Usage: Sync all keys from query with templating + - secretQuery: + path: /source/credentials + key: + regexp: .* + target: + keyPrefix: /target/example-4/ + template: + rawData: 'SECRET-PREFIX-{{ .Data }}' + + ## 5. Usage: Sync single key from query with templating + - secretQuery: + path: /source/credentials/query-data/ key: - regexp: ".*" - key-transform: - - regexp: - source: "d/d/d/(.*)" - target: "d/d/d/$1-final" + regexp: (username|password) + target: + key: /target/example-5 + + template: + data: + user: '{{ .Data.username }}' + pass: '{{ .Data.password }}' + + ## 6. Usage: Sync single key from multiple sources with templating + - secretSources: + - name: username # Username mapping, available as ".Data.username" + secretRef: + key: /source/credentials/username + + - name: password # Password mapping, available as ".Data.password" + secretRef: + key: /source/credentials/password + + - name: dynamic_query # Query mapping, available as "Data.dynamic_query." + secretQuery: + path: /source/credentials + key: + regexp: .* + + target: + key: /target/example-6 + + template: + data: + username: '{{ .Data.username }}' + password: '{{ .Data.password }}' + userpass: '{{ .Data.dynamic_query.username }}/{{ .Data.dynamic_query.password }}' ``` #### Perform sync ```bash secret-sync --source path/to/vault-source.yaml \ - --dest path/to/vault-dest.yaml \ + --target path/to/vault-target.yaml \ --sync path/to/sync-job.yaml # Use --schedule "@every 1m" to override sync job file config. ``` +
+ ### Docs -Check documentation and example usage at [PROPOSAL](docs/proposal.md). +Check documentation and example usage at [DOCS](docs/). diff --git a/cmd/sync.go b/cmd/sync.go index b17a28d..4ea977e 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -51,14 +51,14 @@ func NewSyncCmd() *cobra.Command { cobraCmd.Flags().StringVar(&cmd.flagDstFile, "target", "", "Target store config file. "+ "This is the store where the data will be synced to.") _ = cobraCmd.MarkFlagRequired("target") - cobraCmd.Flags().StringVar(&cmd.flagPlanFile, "plan", "", "Sync job config file. "+ + cobraCmd.Flags().StringVar(&cmd.flagSyncFile, "sync", "", "Sync job config file. "+ "This is the strategy sync template.") - _ = cobraCmd.MarkFlagRequired("plan") + _ = cobraCmd.MarkFlagRequired("sync") - cobraCmd.Flags().StringVar(&cmd.flagSync, "sync", v1alpha1.DefaultSyncJobSchedule, - "Synchronization CRON schedule. Either --sync or --once should be specified.") + cobraCmd.Flags().StringVar(&cmd.flagSchedule, "schedule", v1alpha1.DefaultSyncJobSchedule, + "Sync on CRON schedule. Either --schedule or --once should be specified.") cobraCmd.Flags().BoolVar(&cmd.flagOnce, "once", false, - "Synchronize once and exit. Either --sync or --once should be specified.") + "Synchronize once and exit. Either --schedule or --once should be specified.") return cobraCmd } @@ -66,12 +66,12 @@ func NewSyncCmd() *cobra.Command { type syncCmd struct { flgSrcFile string flagDstFile string - flagPlanFile string - flagSync string + flagSyncFile string + flagSchedule string flagOnce bool source v1alpha1.StoreReader - dest v1alpha1.StoreWriter + target v1alpha1.StoreWriter sync *v1alpha1.SyncJob } @@ -88,26 +88,26 @@ func (cmd *syncCmd) init() error { return err } - // Init dest - destStore, err := loadStore(cmd.flagDstFile) + // Init target + targetStore, err := loadStore(cmd.flagDstFile) if err != nil { return err } - cmd.dest, err = provider.NewClient(context.Background(), destStore) + cmd.target, err = provider.NewClient(context.Background(), targetStore) if err != nil { return err } // Init sync request by loading from file and overriding from cli - cmd.sync, err = loadStrategy(cmd.flagPlanFile) + cmd.sync, err = loadStrategy(cmd.flagSyncFile) if err != nil { return err } if cmd.flagOnce { cmd.sync.RunOnce = cmd.flagOnce } - if cmd.flagSync != "" { - cmd.sync.Schedule = cmd.flagSync + if cmd.flagSchedule != "" { + cmd.sync.Schedule = cmd.flagSchedule } return nil @@ -116,7 +116,7 @@ func (cmd *syncCmd) init() error { func (cmd *syncCmd) run(syncReq *v1alpha1.SyncJob) error { // Run once if syncReq.RunOnce { - resp, err := storesync.Sync(context.Background(), cmd.source, cmd.dest, syncReq.Sync) + resp, err := storesync.Sync(context.Background(), cmd.source, cmd.target, syncReq.Sync) if err != nil { return err } @@ -136,7 +136,7 @@ func (cmd *syncCmd) run(syncReq *v1alpha1.SyncJob) error { select { case <-cronTicker.C: logrus.Info("Handling a new sync request...") - resp, err := storesync.Sync(context.Background(), cmd.source, cmd.dest, syncReq.Sync) + resp, err := storesync.Sync(context.Background(), cmd.source, cmd.target, syncReq.Sync) if err != nil { return err } diff --git a/cmd/sync_test.go b/cmd/sync_test.go index b021f10..d791d2e 100644 --- a/cmd/sync_test.go +++ b/cmd/sync_test.go @@ -29,8 +29,8 @@ func TestSync(t *testing.T) { syncCmd := NewSyncCmd() syncCmd.SetArgs([]string{ "--source", storeFile(t, "testdata"), - "--target", storeFile(t, filepath.Join(os.TempDir(), "dest")), - "--plan", "testdata/syncjob.yaml", + "--target", storeFile(t, filepath.Join(os.TempDir(), "target")), + "--sync", "testdata/syncjob.yaml", "--once", }) err := syncCmd.Execute() diff --git a/cmd/testdata/source/credentials/example b/cmd/testdata/source/credentials/example deleted file mode 100644 index 6320cd2..0000000 --- a/cmd/testdata/source/credentials/example +++ /dev/null @@ -1 +0,0 @@ -data \ No newline at end of file diff --git a/cmd/testdata/store-file-dest.yaml b/cmd/testdata/store-file-dest.yaml index d4e09ae..78978bc 100644 --- a/cmd/testdata/store-file-dest.yaml +++ b/cmd/testdata/store-file-dest.yaml @@ -1,2 +1,2 @@ file: - dirPath: "/tmp/dest" + dirPath: "/tmp/target" diff --git a/cmd/testdata/store-vault.yaml b/cmd/testdata/store-vault.yaml index f62ff6c..1ceb310 100644 --- a/cmd/testdata/store-vault.yaml +++ b/cmd/testdata/store-vault.yaml @@ -1,6 +1,6 @@ vault: address: http://0.0.0.0:8200 - unsealKeysPath: secret + storePath: secret role: '' authPath: userpass tokenPath: '' diff --git a/cmd/testdata/syncjob.yaml b/cmd/testdata/syncjob.yaml index 0922ddd..16bc78d 100644 --- a/cmd/testdata/syncjob.yaml +++ b/cmd/testdata/syncjob.yaml @@ -11,115 +11,194 @@ auditLogPath: "path/to/file" ## Defines how the secrets will be synced sync: - ## 1. Usage: Sync key from ref (all possibilities) + ## 1. Usage: Sync key from ref - secretRef: - key: /source/credentials/example - - - secretRef: - key: /source/credentials/example - template: - rawData: "pre-{{ .Data }}-post" - - - secretRef: - key: /source/credentials/example - template: - data: - auth: "example/pre-{{ .Data }}-post" - - - secretRef: - key: /source/credentials/example - target: - key: /target/credentials/example - - - secretRef: - key: /source/credentials/example - target: - key: /target/credentials/example - template: - rawData: "pre-{{ .Data }}-post" - - - secretRef: - key: /source/credentials/example - target: - key: /target/credentials/example - template: - data: - auth: "example/pre-{{ .Data }}-post" - - ## 2. Usage: Sync all keys from query individually (all possibilities) - - secretQuery: - path: /source/credentials - key: - regexp: (username|password) - - - secretQuery: - path: /source/credentials - key: - regexp: (username|password) - template: - auth: "example/pre-{{ .Data }}-post" + key: /source/credentials/username + target: # If not specified, will be synced under the same key + key: /target/example-1 + ## 2. Usage: Sync all keys from query - secretQuery: path: /source/credentials key: - regexp: (username|password) - template: - data: - newKey: "pre-{{ .Data }}-post" + regexp: .* + target: # If not specified, all keys will be synced under the same path + keyPrefix: /target/example-2/ - - secretQuery: - path: /source/credentials - key: - regexp: (username|password) + ## 3. Usage: Sync key from ref with templating + - secretRef: + key: /source/credentials/password target: - keyPrefix: /target/credentials/new/ + key: /target/example-3 - - secretQuery: - path: /source/credentials - key: - regexp: (username|password) - target: - keyPrefix: /target/credentials/new/ + # Template defines how the secret will be synced to target store. + # Either "rawData" or "data" should be specified, not both. template: - auth: "example/pre-{{ .Data }}-post" + rawData: '{{ .Data }}' # Save as raw (accepts multiline string) + data: # Save as map (accepts nested values) + example: '{{ .Data }}' + ## 4. Usage: Sync all keys from query with templating - secretQuery: path: /source/credentials key: - regexp: (username|password) + regexp: .* target: - keyPrefix: /target/credentials/new/ + keyPrefix: /target/example-4/ template: - data: - newKey: "pre-{{ .Data }}-post" + rawData: 'SECRET-PREFIX-{{ .Data }}' - ## 3. Usage: Sync all keys from query into one key (all possibilities) + ## 5. Usage: Sync single key from query with templating - secretQuery: - path: /source/credentials + path: /source/credentials/query-data/ key: regexp: (username|password) target: - key: /target/credentials/key-from-query + key: /target/example-5 + template: data: - username: '{{ .Data.username }}' - password: '{{ .Data.password }}' + user: '{{ .Data.username }}' + pass: '{{ .Data.password }}' - ## 4. Usage: Sync all keys from different sources into one key (all possibilities) + ## 6. Usage: Sync single key from multiple sources with templating - secretSources: - - name: example + - name: username # Username mapping, available as ".Data.username" + secretRef: + key: /source/credentials/username + + - name: password # Password mapping, available as ".Data.password" secretRef: - key: /source/credentials/example + key: /source/credentials/password - - name: credentials + - name: dynamic_query # Query mapping, available as "Data.dynamic_query." secretQuery: path: /source/credentials key: - regexp: (username|password) + regexp: .* + target: - key: /target/credentials/key-from-sources + key: /target/example-6 + template: data: - example: '{{ .Data.example }}' - username: '{{ .Data.credentials.username }}' - password: '{{ .Data.credentials.password }}' + username: '{{ .Data.username }}' + password: '{{ .Data.password }}' + userpass: '{{ .Data.dynamic_query.username }}/{{ .Data.dynamic_query.password }}' + + +## TODO: Move these items to tests +## Defines how the secrets will be synced +# sync: +# ## 1. Usage: Sync key from ref (all possibilities) +# - secretRef: +# key: /source/credentials/example +# +# - secretRef: +# key: /source/credentials/example +# template: +# rawData: "pre-{{ .Data }}-post" +# +# - secretRef: +# key: /source/credentials/example +# template: +# data: +# auth: "example/pre-{{ .Data }}-post" +# +# - secretRef: +# key: /source/credentials/example +# target: +# key: /target/credentials/example +# +# - secretRef: +# key: /source/credentials/example +# target: +# key: /target/credentials/example +# template: +# rawData: "pre-{{ .Data }}-post" +# +# - secretRef: +# key: /source/credentials/example +# target: +# key: /target/credentials/example +# template: +# data: +# auth: "example/pre-{{ .Data }}-post" +# +# ## 2. Usage: Sync all keys from query individually (all possibilities) +# - secretQuery: +# path: /source/credentials +# key: +# regexp: (username|password) +# +# - secretQuery: +# path: /source/credentials +# key: +# regexp: (username|password) +# template: +# auth: "example/pre-{{ .Data }}-post" +# +# - secretQuery: +# path: /source/credentials +# key: +# regexp: (username|password) +# template: +# data: +# newKey: "pre-{{ .Data }}-post" +# +# - secretQuery: +# path: /source/credentials +# key: +# regexp: (username|password) +# target: +# keyPrefix: /target/credentials/new/ +# +# - secretQuery: +# path: /source/credentials +# key: +# regexp: (username|password) +# target: +# keyPrefix: /target/credentials/new/ +# template: +# auth: "example/pre-{{ .Data }}-post" +# +# - secretQuery: +# path: /source/credentials +# key: +# regexp: (username|password) +# target: +# keyPrefix: /target/credentials/new/ +# template: +# data: +# newKey: "pre-{{ .Data }}-post" +# +# ## 3. Usage: Sync all keys from query into one key (all possibilities) +# - secretQuery: +# path: /source/credentials +# key: +# regexp: (username|password) +# target: +# key: /target/credentials/key-from-query +# template: +# data: +# username: '{{ .Data.username }}' +# password: '{{ .Data.password }}' +# +# ## 4. Usage: Sync all keys from different sources into one key (all possibilities) +# - secretSources: +# - name: example +# secretRef: +# key: /source/credentials/example +# +# - name: credentials +# secretQuery: +# path: /source/credentials +# key: +# regexp: (username|password) +# target: +# key: /target/credentials/key-from-sources +# template: +# data: +# example: '{{ .Data.example }}' +# username: '{{ .Data.credentials.username }}' +# password: '{{ .Data.credentials.password }}' diff --git a/pkg/apis/v1alpha1/provider_vault_types.go b/pkg/apis/v1alpha1/provider_vault_types.go index e0a7227..0b383ee 100644 --- a/pkg/apis/v1alpha1/provider_vault_types.go +++ b/pkg/apis/v1alpha1/provider_vault_types.go @@ -16,10 +16,10 @@ package v1alpha1 // VaultProvider uses Hashicorp Vault as a backend. type VaultProvider struct { - Address string `json:"address"` - UnsealKeysPath string `json:"unsealKeysPath"` - Role string `json:"role"` - AuthPath string `json:"authPath"` - TokenPath string `json:"tokenPath"` - Token string `json:"token"` + Address string `json:"address"` + StorePath string `json:"storePath"` + Role string `json:"role"` + AuthPath string `json:"authPath"` + TokenPath string `json:"tokenPath"` + Token string `json:"token"` } diff --git a/pkg/provider/vault/client.go b/pkg/provider/vault/client.go index b94c0ae..d22848c 100644 --- a/pkg/provider/vault/client.go +++ b/pkg/provider/vault/client.go @@ -133,7 +133,7 @@ func (c *client) SetSecret(_ context.Context, key v1alpha1.SecretRef, value []by // Not used since it has high memory footprint and does not handle search. // It could (potentially) be useful. // DEPRECATED -// nolint +//nolint func (c *client) recursiveList(ctx context.Context, path string) ([]v1alpha1.SecretRef, error) { // List API request response, err := c.apiClient.RawClient().Logical().List(fmt.Sprintf("%s/metadata/%s", c.apiKeyPath, path)) diff --git a/pkg/provider/vault/provider.go b/pkg/provider/vault/provider.go index 46115e4..e57d514 100644 --- a/pkg/provider/vault/provider.go +++ b/pkg/provider/vault/provider.go @@ -39,7 +39,7 @@ func (p *Provider) NewClient(_ context.Context, backend v1alpha1.ProviderBackend return &client{ apiClient: apiClient, - apiKeyPath: vaultCfg.UnsealKeysPath, + apiKeyPath: vaultCfg.StorePath, }, nil } @@ -51,8 +51,8 @@ func (p *Provider) Validate(backend v1alpha1.ProviderBackend) error { if vaultCfg.Address == "" { return fmt.Errorf("empty .Vault.Address") } - if vaultCfg.UnsealKeysPath == "" { - return fmt.Errorf("empty .Vault.UnsealKeysPath") + if vaultCfg.StorePath == "" { + return fmt.Errorf("empty .Vault.StorePath") } if vaultCfg.AuthPath == "" { return fmt.Errorf("empty .Vault.AuthPath") diff --git a/pkg/storesync/processor.go b/pkg/storesync/processor.go index 005322f..3495566 100644 --- a/pkg/storesync/processor.go +++ b/pkg/storesync/processor.go @@ -1,3 +1,17 @@ +// Copyright © 2023 Cisco +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package storesync import ( @@ -5,10 +19,11 @@ import ( "context" "encoding/json" "fmt" - "golang.org/x/sync/errgroup" "sync" "text/template" + "golang.org/x/sync/errgroup" + "github.com/bank-vaults/secret-sync/pkg/apis/v1alpha1" ) diff --git a/pkg/storesync/storesync.go b/pkg/storesync/storesync.go index 173810f..4d11462 100644 --- a/pkg/storesync/storesync.go +++ b/pkg/storesync/storesync.go @@ -24,6 +24,7 @@ import ( "time" "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" "github.com/bank-vaults/secret-sync/pkg/apis/v1alpha1" ) @@ -37,91 +38,105 @@ type Status struct { SyncedAt time.Time // completion timestamp } -// Sync will synchronize keys from source to dest based on provided specs. +// Sync will synchronize keys from source to target based on provided specs. func Sync(ctx context.Context, source v1alpha1.StoreReader, - dest v1alpha1.StoreWriter, + target v1alpha1.StoreWriter, requests []v1alpha1.SyncRequest, ) (*Status, error) { // Validate if source == nil { return nil, fmt.Errorf("source is nil") } - if dest == nil { - return nil, fmt.Errorf("dest is nil") + if target == nil { + return nil, fmt.Errorf("target is nil") } if len(requests) == 0 { return nil, fmt.Errorf("nothing to sync") } - // Get sync data for each sync request and . - // Do each fetch in a separate goroutine (there could be API requests). + // Define data stores syncMu := sync.Mutex{} syncPlan := make(map[v1alpha1.SecretRef]SyncPlan) processor := newProcessor(source) - { - extractWg := sync.WaitGroup{} - for id, req := range requests { - extractWg.Add(1) - go func(id int, req v1alpha1.SyncRequest) { - defer extractWg.Done() + // Get sync plan for each request in a separate goroutine. + // If the same secret needs to be synced more than once, abort sync. + fetchGroup, fetchCtx := errgroup.WithContext(ctx) + for id, req := range requests { + func(id int, req v1alpha1.SyncRequest) { + fetchGroup.Go(func() error { // Fetch keys to store - plans, err := processor.GetSyncPlan(ctx, id, req) + plans, err := processor.GetSyncPlan(fetchCtx, id, req) if err != nil { logrus.WithField("z-req", req).Warnf("Failed to fetch reqID = %d sync plan, reason: %v", id, err) - return + return nil } // Add to sync data syncMu.Lock() + defer syncMu.Unlock() for ref, plan := range plans { + if _, exists := syncPlan[ref]; exists { + // This is a critical error; stop everything + return fmt.Errorf("key %v was schedule for sync more than once", ref) + } syncPlan[ref] = plan } - syncMu.Unlock() - }(id, req) - } - extractWg.Wait() + return nil + }) + }(id, req) } - // Sync keys between source and dest read from sync queue. - // Do sync for each key in a separate goroutine (there will be API requests). - var successCounter atomic.Uint32 - { - syncWg := sync.WaitGroup{} - for ref, plan := range syncPlan { - syncWg.Add(1) - go func(ref v1alpha1.SecretRef, plan SyncPlan) { - defer syncWg.Done() - - err := dest.SetSecret(ctx, ref, plan.Data) - if err != nil { - if err == v1alpha1.ErrKeyNotFound { // not found, soft warn - logrus.WithField("z-req", plan.Request). - Warnf("Skipped syncing reqID = %d for key %s, reason: %v", plan.RequestID, ref.Key, err) - } else { // otherwise, log error - logrus.WithField("z-req", plan.Request). - Errorf("Failed to sync reqID = %d for key %s, reason: %v", plan.RequestID, ref.Key, err) - } - return + // Wait fetch + if err := fetchGroup.Wait(); err != nil { + return nil, fmt.Errorf("aborted syncing, reason: %w", err) + } + + // Sync requests from source to target store. + // Do sync for each plan item in a separate goroutine. + var syncWg sync.WaitGroup + var syncCounter atomic.Uint32 + for ref, plan := range syncPlan { + syncWg.Add(1) + go func(ref v1alpha1.SecretRef, plan SyncPlan) { + defer syncWg.Done() + + // Sync + var err error + if len(plan.Data) == 0 { + err = fmt.Errorf("empty value") + } else { + err = target.SetSecret(ctx, ref, plan.Data) + } + + // Handle response + if err != nil { + if err == v1alpha1.ErrKeyNotFound { // not found, soft warn + logrus.WithField("z-req", plan.Request). + Warnf("Skipped syncing reqID = %d for key %s, reason: %v", plan.RequestID, ref.Key, err) + } else { // otherwise, log error + logrus.WithField("z-req", plan.Request). + Errorf("Failed to sync reqID = %d for key %s, reason: %v", plan.RequestID, ref.Key, err) } + return + } - logrus.WithField("z-req", plan.Request). - Infof("Successfully synced reqID = %d for key %s", plan.RequestID, ref.Key /* , string(plan.Data) */) - successCounter.Add(1) - }(ref, plan) - } - syncWg.Wait() + logrus.WithField("z-req", plan.Request). + Infof("Successfully synced reqID = %d for key %s", plan.RequestID, ref.Key /* , string(plan.Data) */) + syncCounter.Add(1) + }(ref, plan) } + syncWg.Wait() // Return response + syncCount := syncCounter.Load() totalCount := uint32(len(syncPlan)) - successCount := successCounter.Load() return &Status{ Total: totalCount, - Synced: successCount, - Success: totalCount == successCount, - Status: fmt.Sprintf("Synced %d out of total %d keys", successCount, totalCount), + Synced: syncCount, + Success: totalCount == syncCount, + Status: fmt.Sprintf("Synced %d out of total %d keys", syncCount, totalCount), SyncedAt: time.Now(), }, nil } diff --git a/pkg/storesync/storesync_test.go b/pkg/storesync/storesync_test.go index 566284f..a0d3882 100644 --- a/pkg/storesync/storesync_test.go +++ b/pkg/storesync/storesync_test.go @@ -106,7 +106,7 @@ package storesync_test // client, err := provider.NewClient(context.Background(), &v1alpha1.ProviderBackend{ // Vault: &v1alpha1.VaultProvider{ // Address: addr, -// UnsealKeysPath: "secret", +// storePath: "secret", // AuthPath: "userpass", // Token: token, // },