This PR adds feature to export secrets in natural format

main.go

@@ -222,14 +222,16 @@ type Options struct {
 		All     bool `cli:"-a, --all"`
 		Deleted bool `cli:"-d, --deleted"`
 		//These do nothing but are kept for backwards-compat
-		OnlyAlive bool `cli:"-o, --only-alive"`
-		Shallow   bool `cli:"-s, --shallow"`
+		OnlyAlive          bool `cli:"-o, --only-alive"`
+		Shallow            bool `cli:"-s, --shallow"`
+		NoStringConversion bool `cli:"-n, --no-string-conversion"`
 	} `cli:"export"`
 
 	Import struct {
 		IgnoreDestroyed bool `cli:"-I, --ignore-destroyed"`
 		IgnoreDeleted   bool `cli:"-i, --ignore-deleted"`
 		Shallow         bool `cli:"-s, --shallow"`
+		AppendOnly      bool `cli:"-a, --appendonly"`
 	} `cli:"import"`
 
 	Move struct {
@@ -2378,7 +2380,7 @@ redeleting them.
 
 	r.Dispatch("export", &Help{
 		Summary: "Export one or more subtrees for migration / backup purposes",
-		Usage:   "safe export [-ad] PATH [PATH ...]",
+		Usage:   "safe export [-adn] PATH [PATH ...]",
 		Type:    NonDestructiveCommand,
 		Description: `
 Normally, the export will get only the latest version of each secret, and encode it in a format that is backwards-
@@ -2387,6 +2389,7 @@ compatible with pre-1.0.0 versions of safe (and newer versions).
 incompatible with versions of safe prior to v1.0.0
 -d (--deleted) will cause safe to undelete, read, and then redelete deleted secrets in order to encode them in the
 backup. Without this, deleted versions will be ignored.
+-n (--no-string-conversion) will use v3 export and does not convert values to strings. This is incompatible with v1/v2
 `}, func(command string, args ...string) error {
 		rc.Apply(opt.UseTarget)
 		if len(args) < 1 {
@@ -2431,6 +2434,7 @@ backup. Without this, deleted versions will be ignored.
 				FetchAllVersions:    opt.Export.All,
 				GetDeletedVersions:  opt.Export.Deleted,
 				AllowDeletedSecrets: opt.Export.Deleted,
+				AsStrings:           !opt.Export.NoStringConversion,
 			})
 			if err != nil {
 				return err
@@ -2440,6 +2444,7 @@ backup. Without this, deleted versions will be ignored.
 		}
 
 		var mustV2Export bool
+		mustV2Export = opt.Export.All || opt.Export.NoStringConversion
 		//Determine if we can get away with a v1 export
 		for _, s := range secrets {
 			if len(s.Versions) > 1 {
@@ -2459,7 +2464,12 @@ backup. Without this, deleted versions will be ignored.
 		}
 
 		v2Export := func() error {
-			export := exportFormat{ExportVersion: 2, Data: map[string]exportSecret{}, RequiresVersioning: map[string]bool{}}
+
+			exportVersionNum := uint(2)
+			if opt.Export.NoStringConversion {
+				exportVersionNum = 3
+			}
+			export := exportFormat{ExportVersion: exportVersionNum, Data: map[string]exportSecret{}, RequiresVersioning: map[string]bool{}}
 
 			for _, secret := range secrets {
 				if len(secret.Versions) > 1 {
@@ -2477,11 +2487,11 @@ backup. Without this, deleted versions will be ignored.
 					thisVersion := exportVersion{
 						Deleted:   version.State == vault.SecretStateDeleted && opt.Export.Deleted,
 						Destroyed: version.State == vault.SecretStateDestroyed || (version.State == vault.SecretStateDeleted && !opt.Export.Deleted),
-						Value:     map[string]string{},
+						Value:     map[string]interface{}{},
 					}
 
 					for _, key := range version.Data.Keys() {
-						thisVersion.Value[key] = version.Data.Get(key)
+						thisVersion.Value[key] = version.Data.GetAsInterface(key)
 					}
 
 					thisSecret.Versions = append(thisSecret.Versions, thisVersion)
@@ -2524,6 +2534,7 @@ backup. Without this, deleted versions will be ignored.
 rting garbage data and then destroying it (which is originally done to preserve version numbering).
 -i (--ignore-deleted) will ignore deleted versions from being written during the import.
 -s (--shallow) will write only the latest version for each secret.
+-a (--appendonly) will only write latest alive version if exists.
 `}, func(command string, args ...string) error {
 		rc.Apply(opt.UseTarget)
 		b, err := ioutil.ReadAll(os.Stdin)
@@ -2622,7 +2633,7 @@ rting garbage data and then destroying it (which is originally done to preserve
 					}
 					data := vault.NewSecret()
 					for k, v := range secret.Versions[i].Value {
-						data.Set(k, v, false)
+						data.SetAsInterface(k, v, false)
 					}
 					s.Versions = append(s.Versions, vault.SecretVersion{
 						Number: firstVersion + uint(i),
@@ -2632,8 +2643,9 @@ rting garbage data and then destroying it (which is originally done to preserve
 				}
 
 				err := s.Copy(v, s.Path, vault.TreeCopyOpts{
-					Clear: true,
-					Pad:   !(opt.Import.IgnoreDestroyed || opt.Import.Shallow),
+					Clear:      !opt.Import.AppendOnly,
+					Pad:        !(opt.Import.IgnoreDestroyed || opt.Import.Shallow),
+					AppendOnly: opt.Import.AppendOnly,
 				})
 				if err != nil {
 					return err
@@ -2654,7 +2666,7 @@ rting garbage data and then destroying it (which is originally done to preserve
 			if len(v) == 1 {
 				if meta, isMap := (v[0]).(map[string]interface{}); isMap {
 					version, isFloat64 := meta["export_version"].(float64)
-					if isFloat64 && version == 2 {
+					if (isFloat64 && version == 2) || (isFloat64 && version == 3) {
 						fn = v2Import
 					}
 				}
@@ -4478,7 +4490,7 @@ type exportSecret struct {
 }
 
 type exportVersion struct {
-	Deleted   bool              `json:"deleted,omitempty"`
-	Destroyed bool              `json:"destroyed,omitempty"`
-	Value     map[string]string `json:"value,omitempty"`
+	Deleted   bool                   `json:"deleted,omitempty"`
+	Destroyed bool                   `json:"destroyed,omitempty"`
+	Value     map[string]interface{} `json:"value,omitempty"`
 }

vault/secret.go

@@ -18,11 +18,11 @@ import (
 // A Secret contains a set of key/value pairs that store anything you
 // want, including passwords, RSAKey keys, usernames, etc.
 type Secret struct {
-	data map[string]string
+	data map[string]interface{}
 }
 
 func NewSecret() *Secret {
-	return &Secret{make(map[string]string)}
+	return &Secret{make(map[string]interface{})}
 }
 
 func (s Secret) MarshalJSON() ([]byte, error) {
@@ -41,8 +41,29 @@ func (s *Secret) Has(key string) bool {
 
 // Get retrieves the value of the given key, or "" if no such key exists.
 func (s *Secret) Get(key string) string {
-	x, _ := s.data[key]
-	return x
+	xx, ok := s.data[key]
+	if !ok {
+		return ""
+	}
+	switch x := xx.(type) {
+	case string:
+		return x
+	case []byte:
+		return string(x)
+	default:
+		res, _ := json.Marshal(x)
+		return string(res)
+	}
+
+}
+
+func (s *Secret) GetAsInterface(key string) interface{} {
+	xx, ok := s.data[key]
+	if !ok {
+		// return empty interface
+		return nil
+	}
+	return xx
 }
 
 func (s *Secret) Keys() []string {
@@ -64,6 +85,15 @@ func (s *Secret) Set(key, value string, skipIfExists bool) error {
 	return nil
 }
 
+// Set interface
+func (s *Secret) SetAsInterface(key string, value interface{}, skipIfExists bool) error {
+	if s.Has(key) && skipIfExists {
+		return ansi.Errorf("@R{BUG: Something tried to overwrite the} @C{%s} @R{key, but it already existed, and --no-clobber was specified}", key)
+	}
+	s.data[key] = value
+	return nil
+}
+
 // Delete removes the entry with the given key from the Secret.
 // Returns true if there was a matching object to delete. False otherwise.
 func (s *Secret) Delete(key string) bool {
@@ -269,6 +299,29 @@ func (s *Secret) YAML() string {
 	return string(b)
 }
 
+// Eqals compares two secrets and returns true if they are equal
+func (s *Secret) Equals(in *Secret) bool {
+
+	if (s == nil && in == nil) || (s == in) {
+		return true
+	}
+	if (s == nil && in != nil) || (s != nil && in == nil) {
+		return false
+	}
+	if (s.data == nil && in.data != nil) || (s.data != nil && in.data == nil) {
+		return false
+	}
+	a, err := json.Marshal(s.data)
+	if err != nil {
+		return false
+	}
+	b, err := json.Marshal(in.data)
+	if err != nil {
+		return false
+	}
+	return string(a) == string(b)
+}
+
 // SingleValue converts a secret to a string representing the value extracted.
 // Returns an error if there are not exactly one results in the secret
 // object
@@ -278,7 +331,16 @@ func (s *Secret) SingleValue() (string, error) {
 	}
 	var ret string
 	for _, v := range s.data {
-		ret = v
+
+		switch x := v.(type) {
+		case string:
+			ret = x
+		case []byte:
+			ret = string(x)
+		default:
+			res, _ := json.Marshal(x)
+			ret = string(res)
+		}
 	}
 	return ret, nil
 }

vault/tree.go

@@ -113,7 +113,7 @@ type secretTree struct {
 	Branches     []secretTree
 	Type         uint
 	MountVersion uint
-	Value        string
+	Value        interface{}
 	Version      uint
 	Deleted      bool
 	Destroyed    bool
@@ -238,7 +238,8 @@ func (t secretTree) convertToSecrets() Secrets {
 				}
 
 				for _, key := range version.Branches {
-					thisVersion.Data.Set(key.Basename(), key.Value, false)
+					//thisVersion.Data.Set(key.Basename(), key.Value, false)
+					thisVersion.Data.SetAsInterface(key.Basename(), key.Value, false)
 				}
 
 				thisEntry.Versions = append(thisEntry.Versions, thisVersion)
@@ -308,6 +309,8 @@ type TreeOpts struct {
 	GetDeletedVersions bool
 	//Only perform gets. If the target is not a secret, then an error is returned
 	GetOnly bool
+	//All key values are retrieved as strings
+	AsStrings bool
 }
 
 func (v *Vault) constructTree(path string, opts TreeOpts) (*secretTree, error) {
@@ -479,6 +482,8 @@ type TreeCopyOpts struct {
 	Clear bool
 	//Pad will insert dummy versions that have been truncated by Vault
 	Pad bool
+	//AppendOnly
+	AppendOnly bool
 }
 
 func (s SecretEntry) Copy(v *Vault, dst string, opts TreeCopyOpts) error {
@@ -489,6 +494,40 @@ func (s SecretEntry) Copy(v *Vault, dst string, opts TreeCopyOpts) error {
 		}
 	}
 
+	if opts.AppendOnly {
+		var latest *SecretVersion
+		if len(s.Versions) > 0 && s.Versions[len(s.Versions)-1].State == SecretStateAlive {
+			latest = &s.Versions[len(s.Versions)-1]
+		}
+
+		if latest != nil {
+			updateRequired := false
+			// get latest version from client
+			existingSecret := &Secret{
+				data: make(map[string]interface{}),
+			}
+
+			_, err := v.client.Get(dst, &existingSecret.data, &vaultkv.KVGetOpts{Version: 0})
+			// if err is nil, then the secret exists
+			if err == nil {
+				// compare the latest version of the secret with the existing secret
+				// if they are the same, then we don't need to do anything
+				updateRequired = !latest.Data.Equals(existingSecret)
+
+			} else {
+				updateRequired = true
+			}
+			if updateRequired {
+				_, err = v.Client().Set(dst, latest.Data.data, nil)
+				if err != nil {
+					return fmt.Errorf("Could not write secret to path `%s': %s", dst, err)
+				}
+			}
+
+		}
+		return nil
+	}
+
 	var toDelete, toDestroy []uint
 
 	if opts.Pad && len(s.Versions) > 0 {
@@ -503,9 +542,9 @@ func (s SecretEntry) Copy(v *Vault, dst string, opts TreeCopyOpts) error {
 	}
 
 	for _, version := range s.Versions {
-		var toWrite map[string]string
+		var toWrite map[string]interface{}
 		if version.State == SecretStateDestroyed {
-			toWrite = map[string]string{"TO_DESTROY": "TO_DESTROY"}
+			toWrite = map[string]interface{}{"TO_DESTROY": "TO_DESTROY"}
 		} else {
 			toWrite = version.Data.data
 		}
@@ -810,6 +849,7 @@ func (w *treeWorker) workGet(t secretTree) ([]secretTree, error) {
 	}
 
 	s, err := w.vault.Read(EncodePath(path, "", uint64(t.Version)))
+
 	//For v1 backends, this is the first non-list Vault access.
 	// If we're unable to get a path that we could list because of permissions,
 	// don't explode.
@@ -820,6 +860,10 @@ func (w *treeWorker) workGet(t secretTree) ([]secretTree, error) {
 		return nil, err
 	}
 
+	if w.opts.AsStrings {
+		s, err = w.vault.DataAsString(s)
+	}
+
 	if t.Deleted {
 		w.vault.client.Delete(path, &vaultkv.KVDeleteOpts{Versions: []uint{t.Version}})
 		if err != nil {
@@ -838,7 +882,7 @@ func (w *treeWorker) workGet(t secretTree) ([]secretTree, error) {
 		ret = append(ret, secretTree{
 			Name:    path + ":" + key,
 			Type:    treeTypeKey,
-			Value:   string(s.data[key]),
+			Value:   s.data[key],
 			Version: version,
 			Deleted: t.Deleted,
 		})