From 45076418c77549229bc2a1636ffe9f3817e8a97e Mon Sep 17 00:00:00 2001 From: Jason Priebe Date: Wed, 14 Aug 2019 15:39:21 -0400 Subject: [PATCH 1/5] added encrypted-regex option --- cmd/sops/edit.go | 2 ++ cmd/sops/encrypt.go | 2 ++ cmd/sops/main.go | 33 ++++++++++++++++++++++----- config/config.go | 18 +++++++++++++-- config/config_test.go | 14 ++++++++++++ sops.go | 34 ++++++++++++++++++++++++---- sops_test.go | 52 +++++++++++++++++++++++++++++++++++++++++++ stores/stores.go | 22 +++++++++++++++--- 8 files changed, 163 insertions(+), 14 deletions(-) diff --git a/cmd/sops/edit.go b/cmd/sops/edit.go index 341f1c6be..14013b15d 100644 --- a/cmd/sops/edit.go +++ b/cmd/sops/edit.go @@ -37,6 +37,7 @@ type editExampleOpts struct { editOpts UnencryptedSuffix string EncryptedSuffix string + EncryptedRegex string KeyGroups []sops.KeyGroup GroupThreshold int } @@ -65,6 +66,7 @@ func editExample(opts editExampleOpts) ([]byte, error) { KeyGroups: opts.KeyGroups, UnencryptedSuffix: opts.UnencryptedSuffix, EncryptedSuffix: opts.EncryptedSuffix, + EncryptedRegex: opts.EncryptedRegex, Version: version.Version, ShamirThreshold: opts.GroupThreshold, }, diff --git a/cmd/sops/encrypt.go b/cmd/sops/encrypt.go index f34924c90..0d80b5020 100644 --- a/cmd/sops/encrypt.go +++ b/cmd/sops/encrypt.go @@ -22,6 +22,7 @@ type encryptOpts struct { KeyServices []keyservice.KeyServiceClient UnencryptedSuffix string EncryptedSuffix string + EncryptedRegex string KeyGroups []sops.KeyGroup GroupThreshold int } @@ -76,6 +77,7 @@ func encrypt(opts encryptOpts) (encryptedFile []byte, err error) { KeyGroups: opts.KeyGroups, UnencryptedSuffix: opts.UnencryptedSuffix, EncryptedSuffix: opts.EncryptedSuffix, + EncryptedRegex: opts.EncryptedRegex, Version: version.Version, ShamirThreshold: opts.GroupThreshold, }, diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 311f402b0..70effd87a 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -439,6 +439,10 @@ func main() { Name: "encrypted-suffix", Usage: "override the encrypted key suffix. When empty, all keys will be encrypted, unless otherwise marked with unencrypted-suffix.", }, + cli.StringFlag{ + Name: "encrypted-regex", + Usage: "set the encrypted key suffix. When specified, only keys matching the regex will be encrypted.", + }, cli.StringFlag{ Name: "config", Usage: "path to sops' config file. If set, sops will not search for the config file recursively.", @@ -491,6 +495,7 @@ func main() { unencryptedSuffix := c.String("unencrypted-suffix") encryptedSuffix := c.String("encrypted-suffix") + encryptedRegex := c.String("encrypted-regex") conf, err := loadConfig(c, fileName, nil) if err != nil { return toExitError(err) @@ -503,13 +508,29 @@ func main() { if encryptedSuffix == "" { encryptedSuffix = conf.EncryptedSuffix } + if encryptedRegex == "" { + encryptedRegex = conf.EncryptedRegex + } } - if unencryptedSuffix != "" && encryptedSuffix != "" { - return common.NewExitError("Error: cannot use both encrypted_suffix and unencrypted_suffix in the same file", codes.ErrorConflictingParameters) + + cryptRuleCount := 0 + if unencryptedSuffix != "" { + cryptRuleCount++ + } + if encryptedSuffix != "" { + cryptRuleCount++ } - // only supply the default UnencryptedSuffix when EncryptedSuffix is not provided - if unencryptedSuffix == "" && encryptedSuffix == "" { - unencryptedSuffix = sops.DefaultUnencryptedSuffix + if encryptedRegex != "" { + cryptRuleCount++ + } + + if cryptRuleCount > 1 { + return common.NewExitError("Error: cannot use more than one of encrypted_suffix, unencrypted_suffix, or encrypted_regex in the same file", codes.ErrorConflictingParameters) + } + + // only supply the default UnencryptedSuffix when EncryptedSuffix and EncryptedRegex are not provided + if cryptRuleCount == 0 { + return common.NewExitError("Error: cannot use both encrypted_suffix and unencrypted_suffix in the same file", codes.ErrorConflictingParameters) } inputStore := inputStore(c, fileName) @@ -535,6 +556,7 @@ func main() { Cipher: aes.NewCipher(), UnencryptedSuffix: unencryptedSuffix, EncryptedSuffix: encryptedSuffix, + EncryptedRegex: encryptedRegex, KeyServices: svcs, KeyGroups: groups, GroupThreshold: threshold, @@ -656,6 +678,7 @@ func main() { editOpts: opts, UnencryptedSuffix: unencryptedSuffix, EncryptedSuffix: encryptedSuffix, + EncryptedRegex: encryptedRegex, KeyGroups: groups, GroupThreshold: threshold, }) diff --git a/config/config.go b/config/config.go index 6ae958521..dd9e8f185 100644 --- a/config/config.go +++ b/config/config.go @@ -111,6 +111,7 @@ type creationRule struct { ShamirThreshold int `yaml:"shamir_threshold"` UnencryptedSuffix string `yaml:"unencrypted_suffix"` EncryptedSuffix string `yaml:"encrypted_suffix"` + EncryptedRegex string `yaml:"encrypted_regex"` } // Load loads a sops config file into a temporary struct @@ -128,6 +129,7 @@ type Config struct { ShamirThreshold int UnencryptedSuffix string EncryptedSuffix string + EncryptedRegex string Destination publish.Destination } @@ -187,8 +189,19 @@ func loadConfigFile(confPath string) (*configFile, error) { } func configFromRule(rule *creationRule, kmsEncryptionContext map[string]*string) (*Config, error) { - if rule.UnencryptedSuffix != "" && rule.EncryptedSuffix != "" { - return nil, fmt.Errorf("error loading config: cannot use both encrypted_suffix and unencrypted_suffix for the same rule") + cryptRuleCount := 0 + if rule.UnencryptedSuffix != "" { + cryptRuleCount++ + } + if rule.EncryptedSuffix != "" { + cryptRuleCount++ + } + if rule.EncryptedRegex != "" { + cryptRuleCount++ + } + + if cryptRuleCount > 1 { + return nil, fmt.Errorf("error loading config: cannot use more than one of encrypted_suffix, unencrypted_suffix, or encrypted_regex for the same rule") } groups, err := getKeyGroupsFromCreationRule(rule, kmsEncryptionContext) @@ -201,6 +214,7 @@ func configFromRule(rule *creationRule, kmsEncryptionContext map[string]*string) ShamirThreshold: rule.ShamirThreshold, UnencryptedSuffix: rule.UnencryptedSuffix, EncryptedSuffix: rule.EncryptedSuffix, + EncryptedRegex: rule.EncryptedRegex, }, nil } diff --git a/config/config_test.go b/config/config_test.go index 556290325..4823c7c1d 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -122,6 +122,14 @@ creation_rules: version: fooversion `) +var sampleConfigWithRegexParameters = []byte(` +creation_rules: + - path_regex: foobar* + kms: "1" + pgp: "2" + encrypted_regex: ^enc: + `) + var sampleConfigWithInvalidParameters = []byte(` creation_rules: - path_regex: foobar* @@ -273,6 +281,12 @@ func TestLoadConfigFileWithEncryptedSuffix(t *testing.T) { assert.Equal(t, "_enc", conf.EncryptedSuffix) } +func TestLoadConfigFileWithEncryptedRegex(t *testing.T) { + conf, err := loadForFileFromBytes(sampleConfigWithRegexParameters, "barbar", nil) + assert.Equal(t, nil, err) + assert.Equal(t, "^enc:", conf.EncryptedRegex) +} + func TestLoadConfigFileWithInvalidParameters(t *testing.T) { _, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithInvalidParameters, t), "foobar", nil) assert.NotNil(t, err) diff --git a/sops.go b/sops.go index 814ec5329..6485f8717 100644 --- a/sops.go +++ b/sops.go @@ -41,6 +41,7 @@ import ( "crypto/sha512" "fmt" "reflect" + "regexp" "strconv" "strings" "time" @@ -285,9 +286,10 @@ func (branch TreeBranch) walkBranch(in TreeBranch, path []string, onLeaves func( // Encrypt walks over the tree and encrypts all values with the provided cipher, // except those whose key ends with the UnencryptedSuffix specified on the -// Metadata struct, or those not ending with EncryptedSuffix, if EncryptedSuffix -// is provided (by default it is not). If encryption is successful, it returns -// the MAC for the encrypted tree. +// Metadata struct, those not ending with EncryptedSuffix, if EncryptedSuffix +// is provided (by default it is not), or those not matching EncryptedRegex, +// if EncryptedRegex is provided (by default it is not). If encryption is +// successful, it returns the MAC for the encrypted tree. func (tree Tree) Encrypt(key []byte, cipher Cipher) (string, error) { audit.SubmitEvent(audit.EncryptEvent{ File: tree.FilePath, @@ -321,6 +323,16 @@ func (tree Tree) Encrypt(key []byte, cipher Cipher) (string, error) { } } } + if tree.Metadata.EncryptedRegex != "" { + encrypted = false + for _, p := range path { + matched, _ := regexp.Match(tree.Metadata.EncryptedRegex, []byte(p)) + if matched { + encrypted = true + break + } + } + } if encrypted { var err error pathString := strings.Join(path, ":") + ":" @@ -343,7 +355,10 @@ func (tree Tree) Encrypt(key []byte, cipher Cipher) (string, error) { return fmt.Sprintf("%X", hash.Sum(nil)), nil } -// Decrypt walks over the tree and decrypts all values with the provided cipher, except those whose key ends with the UnencryptedSuffix specified on the Metadata struct or those not ending with EncryptedSuffix, if EncryptedSuffix is provided (by default it is not). +// Decrypt walks over the tree and decrypts all values with the provided cipher, +// except those whose key ends with the UnencryptedSuffix specified on the Metadata struct, +// those not ending with EncryptedSuffix, if EncryptedSuffix is provided (by default it is not), +// or those not matching EncryptedRegex, if EncryptedRegex is provided (by default it is not). // If decryption is successful, it returns the MAC for the decrypted tree. func (tree Tree) Decrypt(key []byte, cipher Cipher) (string, error) { log.Debug("Decrypting tree") @@ -371,6 +386,16 @@ func (tree Tree) Decrypt(key []byte, cipher Cipher) (string, error) { } } } + if tree.Metadata.EncryptedRegex != "" { + encrypted = false + for _, p := range path { + matched, _ := regexp.Match(tree.Metadata.EncryptedRegex, []byte(p)) + if matched { + encrypted = true + break + } + } + } var v interface{} if encrypted { var err error @@ -441,6 +466,7 @@ type Metadata struct { LastModified time.Time UnencryptedSuffix string EncryptedSuffix string + EncryptedRegex string MessageAuthenticationCode string Version string KeyGroups []KeyGroup diff --git a/sops_test.go b/sops_test.go index db9fe82d6..d1d7fbc88 100644 --- a/sops_test.go +++ b/sops_test.go @@ -137,6 +137,58 @@ func TestEncryptedSuffix(t *testing.T) { } } +func TestEncryptedRegex(t *testing.T) { + branches := TreeBranches{ + TreeBranch{ + TreeItem{ + Key: "enc:foo", + Value: "bar", + }, + TreeItem{ + Key: "bar", + Value: TreeBranch{ + TreeItem{ + Key: "foo", + Value: "bar", + }, + }, + }, + }, + } + tree := Tree{Branches: branches, Metadata: Metadata{EncryptedRegex: "^enc:"}} + expected := TreeBranch{ + TreeItem{ + Key: "enc:foo", + Value: "rab", + }, + TreeItem{ + Key: "bar", + Value: TreeBranch{ + TreeItem{ + Key: "foo", + Value: "bar", + }, + }, + }, + } + cipher := reverseCipher{} + _, err := tree.Encrypt(bytes.Repeat([]byte("f"), 32), cipher) + if err != nil { + t.Errorf("Encrypting the tree failed: %s", err) + } + if !reflect.DeepEqual(tree.Branches[0], expected) { + t.Errorf("Trees don't match: \ngot \t\t%+v,\n expected \t\t%+v", tree.Branches[0], expected) + } + _, err = tree.Decrypt(bytes.Repeat([]byte("f"), 32), cipher) + if err != nil { + t.Errorf("Decrypting the tree failed: %s", err) + } + expected[0].Value = "bar" + if !reflect.DeepEqual(tree.Branches[0], expected) { + t.Errorf("Trees don't match: \ngot\t\t\t%+v,\nexpected\t\t%+v", tree.Branches[0], expected) + } +} + type MockCipher struct{} func (m MockCipher) Encrypt(value interface{}, key []byte, path string) (string, error) { diff --git a/stores/stores.go b/stores/stores.go index efe6f94cd..0194c92db 100644 --- a/stores/stores.go +++ b/stores/stores.go @@ -45,6 +45,7 @@ type Metadata struct { PGPKeys []pgpkey `yaml:"pgp" json:"pgp"` UnencryptedSuffix string `yaml:"unencrypted_suffix,omitempty" json:"unencrypted_suffix,omitempty"` EncryptedSuffix string `yaml:"encrypted_suffix,omitempty" json:"encrypted_suffix,omitempty"` + EncryptedRegex string `yaml:"encrypted_regex,omitempty" json:"encrypted_regex,omitempty"` Version string `yaml:"version" json:"version"` } @@ -90,6 +91,7 @@ func MetadataFromInternal(sopsMetadata sops.Metadata) Metadata { m.LastModified = sopsMetadata.LastModified.Format(time.RFC3339) m.UnencryptedSuffix = sopsMetadata.UnencryptedSuffix m.EncryptedSuffix = sopsMetadata.EncryptedSuffix + m.EncryptedRegex = sopsMetadata.EncryptedRegex m.MessageAuthenticationCode = sopsMetadata.MessageAuthenticationCode m.Version = sopsMetadata.Version m.ShamirThreshold = sopsMetadata.ShamirThreshold @@ -183,10 +185,23 @@ func (m *Metadata) ToInternal() (sops.Metadata, error) { if err != nil { return sops.Metadata{}, err } - if m.UnencryptedSuffix != "" && m.EncryptedSuffix != "" { - return sops.Metadata{}, fmt.Errorf("Cannot use both encrypted_suffix and unencrypted_suffix in the same file") + + cryptRuleCount := 0 + if m.UnencryptedSuffix != "" { + cryptRuleCount++ + } + if m.EncryptedSuffix != "" { + cryptRuleCount++ + } + if m.EncryptedRegex != "" { + cryptRuleCount++ } - if m.UnencryptedSuffix == "" && m.EncryptedSuffix == "" { + + if cryptRuleCount > 1 { + return sops.Metadata{}, fmt.Errorf("Cannot use more than one of encrypted_suffix, unencrypted_suffix, or encrypted_regex in the same file") + } + + if cryptRuleCount == 0 { m.UnencryptedSuffix = sops.DefaultUnencryptedSuffix } return sops.Metadata{ @@ -196,6 +211,7 @@ func (m *Metadata) ToInternal() (sops.Metadata, error) { MessageAuthenticationCode: m.MessageAuthenticationCode, UnencryptedSuffix: m.UnencryptedSuffix, EncryptedSuffix: m.EncryptedSuffix, + EncryptedRegex: m.EncryptedRegex, LastModified: lastModified, }, nil } From d5f7c886cf4ba63cce2f8fc63fdb02e82e320f0b Mon Sep 17 00:00:00 2001 From: Jason Priebe Date: Thu, 15 Aug 2019 08:31:27 -0400 Subject: [PATCH 2/5] corrected a mistake in handling the interaction among unencrypted-suffix, ecnrypted-suffix, and encrypted-regex --- cmd/sops/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 70effd87a..2c0b00fb8 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -530,7 +530,7 @@ func main() { // only supply the default UnencryptedSuffix when EncryptedSuffix and EncryptedRegex are not provided if cryptRuleCount == 0 { - return common.NewExitError("Error: cannot use both encrypted_suffix and unencrypted_suffix in the same file", codes.ErrorConflictingParameters) + unencryptedSuffix = sops.DefaultUnencryptedSuffix } inputStore := inputStore(c, fileName) From d5199e371f8231511e675532f8713e46b2a9d297 Mon Sep 17 00:00:00 2001 From: Jason Priebe Date: Thu, 15 Aug 2019 08:46:02 -0400 Subject: [PATCH 3/5] added example of encrypted-regex --- README.rst | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index c70165d55..2ba315e14 100644 --- a/README.rst +++ b/README.rst @@ -1193,10 +1193,21 @@ The unencrypted suffix can be set to a different value using the Conversely, you can opt in to only encrypt some values in a YAML or JSON file, by adding a chosen suffix to those keys and passing it to the ``--encrypted-suffix`` option. +A third method is to use the ``--encrypted-regex`` which will only encrypt values under +keys that match the supplied regular expression. For example, this command: + +.. code:: bash + + $ sops --encrypt --encrypted-regex '&(data|stringData)' k8s-secrets.yaml + +will encrypt the values under the ``data`` and ``stringData`` keys in a YAML file +containing kubernetes secrets. It will not encrypt other values that help you to +navigate the file, like ``metadata`` which contains the secrets' names. + You can also specify these options in the ``.sops.yaml`` config file. -Note: these two options ``--unencrypted-suffix`` and ``--encrypted-suffix`` are mutually exclusive and -cannot both be used in the same file. +Note: these three options ``--unencrypted-suffix``, ``--encrypted-suffix``, and ``--encrypted-regex`` are +mutually exclusive and cannot both be used in the same file. Encryption Protocol ------------------- From 34be9e9edf95d357ea6c8c64e7d61bdeec79c912 Mon Sep 17 00:00:00 2001 From: Jason Priebe Date: Thu, 15 Aug 2019 10:27:15 -0400 Subject: [PATCH 4/5] got config_test passing --- config/config_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index 4823c7c1d..1d42dc065 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -124,10 +124,10 @@ creation_rules: var sampleConfigWithRegexParameters = []byte(` creation_rules: - - path_regex: foobar* - kms: "1" - pgp: "2" - encrypted_regex: ^enc: + - path_regex: barbar* + kms: "1" + pgp: "2" + encrypted_regex: "^enc:" `) var sampleConfigWithInvalidParameters = []byte(` @@ -282,7 +282,7 @@ func TestLoadConfigFileWithEncryptedSuffix(t *testing.T) { } func TestLoadConfigFileWithEncryptedRegex(t *testing.T) { - conf, err := loadForFileFromBytes(sampleConfigWithRegexParameters, "barbar", nil) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithRegexParameters, t), "barbar", nil) assert.Equal(t, nil, err) assert.Equal(t, "^enc:", conf.EncryptedRegex) } From d8db56650a1d09bd6bb44e43af136bc531e5cd5f Mon Sep 17 00:00:00 2001 From: Jason Priebe Date: Wed, 21 Aug 2019 12:10:57 -0400 Subject: [PATCH 5/5] Update README.rst Co-Authored-By: Adrian Utrilla --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 2ba315e14..dbeecdbb8 100644 --- a/README.rst +++ b/README.rst @@ -1207,7 +1207,7 @@ navigate the file, like ``metadata`` which contains the secrets' names. You can also specify these options in the ``.sops.yaml`` config file. Note: these three options ``--unencrypted-suffix``, ``--encrypted-suffix``, and ``--encrypted-regex`` are -mutually exclusive and cannot both be used in the same file. +mutually exclusive and cannot all be used in the same file. Encryption Protocol -------------------