-
Notifications
You must be signed in to change notification settings - Fork 1.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
3 changed files
with
181 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,78 @@ | ||
package decoders | ||
|
||
import ( | ||
"bytes" | ||
"encoding/base64" | ||
"strings" | ||
|
||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources" | ||
) | ||
|
||
type Base64 struct{} | ||
|
||
var ( | ||
b64Charset = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=") | ||
b64EndChars = "+/=" | ||
) | ||
|
||
func getSubstringsOfCharacterSet(data []byte, charset []byte, threshold int) []string { | ||
count := 0 | ||
substrings := []string{} | ||
letters := strings.Builder{} | ||
if len(data) == 0 { | ||
return nil | ||
} | ||
for _, char := range string(data) { | ||
if bytes.ContainsRune(charset, char) { | ||
letters.WriteRune(char) | ||
count++ | ||
} else { | ||
if count > threshold { | ||
substrings = appendB64Substring(letters, substrings) | ||
} | ||
letters.Reset() | ||
count = 0 | ||
} | ||
} | ||
|
||
if count > threshold && len(letters.String()) > 0 { | ||
substrings = appendB64Substring(letters, substrings) | ||
} | ||
|
||
return substrings | ||
} | ||
|
||
func appendB64Substring(letters strings.Builder, substrings []string) []string { | ||
|
||
substring := strings.TrimLeft(letters.String(), b64EndChars) | ||
// handle key=value | ||
if strings.Contains(strings.TrimRight(substring, b64EndChars), "=") { | ||
split := strings.SplitN(substring, "=", 2) | ||
substrings = append(substrings, split[len(split)-1]) | ||
} else { | ||
substrings = append(substrings, substring) | ||
} | ||
return substrings | ||
} | ||
|
||
func (d *Base64) FromChunk(chunk *sources.Chunk) *sources.Chunk { | ||
|
||
encodedSubstrings := getSubstringsOfCharacterSet(chunk.Data, b64Charset, 20) | ||
decodedSubstrings := map[string][]byte{} | ||
|
||
for _, str := range encodedSubstrings { | ||
dec, err := base64.StdEncoding.DecodeString(str) | ||
if err == nil && len(dec) > 0 { | ||
decodedSubstrings[str] = dec | ||
} | ||
} | ||
|
||
if len(decodedSubstrings) > 0 { | ||
for substring, dec := range decodedSubstrings { | ||
chunk.Data = bytes.Replace(chunk.Data, []byte(substring), dec, 1) | ||
} | ||
return chunk | ||
} | ||
|
||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,102 @@ | ||
package decoders | ||
|
||
import ( | ||
"testing" | ||
|
||
"github.com/kylelemons/godebug/pretty" | ||
|
||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors" | ||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources" | ||
) | ||
|
||
func TestBase64_FromChunk(t *testing.T) { | ||
tests := []struct { | ||
name string | ||
chunk *sources.Chunk | ||
want *sources.Chunk | ||
}{ | ||
{ | ||
name: "only b64 chunk", | ||
chunk: &sources.Chunk{ | ||
Data: []byte(`bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q=`), | ||
}, | ||
want: &sources.Chunk{ | ||
Data: []byte(`longer-encoded-secret-test`), | ||
}, | ||
}, | ||
{ | ||
name: "mixed content", | ||
chunk: &sources.Chunk{ | ||
Data: []byte(`token: bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q=`), | ||
}, | ||
want: &sources.Chunk{ | ||
Data: []byte(`token: longer-encoded-secret-test`), | ||
}, | ||
}, | ||
{ | ||
name: "no chunk", | ||
chunk: &sources.Chunk{ | ||
Data: []byte(``), | ||
}, | ||
want: nil, | ||
}, | ||
{ | ||
name: "env var (looks like all b64 decodable but has `=` in the middle)", | ||
chunk: &sources.Chunk{ | ||
Data: []byte(`some-encoded-secret=dGVzdHNlY3JldA==`), | ||
}, | ||
want: &sources.Chunk{ | ||
Data: []byte(`some-encoded-secret=testsecret`), | ||
}, | ||
}, | ||
{ | ||
name: "has longer b64 inside", | ||
chunk: &sources.Chunk{ | ||
Data: []byte(`some-encoded-secret="bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q="`), | ||
}, | ||
want: &sources.Chunk{ | ||
Data: []byte(`some-encoded-secret="longer-encoded-secret-test"`), | ||
}, | ||
}, | ||
{ | ||
name: "many possible substrings", | ||
chunk: &sources.Chunk{ | ||
Data: []byte(`Many substrings in this slack message could be base64 decoded | ||
but only dGhpcyBlbmNhcHN1bGF0ZWQgc2VjcmV0 should be decoded.`), | ||
}, | ||
want: &sources.Chunk{ | ||
Data: []byte(`Many substrings in this slack message could be base64 decoded | ||
but only this encapsulated secret should be decoded.`), | ||
}, | ||
}, | ||
} | ||
for _, tt := range tests { | ||
t.Run(tt.name, func(t *testing.T) { | ||
d := &Base64{} | ||
got := d.FromChunk(tt.chunk) | ||
if tt.want != nil { | ||
if got == nil { | ||
t.Fatal("got nil, did not want nil") | ||
} | ||
if diff := pretty.Compare(string(got.Data), string(tt.want.Data)); diff != "" { | ||
t.Errorf("Base64FromChunk() %s diff: (-got +want)\n%s", tt.name, diff) | ||
} | ||
} else { | ||
if got != nil { | ||
t.Error("Expected nil chunk") | ||
} | ||
} | ||
}) | ||
} | ||
} | ||
|
||
func BenchmarkFromChunk(benchmark *testing.B) { | ||
d := Base64{} | ||
for name, data := range detectors.MustGetBenchmarkData() { | ||
benchmark.Run(name, func(b *testing.B) { | ||
for n := 0; n < b.N; n++ { | ||
d.FromChunk(&sources.Chunk{Data: data}) | ||
} | ||
}) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -7,6 +7,7 @@ import ( | |
func DefaultDecoders() []Decoder { | ||
return []Decoder{ | ||
&Plain{}, | ||
&Base64{}, | ||
} | ||
} | ||
|
||
|