From af979d46202da8f3f24a8a626dfb3d529b405295 Mon Sep 17 00:00:00 2001 From: Bill Rich Date: Mon, 11 Apr 2022 15:38:08 -0800 Subject: [PATCH] Use .Line to reduce diff strangeness (#380) --- main.go | 9 +++++++-- pkg/sources/git/git.go | 4 +++- pkg/sources/git/git_test.go | 22 +++++++++++----------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/main.go b/main.go index 82362c46381f..2cb747d7d72e 100644 --- a/main.go +++ b/main.go @@ -33,6 +33,7 @@ var ( cli = kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.") cmd string debug = cli.Flag("debug", "Run in debug mode.").Bool() + trace = cli.Flag("trace", "Run in trace mode.").Bool() jsonOut = cli.Flag("json", "Output in JSON format.").Short('j').Bool() jsonLegacy = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool() concurrency = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int() @@ -96,10 +97,14 @@ func init() { if *jsonOut { logrus.SetFormatter(&logrus.JSONFormatter{}) } - if *debug { + switch { + case *trace: + logrus.SetLevel(logrus.TraceLevel) + logrus.Debugf("running version %s", version.BuildVersion) + case *debug: logrus.SetLevel(logrus.DebugLevel) logrus.Debugf("running version %s", version.BuildVersion) - } else { + default: logrus.SetLevel(logrus.InfoLevel) } } diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go index 6bcb8e40e926..c3d86e64f5a6 100644 --- a/pkg/sources/git/git.go +++ b/pkg/sources/git/git.go @@ -278,6 +278,7 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp var depth int64 for file := range fileChan { + log.WithField("commit", file.PatchHeader.SHA).WithField("file", file.NewName).Trace("Scanning file from git") if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth { log.Debugf("reached max depth") break @@ -311,9 +312,10 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp newLineNumber := frag.NewPosition for _, line := range frag.Lines { if line.Op == gitdiff.OpAdd { - newLines.WriteString(strings.ReplaceAll(line.String(), "\n", " ") + "\n") + newLines.WriteString(line.Line) } } + log.WithField("fragment", newLines.String()).Trace("detecting fragment") metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, newLineNumber) chunksChan <- &sources.Chunk{ SourceName: s.sourceName, diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.go index 61b9da20d32a..b524ee73d46b 100644 --- a/pkg/sources/git/git_test.go +++ b/pkg/sources/git/git_test.go @@ -169,17 +169,17 @@ func TestSource_Chunks_Integration(t *testing.T) { }, }, expectedChunkData: map[string]*byteCompare{ - "70001020fab32b1fcf2f1f0e5c66424eae649826-aws": {B: []byte("+[default] \n+aws_access_key_id = AKIAXYZDQCEN4B6JSJQI \n+aws_secret_access_key = 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 \n+output = json \n+region = us-east-2 \n")}, - "a6f8aa55736d4a85be31a0048a4607396898647a-bump": {B: []byte("+f \n")}, - "07d96d011005fe8296bdd237c13a06a72e96783d-bump": {B: []byte("+ s \n")}, - "2f251b8c1e72135a375b659951097ec7749d4af9-bump": {B: []byte("+ \n")}, - "e6c8bbabd8796ea3cd85bfc2e55b27e0a491747f-bump": {B: []byte("+oops \n")}, - "735b52b0eb40610002bb1088e902bd61824eb305-bump": {B: []byte("+oops \n")}, - "ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("+ \n")}, - "27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("+yup, just did that \n+ \n+github_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\" \n+ \n+oh, goodness. there's another one!\n")}, - "8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("+oops might drop a slack token here \n+ \n+github_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\" \n+ \n+yup, just did that\n"), Multi: true}, - "8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("+blah blaj \n+ \n+this is the secret: AKIA2E0A8F3B244C9986 \n+ \n+okay thank you bye\n"), Multi: true}, - "90c75f884c65dc3638ca1610bd9844e668f213c2-aws": {B: []byte("+this is the secret: [Default] \n+Access key Id: AKIAILE3JG6KMS3HZGCA \n+Secret Access Key: 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 \n"), Multi: true}, + "70001020fab32b1fcf2f1f0e5c66424eae649826-aws": {B: []byte("[default]\naws_access_key_id = AKIAXYZDQCEN4B6JSJQI\naws_secret_access_key = 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\noutput = json\nregion = us-east-2\n")}, + "a6f8aa55736d4a85be31a0048a4607396898647a-bump": {B: []byte("f\n")}, + "07d96d011005fe8296bdd237c13a06a72e96783d-bump": {B: []byte(" s \n")}, + "2f251b8c1e72135a375b659951097ec7749d4af9-bump": {B: []byte(" \n")}, + "e6c8bbabd8796ea3cd85bfc2e55b27e0a491747f-bump": {B: []byte("oops \n")}, + "735b52b0eb40610002bb1088e902bd61824eb305-bump": {B: []byte("oops\n")}, + "ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("\n")}, + "27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("yup, just did that\n\ngithub_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\"\n\noh, goodness. there's another one!")}, + "8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("oops might drop a slack token here\n\ngithub_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\"\n\nyup, just did that"), Multi: true}, + "8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("blah blaj\n\nthis is the secret: AKIA2E0A8F3B244C9986\n\nokay thank you bye"), Multi: true}, + "90c75f884c65dc3638ca1610bd9844e668f213c2-aws": {B: []byte("this is the secret: [Default]\nAccess key Id: AKIAILE3JG6KMS3HZGCA\nSecret Access Key: 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\n"), Multi: true}, }, }, }