Skip to content

Commit

Permalink
feat: base branch diff (#1158)
Browse files Browse the repository at this point in the history 
* feat: scan only changed files

* refactor: remove use of xerrors

* refactor: allow orchestrator to be reused

* chore: include base branch in scan id

* refactor: more orchestrator refactoring

* feat: fetch base branch if it doesn't exist

* feat: scan base branch

* feat: filter out existing findings

* feat: account for renames and line moves

* feat: pass base branch as env var rather than command option

* feat: improve diff output

* fix: make renames work correctly

* refactor: minor tidying of ignore

* fix: cope with no repository

* test: update filelist tests

* test: update composition helper

* fix: include target in scan id (#1160)

fix: include absolute target in scan id
  • Loading branch information
@didroe
didroe authored Aug 1, 2023
1 parent 2fa6faa commit e756629
Show file tree
Hide file tree
Showing 39 changed files with 1,251 additions and 597 deletions.
7 changes: 3 additions & 4 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@ require (
github.com/ghodss/yaml v1.0.0
github.com/gitsight/go-vcsurl v1.0.1
github.com/go-enry/go-enry/v2 v2.8.4
github.com/go-git/go-billy/v5 v5.4.1
github.com/go-git/go-git/v5 v5.8.1
github.com/go-git/go-git/v5 v5.7.0
github.com/google/go-github v17.0.0+incompatible
github.com/google/uuid v1.3.0
github.com/hhatto/gocloc v0.5.1
Expand Down Expand Up @@ -68,7 +67,6 @@ require (
)

require (
dario.cat/mergo v1.0.0 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
github.com/acomagu/bufpipe v1.0.4 // indirect
Expand All @@ -79,6 +77,7 @@ require (
github.com/emirpasic/gods v1.18.1 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.4.1 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
Expand All @@ -87,6 +86,7 @@ require (
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/imdario/mergo v0.3.15 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
Expand Down Expand Up @@ -131,7 +131,6 @@ require (
github.com/subosito/gotenv v1.4.2 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/text v0.11.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1
Expand Down
10 changes: 4 additions & 6 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
Expand Down Expand Up @@ -135,8 +133,8 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
github.com/go-git/go-git/v5 v5.8.1 h1:Zo79E4p7TRk0xoRgMq0RShiTHGKcKI4+DI6BfJc/Q+A=
github.com/go-git/go-git/v5 v5.8.1/go.mod h1:FHFuoD6yGz5OSKEBK+aWN9Oah0q54Jxl0abmj6GnqAo=
github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2aE=
github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
Expand Down Expand Up @@ -241,6 +239,8 @@ github.com/hhatto/gocloc v0.5.1 h1:VFTm5VdllyGxvAjvXmYQDgthaVAUwPPJphebL+fHz2c=
github.com/hhatto/gocloc v0.5.1/go.mod h1:pTtvBwdm0Mhqjkqu1g9uXplkncP/CHnhhDOigLj9/ek=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
Expand Down Expand Up @@ -692,8 +692,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
Expand Down
31 changes: 17 additions & 14 deletions new/detector/composition/testhelper/testhelper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,21 @@ import (
"strings"
"testing"

"github.com/bradleyjkemp/cupaloy"
"github.com/hhatto/gocloc"
"github.com/rs/zerolog"
"gopkg.in/yaml.v3"

"github.com/bearer/bearer/pkg/commands"
"github.com/bearer/bearer/pkg/commands/process/orchestrator/filelist"
"github.com/bearer/bearer/pkg/commands/process/filelist"
"github.com/bearer/bearer/pkg/commands/process/filelist/files"
"github.com/bearer/bearer/pkg/commands/process/orchestrator/work"
"github.com/bearer/bearer/pkg/commands/process/orchestrator/worker"
"github.com/bearer/bearer/pkg/commands/process/settings"
"github.com/bearer/bearer/pkg/commands/process/worker"
"github.com/bearer/bearer/pkg/commands/process/worker/work"
"github.com/bearer/bearer/pkg/flag"
"github.com/bearer/bearer/pkg/report/output"
"github.com/bearer/bearer/pkg/types"
util "github.com/bearer/bearer/pkg/util/output"
"github.com/bradleyjkemp/cupaloy"
"github.com/hhatto/gocloc"
"github.com/rs/zerolog"
"gopkg.in/yaml.v3"
)

type Runner struct {
Expand Down Expand Up @@ -91,16 +93,16 @@ func (runner *Runner) RunTest(t *testing.T, testdataPath string, snapshotPath st
Languages: map[string]*gocloc.Language{},
MaxPathLength: 0,
}
files, err := filelist.Discover(testdataPath, &dummyGoclocResult, runner.config)
fileList, err := filelist.Discover(nil, testdataPath, &dummyGoclocResult, runner.config)
if err != nil {
t.Fatalf("failed to discover files: %s", err)
}

if len(files) == 0 {
if len(fileList.Files) == 0 {
t.Fatal("no scannable files found")
}

for _, file := range files {
for _, file := range fileList.Files {
myfile := file
ext := filepath.Ext(file.FilePath)
testName := strings.TrimSuffix(file.FilePath, ext) + ".yml"
Expand All @@ -110,7 +112,7 @@ func (runner *Runner) RunTest(t *testing.T, testdataPath string, snapshotPath st
}
}

func (runner *Runner) scanSingleFile(t *testing.T, testDataPath string, fileRelativePath work.File, snapshotsPath string) {
func (runner *Runner) scanSingleFile(t *testing.T, testDataPath string, fileRelativePath files.File, snapshotsPath string) {
detectorsReportFile, err := os.CreateTemp("", "report.jsonl")
if err != nil {
t.Fatalf("failed to create tmp report file: %s", err)
Expand All @@ -122,16 +124,16 @@ func (runner *Runner) scanSingleFile(t *testing.T, testDataPath string, fileRela
t.Fatalf("failed to get absolute path of report file: %s", err)
}

response := runner.worker.Scan(context.Background(), work.ProcessRequest{
_, err = runner.worker.Scan(context.Background(), work.ProcessRequest{
File: fileRelativePath,
ReportPath: detectorsReportPath,
Repository: work.Repository{
Dir: testDataPath,
},
})

if response.Error != "" {
t.Fatalf("failed to do scan %s", response.Error)
if err != nil {
t.Fatalf("failed to do scan %s", err)
}

runner.config.Scan.Target = testDataPath
Expand All @@ -140,6 +142,7 @@ func (runner *Runner) scanSingleFile(t *testing.T, testDataPath string, fileRela
Path: detectorsReportPath,
},
runner.config,
nil,
)

report, _ := util.ReportYAML(detections)
Expand Down
9 changes: 4 additions & 5 deletions pkg/commands/app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"github.com/bearer/bearer/pkg/commands/artifact"
"github.com/bearer/bearer/pkg/flag"
"github.com/spf13/cobra"
"golang.org/x/xerrors"
)

// VersionInfo holds the bearer version
Expand Down Expand Up @@ -86,20 +85,20 @@ func NewConfigCommand() *cobra.Command {
Short: "Scan config files for misconfigurations",
PreRunE: func(cmd *cobra.Command, args []string) error {
if err := configFlags.Bind(cmd); err != nil {
return xerrors.Errorf("flag bind error: %w", err)
return fmt.Errorf("flag bind error: %w", err)
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
if err := configFlags.Bind(cmd); err != nil {
return xerrors.Errorf("flag bind error: %w", err)
return fmt.Errorf("flag bind error: %w", err)
}
options, err := configFlags.ToOptions(args)
if err != nil {
return xerrors.Errorf("flag error: %w", err)
return fmt.Errorf("flag error: %w", err)
}

return artifact.Run(cmd.Context(), options, artifact.TargetFilesystem)
return artifact.Run(cmd.Context(), options)
},
SilenceErrors: true,
SilenceUsage: true,
Expand Down
Loading

0 comments on commit e756629

Please sign in to comment.