From aea9bb47a3e41a6f326380ce0f90d709a8620be5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Fabianski?= Date: Thu, 14 Mar 2024 18:13:28 +0100 Subject: [PATCH] feat: allow test files to be scanned --- e2e/flags/.snapshots/TestInitCommand | 1 + .../.snapshots/TestMetadataFlags-help-scan | 1 + .../.snapshots/TestMetadataFlags-scan-help | 1 + ...ReportFlagsShouldFail-invalid-context-flag | 1 + ...lagsShouldFail-invalid-format-flag-privacy | 1 + ...agsShouldFail-invalid-format-flag-security | 1 + ...tReportFlagsShouldFail-invalid-report-flag | 1 + .../process/orchestrator/worker/worker.go | 3 +++ internal/detectors/detectors.go | 4 ++++ .../internal/testhelper/testhelper.go | 2 +- internal/flag/scan_flags.go | 7 ++++++ internal/flag/types/types.go | 1 + internal/util/file/file.go | 23 +++++++++++++++---- 13 files changed, 42 insertions(+), 5 deletions(-) diff --git a/e2e/flags/.snapshots/TestInitCommand b/e2e/flags/.snapshots/TestInitCommand index a9eabefb7..b92ad3257 100644 --- a/e2e/flags/.snapshots/TestInitCommand +++ b/e2e/flags/.snapshots/TestInitCommand @@ -26,4 +26,5 @@ scan: scanner: - sast skip-path: [] + skip-test: true diff --git a/e2e/flags/.snapshots/TestMetadataFlags-help-scan b/e2e/flags/.snapshots/TestMetadataFlags-help-scan index 80ea983df..ef57a9ca6 100644 --- a/e2e/flags/.snapshots/TestMetadataFlags-help-scan +++ b/e2e/flags/.snapshots/TestMetadataFlags-help-scan @@ -36,6 +36,7 @@ Scan Flags --quiet Suppress non-essential messages --scanner strings Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast]) --skip-path strings Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql + --skip-test Disable automatic skipping of test files (default true) General Flags --api-key string Use your Bearer API Key to send the report to Bearer. diff --git a/e2e/flags/.snapshots/TestMetadataFlags-scan-help b/e2e/flags/.snapshots/TestMetadataFlags-scan-help index 80ea983df..ef57a9ca6 100644 --- a/e2e/flags/.snapshots/TestMetadataFlags-scan-help +++ b/e2e/flags/.snapshots/TestMetadataFlags-scan-help @@ -36,6 +36,7 @@ Scan Flags --quiet Suppress non-essential messages --scanner strings Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast]) --skip-path strings Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql + --skip-test Disable automatic skipping of test files (default true) General Flags --api-key string Use your Bearer API Key to send the report to Bearer. diff --git a/e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-context-flag b/e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-context-flag index fa6b80bc7..8a77f3218 100644 --- a/e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-context-flag +++ b/e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-context-flag @@ -37,6 +37,7 @@ Scan Flags --quiet Suppress non-essential messages --scanner strings Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast]) --skip-path strings Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql + --skip-test Disable automatic skipping of test files (default true) General Flags --api-key string Use your Bearer API Key to send the report to Bearer. diff --git a/e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-format-flag-privacy b/e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-format-flag-privacy index 8b9114df2..31f37ac8b 100644 --- a/e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-format-flag-privacy +++ b/e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-format-flag-privacy @@ -37,6 +37,7 @@ Scan Flags --quiet Suppress non-essential messages --scanner strings Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast]) --skip-path strings Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql + --skip-test Disable automatic skipping of test files (default true) General Flags --api-key string Use your Bearer API Key to send the report to Bearer. diff --git a/e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-format-flag-security b/e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-format-flag-security index a6276a8d6..4d864af34 100644 --- a/e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-format-flag-security +++ b/e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-format-flag-security @@ -37,6 +37,7 @@ Scan Flags --quiet Suppress non-essential messages --scanner strings Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast]) --skip-path strings Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql + --skip-test Disable automatic skipping of test files (default true) General Flags --api-key string Use your Bearer API Key to send the report to Bearer. diff --git a/e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-report-flag b/e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-report-flag index cc33c1019..05b623e36 100644 --- a/e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-report-flag +++ b/e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-report-flag @@ -37,6 +37,7 @@ Scan Flags --quiet Suppress non-essential messages --scanner strings Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast]) --skip-path strings Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql + --skip-test Disable automatic skipping of test files (default true) General Flags --api-key string Use your Bearer API Key to send the report to Bearer. diff --git a/internal/commands/process/orchestrator/worker/worker.go b/internal/commands/process/orchestrator/worker/worker.go index a06f456a8..fe50b76c9 100644 --- a/internal/commands/process/orchestrator/worker/worker.go +++ b/internal/commands/process/orchestrator/worker/worker.go @@ -34,11 +34,13 @@ type Worker struct { classifer *classification.Classifier enabledScanners []string sastScanner *scanner.Scanner + skipTest bool } func (worker *Worker) Setup(config config.Config) error { worker.debug = config.Debug worker.enabledScanners = config.Scan.Scanner + worker.skipTest = config.Scan.SkipTest if slices.Contains(worker.enabledScanners, "sast") { classifier, err := classification.NewClassifier(&classification.Config{Config: config}) @@ -86,6 +88,7 @@ func (worker *Worker) Scan(ctx context.Context, scanRequest work.ProcessRequest) fileStats, worker.enabledScanners, worker.sastScanner, + worker.skipTest, ) if ctx.Err() != nil { diff --git a/internal/detectors/detectors.go b/internal/detectors/detectors.go index 6f2bbbdf9..d55194d3e 100644 --- a/internal/detectors/detectors.go +++ b/internal/detectors/detectors.go @@ -140,6 +140,7 @@ func Extract( fileStats *stats.FileStats, enabledScanners []string, sastScanner *scanner.Scanner, + skipTest bool, ) error { return ExtractWithDetectors( ctx, @@ -149,6 +150,7 @@ func Extract( fileStats, Registrations(enabledScanners), sastScanner, + skipTest, ) } @@ -160,6 +162,7 @@ func ExtractWithDetectors( fileStats *stats.FileStats, allDetectors []InitializedDetector, sastScanner *scanner.Scanner, + skipTest bool, ) error { activeDetectors := make(map[InitializedDetector]activeDetector) @@ -167,6 +170,7 @@ func ExtractWithDetectors( if err := file.IterateFilesList( rootDir, []string{filename}, + skipTest, func(dir *file.Path) (bool, error) { for _, detector := range allDetectors { active, isActive := activeDetectors[detector] diff --git a/internal/detectors/internal/testhelper/testhelper.go b/internal/detectors/internal/testhelper/testhelper.go index 0bf3c8e09..1803526ab 100644 --- a/internal/detectors/internal/testhelper/testhelper.go +++ b/internal/detectors/internal/testhelper/testhelper.go @@ -53,7 +53,7 @@ func Extract( } for _, filename := range files { - err = detectors.ExtractWithDetectors(context.Background(), path, filename, &report, nil, registrations, nil) + err = detectors.ExtractWithDetectors(context.Background(), path, filename, &report, nil, registrations, nil, true) if !assert.Nil(t, err) { t.Errorf("report has errored %s", err) } diff --git a/internal/flag/scan_flags.go b/internal/flag/scan_flags.go index 4c21a74b5..6490259d2 100644 --- a/internal/flag/scan_flags.go +++ b/internal/flag/scan_flags.go @@ -34,6 +34,12 @@ var ( Value: []string{}, Usage: "Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql", }) + SkipTestFlag = ScanFlagGroup.add(flagtypes.Flag{ + Name: "skip-test", + ConfigName: "scan.skip-test", + Value: true, + Usage: "Disable automatic skipping of test files", + }) DisableDomainResolutionFlag = ScanFlagGroup.add(flagtypes.Flag{ Name: "disable-domain-resolution", ConfigName: "scan.disable-domain-resolution", @@ -162,6 +168,7 @@ func (scanFlagGroup) SetOptions(options *flagtypes.Options, args []string) error options.ScanOptions = flagtypes.ScanOptions{ SkipPath: getStringSlice(SkipPathFlag), + SkipTest: getBool(SkipTestFlag), DisableDomainResolution: getBool(DisableDomainResolutionFlag), DomainResolutionTimeout: getDuration(DomainResolutionTimeoutFlag), InternalDomains: getStringSlice(InternalDomainsFlag), diff --git a/internal/flag/types/types.go b/internal/flag/types/types.go index 7af9d413c..f9efadfcb 100644 --- a/internal/flag/types/types.go +++ b/internal/flag/types/types.go @@ -60,6 +60,7 @@ type Options struct { type ScanOptions struct { Target string `mapstructure:"target" json:"target" yaml:"target"` + SkipTest bool `mapstructure:"skip-test" json:"skip-test" yaml:"skip-test"` SkipPath []string `mapstructure:"skip-path" json:"skip-path" yaml:"skip-path"` DisableDomainResolution bool `mapstructure:"disable-domain-resolution" json:"disable-domain-resolution" yaml:"disable-domain-resolution"` DomainResolutionTimeout time.Duration `mapstructure:"domain-resolution-timeout" json:"domain-resolution-timeout" yaml:"domain-resolution-timeout"` diff --git a/internal/util/file/file.go b/internal/util/file/file.go index 3016d1412..f2f581fb2 100644 --- a/internal/util/file/file.go +++ b/internal/util/file/file.go @@ -27,13 +27,16 @@ type Line struct { Strip bool } -var ignoredFilenames = []*regexp.Regexp{ - regexp.MustCompile(`(^|/)\.git/`), +var ignoreTestFiles = []*regexp.Regexp{ regexp.MustCompile(`(^|/)(?i:_*tests?_*)/`), regexp.MustCompile(`(^|/)specs?/`), - regexp.MustCompile(`(^|/)testing/`), regexp.MustCompile(`(^|/|[_-])(spec|test)s?\.`), regexp.MustCompile(`(?i:unit[-_]?tests?)`), +} + +var ignoredFilenames = []*regexp.Regexp{ + regexp.MustCompile(`(^|/)\.git/`), + regexp.MustCompile(`(^|/)testing/`), regexp.MustCompile(`(^|/)_*mocks?_*`), regexp.MustCompile(`(^|/)fixtures/`), regexp.MustCompile(`\.log$`), @@ -110,7 +113,13 @@ func (path *Path) Exists() bool { return true } -func IterateFilesList(rootDir string, files []string, allowDir AllowDirFunction, visitFile VisitFileFunction) error { +func IterateFilesList( + rootDir string, + files []string, + skipTest bool, + allowDir AllowDirFunction, + visitFile VisitFileFunction, +) error { gitIgnore := getGitIgnore(rootDir) rootDir, err := filepath.Abs(rootDir) @@ -145,6 +154,12 @@ func IterateFilesList(rootDir string, files []string, allowDir AllowDirFunction, } if regex.AnyMatch(ignoredFilenames, relativePath) { + log.Debug().Msgf("%s: skipping due to filename: other", path) + continue + } + + if skipTest && regex.AnyMatch(ignoreTestFiles, relativePath) { + log.Debug().Msgf("%s: skipping due to filename: test", path) continue }