Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: allow test files to be scanned #1542

Merged
merged 2 commits into from
Mar 15, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions e2e/flags/.snapshots/TestInitCommand
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,5 @@ scan:
scanner:
- sast
skip-path: []
skip-test: true

1 change: 1 addition & 0 deletions e2e/flags/.snapshots/TestMetadataFlags-help-scan
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ Scan Flags
--quiet Suppress non-essential messages
--scanner strings Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast])
--skip-path strings Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql
--skip-test Disable automatic skipping of test files (default true)

General Flags
--api-key string Use your Bearer API Key to send the report to Bearer.
Expand Down
1 change: 1 addition & 0 deletions e2e/flags/.snapshots/TestMetadataFlags-scan-help
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ Scan Flags
--quiet Suppress non-essential messages
--scanner strings Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast])
--skip-path strings Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql
--skip-test Disable automatic skipping of test files (default true)

General Flags
--api-key string Use your Bearer API Key to send the report to Bearer.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ Scan Flags
--quiet Suppress non-essential messages
--scanner strings Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast])
--skip-path strings Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql
--skip-test Disable automatic skipping of test files (default true)

General Flags
--api-key string Use your Bearer API Key to send the report to Bearer.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ Scan Flags
--quiet Suppress non-essential messages
--scanner strings Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast])
--skip-path strings Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql
--skip-test Disable automatic skipping of test files (default true)

General Flags
--api-key string Use your Bearer API Key to send the report to Bearer.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ Scan Flags
--quiet Suppress non-essential messages
--scanner strings Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast])
--skip-path strings Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql
--skip-test Disable automatic skipping of test files (default true)

General Flags
--api-key string Use your Bearer API Key to send the report to Bearer.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ Scan Flags
--quiet Suppress non-essential messages
--scanner strings Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast])
--skip-path strings Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql
--skip-test Disable automatic skipping of test files (default true)

General Flags
--api-key string Use your Bearer API Key to send the report to Bearer.
Expand Down
3 changes: 3 additions & 0 deletions internal/commands/process/orchestrator/worker/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,13 @@ type Worker struct {
classifer *classification.Classifier
enabledScanners []string
sastScanner *scanner.Scanner
skipTest bool
}

func (worker *Worker) Setup(config config.Config) error {
worker.debug = config.Debug
worker.enabledScanners = config.Scan.Scanner
worker.skipTest = config.Scan.SkipTest

if slices.Contains(worker.enabledScanners, "sast") {
classifier, err := classification.NewClassifier(&classification.Config{Config: config})
Expand Down Expand Up @@ -86,6 +88,7 @@ func (worker *Worker) Scan(ctx context.Context, scanRequest work.ProcessRequest)
fileStats,
worker.enabledScanners,
worker.sastScanner,
worker.skipTest,
)

if ctx.Err() != nil {
Expand Down
4 changes: 4 additions & 0 deletions internal/detectors/detectors.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ func Extract(
fileStats *stats.FileStats,
enabledScanners []string,
sastScanner *scanner.Scanner,
skipTest bool,
) error {
return ExtractWithDetectors(
ctx,
Expand All @@ -149,6 +150,7 @@ func Extract(
fileStats,
Registrations(enabledScanners),
sastScanner,
skipTest,
)
}

Expand All @@ -160,13 +162,15 @@ func ExtractWithDetectors(
fileStats *stats.FileStats,
allDetectors []InitializedDetector,
sastScanner *scanner.Scanner,
skipTest bool,
) error {

activeDetectors := make(map[InitializedDetector]activeDetector)

if err := file.IterateFilesList(
rootDir,
[]string{filename},
skipTest,
func(dir *file.Path) (bool, error) {
for _, detector := range allDetectors {
active, isActive := activeDetectors[detector]
Expand Down
2 changes: 1 addition & 1 deletion internal/detectors/internal/testhelper/testhelper.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func Extract(
}

for _, filename := range files {
err = detectors.ExtractWithDetectors(context.Background(), path, filename, &report, nil, registrations, nil)
err = detectors.ExtractWithDetectors(context.Background(), path, filename, &report, nil, registrations, nil, true)
if !assert.Nil(t, err) {
t.Errorf("report has errored %s", err)
}
Expand Down
7 changes: 7 additions & 0 deletions internal/flag/scan_flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,12 @@ var (
Value: []string{},
Usage: "Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql",
})
SkipTestFlag = ScanFlagGroup.add(flagtypes.Flag{
Name: "skip-test",
ConfigName: "scan.skip-test",
Value: true,
Usage: "Disable automatic skipping of test files",
})
DisableDomainResolutionFlag = ScanFlagGroup.add(flagtypes.Flag{
Name: "disable-domain-resolution",
ConfigName: "scan.disable-domain-resolution",
Expand Down Expand Up @@ -162,6 +168,7 @@ func (scanFlagGroup) SetOptions(options *flagtypes.Options, args []string) error

options.ScanOptions = flagtypes.ScanOptions{
SkipPath: getStringSlice(SkipPathFlag),
SkipTest: getBool(SkipTestFlag),
DisableDomainResolution: getBool(DisableDomainResolutionFlag),
DomainResolutionTimeout: getDuration(DomainResolutionTimeoutFlag),
InternalDomains: getStringSlice(InternalDomainsFlag),
Expand Down
1 change: 1 addition & 0 deletions internal/flag/types/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ type Options struct {

type ScanOptions struct {
Target string `mapstructure:"target" json:"target" yaml:"target"`
SkipTest bool `mapstructure:"skip-test" json:"skip-test" yaml:"skip-test"`
SkipPath []string `mapstructure:"skip-path" json:"skip-path" yaml:"skip-path"`
DisableDomainResolution bool `mapstructure:"disable-domain-resolution" json:"disable-domain-resolution" yaml:"disable-domain-resolution"`
DomainResolutionTimeout time.Duration `mapstructure:"domain-resolution-timeout" json:"domain-resolution-timeout" yaml:"domain-resolution-timeout"`
Expand Down
23 changes: 19 additions & 4 deletions internal/util/file/file.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,16 @@ type Line struct {
Strip bool
}

var ignoredFilenames = []*regexp.Regexp{
regexp.MustCompile(`(^|/)\.git/`),
var ignoreTestFiles = []*regexp.Regexp{
regexp.MustCompile(`(^|/)(?i:_*tests?_*)/`),
regexp.MustCompile(`(^|/)specs?/`),
regexp.MustCompile(`(^|/)testing/`),
regexp.MustCompile(`(^|/|[_-])(spec|test)s?\.`),
regexp.MustCompile(`(?i:unit[-_]?tests?)`),
}

var ignoredFilenames = []*regexp.Regexp{
regexp.MustCompile(`(^|/)\.git/`),
regexp.MustCompile(`(^|/)testing/`),
regexp.MustCompile(`(^|/)_*mocks?_*`),
regexp.MustCompile(`(^|/)fixtures/`),
regexp.MustCompile(`\.log$`),
Expand Down Expand Up @@ -110,7 +113,13 @@ func (path *Path) Exists() bool {
return true
}

func IterateFilesList(rootDir string, files []string, allowDir AllowDirFunction, visitFile VisitFileFunction) error {
func IterateFilesList(
rootDir string,
files []string,
skipTest bool,
allowDir AllowDirFunction,
visitFile VisitFileFunction,
) error {
gitIgnore := getGitIgnore(rootDir)

rootDir, err := filepath.Abs(rootDir)
Expand Down Expand Up @@ -145,6 +154,12 @@ func IterateFilesList(rootDir string, files []string, allowDir AllowDirFunction,
}

if regex.AnyMatch(ignoredFilenames, relativePath) {
log.Debug().Msgf("%s: skipping due to filename: other", path)
continue
}

if skipTest && regex.AnyMatch(ignoreTestFiles, relativePath) {
log.Debug().Msgf("%s: skipping due to filename: test", path)
continue
}

Expand Down
Loading