Skip to content

Commit

Permalink
fix: include discarded rules in rule check (#1653)
Browse files Browse the repository at this point in the history
  • Loading branch information
didroe committed Jul 16, 2024
1 parent 292f365 commit a179682
Show file tree
Hide file tree
Showing 5 changed files with 66 additions and 44 deletions.
2 changes: 1 addition & 1 deletion pkg/commands/artifact/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -432,7 +432,7 @@ func (r *runner) Report(
}
}

if len(r.scanSettings.Rules) == 0 && slices.Contains(r.scanSettings.Scan.Scanner, flag.ScannerSAST) && r.scanSettings.Report.Report == flag.ReportSecurity {
if r.scanSettings.LoadedRuleCount == 0 && slices.Contains(r.scanSettings.Scan.Scanner, flag.ScannerSAST) && r.scanSettings.Report.Report == flag.ReportSecurity {
return false, fmt.Errorf("%d rules found for supported language, default rules could not be downloaded or possibly disabled without using --external-rule-dir", len(r.scanSettings.Rules))
}

Expand Down
1 change: 1 addition & 0 deletions pkg/commands/process/settings/loader/loader.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ func FromOptions(
IgnoreGit: opts.GeneralOptions.IgnoreGit,
Policies: policies,
Rules: result.Rules,
LoadedRuleCount: result.LoadedRuleCount,
BuiltInRules: result.BuiltInRules,
CacheUsed: result.CacheUsed,
BearerRulesVersion: result.BearerRulesVersion,
Expand Down
73 changes: 41 additions & 32 deletions pkg/commands/process/settings/rules/loader.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
"gopkg.in/yaml.v3"

"github.com/bearer/bearer/pkg/commands/process/settings"
"github.com/bearer/bearer/pkg/engine"
flagtypes "github.com/bearer/bearer/pkg/flag/types"
"github.com/bearer/bearer/pkg/util/output"
"github.com/bearer/bearer/pkg/version_check"
Expand All @@ -29,14 +28,14 @@ func loadDefinitionsFromRemote(
definitions map[string]settings.RuleDefinition,
options flagtypes.RuleOptions,
versionMeta *version_check.VersionMeta,
) error {
) (int, error) {
if options.DisableDefaultRules {
return nil
return 0, nil
}

if versionMeta.Rules.Version == nil {
log.Debug().Msg("No rule packages found")
return nil
return 0, nil
}

urls := make([]string, 0, len(versionMeta.Rules.Packages))
Expand All @@ -45,91 +44,98 @@ func loadDefinitionsFromRemote(
urls = append(urls, value)
}

if err := readDefinitionsFromUrls(definitions, urls); err != nil {
return fmt.Errorf("loading rules failed: %s", err)
count, err := readDefinitionsFromUrls(definitions, urls)
if err != nil {
return 0, fmt.Errorf("loading rules failed: %s", err)
}

return nil
return count, nil
}

func readDefinitionsFromUrls(ruleDefinitions map[string]settings.RuleDefinition, languageDownloads []string) (err error) {
func readDefinitionsFromUrls(ruleDefinitions map[string]settings.RuleDefinition, languageDownloads []string) (int, error) {
bearerRulesDir := bearerRulesDir()
if _, err := os.Stat(bearerRulesDir); errors.Is(err, os.ErrNotExist) {
err := os.Mkdir(bearerRulesDir, os.ModePerm)
if err != nil {
return fmt.Errorf("could not create bearer-rules directory: %s", err)
return 0, fmt.Errorf("could not create bearer-rules directory: %s", err)
}
}

count := 0
for _, languagePackageUrl := range languageDownloads {
// Prepare filepath
urlHash := md5.Sum([]byte(languagePackageUrl))
filepath, err := filepath.Abs(filepath.Join(bearerRulesDir, fmt.Sprintf("%x.tar.gz", urlHash)))

if err != nil {
return err
return 0, err
}

var languageCount int
if _, err := os.Stat(filepath); err == nil {
log.Trace().Msgf("Using local cache for rule package: %s", languagePackageUrl)
file, err := os.Open(filepath)
if err != nil {
return err
return 0, err
}
defer file.Close()

if err = readRuleDefinitionZip(ruleDefinitions, file); err != nil {
return err
languageCount, err = readRuleDefinitionZip(ruleDefinitions, file)
if err != nil {
return 0, err
}
} else {
log.Trace().Msgf("Downloading rule package: %s", languagePackageUrl)
httpClient := &http.Client{Timeout: 60 * time.Second}
resp, err := httpClient.Get(languagePackageUrl)
if err != nil {
return err
return 0, err
}
defer resp.Body.Close()

// Create file in rules dir
file, err := os.Create(filepath)
if err != nil {
return err
return 0, err
}
defer file.Close()

// Copy the contents of the downloaded archive to the file
if _, err := io.Copy(file, resp.Body); err != nil {
return err
return 0, err
}
// reset file pointer to start of file
_, err = file.Seek(0, 0)
if err != nil {
return err
return 0, err
}

if err = readRuleDefinitionZip(ruleDefinitions, file); err != nil {
return err
languageCount, err = readRuleDefinitionZip(ruleDefinitions, file)
if err != nil {
return 0, err
}
}

count += languageCount
}

return nil
return count, nil
}

func readRuleDefinitionZip(ruleDefinitions map[string]settings.RuleDefinition, file *os.File) error {
func readRuleDefinitionZip(ruleDefinitions map[string]settings.RuleDefinition, file *os.File) (int, error) {
gzr, err := gzip.NewReader(file)
if err != nil {
return err
return 0, err
}
defer gzr.Close()

tr := tar.NewReader(gzr)
count := 0
for {
header, err := tr.Next()
if err == io.EOF {
break
} else if err != nil {
return err
return 0, err
}

if !isRuleFile(header.Name) {
Expand All @@ -139,34 +145,35 @@ func readRuleDefinitionZip(ruleDefinitions map[string]settings.RuleDefinition, f
data := make([]byte, header.Size)
_, err = io.ReadFull(tr, data)
if err != nil {
return fmt.Errorf("failed to read file %s: %w", header.Name, err)
return 0, fmt.Errorf("failed to read file %s: %w", header.Name, err)
}

var ruleDefinition settings.RuleDefinition
err = yaml.Unmarshal(data, &ruleDefinition)
if err != nil {
return fmt.Errorf("failed to unmarshal rule %s: %w", header.Name, err)
return 0, fmt.Errorf("failed to unmarshal rule %s: %w", header.Name, err)
}

id := ruleDefinition.Metadata.ID
_, ruleExists := ruleDefinitions[id]
if ruleExists {
return fmt.Errorf("duplicate built-in rule ID %s", id)
return 0, fmt.Errorf("duplicate built-in rule ID %s", id)
}

ruleDefinitions[id] = ruleDefinition
count += 1
}

return nil
return count, nil
}

func loadCustomDefinitions(
engine engine.Engine,
definitions map[string]settings.RuleDefinition,
isBuiltIn bool,
dir fs.FS,
languageIDs []string,
) error {
) (int, error) {
count := 0
loadedDefinitions := make(map[string]settings.RuleDefinition)
if err := fs.WalkDir(dir, ".", func(path string, dirEntry fs.DirEntry, err error) error {
if err != nil {
Expand Down Expand Up @@ -214,6 +221,8 @@ func loadCustomDefinitions(
}
}

count += 1

if !supported {
log.Debug().Msgf(
"rule file has no supported languages[%s] %s",
Expand All @@ -231,7 +240,7 @@ func loadCustomDefinitions(

return nil
}); err != nil {
return err
return 0, err
}

for id, definition := range loadedDefinitions {
Expand All @@ -240,7 +249,7 @@ func loadCustomDefinitions(
}
}

return nil
return count, nil
}

func bearerRulesDir() string {
Expand Down
17 changes: 14 additions & 3 deletions pkg/commands/process/settings/rules/rules.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ const (
type LoadRulesResult struct {
BuiltInRules map[string]*settings.Rule
Rules map[string]*settings.Rule
LoadedRuleCount int
CacheUsed bool
BearerRulesVersion string
}
Expand Down Expand Up @@ -53,11 +54,16 @@ func Load(

log.Debug().Msg("Loading rules")

if err := loadDefinitionsFromRemote(definitions, options, versionMeta); err != nil {
count := 0

remoteCount, err := loadDefinitionsFromRemote(definitions, options, versionMeta)
if err != nil {
return result, fmt.Errorf("error loading remote rules: %w", err)
}

if err := loadCustomDefinitions(engine, builtInDefinitions, true, builtInRulesFS, nil); err != nil {
count += remoteCount

if _, err := loadCustomDefinitions(builtInDefinitions, true, builtInRulesFS, nil); err != nil {
return result, fmt.Errorf("error loading built-in rules: %w", err)
}

Expand All @@ -66,10 +72,14 @@ func Load(
dirname, _ := os.UserHomeDir()
dir = filepath.Join(dirname, dir[2:])
}

log.Debug().Msgf("loading external rules from: %s", dir)
if err := loadCustomDefinitions(engine, definitions, false, os.DirFS(dir), foundLanguageIDs); err != nil {
externalCount, err := loadCustomDefinitions(definitions, false, os.DirFS(dir), foundLanguageIDs)
if err != nil {
return result, fmt.Errorf("external rules %w", err)
}

count += externalCount
}

if err := validateRuleOptionIDs(options, definitions, builtInDefinitions); err != nil {
Expand All @@ -81,6 +91,7 @@ func Load(

result.Rules = BuildRules(definitions, enabledRules)
result.BuiltInRules = BuildRules(builtInDefinitions, builtInRules)
result.LoadedRuleCount = count

for _, definition := range definitions {
id := definition.Metadata.ID
Expand Down
17 changes: 9 additions & 8 deletions pkg/commands/process/settings/settings.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,15 @@ type Config struct {
Target string `mapstructure:"target" json:"target" yaml:"target"`
IgnoreFile string `mapstructure:"ignore_file" json:"ignore_file" yaml:"ignore_file"`
Rules map[string]*Rule `mapstructure:"rules" json:"rules" yaml:"rules"`
BuiltInRules map[string]*Rule `mapstructure:"built_in_rules" json:"built_in_rules" yaml:"built_in_rules"`
CacheUsed bool `mapstructure:"cache_used" json:"cache_used" yaml:"cache_used"`
BearerRulesVersion string `mapstructure:"bearer_rules_version" json:"bearer_rules_version" yaml:"bearer_rules_version"`
NoColor bool `mapstructure:"no_color" json:"no_color" yaml:"no_color"`
Debug bool `mapstructure:"debug" json:"debug" yaml:"debug"`
LogLevel string `mapstructure:"log_level" json:"log_level" yaml:"log_level"`
DebugProfile bool `mapstructure:"debug_profile" json:"debug_profile" yaml:"debug_profile"`
IgnoreGit bool `mapstructure:"ignore_git" json:"ignore_git" yaml:"ignore_git"`
LoadedRuleCount int
BuiltInRules map[string]*Rule `mapstructure:"built_in_rules" json:"built_in_rules" yaml:"built_in_rules"`
CacheUsed bool `mapstructure:"cache_used" json:"cache_used" yaml:"cache_used"`
BearerRulesVersion string `mapstructure:"bearer_rules_version" json:"bearer_rules_version" yaml:"bearer_rules_version"`
NoColor bool `mapstructure:"no_color" json:"no_color" yaml:"no_color"`
Debug bool `mapstructure:"debug" json:"debug" yaml:"debug"`
LogLevel string `mapstructure:"log_level" json:"log_level" yaml:"log_level"`
DebugProfile bool `mapstructure:"debug_profile" json:"debug_profile" yaml:"debug_profile"`
IgnoreGit bool `mapstructure:"ignore_git" json:"ignore_git" yaml:"ignore_git"`
}

type Processor struct {
Expand Down

0 comments on commit a179682

Please sign in to comment.