Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: include discarded rules in rule check #1653

Merged
merged 1 commit into from
Jul 16, 2024
Merged

fix: include discarded rules in rule check #1653

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pkg/commands/artifact/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -432,7 +432,7 @@ func (r *runner) Report(
}
}

if len(r.scanSettings.Rules) == 0 && slices.Contains(r.scanSettings.Scan.Scanner, flag.ScannerSAST) && r.scanSettings.Report.Report == flag.ReportSecurity {
if r.scanSettings.LoadedRuleCount == 0 && slices.Contains(r.scanSettings.Scan.Scanner, flag.ScannerSAST) && r.scanSettings.Report.Report == flag.ReportSecurity {
return false, fmt.Errorf("%d rules found for supported language, default rules could not be downloaded or possibly disabled without using --external-rule-dir", len(r.scanSettings.Rules))
}

Expand Down
1 change: 1 addition & 0 deletions pkg/commands/process/settings/loader/loader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ func FromOptions(
IgnoreGit: opts.GeneralOptions.IgnoreGit,
Policies: policies,
Rules: result.Rules,
LoadedRuleCount: result.LoadedRuleCount,
BuiltInRules: result.BuiltInRules,
CacheUsed: result.CacheUsed,
BearerRulesVersion: result.BearerRulesVersion,
Expand Down
73 changes: 41 additions & 32 deletions pkg/commands/process/settings/rules/loader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
"gopkg.in/yaml.v3"

"github.com/bearer/bearer/pkg/commands/process/settings"
"github.com/bearer/bearer/pkg/engine"
flagtypes "github.com/bearer/bearer/pkg/flag/types"
"github.com/bearer/bearer/pkg/util/output"
"github.com/bearer/bearer/pkg/version_check"
Expand All @@ -29,14 +28,14 @@ func loadDefinitionsFromRemote(
definitions map[string]settings.RuleDefinition,
options flagtypes.RuleOptions,
versionMeta *version_check.VersionMeta,
) error {
) (int, error) {
if options.DisableDefaultRules {
return nil
return 0, nil
}

if versionMeta.Rules.Version == nil {
log.Debug().Msg("No rule packages found")
return nil
return 0, nil
}

urls := make([]string, 0, len(versionMeta.Rules.Packages))
Expand All @@ -45,91 +44,98 @@ func loadDefinitionsFromRemote(
urls = append(urls, value)
}

if err := readDefinitionsFromUrls(definitions, urls); err != nil {
return fmt.Errorf("loading rules failed: %s", err)
count, err := readDefinitionsFromUrls(definitions, urls)
if err != nil {
return 0, fmt.Errorf("loading rules failed: %s", err)
}

return nil
return count, nil
}

func readDefinitionsFromUrls(ruleDefinitions map[string]settings.RuleDefinition, languageDownloads []string) (err error) {
func readDefinitionsFromUrls(ruleDefinitions map[string]settings.RuleDefinition, languageDownloads []string) (int, error) {
bearerRulesDir := bearerRulesDir()
if _, err := os.Stat(bearerRulesDir); errors.Is(err, os.ErrNotExist) {
err := os.Mkdir(bearerRulesDir, os.ModePerm)
if err != nil {
return fmt.Errorf("could not create bearer-rules directory: %s", err)
return 0, fmt.Errorf("could not create bearer-rules directory: %s", err)
}
}

count := 0
for _, languagePackageUrl := range languageDownloads {
// Prepare filepath
urlHash := md5.Sum([]byte(languagePackageUrl))
filepath, err := filepath.Abs(filepath.Join(bearerRulesDir, fmt.Sprintf("%x.tar.gz", urlHash)))

if err != nil {
return err
return 0, err
}

var languageCount int
if _, err := os.Stat(filepath); err == nil {
log.Trace().Msgf("Using local cache for rule package: %s", languagePackageUrl)
file, err := os.Open(filepath)
if err != nil {
return err
return 0, err
}
defer file.Close()

if err = readRuleDefinitionZip(ruleDefinitions, file); err != nil {
return err
languageCount, err = readRuleDefinitionZip(ruleDefinitions, file)
if err != nil {
return 0, err
}
} else {
log.Trace().Msgf("Downloading rule package: %s", languagePackageUrl)
httpClient := &http.Client{Timeout: 60 * time.Second}
resp, err := httpClient.Get(languagePackageUrl)
if err != nil {
return err
return 0, err
}
defer resp.Body.Close()

// Create file in rules dir
file, err := os.Create(filepath)
if err != nil {
return err
return 0, err
}
defer file.Close()

// Copy the contents of the downloaded archive to the file
if _, err := io.Copy(file, resp.Body); err != nil {
return err
return 0, err
}
// reset file pointer to start of file
_, err = file.Seek(0, 0)
if err != nil {
return err
return 0, err
}

if err = readRuleDefinitionZip(ruleDefinitions, file); err != nil {
return err
languageCount, err = readRuleDefinitionZip(ruleDefinitions, file)
if err != nil {
return 0, err
}
}

count += languageCount
}

return nil
return count, nil
}

func readRuleDefinitionZip(ruleDefinitions map[string]settings.RuleDefinition, file *os.File) error {
func readRuleDefinitionZip(ruleDefinitions map[string]settings.RuleDefinition, file *os.File) (int, error) {
gzr, err := gzip.NewReader(file)
if err != nil {
return err
return 0, err
}
defer gzr.Close()

tr := tar.NewReader(gzr)
count := 0
for {
header, err := tr.Next()
if err == io.EOF {
break
} else if err != nil {
return err
return 0, err
}

if !isRuleFile(header.Name) {
Expand All @@ -139,34 +145,35 @@ func readRuleDefinitionZip(ruleDefinitions map[string]settings.RuleDefinition, f
data := make([]byte, header.Size)
_, err = io.ReadFull(tr, data)
if err != nil {
return fmt.Errorf("failed to read file %s: %w", header.Name, err)
return 0, fmt.Errorf("failed to read file %s: %w", header.Name, err)
}

var ruleDefinition settings.RuleDefinition
err = yaml.Unmarshal(data, &ruleDefinition)
if err != nil {
return fmt.Errorf("failed to unmarshal rule %s: %w", header.Name, err)
return 0, fmt.Errorf("failed to unmarshal rule %s: %w", header.Name, err)
}

id := ruleDefinition.Metadata.ID
_, ruleExists := ruleDefinitions[id]
if ruleExists {
return fmt.Errorf("duplicate built-in rule ID %s", id)
return 0, fmt.Errorf("duplicate built-in rule ID %s", id)
}

ruleDefinitions[id] = ruleDefinition
count += 1
}

return nil
return count, nil
}

func loadCustomDefinitions(
engine engine.Engine,
definitions map[string]settings.RuleDefinition,
isBuiltIn bool,
dir fs.FS,
languageIDs []string,
) error {
) (int, error) {
count := 0
loadedDefinitions := make(map[string]settings.RuleDefinition)
if err := fs.WalkDir(dir, ".", func(path string, dirEntry fs.DirEntry, err error) error {
if err != nil {
Expand Down Expand Up @@ -214,6 +221,8 @@ func loadCustomDefinitions(
}
}

count += 1

if !supported {
log.Debug().Msgf(
"rule file has no supported languages[%s] %s",
Expand All @@ -231,7 +240,7 @@ func loadCustomDefinitions(

return nil
}); err != nil {
return err
return 0, err
}

for id, definition := range loadedDefinitions {
Expand All @@ -240,7 +249,7 @@ func loadCustomDefinitions(
}
}

return nil
return count, nil
}

func bearerRulesDir() string {
Expand Down
17 changes: 14 additions & 3 deletions pkg/commands/process/settings/rules/rules.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ const (
type LoadRulesResult struct {
BuiltInRules map[string]*settings.Rule
Rules map[string]*settings.Rule
LoadedRuleCount int
CacheUsed bool
BearerRulesVersion string
}
Expand Down Expand Up @@ -53,11 +54,16 @@ func Load(

log.Debug().Msg("Loading rules")

if err := loadDefinitionsFromRemote(definitions, options, versionMeta); err != nil {
count := 0

remoteCount, err := loadDefinitionsFromRemote(definitions, options, versionMeta)
if err != nil {
return result, fmt.Errorf("error loading remote rules: %w", err)
}

if err := loadCustomDefinitions(engine, builtInDefinitions, true, builtInRulesFS, nil); err != nil {
count += remoteCount

if _, err := loadCustomDefinitions(builtInDefinitions, true, builtInRulesFS, nil); err != nil {
return result, fmt.Errorf("error loading built-in rules: %w", err)
}

Expand All @@ -66,10 +72,14 @@ func Load(
dirname, _ := os.UserHomeDir()
dir = filepath.Join(dirname, dir[2:])
}

log.Debug().Msgf("loading external rules from: %s", dir)
if err := loadCustomDefinitions(engine, definitions, false, os.DirFS(dir), foundLanguageIDs); err != nil {
externalCount, err := loadCustomDefinitions(definitions, false, os.DirFS(dir), foundLanguageIDs)
if err != nil {
return result, fmt.Errorf("external rules %w", err)
}

count += externalCount
}

if err := validateRuleOptionIDs(options, definitions, builtInDefinitions); err != nil {
Expand All @@ -81,6 +91,7 @@ func Load(

result.Rules = BuildRules(definitions, enabledRules)
result.BuiltInRules = BuildRules(builtInDefinitions, builtInRules)
result.LoadedRuleCount = count

for _, definition := range definitions {
id := definition.Metadata.ID
Expand Down
17 changes: 9 additions & 8 deletions pkg/commands/process/settings/settings.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,15 @@ type Config struct {
Target string `mapstructure:"target" json:"target" yaml:"target"`
IgnoreFile string `mapstructure:"ignore_file" json:"ignore_file" yaml:"ignore_file"`
Rules map[string]*Rule `mapstructure:"rules" json:"rules" yaml:"rules"`
BuiltInRules map[string]*Rule `mapstructure:"built_in_rules" json:"built_in_rules" yaml:"built_in_rules"`
CacheUsed bool `mapstructure:"cache_used" json:"cache_used" yaml:"cache_used"`
BearerRulesVersion string `mapstructure:"bearer_rules_version" json:"bearer_rules_version" yaml:"bearer_rules_version"`
NoColor bool `mapstructure:"no_color" json:"no_color" yaml:"no_color"`
Debug bool `mapstructure:"debug" json:"debug" yaml:"debug"`
LogLevel string `mapstructure:"log_level" json:"log_level" yaml:"log_level"`
DebugProfile bool `mapstructure:"debug_profile" json:"debug_profile" yaml:"debug_profile"`
IgnoreGit bool `mapstructure:"ignore_git" json:"ignore_git" yaml:"ignore_git"`
LoadedRuleCount int
BuiltInRules map[string]*Rule `mapstructure:"built_in_rules" json:"built_in_rules" yaml:"built_in_rules"`
CacheUsed bool `mapstructure:"cache_used" json:"cache_used" yaml:"cache_used"`
BearerRulesVersion string `mapstructure:"bearer_rules_version" json:"bearer_rules_version" yaml:"bearer_rules_version"`
NoColor bool `mapstructure:"no_color" json:"no_color" yaml:"no_color"`
Debug bool `mapstructure:"debug" json:"debug" yaml:"debug"`
LogLevel string `mapstructure:"log_level" json:"log_level" yaml:"log_level"`
DebugProfile bool `mapstructure:"debug_profile" json:"debug_profile" yaml:"debug_profile"`
IgnoreGit bool `mapstructure:"ignore_git" json:"ignore_git" yaml:"ignore_git"`
}

type Processor struct {
Expand Down
Loading