From 70c7f6dc2baae2a7979d690da2bf08f8e60167c5 Mon Sep 17 00:00:00 2001 From: elsapet Date: Mon, 18 Sep 2023 15:47:46 +0200 Subject: [PATCH] fix: add detector lang to dependency --- .../detectors/dependencies/dependencies.go | 79 +++++-------------- internal/detectors/dependencies/npm/npm.go | 2 +- .../dependencies/package-json/package-json.go | 2 +- .../dependencies/project-json/project-json.go | 2 +- .../dependencies/yarnlock/yarnlock.go | 2 +- internal/report/detections/detections.go | 11 +-- internal/report/detectors/detectors.go | 1 + .../output/dataflow/components/components.go | 25 +++--- .../output/dataflow/types/components.go | 9 ++- internal/report/output/security/security.go | 19 ++--- internal/report/report.go | 2 +- internal/report/writer/detectors.go | 9 ++- 12 files changed, 63 insertions(+), 100 deletions(-) diff --git a/internal/detectors/dependencies/dependencies.go b/internal/detectors/dependencies/dependencies.go index 40565f165..cd36505fe 100644 --- a/internal/detectors/dependencies/dependencies.go +++ b/internal/detectors/dependencies/dependencies.go @@ -40,54 +40,6 @@ func (detector *detector) AcceptDir(dir *file.Path) (bool, error) { return true, nil } -func DetectorsForLanguage(language string) []string { - switch language { - case "C#": - return []string{ - "nuget", - "packageconfig", - "paketdependencies", - } - case "Go": - return []string{ - "gosum", - } - case "Java": - return []string{ - "buildgradle", - "ivy", - "mvnplugin", - "pomxml", - } - case "Javascript": - return []string{ - "npm", - "packagejson", - "projectjson", - "yarnlock", - } - case "PHP": - return []string{ - "composerlock", - "composerjson", - } - case "Python": - return []string{ - "pipdeptree", - "piplock", - "poetry", - "pyproject", - "requirements", - } - case "Ruby": - return []string{ - "gemfile", - } - } - - return []string{} -} - func (detector *detector) ProcessFile(file *file.FileInfo, dir *file.Path, report report.Report) (bool, error) { switch file.Base { case "Gemfile.lock": @@ -145,19 +97,24 @@ func discoverDependency(report report.Report, file *file.FileInfo, discover func for _, dep := range result.Dependencies { startColumnNumber := int(dep.Column) lineNumber := int(dep.Line) - report.AddDependency(detectors.Type(result.Provider), dependencies.Dependency{ - Group: dep.Group, - Name: dep.Name, - Version: dep.Version, - PackageManager: result.PackageManager, - }, source.Source{ - Language: file.Language, - LanguageType: file.LanguageTypeString(), - Filename: file.RelativePath, - StartColumnNumber: &startColumnNumber, - StartLineNumber: &lineNumber, - EndLineNumber: &lineNumber, - }) + report.AddDependency( + detectors.Type(result.Provider), + detectors.Language(result.Language), + dependencies.Dependency{ + Group: dep.Group, + Name: dep.Name, + Version: dep.Version, + PackageManager: result.PackageManager, + }, + source.Source{ + Language: file.Language, + LanguageType: file.LanguageTypeString(), + Filename: file.RelativePath, + StartColumnNumber: &startColumnNumber, + StartLineNumber: &lineNumber, + EndLineNumber: &lineNumber, + }, + ) } return true, nil diff --git a/internal/detectors/dependencies/npm/npm.go b/internal/detectors/dependencies/npm/npm.go index 1f87f88da..0b4209daa 100644 --- a/internal/detectors/dependencies/npm/npm.go +++ b/internal/detectors/dependencies/npm/npm.go @@ -54,7 +54,7 @@ var queryRequires = parser.QueryMustCompile(language, ` func Discover(f *file.FileInfo) (report *depsbase.DiscoveredDependency) { report = &depsbase.DiscoveredDependency{} report.Provider = "npm" - report.Language = "Javascript" + report.Language = "JavaScript" report.PackageManager = "npm" tree, err := parser.ParseFile(f, f.Path, language) if err != nil { diff --git a/internal/detectors/dependencies/package-json/package-json.go b/internal/detectors/dependencies/package-json/package-json.go index 69bbd838b..f7416faf2 100644 --- a/internal/detectors/dependencies/package-json/package-json.go +++ b/internal/detectors/dependencies/package-json/package-json.go @@ -30,7 +30,7 @@ var queryDependencies = parser.QueryMustCompile(language, ` func Discover(f *file.FileInfo) (report *depsbase.DiscoveredDependency) { report = &depsbase.DiscoveredDependency{} report.Provider = "package-json" - report.Language = "Javascript" + report.Language = "JavaScript" report.PackageManager = "npm" tree, err := parser.ParseFile(f, f.Path, language) if err != nil { diff --git a/internal/detectors/dependencies/project-json/project-json.go b/internal/detectors/dependencies/project-json/project-json.go index f18f13ccd..272b2b5ff 100644 --- a/internal/detectors/dependencies/project-json/project-json.go +++ b/internal/detectors/dependencies/project-json/project-json.go @@ -30,7 +30,7 @@ var queryDependencies = parser.QueryMustCompile(language, ` func Discover(f *file.FileInfo) (report *depsbase.DiscoveredDependency) { report = &depsbase.DiscoveredDependency{} report.Provider = "package-json" - report.Language = "Javascript" + report.Language = "JavaScript" report.PackageManager = "nuget" tree, err := parser.ParseFile(f, f.Path, language) if err != nil { diff --git a/internal/detectors/dependencies/yarnlock/yarnlock.go b/internal/detectors/dependencies/yarnlock/yarnlock.go index cef65407a..f006d6326 100644 --- a/internal/detectors/dependencies/yarnlock/yarnlock.go +++ b/internal/detectors/dependencies/yarnlock/yarnlock.go @@ -22,7 +22,7 @@ var dependencyLockedVersionRegexp *regexp.Regexp func Discover(f *file.FileInfo) (report *depsbase.DiscoveredDependency) { report = &depsbase.DiscoveredDependency{} report.Provider = "yarn.lock" - report.Language = "Javascript" + report.Language = "JavaScript" report.PackageManager = "npm" fileBytes, err := os.ReadFile(f.AbsolutePath) diff --git a/internal/report/detections/detections.go b/internal/report/detections/detections.go index 3b03df5a0..aed594aef 100644 --- a/internal/report/detections/detections.go +++ b/internal/report/detections/detections.go @@ -62,9 +62,10 @@ type FrameworkDetection struct { } type Detection struct { - Type DetectionType `json:"type" yaml:"type"` - DetectorType detectors.Type `json:"detector_type" yaml:"detector_type"` - CommitSHA string `json:"commit_sha,omitempty" yaml:"commit_sha,omitempty"` - Source source.Source `json:"source" yaml:"source"` - Value interface{} `json:"value" yaml:"value"` + Type DetectionType `json:"type" yaml:"type"` + DetectorType detectors.Type `json:"detector_type" yaml:"detector_type"` + DetectorLanguage detectors.Language `json:"detector_language,omitempty" yaml:"detector_language,omitempty"` + CommitSHA string `json:"commit_sha,omitempty" yaml:"commit_sha,omitempty"` + Source source.Source `json:"source" yaml:"source"` + Value interface{} `json:"value" yaml:"value"` } diff --git a/internal/report/detectors/detectors.go b/internal/report/detectors/detectors.go index 22b52d431..5311ef80d 100644 --- a/internal/report/detectors/detectors.go +++ b/internal/report/detectors/detectors.go @@ -1,6 +1,7 @@ package detectors type Type string +type Language string const ( DetectorDependencies Type = "dependencies" diff --git a/internal/report/output/dataflow/components/components.go b/internal/report/output/dataflow/components/components.go index 9c5357929..dc9bbed91 100644 --- a/internal/report/output/dataflow/components/components.go +++ b/internal/report/output/dataflow/components/components.go @@ -20,9 +20,10 @@ type Holder struct { } type dependency struct { - name string - filename string - version string + name string + filename string + version string + detectorLanguage string } type component struct { @@ -103,6 +104,7 @@ func (holder *Holder) AddDependency(classifiedDetection dependenciesclassificati holder.addDependency( string(classifiedDetection.DetectorType), + string(classifiedDetection.DetectorLanguage), classifiedDetection.Source.Filename, name, version, @@ -163,6 +165,7 @@ func (holder *Holder) AddFramework(classifiedDetection frameworkclassification.C // addComponent adds component to hash list and at the same time blocks duplicates func (holder *Holder) addDependency( detectorName string, + detectorLanguage string, fileName string, name string, version string, @@ -174,9 +177,10 @@ func (holder *Holder) addDependency( holder.dependencies[detectorName] = append( holder.dependencies[detectorName], &dependency{ - name: name, - version: version, - filename: fileName, + name: name, + version: version, + filename: fileName, + detectorLanguage: detectorLanguage, }, ) } @@ -235,10 +239,11 @@ func (holder *Holder) ToDataFlowForDependencies() []types.Dependency { for detectorName, dependencies := range holder.dependencies { for _, dependency := range dependencies { data = append(data, types.Dependency{ - Name: dependency.name, - Version: dependency.version, - Filename: dependency.filename, - Detector: detectorName, + Name: dependency.name, + Version: dependency.version, + Filename: dependency.filename, + Detector: detectorName, + DetectorLanguage: dependency.detectorLanguage, }) } } diff --git a/internal/report/output/dataflow/types/components.go b/internal/report/output/dataflow/types/components.go index c9434d1b5..41119c4d5 100644 --- a/internal/report/output/dataflow/types/components.go +++ b/internal/report/output/dataflow/types/components.go @@ -9,10 +9,11 @@ type Component struct { } type Dependency struct { - Name string `json:"name" yaml:"name"` - Version string `json:"version" yaml:"version"` - Filename string `json:"filename" yaml:"filename"` - Detector string `json:"detector" yaml:"detector"` + Name string `json:"name" yaml:"name"` + Version string `json:"version" yaml:"version"` + Filename string `json:"filename" yaml:"filename"` + Detector string `json:"detector" yaml:"detector"` + DetectorLanguage string `json:"-" yaml:"-"` } type ComponentLocation struct { diff --git a/internal/report/output/security/security.go b/internal/report/output/security/security.go index 7c5620f46..59c2d3590 100644 --- a/internal/report/output/security/security.go +++ b/internal/report/output/security/security.go @@ -17,7 +17,6 @@ import ( "github.com/bearer/bearer/internal/classification/db" "github.com/bearer/bearer/internal/commands/process/settings" - "github.com/bearer/bearer/internal/detectors/dependencies" "github.com/bearer/bearer/internal/report/basebranchfindings" globaltypes "github.com/bearer/bearer/internal/types" "github.com/bearer/bearer/internal/util/file" @@ -548,29 +547,21 @@ func writeRuleListToString( sort.Slice(languageSlice, func(i, j int) bool { return len(languageSlice[i].Files) > len(languageSlice[j].Files) }) - unsupportedLanguages := make(map[string]int) + unsupportedLanguages := make(map[string]bool) for _, lang := range languageSlice { if ruleCount, ok := ruleCountPerLang[lang.Name]; ok { tbl.AddRow(lang.Name, ruleCount.DefaultRuleCount, ruleCount.CustomRuleCount, len(languages[lang.Name].Files)) } else { - for _, detector := range dependencies.DetectorsForLanguage(lang.Name) { - if _, ok := unsupportedLanguages[lang.Name]; ok { + for _, reportedDependency := range reportedDependencies { + if unsupportedLanguages[reportedDependency.DetectorLanguage] { break } - for _, reportedDependency := range reportedDependencies { - if reportedDependency.Detector == detector { - unsupportedLanguages[lang.Name] = len(languages[lang.Name].Files) - break - } - } + unsupportedLanguages[lang.Name] = true + tbl.AddRow(lang.Name, 0, 0, len(languages[lang.Name].Files)) } } } - for language, filesCount := range unsupportedLanguages { - tbl.AddRow(language, 0, 0, filesCount) - } - tbl.Print() if len(unsupportedLanguages) > 0 { diff --git a/internal/report/report.go b/internal/report/report.go index 44185cfe8..2a12789ae 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -19,7 +19,7 @@ type Report interface { datatype.ReportDataType AddInterface(detectorType detectors.Type, data interfaces.Interface, source source.Source) AddFramework(detectorType detectors.Type, frameworkType frameworks.Type, data interface{}, source source.Source) - AddDependency(detectorType detectors.Type, dependency dependencies.Dependency, source source.Source) + AddDependency(detectorType detectors.Type, detectorLanguage detectors.Language, dependency dependencies.Dependency, source source.Source) AddSecretLeak(secret secret.Secret, source source.Source) AddError(filePath string, err error) } diff --git a/internal/report/writer/detectors.go b/internal/report/writer/detectors.go index ba992c801..95196e425 100644 --- a/internal/report/writer/detectors.go +++ b/internal/report/writer/detectors.go @@ -176,11 +176,18 @@ func (report *Detectors) AddDetection(detectionType detections.DetectionType, de func (report *Detectors) AddDependency( detectorType detectors.Type, + detectorLanguage detectors.Language, dependency dependencies.Dependency, source source.Source, ) { - detection := &detections.Detection{DetectorType: detectorType, Value: dependency, Source: source, Type: detections.TypeDependency} + detection := &detections.Detection{ + DetectorType: detectorType, + DetectorLanguage: detectorLanguage, + Value: dependency, + Source: source, + Type: detections.TypeDependency, + } classifiedDetection, err := report.Classifier.Dependencies.Classify(*detection) if err != nil { report.AddError(source.Filename, fmt.Errorf("classification dependencies error: %s", err))