Skip to content

Commit

Permalink
feat: detect at match node rather than root node (#466)
Browse files Browse the repository at this point in the history 
* refactor: determine match node at pattern compilation time

* feat: make queries match at match node rather than root node

* fix: add missing error handling

* refactor: make detectors only return detection data
  • Loading branch information
@didroe
didroe authored Jan 30, 2023
1 parent bd6a9a5 commit 1d74d73
Show file tree
Hide file tree
Showing 33 changed files with 252 additions and 376 deletions.
102 changes: 49 additions & 53 deletions new/detector/composition/composition.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,83 +12,79 @@ import (
"github.com/bearer/curio/pkg/report/detectors"
"github.com/bearer/curio/pkg/report/schema"
"github.com/bearer/curio/pkg/report/source"
"github.com/bearer/curio/pkg/util/file"
)

func ReportDetections(report reportdetections.ReportDetection, compositionDetections []*detectortypes.CompositionDetection) {
func ReportDetections(report reportdetections.ReportDetection, file *file.FileInfo, detections []*detectortypes.Detection) {
pluralizer := pluralize.NewClient()

for _, compositionDetection := range compositionDetections {
for _, detection := range detections {
data := detection.Data.(custom.Data)

for _, detection := range compositionDetection.Detections {
if len(data.Datatypes) == 0 {
report.AddDetection(reportdetections.TypeCustomRisk,
detectors.Type(detection.DetectorType),
source.New(
file,
file.Path,
detection.MatchNode.LineNumber(),
detection.MatchNode.ColumnNumber(),
data.Pattern,
),
schema.Parent{
LineNumber: detection.MatchNode.LineNumber(),
Content: detection.MatchNode.Content(),
})
}

data := detection.Data.(custom.Data)
for _, datatypeDetection := range data.Datatypes {
datatypeData := datatypeDetection.Data.(datatype.Data)

if len(data.Datatypes) == 0 {
report.AddDetection(reportdetections.TypeCustomRisk,
detectors.Type(compositionDetection.RuleName),
source.New(
compositionDetection.File,
compositionDetection.File.Path,
detection.MatchNode.LineNumber(),
detection.MatchNode.ColumnNumber(),
data.Pattern,
),
schema.Parent{
report.AddDetection(
reportdetections.TypeCustomClassified,
detectors.Type(detection.DetectorType),
source.New(
file,
file.Path,
datatypeDetection.MatchNode.LineNumber(),
datatypeDetection.MatchNode.ColumnNumber(),
"",
),
schema.Schema{
ObjectName: datatypeData.Name,
NormalizedObjectName: pluralizer.Singular(strings.ToLower(datatypeData.Name)),
Classification: datatypeData.Classification,
Parent: &schema.Parent{
LineNumber: detection.MatchNode.LineNumber(),
Content: detection.MatchNode.Content(),
})
}
},
},
)

for _, datatypeDetection := range data.Datatypes {
datatypeData := datatypeDetection.Data.(datatype.Data)
for _, property := range datatypeData.Properties {

report.AddDetection(
reportdetections.TypeCustomClassified,
detectors.Type(compositionDetection.RuleName),
detectors.Type(detection.DetectorType),
source.New(
compositionDetection.File,
compositionDetection.File.Path,
datatypeDetection.MatchNode.LineNumber(),
datatypeDetection.MatchNode.ColumnNumber(),
file,
file.Path,
property.Detection.MatchNode.LineNumber(),
property.Detection.MatchNode.ColumnNumber(),
"",
),
schema.Schema{
ObjectName: datatypeData.Name,
NormalizedObjectName: pluralizer.Singular(strings.ToLower(datatypeData.Name)),
Classification: datatypeData.Classification,
NormalizedObjectName: pluralizer.Singular(strings.ToLower(property.Name)),
FieldName: property.Name,
NormalizedFieldName: pluralizer.Singular(strings.ToLower(property.Name)),
Classification: property.Classification,
Parent: &schema.Parent{
LineNumber: detection.MatchNode.LineNumber(),
Content: detection.MatchNode.Content(),
},
},
)

for _, property := range datatypeData.Properties {

report.AddDetection(
reportdetections.TypeCustomClassified,
detectors.Type(compositionDetection.RuleName),
source.New(
compositionDetection.File,
compositionDetection.File.Path,
property.Detection.MatchNode.LineNumber(),
property.Detection.MatchNode.ColumnNumber(),
"",
),
schema.Schema{
ObjectName: datatypeData.Name,
NormalizedObjectName: pluralizer.Singular(strings.ToLower(property.Name)),
FieldName: property.Name,
NormalizedFieldName: pluralizer.Singular(strings.ToLower(property.Name)),
Classification: property.Classification,
Parent: &schema.Parent{
LineNumber: detection.MatchNode.LineNumber(),
Content: detection.MatchNode.Content(),
},
},
)

}
}
}
}
Expand Down
100 changes: 41 additions & 59 deletions new/detector/composition/ruby/.snapshots/TestRuby-object.rb
View file
Original file line number Diff line number Diff line change
@@ -1,72 +1,54 @@
[
{
"RuleName": "logger",
"File": {
"AbsolutePath": "/Users/vjeranfistric/go/src/github.com/bearer/curio/new/detector/composition/ruby/testdata/testcases/object.rb",
"RelativePath": "object.rb",
"FileInfo": {},
"IsGenerated": false,
"Extension": ".rb",
"Base": "object.rb",
"IsConfiguration": false,
"IsDotFile": false,
"IsVendor": false,
"Language": "Ruby",
"LanguageType": 2
},
"Detections": [
{
"MatchNode": {},
"ContextNode": null,
"Data": {
"Pattern": "logger.info($\u003cDATA_TYPE\u003e)\n",
"Datatypes": [
{
"MatchNode": {},
"ContextNode": {},
"Data": {
"Name": "user",
"DetectorType": "logger",
"MatchNode": {},
"Data": {
"Pattern": "logger.info($\u003cDATA_TYPE\u003e)\n",
"Datatypes": [
{
"DetectorType": "datatype",
"MatchNode": {},
"Data": {
"Name": "user",
"Classification": {
"name": "user",
"data_type": {
"name": "Unique Identifier",
"uuid": "12d44ae0-1df7-4faf-9fb1-b46cc4b4dce9",
"category_uuid": "14124881-6b92-4fc5-8005-ea7c1c09592e"
},
"decision": {
"state": "valid",
"reason": "valid_object_with_valid_properties"
}
},
"Properties": [
{
"Name": "name",
"Detection": {
"DetectorType": "object",
"MatchNode": {},
"Data": {
"Name": "name"
}
},
"Classification": {
"name": "user",
"name": "name",
"data_type": {
"name": "Unique Identifier",
"uuid": "12d44ae0-1df7-4faf-9fb1-b46cc4b4dce9",
"name": "Fullname",
"uuid": "1617291b-bc22-4267-ad5e-8054b2505958",
"category_uuid": "14124881-6b92-4fc5-8005-ea7c1c09592e"
},
"decision": {
"state": "valid",
"reason": "valid_object_with_valid_properties"
}
},
"Properties": [
{
"Name": "name",
"Detection": {
"MatchNode": {},
"ContextNode": null,
"Data": {
"Name": "name"
}
},
"Classification": {
"name": "name",
"data_type": {
"name": "Fullname",
"uuid": "1617291b-bc22-4267-ad5e-8054b2505958",
"category_uuid": "14124881-6b92-4fc5-8005-ea7c1c09592e"
},
"decision": {
"state": "valid",
"reason": "known_pattern"
}
}
"reason": "known_pattern"
}
]
}
}
}
]
]
}
}
}
]
]
}
}
]
21 changes: 5 additions & 16 deletions new/detector/composition/ruby/ruby.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"github.com/bearer/curio/new/detector/implementation/ruby/object"
"github.com/bearer/curio/new/detector/implementation/ruby/property"
"github.com/bearer/curio/new/language"
"github.com/bearer/curio/new/language/tree"

"github.com/bearer/curio/pkg/classification"
"github.com/bearer/curio/pkg/commands/process/settings"
Expand Down Expand Up @@ -121,7 +120,7 @@ func (composition *Composition) Close() {
}
}

func (composition *Composition) DetectFromFile(file *file.FileInfo) ([]*detectortypes.CompositionDetection, error) {
func (composition *Composition) DetectFromFile(file *file.FileInfo) ([]*detectortypes.Detection, error) {
if file.Language != "Ruby" {
return nil, nil
}
Expand All @@ -138,25 +137,15 @@ func (composition *Composition) DetectFromFile(file *file.FileInfo) ([]*detector

evaluator := evaluator.New(composition.lang, composition.detectorSet, tree, file.FileInfo.Name())

return composition.extractCustomDetectors(evaluator, tree, file)

}

func (composition *Composition) extractCustomDetectors(evaluator detectortypes.Evaluator, tree *tree.Tree, file *file.FileInfo) ([]*detectortypes.CompositionDetection, error) {
customDetections := []*detectortypes.CompositionDetection{}

var result []*detectortypes.Detection
for _, detectorType := range composition.customDetectorTypes {
detections, err := evaluator.ForTree(tree.RootNode(), detectorType)
detections, err := evaluator.ForTree(tree.RootNode(), detectorType, false)
if err != nil {
return nil, err
}

customDetections = append(customDetections, &detectortypes.CompositionDetection{
RuleName: detectorType,
File: file,
Detections: detections,
})
result = append(result, detections...)
}

return customDetections, nil
return result, nil
}
51 changes: 32 additions & 19 deletions new/detector/evaluator/evaluator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,28 +40,34 @@ func (evaluator *evaluator) FileName() string {
return evaluator.fileName
}

func (evaluator *evaluator) ForTree(rootNode *langtree.Node, detectorType string) ([]*types.Detection, error) {
func (evaluator *evaluator) ForTree(
rootNode *langtree.Node,
detectorType string,
followFlow bool,
) ([]*types.Detection, error) {
var result []*types.Detection

if err := rootNode.Walk(func(node *langtree.Node, visitChildren func() error) error {
if !node.Equal(rootNode) && !evaluator.lang.DescendIntoDetectionNode(node) {
return nil
}

detections, err := evaluator.nonUnifiedNodeDetections(node, detectorType)
if err != nil {
return err
}

result = append(result, detections...)

for _, unifiedNode := range node.UnifiedNodes() {
unifiedNodeDetections, err := evaluator.ForTree(unifiedNode, detectorType)
if err != nil {
return err
}

result = append(result, unifiedNodeDetections...)
}
if followFlow {
for _, unifiedNode := range node.UnifiedNodes() {
unifiedNodeDetections, err := evaluator.ForTree(unifiedNode, detectorType, true)
if err != nil {
return err
}

if evaluator.lang.IsTerminalDetectionNode(node) {
return nil
result = append(result, unifiedNodeDetections...)
}
}

return visitChildren()
Expand All @@ -75,19 +81,22 @@ func (evaluator *evaluator) ForTree(rootNode *langtree.Node, detectorType string
func (evaluator *evaluator) ForNode(
node *langtree.Node,
detectorType string,
followFlow bool,
) ([]*types.Detection, error) {
detections, err := evaluator.nonUnifiedNodeDetections(node, detectorType)
if err != nil {
return nil, err
}

for _, unifiedNode := range node.UnifiedNodes() {
unifiedNodeDetections, err := evaluator.ForNode(unifiedNode, detectorType)
if err != nil {
return nil, err
}
if followFlow {
for _, unifiedNode := range node.UnifiedNodes() {
unifiedNodeDetections, err := evaluator.ForNode(unifiedNode, detectorType, true)
if err != nil {
return nil, err
}

detections = append(detections, unifiedNodeDetections...)
detections = append(detections, unifiedNodeDetections...)
}
}

return detections, nil
Expand All @@ -111,7 +120,11 @@ func (evaluator *evaluator) nonUnifiedNodeDetections(
return detections, nil
}

evaluator.detectAtNode(node, detectorType) //nolint:errcheck
err := evaluator.detectAtNode(node, detectorType)
if err != nil {
return nil, err
}

return nodeDetections[detectorType], nil
}

Expand All @@ -138,7 +151,7 @@ func (evaluator *evaluator) TreeHas(rootNode *langtree.Node, detectorType string
}

func (evaluator *evaluator) NodeHas(node *langtree.Node, detectorType string) (bool, error) {
detections, err := evaluator.ForNode(node, detectorType)
detections, err := evaluator.ForNode(node, detectorType, true)
if err != nil {
return false, err
}
Expand Down
Loading

0 comments on commit 1d74d73

Please sign in to comment.