Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(recipes): add exclusion patterns when wildcard is used #372

Merged
merged 2 commits into from
Jan 11, 2023
Merged

feat(recipes): add exclusion patterns when wildcard is used #372

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
73fbf52
feat(recipes): add exclusion patterns when wildcard is used
cfabianski Jan 10, 2023
2e450c2
refactor: update interfaces to return only recipeMatch
cfabianski Jan 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions pkg/classification/db/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,13 @@ type DefaultDB struct {
}

type Recipe struct {
URLS []string `json:"urls" yaml:"urls"`
Name string `json:"name" yaml:"name"`
Type string `json:"type" yaml:"type"`
SubType string `json:"sub_type" yaml:"sub_type"`
Packages []Package `json:"packages" yaml:"packages"`
UUID string `json:"uuid" yaml:"uuid"`
URLS []string `json:"urls" yaml:"urls"`
ExcludeURLS []string `json:"exclude_urls" yaml:"exclude_urls"`
Name string `json:"name" yaml:"name"`
Type string `json:"type" yaml:"type"`
SubType string `json:"sub_type" yaml:"sub_type"`
Packages []Package `json:"packages" yaml:"packages"`
UUID string `json:"uuid" yaml:"uuid"`
}

type Package struct {
Expand Down
5 changes: 2 additions & 3 deletions pkg/classification/db/recipes/google_cloud_apis.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@
"metadata": { "version": "1.0" },
"name": "Google Cloud APIs",
"type": "external_service",
"urls": [
"https://*.googleapis.com"
],
"urls": ["https://*.googleapis.com"],
"exclude_urls": ["https://ajax.googleapis.com"],
"packages": [
{
"name": "Google.Apis",
Expand Down
63 changes: 53 additions & 10 deletions pkg/classification/interfaces/interfaces.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,12 @@ type Config struct {
}

type Recipe struct {
UUID string
Name string
Type string
SubType string
URLS []RecipeURL
UUID string
Name string
Type string
SubType string
URLS []RecipeURL
ExcludeURLS []RecipeURL
}

type RecipeURL struct {
Expand All @@ -57,6 +58,7 @@ type RecipeURLMatch struct {
RecipeUUID string
RecipeName string
RecipeSubType string
ExcludedURL bool
}

var ErrInvalidRecipes = errors.New("invalid interface recipe")
Expand All @@ -75,9 +77,9 @@ func New(config Config) (*Classifier, error) {
var preparedRecipes []Recipe
for _, recipe := range config.Recipes {
preparedRecipe := Recipe{
UUID: recipe.UUID,
Name: recipe.Name,
Type: recipe.Type,
UUID: recipe.UUID,
Name: recipe.Name,
Type: recipe.Type,
SubType: recipe.SubType,
}
for _, recipeURL := range recipe.URLS {
Expand All @@ -92,6 +94,20 @@ func New(config Config) (*Classifier, error) {
}
preparedRecipe.URLS = append(preparedRecipe.URLS, preparedRecipeURL)
}

for _, excludedRecipeURL := range recipe.ExcludeURLS {
regexpMatcher, err := url.PrepareRegexpMatcher(excludedRecipeURL)
if err != nil {
return nil, ErrInvalidRecipes
}

preparedRecipeURL := RecipeURL{
URL: excludedRecipeURL,
RegexpMatcher: regexpMatcher,
}
preparedRecipe.ExcludeURLS = append(preparedRecipe.ExcludeURLS, preparedRecipeURL)
}

preparedRecipes = append(preparedRecipes, preparedRecipe)
}

Expand Down Expand Up @@ -184,12 +200,26 @@ func (classifier *Classifier) Classify(data detections.Detection) (*ClassifiedIn
if err != nil {
return nil, err
}

if recipeMatch != nil {
if recipeMatch.ExcludedURL {
return &ClassifiedInterface{
Detection: &data,
Classification: &Classification{
URL: value,
Decision: classify.ClassificationDecision{
State: classify.Invalid,
Reason: "ignored_url_in_recipe",
},
},
}, nil
}

classifiedInterface := &ClassifiedInterface{
Detection: &data,
Classification: &Classification{
URL: recipeMatch.DetectionURLPart,
RecipeMatch: true,
URL: recipeMatch.DetectionURLPart,
RecipeMatch: true,
RecipeUUID: recipeMatch.RecipeUUID,
RecipeName: recipeMatch.RecipeName,
RecipeSubType: recipeMatch.RecipeSubType,
Expand Down Expand Up @@ -233,6 +263,19 @@ func (classifier *Classifier) FindMatchingRecipeUrl(detectionURL string) (*Recip

matchSize := 0
for _, recipe := range classifier.Recipes {
for _, recipeURL := range recipe.ExcludeURLS {
match, err := url.Match(detectionURL, recipeURL.RegexpMatcher)
if err != nil {
return nil, err
}

if match != "" {
return &RecipeURLMatch{
ExcludedURL: true,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since we have this info, we could include it (may be useful for debugging). I'll leave it up to you!

Suggested change
ExcludedURL: true,
ExcludedURL: true,
DetectionURLPart: match,
RecipeURL: recipeURL.URL,
RecipeUUID: recipe.UUID,
RecipeName: recipe.Name,
RecipeSubType: recipe.SubType,

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure yet
I'll merge it like this let's see how useful it is :)

}, nil
}
}

for _, recipeURL := range recipe.URLS {
match, err := url.Match(detectionURL, recipeURL.RegexpMatcher)
if err != nil {
Expand Down
51 changes: 39 additions & 12 deletions pkg/classification/interfaces/interfaces_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,11 @@ func TestInterface(t *testing.T) {
},
},
Want: &interfaces.Classification{
URL: "https://api.stripe.com",
RecipeName: "Stripe",
URL: "https://api.stripe.com",
RecipeName: "Stripe",
RecipeSubType: "third_party",
RecipeUUID: "c24b836a-d035-49dc-808f-1912f16f690d",
RecipeMatch: true,
RecipeUUID: "c24b836a-d035-49dc-808f-1912f16f690d",
RecipeMatch: true,
Decision: classify.ClassificationDecision{
State: classify.Valid,
Reason: "recipe_match",
Expand Down Expand Up @@ -80,6 +80,33 @@ func TestInterface(t *testing.T) {
},
},
},
{
Name: "when there is a matching recipe with a wildcard but it is part of the exclusion list",
Input: detections.Detection{
Value: reportinterfaces.Interface{
Type: reportinterfaces.TypeURL,
Value: &values.Value{
Parts: []values.Part{
&values.String{
Type: values.PartTypeString,
Value: "https://",
},
&values.String{
Type: values.PartTypeString,
Value: "ajax.googleapis.com",
},
},
},
},
},
Want: &interfaces.Classification{
URL: "https://ajax.googleapis.com",
Decision: classify.ClassificationDecision{
State: classify.Invalid,
Reason: "ignored_url_in_recipe",
},
},
},
{
Name: "when there is a matching recipe with a wildcard",
Input: detections.Detection{
Expand All @@ -100,11 +127,11 @@ func TestInterface(t *testing.T) {
},
},
Want: &interfaces.Classification{
URL: "http://*.stripe.com",
RecipeName: "Stripe",
URL: "http://*.stripe.com",
RecipeName: "Stripe",
RecipeSubType: "third_party",
RecipeUUID: "c24b836a-d035-49dc-808f-1912f16f690d",
RecipeMatch: true,
RecipeUUID: "c24b836a-d035-49dc-808f-1912f16f690d",
RecipeMatch: true,
Decision: classify.ClassificationDecision{
State: classify.Potential,
Reason: "recipe_match_with_wildcard",
Expand All @@ -131,11 +158,11 @@ func TestInterface(t *testing.T) {
},
},
Want: &interfaces.Classification{
URL: "https://googleapis.com/auth/spreadsheets",
RecipeName: "Google Spreadsheets",
URL: "https://googleapis.com/auth/spreadsheets",
RecipeName: "Google Spreadsheets",
RecipeSubType: "third_party",
RecipeUUID: "ebe2e05e-bc56-4204-9329-d9b8d3cf1837",
RecipeMatch: true,
RecipeUUID: "ebe2e05e-bc56-4204-9329-d9b8d3cf1837",
RecipeMatch: true,
Decision: classify.ClassificationDecision{
State: classify.Valid,
Reason: "recipe_match",
Expand Down