From 7e2925d86ed5f2c3b5ef0514a6f82d214496903b Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Mon, 7 Oct 2024 22:59:05 +0700 Subject: [PATCH 01/11] feat: global matchers Signed-off-by: Dwi Siswanto Co-authored-by: Ice3man543 --- internal/runner/runner.go | 2 + pkg/output/format_screen.go | 5 ++ pkg/output/output.go | 2 + .../common/globalmatchers/globalmatchers.go | 62 +++++++++++++++++++ pkg/protocols/http/http.go | 3 + pkg/protocols/http/operators.go | 5 ++ pkg/protocols/http/request.go | 11 +++- pkg/protocols/protocols.go | 3 + pkg/templates/compile.go | 22 +++++++ 9 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 pkg/protocols/common/globalmatchers/globalmatchers.go diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 0b6da592d3..516bd7ca50 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -48,6 +48,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit" @@ -475,6 +476,7 @@ func (r *Runner) RunEnumeration() error { TemporaryDirectory: r.tmpDir, Parser: r.parser, FuzzParamsFrequency: fuzzFreqCache, + GlobalMatchers: globalmatchers.New(), } if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) { diff --git a/pkg/output/format_screen.go b/pkg/output/format_screen.go index 9d0f03efaf..88e13d0950 100644 --- a/pkg/output/format_screen.go +++ b/pkg/output/format_screen.go @@ -39,6 +39,11 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte { } } + if output.Passive { + builder.WriteString("] [") + builder.WriteString(w.aurora.BrightMagenta("passive").String()) + } + builder.WriteString("] [") builder.WriteString(w.aurora.BrightBlue(output.Type).String()) builder.WriteString("] ") diff --git a/pkg/output/output.go b/pkg/output/output.go index fbc9f71306..7e4d86d8f5 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -184,6 +184,8 @@ type ResultEvent struct { MatcherStatus bool `json:"matcher-status"` // Lines is the line count for the specified match Lines []int `json:"matched-line,omitempty"` + // Passive specifies whether the match was discovered passively in the response + Passive bool `json:"passive,omitempty"` // IssueTrackers is the metadata for issue trackers IssueTrackers map[string]IssueTrackerMetadata `json:"issue_trackers,omitempty"` diff --git a/pkg/protocols/common/globalmatchers/globalmatchers.go b/pkg/protocols/common/globalmatchers/globalmatchers.go new file mode 100644 index 0000000000..e5982dbd75 --- /dev/null +++ b/pkg/protocols/common/globalmatchers/globalmatchers.go @@ -0,0 +1,62 @@ +package globalmatchers + +import ( + "github.com/projectdiscovery/nuclei/v3/pkg/model" + "github.com/projectdiscovery/nuclei/v3/pkg/operators" + "github.com/projectdiscovery/nuclei/v3/pkg/output" + "golang.org/x/exp/maps" +) + +// Storage is a struct that holds the global matchers +type Storage struct { + requests []*Item +} + +// Item is a struct that holds the global matchers +// details for a template +type Item struct { + TemplateID string + TemplatePath string + TemplateInfo model.Info + Operators []*operators.Operators +} + +// New creates a new storage for global matchers +func New() *Storage { + return &Storage{} +} + +// AddOperator adds a new operator to the global matchers +func (s *Storage) AddOperator(item *Item) { + s.requests = append(s.requests, item) +} + +// HasMatchers returns true if we have global matchers +func (s *Storage) HasMatchers() bool { + return len(s.requests) > 0 +} + +// Match matches the global matchers against the response +func (s *Storage) Match( + event output.InternalEvent, + matchFunc operators.MatchFunc, + extractFunc operators.ExtractFunc, + isDebug bool, + callback func(event output.InternalEvent, result *operators.Result), +) { + for _, item := range s.requests { + for _, operator := range item.Operators { + result, matched := operator.Execute(event, matchFunc, extractFunc, isDebug) + if !matched { + continue + } + + eventCopy := maps.Clone(event) + eventCopy["template-id"] = item.TemplateID + eventCopy["template-info"] = item.TemplateInfo + eventCopy["template-path"] = item.TemplatePath + eventCopy["passive"] = true + callback(eventCopy, result) + } + } +} diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go index 844bf8c579..bbde74dcb4 100644 --- a/pkg/protocols/http/http.go +++ b/pkg/protocols/http/http.go @@ -223,6 +223,9 @@ type Request struct { // FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR FuzzPreConditionOperator string `yaml:"pre-condition-operator,omitempty" json:"pre-condition-operator,omitempty" jsonschema:"title=condition between the filters,description=Operator to use between multiple per-conditions,enum=and,enum=or"` fuzzPreConditionOperator matchers.ConditionType `yaml:"-" json:"-"` + // description: | + // Passive specifies whether the request should be made in passive mode + Passive bool `yaml:"passive,omitempty" json:"passive,omitempty" jsonschema:"title=passive mode,description=Passive mode for the request"` } func (e Request) JSONSchemaExtend(schema *jsonschema.Schema) { diff --git a/pkg/protocols/http/operators.go b/pkg/protocols/http/operators.go index d630bfd8b0..9a66f54d92 100644 --- a/pkg/protocols/http/operators.go +++ b/pkg/protocols/http/operators.go @@ -166,6 +166,10 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent if types.ToString(wrapped.InternalEvent["path"]) != "" { fields.Path = types.ToString(wrapped.InternalEvent["path"]) } + var isPassive bool + if value, ok := wrapped.InternalEvent["passive"]; ok { + isPassive = value.(bool) + } data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), @@ -183,6 +187,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent Timestamp: time.Now(), MatcherStatus: true, IP: fields.Ip, + Passive: isPassive, Request: types.ToString(wrapped.InternalEvent["request"]), Response: request.truncateResponse(wrapped.InternalEvent["response"]), CURLCommand: types.ToString(wrapped.InternalEvent["curl-command"]), diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 994e065582..f74020eefb 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -973,13 +973,22 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ // prune signature internal values if any request.pruneSignatureInternalValues(generatedRequest.meta) - event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(generatedRequest.dynamicValues, finalEvent), request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { + interimEvent := generators.MergeMaps(generatedRequest.dynamicValues, finalEvent) + isDebug := request.options.Options.Debug || request.options.Options.DebugResponse + event := eventcreator.CreateEventWithAdditionalOptions(request, interimEvent, isDebug, func(internalWrappedEvent *output.InternalWrappedEvent) { internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta }) + if hasInteractMatchers { event.UsesInteractsh = true } + if request.options.GlobalMatchers.HasMatchers() { + request.options.GlobalMatchers.Match(interimEvent, request.Match, request.Extract, isDebug, func(event output.InternalEvent, result *operators.Result) { + callback(eventcreator.CreateEventWithOperatorResults(request, event, result)) + }) + } + // if requrlpattern is enabled, only then it is reflected in result event else it is empty string // consult @Ice3man543 before changing this logic (context: vuln_hash) if request.options.ExportReqURLPattern { diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index a9d50c1481..9ead70321d 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -25,6 +25,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/progress" "github.com/projectdiscovery/nuclei/v3/pkg/projectfile" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers" @@ -126,6 +127,8 @@ type ExecutorOptions struct { // ExportReqURLPattern exports the request URL pattern // in ResultEvent it contains the exact url pattern (ex: {{BaseURL}}/{{randstr}}/xyz) used in the request ExportReqURLPattern bool + // GlobalMatchers is the storage for global matchers with http passive templates + GlobalMatchers *globalmatchers.Storage } // todo: centralizing components is not feasible with current clogged architecture diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index 01af3a999b..47761a1316 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -20,6 +20,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/offlinehttp" "github.com/projectdiscovery/nuclei/v3/pkg/templates/signer" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec" @@ -81,6 +82,18 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Executo if err != nil { return nil, err } + if template.checkHTTPContainsGlobalMatchers() { + item := &globalmatchers.Item{ + TemplateID: template.ID, + TemplatePath: filePath, + TemplateInfo: template.Info, + } + for _, request := range template.RequestsHTTP { + item.Operators = append(item.Operators, request.CompiledOperators) + } + options.GlobalMatchers.AddOperator(item) + return nil, nil + } // Compile the workflow request if len(template.Workflows) > 0 { compiled := &template.Workflow @@ -96,6 +109,15 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Executo return template, nil } +func (template *Template) checkHTTPContainsGlobalMatchers() bool { + for _, request := range template.RequestsHTTP { + if request.Passive { + return true + } + } + return false +} + // parseSelfContainedRequests parses the self contained template requests. func (template *Template) parseSelfContainedRequests() { if template.Signature.Value.String() != "" { From 86e8f283b99cf1b6a98c3a20be84d0df3ad3023a Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 8 Oct 2024 12:25:16 +0700 Subject: [PATCH 02/11] feat(globalmatchers): make `Callback` as type Signed-off-by: Dwi Siswanto --- pkg/protocols/common/globalmatchers/globalmatchers.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/protocols/common/globalmatchers/globalmatchers.go b/pkg/protocols/common/globalmatchers/globalmatchers.go index e5982dbd75..66658ccf85 100644 --- a/pkg/protocols/common/globalmatchers/globalmatchers.go +++ b/pkg/protocols/common/globalmatchers/globalmatchers.go @@ -12,6 +12,10 @@ type Storage struct { requests []*Item } +// Callback is called when a global matcher is matched. +// It receives internal event & result of the operator execution. +type Callback func(event output.InternalEvent, result *operators.Result) + // Item is a struct that holds the global matchers // details for a template type Item struct { @@ -42,7 +46,7 @@ func (s *Storage) Match( matchFunc operators.MatchFunc, extractFunc operators.ExtractFunc, isDebug bool, - callback func(event output.InternalEvent, result *operators.Result), + callback Callback, ) { for _, item := range s.requests { for _, operator := range item.Operators { From 1f78cdaf7d874813b2a6a3a8b7609318a7fac2ad Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 9 Oct 2024 03:42:50 +0700 Subject: [PATCH 03/11] feat: update `passive` term to `(matchers-)static` Signed-off-by: Dwi Siswanto --- pkg/output/format_screen.go | 4 ++-- pkg/output/output.go | 5 +++-- pkg/protocols/common/globalmatchers/globalmatchers.go | 2 +- pkg/protocols/http/http.go | 6 ++++-- pkg/protocols/http/operators.go | 8 ++++---- pkg/protocols/protocols.go | 2 +- pkg/templates/compile.go | 2 +- 7 files changed, 16 insertions(+), 13 deletions(-) diff --git a/pkg/output/format_screen.go b/pkg/output/format_screen.go index 88e13d0950..7c07d64b92 100644 --- a/pkg/output/format_screen.go +++ b/pkg/output/format_screen.go @@ -39,9 +39,9 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte { } } - if output.Passive { + if output.MatchersStatic { builder.WriteString("] [") - builder.WriteString(w.aurora.BrightMagenta("passive").String()) + builder.WriteString(w.aurora.BrightMagenta("static").String()) } builder.WriteString("] [") diff --git a/pkg/output/output.go b/pkg/output/output.go index 7e4d86d8f5..5182d74605 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -184,8 +184,9 @@ type ResultEvent struct { MatcherStatus bool `json:"matcher-status"` // Lines is the line count for the specified match Lines []int `json:"matched-line,omitempty"` - // Passive specifies whether the match was discovered passively in the response - Passive bool `json:"passive,omitempty"` + // MatchersStatic identifies whether the matches was detected statically in the response + // of another template's result event + MatchersStatic bool `json:"matcher-static,omitempty"` // IssueTrackers is the metadata for issue trackers IssueTrackers map[string]IssueTrackerMetadata `json:"issue_trackers,omitempty"` diff --git a/pkg/protocols/common/globalmatchers/globalmatchers.go b/pkg/protocols/common/globalmatchers/globalmatchers.go index 66658ccf85..01557237e8 100644 --- a/pkg/protocols/common/globalmatchers/globalmatchers.go +++ b/pkg/protocols/common/globalmatchers/globalmatchers.go @@ -59,7 +59,7 @@ func (s *Storage) Match( eventCopy["template-id"] = item.TemplateID eventCopy["template-info"] = item.TemplateInfo eventCopy["template-path"] = item.TemplatePath - eventCopy["passive"] = true + eventCopy["matcher-static"] = true callback(eventCopy, result) } } diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go index bbde74dcb4..1a18f15a98 100644 --- a/pkg/protocols/http/http.go +++ b/pkg/protocols/http/http.go @@ -224,8 +224,10 @@ type Request struct { FuzzPreConditionOperator string `yaml:"pre-condition-operator,omitempty" json:"pre-condition-operator,omitempty" jsonschema:"title=condition between the filters,description=Operator to use between multiple per-conditions,enum=and,enum=or"` fuzzPreConditionOperator matchers.ConditionType `yaml:"-" json:"-"` // description: | - // Passive specifies whether the request should be made in passive mode - Passive bool `yaml:"passive,omitempty" json:"passive,omitempty" jsonschema:"title=passive mode,description=Passive mode for the request"` + // MatchersStatic marks matchers as static and applies globally to all events (request and response) from other templates. + // + // Note that if this is enabled, no requests will be sent from this template. + MatchersStatic bool `yaml:"matchers-static,omitempty" json:"matchers-static,omitempty" jsonschema:"title=static matchers option,description=marks matchers as static and applies them globally to all events from other templates"` } func (e Request) JSONSchemaExtend(schema *jsonschema.Schema) { diff --git a/pkg/protocols/http/operators.go b/pkg/protocols/http/operators.go index 9a66f54d92..9e2fa8e278 100644 --- a/pkg/protocols/http/operators.go +++ b/pkg/protocols/http/operators.go @@ -166,9 +166,9 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent if types.ToString(wrapped.InternalEvent["path"]) != "" { fields.Path = types.ToString(wrapped.InternalEvent["path"]) } - var isPassive bool - if value, ok := wrapped.InternalEvent["passive"]; ok { - isPassive = value.(bool) + var isMatchersStatic bool + if value, ok := wrapped.InternalEvent["matchers-static"]; ok { + isMatchersStatic = value.(bool) } data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), @@ -186,8 +186,8 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), MatcherStatus: true, + MatchersStatic: isMatchersStatic, IP: fields.Ip, - Passive: isPassive, Request: types.ToString(wrapped.InternalEvent["request"]), Response: request.truncateResponse(wrapped.InternalEvent["response"]), CURLCommand: types.ToString(wrapped.InternalEvent["curl-command"]), diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 9ead70321d..12db54d16d 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -127,7 +127,7 @@ type ExecutorOptions struct { // ExportReqURLPattern exports the request URL pattern // in ResultEvent it contains the exact url pattern (ex: {{BaseURL}}/{{randstr}}/xyz) used in the request ExportReqURLPattern bool - // GlobalMatchers is the storage for global matchers with http passive templates + // GlobalMatchers is the storage for (static) global matchers GlobalMatchers *globalmatchers.Storage } diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index 47761a1316..5960cc71f9 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -111,7 +111,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Executo func (template *Template) checkHTTPContainsGlobalMatchers() bool { for _, request := range template.RequestsHTTP { - if request.Passive { + if request.MatchersStatic { return true } } From 7ffa809d0c99b549928c6b2dd494e41cdcd0fcb8 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 9 Oct 2024 03:48:57 +0700 Subject: [PATCH 04/11] feat(globalmatchers): add `origin-template-*` event also use `Set` method instead of `maps.Clone` Signed-off-by: Dwi Siswanto --- .../common/globalmatchers/globalmatchers.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/protocols/common/globalmatchers/globalmatchers.go b/pkg/protocols/common/globalmatchers/globalmatchers.go index 01557237e8..543f7d64c9 100644 --- a/pkg/protocols/common/globalmatchers/globalmatchers.go +++ b/pkg/protocols/common/globalmatchers/globalmatchers.go @@ -4,7 +4,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/model" "github.com/projectdiscovery/nuclei/v3/pkg/operators" "github.com/projectdiscovery/nuclei/v3/pkg/output" - "golang.org/x/exp/maps" ) // Storage is a struct that holds the global matchers @@ -50,17 +49,20 @@ func (s *Storage) Match( ) { for _, item := range s.requests { for _, operator := range item.Operators { + event.Set("origin-template-id", event["template-id"]) + event.Set("origin-template-info", event["template-info"]) + event.Set("origin-template-path", event["template-path"]) + event.Set("template-id", item.TemplateID) + event.Set("template-info", item.TemplateInfo) + event.Set("template-path", item.TemplatePath) + event.Set("matchers-static", true) + result, matched := operator.Execute(event, matchFunc, extractFunc, isDebug) if !matched { continue } - eventCopy := maps.Clone(event) - eventCopy["template-id"] = item.TemplateID - eventCopy["template-info"] = item.TemplateInfo - eventCopy["template-path"] = item.TemplatePath - eventCopy["matcher-static"] = true - callback(eventCopy, result) + callback(event, result) } } } From eb42bf46eafb1a67e43b6930149b07fd5374f4e8 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 10 Oct 2024 22:06:16 +0700 Subject: [PATCH 05/11] feat: update `matchers-static` term to `global-matchers` Signed-off-by: Dwi Siswanto --- pkg/output/format_screen.go | 4 ++-- pkg/output/output.go | 4 ++-- pkg/protocols/common/globalmatchers/globalmatchers.go | 2 +- pkg/protocols/http/http.go | 6 ++---- pkg/protocols/http/operators.go | 8 ++++---- pkg/protocols/protocols.go | 2 +- pkg/templates/compile.go | 2 +- 7 files changed, 13 insertions(+), 15 deletions(-) diff --git a/pkg/output/format_screen.go b/pkg/output/format_screen.go index 7c07d64b92..3cdcec1e15 100644 --- a/pkg/output/format_screen.go +++ b/pkg/output/format_screen.go @@ -39,9 +39,9 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte { } } - if output.MatchersStatic { + if output.GlobalMatchers { builder.WriteString("] [") - builder.WriteString(w.aurora.BrightMagenta("static").String()) + builder.WriteString(w.aurora.BrightMagenta("global").String()) } builder.WriteString("] [") diff --git a/pkg/output/output.go b/pkg/output/output.go index 5182d74605..9d84fd20fc 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -184,9 +184,9 @@ type ResultEvent struct { MatcherStatus bool `json:"matcher-status"` // Lines is the line count for the specified match Lines []int `json:"matched-line,omitempty"` - // MatchersStatic identifies whether the matches was detected statically in the response + // GlobalMatchers identifies whether the matches was detected in the response // of another template's result event - MatchersStatic bool `json:"matcher-static,omitempty"` + GlobalMatchers bool `json:"global-matchers,omitempty"` // IssueTrackers is the metadata for issue trackers IssueTrackers map[string]IssueTrackerMetadata `json:"issue_trackers,omitempty"` diff --git a/pkg/protocols/common/globalmatchers/globalmatchers.go b/pkg/protocols/common/globalmatchers/globalmatchers.go index 543f7d64c9..e8243a0cbb 100644 --- a/pkg/protocols/common/globalmatchers/globalmatchers.go +++ b/pkg/protocols/common/globalmatchers/globalmatchers.go @@ -55,7 +55,7 @@ func (s *Storage) Match( event.Set("template-id", item.TemplateID) event.Set("template-info", item.TemplateInfo) event.Set("template-path", item.TemplatePath) - event.Set("matchers-static", true) + event.Set("global-matchers", true) result, matched := operator.Execute(event, matchFunc, extractFunc, isDebug) if !matched { diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go index 1a18f15a98..bd089edd47 100644 --- a/pkg/protocols/http/http.go +++ b/pkg/protocols/http/http.go @@ -224,10 +224,8 @@ type Request struct { FuzzPreConditionOperator string `yaml:"pre-condition-operator,omitempty" json:"pre-condition-operator,omitempty" jsonschema:"title=condition between the filters,description=Operator to use between multiple per-conditions,enum=and,enum=or"` fuzzPreConditionOperator matchers.ConditionType `yaml:"-" json:"-"` // description: | - // MatchersStatic marks matchers as static and applies globally to all events (request and response) from other templates. - // - // Note that if this is enabled, no requests will be sent from this template. - MatchersStatic bool `yaml:"matchers-static,omitempty" json:"matchers-static,omitempty" jsonschema:"title=static matchers option,description=marks matchers as static and applies them globally to all events from other templates"` + // GlobalMatchers marks matchers as static and applies globally to all result events from other templates + GlobalMatchers bool `yaml:"global-matchers,omitempty" json:"global-matchers,omitempty" jsonschema:"title=global matchers,description=marks matchers as static and applies globally to all result events from other templates"` } func (e Request) JSONSchemaExtend(schema *jsonschema.Schema) { diff --git a/pkg/protocols/http/operators.go b/pkg/protocols/http/operators.go index 9e2fa8e278..9e7d58af0a 100644 --- a/pkg/protocols/http/operators.go +++ b/pkg/protocols/http/operators.go @@ -166,9 +166,9 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent if types.ToString(wrapped.InternalEvent["path"]) != "" { fields.Path = types.ToString(wrapped.InternalEvent["path"]) } - var isMatchersStatic bool - if value, ok := wrapped.InternalEvent["matchers-static"]; ok { - isMatchersStatic = value.(bool) + var isGlobalMatchers bool + if value, ok := wrapped.InternalEvent["global-matchers"]; ok { + isGlobalMatchers = value.(bool) } data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), @@ -186,8 +186,8 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), MatcherStatus: true, - MatchersStatic: isMatchersStatic, IP: fields.Ip, + GlobalMatchers: isGlobalMatchers, Request: types.ToString(wrapped.InternalEvent["request"]), Response: request.truncateResponse(wrapped.InternalEvent["response"]), CURLCommand: types.ToString(wrapped.InternalEvent["curl-command"]), diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 12db54d16d..9ead70321d 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -127,7 +127,7 @@ type ExecutorOptions struct { // ExportReqURLPattern exports the request URL pattern // in ResultEvent it contains the exact url pattern (ex: {{BaseURL}}/{{randstr}}/xyz) used in the request ExportReqURLPattern bool - // GlobalMatchers is the storage for (static) global matchers + // GlobalMatchers is the storage for global matchers with http passive templates GlobalMatchers *globalmatchers.Storage } diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index 5960cc71f9..5de83d3e05 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -111,7 +111,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Executo func (template *Template) checkHTTPContainsGlobalMatchers() bool { for _, request := range template.RequestsHTTP { - if request.MatchersStatic { + if request.GlobalMatchers { return true } } From d9d83abf8ba33e374943305dacc196de80bf8623 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Fri, 11 Oct 2024 02:29:25 +0700 Subject: [PATCH 06/11] feat(globalmatchers): clone event before `operator.Execute` Signed-off-by: Dwi Siswanto --- .../common/globalmatchers/globalmatchers.go | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/pkg/protocols/common/globalmatchers/globalmatchers.go b/pkg/protocols/common/globalmatchers/globalmatchers.go index e8243a0cbb..15c73bd64d 100644 --- a/pkg/protocols/common/globalmatchers/globalmatchers.go +++ b/pkg/protocols/common/globalmatchers/globalmatchers.go @@ -1,6 +1,8 @@ package globalmatchers import ( + "maps" + "github.com/projectdiscovery/nuclei/v3/pkg/model" "github.com/projectdiscovery/nuclei/v3/pkg/operators" "github.com/projectdiscovery/nuclei/v3/pkg/output" @@ -49,20 +51,21 @@ func (s *Storage) Match( ) { for _, item := range s.requests { for _, operator := range item.Operators { - event.Set("origin-template-id", event["template-id"]) - event.Set("origin-template-info", event["template-info"]) - event.Set("origin-template-path", event["template-path"]) - event.Set("template-id", item.TemplateID) - event.Set("template-info", item.TemplateInfo) - event.Set("template-path", item.TemplatePath) - event.Set("global-matchers", true) + newEvent := maps.Clone(event) + newEvent.Set("origin-template-id", event["template-id"]) + newEvent.Set("origin-template-info", event["template-info"]) + newEvent.Set("origin-template-path", event["template-path"]) + newEvent.Set("template-id", item.TemplateID) + newEvent.Set("template-info", item.TemplateInfo) + newEvent.Set("template-path", item.TemplatePath) + newEvent.Set("global-matchers", true) - result, matched := operator.Execute(event, matchFunc, extractFunc, isDebug) + result, matched := operator.Execute(newEvent, matchFunc, extractFunc, isDebug) if !matched { continue } - callback(event, result) + callback(newEvent, result) } } } From 4bbb949659f7f60fd4c72e6f01e1a01e9b65d612 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Fri, 11 Oct 2024 04:04:17 +0700 Subject: [PATCH 07/11] fix(tmplexec): don't store `matched` on `global-matchers` templ This will end up generating 2 events from the same `scan.ScanContext` if one of the templates has `global-matchers` enabled. This way, non- `global-matchers` templates can enter the `writeFailureCallback` func to log failure output. Signed-off-by: Dwi Siswanto --- pkg/tmplexec/exec.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go index 279d03d849..0caefe6024 100644 --- a/pkg/tmplexec/exec.go +++ b/pkg/tmplexec/exec.go @@ -174,7 +174,15 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { if !event.HasOperatorResult() && event.InternalEvent != nil { lastMatcherEvent = event } else { - if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) { + var isGlobalMatchers bool + isGlobalMatchers, _ = event.InternalEvent["global-matchers"].(bool) + // NOTE(dwisiswant0): Don't store `matched` on a `global-matchers` template. + // This will end up generating 2 events from the same `scan.ScanContext` if + // one of the templates has `global-matchers` enabled. This way, + // non-`global-matchers` templates can enter the `writeFailureCallback` + // func to log failure output. + wr := writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) + if wr && !isGlobalMatchers { matched.Store(true) } else { lastMatcherEvent = event From 123d8529f8c1f76cb2616a816ca4b551a7643a76 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Fri, 11 Oct 2024 20:18:38 +0700 Subject: [PATCH 08/11] feat(globalmatchers): initializes `requests` on `New` Signed-off-by: Dwi Siswanto --- pkg/protocols/common/globalmatchers/globalmatchers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/protocols/common/globalmatchers/globalmatchers.go b/pkg/protocols/common/globalmatchers/globalmatchers.go index 15c73bd64d..73838e0d71 100644 --- a/pkg/protocols/common/globalmatchers/globalmatchers.go +++ b/pkg/protocols/common/globalmatchers/globalmatchers.go @@ -28,7 +28,7 @@ type Item struct { // New creates a new storage for global matchers func New() *Storage { - return &Storage{} + return &Storage{requests: make([]*Item, 0)} } // AddOperator adds a new operator to the global matchers From 4ffeb1a70ee9ef0ade6c5bd93c620eb5dae6837d Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Fri, 11 Oct 2024 20:19:21 +0700 Subject: [PATCH 09/11] feat(globalmatchers): add `hasStorage` method Signed-off-by: Dwi Siswanto --- .../common/globalmatchers/globalmatchers.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/protocols/common/globalmatchers/globalmatchers.go b/pkg/protocols/common/globalmatchers/globalmatchers.go index 73838e0d71..321759f927 100644 --- a/pkg/protocols/common/globalmatchers/globalmatchers.go +++ b/pkg/protocols/common/globalmatchers/globalmatchers.go @@ -31,13 +31,26 @@ func New() *Storage { return &Storage{requests: make([]*Item, 0)} } +// hasStorage checks if the Storage is initialized +func (s *Storage) hasStorage() bool { + return s != nil +} + // AddOperator adds a new operator to the global matchers func (s *Storage) AddOperator(item *Item) { + if !s.hasStorage() { + return + } + s.requests = append(s.requests, item) } // HasMatchers returns true if we have global matchers func (s *Storage) HasMatchers() bool { + if !s.hasStorage() { + return false + } + return len(s.requests) > 0 } From 44b82ad4b21f1087510de207bb343f625f0485c9 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sat, 12 Oct 2024 01:54:26 +0700 Subject: [PATCH 10/11] refactor(templates): rename global matchers checks method Signed-off-by: Dwi Siswanto --- pkg/templates/compile.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index 5de83d3e05..b4005afe81 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -82,7 +82,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Executo if err != nil { return nil, err } - if template.checkHTTPContainsGlobalMatchers() { + if template.isGlobalMatchersEnabled() { item := &globalmatchers.Item{ TemplateID: template.ID, TemplatePath: filePath, @@ -109,7 +109,17 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Executo return template, nil } -func (template *Template) checkHTTPContainsGlobalMatchers() bool { +// isGlobalMatchersEnabled checks if any of requests in the template +// have global matchers enabled. It iterates through all requests and +// returns true if at least one request has global matchers enabled; +// otherwise, it returns false. +// +// Note: This method only checks the `RequestsHTTP` +// field of the template, which is specific to http-protocol-based +// templates. +// +// TODO: support all protocols. +func (template *Template) isGlobalMatchersEnabled() bool { for _, request := range template.RequestsHTTP { if request.GlobalMatchers { return true From 12f6cb5793ebb90b849f8c34948c75df61229e77 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sat, 12 Oct 2024 02:23:39 +0700 Subject: [PATCH 11/11] fix(loader): handle nil `templates.Template` pointer Signed-off-by: Dwi Siswanto --- pkg/catalog/loader/loader.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go index 48f14a4054..94818020b7 100644 --- a/pkg/catalog/loader/loader.go +++ b/pkg/catalog/loader/loader.go @@ -353,6 +353,20 @@ func (store *Store) areWorkflowOrTemplatesValid(filteredTemplatePaths map[string if isParsingError("Error occurred parsing template %s: %s\n", templatePath, err) { areTemplatesValid = false } + } else if template == nil { + // NOTE(dwisiswant0): possibly global matchers template. + // This could definitely be handled better, for example by returning an + // `ErrGlobalMatchersTemplate` during `templates.Parse` and checking it + // with `errors.Is`. + // + // However, I’m not sure if every reference to it should be handled + // that way. Returning a `templates.Template` pointer would mean it’s + // an active template (sending requests), and adding a specific field + // like `isGlobalMatchers` in `templates.Template` (then checking it + // with a `*templates.Template.IsGlobalMatchersEnabled` method) would + // just introduce more unknown issues - like during template + // clustering, AFAIK. + continue } else { if existingTemplatePath, found := templateIDPathMap[template.ID]; !found { templateIDPathMap[template.ID] = templatePath