diff --git a/cmd/crowdsec-cli/metrics.go b/cmd/crowdsec-cli/metrics.go index 8ab3f01bd7d..3796bdb737e 100644 --- a/cmd/crowdsec-cli/metrics.go +++ b/cmd/crowdsec-cli/metrics.go @@ -58,6 +58,7 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error }{} acquis_stats := map[string]map[string]int{} parsers_stats := map[string]map[string]int{} + whitelist_stats := map[string]map[string]int{} buckets_stats := map[string]map[string]int{} lapi_stats := map[string]map[string]int{} lapi_machine_stats := map[string]map[string]map[string]int{} @@ -173,6 +174,18 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error parsers_stats[name] = make(map[string]int) } parsers_stats[name]["unparsed"] += ival + case "cs_node_hits_wl_ok_total": + subName := fmt.Sprintf("%s:[%s]", name, metric.Labels["expression"]) + if _, ok := whitelist_stats[subName]; !ok { + whitelist_stats[subName] = make(map[string]int) + } + whitelist_stats[subName]["wl"] += ival + case "cs_node_hits_wl_ko_total": + subName := fmt.Sprintf("%s:[%s]", name, metric.Labels["expression"]) + if _, ok := whitelist_stats[subName]; !ok { + whitelist_stats[subName] = make(map[string]int) + } + whitelist_stats[subName]["nowl"] += ival case "cs_lapi_route_requests_total": if _, ok := lapi_stats[route]; !ok { lapi_stats[route] = make(map[string]int) @@ -237,6 +250,7 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error acquisStatsTable(out, acquis_stats) bucketStatsTable(out, buckets_stats) parserStatsTable(out, parsers_stats) + whitelistStatsTable(out, whitelist_stats) lapiStatsTable(out, lapi_stats) lapiMachineStatsTable(out, lapi_machine_stats) lapiBouncerStatsTable(out, lapi_bouncer_stats) @@ -252,6 +266,7 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error stats["acquisition"] = acquis_stats stats["buckets"] = buckets_stats stats["parsers"] = parsers_stats + stats["whitelist"] = whitelist_stats stats["lapi"] = lapi_stats stats["lapi_machine"] = lapi_machine_stats stats["lapi_bouncer"] = lapi_bouncer_stats @@ -282,7 +297,6 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error var noUnit bool - func runMetrics(cmd *cobra.Command, args []string) error { if err := csConfig.LoadPrometheus(); err != nil { return fmt.Errorf("failed to load prometheus config: %w", err) @@ -311,7 +325,6 @@ func runMetrics(cmd *cobra.Command, args []string) error { return nil } - func NewMetricsCmd() *cobra.Command { cmdMetrics := &cobra.Command{ Use: "metrics", @@ -319,7 +332,7 @@ func NewMetricsCmd() *cobra.Command { Long: `Fetch metrics from the prometheus server and display them in a human-friendly way`, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - RunE: runMetrics, + RunE: runMetrics, } cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url (http://:/metrics)") cmdMetrics.PersistentFlags().BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units") diff --git a/cmd/crowdsec-cli/metrics_table.go b/cmd/crowdsec-cli/metrics_table.go index 69706c7acf2..bb16734eace 100644 --- a/cmd/crowdsec-cli/metrics_table.go +++ b/cmd/crowdsec-cli/metrics_table.go @@ -129,6 +129,20 @@ func parserStatsTable(out io.Writer, stats map[string]map[string]int) { } } +func whitelistStatsTable(out io.Writer, stats map[string]map[string]int) { + t := newTable(out) + t.SetRowLines(false) + t.SetHeaders("Whitelist", "WL", "NoWL") + t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) + keys := []string{"wl", "nowl"} + if numRows, err := metricsToTable(t, stats, keys); err != nil { + log.Warningf("while collecting metrics stats: %s", err) + } else if numRows > 0 { + renderTableTitle(out, "\nWhitelist Metrics:") + t.Render() + } +} + func stashStatsTable(out io.Writer, stats map[string]struct { Type string Count int diff --git a/cmd/crowdsec/metrics.go b/cmd/crowdsec/metrics.go index 103becceded..eff2ef4a1b8 100644 --- a/cmd/crowdsec/metrics.go +++ b/cmd/crowdsec/metrics.go @@ -173,7 +173,7 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { } else { log.Infof("Loading prometheus collectors") prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo, - parser.NodesHits, parser.NodesHitsOk, parser.NodesHitsKo, + parser.NodesHits, parser.NodesHitsOk, parser.NodesHitsKo, parser.NodesHitsWl, parser.NodesHitsWlOk, parser.NodesHitsWlKo, globalCsInfo, globalParsingHistogram, globalPourHistogram, v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions, v1.LapiResponseTime, leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount, diff --git a/pkg/parser/node.go b/pkg/parser/node.go index 9f848535ff4..0fd1fd348b3 100644 --- a/pkg/parser/node.go +++ b/pkg/parser/node.go @@ -9,7 +9,6 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" "github.com/davecgh/go-spew/spew" - "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" yaml "gopkg.in/yaml.v2" @@ -168,22 +167,23 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri NodeState = true } - if n.Name != "" { - NodesHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc() - } + n.IncPromHits(p) exprErr := error(nil) - isWhitelisted := n.CheckIPsWL(p.ParseIPSources()) + isWhitelisted := n.CheckIPsWL(p) if !isWhitelisted { - isWhitelisted, exprErr = n.CheckExprWL(cachedExprEnv) + isWhitelisted, exprErr = n.CheckExprWL(cachedExprEnv, p) } if exprErr != nil { // Previous code returned nil if there was an error, so we keep this behavior return false, nil //nolint:nilerr } - if isWhitelisted && !p.Whitelisted { - p.Whitelisted = true - p.WhitelistReason = n.Whitelist.Reason + + if isWhitelisted { + if !p.Whitelisted { + p.Whitelisted = true + p.WhitelistReason = n.Whitelist.Reason + } /*huglily wipe the ban order if the event is whitelisted and it's an overflow */ if p.Type == types.OVFLW { /*don't do this at home kids */ ips := []string{} @@ -331,16 +331,12 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri //grok or leafs failed, don't process statics if !NodeState { - if n.Name != "" { - NodesHitsKo.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc() - } + n.IncKoNodeHits(p) clog.Debugf("Event leaving node : ko") return NodeState, nil } - if n.Name != "" { - NodesHitsOk.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc() - } + n.IncOkNodeHits(p) /* This is to apply statics when the node either was whitelisted, or is not a whitelist (it has no expr/ips wl) diff --git a/pkg/parser/node_metrics.go b/pkg/parser/node_metrics.go new file mode 100644 index 00000000000..7cb0edb4ec0 --- /dev/null +++ b/pkg/parser/node_metrics.go @@ -0,0 +1,96 @@ +package parser + +import ( + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/prometheus/client_golang/prometheus" +) + +var NodesHits = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_node_hits_total", + Help: "Total events entered node.", + }, + []string{"source", "type", "name"}, +) + +var NodesHitsOk = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_node_hits_ok_total", + Help: "Total events successfully exited node.", + }, + []string{"source", "type", "name"}, +) + +var NodesHitsKo = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_node_hits_ko_total", + Help: "Total events unsuccessfully exited node.", + }, + []string{"source", "type", "name"}, +) + +var NodesHitsWl = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_node_hits_wl_total", + Help: "Total events entered whitelists.", + }, + []string{"name", "source", "type"}, +) + +var NodesHitsWlOk = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_node_hits_wl_ok_total", + Help: "Total events successfully exited node.", + }, + []string{"name", "expression", "source", "type"}, +) + +var NodesHitsWlKo = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_node_hits_wl_ko_total", + Help: "Total events unsuccessfully exited node.", + }, + []string{"name", "expression", "source", "type"}, +) + +// Increase the number of hits for a node excluding whitelists +func (n *Node) IncPromHits(p *types.Event) { + if n.Name == "" { + return + } + if !n.ContainsWLs() { + NodesHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc() + } +} + +// Increase the number of ok hits for a node excluding whitelists +func (n *Node) IncOkNodeHits(p *types.Event) { + if n.Name == "" || n.ContainsWLs() { + return + } + NodesHitsOk.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc() +} + +// Increase the number of ko hits for a node excluding whitelists +func (n *Node) IncKoNodeHits(p *types.Event) { + if n.Name == "" || n.ContainsWLs() { + return + } + NodesHitsKo.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc() +} + +// Increase the number of ok hits for a whitelist node +func (n *Node) IncOkWLHits(p *types.Event, expr string) { + if n.Name == "" || !n.ContainsWLs() { + return + } + NodesHitsWlOk.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name, "expression": expr}).Inc() +} + +// Increase the number of ko hits for a whitelist node +func (n *Node) IncKoWLHits(p *types.Event, expr string) { + if n.Name == "" || !n.ContainsWLs() { + return + } + NodesHitsWlKo.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name, "expression": expr}).Inc() +} diff --git a/pkg/parser/runtime.go b/pkg/parser/runtime.go index ecc1b4d85b9..fae2d06f412 100644 --- a/pkg/parser/runtime.go +++ b/pkg/parser/runtime.go @@ -16,7 +16,6 @@ import ( "github.com/antonmedv/expr" "github.com/mohae/deepcopy" - "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -194,30 +193,6 @@ func (n *Node) ProcessStatics(statics []ExtraField, event *types.Event) error { return nil } -var NodesHits = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_node_hits_total", - Help: "Total events entered node.", - }, - []string{"source", "type", "name"}, -) - -var NodesHitsOk = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_node_hits_ok_total", - Help: "Total events successfully exited node.", - }, - []string{"source", "type", "name"}, -) - -var NodesHitsKo = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_node_hits_ko_total", - Help: "Total events unsuccessfully exited node.", - }, - []string{"source", "type", "name"}, -) - func stageidx(stage string, stages []string) int { for i, v := range stages { if stage == v { diff --git a/pkg/parser/whitelist.go b/pkg/parser/whitelist.go index 8c18e70c324..da9c8767e70 100644 --- a/pkg/parser/whitelist.go +++ b/pkg/parser/whitelist.go @@ -38,12 +38,12 @@ func (n *Node) ContainsIPLists() bool { return len(n.Whitelist.B_Ips) > 0 || len(n.Whitelist.B_Cidrs) > 0 } -func (n *Node) CheckIPsWL(srcs []net.IP) bool { +func (n *Node) CheckIPsWL(p *types.Event) bool { isWhitelisted := false if !n.ContainsIPLists() { return isWhitelisted } - for _, src := range srcs { + for _, src := range p.ParseIPSources() { if isWhitelisted { break } @@ -51,23 +51,27 @@ func (n *Node) CheckIPsWL(srcs []net.IP) bool { if v.Equal(src) { n.Logger.Debugf("Event from [%s] is whitelisted by IP (%s), reason [%s]", src, v, n.Whitelist.Reason) isWhitelisted = true + n.IncOkWLHits(p, v.String()) break } + n.IncKoWLHits(p, v.String()) n.Logger.Tracef("whitelist: %s is not eq [%s]", src, v) } for _, v := range n.Whitelist.B_Cidrs { if v.Contains(src) { n.Logger.Debugf("Event from [%s] is whitelisted by CIDR (%s), reason [%s]", src, v, n.Whitelist.Reason) isWhitelisted = true + n.IncOkWLHits(p, v.String()) break } + n.IncKoWLHits(p, v.String()) n.Logger.Tracef("whitelist: %s not in [%s]", src, v) } } return isWhitelisted } -func (n *Node) CheckExprWL(cachedExprEnv map[string]interface{}) (bool, error) { +func (n *Node) CheckExprWL(cachedExprEnv map[string]interface{}, p *types.Event) (bool, error) { isWhitelisted := false if !n.ContainsExprLists() { @@ -79,7 +83,8 @@ func (n *Node) CheckExprWL(cachedExprEnv map[string]interface{}) (bool, error) { if isWhitelisted { break } - output, err := expr.Run(e.Filter, cachedExprEnv) + + output, err := expr.Run(e.Filter, cachedExprEnv) if err != nil { n.Logger.Warningf("failed to run whitelist expr : %v", err) n.Logger.Debug("Event leaving node : ko") @@ -93,6 +98,9 @@ func (n *Node) CheckExprWL(cachedExprEnv map[string]interface{}) (bool, error) { if out { n.Logger.Debugf("Event is whitelisted by expr, reason [%s]", n.Whitelist.Reason) isWhitelisted = true + n.IncOkWLHits(p, fmt.Sprintf("%d", eidx)) + } else { + n.IncKoWLHits(p, fmt.Sprintf("%d", eidx)) } default: n.Logger.Errorf("unexpected type %t (%v) while running '%s'", output, output, n.Whitelist.Exprs[eidx]) diff --git a/pkg/parser/whitelist_test.go b/pkg/parser/whitelist_test.go index 8796aaedafe..501c655243d 100644 --- a/pkg/parser/whitelist_test.go +++ b/pkg/parser/whitelist_test.go @@ -289,9 +289,9 @@ func TestWhitelistCheck(t *testing.T) { var err error node.Whitelist = tt.whitelist node.CompileWLs() - isWhitelisted := node.CheckIPsWL(tt.event.ParseIPSources()) + isWhitelisted := node.CheckIPsWL(tt.event) if !isWhitelisted { - isWhitelisted, err = node.CheckExprWL(map[string]interface{}{"evt": tt.event}) + isWhitelisted, err = node.CheckExprWL(map[string]interface{}{"evt": tt.event}, tt.event) } require.NoError(t, err) require.Equal(t, tt.expected, isWhitelisted)