From dbfafd78e888f98afb55bfe9f4d4607df8f5ae03 Mon Sep 17 00:00:00 2001 From: Jeremy Mill Date: Fri, 15 Apr 2022 14:44:49 -0400 Subject: [PATCH 1/4] add custom prometheus labes Signed-off-by: Jeremy Mill --- README.md | 5 +++++ config.go | 18 +++++++++++++++--- config_example.yaml | 4 ++++ handlers.go | 6 +++++- main.go | 2 +- stats_prometheus.go | 22 +++++++++++++--------- types/types.go | 1 + 7 files changed, 44 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 05c8c03e2..d1a98afc9 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,11 @@ customfields: # custom fields are added to falco events Ckey: "CValue" mutualtlsfilespath: "/etc/certs" # folder which will used to store client.crt, client.key and ca.crt files for mutual tls (default: "/etc/certs") +customprometheus: # custom labels to add to prometheus + Akey: "AValue" + Bkey: "BValue" + Ckey: "CValue" + slack: webhookurl: "" # Slack WebhookURL (ex: https://hooks.slack.com/services/XXXX/YYYY/ZZZZ), if not empty, Slack output is enabled #footer: "" # Slack footer diff --git a/config.go b/config.go index 59d0e9e6a..56cb4a109 100644 --- a/config.go +++ b/config.go @@ -19,9 +19,10 @@ import ( func getConfig() *types.Configuration { c := &types.Configuration{ - Customfields: make(map[string]string), - Webhook: types.WebhookOutputConfig{CustomHeaders: make(map[string]string)}, - CloudEvents: types.CloudEventsOutputConfig{Extensions: make(map[string]string)}, + Customfields: make(map[string]string), + CustomPrometheus: make(map[string]string), + Webhook: types.WebhookOutputConfig{CustomHeaders: make(map[string]string)}, + CloudEvents: types.CloudEventsOutputConfig{Extensions: make(map[string]string)}, } configFile := kingpin.Flag("config-file", "config file").Short('c').ExistingFile() @@ -327,6 +328,7 @@ func getConfig() *types.Configuration { } v.GetStringMapString("customfields") + v.GetStringMapString("customprometheus") v.GetStringMapString("Webhook.CustomHeaders") v.GetStringMapString("CloudEvents.Extensions") if err := v.Unmarshal(c); err != nil { @@ -343,6 +345,16 @@ func getConfig() *types.Configuration { } } + if value, present := os.LookupEnv("CUSTOMPROMETHEUS"); present { + customprometheus := strings.Split(value, ",") + for _, label := range customprometheus { + tagkeys := strings.Split(label, ":") + if len(tagkeys) == 2 { + c.CustomPrometheus[tagkeys[0]] = tagkeys[1] + } + } + } + if value, present := os.LookupEnv("WEBHOOK_CUSTOMHEADERS"); present { customfields := strings.Split(value, ",") for _, label := range customfields { diff --git a/config_example.yaml b/config_example.yaml index fcb6d11a8..9da9ffec9 100644 --- a/config_example.yaml +++ b/config_example.yaml @@ -5,6 +5,10 @@ customfields: # custom fields are added to falco events Akey: "AValue" Bkey: "BValue" Ckey: "CValue" +customprometheus: # custom labels to add to prometheus + Akey: "AValue" + Bkey: "BValue" + Ckey: "CValue" mutualtlsfilespath: "/etc/certs" # folder which will used to store client.crt, client.key and ca.crt files for mutual tls (default: "/etc/certs") slack: diff --git a/handlers.go b/handlers.go index 045b7bca5..2e8d2e584 100644 --- a/handlers.go +++ b/handlers.go @@ -106,7 +106,11 @@ func newFalcoPayload(payload io.Reader) (types.FalcoPayload, error) { nullClient.CountMetric("falco.accepted", 1, []string{"priority:" + falcopayload.Priority.String()}) stats.Falco.Add(strings.ToLower(falcopayload.Priority.String()), 1) - promStats.Falco.With(map[string]string{"rule": falcopayload.Rule, "priority": falcopayload.Priority.String(), "k8s_ns_name": kn, "k8s_pod_name": kp}).Inc() + promLabels := map[string]string{"rule": falcopayload.Rule, "priority": falcopayload.Priority.String(), "k8s_ns_name": kn, "k8s_pod_name": kp} + for key, value := range config.CustomPrometheus { + promLabels[key] = value + } + promStats.Falco.With(promLabels).Inc() if config.Debug == true { body, _ := json.Marshal(falcopayload) diff --git a/main.go b/main.go index 4b565d34d..465542f7c 100644 --- a/main.go +++ b/main.go @@ -62,7 +62,7 @@ var ( func init() { config = getConfig() stats = getInitStats() - promStats = getInitPromStats() + promStats = getInitPromStats(config) nullClient = &outputs.Client{ OutputType: "null", diff --git a/stats_prometheus.go b/stats_prometheus.go index daf6a43d3..c00131fe4 100644 --- a/stats_prometheus.go +++ b/stats_prometheus.go @@ -7,9 +7,9 @@ import ( "github.com/falcosecurity/falcosidekick/types" ) -func getInitPromStats() *types.PromStatistics { +func getInitPromStats(config *types.Configuration) *types.PromStatistics { promStats = &types.PromStatistics{ - Falco: getFalcoNewCounterVec(), + Falco: getFalcoNewCounterVec(config), Inputs: getInputNewCounterVec(), Outputs: getOutputNewCounterVec(), } @@ -34,16 +34,20 @@ func getOutputNewCounterVec() *prometheus.CounterVec { ) } -func getFalcoNewCounterVec() *prometheus.CounterVec { +func getFalcoNewCounterVec(config *types.Configuration) *prometheus.CounterVec { + labelnames := []string{ + "rule", + "priority", + "k8s_ns_name", + "k8s_pod_name", + } + for key := range config.CustomPrometheus { + labelnames = append(labelnames, key) + } return promauto.NewCounterVec( prometheus.CounterOpts{ Name: "falco_events", }, - []string{ - "rule", - "priority", - "k8s_ns_name", - "k8s_pod_name", - }, + labelnames, ) } diff --git a/types/types.go b/types/types.go index 7e9e1f554..f452782dc 100644 --- a/types/types.go +++ b/types/types.go @@ -25,6 +25,7 @@ type Configuration struct { ListenAddress string ListenPort int Customfields map[string]string + CustomPrometheus map[string]string Slack SlackOutputConfig Cliq CliqOutputConfig Mattermost MattermostOutputConfig From 3ded9f0d4fcb1a9114bcd0cbf1f1c979088c81b3 Mon Sep 17 00:00:00 2001 From: Jeremy Mill Date: Fri, 15 Apr 2022 15:25:14 -0400 Subject: [PATCH 2/4] use customfields in prometheus Signed-off-by: Jeremy Mill --- README.md | 5 ----- config.go | 18 +++--------------- config_example.yaml | 4 ---- handlers.go | 2 +- stats_prometheus.go | 2 +- types/types.go | 1 - 6 files changed, 5 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index d1a98afc9..05c8c03e2 100644 --- a/README.md +++ b/README.md @@ -173,11 +173,6 @@ customfields: # custom fields are added to falco events Ckey: "CValue" mutualtlsfilespath: "/etc/certs" # folder which will used to store client.crt, client.key and ca.crt files for mutual tls (default: "/etc/certs") -customprometheus: # custom labels to add to prometheus - Akey: "AValue" - Bkey: "BValue" - Ckey: "CValue" - slack: webhookurl: "" # Slack WebhookURL (ex: https://hooks.slack.com/services/XXXX/YYYY/ZZZZ), if not empty, Slack output is enabled #footer: "" # Slack footer diff --git a/config.go b/config.go index 56cb4a109..59d0e9e6a 100644 --- a/config.go +++ b/config.go @@ -19,10 +19,9 @@ import ( func getConfig() *types.Configuration { c := &types.Configuration{ - Customfields: make(map[string]string), - CustomPrometheus: make(map[string]string), - Webhook: types.WebhookOutputConfig{CustomHeaders: make(map[string]string)}, - CloudEvents: types.CloudEventsOutputConfig{Extensions: make(map[string]string)}, + Customfields: make(map[string]string), + Webhook: types.WebhookOutputConfig{CustomHeaders: make(map[string]string)}, + CloudEvents: types.CloudEventsOutputConfig{Extensions: make(map[string]string)}, } configFile := kingpin.Flag("config-file", "config file").Short('c').ExistingFile() @@ -328,7 +327,6 @@ func getConfig() *types.Configuration { } v.GetStringMapString("customfields") - v.GetStringMapString("customprometheus") v.GetStringMapString("Webhook.CustomHeaders") v.GetStringMapString("CloudEvents.Extensions") if err := v.Unmarshal(c); err != nil { @@ -345,16 +343,6 @@ func getConfig() *types.Configuration { } } - if value, present := os.LookupEnv("CUSTOMPROMETHEUS"); present { - customprometheus := strings.Split(value, ",") - for _, label := range customprometheus { - tagkeys := strings.Split(label, ":") - if len(tagkeys) == 2 { - c.CustomPrometheus[tagkeys[0]] = tagkeys[1] - } - } - } - if value, present := os.LookupEnv("WEBHOOK_CUSTOMHEADERS"); present { customfields := strings.Split(value, ",") for _, label := range customfields { diff --git a/config_example.yaml b/config_example.yaml index 9da9ffec9..fcb6d11a8 100644 --- a/config_example.yaml +++ b/config_example.yaml @@ -5,10 +5,6 @@ customfields: # custom fields are added to falco events Akey: "AValue" Bkey: "BValue" Ckey: "CValue" -customprometheus: # custom labels to add to prometheus - Akey: "AValue" - Bkey: "BValue" - Ckey: "CValue" mutualtlsfilespath: "/etc/certs" # folder which will used to store client.crt, client.key and ca.crt files for mutual tls (default: "/etc/certs") slack: diff --git a/handlers.go b/handlers.go index 2e8d2e584..bd60c544e 100644 --- a/handlers.go +++ b/handlers.go @@ -107,7 +107,7 @@ func newFalcoPayload(payload io.Reader) (types.FalcoPayload, error) { nullClient.CountMetric("falco.accepted", 1, []string{"priority:" + falcopayload.Priority.String()}) stats.Falco.Add(strings.ToLower(falcopayload.Priority.String()), 1) promLabels := map[string]string{"rule": falcopayload.Rule, "priority": falcopayload.Priority.String(), "k8s_ns_name": kn, "k8s_pod_name": kp} - for key, value := range config.CustomPrometheus { + for key, value := range config.Customfields { promLabels[key] = value } promStats.Falco.With(promLabels).Inc() diff --git a/stats_prometheus.go b/stats_prometheus.go index c00131fe4..0d24acfb1 100644 --- a/stats_prometheus.go +++ b/stats_prometheus.go @@ -41,7 +41,7 @@ func getFalcoNewCounterVec(config *types.Configuration) *prometheus.CounterVec { "k8s_ns_name", "k8s_pod_name", } - for key := range config.CustomPrometheus { + for key := range config.Customfields { labelnames = append(labelnames, key) } return promauto.NewCounterVec( diff --git a/types/types.go b/types/types.go index f452782dc..7e9e1f554 100644 --- a/types/types.go +++ b/types/types.go @@ -25,7 +25,6 @@ type Configuration struct { ListenAddress string ListenPort int Customfields map[string]string - CustomPrometheus map[string]string Slack SlackOutputConfig Cliq CliqOutputConfig Mattermost MattermostOutputConfig From 8761d4659b0c733df2f9e5f6afc4c46db0813e36 Mon Sep 17 00:00:00 2001 From: Jeremy Mill Date: Sat, 16 Apr 2022 18:04:49 -0400 Subject: [PATCH 3/4] detect unit testing and skip init Signed-off-by: Jeremy Mill --- main.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/main.go b/main.go index 465542f7c..9e97b7281 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "net/http" + "os" "strings" "github.com/DataDog/datadog-go/statsd" @@ -60,6 +61,13 @@ var ( ) func init() { + // detect unit testing and skip init. + // see: https://github.com/alecthomas/kingpin/issues/187 + testing := (strings.HasSuffix(os.Args[0], ".test") || + strings.HasSuffix(os.Args[0], "__debug_bin")) + if testing { + return + } config = getConfig() stats = getInitStats() promStats = getInitPromStats(config) From dd989ed5de01a3669385a81b8e43583710e47f74 Mon Sep 17 00:00:00 2001 From: Jeremy Mill Date: Sat, 16 Apr 2022 18:05:20 -0400 Subject: [PATCH 4/4] validate prom labels and add testing Signed-off-by: Jeremy Mill --- stats_prometheus.go | 12 +++++++++++- stats_prometheus_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 stats_prometheus_test.go diff --git a/stats_prometheus.go b/stats_prometheus.go index 0d24acfb1..a898d5615 100644 --- a/stats_prometheus.go +++ b/stats_prometheus.go @@ -1,6 +1,9 @@ package main import ( + "log" + "regexp" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -42,7 +45,14 @@ func getFalcoNewCounterVec(config *types.Configuration) *prometheus.CounterVec { "k8s_pod_name", } for key := range config.Customfields { - labelnames = append(labelnames, key) + matched, err := regexp.MatchString("^[a-zA-Z_:][a-zA-Z0-9_:]*$", key) + if err != nil { + log.Printf("Error matching prometheus label from custom fields. Err: %s", err) + continue + } + if matched { + labelnames = append(labelnames, key) + } } return promauto.NewCounterVec( prometheus.CounterOpts{ diff --git a/stats_prometheus_test.go b/stats_prometheus_test.go new file mode 100644 index 000000000..58c122dbc --- /dev/null +++ b/stats_prometheus_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/falcosecurity/falcosidekick/types" +) + +func TestFalcoNewCounterVec(t *testing.T) { + c := &types.Configuration{ + Customfields: make(map[string]string), + } + c.Customfields["test"] = "foo" + c.Customfields["should*fail"] = "bar" + + cv := getFalcoNewCounterVec(c) + shouldbe := []string{"rule", "priority", "k8s_ns_name", "k8s_pod_name", "test"} + mm, err := cv.GetMetricWithLabelValues(shouldbe...) + if err != nil { + t.Errorf("Error getting Metrics from promauto") + } + metricDescString := mm.Desc().String() + require.Equal(t, metricDescString, "Desc{fqName: \"falco_events\", help: \"\", constLabels: {}, variableLabels: [rule priority k8s_ns_name k8s_pod_name test]}") +}