From 316d790997e2f9ea3640965b6e7e6bfd8769391e Mon Sep 17 00:00:00 2001 From: Dieter Plaetinck Date: Wed, 10 Jan 2018 17:14:48 +0100 Subject: [PATCH 1/4] mt-index-cat: add tags validation filter --- cmd/mt-index-cat/main.go | 15 +++++++++++++++ docs/tools.md | 2 ++ vendor/gopkg.in/raintank/schema.v1/metric.go | 13 ++++++------- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/cmd/mt-index-cat/main.go b/cmd/mt-index-cat/main.go index 8076b0e4e2..548862da37 100644 --- a/cmd/mt-index-cat/main.go +++ b/cmd/mt-index-cat/main.go @@ -26,6 +26,7 @@ func main() { var addr string var prefix string var substr string + var tags string var from string var maxAge string var verbose bool @@ -35,6 +36,7 @@ func main() { globalFlags.StringVar(&addr, "addr", "http://localhost:6060", "graphite/metrictank address") globalFlags.StringVar(&prefix, "prefix", "", "only show metrics that have this prefix") globalFlags.StringVar(&substr, "substr", "", "only show metrics that have this substring") + globalFlags.StringVar(&tags, "tags", "", "tag filter. empty (default), or 'valid', or 'invalid'") globalFlags.StringVar(&from, "from", "30min", "for vegeta outputs, will generate requests for data starting from now minus... eg '30min', '5h', '14d', etc. or a unix timestamp") globalFlags.StringVar(&maxAge, "max-age", "6h30min", "max age (last update diff with now) of metricdefs. use 0 to disable") globalFlags.IntVar(&limit, "limit", 0, "only show this many metrics. use 0 to disable") @@ -111,6 +113,12 @@ func main() { os.Exit(1) } + if tags != "" && tags != "valid" && tags != "invalid" { + log.Println("invalid tags filter") + flag.Usage() + os.Exit(1) + } + globalFlags.Parse(os.Args[1:cassI]) cassFlags.Parse(os.Args[cassI+1 : len(os.Args)-1]) cassandra.Enabled = true @@ -157,6 +165,13 @@ func main() { for _, d := range defs { if prefix == "" || strings.HasPrefix(d.Metric, prefix) { if substr == "" || strings.Contains(d.Metric, substr) { + if tags != "" { + valid := schema.ValidateTags(d.Tags) + // skip the metric if the validation result is not what we want + if valid != (tags == "valid") { + continue + } + } show(d) shown += 1 if shown == limit { diff --git a/docs/tools.md b/docs/tools.md index 9b368cd6f9..11a2257cad 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -70,6 +70,8 @@ global config flags: only show metrics that have this prefix -substr string only show metrics that have this substring + -tags string + tag filter. empty (default), or 'valid', or 'invalid' -verbose print stats to stderr diff --git a/vendor/gopkg.in/raintank/schema.v1/metric.go b/vendor/gopkg.in/raintank/schema.v1/metric.go index 1de6dd1d1f..59204d9252 100644 --- a/vendor/gopkg.in/raintank/schema.v1/metric.go +++ b/vendor/gopkg.in/raintank/schema.v1/metric.go @@ -62,7 +62,7 @@ func (m *MetricData) Validate() error { if m.Mtype == "" || (m.Mtype != "gauge" && m.Mtype != "rate" && m.Mtype != "count" && m.Mtype != "counter" && m.Mtype != "timestamp") { return errInvalidMtype } - if !validateTags(m.Tags) { + if !ValidateTags(m.Tags) { return errInvalidTagFormat } return nil @@ -208,7 +208,7 @@ func (m *MetricDefinition) Validate() error { if m.Mtype == "" || (m.Mtype != "gauge" && m.Mtype != "rate" && m.Mtype != "count" && m.Mtype != "counter" && m.Mtype != "timestamp") { return errInvalidMtype } - if !validateTags(m.Tags) { + if !ValidateTags(m.Tags) { return errInvalidTagFormat } return nil @@ -263,11 +263,10 @@ func MetricDefinitionFromMetricData(d *MetricData) *MetricDefinition { return md } -// validateTags verifies that all the tags are of a valid format. If one or more -// are invalid it returns false, otherwise true. -// a valid format is anything that looks like key=value, the length of key and -// value must be >0 and both must not contain the ; character. -func validateTags(tags []string) bool { +// ValidateTags returns whether all tags are in a valid format. +// a valid format is anything that looks like key=value, +// the length of key and value must be >0 and both must not contain the ; character. +func ValidateTags(tags []string) bool { for _, t := range tags { if len(t) == 0 { return false From 8ede42165fada82f50adf87c784446e6bdd6b98d Mon Sep 17 00:00:00 2001 From: Dieter Plaetinck Date: Wed, 10 Jan 2018 17:15:13 +0100 Subject: [PATCH 2/4] cleanups --- vendor/gopkg.in/raintank/schema.v1/metric.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/vendor/gopkg.in/raintank/schema.v1/metric.go b/vendor/gopkg.in/raintank/schema.v1/metric.go index 59204d9252..e00403b1a2 100644 --- a/vendor/gopkg.in/raintank/schema.v1/metric.go +++ b/vendor/gopkg.in/raintank/schema.v1/metric.go @@ -272,26 +272,24 @@ func ValidateTags(tags []string) bool { return false } - // any = must not be the first character - if t[0] == 61 { + if t[0] == '=' { return false } foundEqual := false for pos := 0; pos < len(t); pos++ { - // no ; allowed - if t[pos] == 59 { + if t[pos] == ';' { return false } if !foundEqual { // no ! allowed in key - if t[pos] == 33 { + if t[pos] == '!' { return false } // found the first =, so this will be the separator between key & value - if t[pos] == 61 { + if t[pos] == '=' { // first equal sign must not be the last character if pos == len(t)-1 { return false From b8f494d110dedc009806211da24d56bec76e89ad Mon Sep 17 00:00:00 2001 From: Dieter Plaetinck Date: Wed, 10 Jan 2018 19:32:15 +0100 Subject: [PATCH 3/4] add tag filters none & some --- cmd/mt-index-cat/main.go | 26 +++++++++++++++++++++----- docs/tools.md | 9 ++++++++- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/cmd/mt-index-cat/main.go b/cmd/mt-index-cat/main.go index 548862da37..b715ccfe99 100644 --- a/cmd/mt-index-cat/main.go +++ b/cmd/mt-index-cat/main.go @@ -36,7 +36,7 @@ func main() { globalFlags.StringVar(&addr, "addr", "http://localhost:6060", "graphite/metrictank address") globalFlags.StringVar(&prefix, "prefix", "", "only show metrics that have this prefix") globalFlags.StringVar(&substr, "substr", "", "only show metrics that have this substring") - globalFlags.StringVar(&tags, "tags", "", "tag filter. empty (default), or 'valid', or 'invalid'") + globalFlags.StringVar(&tags, "tags", "", "tag filter. empty (default), 'some', 'none', 'valid', or 'invalid'") globalFlags.StringVar(&from, "from", "30min", "for vegeta outputs, will generate requests for data starting from now minus... eg '30min', '5h', '14d', etc. or a unix timestamp") globalFlags.StringVar(&maxAge, "max-age", "6h30min", "max age (last update diff with now) of metricdefs. use 0 to disable") globalFlags.IntVar(&limit, "limit", 0, "only show this many metrics. use 0 to disable") @@ -57,6 +57,13 @@ func main() { fmt.Printf("global config flags:\n\n") globalFlags.PrintDefaults() fmt.Println() + fmt.Println("tags filter:") + fmt.Println(" '' no filtering based on tags") + fmt.Println(" 'none' only show metrics that have no tags") + fmt.Println(" 'some' only show metrics that have one or more tags") + fmt.Println(" 'valid' only show metrics whose tags (if any) are valid") + fmt.Println(" 'invalid' only show metrics that have one or more invalid tags") + fmt.Println() fmt.Printf("idxtype: only 'cass' supported for now\n\n") fmt.Printf("cass config flags:\n\n") cassFlags.PrintDefaults() @@ -113,7 +120,7 @@ func main() { os.Exit(1) } - if tags != "" && tags != "valid" && tags != "invalid" { + if tags != "" && tags != "valid" && tags != "invalid" && tags != "some" && tags != "none" { log.Println("invalid tags filter") flag.Usage() os.Exit(1) @@ -166,11 +173,20 @@ func main() { if prefix == "" || strings.HasPrefix(d.Metric, prefix) { if substr == "" || strings.Contains(d.Metric, substr) { if tags != "" { - valid := schema.ValidateTags(d.Tags) - // skip the metric if the validation result is not what we want - if valid != (tags == "valid") { + if tags == "none" && len(d.Tags) != 0 { + continue + } + if tags == "some" && len(d.Tags) == 0 { continue } + if tags == "valid" || tags == "invalid" { + valid := schema.ValidateTags(d.Tags) + + // skip the metric if the validation result is not what we want + if valid != (tags == "valid") { + continue + } + } } show(d) shown += 1 diff --git a/docs/tools.md b/docs/tools.md index 11a2257cad..d25713b445 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -71,10 +71,17 @@ global config flags: -substr string only show metrics that have this substring -tags string - tag filter. empty (default), or 'valid', or 'invalid' + tag filter. empty (default), 'some', 'none', 'valid', or 'invalid' -verbose print stats to stderr +tags filter: + '' no filtering based on tags + 'none' only show metrics that have no tags + 'some' only show metrics that have one or more tags + 'valid' only show metrics whose tags (if any) are valid + 'invalid' only show metrics that have one or more invalid tags + idxtype: only 'cass' supported for now cass config flags: From 0a212974d0bb4bfc9dce4104c73977999282e456 Mon Sep 17 00:00:00 2001 From: Dieter Plaetinck Date: Thu, 11 Jan 2018 13:58:16 +0100 Subject: [PATCH 4/4] cleaner filter-skip conditions --- cmd/mt-index-cat/main.go | 48 +++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/cmd/mt-index-cat/main.go b/cmd/mt-index-cat/main.go index b715ccfe99..e46a35eba7 100644 --- a/cmd/mt-index-cat/main.go +++ b/cmd/mt-index-cat/main.go @@ -170,31 +170,33 @@ func main() { shown := 0 for _, d := range defs { - if prefix == "" || strings.HasPrefix(d.Metric, prefix) { - if substr == "" || strings.Contains(d.Metric, substr) { - if tags != "" { - if tags == "none" && len(d.Tags) != 0 { - continue - } - if tags == "some" && len(d.Tags) == 0 { - continue - } - if tags == "valid" || tags == "invalid" { - valid := schema.ValidateTags(d.Tags) - - // skip the metric if the validation result is not what we want - if valid != (tags == "valid") { - continue - } - } - } - show(d) - shown += 1 - if shown == limit { - break - } + // note that prefix and substr can be "", meaning filter disabled. + // the conditions handle this fine as well. + if !strings.HasPrefix(d.Metric, prefix) { + continue + } + if !strings.Contains(d.Metric, substr) { + continue + } + if tags == "none" && len(d.Tags) != 0 { + continue + } + if tags == "some" && len(d.Tags) == 0 { + continue + } + if tags == "valid" || tags == "invalid" { + valid := schema.ValidateTags(d.Tags) + + // skip the metric if the validation result is not what we want + if valid != (tags == "valid") { + continue } } + show(d) + shown += 1 + if shown == limit { + break + } } if verbose {