Allow wildcard filtering of varnish stats

closes #1275

CHANGELOG.md

@@ -24,6 +24,7 @@ time before a new metric is included by the plugin.
 - [#1264](https://github.com/influxdata/telegraf/pull/1264): Add SSL config options to http_response plugin.
 - [#1272](https://github.com/influxdata/telegraf/pull/1272): graphite parser: add ability to specify multiple tag keys, for consistency with influxdb parser.
 - [#1265](https://github.com/influxdata/telegraf/pull/1265): Make dns lookups for chrony configurable. Thanks @zbindenren!
+- [#1275](https://github.com/influxdata/telegraf/pull/1275): Allow wildcard filtering of varnish stats.
 
 ### Bugfixes
 

Godeps

@@ -16,7 +16,7 @@ github.com/eapache/go-resiliency b86b1ec0dd4209a588dc1285cdd471e73525c0b3
 github.com/eapache/queue ded5959c0d4e360646dc9e9908cff48666781367
 github.com/eclipse/paho.mqtt.golang 0f7a459f04f13a41b7ed752d47944528d4bf9a86
 github.com/go-sql-driver/mysql 1fca743146605a172a266e1654e01e5cd5669bee
-github.com/gobwas/glob d877f6352135181470c40c73ebb81aefa22115fa
+github.com/gobwas/glob 49571a1557cd20e6a2410adc6421f85b66c730b5
 github.com/golang/protobuf 552c7b9542c194800fd493123b3798ef0a832032
 github.com/golang/snappy 427fb6fc07997f43afa32f35e850833760e489a7
 github.com/gonuts/go-shellquote e842a11b24c6abfb3dd27af69a17f482e4b483c2

internal/internal.go

@@ -16,6 +16,8 @@ import (
 	"strings"
 	"time"
 	"unicode"
+
+	"github.com/gobwas/glob"
 )
 
 const alphanum string = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
@@ -205,3 +207,24 @@ func WaitTimeout(c *exec.Cmd, timeout time.Duration) error {
 		return TimeoutErr
 	}
 }
+
+// CompileFilter takes a list of glob "filters", ie:
+//   ["MAIN.*", "CPU.*", "NET"]
+// and compiles them into a glob object. This glob object can
+// then be used to match keys to the filter.
+func CompileFilter(filters []string) (glob.Glob, error) {
+	var out glob.Glob
+
+	// return if there is nothing to compile
+	if len(filters) == 0 {
+		return out, nil
+	}
+
+	var err error
+	if len(filters) == 1 {
+		out, err = glob.Compile(filters[0])
+	} else {
+		out, err = glob.Compile("{" + strings.Join(filters, ",") + "}")
+	}
+	return out, err
+}

internal/internal_test.go

@@ -106,3 +106,34 @@ func TestRunError(t *testing.T) {
 
 	assert.Error(t, err)
 }
+
+func TestCompileFilter(t *testing.T) {
+	f, err := CompileFilter([]string{})
+	assert.NoError(t, err)
+	assert.Nil(t, f)
+
+	f, err = CompileFilter([]string{"cpu"})
+	assert.NoError(t, err)
+	assert.True(t, f.Match("cpu"))
+	assert.False(t, f.Match("cpu0"))
+	assert.False(t, f.Match("mem"))
+
+	f, err = CompileFilter([]string{"cpu*"})
+	assert.NoError(t, err)
+	assert.True(t, f.Match("cpu"))
+	assert.True(t, f.Match("cpu0"))
+	assert.False(t, f.Match("mem"))
+
+	f, err = CompileFilter([]string{"cpu", "mem"})
+	assert.NoError(t, err)
+	assert.True(t, f.Match("cpu"))
+	assert.False(t, f.Match("cpu0"))
+	assert.True(t, f.Match("mem"))
+
+	f, err = CompileFilter([]string{"cpu", "mem", "net*"})
+	assert.NoError(t, err)
+	assert.True(t, f.Match("cpu"))
+	assert.False(t, f.Match("cpu0"))
+	assert.True(t, f.Match("mem"))
+	assert.True(t, f.Match("network"))
+}

internal/models/filter.go

@@ -2,11 +2,11 @@ package internal_models
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/gobwas/glob"
 
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
 )
 
 // TagFilter is the name of a tag, and the values on which to filter
@@ -42,62 +42,48 @@ type Filter struct {
 // Compile all Filter lists into glob.Glob objects.
 func (f *Filter) CompileFilter() error {
 	var err error
-	f.nameDrop, err = compileFilter(f.NameDrop)
+	f.nameDrop, err = internal.CompileFilter(f.NameDrop)
 	if err != nil {
 		return fmt.Errorf("Error compiling 'namedrop', %s", err)
 	}
-	f.namePass, err = compileFilter(f.NamePass)
+	f.namePass, err = internal.CompileFilter(f.NamePass)
 	if err != nil {
 		return fmt.Errorf("Error compiling 'namepass', %s", err)
 	}
 
-	f.fieldDrop, err = compileFilter(f.FieldDrop)
+	f.fieldDrop, err = internal.CompileFilter(f.FieldDrop)
 	if err != nil {
 		return fmt.Errorf("Error compiling 'fielddrop', %s", err)
 	}
-	f.fieldPass, err = compileFilter(f.FieldPass)
+	f.fieldPass, err = internal.CompileFilter(f.FieldPass)
 	if err != nil {
 		return fmt.Errorf("Error compiling 'fieldpass', %s", err)
 	}
 
-	f.tagExclude, err = compileFilter(f.TagExclude)
+	f.tagExclude, err = internal.CompileFilter(f.TagExclude)
 	if err != nil {
 		return fmt.Errorf("Error compiling 'tagexclude', %s", err)
 	}
-	f.tagInclude, err = compileFilter(f.TagInclude)
+	f.tagInclude, err = internal.CompileFilter(f.TagInclude)
 	if err != nil {
 		return fmt.Errorf("Error compiling 'taginclude', %s", err)
 	}
 
 	for i, _ := range f.TagDrop {
-		f.TagDrop[i].filter, err = compileFilter(f.TagDrop[i].Filter)
+		f.TagDrop[i].filter, err = internal.CompileFilter(f.TagDrop[i].Filter)
 		if err != nil {
 			return fmt.Errorf("Error compiling 'tagdrop', %s", err)
 		}
 	}
 	for i, _ := range f.TagPass {
-		f.TagPass[i].filter, err = compileFilter(f.TagPass[i].Filter)
+		f.TagPass[i].filter, err = internal.CompileFilter(f.TagPass[i].Filter)
 		if err != nil {
 			return fmt.Errorf("Error compiling 'tagpass', %s", err)
 		}
 	}
 	return nil
 }
 
-func compileFilter(filter []string) (glob.Glob, error) {
-	if len(filter) == 0 {
-		return nil, nil
-	}
-	var g glob.Glob
-	var err error
-	if len(filter) == 1 {
-		g, err = glob.Compile(filter[0])
-	} else {
-		g, err = glob.Compile("{" + strings.Join(filter, ",") + "}")
-	}
-	return g, err
-}
-
 func (f *Filter) ShouldMetricPass(metric telegraf.Metric) bool {
 	if f.ShouldNamePass(metric.Name()) && f.ShouldTagsPass(metric.Tags()) {
 		return true

plugins/inputs/varnish/varnish.go

@@ -10,33 +10,37 @@ import (
 	"os/exec"
 	"strconv"
 	"strings"
+	"time"
+
+	"github.com/gobwas/glob"
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/plugins/inputs"
-	"time"
 )
 
-const (
-	kwAll = "all"
-)
+type runner func(cmdName string) (*bytes.Buffer, error)
 
 // Varnish is used to store configuration values
 type Varnish struct {
 	Stats  []string
 	Binary string
+
+	filter glob.Glob
+	run    runner
 }
 
 var defaultStats = []string{"MAIN.cache_hit", "MAIN.cache_miss", "MAIN.uptime"}
 var defaultBinary = "/usr/bin/varnishstat"
 
-var varnishSampleConfig = `
+var sampleConfig = `
   ## The default location of the varnishstat binary can be overridden with:
   binary = "/usr/bin/varnishstat"
 
   ## By default, telegraf gather stats for 3 metric points.
   ## Setting stats will override the defaults shown below.
-  ## stats may also be set to ["all"], which will collect all stats
+  ## Glob matching can be used, ie, stats = ["MAIN.*"]
+  ## stats may also be set to ["*"], which will collect all stats
   stats = ["MAIN.cache_hit", "MAIN.cache_miss", "MAIN.uptime"]
 `
 
@@ -46,44 +50,11 @@ func (s *Varnish) Description() string {
 
 // SampleConfig displays configuration instructions
 func (s *Varnish) SampleConfig() string {
-	return fmt.Sprintf(varnishSampleConfig, strings.Join(defaultStats, "\",\""))
-}
-
-func (s *Varnish) setDefaults() {
-	if len(s.Stats) == 0 {
-		s.Stats = defaultStats
-	}
-
-	if s.Binary == "" {
-		s.Binary = defaultBinary
-	}
-}
-
-// Builds a filter function that will indicate whether a given stat should
-// be reported
-func (s *Varnish) statsFilter() func(string) bool {
-	s.setDefaults()
-
-	// Build a set for constant-time lookup of whether stats should be included
-	filter := make(map[string]struct{})
-	for _, s := range s.Stats {
-		filter[s] = struct{}{}
-	}
-
-	// Create a function that respects the kwAll by always returning true
-	// if it is set
-	return func(stat string) bool {
-		if s.Stats[0] == kwAll {
-			return true
-		}
-
-		_, found := filter[stat]
-		return found
-	}
+	return sampleConfig
 }
 
 // Shell out to varnish_stat and return the output
-var varnishStat = func(cmdName string) (*bytes.Buffer, error) {
+func varnishRunner(cmdName string) (*bytes.Buffer, error) {
 	cmdArgs := []string{"-1"}
 
 	cmd := exec.Command(cmdName, cmdArgs...)
@@ -104,13 +75,27 @@ var varnishStat = func(cmdName string) (*bytes.Buffer, error) {
 // 'section' tag and all stats that share that prefix will be reported as fields
 // with that tag
 func (s *Varnish) Gather(acc telegraf.Accumulator) error {
-	s.setDefaults()
-	out, err := varnishStat(s.Binary)
+	if s.filter == nil {
+		var err error
+		if len(s.Stats) == 0 {
+			s.filter, err = internal.CompileFilter(defaultStats)
+		} else {
+			// legacy support, change "all" -> "*":
+			if s.Stats[0] == "all" {
+				s.Stats[0] = "*"
+			}
+			s.filter, err = internal.CompileFilter(s.Stats)
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	out, err := s.run(s.Binary)
 	if err != nil {
 		return fmt.Errorf("error gathering metrics: %s", err)
 	}
 
-	statsFilter := s.statsFilter()
 	sectionMap := make(map[string]map[string]interface{})
 	scanner := bufio.NewScanner(out)
 	for scanner.Scan() {
@@ -125,7 +110,7 @@ func (s *Varnish) Gather(acc telegraf.Accumulator) error {
 		stat := cols[0]
 		value := cols[1]
 
-		if !statsFilter(stat) {
+		if s.filter != nil && !s.filter.Match(stat) {
 			continue
 		}
 
@@ -138,7 +123,7 @@ func (s *Varnish) Gather(acc telegraf.Accumulator) error {
 			sectionMap[section] = make(map[string]interface{})
 		}
 
-		sectionMap[section][field], err = strconv.Atoi(value)
+		sectionMap[section][field], err = strconv.ParseUint(value, 10, 64)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Expected a numeric value for %s = %v\n",
 				stat, value)
@@ -160,5 +145,9 @@ func (s *Varnish) Gather(acc telegraf.Accumulator) error {
 }
 
 func init() {
-	inputs.Add("varnish", func() telegraf.Input { return &Varnish{} })
+	inputs.Add("varnish", func() telegraf.Input {
+		return &Varnish{
+			run: varnishRunner,
+		}
+	})
 }