Add -diff option

README.md

@@ -6,12 +6,13 @@ Json custom metrics plugin for mackerel.io agent.
 ## Synopsis
 
 ```
-mackerel-plugin-json -url=<url to get JSON> [-prefix=<prefix for a metric name>] [-insecure] [-include=<expression>] [-exclude=<expression>]
+mackerel-plugin-json -url=<url to get JSON> [-prefix=<prefix for a metric name>] [-insecure] [-include=<expression>] [-exclude=<expression>] [-diff=<expression>]
 ```
 
 - `-url` always needs to be specified.
 - If you want to skip a SSL certificate validation (e.g. using a self-signed certificate), you need to specify `-insecure`.
 - If you want to get only metrics that matches a regular expression, specify the expression with `-include`. If you want to get only ones that doesn't match an expression, use `-exclude`.
+- If you want to get metrics as differential, specify the expression with `-diff` that matches to metric name.
 - Metrics that have non-number value (string, `null`, etc.) are omitted.
 - Arrays are supported. Metrics' name will contain a serial number like `custom.elements.0.xxx`, `custom.elements.1.xxx` and so on.
 

lib/json.go

@@ -1,18 +1,23 @@
 package mpjson
 
 import (
+	"crypto/sha1"
 	"crypto/tls"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"os"
+	"path/filepath"
 	"reflect"
 	"regexp"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/mackerelio/golib/pluginutil"
 )
 
 // JSONPlugin plugin for JSON
@@ -26,6 +31,13 @@ type JSONPlugin struct {
 	Stdin              bool
 	ExcludeExp         *regexp.Regexp
 	IncludeExp         *regexp.Regexp
+	DiffExp            *regexp.Regexp
+}
+
+// Values represents metric values
+type Values struct {
+	LastTimestamp int64              `json:"last_timestamp"`
+	Metrics       map[string]float64 `json:"metrics"`
 }
 
 func (p JSONPlugin) traverseMap(content interface{}, path []string) (map[string]float64, error) {
@@ -112,6 +124,81 @@ func (p JSONPlugin) FetchMetrics() (map[string]float64, error) {
 	return p.traverseMap(content, []string{p.Prefix})
 }
 
+func (p JSONPlugin) calcDiff(metrics map[string]float64, ts int64) (map[string]float64, error) {
+	lastValues := p.fetchLastValues()
+	diffTime := float64(ts - lastValues.LastTimestamp)
+	if lastValues.LastTimestamp != 0 && diffTime > 600 {
+		log.Println("Too long duration")
+		diffTime = 0 // do not calc diff
+	}
+
+	err := p.saveValues(Values{
+		LastTimestamp: ts,
+		Metrics:       metrics,
+	})
+	if err != nil {
+		log.Printf("Couldn't save values: %s", err)
+	}
+
+	for path, value := range metrics {
+		if !p.DiffExp.MatchString(path) {
+			continue
+		}
+		if lastValue, ok := lastValues.Metrics[path]; ok && diffTime > 0 {
+			diff := (value - lastValue) * 60 / diffTime
+			if diff >= 0 {
+				metrics[path] = diff
+			} else {
+				log.Printf("Counter %s seems to be reset.", path)
+				metrics[path] = 0
+			}
+		} else {
+			metrics[path] = 0 // cannot calc diff
+		}
+	}
+
+	return metrics, nil
+}
+
+func (p JSONPlugin) tempfilename() string {
+	// tempfiles have an unique name for each configuration.
+	filename := fmt.Sprintf("mackerel-plugin-json.%x", sha1.Sum([]byte(p.Prefix+p.URL)))
+	return filepath.Join(pluginutil.PluginWorkDir(), filename)
+}
+
+func (p JSONPlugin) fetchLastValues() Values {
+	v := Values{
+		LastTimestamp: 0,
+		Metrics:       make(map[string]float64),
+	}
+	filename := p.tempfilename()
+	f, err := os.Open(filename)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			log.Printf("Couldn't fetch last values: %s", err)
+		}
+		return v
+	}
+	defer f.Close()
+
+	if err = json.NewDecoder(f).Decode(&v); err != nil {
+		// invalid file format. try to delete
+		os.Remove(filename)
+	}
+	return v
+}
+
+func (p JSONPlugin) saveValues(v Values) error {
+	filename := p.tempfilename()
+	f, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	return json.NewEncoder(f).Encode(v)
+}
+
 // Do do doo
 func Do() {
 	url := flag.String("url", "", "URL to get a JSON")
@@ -120,6 +207,7 @@ func Do() {
 	insecure := flag.Bool("insecure", false, "Skip certificate verifications")
 	exclude := flag.String("exclude", `^$`, "Exclude metrics that matches the expression")
 	include := flag.String("include", ``, "Output metrics that matches the expression")
+	diff := flag.String("diff", ``, "Calculate difference of metrics that matches the expression")
 	flag.Parse()
 
 	if (*url == "") && (*stdin == false) {
@@ -149,9 +237,21 @@ func Do() {
 		fmt.Printf("include expression is invalid: %s", err)
 		os.Exit(1)
 	}
+	if *diff != "" {
+		jsonplugin.DiffExp, err = regexp.Compile(*diff)
+		if err != nil {
+			fmt.Printf("diff expression is invalid: %s", err)
+			os.Exit(1)
+		}
+	}
 
 	metrics, _ := jsonplugin.FetchMetrics()
+
+	ts := time.Now().Unix()
+	if jsonplugin.DiffExp != nil {
+		metrics, _ = jsonplugin.calcDiff(metrics, ts)
+	}
 	for k, v := range metrics {
-		fmt.Printf("%s\t%f\t%d\n", k, v, time.Now().Unix())
+		fmt.Printf("%s\t%f\t%d\n", k, v, ts)
 	}
 }

lib/json_test.go

@@ -4,8 +4,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"regexp"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -86,3 +88,53 @@ func TestOutputMetric(t *testing.T) {
 	assert.EqualValues(t, "", path)
 	assert.EqualValues(t, 0, value)
 }
+
+func TestCalcDiff(t *testing.T) {
+	dir, err := ioutil.TempDir("", "makerel-plugin-json-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	orig := os.Getenv("MACKEREL_PLUGIN_WORKDIR")
+	os.Setenv("MACKEREL_PLUGIN_WORKDIR", dir)        // to separate tempfile for each test
+	defer os.Setenv("MACKEREL_PLUGIN_WORKDIR", orig) // restore
+
+	var p JSONPlugin
+	p.Prefix = "testprefix"
+	p.DiffExp = regexp.MustCompile(`foo`)
+	p.ExcludeExp = regexp.MustCompile(`^$`)
+	p.IncludeExp = regexp.MustCompile(``)
+
+	ts := time.Now().Unix()
+	{
+		bytes, _ := ioutil.ReadFile("testdata/diff_before.json")
+		var content interface{}
+		err := json.Unmarshal(bytes, &content)
+		if err != nil {
+			panic(err)
+		}
+		stat, err := p.traverseMap(content, []string{p.Prefix})
+		assert.Nil(t, err)
+		stat, err = p.calcDiff(stat, ts)
+		assert.EqualValues(t, 0, stat[p.Prefix+".foo_1"]) // at first
+		assert.EqualValues(t, 0, stat[p.Prefix+".foo_2"]) // at first
+		assert.EqualValues(t, 300, stat[p.Prefix+".bar"])
+	}
+
+	ts += 60
+	{
+		bytes, _ := ioutil.ReadFile("testdata/diff_after.json")
+		var content interface{}
+		err := json.Unmarshal(bytes, &content)
+		if err != nil {
+			panic(err)
+		}
+		stat, err := p.traverseMap(content, []string{p.Prefix})
+		assert.Nil(t, err)
+		stat, err = p.calcDiff(stat, ts)
+		assert.EqualValues(t, 0, stat[p.Prefix+".foo_1"])   // reset
+		assert.EqualValues(t, 100, stat[p.Prefix+".foo_2"]) // diff
+		assert.EqualValues(t, 400, stat[p.Prefix+".bar"])
+	}
+}

lib/testdata/diff_after.json

@@ -0,0 +1,5 @@
+{
+  "foo_1": 50,
+  "foo_2": 300,
+  "bar": 400
+}

lib/testdata/diff_before.json

@@ -0,0 +1,5 @@
+{
+  "foo_1": 100,
+  "foo_2": 200,
+  "bar": 300
+}