From b2d7fbf23cf90791b1b08e572f9ab9117553a020 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Mon, 20 Feb 2017 18:08:49 +0100
Subject: [PATCH 01/29] go-carbon: support multiple aggregationMethods per
 pattern

note that whisper files and structures are untouched. We only use
this extended format for metrictank.

also: if failure to parse, return error
---
 .../lomik/go-carbon/persister/whisper.go      |  2 +-
 .../persister/whisper_aggregation.go          | 39 ++++++++++---------
 2 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/vendor/github.com/lomik/go-carbon/persister/whisper.go b/vendor/github.com/lomik/go-carbon/persister/whisper.go
index 8e04776be6..ddb3c46657 100644
--- a/vendor/github.com/lomik/go-carbon/persister/whisper.go
+++ b/vendor/github.com/lomik/go-carbon/persister/whisper.go
@@ -129,7 +129,7 @@ func store(p *Whisper, values *points.Points) {
 			return
 		}
 
-		w, err = whisper.CreateWithOptions(path, schema.Retentions, aggr.aggregationMethod, float32(aggr.xFilesFactor), &whisper.Options{
+		w, err = whisper.CreateWithOptions(path, schema.Retentions, aggr.aggregationMethod[0], float32(aggr.xFilesFactor), &whisper.Options{
 			Sparse: p.sparse,
 		})
 		if err != nil {
diff --git a/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go b/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
index 444406298f..4b17511dfa 100644
--- a/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
+++ b/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
@@ -5,6 +5,7 @@ Schemas read code from https://github.com/grobian/carbonwriter/
 */
 
 import (
+	"fmt"
 	"regexp"
 	"strconv"
 	"strings"
@@ -19,7 +20,7 @@ type whisperAggregationItem struct {
 	pattern              *regexp.Regexp
 	xFilesFactor         float64
 	aggregationMethodStr string
-	aggregationMethod    whisper.AggregationMethod
+	aggregationMethod    []whisper.AggregationMethod
 }
 
 // WhisperAggregation ...
@@ -37,7 +38,7 @@ func NewWhisperAggregation() *WhisperAggregation {
 			pattern:              nil,
 			xFilesFactor:         0.5,
 			aggregationMethodStr: "average",
-			aggregationMethod:    whisper.Average,
+			aggregationMethod:    []whisper.AggregationMethod{whisper.Average},
 		},
 	}
 }
@@ -77,25 +78,27 @@ func ReadWhisperAggregation(file string) (*WhisperAggregation, error) {
 		if err != nil {
 			logrus.Errorf("failed to parse xFilesFactor '%s' in %s: %s",
 				s.ValueOf("xFilesFactor"), item.name, err.Error())
-			continue
+			return nil, err
 		}
 
 		item.aggregationMethodStr = s.ValueOf("aggregationMethod")
-		switch item.aggregationMethodStr {
-		case "average", "avg":
-			item.aggregationMethod = whisper.Average
-		case "sum":
-			item.aggregationMethod = whisper.Sum
-		case "last":
-			item.aggregationMethod = whisper.Last
-		case "max":
-			item.aggregationMethod = whisper.Max
-		case "min":
-			item.aggregationMethod = whisper.Min
-		default:
-			logrus.Errorf("unknown aggregation method '%s'",
-				s.ValueOf("aggregationMethod"))
-			continue
+		methodStrs := strings.Split(item.aggregationMethodStr, ",")
+		for _, methodStr := range methodStrs {
+			switch methodStr {
+			case "average", "avg":
+				item.aggregationMethod = append(item.aggregationMethod, whisper.Average)
+			case "sum":
+				item.aggregationMethod = append(item.aggregationMethod, whisper.Sum)
+			case "last":
+				item.aggregationMethod = append(item.aggregationMethod, whisper.Last)
+			case "max":
+				item.aggregationMethod = append(item.aggregationMethod, whisper.Max)
+			case "min":
+				item.aggregationMethod = append(item.aggregationMethod, whisper.Min)
+			default:
+				logrus.Errorf("unknown aggregation method '%s'", methodStr)
+				return result, fmt.Errorf("unknown aggregation method %q", methodStr)
+			}
 		}
 
 		logrus.Debugf("[persister] Adding aggregation [%s] pattern = %s aggregationMethod = %s xFilesFactor = %f",

From eca803104f967303aa4fdac3b2480c24793270d0 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Tue, 28 Feb 2017 20:31:43 +0100
Subject: [PATCH 02/29] go-carbon: support extra fields for retentions

ChunkSpan, NumChunks and Ready fields are now supported,
also optionally specified via the config file
---
 .../go-carbon/persister/whisper_schema.go     | 116 ++++++++++++++++--
 vendor/github.com/lomik/go-whisper/whisper.go |  24 ++--
 2 files changed, 122 insertions(+), 18 deletions(-)

diff --git a/vendor/github.com/lomik/go-carbon/persister/whisper_schema.go b/vendor/github.com/lomik/go-carbon/persister/whisper_schema.go
index 881eaaa38c..2e473998b4 100644
--- a/vendor/github.com/lomik/go-carbon/persister/whisper_schema.go
+++ b/vendor/github.com/lomik/go-carbon/persister/whisper_schema.go
@@ -5,6 +5,7 @@
 package persister
 
 import (
+	"errors"
 	"fmt"
 	"regexp"
 	"sort"
@@ -13,8 +14,57 @@ import (
 
 	"github.com/alyu/configparser"
 	"github.com/lomik/go-whisper"
+	"github.com/raintank/dur"
+	"github.com/raintank/metrictank/mdata/chunk"
 )
 
+const Month_sec = 60 * 60 * 24 * 28
+
+var ChunkSpans = [32]uint32{
+	1,
+	5,
+	10,
+	15,
+	20,
+	30,
+	60,        // 1m
+	90,        // 1.5m
+	2 * 60,    // 2m
+	3 * 60,    // 3m
+	5 * 60,    // 5m
+	10 * 60,   // 10m
+	15 * 60,   // 15m
+	20 * 60,   // 20m
+	30 * 60,   // 30m
+	45 * 60,   // 45m
+	3600,      // 1h
+	90 * 60,   // 1.5h
+	2 * 3600,  // 2h
+	150 * 60,  // 2.5h
+	3 * 3600,  // 3h
+	4 * 3600,  // 4h
+	5 * 3600,  // 5h
+	6 * 3600,  // 6h
+	7 * 3600,  // 7h
+	8 * 3600,  // 8h
+	9 * 3600,  // 9h
+	10 * 3600, // 10h
+	12 * 3600, // 12h
+	15 * 3600, // 15h
+	18 * 3600, // 18h
+	24 * 3600, // 24h
+}
+
+type SpanCode uint8
+
+var RevChunkSpans = make(map[uint32]SpanCode, len(ChunkSpans))
+
+func init() {
+	for k, v := range ChunkSpans {
+		RevChunkSpans[v] = SpanCode(k)
+	}
+}
+
 // Schema represents one schema setting
 type Schema struct {
 	Name         string
@@ -47,7 +97,7 @@ func ParseRetentionDefs(retentionDefs string) (whisper.Retentions, error) {
 	for _, retentionDef := range strings.Split(retentionDefs, ",") {
 		retentionDef = strings.TrimSpace(retentionDef)
 		parts := strings.Split(retentionDef, ":")
-		if len(parts) != 2 {
+		if len(parts) < 2 || len(parts) > 5 {
 			return nil, fmt.Errorf("bad retentions spec %q", retentionDef)
 		}
 
@@ -55,19 +105,67 @@ func ParseRetentionDefs(retentionDefs string) (whisper.Retentions, error) {
 		val1, err1 := strconv.ParseInt(parts[0], 10, 0)
 		val2, err2 := strconv.ParseInt(parts[1], 10, 0)
 
+		var retention *whisper.Retention
+		var err error
 		if err1 == nil && err2 == nil {
-			retention := whisper.NewRetention(int(val1), int(val2))
-			retentions = append(retentions, &retention)
-			continue
+			ret := whisper.NewRetention(int(val1), int(val2))
+			retention = &ret
+		} else {
+			// try new format
+			retention, err = whisper.ParseRetentionDef(retentionDef)
+			if err != nil {
+				return nil, err
+			}
 		}
-
-		// try new format
-		retention, err := whisper.ParseRetentionDef(retentionDef)
-		if err != nil {
-			return nil, err
+		if len(parts) >= 3 {
+			retention.ChunkSpan, err = dur.ParseUNsec(parts[2])
+			if err != nil {
+				return nil, err
+			}
+			if (Month_sec % retention.ChunkSpan) != 0 {
+				return nil, errors.New("chunkSpan must fit without remainders into month_sec (28*24*60*60)")
+			}
+			_, ok := chunk.RevChunkSpans[retention.ChunkSpan]
+			if !ok {
+				return nil, fmt.Errorf("chunkSpan %s is not a valid value (https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans)", parts[2])
+			}
+		} else {
+			// default to a valid chunkspan that can hold at least 100 points, or select the largest one otherwise.
+			approxSpan := uint32(retention.SecondsPerPoint() * 100)
+			var span uint32
+			for _, span = range ChunkSpans {
+				if span >= approxSpan {
+					break
+				}
+			}
+			retention.ChunkSpan = span
+		}
+		retention.NumChunks = 2
+		if len(parts) >= 4 {
+			i, err := strconv.Atoi(parts[3])
+			if err != nil {
+				return nil, err
+			}
+			retention.NumChunks = uint32(i)
 		}
+		retention.Ready = true
+		if len(parts) == 5 {
+			retention.Ready, err = strconv.ParseBool(parts[4])
+			if err != nil {
+				return nil, err
+			}
+		}
+
 		retentions = append(retentions, retention)
 	}
+	prevInterval := 0
+	for _, r := range retentions {
+		if r.SecondsPerPoint() <= prevInterval {
+			// for users' own sanity, and also so we can reference to archive 0, 1, 2 and it means what you expect
+			return nil, errors.New("aggregation archives must be in ascending order")
+		}
+		prevInterval = r.SecondsPerPoint()
+	}
 	return retentions, nil
 }
 
diff --git a/vendor/github.com/lomik/go-whisper/whisper.go b/vendor/github.com/lomik/go-whisper/whisper.go
index de1595182b..1ff4d4667a 100644
--- a/vendor/github.com/lomik/go-whisper/whisper.go
+++ b/vendor/github.com/lomik/go-whisper/whisper.go
@@ -96,7 +96,7 @@ func parseRetentionPart(retentionPart string) (int, error) {
 */
 func ParseRetentionDef(retentionDef string) (*Retention, error) {
 	parts := strings.Split(retentionDef, ":")
-	if len(parts) != 2 {
+	if len(parts) < 2 {
 		return nil, fmt.Errorf("Not enough parts in retentionDef [%v]", retentionDef)
 	}
 	precision, err := parseRetentionPart(parts[0])
@@ -104,13 +104,14 @@ func ParseRetentionDef(retentionDef string) (*Retention, error) {
 		return nil, fmt.Errorf("Failed to parse precision: %v", err)
 	}
 
-	points, err := parseRetentionPart(parts[1])
+	ttl, err := parseRetentionPart(parts[1])
 	if err != nil {
 		return nil, fmt.Errorf("Failed to parse points: %v", err)
 	}
-	points /= precision
 
-	return &Retention{precision, points}, err
+	return &Retention{
+		secondsPerPoint: precision,
+		numberOfPoints:  ttl / precision}, err
 }
 
 func ParseRetentionDefs(retentionDefs string) (Retentions, error) {
@@ -789,10 +790,15 @@ func (whisper *Whisper) readInt(offset int64) (int, error) {
   it records.
 */
 type Retention struct {
-	secondsPerPoint int
-	numberOfPoints  int
+	secondsPerPoint int    // interval in seconds
+	numberOfPoints  int    // ~ttl
+	ChunkSpan       uint32 // duration of chunk of aggregated metric for storage, controls how many aggregated points go into 1 chunk
+	NumChunks       uint32 // number of chunks to keep in memory. remember, for a query from now until 3 months ago, we will end up querying the memory server as well.
+	Ready           bool   // ready for reads?
 }
 
+// note missing, but can be gotten via MaxRetention: TTL       uint32 // how many seconds to keep the chunk in cassandra
+
 func (retention *Retention) MaxRetention() int {
 	return retention.secondsPerPoint * retention.numberOfPoints
 }
@@ -811,8 +817,8 @@ func (retention *Retention) NumberOfPoints() int {
 
 func NewRetention(secondsPerPoint, numberOfPoints int) Retention {
 	return Retention{
-		secondsPerPoint,
-		numberOfPoints,
+		secondsPerPoint: secondsPerPoint,
+		numberOfPoints:  numberOfPoints,
 	}
 }
 
@@ -998,7 +1004,7 @@ func unpackFloat64(b []byte) float64 {
 }
 
 func unpackArchiveInfo(b []byte) *archiveInfo {
-	return &archiveInfo{Retention{unpackInt(b[IntSize : IntSize*2]), unpackInt(b[IntSize*2 : IntSize*3])}, unpackInt(b[:IntSize])}
+	return &archiveInfo{Retention{secondsPerPoint: unpackInt(b[IntSize : IntSize*2]), numberOfPoints: unpackInt(b[IntSize*2 : IntSize*3])}, unpackInt(b[:IntSize])}
 }
 
 func unpackDataPoint(b []byte) dataPoint {

From 8d14264b717ef87b2cfe04f4b318f1883ad0264b Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Wed, 1 Mar 2017 11:22:01 +0100
Subject: [PATCH 03/29] go-carbon: upon schema/agg match, also return index

so we can efficiently reference a schema or aggregation definition
also make WhisperAggregation.Match public
---
 vendor/github.com/lomik/go-carbon/persister/whisper.go    | 4 ++--
 .../lomik/go-carbon/persister/whisper_aggregation.go      | 8 ++++----
 .../lomik/go-carbon/persister/whisper_schema.go           | 8 ++++----
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/vendor/github.com/lomik/go-carbon/persister/whisper.go b/vendor/github.com/lomik/go-carbon/persister/whisper.go
index ddb3c46657..932e77cab1 100644
--- a/vendor/github.com/lomik/go-carbon/persister/whisper.go
+++ b/vendor/github.com/lomik/go-carbon/persister/whisper.go
@@ -104,13 +104,13 @@ func store(p *Whisper, values *points.Points) {
 			return
 		}
 
-		schema, ok := p.schemas.Match(values.Metric)
+		_, schema, ok := p.schemas.Match(values.Metric)
 		if !ok {
 			logrus.Errorf("[persister] No storage schema defined for %s", values.Metric)
 			return
 		}
 
-		aggr := p.aggregation.match(values.Metric)
+		_, aggr := p.aggregation.Match(values.Metric)
 		if aggr == nil {
 			logrus.Errorf("[persister] No storage aggregation defined for %s", values.Metric)
 			return
diff --git a/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go b/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
index 4b17511dfa..3066cca61b 100644
--- a/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
+++ b/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
@@ -112,11 +112,11 @@ func ReadWhisperAggregation(file string) (*WhisperAggregation, error) {
 }
 
 // Match find schema for metric
-func (a *WhisperAggregation) match(metric string) *whisperAggregationItem {
-	for _, s := range a.Data {
+func (a *WhisperAggregation) Match(metric string) (uint16, *whisperAggregationItem) {
+	for i, s := range a.Data {
 		if s.pattern.MatchString(metric) {
-			return s
+			return uint16(i), s
 		}
 	}
-	return a.Default
+	return uint16(len(a.Data)), a.Default // default has the index of one more than last of what's in a.Data
 }
diff --git a/vendor/github.com/lomik/go-carbon/persister/whisper_schema.go b/vendor/github.com/lomik/go-carbon/persister/whisper_schema.go
index 2e473998b4..e1369ec553 100644
--- a/vendor/github.com/lomik/go-carbon/persister/whisper_schema.go
+++ b/vendor/github.com/lomik/go-carbon/persister/whisper_schema.go
@@ -82,13 +82,13 @@ func (s WhisperSchemas) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
 func (s WhisperSchemas) Less(i, j int) bool { return s[i].Priority >= s[j].Priority }
 
 // Match finds the schema for metric or returns false if none found
-func (s WhisperSchemas) Match(metric string) (Schema, bool) {
-	for _, schema := range s {
+func (s WhisperSchemas) Match(metric string) (uint16, Schema, bool) {
+	for i, schema := range s {
 		if schema.Pattern.MatchString(metric) {
-			return schema, true
+			return uint16(i), schema, true
 		}
 	}
-	return Schema{}, false
+	return 0, Schema{}, false
 }
 
 // ParseRetentionDefs parses retention definitions into a Retentions structure

From dcf711607212b5593c58fc762443346bda0766b5 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Wed, 1 Mar 2017 13:11:17 +0100
Subject: [PATCH 04/29] go-carbon: expose WhisperAggregationItem and properties

because we want to work with them in metrictank
---
 .../lomik/go-carbon/persister/whisper.go      |  4 +--
 .../persister/whisper_aggregation.go          | 36 +++++++++----------
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/vendor/github.com/lomik/go-carbon/persister/whisper.go b/vendor/github.com/lomik/go-carbon/persister/whisper.go
index 932e77cab1..932d291556 100644
--- a/vendor/github.com/lomik/go-carbon/persister/whisper.go
+++ b/vendor/github.com/lomik/go-carbon/persister/whisper.go
@@ -120,7 +120,7 @@ func store(p *Whisper, values *points.Points) {
 			"retention":    schema.RetentionStr,
 			"schema":       schema.Name,
 			"aggregation":  aggr.name,
-			"xFilesFactor": aggr.xFilesFactor,
+			"xFilesFactor": aggr.XFilesFactor,
 			"method":       aggr.aggregationMethodStr,
 		}).Debugf("[persister] Creating %s", path)
 
@@ -129,7 +129,7 @@ func store(p *Whisper, values *points.Points) {
 			return
 		}
 
-		w, err = whisper.CreateWithOptions(path, schema.Retentions, aggr.aggregationMethod[0], float32(aggr.xFilesFactor), &whisper.Options{
+		w, err = whisper.CreateWithOptions(path, schema.Retentions, aggr.AggregationMethod[0], float32(aggr.XFilesFactor), &whisper.Options{
 			Sparse: p.sparse,
 		})
 		if err != nil {
diff --git a/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go b/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
index 3066cca61b..cd88bb2ed4 100644
--- a/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
+++ b/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
@@ -15,30 +15,30 @@ import (
 	"github.com/lomik/go-whisper"
 )
 
-type whisperAggregationItem struct {
+type WhisperAggregationItem struct {
 	name                 string
 	pattern              *regexp.Regexp
-	xFilesFactor         float64
+	XFilesFactor         float64
 	aggregationMethodStr string
-	aggregationMethod    []whisper.AggregationMethod
+	AggregationMethod    []whisper.AggregationMethod
 }
 
 // WhisperAggregation ...
 type WhisperAggregation struct {
-	Data    []*whisperAggregationItem
-	Default *whisperAggregationItem
+	Data    []*WhisperAggregationItem
+	Default *WhisperAggregationItem
 }
 
 // NewWhisperAggregation create instance of WhisperAggregation
 func NewWhisperAggregation() *WhisperAggregation {
 	return &WhisperAggregation{
-		Data: make([]*whisperAggregationItem, 0),
-		Default: &whisperAggregationItem{
+		Data: make([]*WhisperAggregationItem, 0),
+		Default: &WhisperAggregationItem{
 			name:                 "default",
 			pattern:              nil,
-			xFilesFactor:         0.5,
+			XFilesFactor:         0.5,
 			aggregationMethodStr: "average",
-			aggregationMethod:    []whisper.AggregationMethod{whisper.Average},
+			AggregationMethod:    []whisper.AggregationMethod{whisper.Average},
 		},
 	}
 }
@@ -58,7 +58,7 @@ func ReadWhisperAggregation(file string) (*WhisperAggregation, error) {
 	result := NewWhisperAggregation()
 
 	for _, s := range sections {
-		item := &whisperAggregationItem{}
+		item := &WhisperAggregationItem{}
 		// this is mildly stupid, but I don't feel like forking
 		// configparser just for this
 		item.name =
@@ -74,7 +74,7 @@ func ReadWhisperAggregation(file string) (*WhisperAggregation, error) {
 			return nil, err
 		}
 
-		item.xFilesFactor, err = strconv.ParseFloat(s.ValueOf("xFilesFactor"), 64)
+		item.XFilesFactor, err = strconv.ParseFloat(s.ValueOf("xFilesFactor"), 64)
 		if err != nil {
 			logrus.Errorf("failed to parse xFilesFactor '%s' in %s: %s",
 				s.ValueOf("xFilesFactor"), item.name, err.Error())
@@ -86,15 +86,15 @@ func ReadWhisperAggregation(file string) (*WhisperAggregation, error) {
 		for _, methodStr := range methodStrs {
 			switch methodStr {
 			case "average", "avg":
-				item.aggregationMethod = append(item.aggregationMethod, whisper.Average)
+				item.AggregationMethod = append(item.AggregationMethod, whisper.Average)
 			case "sum":
-				item.aggregationMethod = append(item.aggregationMethod, whisper.Sum)
+				item.AggregationMethod = append(item.AggregationMethod, whisper.Sum)
 			case "last":
-				item.aggregationMethod = append(item.aggregationMethod, whisper.Last)
+				item.AggregationMethod = append(item.AggregationMethod, whisper.Last)
 			case "max":
-				item.aggregationMethod = append(item.aggregationMethod, whisper.Max)
+				item.AggregationMethod = append(item.AggregationMethod, whisper.Max)
 			case "min":
-				item.aggregationMethod = append(item.aggregationMethod, whisper.Min)
+				item.AggregationMethod = append(item.AggregationMethod, whisper.Min)
 			default:
 				logrus.Errorf("unknown aggregation method '%s'", methodStr)
 				return result, fmt.Errorf("unknown aggregation method %q", methodStr)
@@ -103,7 +103,7 @@ func ReadWhisperAggregation(file string) (*WhisperAggregation, error) {
 
 		logrus.Debugf("[persister] Adding aggregation [%s] pattern = %s aggregationMethod = %s xFilesFactor = %f",
 			item.name, s.ValueOf("pattern"),
-			item.aggregationMethodStr, item.xFilesFactor)
+			item.aggregationMethodStr, item.XFilesFactor)
 
 		result.Data = append(result.Data, item)
 	}
@@ -112,7 +112,7 @@ func ReadWhisperAggregation(file string) (*WhisperAggregation, error) {
 }
 
 // Match find schema for metric
-func (a *WhisperAggregation) Match(metric string) (uint16, *whisperAggregationItem) {
+func (a *WhisperAggregation) Match(metric string) (uint16, *WhisperAggregationItem) {
 	for i, s := range a.Data {
 		if s.pattern.MatchString(metric) {
 			return uint16(i), s

From 8f800f344f7c9906f32a819134224992931e3a5d Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Mon, 20 Feb 2017 18:12:32 +0100
Subject: [PATCH 05/29] rework archives & consolidation

1) use storage-schemas.conf and storage-aggregation.conf
to configure numchunks, chunkspans, ttls and aggregations.
For carbon also to find out the raw interval.
For other input plugins we only honor interval specified in the data

Note:
- persist message now need to mention which metric names, in case
metrics need to be created on the other end.

2) support configurable retention schemes based on patterns, like
graphite. adjust alignRequests accordingly

3) support configurable aggregation bands to control which rollups
get created per metric pattern.

4) add back 'last' roll-up aggregation and runtime consolidation

It was removed in https://github.com/raintank/metrictank/pull/142
 / 2be35468ea2c486f3c2fc201343a34f79b36055b
based on a discussion in https://github.com/raintank/strategy/issues/11
on the premise that storing this extra series for all our metrics wasn't
worth it and it's not queryable via consolidateBy which only does sum,
avg, min and max.

However:
1) now aggregations are configurable:
   One only enables specific aggregation bands on an as-needed basis.
2) graphite supports last rollups, so we must too.
3) for certain types of data (e.g. counters) it's
   simply the best (better than all the rest) approach
4) storage level aggregation is not as tied to consolidateBy() as we
   originally made it out to be, so that point doesn't really apply
   anymore.
5) we also restore the 'last' runtime consolidation code,
   even though we will no longer do runtime consolidation in MT for now
   but the upcoming normalization uses the same code.
   As we should use 'last' for normalization if rollups was 'last'.
   (see next commit for more info)
---
 api/dataprocessor_test.go                     |  16 ++-
 api/graphite.go                               |  12 +-
 api/models/request.go                         |   6 +-
 api/query_engine.go                           | 121 +++++++++++-------
 batch/aggregator.go                           |  13 ++
 consolidation/consolidation.go                |  13 +-
 docker/docker-cluster/metrictank.ini          |  25 ----
 .../docker-compose.yml                        |   2 +
 .../metrictank.ini                            |  25 ----
 .../storage-aggregation.conf                  |   4 +
 .../storage-schemas.conf                      |   3 +
 docker/docker-dev/docker-compose.yml          |   2 +
 docs/config.md                                |  23 ----
 idx/cassandra/cassandra.go                    |   8 +-
 idx/idx.go                                    |   6 +-
 idx/memory/memory.go                          |  21 ++-
 input/carbon/carbon.go                        |  31 +----
 input/input.go                                |   7 +-
 mdata/aggmetric.go                            |  24 +++-
 mdata/aggmetric_test.go                       |   3 +-
 mdata/aggmetrics.go                           |  21 +--
 mdata/aggregation.go                          |   3 +
 mdata/aggregator.go                           |  79 +++++++++---
 mdata/aggregator_test.go                      |   4 +
 mdata/aggsettings.go                          |  82 ------------
 mdata/ifaces.go                               |   9 +-
 mdata/init.go                                 |  39 +++++-
 mdata/notifier.go                             |  11 +-
 mdata/notifierKafka/notifierKafka.go          |   1 +
 mdata/notifierNsq/notifierNsq.go              |  12 +-
 mdata/schema.go                               |  61 +++++++++
 metrictank-sample.ini                         |  25 ----
 metrictank.go                                 |  41 +-----
 scripts/config/metrictank-docker.ini          |  25 ----
 scripts/config/metrictank-package.ini         |  25 ----
 scripts/config/storage-schemas.conf           |   2 +-
 usage/usage.go                                |   7 +-
 usage/usage_test.go                           |   2 +-
 38 files changed, 396 insertions(+), 418 deletions(-)
 create mode 100644 docker/docker-dev-custom-cfg-kafka/storage-aggregation.conf
 create mode 100644 docker/docker-dev-custom-cfg-kafka/storage-schemas.conf
 delete mode 100644 mdata/aggsettings.go
 create mode 100644 mdata/schema.go

diff --git a/api/dataprocessor_test.go b/api/dataprocessor_test.go
index 7eeeea4d2a..f38cabccaf 100644
--- a/api/dataprocessor_test.go
+++ b/api/dataprocessor_test.go
@@ -155,6 +155,20 @@ func TestConsolidationFunctions(t *testing.T) {
 				{Val: 3, Ts: 1449178151},
 				{Val: 4, Ts: 1449178161},
 			},
+			consolidation.Lst,
+			2,
+			[]schema.Point{
+				{2, 1449178141},
+				{4, 1449178161},
+			},
+		},
+		{
+			[]schema.Point{
+				{1, 1449178131},
+				{2, 1449178141},
+				{3, 1449178151},
+				{4, 1449178161},
+			},
 			consolidation.Min,
 			2,
 			[]schema.Point{
@@ -566,7 +580,7 @@ func TestGetSeriesFixed(t *testing.T) {
 			for to := uint32(31); to <= 40; to++ { // should always yield result with last point at 30 (because to is exclusive)
 				name := fmt.Sprintf("case.data.offset.%d.query:%d-%d", offset, from, to)
 
-				metric := metrics.GetOrCreate(name)
+				metric := metrics.GetOrCreate(name, name)
 				metric.Add(offset, 10)    // this point will always be quantized to 10
 				metric.Add(10+offset, 20) // this point will always be quantized to 20, so it should be selected
 				metric.Add(20+offset, 30) // this point will always be quantized to 30, so it should be selected
diff --git a/api/graphite.go b/api/graphite.go
index aee3343a02..d4aa446bfb 100644
--- a/api/graphite.go
+++ b/api/graphite.go
@@ -213,8 +213,10 @@ func (s *Server) renderMetrics(ctx *middleware.Context, request models.GraphiteR
 	consolidatorForPattern := make(map[string]string)
 	patterns := make([]string, 0)
 	type locatedDef struct {
-		def  schema.MetricDefinition
-		node cluster.Node
+		def     schema.MetricDefinition
+		node    cluster.Node
+		SchemaI uint16
+		AggI    uint16
 	}
 
 	//locatedDefs[<pattern>][<def.id>]locatedDef
@@ -245,7 +247,7 @@ func (s *Server) renderMetrics(ctx *middleware.Context, request models.GraphiteR
 				continue
 			}
 			for _, def := range metric.Defs {
-				locatedDefs[s.Pattern][def.Id] = locatedDef{def, s.Node}
+				locatedDefs[s.Pattern][def.Id] = locatedDef{def, s.Node, metric.SchemaI, metric.AggI}
 			}
 		}
 	}
@@ -263,7 +265,7 @@ func (s *Server) renderMetrics(ctx *middleware.Context, request models.GraphiteR
 			// def.Name is like foo.concretebar
 			// so we want target to contain the concrete graphite name, potentially wrapped with consolidateBy().
 			target := strings.Replace(targetForPattern[pattern], pattern, def.Name, -1)
-			reqs = append(reqs, models.NewReq(def.Id, target, fromUnix, toUnix, request.MaxDataPoints, uint32(def.Interval), consolidator, locdef.node))
+			reqs = append(reqs, models.NewReq(def.Id, target, fromUnix, toUnix, request.MaxDataPoints, uint32(def.Interval), consolidator, locdef.node, locdef.SchemaI, locdef.AggI))
 		}
 	}
 
@@ -279,7 +281,7 @@ func (s *Server) renderMetrics(ctx *middleware.Context, request models.GraphiteR
 			ctx.Req.Method, from, to, len(request.Targets), request.MaxDataPoints)
 	}
 
-	reqs, err = alignRequests(uint32(time.Now().Unix()), reqs, s.MemoryStore.AggSettings())
+	reqs, err = alignRequests(uint32(time.Now().Unix()), reqs)
 	if err != nil {
 		log.Error(3, "HTTP Render alignReq error: %s", err)
 		response.Write(ctx, response.WrapError(err))
diff --git a/api/models/request.go b/api/models/request.go
index 145b62814f..bcce9d917f 100644
--- a/api/models/request.go
+++ b/api/models/request.go
@@ -17,6 +17,8 @@ type Req struct {
 	RawInterval  uint32                     `json:"rawInterval"` // the interval of the raw metric before any consolidation
 	Consolidator consolidation.Consolidator `json:"consolidator"`
 	Node         cluster.Node               `json:"-"`
+	SchemaI      uint16                     `json:"schemaI"`
+	AggI         uint16                     `json:"aggI"`
 
 	// these fields need some more coordination and are typically set later
 	Archive      int    `json:"archive"`      // 0 means original data, 1 means first agg level, 2 means 2nd, etc.
@@ -26,7 +28,7 @@ type Req struct {
 	AggNum       uint32 `json:"aggNum"`       // how many points to consolidate together at runtime, after fetching from the archive
 }
 
-func NewReq(key, target string, from, to, maxPoints, rawInterval uint32, consolidator consolidation.Consolidator, node cluster.Node) Req {
+func NewReq(key, target string, from, to, maxPoints, rawInterval uint32, consolidator consolidation.Consolidator, node cluster.Node, schemaI, aggI uint16) Req {
 	return Req{
 		key,
 		target,
@@ -36,6 +38,8 @@ func NewReq(key, target string, from, to, maxPoints, rawInterval uint32, consoli
 		rawInterval,
 		consolidator,
 		node,
+		schemaI,
+		aggI,
 		-1, // this is supposed to be updated still!
 		0,  // this is supposed to be updated still
 		0,  // this is supposed to be updated still
diff --git a/api/query_engine.go b/api/query_engine.go
index 5e9e15c8f2..6e80f45df2 100644
--- a/api/query_engine.go
+++ b/api/query_engine.go
@@ -1,6 +1,8 @@
 package api
 
 import (
+	"errors"
+
 	"github.com/raintank/metrictank/api/models"
 	"github.com/raintank/metrictank/mdata"
 	"github.com/raintank/metrictank/stats"
@@ -15,84 +17,111 @@ var (
 	reqRenderPointsFetched = stats.NewMeter32("api.request.render.points_fetched", false)
 	// metric api.request.render.series is the number of points the request will return.
 	reqRenderPointsReturned = stats.NewMeter32("api.request.render.points_returned", false)
+
+	errUnSatisfiable = errors.New("request cannot be satisfied due to lack of available retentions")
 )
 
 // alignRequests updates the requests with all details for fetching, making sure all metrics are in the same, optimal interval
-// luckily, all metrics still use the same aggSettings, making this a bit simpler
 // note: it is assumed that all requests have the same from & to.
 // also takes a "now" value which we compare the TTL against
-func alignRequests(now uint32, reqs []models.Req, s mdata.AggSettings) ([]models.Req, error) {
+func alignRequests(now uint32, reqs []models.Req) ([]models.Req, error) {
 
 	tsRange := (reqs[0].To - reqs[0].From)
 
-	// prefer raw first and foremost.
-	// if raw interval doesn't retain data long enough, we must
-	// find the highest res rollup archive that retains all the data we need.
+	var listIntervals []uint32
+	var seenIntervals = make(map[uint32]struct{})
+
+	// set preliminary settings. may be adjusted further down
+	// but for now:
+	// for each req, find the highest res archive
+	// (starting with raw, then rollups in decreasing precision)
+	// that retains all the data we need.
 	// fallback to lowest res option (which *should* have the longest TTL)
 
-	archive := 0
-	ttl := s.RawTTL
+	for i := range reqs {
+		req := &reqs[i]
+		retentions := mdata.GetRetentions(req.SchemaI)
+		req.Archive = -1
 
-	if now-s.RawTTL > reqs[0].From {
-		for i, agg := range s.Aggs {
+		for i, ret := range retentions {
 			// skip non-ready option.
-			if !agg.Ready {
+			if !ret.Ready {
 				continue
 			}
-			archive = i + 1
-			ttl = agg.TTL
-			if now-agg.TTL <= reqs[0].From {
+
+			req.Archive = i
+			req.TTL = uint32(ret.MaxRetention())
+			req.ArchInterval = uint32(ret.SecondsPerPoint())
+
+			if now-req.TTL <= req.From {
 				break
 			}
 		}
 
-	}
+		if req.Archive == -1 {
+			return nil, errUnSatisfiable
+		}
 
-	var interval uint32
-	var listRawIntervals []uint32 // note: only populated when archive 0
-	// the first (raw) uses the LCM of different raw intervals, since that's what needed to fulfill a raw request.
-	// in edge cases (poorly configured setups) this might make raw interval > 1st rollup
-	if archive == 0 {
-		seenRawIntervals := make(map[uint32]struct{})
-		for _, req := range reqs {
-			if _, ok := seenRawIntervals[req.RawInterval]; !ok {
-				listRawIntervals = append(listRawIntervals, req.RawInterval)
-				seenRawIntervals[req.RawInterval] = struct{}{}
-			}
+		if _, ok := seenIntervals[req.ArchInterval]; !ok {
+			listIntervals = append(listIntervals, req.ArchInterval)
+			seenIntervals[req.ArchInterval] = struct{}{}
 		}
-		interval = util.Lcm(listRawIntervals)
-	} else {
-		interval = s.Aggs[archive-1].Span
 	}
 
-	/* we now just need to update the following properties for each req:
-	   archive      int    // 0 means original data, 1 means first agg level, 2 means 2nd, etc.
-	   archInterval uint32 // the interval corresponding to the archive we'll fetch
-	   outInterval  uint32 // the interval of the output data, after any runtime consolidation
-	   aggNum       uint32 // how many points to consolidate together at runtime, after fetching from the archive
-	*/
+	// due to different retentions coming into play, different requests may end up with different resolutions
+	// we all need to emit them at the same interval, the LCM interval >= interval of the req
+
+	interval := util.Lcm(listIntervals)
+
+	// now, for all our requests, set all their properties.  we may have to apply runtime consolidation to get the
+	// correct output interval if out interval != native.  In that case, we also check whether we can fulfill
+	// the request by reading from an archive instead (i.e. whether it has the correct interval.
+	// the TTL of lower resolution archives is always assumed to be at least as long so we don't have to check that)
 
-	// only apply runtime consolidation (pre data processing in graphite api) if we have to due to non uniform raw intervals
-	runtimeConsolidate := archive == 0 && len(listRawIntervals) > 1
 	var pointsFetch uint32
 	for i := range reqs {
 		req := &reqs[i]
-		req.Archive = archive
-		req.TTL = ttl
-		req.OutInterval = interval
-
-		req.ArchInterval = interval
-		req.AggNum = 1
-		if runtimeConsolidate {
-			req.ArchInterval = req.RawInterval
-			req.AggNum = req.OutInterval / req.RawInterval
+		if req.ArchInterval == interval {
+			// the easy case. we can satisfy this req with what we already found
+			// just have to set a few more options
+			req.OutInterval = req.ArchInterval
+			req.AggNum = 1
+
+		} else {
+			// the harder case. due to other reqs with different retention settings
+			// we have to deliver an interval higher than what we originally came up with
+
+			// let's see first if we can deliver it via lower-res rollup archives, if we have any
+			retentions := mdata.GetRetentions(req.SchemaI)
+			for i, ret := range retentions[req.Archive+1:] {
+				archInterval := uint32(ret.SecondsPerPoint())
+				if interval == archInterval && ret.Ready {
+					// we're in luck. this will be more efficient than runtime consolidation
+					req.Archive = req.Archive + 1 + i
+					req.ArchInterval = uint32(ret.SecondsPerPoint())
+					req.TTL = uint32(ret.MaxRetention())
+					req.OutInterval = req.ArchInterval
+					req.AggNum = 1
+					break
+				}
+
+			}
+			if req.ArchInterval != interval {
+				// we have not been able to find an archive matching the desired output interval
+				// we will have to apply runtime consolidation
+				// we use the initially found archive as starting point. there could be some cases - if you have exotic settings -
+				// where it may be more efficient to pick a lower res archive as starting point (it would still require an interval
+				// divisible by the output interval) but let's not worry about that edge case.
+				req.OutInterval = interval
+				req.AggNum = interval / req.ArchInterval
+			}
 		}
 		pointsFetch += tsRange / req.ArchInterval
+		reqRenderChosenArchive.Value(req.Archive)
 	}
 
 	reqRenderPointsFetched.ValueUint32(pointsFetch)
 	reqRenderPointsReturned.ValueUint32(uint32(len(reqs)) * tsRange / interval)
-	reqRenderChosenArchive.Values(archive, len(reqs))
 
 	return reqs, nil
 }
diff --git a/batch/aggregator.go b/batch/aggregator.go
index f7384a1722..178cd71582 100644
--- a/batch/aggregator.go
+++ b/batch/aggregator.go
@@ -41,6 +41,19 @@ func Cnt(in []schema.Point) float64 {
 	return valid
 }
 
+func Lst(in []schema.Point) float64 {
+	if len(in) == 0 {
+		panic("last() called in aggregator with 0 terms")
+	}
+	lst := math.NaN()
+	for _, v := range in {
+		if !math.IsNaN(v.Val) {
+			lst = v.Val
+		}
+	}
+	return lst
+}
+
 func Min(in []schema.Point) float64 {
 	if len(in) == 0 {
 		panic("min() called in aggregator with 0 terms")
diff --git a/consolidation/consolidation.go b/consolidation/consolidation.go
index d4bcb0d2ce..3962ae0f84 100644
--- a/consolidation/consolidation.go
+++ b/consolidation/consolidation.go
@@ -18,6 +18,7 @@ const (
 	None Consolidator = iota
 	Avg
 	Cnt // not available through http api
+	Lst
 	Min
 	Max
 	Sum
@@ -32,6 +33,8 @@ func (c Consolidator) String() string {
 		return "AverageConsolidator"
 	case Cnt:
 		return "CountConsolidator"
+	case Lst:
+		return "LastConsolidator"
 	case Min:
 		return "MinimumConsolidator"
 	case Max:
@@ -52,6 +55,8 @@ func (c Consolidator) Archive() string {
 		panic("avg consolidator has no matching Archive(). you need sum and cnt")
 	case Cnt:
 		return "cnt"
+	case Lst:
+		return "lst"
 	case Min:
 		return "min"
 	case Max:
@@ -66,6 +71,8 @@ func FromArchive(archive string) Consolidator {
 	switch archive {
 	case "cnt":
 		return Cnt
+	case "lst":
+		return Lst
 	case "min":
 		return Min
 	case "max":
@@ -84,6 +91,8 @@ func GetAggFunc(consolidator Consolidator) batch.AggFunc {
 		consFunc = batch.Avg
 	case Cnt:
 		consFunc = batch.Cnt
+	case Lst:
+		consFunc = batch.Lst
 	case Min:
 		consFunc = batch.Min
 	case Max:
@@ -107,6 +116,8 @@ func GetConsolidator(def *schema.MetricDefinition, pref string) (Consolidator, e
 	switch consolidateBy {
 	case "avg", "average":
 		consolidator = Avg
+	case "last":
+		consolidator = Lst
 	case "min":
 		consolidator = Min
 	case "max":
@@ -120,7 +131,7 @@ func GetConsolidator(def *schema.MetricDefinition, pref string) (Consolidator, e
 }
 
 func Validate(fn string) error {
-	if fn == "avg" || fn == "average" || fn == "min" || fn == "max" || fn == "sum" {
+	if fn == "avg" || fn == "average" || fn == "last" || fn == "min" || fn == "max" || fn == "sum" {
 		return nil
 	}
 	return errUnknownConsolidationFunction
diff --git a/docker/docker-cluster/metrictank.ini b/docker/docker-cluster/metrictank.ini
index fb37e1a05e..a5270a5c4f 100644
--- a/docker/docker-cluster/metrictank.ini
+++ b/docker/docker-cluster/metrictank.ini
@@ -10,16 +10,6 @@ accounting-period = 5min
 
 # see https://github.com/raintank/metrictank/blob/master/docs/memory-server.md for more details
 
-# duration of raw chunks. e.g. 10min, 30min, 1h, 90min...
-# must be valid value as described here https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans
-chunkspan = 10min
-# number of raw chunks to keep in in-memory ring buffer
-# See https://github.com/raintank/metrictank/blob/master/docs/memory-server.md for details and trade-offs, especially when compared to chunk-cache
-# (settings further down) which may be a more effective method to cache data and alleviate workload for cassandra.
-numchunks = 7
-# minimum wait before raw metrics are removed from storage
-ttl = 35d
-
 # max age for a chunk before to be considered stale and to be persisted to Cassandra
 chunk-max-stale = 1h
 # max age for a metric before to be considered stale and to be purged from memory
@@ -32,18 +22,6 @@ gc-interval = 1h
 # in clusters, best to assure the primary has saved all the data that a newly warmup instance will need to query, to prevent gaps in charts
 warm-up-period = 1h
 
-# settings for rollups (aggregation for archives)
-# comma-separated list of archive specifications.
-# archive specification is of the form: aggSpan:chunkSpan:numChunks:TTL[:ready as bool. default true]
-# with these aggregation rules: 5min:1h:2:3mon,1h:6h:2:1y:false you get:
-# - 5 min of data, store in a chunk that lasts 1hour, keep 2 chunks in in-memory ring buffer, keep for 3months in cassandra
-# - 1hr worth of data, in chunks of 6 hours, 2 chunks in in-memory ring buffer, keep for 1 year, but this series is not ready yet for querying.
-# When running a cluster of metrictank instances, all instances should have the same agg-settings.
-# Note:
-# * chunk spans must be valid values as described here https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans
-# * numchunks -like the global setting- has nuanced use compared to chunk cache. see https://github.com/raintank/metrictank/blob/master/docs/memory-server.md
-agg-settings =
-
 ## metric data storage in cassandra ##
 
 # see https://github.com/raintank/metrictank/blob/master/docs/cassandra.md for more details
@@ -159,9 +137,6 @@ enabled = true
 addr = :2003
 # represents the "partition" of your data if you decide to partition your data.
 partition = 0
-# needed to know your raw resolution for your metrics. see http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-schemas-conf
-# NOTE: does NOT use aggregation and retention settings from this file.  We use agg-settings and ttl for that.
-schemas-file = /etc/raintank/storage-schemas.conf
 
 ### kafka-mdm input (optional, recommended)
 [kafka-mdm-in]
diff --git a/docker/docker-dev-custom-cfg-kafka/docker-compose.yml b/docker/docker-dev-custom-cfg-kafka/docker-compose.yml
index 79332f2c59..46b1b01d33 100644
--- a/docker/docker-dev-custom-cfg-kafka/docker-compose.yml
+++ b/docker/docker-dev-custom-cfg-kafka/docker-compose.yml
@@ -10,6 +10,8 @@ services:
     volumes:
       - ../../build/metrictank:/usr/bin/metrictank
       - ./metrictank.ini:/etc/raintank/metrictank.ini
+      - ./storage-schemas.conf:/etc/raintank/storage-schemas.conf
+      - ./storage-aggregation.conf:/etc/raintank/storage-aggregation.conf
     environment:
      WAIT_HOSTS: cassandra:9042,kafka:9092
      WAIT_TIMEOUT: 60
diff --git a/docker/docker-dev-custom-cfg-kafka/metrictank.ini b/docker/docker-dev-custom-cfg-kafka/metrictank.ini
index 280b4a8949..45ebe3f061 100644
--- a/docker/docker-dev-custom-cfg-kafka/metrictank.ini
+++ b/docker/docker-dev-custom-cfg-kafka/metrictank.ini
@@ -10,16 +10,6 @@ accounting-period = 5min
 
 # see https://github.com/raintank/metrictank/blob/master/docs/memory-server.md for more details
 
-# duration of raw chunks. e.g. 10min, 30min, 1h, 90min...
-# must be valid value as described here https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans
-chunkspan = 2min
-# number of raw chunks to keep in in-memory ring buffer
-# See https://github.com/raintank/metrictank/blob/master/docs/memory-server.md for details and trade-offs, especially when compared to chunk-cache
-# (settings further down) which may be a more effective method to cache data and alleviate workload for cassandra.
-numchunks = 2
-# minimum wait before raw metrics are removed from storage
-ttl = 35d
-
 # max age for a chunk before to be considered stale and to be persisted to Cassandra
 chunk-max-stale = 1h
 # max age for a metric before to be considered stale and to be purged from memory
@@ -32,18 +22,6 @@ gc-interval = 1h
 # in clusters, best to assure the primary has saved all the data that a newly warmup instance will need to query, to prevent gaps in charts
 warm-up-period = 1h
 
-# settings for rollups (aggregation for archives)
-# comma-separated list of archive specifications.
-# archive specification is of the form: aggSpan:chunkSpan:numChunks:TTL[:ready as bool. default true]
-# with these aggregation rules: 5min:1h:2:3mon,1h:6h:2:1y:false you get:
-# - 5 min of data, store in a chunk that lasts 1hour, keep 2 chunks in in-memory ring buffer, keep for 3months in cassandra
-# - 1hr worth of data, in chunks of 6 hours, 2 chunks in in-memory ring buffer, keep for 1 year, but this series is not ready yet for querying.
-# When running a cluster of metrictank instances, all instances should have the same agg-settings.
-# Note:
-# * chunk spans must be valid values as described here https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans
-# * numchunks -like the global setting- has nuanced use compared to chunk cache. see https://github.com/raintank/metrictank/blob/master/docs/memory-server.md
-agg-settings =
-
 ## metric data storage in cassandra ##
 
 # see https://github.com/raintank/metrictank/blob/master/docs/cassandra.md for more details
@@ -159,9 +137,6 @@ enabled = true
 addr = :2003
 # represents the "partition" of your data if you decide to partition your data.
 partition = 0
-# needed to know your raw resolution for your metrics. see http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-schemas-conf
-# NOTE: does NOT use aggregation and retention settings from this file.  We use agg-settings and ttl for that.
-schemas-file = /etc/raintank/storage-schemas.conf
 
 ### kafka-mdm input (optional, recommended)
 [kafka-mdm-in]
diff --git a/docker/docker-dev-custom-cfg-kafka/storage-aggregation.conf b/docker/docker-dev-custom-cfg-kafka/storage-aggregation.conf
new file mode 100644
index 0000000000..465906149c
--- /dev/null
+++ b/docker/docker-dev-custom-cfg-kafka/storage-aggregation.conf
@@ -0,0 +1,4 @@
+[default]
+pattern = .*
+xFilesFactor = 0.5
+aggregationMethod = avg,min,max
diff --git a/docker/docker-dev-custom-cfg-kafka/storage-schemas.conf b/docker/docker-dev-custom-cfg-kafka/storage-schemas.conf
new file mode 100644
index 0000000000..77c2614104
--- /dev/null
+++ b/docker/docker-dev-custom-cfg-kafka/storage-schemas.conf
@@ -0,0 +1,3 @@
+[default]
+pattern = .*
+retentions = 1s:35d:2min:2
diff --git a/docker/docker-dev/docker-compose.yml b/docker/docker-dev/docker-compose.yml
index 8734269467..53b39b9843 100644
--- a/docker/docker-dev/docker-compose.yml
+++ b/docker/docker-dev/docker-compose.yml
@@ -10,6 +10,8 @@ services:
     volumes:
       - ../../build/metrictank:/usr/bin/metrictank
       - ../../scripts/config/metrictank-docker.ini:/etc/raintank/metrictank.ini
+      - ../../scripts/config/storage-schemas.conf:/etc/raintank/storage-schemas.conf
+      - ../../scripts/config/storage-aggregation.conf:/etc/raintank/storage-aggregation.conf
     environment:
      WAIT_HOSTS: cassandra:9042
      WAIT_TIMEOUT: 60
diff --git a/docs/config.md b/docs/config.md
index a5cc96b15a..a2416f0009 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -35,15 +35,6 @@ accounting-period = 5min
 
 ```
 # see https://github.com/raintank/metrictank/blob/master/docs/memory-server.md for more details
-# duration of raw chunks. e.g. 10min, 30min, 1h, 90min...
-# must be valid value as described here https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans
-chunkspan = 10min
-# number of raw chunks to keep in in-memory ring buffer
-# See https://github.com/raintank/metrictank/blob/master/docs/memory-server.md for details and trade-offs, especially when compared to chunk-cache
-# (settings further down) which may be a more effective method to cache data and alleviate workload for cassandra.
-numchunks = 7
-# minimum wait before raw metrics are removed from storage
-ttl = 35d
 # max age for a chunk before to be considered stale and to be persisted to Cassandra
 chunk-max-stale = 1h
 # max age for a metric before to be considered stale and to be purged from in-memory ring buffer.
@@ -54,17 +45,6 @@ gc-interval = 1h
 # shorter warmup means metrictank will need to query cassandra more if it doesn't have requested data yet.
 # in clusters, best to assure the primary has saved all the data that a newly warmup instance will need to query, to prevent gaps in charts
 warm-up-period = 1h
-# settings for rollups (aggregation for archives)
-# comma-separated list of archive specifications.
-# archive specification is of the form: aggSpan:chunkSpan:numChunks:TTL[:ready as bool. default true]
-# with these aggregation rules: 5min:1h:2:3mon,1h:6h:2:1y:false you get:
-# - 5 min of data, store in a chunk that lasts 1hour, keep 2 chunks in in-memory ring buffer, keep for 3months in cassandra
-# - 1hr worth of data, in chunks of 6 hours, 2 chunks in in-memory ring buffer, keep for 1 year, but this series is not ready yet for querying.
-# When running a cluster of metrictank instances, all instances should have the same agg-settings.
-# Note:
-# * chunk spans must be valid values as described here https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans
-# * numchunks -like the global setting- has nuanced use compared to chunk cache. see https://github.com/raintank/metrictank/blob/master/docs/memory-server.md
-agg-settings =
 ```
 
 ## metric data storage in cassandra ##
@@ -189,9 +169,6 @@ enabled = false
 addr = :2003
 # represents the "partition" of your data if you decide to partition your data.
 partition = 0
-# needed to know your raw resolution for your metrics. see http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-schemas-conf
-# NOTE: does NOT use aggregation and retention settings from this file.  We use agg-settings and ttl for that.
-schemas-file = /path/to/your/schemas-file
 ```
 
 ### kafka-mdm input (optional, recommended)
diff --git a/idx/cassandra/cassandra.go b/idx/cassandra/cassandra.go
index d8dd072954..b8d9392b05 100644
--- a/idx/cassandra/cassandra.go
+++ b/idx/cassandra/cassandra.go
@@ -221,7 +221,7 @@ func (c *CasIdx) Stop() {
 	c.session.Close()
 }
 
-func (c *CasIdx) AddOrUpdate(data *schema.MetricData, partition int32) {
+func (c *CasIdx) AddOrUpdate(data *schema.MetricData, partition int32, schemaI, aggI uint16) {
 	pre := time.Now()
 	existing, inMemory := c.MemoryIdx.Get(data.Id)
 
@@ -232,7 +232,7 @@ func (c *CasIdx) AddOrUpdate(data *schema.MetricData, partition int32) {
 				log.Debug("cassandra-idx def hasn't been seen for a while, updating index.")
 				def := schema.MetricDefinitionFromMetricData(data)
 				def.Partition = partition
-				c.MemoryIdx.AddOrUpdateDef(def)
+				c.MemoryIdx.AddOrUpdateDef(def, schemaI, aggI)
 				c.writeQueue <- writeReq{recvTime: time.Now(), def: def}
 				statUpdateDuration.Value(time.Since(pre))
 			}
@@ -248,14 +248,14 @@ func (c *CasIdx) AddOrUpdate(data *schema.MetricData, partition int32) {
 		}()
 		def := schema.MetricDefinitionFromMetricData(data)
 		def.Partition = partition
-		c.MemoryIdx.AddOrUpdateDef(def)
+		c.MemoryIdx.AddOrUpdateDef(def, schemaI, aggI)
 		c.writeQueue <- writeReq{recvTime: time.Now(), def: def}
 		statUpdateDuration.Value(time.Since(pre))
 		return
 	}
 	def := schema.MetricDefinitionFromMetricData(data)
 	def.Partition = partition
-	c.MemoryIdx.AddOrUpdateDef(def)
+	c.MemoryIdx.AddOrUpdateDef(def, schemaI, aggI)
 	c.writeQueue <- writeReq{recvTime: time.Now(), def: def}
 	statAddDuration.Value(time.Since(pre))
 }
diff --git a/idx/idx.go b/idx/idx.go
index 985d0a21ac..69360ef5df 100644
--- a/idx/idx.go
+++ b/idx/idx.go
@@ -20,6 +20,8 @@ type Node struct {
 	Leaf        bool
 	Defs        []schema.MetricDefinition
 	HasChildren bool
+	SchemaI     uint16 // index in mdata.schemas (not persisted)
+	AggI        uint16 // index in mdata.aggregations (not persisted)
 }
 
 /*
@@ -48,7 +50,7 @@ Interface
 * Stop():
  This will be called when metrictank is shutting down.
 
-* AddOrUpdate(*schema.MetricData, int32):
+* AddOrUpdate(*schema.MetricData, int32, uint16, uint16):
   Every metric received will result in a call to this method to ensure the
   metric has been added to the index. The method is passed the metricData
   payload and the partition id of the metric
@@ -86,7 +88,7 @@ Interface
 type MetricIndex interface {
 	Init() error
 	Stop()
-	AddOrUpdate(*schema.MetricData, int32)
+	AddOrUpdate(*schema.MetricData, int32, uint16, uint16)
 	Get(string) (schema.MetricDefinition, bool)
 	Delete(int, string) ([]schema.MetricDefinition, error)
 	Find(int, string, int64) ([]Node, error)
diff --git a/idx/memory/memory.go b/idx/memory/memory.go
index c7ad4e9afd..9c560b3187 100644
--- a/idx/memory/memory.go
+++ b/idx/memory/memory.go
@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/raintank/metrictank/idx"
+	"github.com/raintank/metrictank/mdata"
 	"github.com/raintank/metrictank/stats"
 	"github.com/raintank/worldping-api/pkg/log"
 	"github.com/rakyll/globalconf"
@@ -58,6 +59,9 @@ type Node struct {
 	Path     string
 	Children []string
 	Defs     []string
+	SchemaI  uint16 // index in mdata.schemas (not persisted)
+	AggI     uint16 // index in mdata.aggregations (not persisted)
+
 }
 
 func (n *Node) HasChildren() bool {
@@ -97,7 +101,7 @@ func (m *MemoryIdx) Stop() {
 	return
 }
 
-func (m *MemoryIdx) AddOrUpdate(data *schema.MetricData, partition int32) {
+func (m *MemoryIdx) AddOrUpdate(data *schema.MetricData, partition int32, schemaI, aggI uint16) {
 	pre := time.Now()
 	m.Lock()
 	defer m.Unlock()
@@ -111,7 +115,7 @@ func (m *MemoryIdx) AddOrUpdate(data *schema.MetricData, partition int32) {
 	}
 
 	def := schema.MetricDefinitionFromMetricData(data)
-	m.add(def)
+	m.add(def, schemaI, aggI)
 	statMetricsActive.Inc()
 	statAddDuration.Value(time.Since(pre))
 }
@@ -127,7 +131,10 @@ func (m *MemoryIdx) Load(defs []schema.MetricDefinition) int {
 		if _, ok := m.DefById[def.Id]; ok {
 			continue
 		}
-		m.add(def)
+		schemaI, _ := mdata.MatchSchema(def.Name)
+		aggI, _ := mdata.MatchAgg(def.Name)
+
+		m.add(def, schemaI, aggI)
 		num++
 		statMetricsActive.Inc()
 		statAddDuration.Value(time.Since(pre))
@@ -136,7 +143,7 @@ func (m *MemoryIdx) Load(defs []schema.MetricDefinition) int {
 	return num
 }
 
-func (m *MemoryIdx) AddOrUpdateDef(def *schema.MetricDefinition) {
+func (m *MemoryIdx) AddOrUpdateDef(def *schema.MetricDefinition, schemaI, aggI uint16) {
 	pre := time.Now()
 	m.Lock()
 	defer m.Unlock()
@@ -147,12 +154,12 @@ func (m *MemoryIdx) AddOrUpdateDef(def *schema.MetricDefinition) {
 		statUpdateDuration.Value(time.Since(pre))
 		return
 	}
-	m.add(def)
+	m.add(def, schemaI, aggI)
 	statMetricsActive.Inc()
 	statAddDuration.Value(time.Since(pre))
 }
 
-func (m *MemoryIdx) add(def *schema.MetricDefinition) {
+func (m *MemoryIdx) add(def *schema.MetricDefinition, schemaI, aggI uint16) {
 	path := def.Name
 	//first check to see if a tree has been created for this OrgId
 	tree, ok := m.Tree[def.OrgId]
@@ -227,6 +234,8 @@ func (m *MemoryIdx) add(def *schema.MetricDefinition) {
 		Path:     path,
 		Children: []string{},
 		Defs:     []string{def.Id},
+		SchemaI:  schemaI,
+		AggI:     aggI,
 	}
 	m.DefById[def.Id] = def
 	statAdd.Inc()
diff --git a/input/carbon/carbon.go b/input/carbon/carbon.go
index 3107c931f4..45e69e561f 100644
--- a/input/carbon/carbon.go
+++ b/input/carbon/carbon.go
@@ -9,10 +9,10 @@ import (
 	"net"
 	"sync"
 
-	"github.com/lomik/go-carbon/persister"
 	"github.com/metrics20/go-metrics20/carbon20"
 	"github.com/raintank/metrictank/cluster"
 	"github.com/raintank/metrictank/input"
+	"github.com/raintank/metrictank/mdata"
 	"github.com/raintank/metrictank/stats"
 	"github.com/raintank/worldping-api/pkg/log"
 	"github.com/rakyll/globalconf"
@@ -29,7 +29,6 @@ type Carbon struct {
 	input.Handler
 	addrStr          string
 	addr             *net.TCPAddr
-	schemas          persister.WhisperSchemas
 	listener         *net.TCPListener
 	handlerWaitGroup sync.WaitGroup
 	quit             chan struct{}
@@ -73,8 +72,6 @@ func (c *Carbon) Name() string {
 
 var Enabled bool
 var addr string
-var schemasFile string
-var schemas persister.WhisperSchemas
 var partitionId int
 
 func ConfigSetup() {
@@ -82,7 +79,6 @@ func ConfigSetup() {
 	inCarbon.BoolVar(&Enabled, "enabled", false, "")
 	inCarbon.StringVar(&addr, "addr", ":2003", "tcp listen address")
 	inCarbon.IntVar(&partitionId, "partition", 0, "partition Id.")
-	inCarbon.StringVar(&schemasFile, "schemas-file", "/path/to/your/schemas-file", "see http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-schemas-conf")
 	globalconf.Register("carbon-in", inCarbon)
 }
 
@@ -90,25 +86,6 @@ func ConfigProcess() {
 	if !Enabled {
 		return
 	}
-	var err error
-	schemas, err = persister.ReadWhisperSchemas(schemasFile)
-	if err != nil {
-		log.Fatal(4, "carbon-in: can't read schemas file %q: %s", schemasFile, err.Error())
-	}
-	var defaultFound bool
-	for _, schema := range schemas {
-		if schema.Pattern.String() == ".*" {
-			defaultFound = true
-		}
-		if len(schema.Retentions) == 0 {
-			log.Fatal(4, "carbon-in: retention setting cannot be empty")
-		}
-	}
-	if !defaultFound {
-		// good graphite health (not sure what graphite does if there's no .*)
-		// but we definitely need to always be able to determine which interval to use
-		log.Fatal(4, "carbon-in: storage-conf does not have a default '.*' pattern")
-	}
 	cluster.Manager.SetPartitions([]int32{int32(partitionId)})
 }
 
@@ -120,7 +97,6 @@ func New() *Carbon {
 	return &Carbon{
 		addrStr:   addr,
 		addr:      addrT,
-		schemas:   schemas,
 		connTrack: NewConnTrack(),
 	}
 }
@@ -196,10 +172,7 @@ func (c *Carbon) handle(conn net.Conn) {
 			continue
 		}
 		name := string(key)
-		s, ok := c.schemas.Match(name)
-		if !ok {
-			log.Fatal(4, "carbon-in: couldn't find a schema for %q - this is impossible since we asserted there was a default with patt .*", name)
-		}
+		_, s := mdata.MatchSchema(name) // note: also called by metrictank DefaultHandler.Process. maybe can be optimized
 		interval := s.Retentions[0].SecondsPerPoint()
 		md := &schema.MetricData{
 			Name:     name,
diff --git a/input/input.go b/input/input.go
index d7b973d381..3f32812f2a 100644
--- a/input/input.go
+++ b/input/input.go
@@ -67,12 +67,15 @@ func (in DefaultHandler) Process(metric *schema.MetricData, partition int32) {
 		return
 	}
 
+	schemaI, _ := mdata.MatchSchema(metric.Name)
+	aggI, _ := mdata.MatchAgg(metric.Name)
+
 	pre := time.Now()
-	in.metricIndex.AddOrUpdate(metric, partition)
+	in.metricIndex.AddOrUpdate(metric, partition, schemaI, aggI)
 	in.pressureIdx.Add(int(time.Since(pre).Nanoseconds()))
 
 	pre = time.Now()
-	m := in.metrics.GetOrCreate(metric.Id)
+	m := in.metrics.GetOrCreate(metric.Id, metric.Name, schemaI, aggI)
 	m.Add(uint32(metric.Time), metric.Value)
 	if in.usage != nil {
 		in.usage.Add(metric.OrgId, metric.Id)
diff --git a/mdata/aggmetric.go b/mdata/aggmetric.go
index 4e89167499..66edb0b144 100644
--- a/mdata/aggmetric.go
+++ b/mdata/aggmetric.go
@@ -6,6 +6,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/lomik/go-carbon/persister"
+	whisper "github.com/lomik/go-whisper"
 	"github.com/raintank/metrictank/cluster"
 	"github.com/raintank/metrictank/consolidation"
 	"github.com/raintank/metrictank/mdata/cache"
@@ -39,21 +41,27 @@ type AggMetric struct {
 
 // NewAggMetric creates a metric with given key, it retains the given number of chunks each chunkSpan seconds long
 // it optionally also creates aggregations with the given settings
-func NewAggMetric(store Store, cachePusher cache.CachePusher, key string, chunkSpan, numChunks uint32, ttl uint32, aggsetting ...AggSetting) *AggMetric {
+// the 0th retention is the native archive of this metric. if there's several others, we create aggregators, using agg.
+// it's the callers responsability to make sure agg is not nil in that case!
+func NewAggMetric(store Store, cachePusher cache.CachePusher, key string, retentions whisper.Retentions, agg *persister.WhisperAggregationItem) *AggMetric {
+
+	// note: during parsing of retentions, we assure there's at least 1.
+	ret := retentions[0]
+
 	m := AggMetric{
 		cachePusher: cachePusher,
 		store:       store,
 		Key:         key,
-		ChunkSpan:   chunkSpan,
-		NumChunks:   numChunks,
-		Chunks:      make([]*chunk.Chunk, 0, numChunks),
-		ttl:         ttl,
+		ChunkSpan:   ret.ChunkSpan,
+		NumChunks:   ret.NumChunks,
+		Chunks:      make([]*chunk.Chunk, 0, ret.NumChunks),
+		ttl:         uint32(ret.MaxRetention()),
 		// we set LastWrite here to make sure a new Chunk doesn't get immediately
 		// garbage collected right after creating it, before we can push to it.
 		lastWrite: uint32(time.Now().Unix()),
 	}
-	for _, as := range aggsetting {
-		m.aggregators = append(m.aggregators, NewAggregator(store, cachePusher, key, as.Span, as.ChunkSpan, as.NumChunks, as.TTL))
+	for _, ret := range retentions[1:] {
+		m.aggregators = append(m.aggregators, NewAggregator(store, cachePusher, key, *ret, *agg))
 	}
 
 	return &m
@@ -119,6 +127,8 @@ func (a *AggMetric) GetAggregated(consolidator consolidation.Consolidator, aggSp
 				panic("avg consolidator has no matching Archive(). you need sum and cnt")
 			case consolidation.Cnt:
 				return a.cntMetric.Get(from, to)
+			case consolidation.Lst:
+				return a.lstMetric.Get(from, to)
 			case consolidation.Min:
 				return a.minMetric.Get(from, to)
 			case consolidation.Max:
diff --git a/mdata/aggmetric_test.go b/mdata/aggmetric_test.go
index 5d8de75471..06d43ff34e 100644
--- a/mdata/aggmetric_test.go
+++ b/mdata/aggmetric_test.go
@@ -231,7 +231,8 @@ func BenchmarkAggMetrics1000Metrics1Day(b *testing.B) {
 	maxT := 3600 * 24 * uint32(b.N) // b.N in days
 	for t := uint32(1); t < maxT; t += 10 {
 		for metricI := 0; metricI < 1000; metricI++ {
-			m := metrics.GetOrCreate(keys[metricI])
+			k := keys[metricI]
+			m := metrics.GetOrCreate(k, k)
 			m.Add(t, float64(t))
 		}
 	}
diff --git a/mdata/aggmetrics.go b/mdata/aggmetrics.go
index e411736376..b33ce34425 100644
--- a/mdata/aggmetrics.go
+++ b/mdata/aggmetrics.go
@@ -13,22 +13,16 @@ type AggMetrics struct {
 	cachePusher cache.CachePusher
 	sync.RWMutex
 	Metrics        map[string]*AggMetric
-	chunkSpan      uint32
-	numChunks      uint32
-	aggSettings    AggSettings // for now we apply the same settings to all AggMetrics. later we may want to have different settings.
 	chunkMaxStale  uint32
 	metricMaxStale uint32
 	gcInterval     time.Duration
 }
 
-func NewAggMetrics(store Store, cachePusher cache.CachePusher, chunkSpan, numChunks, chunkMaxStale, metricMaxStale uint32, ttl uint32, gcInterval time.Duration, aggSettings []AggSetting) *AggMetrics {
+func NewAggMetrics(store Store, cachePusher cache.CachePusher, chunkMaxStale, metricMaxStale uint32, gcInterval time.Duration) *AggMetrics {
 	ms := AggMetrics{
 		store:          store,
 		cachePusher:    cachePusher,
 		Metrics:        make(map[string]*AggMetric),
-		chunkSpan:      chunkSpan,
-		numChunks:      numChunks,
-		aggSettings:    AggSettings{ttl, aggSettings},
 		chunkMaxStale:  chunkMaxStale,
 		metricMaxStale: metricMaxStale,
 		gcInterval:     gcInterval,
@@ -49,8 +43,8 @@ func (ms *AggMetrics) GC() {
 		time.Sleep(diff + time.Minute)
 		log.Info("checking for stale chunks that need persisting.")
 		now := uint32(time.Now().Unix())
-		chunkMinTs := now - (now % ms.chunkSpan) - uint32(ms.chunkMaxStale)
-		metricMinTs := now - (now % ms.chunkSpan) - uint32(ms.metricMaxStale)
+		chunkMinTs := now - uint32(ms.chunkMaxStale)
+		metricMinTs := now - uint32(ms.metricMaxStale)
 
 		// as this is the only goroutine that can delete from ms.Metrics
 		// we only need to lock long enough to get the list of actives metrics.
@@ -85,18 +79,15 @@ func (ms *AggMetrics) Get(key string) (Metric, bool) {
 	return m, ok
 }
 
-func (ms *AggMetrics) GetOrCreate(key string) Metric {
+func (ms *AggMetrics) GetOrCreate(key, name string, schemaI, aggI uint16) Metric {
 	ms.Lock()
 	m, ok := ms.Metrics[key]
 	if !ok {
-		m = NewAggMetric(ms.store, ms.cachePusher, key, ms.chunkSpan, ms.numChunks, ms.aggSettings.RawTTL, ms.aggSettings.Aggs...)
+		agg := GetAgg(aggI)
+		m = NewAggMetric(ms.store, ms.cachePusher, key, GetRetentions(schemaI), &agg)
 		ms.Metrics[key] = m
 		metricsActive.Set(len(ms.Metrics))
 	}
 	ms.Unlock()
 	return m
 }
-
-func (ms *AggMetrics) AggSettings() AggSettings {
-	return ms.aggSettings
-}
diff --git a/mdata/aggregation.go b/mdata/aggregation.go
index 28add15755..facd8f1b3a 100644
--- a/mdata/aggregation.go
+++ b/mdata/aggregation.go
@@ -9,6 +9,7 @@ type Aggregation struct {
 	max float64
 	sum float64
 	cnt float64
+	lst float64
 }
 
 func NewAggregation() *Aggregation {
@@ -23,6 +24,7 @@ func (a *Aggregation) Add(val float64) {
 	a.max = math.Max(val, a.max)
 	a.sum += val
 	a.cnt += 1
+	a.lst = val
 }
 
 func (a *Aggregation) Reset() {
@@ -30,4 +32,5 @@ func (a *Aggregation) Reset() {
 	a.max = -math.MaxFloat64
 	a.sum = 0
 	a.cnt = 0
+	// no need to set a.lst, for a to be valid (cnt > 1), a.lst will always be set properly
 }
diff --git a/mdata/aggregator.go b/mdata/aggregator.go
index c686fe38e4..de2153935a 100644
--- a/mdata/aggregator.go
+++ b/mdata/aggregator.go
@@ -2,6 +2,9 @@ package mdata
 
 import (
 	"fmt"
+
+	"github.com/lomik/go-carbon/persister"
+	whisper "github.com/lomik/go-whisper"
 	"github.com/raintank/metrictank/mdata/cache"
 )
 
@@ -26,24 +29,66 @@ type Aggregator struct {
 	maxMetric       *AggMetric
 	sumMetric       *AggMetric
 	cntMetric       *AggMetric
+	lstMetric       *AggMetric
 }
 
-func NewAggregator(store Store, cachePusher cache.CachePusher, key string, aggSpan, aggChunkSpan, aggNumChunks uint32, ttl uint32) *Aggregator {
-	return &Aggregator{
-		key:       key,
-		span:      aggSpan,
-		agg:       NewAggregation(),
-		minMetric: NewAggMetric(store, cachePusher, fmt.Sprintf("%s_min_%d", key, aggSpan), aggChunkSpan, aggNumChunks, ttl),
-		maxMetric: NewAggMetric(store, cachePusher, fmt.Sprintf("%s_max_%d", key, aggSpan), aggChunkSpan, aggNumChunks, ttl),
-		sumMetric: NewAggMetric(store, cachePusher, fmt.Sprintf("%s_sum_%d", key, aggSpan), aggChunkSpan, aggNumChunks, ttl),
-		cntMetric: NewAggMetric(store, cachePusher, fmt.Sprintf("%s_cnt_%d", key, aggSpan), aggChunkSpan, aggNumChunks, ttl),
+func NewAggregator(store Store, cachePusher cache.CachePusher, key string, ret whisper.Retention, agg persister.WhisperAggregationItem) *Aggregator {
+	if len(agg.AggregationMethod) == 0 {
+		panic("NewAggregator called without aggregations. this should never happen")
+	}
+	span := uint32(ret.SecondsPerPoint())
+	aggregator := &Aggregator{
+		key:  key,
+		span: span,
+		agg:  NewAggregation(),
 	}
+	for _, agg := range agg.AggregationMethod {
+		switch agg {
+		case whisper.Average:
+			if aggregator.sumMetric == nil {
+				aggregator.sumMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_sum_%d", key, span), whisper.Retentions{&ret}, nil)
+			}
+			if aggregator.cntMetric == nil {
+				aggregator.cntMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_cnt_%d", key, span), whisper.Retentions{&ret}, nil)
+			}
+		case whisper.Sum:
+			if aggregator.sumMetric == nil {
+				aggregator.sumMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_sum_%d", key, span), whisper.Retentions{&ret}, nil)
+			}
+		case whisper.Last:
+			if aggregator.lstMetric == nil {
+				aggregator.lstMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_lst_%d", key, span), whisper.Retentions{ret}, nil)
+			}
+		case whisper.Max:
+			if aggregator.maxMetric == nil {
+				aggregator.maxMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_max_%d", key, span), whisper.Retentions{&ret}, nil)
+			}
+		case whisper.Min:
+			if aggregator.minMetric == nil {
+				aggregator.minMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_min_%d", key, span), whisper.Retentions{&ret}, nil)
+			}
+		}
+	}
+	return aggregator
 }
+
+// flush adds points to the aggregation-series and resets aggregation state
 func (agg *Aggregator) flush() {
-	agg.minMetric.Add(agg.currentBoundary, agg.agg.min)
-	agg.maxMetric.Add(agg.currentBoundary, agg.agg.max)
-	agg.sumMetric.Add(agg.currentBoundary, agg.agg.sum)
-	agg.cntMetric.Add(agg.currentBoundary, agg.agg.cnt)
+	if agg.minMetric != nil {
+		agg.minMetric.Add(agg.currentBoundary, agg.agg.min)
+	}
+	if agg.maxMetric != nil {
+		agg.maxMetric.Add(agg.currentBoundary, agg.agg.max)
+	}
+	if agg.sumMetric != nil {
+		agg.sumMetric.Add(agg.currentBoundary, agg.agg.sum)
+	}
+	if agg.cntMetric != nil {
+		agg.cntMetric.Add(agg.currentBoundary, agg.agg.cnt)
+	}
+	if agg.lstMetric != nil {
+		agg.lstMetric.Add(agg.currentBoundary, agg.agg.lst)
+	}
 	//msg := fmt.Sprintf("flushed cnt %v sum %f min %f max %f, reset the block", agg.agg.cnt, agg.agg.sum, agg.agg.min, agg.agg.max)
 	agg.agg.Reset()
 }
@@ -55,23 +100,15 @@ func (agg *Aggregator) Add(ts uint32, val float64) {
 		agg.agg.Add(val)
 		if ts == boundary {
 			agg.flush()
-			//log.Debug("aggregator %d Add(): added to aggregation block, %s because this was the last point for the block", agg.span, agg.flush())
-			//} else {
-			//		log.Debug("aggregator %d Add(): added to aggregation block", agg.span)
-			//	}
 		}
 	} else if boundary > agg.currentBoundary {
 		// store current totals as a new point in their series
 		// if the cnt is still 0, the numbers are invalid, not to be flushed and we can simply reuse the aggregation
 		if agg.agg.cnt != 0 {
 			agg.flush()
-			//		msg = agg.flush() + "and added new point"
-			//	} else {
-			//		msg = "added point to still-unused aggregation block"
 		}
 		agg.currentBoundary = boundary
 		agg.agg.Add(val)
-		//	log.Debug("aggregator %d Add(): %s", agg.span, msg)
 	} else {
 		panic("aggregator: boundary < agg.currentBoundary. ts > lastSeen should already have been asserted")
 	}
diff --git a/mdata/aggregator_test.go b/mdata/aggregator_test.go
index e432d3a647..389daeb766 100644
--- a/mdata/aggregator_test.go
+++ b/mdata/aggregator_test.go
@@ -117,6 +117,10 @@ func TestAggregator(t *testing.T) {
 		{Val: 2, Ts: 120},
 		{Val: 3, Ts: 240},
 	})
+	compare("simple-lst-skip-a-block", agg.lstMetric, []schema.Point{
+		{Val: 5, Ts: 120},
+		{Val: 978894.445, Ts: 240},
+	})
 	compare("simple-sum-skip-a-block", agg.sumMetric, []schema.Point{
 		{Val: 128.4, Ts: 120},
 		{Val: 2451.123 + 1451.123 + 978894.445, Ts: 240},
diff --git a/mdata/aggsettings.go b/mdata/aggsettings.go
deleted file mode 100644
index d8f5c3ca93..0000000000
--- a/mdata/aggsettings.go
+++ /dev/null
@@ -1,82 +0,0 @@
-package mdata
-
-import (
-	"errors"
-	"fmt"
-	"sort"
-	"strconv"
-	"strings"
-
-	"github.com/raintank/dur"
-	"github.com/raintank/metrictank/mdata/chunk"
-)
-
-type AggSetting struct {
-	Span      uint32 // in seconds, controls how many input points go into an aggregated point.
-	ChunkSpan uint32 // duration of chunk of aggregated metric for storage, controls how many aggregated points go into 1 chunk
-	NumChunks uint32 // number of chunks to keep in memory. remember, for a query from now until 3 months ago, we will end up querying the memory server as well.
-	TTL       uint32 // how many seconds to keep the chunk in cassandra
-	Ready     bool   // ready for reads?
-}
-
-type AggSettings struct {
-	RawTTL uint32       // TTL for raw data
-	Aggs   []AggSetting // aggregations
-}
-
-func NewAggSetting(span, chunkSpan, numChunks, ttl uint32, ready bool) AggSetting {
-	return AggSetting{
-		Span:      span,
-		ChunkSpan: chunkSpan,
-		NumChunks: numChunks,
-		TTL:       ttl,
-		Ready:     ready,
-	}
-}
-
-type AggSettingsSpanAsc []AggSetting
-
-func (a AggSettingsSpanAsc) Len() int           { return len(a) }
-func (a AggSettingsSpanAsc) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
-func (a AggSettingsSpanAsc) Less(i, j int) bool { return a[i].Span < a[j].Span }
-
-func ParseAggSettings(input string) ([]AggSetting, error) {
-	set := strings.Split(input, ",")
-	settings := make([]AggSetting, 0)
-	for _, v := range set {
-		if v == "" {
-			continue
-		}
-		fields := strings.Split(v, ":")
-		if len(fields) < 4 {
-			return nil, errors.New("bad agg settings format")
-		}
-		aggSpan := dur.MustParseUNsec("aggsettings", fields[0])
-		aggChunkSpan := dur.MustParseUNsec("aggsettings", fields[1])
-		aggNumChunks := dur.MustParseUNsec("aggsettings", fields[2])
-		aggTTL := dur.MustParseUNsec("aggsettings", fields[3])
-		if (Month_sec % aggChunkSpan) != 0 {
-			return nil, errors.New("aggChunkSpan must fit without remainders into month_sec (28*24*60*60)")
-		}
-		_, ok := chunk.RevChunkSpans[aggChunkSpan]
-		if !ok {
-			return nil, fmt.Errorf("aggChunkSpan %s is not a valid value (https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans)", fields[1])
-		}
-		ready := true
-		var err error
-		if len(fields) == 5 {
-			ready, err = strconv.ParseBool(fields[4])
-			if err != nil {
-				return nil, fmt.Errorf("aggsettings ready: %s", err)
-			}
-		}
-		settings = append(settings, NewAggSetting(aggSpan, aggChunkSpan, aggNumChunks, aggTTL, ready))
-	}
-
-	spanAsc := AggSettingsSpanAsc(settings)
-	if !sort.IsSorted(spanAsc) {
-		return nil, errors.New("aggregation settings need to be ordered by span, in ascending order")
-	}
-
-	return settings, nil
-}
diff --git a/mdata/ifaces.go b/mdata/ifaces.go
index 6abbb2aa28..cd5ec2080a 100644
--- a/mdata/ifaces.go
+++ b/mdata/ifaces.go
@@ -1,12 +1,13 @@
 package mdata
 
-import "github.com/raintank/metrictank/consolidation"
-import "github.com/raintank/metrictank/mdata/chunk"
+import (
+	"github.com/raintank/metrictank/consolidation"
+	"github.com/raintank/metrictank/mdata/chunk"
+)
 
 type Metrics interface {
 	Get(key string) (Metric, bool)
-	GetOrCreate(key string) Metric
-	AggSettings() AggSettings
+	GetOrCreate(key, name string, schemaI, aggI uint16) Metric
 }
 
 type Metric interface {
diff --git a/mdata/init.go b/mdata/init.go
index 3e3734da8a..d35c447a0d 100644
--- a/mdata/init.go
+++ b/mdata/init.go
@@ -3,7 +3,12 @@
 // save states over the network
 package mdata
 
-import "github.com/raintank/metrictank/stats"
+import (
+	"log"
+
+	"github.com/lomik/go-carbon/persister"
+	"github.com/raintank/metrictank/stats"
+)
 
 var (
 	LogLevel int
@@ -37,4 +42,36 @@ var (
 
 	// metric tank.gc_metric is the number of times the metrics GC is about to inspect a metric (series)
 	gcMetric = stats.NewCounter32("tank.gc_metric")
+
+	schemas      persister.WhisperSchemas
+	aggregations *persister.WhisperAggregation
+
+	schemasFile = "/etc/raintank/storage-schemas.conf"
+	aggFile     = "/etc/raintank/storage-aggregation.conf"
 )
+
+func ConfigProcess() {
+	var err error
+	schemas, err = persister.ReadWhisperSchemas(schemasFile)
+	if err != nil {
+		log.Fatalf("can't read schemas file %q: %s", schemasFile, err.Error())
+	}
+	var defaultFound bool
+	for _, schema := range schemas {
+		if schema.Pattern.String() == ".*" {
+			defaultFound = true
+		}
+		if len(schema.Retentions) == 0 {
+			log.Fatal(4, "retention setting cannot be empty")
+		}
+	}
+	if !defaultFound {
+		// good graphite health (not sure what graphite does if there's no .*)
+		// but we definitely need to always be able to determine which interval to use
+		log.Fatal(4, "storage-conf does not have a default '.*' pattern")
+	}
+	aggregations, err = persister.ReadWhisperAggregation(aggFile)
+	if err != nil {
+		log.Fatalf("can't read storage-aggregation file %q: %s", aggFile, err.Error())
+	}
+}
diff --git a/mdata/notifier.go b/mdata/notifier.go
index 610312d857..38cebef3cf 100644
--- a/mdata/notifier.go
+++ b/mdata/notifier.go
@@ -28,6 +28,7 @@ const PersistMessageBatchV1 = 1
 type PersistMessage struct {
 	Instance string `json:"instance"`
 	Key      string `json:"key"`
+	Name     string `json:"name"`
 	T0       uint32 `json:"t0"`
 }
 
@@ -39,6 +40,8 @@ type PersistMessageBatch struct {
 type SavedChunk struct {
 	Key string `json:"key"`
 	T0  uint32 `json:"t0"`
+
+	Name string `json:"name"` // filled in when handler does index lookup
 }
 
 func SendPersistMessage(key string, t0 uint32) {
@@ -82,7 +85,9 @@ func (cl Notifier) Handle(data []byte) {
 				}
 			}
 			if cl.CreateMissingMetrics {
-				agg := cl.Metrics.GetOrCreate(key[0])
+				schemaI, _ := MatchSchema(c.Name)
+				aggI, _ := MatchAgg(c.Name)
+				agg := cl.Metrics.GetOrCreate(key[0], c.Name, schemaI, aggI)
 				if len(key) == 3 {
 					agg.(*AggMetric).SyncAggregatedChunkSaveState(c.T0, consolidator, uint32(aggSpan))
 				} else {
@@ -107,7 +112,9 @@ func (cl Notifier) Handle(data []byte) {
 		messagesReceived.Add(1)
 		// get metric
 		if cl.CreateMissingMetrics {
-			agg := cl.Metrics.GetOrCreate(ms.Key)
+			schemaI, _ := MatchSchema(ms.Name)
+			aggI, _ := MatchAgg(ms.Name)
+			agg := cl.Metrics.GetOrCreate(ms.Key, ms.Name, schemaI, aggI)
 			agg.(*AggMetric).SyncChunkSaveState(ms.T0)
 		} else if agg, ok := cl.Metrics.Get(ms.Key); ok {
 			agg.(*AggMetric).SyncChunkSaveState(ms.T0)
diff --git a/mdata/notifierKafka/notifierKafka.go b/mdata/notifierKafka/notifierKafka.go
index a5b4dbec27..bfbf409279 100644
--- a/mdata/notifierKafka/notifierKafka.go
+++ b/mdata/notifierKafka/notifierKafka.go
@@ -238,6 +238,7 @@ func (c *NotifierKafka) flush() {
 		buf := bytes.NewBuffer(c.bPool.Get())
 		binary.Write(buf, binary.LittleEndian, uint8(mdata.PersistMessageBatchV1))
 		encoder := json.NewEncoder(buf)
+		c.buf[i].Name = def.Name
 		pMsg = mdata.PersistMessageBatch{Instance: c.instance, SavedChunks: c.buf[i : i+1]}
 		err := encoder.Encode(&pMsg)
 		if err != nil {
diff --git a/mdata/notifierNsq/notifierNsq.go b/mdata/notifierNsq/notifierNsq.go
index edf98dc521..5f02106e06 100644
--- a/mdata/notifierNsq/notifierNsq.go
+++ b/mdata/notifierNsq/notifierNsq.go
@@ -4,10 +4,12 @@ import (
 	"bytes"
 	"encoding/binary"
 	"encoding/json"
+	"strings"
 	"time"
 
 	"github.com/bitly/go-hostpool"
 	"github.com/nsqio/go-nsq"
+	"github.com/raintank/metrictank/idx"
 	"github.com/raintank/metrictank/mdata"
 	"github.com/raintank/metrictank/mdata/notifierNsq/instrumented_nsq"
 	"github.com/raintank/metrictank/stats"
@@ -24,9 +26,10 @@ type NotifierNSQ struct {
 	buf      []mdata.SavedChunk
 	instance string
 	mdata.Notifier
+	idx idx.MetricIndex
 }
 
-func New(instance string, metrics mdata.Metrics) *NotifierNSQ {
+func New(instance string, metrics mdata.Metrics, idx idx.MetricIndex) *NotifierNSQ {
 	// metric cluster.notifier.nsq.messages-published is a counter of messages published to the nsq cluster notifier
 	messagesPublished = stats.NewCounter32("cluster.notifier.nsq.messages-published")
 	// metric cluster.notifier.nsq.message_size is the sizes seen of messages through the nsq cluster notifier
@@ -55,6 +58,7 @@ func New(instance string, metrics mdata.Metrics) *NotifierNSQ {
 			Instance: instance,
 			Metrics:  metrics,
 		},
+		idx: idx,
 	}
 	consumer.AddConcurrentHandlers(c, 2)
 
@@ -78,6 +82,12 @@ func (c *NotifierNSQ) HandleMessage(m *nsq.Message) error {
 }
 
 func (c *NotifierNSQ) Send(sc mdata.SavedChunk) {
+	def, ok := c.idx.Get(strings.SplitN(sc.Key, "_", 2)[0])
+	if !ok {
+		log.Error(3, "nsq-cluster: failed to lookup metricDef with id %s", sc.Key)
+		return
+	}
+	sc.Name = def.Name
 	c.in <- sc
 }
 
diff --git a/mdata/schema.go b/mdata/schema.go
new file mode 100644
index 0000000000..b5b198dc5b
--- /dev/null
+++ b/mdata/schema.go
@@ -0,0 +1,61 @@
+package mdata
+
+import (
+	"github.com/lomik/go-carbon/persister"
+	"github.com/lomik/go-whisper"
+	"github.com/raintank/metrictank/util"
+)
+
+// MatchSchema returns the schema for the given metric key, and the index of the schema (to efficiently reference it)
+// it will always find the schema because we made sure there is a catchall '.*' pattern
+func MatchSchema(key string) (uint16, persister.Schema) {
+	i, schema, _ := schemas.Match(key)
+	return i, schema
+}
+
+// MatchAgg returns the aggregation definition for the given metric key, and the index of it (to efficiently reference it)
+// i may be 1 more than the last defined by user, in which case it's the default.
+func MatchAgg(key string) (uint16, *persister.WhisperAggregationItem) {
+	i, agg := aggregations.Match(key)
+	return i, agg
+}
+
+// caller must assure i is valid
+func GetRetentions(i uint16) whisper.Retentions {
+	return schemas[i].Retentions
+}
+
+// caller must assure i is valid
+// note the special case
+func GetAgg(i uint16) persister.WhisperAggregationItem {
+	if i+1 > uint16(len(aggregations.Data)) {
+		return *aggregations.Default
+	}
+	return *aggregations.Data[i]
+}
+
+// TTLs returns a slice of all TTL's seen amongst all archives of all schemas
+func TTLs() []uint32 {
+	ttls := make(map[uint32]struct{})
+	for _, s := range schemas {
+		for _, r := range s.Retentions {
+			ttls[uint32(r.MaxRetention())] = struct{}{}
+		}
+	}
+	var ttlSlice []uint32
+	for ttl := range ttls {
+		ttlSlice = append(ttlSlice, ttl)
+	}
+	return ttlSlice
+}
+
+// MaxChunkSpan returns the largest chunkspan seen amongst all archives of all schemas
+func MaxChunkSpan() uint32 {
+	max := uint32(0)
+	for _, s := range schemas {
+		for _, r := range s.Retentions {
+			max = util.Max(max, r.ChunkSpan)
+		}
+	}
+	return max
+}
diff --git a/metrictank-sample.ini b/metrictank-sample.ini
index 2bfc5ddecb..19baebca4a 100644
--- a/metrictank-sample.ini
+++ b/metrictank-sample.ini
@@ -13,16 +13,6 @@ accounting-period = 5min
 
 # see https://github.com/raintank/metrictank/blob/master/docs/memory-server.md for more details
 
-# duration of raw chunks. e.g. 10min, 30min, 1h, 90min...
-# must be valid value as described here https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans
-chunkspan = 10min
-# number of raw chunks to keep in in-memory ring buffer
-# See https://github.com/raintank/metrictank/blob/master/docs/memory-server.md for details and trade-offs, especially when compared to chunk-cache
-# (settings further down) which may be a more effective method to cache data and alleviate workload for cassandra.
-numchunks = 7
-# minimum wait before raw metrics are removed from storage
-ttl = 35d
-
 # max age for a chunk before to be considered stale and to be persisted to Cassandra
 chunk-max-stale = 1h
 # max age for a metric before to be considered stale and to be purged from in-memory ring buffer.
@@ -35,18 +25,6 @@ gc-interval = 1h
 # in clusters, best to assure the primary has saved all the data that a newly warmup instance will need to query, to prevent gaps in charts
 warm-up-period = 1h
 
-# settings for rollups (aggregation for archives)
-# comma-separated list of archive specifications.
-# archive specification is of the form: aggSpan:chunkSpan:numChunks:TTL[:ready as bool. default true]
-# with these aggregation rules: 5min:1h:2:3mon,1h:6h:2:1y:false you get:
-# - 5 min of data, store in a chunk that lasts 1hour, keep 2 chunks in in-memory ring buffer, keep for 3months in cassandra
-# - 1hr worth of data, in chunks of 6 hours, 2 chunks in in-memory ring buffer, keep for 1 year, but this series is not ready yet for querying.
-# When running a cluster of metrictank instances, all instances should have the same agg-settings.
-# Note:
-# * chunk spans must be valid values as described here https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans
-# * numchunks -like the global setting- has nuanced use compared to chunk cache. see https://github.com/raintank/metrictank/blob/master/docs/memory-server.md
-agg-settings =
-
 ## metric data storage in cassandra ##
 
 # see https://github.com/raintank/metrictank/blob/master/docs/cassandra.md for more details
@@ -162,9 +140,6 @@ enabled = false
 addr = :2003
 # represents the "partition" of your data if you decide to partition your data.
 partition = 0
-# needed to know your raw resolution for your metrics. see http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-schemas-conf
-# NOTE: does NOT use aggregation and retention settings from this file.  We use agg-settings and ttl for that.
-schemas-file = /path/to/your/schemas-file
 
 ### kafka-mdm input (optional, recommended)
 [kafka-mdm-in]
diff --git a/metrictank.go b/metrictank.go
index 62ed7073a2..bb6eaa640b 100644
--- a/metrictank.go
+++ b/metrictank.go
@@ -29,13 +29,11 @@ import (
 	inKafkaMdm "github.com/raintank/metrictank/input/kafkamdm"
 	"github.com/raintank/metrictank/mdata"
 	"github.com/raintank/metrictank/mdata/cache"
-	"github.com/raintank/metrictank/mdata/chunk"
 	"github.com/raintank/metrictank/mdata/notifierKafka"
 	"github.com/raintank/metrictank/mdata/notifierNsq"
 	"github.com/raintank/metrictank/stats"
 	statsConfig "github.com/raintank/metrictank/stats/config"
 	"github.com/raintank/metrictank/usage"
-	"github.com/raintank/metrictank/util"
 	"github.com/raintank/worldping-api/pkg/log"
 	"github.com/rakyll/globalconf"
 )
@@ -57,17 +55,11 @@ var (
 	accountingPeriodStr = flag.String("accounting-period", "5min", "accounting period to track per-org usage metrics")
 
 	// Data:
-	chunkSpanStr = flag.String("chunkspan", "10min", "duration of raw chunks")
-	numChunksInt = flag.Int("numchunks", 7, "number of raw chunks to keep in in-memory ring buffer. See https://github.com/raintank/metrictank/blob/master/docs/memory-server.md for details and trade-offs, especially when compared to chunk-cache")
-	ttlStr       = flag.String("ttl", "35d", "minimum wait before metrics are removed from storage")
-
 	chunkMaxStaleStr  = flag.String("chunk-max-stale", "1h", "max age for a chunk before to be considered stale and to be persisted to Cassandra.")
 	metricMaxStaleStr = flag.String("metric-max-stale", "6h", "max age for a metric before to be considered stale and to be purged from memory.")
 	gcIntervalStr     = flag.String("gc-interval", "1h", "Interval to run garbage collection job.")
 	warmUpPeriodStr   = flag.String("warm-up-period", "1h", "duration before secondary nodes start serving requests")
 
-	aggSettingsStr = flag.String("agg-settings", "", "aggregation settings: <agg span in seconds>:<agg chunkspan in seconds>:<agg numchunks>:<ttl in seconds>[:<ready as bool. default true>] (may be given multiple times as comma-separated list)")
-
 	// Cassandra:
 	cassandraAddrs               = flag.String("cassandra-addrs", "localhost", "cassandra host (may be given multiple times as comma-separated list)")
 	cassandraKeyspace            = flag.String("cassandra-keyspace", "metrictank", "cassandra keyspace to use for storing the metric data table")
@@ -211,6 +203,7 @@ func main() {
 	notifierNsq.ConfigProcess()
 	notifierKafka.ConfigProcess(*instance)
 	statsConfig.ConfigProcess(*instance)
+	mdata.ConfigProcess()
 
 	if !inCarbon.Enabled && !inKafkaMdm.Enabled {
 		log.Fatal(4, "you should enable at least 1 input plugin")
@@ -219,24 +212,9 @@ func main() {
 	sec := dur.MustParseUNsec("warm-up-period", *warmUpPeriodStr)
 	warmupPeriod = time.Duration(sec) * time.Second
 
-	chunkSpan := dur.MustParseUNsec("chunkspan", *chunkSpanStr)
-	numChunks := uint32(*numChunksInt)
 	chunkMaxStale := dur.MustParseUNsec("chunk-max-stale", *chunkMaxStaleStr)
 	metricMaxStale := dur.MustParseUNsec("metric-max-stale", *metricMaxStaleStr)
 	gcInterval := time.Duration(dur.MustParseUNsec("gc-interval", *gcIntervalStr)) * time.Second
-	ttl := dur.MustParseUNsec("ttl", *ttlStr)
-	if (mdata.Month_sec % chunkSpan) != 0 {
-		log.Fatal(4, "chunkSpan must fit without remainders into month_sec (28*24*60*60)")
-	}
-	_, ok := chunk.RevChunkSpans[chunkSpan]
-	if !ok {
-		log.Fatal(4, "chunkSpan %s is not a valid value (https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans)", *chunkSpanStr)
-	}
-
-	aggSettings, err := mdata.ParseAggSettings(*aggSettingsStr)
-	if err != nil {
-		log.Fatal(4, "invalid agg-settings: %s", err)
-	}
 
 	proftrigFreq := dur.MustParseUsec("proftrigger-freq", *proftrigFreqStr)
 	proftrigMinDiff := int(dur.MustParseUNsec("proftrigger-min-diff", *proftrigMinDiffStr))
@@ -273,11 +251,7 @@ func main() {
 	/***********************************
 		Initialize our backendStore
 	***********************************/
-	ttls := []uint32{ttl}
-	for _, agg := range aggSettings {
-		ttls = append(ttls, agg.TTL)
-	}
-	store, err := mdata.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, *cassandraReadConcurrency, *cassandraWriteConcurrency, *cassandraReadQueueSize, *cassandraWriteQueueSize, *cassandraRetries, *cqlProtocolVersion, *cassandraWindowFactor, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, ttls)
+	store, err := mdata.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, *cassandraReadConcurrency, *cassandraWriteConcurrency, *cassandraReadQueueSize, *cassandraWriteQueueSize, *cassandraRetries, *cqlProtocolVersion, *cassandraWindowFactor, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, mdata.TTLs())
 	if err != nil {
 		log.Fatal(4, "failed to initialize cassandra. %s", err)
 	}
@@ -290,7 +264,7 @@ func main() {
 	/***********************************
 		Initialize our MemoryStore
 	***********************************/
-	metrics = mdata.NewAggMetrics(store, ccache, chunkSpan, numChunks, chunkMaxStale, metricMaxStale, ttl, gcInterval, aggSettings)
+	metrics = mdata.NewAggMetrics(store, ccache, chunkMaxStale, metricMaxStale, gcInterval)
 
 	/***********************************
 		Initialize our Inputs
@@ -371,7 +345,7 @@ func main() {
 	}
 
 	if notifierNsq.Enabled {
-		handlers = append(handlers, notifierNsq.New(*instance, metrics))
+		handlers = append(handlers, notifierNsq.New(*instance, metrics, metricIndex))
 	}
 
 	mdata.InitPersistNotifier(handlers...)
@@ -392,11 +366,8 @@ func main() {
 	// When the timer becomes 0 it means the in-memory buffer has been able to fully populate so that if you stop a primary
 	// and it was able to save its complete chunks, this node will be able to take over without dataloss.
 	// You can upgrade a candidate to primary while the timer is not 0 yet, it just means it may have missing data in the chunks that it will save.
-	highestChunkSpan := chunkSpan
-	for _, agg := range aggSettings {
-		highestChunkSpan = util.Max(chunkSpan, agg.ChunkSpan)
-	}
-	stats.NewTimeDiffReporter32("cluster.self.promotion_wait", (uint32(time.Now().Unix())/highestChunkSpan+1)*highestChunkSpan)
+	maxChunkSpan := mdata.MaxChunkSpan()
+	stats.NewTimeDiffReporter32("cluster.self.promotion_wait", (uint32(time.Now().Unix())/maxChunkSpan+1)*maxChunkSpan)
 
 	/***********************************
 		Set our status so we can accept
diff --git a/scripts/config/metrictank-docker.ini b/scripts/config/metrictank-docker.ini
index be4d5147c4..a6939c7f97 100644
--- a/scripts/config/metrictank-docker.ini
+++ b/scripts/config/metrictank-docker.ini
@@ -10,16 +10,6 @@ accounting-period = 5min
 
 # see https://github.com/raintank/metrictank/blob/master/docs/memory-server.md for more details
 
-# duration of raw chunks. e.g. 10min, 30min, 1h, 90min...
-# must be valid value as described here https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans
-chunkspan = 10min
-# number of raw chunks to keep in in-memory ring buffer
-# See https://github.com/raintank/metrictank/blob/master/docs/memory-server.md for details and trade-offs, especially when compared to chunk-cache
-# (settings further down) which may be a more effective method to cache data and alleviate workload for cassandra.
-numchunks = 7
-# minimum wait before raw metrics are removed from storage
-ttl = 35d
-
 # max age for a chunk before to be considered stale and to be persisted to Cassandra
 chunk-max-stale = 1h
 # max age for a metric before to be considered stale and to be purged from memory
@@ -32,18 +22,6 @@ gc-interval = 1h
 # in clusters, best to assure the primary has saved all the data that a newly warmup instance will need to query, to prevent gaps in charts
 warm-up-period = 1h
 
-# settings for rollups (aggregation for archives)
-# comma-separated list of archive specifications.
-# archive specification is of the form: aggSpan:chunkSpan:numChunks:TTL[:ready as bool. default true]
-# with these aggregation rules: 5min:1h:2:3mon,1h:6h:2:1y:false you get:
-# - 5 min of data, store in a chunk that lasts 1hour, keep 2 chunks in in-memory ring buffer, keep for 3months in cassandra
-# - 1hr worth of data, in chunks of 6 hours, 2 chunks in in-memory ring buffer, keep for 1 year, but this series is not ready yet for querying.
-# When running a cluster of metrictank instances, all instances should have the same agg-settings.
-# Note:
-# * chunk spans must be valid values as described here https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans
-# * numchunks -like the global setting- has nuanced use compared to chunk cache. see https://github.com/raintank/metrictank/blob/master/docs/memory-server.md
-agg-settings =
-
 ## metric data storage in cassandra ##
 
 # see https://github.com/raintank/metrictank/blob/master/docs/cassandra.md for more details
@@ -159,9 +137,6 @@ enabled = true
 addr = :2003
 # represents the "partition" of your data if you decide to partition your data.
 partition = 0
-# needed to know your raw resolution for your metrics. see http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-schemas-conf
-# NOTE: does NOT use aggregation and retention settings from this file.  We use agg-settings and ttl for that.
-schemas-file = /etc/raintank/storage-schemas.conf
 
 ### kafka-mdm input (optional, recommended)
 [kafka-mdm-in]
diff --git a/scripts/config/metrictank-package.ini b/scripts/config/metrictank-package.ini
index 57eed8d866..7476f76300 100644
--- a/scripts/config/metrictank-package.ini
+++ b/scripts/config/metrictank-package.ini
@@ -10,16 +10,6 @@ accounting-period = 5min
 
 # see https://github.com/raintank/metrictank/blob/master/docs/memory-server.md for more details
 
-# duration of raw chunks. e.g. 10min, 30min, 1h, 90min...
-# must be valid value as described here https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans
-chunkspan = 10min
-# number of raw chunks to keep in in-memory ring buffer
-# See https://github.com/raintank/metrictank/blob/master/docs/memory-server.md for details and trade-offs, especially when compared to chunk-cache
-# (settings further down) which may be a more effective method to cache data and alleviate workload for cassandra.
-numchunks = 7
-# minimum wait before raw metrics are removed from storage
-ttl = 35d
-
 # max age for a chunk before to be considered stale and to be persisted to Cassandra
 chunk-max-stale = 1h
 # max age for a metric before to be considered stale and to be purged from memory
@@ -32,18 +22,6 @@ gc-interval = 1h
 # in clusters, best to assure the primary has saved all the data that a newly warmup instance will need to query, to prevent gaps in charts
 warm-up-period = 1h
 
-# settings for rollups (aggregation for archives)
-# comma-separated list of archive specifications.
-# archive specification is of the form: aggSpan:chunkSpan:numChunks:TTL[:ready as bool. default true]
-# with these aggregation rules: 5min:1h:2:3mon,1h:6h:2:1y:false you get:
-# - 5 min of data, store in a chunk that lasts 1hour, keep 2 chunks in in-memory ring buffer, keep for 3months in cassandra
-# - 1hr worth of data, in chunks of 6 hours, 2 chunks in in-memory ring buffer, keep for 1 year, but this series is not ready yet for querying.
-# When running a cluster of metrictank instances, all instances should have the same agg-settings.
-# Note:
-# * chunk spans must be valid values as described here https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans
-# * numchunks -like the global setting- has nuanced use compared to chunk cache. see https://github.com/raintank/metrictank/blob/master/docs/memory-server.md
-agg-settings =
-
 ## metric data storage in cassandra ##
 
 # see https://github.com/raintank/metrictank/blob/master/docs/cassandra.md for more details
@@ -159,9 +137,6 @@ enabled = true
 addr = :2003
 # represents the "partition" of your data if you decide to partition your data.
 partition = 0
-# needed to know your raw resolution for your metrics. see http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-schemas-conf
-# NOTE: does NOT use aggregation and retention settings from this file.  We use agg-settings and ttl for that.
-schemas-file = /etc/raintank/storage-schemas.conf
 
 ### kafka-mdm input (optional, recommended)
 [kafka-mdm-in]
diff --git a/scripts/config/storage-schemas.conf b/scripts/config/storage-schemas.conf
index d13cde4086..da6b38fa3c 100644
--- a/scripts/config/storage-schemas.conf
+++ b/scripts/config/storage-schemas.conf
@@ -1,3 +1,3 @@
 [default]
 pattern = .*
-retentions = 1s:1d
+retentions = 1s:35d:10min:7
diff --git a/usage/usage.go b/usage/usage.go
index 4d3ac4f90a..0701a08226 100644
--- a/usage/usage.go
+++ b/usage/usage.go
@@ -101,10 +101,13 @@ func (u *Usage) Report() {
 		met.Value = val
 		met.SetId()
 
-		m := metrics.GetOrCreate(met.Id)
+		schemaI, _ := mdata.MatchSchema(name)
+		aggI, _ := mdata.MatchAgg(name)
+
+		m := metrics.GetOrCreate(met.Id, met.Metric, schemaI, aggI)
 		m.Add(uint32(met.Time), met.Value)
 		//TODO: how to set the partition of the metric?  We probably just need to publish the metric to our Input Plugin
-		metricIndex.AddOrUpdate(met, 0)
+		metricIndex.AddOrUpdate(met, 0, schemaI, aggI)
 	}
 	for {
 		ticker := tick()
diff --git a/usage/usage_test.go b/usage/usage_test.go
index 722632b83b..b16a09a8d4 100644
--- a/usage/usage_test.go
+++ b/usage/usage_test.go
@@ -30,7 +30,7 @@ func (f *FakeAggMetrics) Get(key string) (mdata.Metric, bool) {
 	f.Unlock()
 	return m, ok
 }
-func (f *FakeAggMetrics) GetOrCreate(key string) mdata.Metric {
+func (f *FakeAggMetrics) GetOrCreate(key, name string) mdata.Metric {
 	f.Lock()
 	m, ok := f.Metrics[key]
 	if !ok {

From 49214b58c8ff29b261fbb44353d447168536a8c4 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Fri, 3 Mar 2017 09:10:05 +0100
Subject: [PATCH 06/29] fix runtime consolidation and normalization

"runtime consolidation" used to be controlled by consolidateBy and
affect consolidation after api processing by (and in) graphite,
as well as before (when normalizing data before processing).
Now these two are distinct:

- only the post-function processing should be affected by consolidateBy,
  and should be referred to as "runtime consolidation".
  Since this code is currently in graphite, metrictank shouldn't really
  do anything.  In particular we still parse out
  consolidateBy(..., "func") since that's what graphite-metrictank sends,
  but we ignore it

- the only runtime consolidation metrictank currently does, is normalizing
  archives (making sure they are in the same resolution) before feeding
  them into the api processing pipeline. To avoid confusion this is now
  called normalizing, akin to `normalize` in graphite's functions.py
  This is an extension of the consolidation mechanism used to create the rollup archive.
  This used to be controlled by consolidateBy, but not anymore. Now:
  * the archive being read from is whathever is the primary (first)
  aggregationMethod defined in storage-aggregation.conf .
  * consolidation method for normalization is the same (always).
  Both are controlled via the consolidation property of Req.
  Later we can extend this to choose the method via queries, but it'll
  have to use a mechanism separate of consolidateBy.
---
 api/dataprocessor.go           | 18 ++++++++---------
 api/graphite.go                | 15 ++++++---------
 api/models/request.go          |  4 ++--
 consolidation/consolidation.go | 35 ++++------------------------------
 4 files changed, 21 insertions(+), 51 deletions(-)

diff --git a/api/dataprocessor.go b/api/dataprocessor.go
index 776ec90f3f..33d2794f48 100644
--- a/api/dataprocessor.go
+++ b/api/dataprocessor.go
@@ -316,22 +316,22 @@ func (s *Server) getTargetsLocal(reqs []models.Req) ([]models.Series, error) {
 
 func (s *Server) getTarget(req models.Req) (points []schema.Point, interval uint32, err error) {
 	defer doRecover(&err)
-	readConsolidated := req.Archive != 0   // do we need to read from a downsampled series?
-	runtimeConsolidation := req.AggNum > 1 // do we need to compress any points at runtime?
+	readRollup := req.Archive != 0 // do we need to read from a downsampled series?
+	normalize := req.AggNum > 1    // do we need to compress any points at runtime?
 
 	if LogLevel < 2 {
-		if runtimeConsolidation {
-			log.Debug("DP getTarget() %s runtimeConsolidation: true. agg factor: %d -> output interval: %d", req, req.AggNum, req.OutInterval)
+		if normalize {
+			log.Debug("DP getTarget() %s normalize: true. agg factor: %d -> output interval: %d", req, req.AggNum, req.OutInterval)
 		} else {
-			log.Debug("DP getTarget() %s runtimeConsolidation: false. output interval: %d", req, req.OutInterval)
+			log.Debug("DP getTarget() %s normalize: false. output interval: %d", req, req.OutInterval)
 		}
 	}
 
-	if !readConsolidated && !runtimeConsolidation {
+	if !readRollup && !normalize {
 		return s.getSeriesFixed(req, consolidation.None), req.OutInterval, nil
-	} else if !readConsolidated && runtimeConsolidation {
+	} else if !readRollup && normalize {
 		return consolidate(s.getSeriesFixed(req, consolidation.None), req.AggNum, req.Consolidator), req.OutInterval, nil
-	} else if readConsolidated && !runtimeConsolidation {
+	} else if readRollup && !normalize {
 		if req.Consolidator == consolidation.Avg {
 			return divide(
 				s.getSeriesFixed(req, consolidation.Sum),
@@ -341,7 +341,7 @@ func (s *Server) getTarget(req models.Req) (points []schema.Point, interval uint
 			return s.getSeriesFixed(req, req.Consolidator), req.OutInterval, nil
 		}
 	} else {
-		// readConsolidated && runtimeConsolidation
+		// readRollup && normalize
 		if req.Consolidator == consolidation.Avg {
 			return divide(
 				consolidate(s.getSeriesFixed(req, consolidation.Sum), req.AggNum, consolidation.Sum),
diff --git a/api/graphite.go b/api/graphite.go
index d4aa446bfb..0047c2b7c1 100644
--- a/api/graphite.go
+++ b/api/graphite.go
@@ -15,6 +15,7 @@ import (
 	"github.com/raintank/metrictank/cluster"
 	"github.com/raintank/metrictank/consolidation"
 	"github.com/raintank/metrictank/idx"
+	"github.com/raintank/metrictank/mdata"
 	"github.com/raintank/metrictank/stats"
 	"github.com/raintank/worldping-api/pkg/log"
 	"gopkg.in/raintank/schema.v1"
@@ -209,8 +210,6 @@ func (s *Server) renderMetrics(ctx *middleware.Context, request models.GraphiteR
 
 	reqs := make([]models.Req, 0)
 
-	// consolidatorForPattern[<pattern>]<consolidateBy>
-	consolidatorForPattern := make(map[string]string)
 	patterns := make([]string, 0)
 	type locatedDef struct {
 		def     schema.MetricDefinition
@@ -224,12 +223,11 @@ func (s *Server) renderMetrics(ctx *middleware.Context, request models.GraphiteR
 	//targetForPattern[<pattern>]<target>
 	targetForPattern := make(map[string]string)
 	for _, target := range targets {
-		pattern, consolidateBy, err := parseTarget(target)
+		pattern, _, err := parseTarget(target)
 		if err != nil {
 			ctx.Error(http.StatusBadRequest, err.Error())
 			return
 		}
-		consolidatorForPattern[pattern] = consolidateBy
 		patterns = append(patterns, pattern)
 		targetForPattern[pattern] = target
 		locatedDefs[pattern] = make(map[string]locatedDef)
@@ -255,11 +253,10 @@ func (s *Server) renderMetrics(ctx *middleware.Context, request models.GraphiteR
 	for pattern, ldefs := range locatedDefs {
 		for _, locdef := range ldefs {
 			def := locdef.def
-			consolidator, err := consolidation.GetConsolidator(&def, consolidatorForPattern[pattern])
-			if err != nil {
-				response.Write(ctx, response.NewError(http.StatusBadRequest, err.Error()))
-				return
-			}
+			// set consolidator that will be used to normalize raw data before feeding into processing functions
+			// not to be confused with runtime consolidation which happens in the graphite api, after all processing.
+			fn := mdata.GetAgg(locdef.AggI).AggregationMethod[0]
+			consolidator := consolidation.Consolidator(fn) // we use the same number assignments so we can cast them
 			// target is like foo.bar or foo.* or consolidateBy(foo.*,'sum')
 			// pattern is like foo.bar or foo.*
 			// def.Name is like foo.concretebar
diff --git a/api/models/request.go b/api/models/request.go
index bcce9d917f..5dc007364a 100644
--- a/api/models/request.go
+++ b/api/models/request.go
@@ -14,8 +14,8 @@ type Req struct {
 	From         uint32                     `json:"from"`
 	To           uint32                     `json:"to"`
 	MaxPoints    uint32                     `json:"maxPoints"`
-	RawInterval  uint32                     `json:"rawInterval"` // the interval of the raw metric before any consolidation
-	Consolidator consolidation.Consolidator `json:"consolidator"`
+	RawInterval  uint32                     `json:"rawInterval"`  // the interval of the raw metric before any consolidation
+	Consolidator consolidation.Consolidator `json:"consolidator"` // consolidation method for rollup archive and normalization. (not runtime consolidation)
 	Node         cluster.Node               `json:"-"`
 	SchemaI      uint16                     `json:"schemaI"`
 	AggI         uint16                     `json:"aggI"`
diff --git a/consolidation/consolidation.go b/consolidation/consolidation.go
index 3962ae0f84..9842a52040 100644
--- a/consolidation/consolidation.go
+++ b/consolidation/consolidation.go
@@ -4,8 +4,8 @@ package consolidation
 import (
 	"errors"
 	"fmt"
+
 	"github.com/raintank/metrictank/batch"
-	"gopkg.in/raintank/schema.v1"
 )
 
 // consolidator is a highlevel description of a point consolidation method
@@ -17,11 +17,11 @@ var errUnknownConsolidationFunction = errors.New("unknown consolidation function
 const (
 	None Consolidator = iota
 	Avg
-	Cnt // not available through http api
+	Sum
 	Lst
-	Min
 	Max
-	Sum
+	Min
+	Cnt // not available through http api
 )
 
 // String provides human friendly names
@@ -103,33 +103,6 @@ func GetAggFunc(consolidator Consolidator) batch.AggFunc {
 	return consFunc
 }
 
-func GetConsolidator(def *schema.MetricDefinition, pref string) (Consolidator, error) {
-	consolidateBy := pref
-
-	if consolidateBy == "" {
-		consolidateBy = "avg"
-		if def.Mtype == "counter" {
-			consolidateBy = "max"
-		}
-	}
-	var consolidator Consolidator
-	switch consolidateBy {
-	case "avg", "average":
-		consolidator = Avg
-	case "last":
-		consolidator = Lst
-	case "min":
-		consolidator = Min
-	case "max":
-		consolidator = Max
-	case "sum":
-		consolidator = Sum
-	default:
-		return consolidator, errUnknownConsolidationFunction
-	}
-	return consolidator, nil
-}
-
 func Validate(fn string) error {
 	if fn == "avg" || fn == "average" || fn == "last" || fn == "min" || fn == "max" || fn == "sum" {
 		return nil

From 99e686b50def78179ed6ffb32080e6301250aa54 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Wed, 1 Mar 2017 17:11:27 +0100
Subject: [PATCH 07/29] fix tests

note: a bunch of the handler/processing tests from the input package
are removed.  They were testing a whole bunch of things at once
(index, aggmetrics, handler, input plugin, and libraries they depend on)
unit tests should test very specific units of code, but in the case of
the input package, the code they would test would become so trivial and
obviously correct, because the input plugins barely do anything
themselves.

also:
* make the tests a bit more descriptive
* make testAlignRequests easier to understand by splitting it up
---
 api/dataprocessor_test.go                     |  61 +-
 api/query_engine_test.go                      | 530 ++++++++++++------
 idx/cassandra/cassandra_test.go               |  16 +-
 idx/memory/memory_find_test.go                |   8 +-
 idx/memory/memory_test.go                     |  34 +-
 input/carbon/carbon_test.go                   | 111 ----
 input/input_test.go                           |  78 +--
 input/kafkamdm/kafkamdm_test.go               |  94 ----
 mdata/aggmetric_test.go                       |  94 ++--
 mdata/aggregator_test.go                      |  18 +-
 mdata/init.go                                 |  11 +-
 mdata/schema.go                               |  44 +-
 usage/usage_test.go                           |  17 +-
 vendor/github.com/lomik/go-whisper/whisper.go |  10 +
 14 files changed, 538 insertions(+), 588 deletions(-)
 delete mode 100644 input/carbon/carbon_test.go
 delete mode 100644 input/kafkamdm/kafkamdm_test.go

diff --git a/api/dataprocessor_test.go b/api/dataprocessor_test.go
index f38cabccaf..6a9341e500 100644
--- a/api/dataprocessor_test.go
+++ b/api/dataprocessor_test.go
@@ -2,6 +2,13 @@ package api
 
 import (
 	"fmt"
+	"math"
+	"math/rand"
+	"reflect"
+	"testing"
+	"time"
+
+	whisper "github.com/lomik/go-whisper"
 	"github.com/raintank/metrictank/api/models"
 	"github.com/raintank/metrictank/cluster"
 	"github.com/raintank/metrictank/consolidation"
@@ -10,11 +17,6 @@ import (
 	"github.com/raintank/metrictank/mdata/cache/accnt"
 	"github.com/raintank/metrictank/mdata/chunk"
 	"gopkg.in/raintank/schema.v1"
-	"math"
-	"math/rand"
-	"reflect"
-	"testing"
-	"time"
 )
 
 type testCase struct {
@@ -556,20 +558,24 @@ func TestPrevBoundary(t *testing.T) {
 }
 
 // TestGetSeriesFixed assures that series data is returned in proper form.
+// for each case, we generate a new series of 5 points to cover every possible combination of:
+// * every possible data   offset (against its quantized version)       e.g. offset between 0 and interval-1
+// * every possible `from` offset (against its quantized query results) e.g. offset between 0 and interval-1
+// * every possible `to`   offset (against its quantized query results) e.g. offset between 0 and interval-1
+// and asserts that we get the appropriate data back in all possible query (based on to/from) of the raw data
+
 func TestGetSeriesFixed(t *testing.T) {
 	cluster.Init("default", "test", time.Now(), "http", 6060)
 	store := mdata.NewDevnullStore()
-	metrics := mdata.NewAggMetrics(store, &cache.MockCache{}, 600, 10, 0, 0, 0, 0, []mdata.AggSetting{})
+
+	mdata.SetOnlyDefaultAgg(whisper.Average, whisper.Min, whisper.Max)
+	mdata.SetSingleSchema(whisper.NewRetentionMT(10, 100, 600, 10, true))
+
+	metrics := mdata.NewAggMetrics(store, &cache.MockCache{}, 0, 0, 0)
 	srv, _ := NewServer()
 	srv.BindBackendStore(store)
 	srv.BindMemoryStore(metrics)
 
-	// the tests below cycles through every possible combination of:
-	// * every possible data   offset (against its quantized version)       e.g. offset between 0 and interval-1
-	// * every possible `from` offset (against its quantized query results) e.g. offset between 0 and interval-1
-	// * every possible `to`   offset (against its quantized query results) e.g. offset between 0 and interval-1
-	// and asserts that we get the appropriate data back in all possible scenarios.
-
 	expected := []schema.Point{
 		{Val: 20, Ts: 20},
 		{Val: 30, Ts: 30},
@@ -580,13 +586,13 @@ func TestGetSeriesFixed(t *testing.T) {
 			for to := uint32(31); to <= 40; to++ { // should always yield result with last point at 30 (because to is exclusive)
 				name := fmt.Sprintf("case.data.offset.%d.query:%d-%d", offset, from, to)
 
-				metric := metrics.GetOrCreate(name, name)
+				metric := metrics.GetOrCreate(name, name, 0, 0)
 				metric.Add(offset, 10)    // this point will always be quantized to 10
 				metric.Add(10+offset, 20) // this point will always be quantized to 20, so it should be selected
 				metric.Add(20+offset, 30) // this point will always be quantized to 30, so it should be selected
 				metric.Add(30+offset, 40) // this point will always be quantized to 40
 				metric.Add(40+offset, 50) // this point will always be quantized to 50
-				req := models.NewReq(name, name, from, to, 1000, 10, consolidation.Avg, cluster.Manager.ThisNode())
+				req := models.NewReq(name, name, from, to, 1000, 10, consolidation.Avg, cluster.Manager.ThisNode(), 0, 0)
 				req.ArchInterval = 10
 				points := srv.getSeriesFixed(req, consolidation.None)
 				if !reflect.DeepEqual(expected, points) {
@@ -597,14 +603,15 @@ func TestGetSeriesFixed(t *testing.T) {
 	}
 }
 
-func reqRaw(key string, from, to, maxPoints, rawInterval uint32, consolidator consolidation.Consolidator) models.Req {
-	req := models.NewReq(key, key, from, to, maxPoints, rawInterval, consolidator, cluster.Manager.ThisNode())
+func reqRaw(key string, from, to, maxPoints, rawInterval uint32, consolidator consolidation.Consolidator, schemaI, aggI uint16) models.Req {
+	req := models.NewReq(key, key, from, to, maxPoints, rawInterval, consolidator, cluster.Manager.ThisNode(), schemaI, aggI)
 	return req
 }
-func reqOut(key string, from, to, maxPoints, rawInterval uint32, consolidator consolidation.Consolidator, archive int, archInterval, outInterval, aggNum uint32) models.Req {
-	req := models.NewReq(key, key, from, to, maxPoints, rawInterval, consolidator, cluster.Manager.ThisNode())
+func reqOut(key string, from, to, maxPoints, rawInterval uint32, consolidator consolidation.Consolidator, schemaI, aggI uint16, archive int, archInterval, ttl, outInterval, aggNum uint32) models.Req {
+	req := models.NewReq(key, key, from, to, maxPoints, rawInterval, consolidator, cluster.Manager.ThisNode(), schemaI, aggI)
 	req.Archive = archive
 	req.ArchInterval = archInterval
+	req.TTL = ttl
 	req.OutInterval = outInterval
 	req.AggNum = aggNum
 	return req
@@ -700,7 +707,7 @@ func TestMergeSeries(t *testing.T) {
 func TestRequestContextWithoutConsolidator(t *testing.T) {
 	metric := "metric1"
 	archInterval := uint32(10)
-	req := reqRaw(metric, 44, 88, 100, 10, consolidation.None)
+	req := reqRaw(metric, 44, 88, 100, 10, consolidation.None, 0, 0)
 	req.ArchInterval = archInterval
 	ctx := newRequestContext(&req, consolidation.None)
 
@@ -725,7 +732,7 @@ func TestRequestContextWithConsolidator(t *testing.T) {
 	archInterval := uint32(10)
 	from := uint32(44)
 	to := uint32(88)
-	req := reqRaw(metric, from, to, 100, 10, consolidation.Sum)
+	req := reqRaw(metric, from, to, 100, 10, consolidation.Sum, 0, 0)
 	req.ArchInterval = archInterval
 	ctx := newRequestContext(&req, consolidation.Sum)
 
@@ -792,7 +799,8 @@ func TestGetSeriesCachedStore(t *testing.T) {
 	srv, _ := NewServer()
 	store := mdata.NewMockStore()
 	srv.BindBackendStore(store)
-	metrics := mdata.NewAggMetrics(store, &cache.MockCache{}, 1, 1, 0, 0, 0, 0, []mdata.AggSetting{})
+
+	metrics := mdata.NewAggMetrics(store, &cache.MockCache{}, 0, 0, 0)
 	srv.BindMemoryStore(metrics)
 	metric := "metric1"
 	var c *cache.CCache
@@ -851,7 +859,7 @@ func TestGetSeriesCachedStore(t *testing.T) {
 				}
 
 				// create a request for the current range
-				req := reqRaw(metric, from, to, span, 1, consolidation.None)
+				req := reqRaw(metric, from, to, span, 1, consolidation.None, 0, 0)
 				req.ArchInterval = 1
 				ctx := newRequestContext(&req, consolidation.None)
 				iters := srv.getSeriesCachedStore(ctx, to)
@@ -966,9 +974,8 @@ func TestGetSeriesCachedStore(t *testing.T) {
 func TestGetSeriesAggMetrics(t *testing.T) {
 	cluster.Init("default", "test", time.Now(), "http", 6060)
 	store := mdata.NewMockStore()
-	chunkSpan := uint32(600)
-	numChunks := uint32(10)
-	metrics := mdata.NewAggMetrics(store, &cache.MockCache{}, chunkSpan, numChunks, 0, 0, 0, 0, []mdata.AggSetting{})
+
+	metrics := mdata.NewAggMetrics(store, &cache.MockCache{}, 0, 0, 0)
 	srv, _ := NewServer()
 	srv.BindBackendStore(store)
 	srv.BindMemoryStore(metrics)
@@ -976,11 +983,11 @@ func TestGetSeriesAggMetrics(t *testing.T) {
 	to := uint32(1888)
 	metricKey := "metric1"
 	archInterval := uint32(10)
-	req := reqRaw(metricKey, from, to, 100, 10, consolidation.None)
+	req := reqRaw(metricKey, from, to, 100, 10, consolidation.None, 0, 0)
 	req.ArchInterval = archInterval
 	ctx := newRequestContext(&req, consolidation.None)
 
-	metric := metrics.GetOrCreate(metricKey)
+	metric := metrics.GetOrCreate(metricKey, metricKey, 0, 0)
 	for i := uint32(50); i < 3000; i++ {
 		metric.Add(i, float64(i^2))
 	}
diff --git a/api/query_engine_test.go b/api/query_engine_test.go
index 3168004427..962e6c936d 100644
--- a/api/query_engine_test.go
+++ b/api/query_engine_test.go
@@ -1,209 +1,363 @@
 package api
 
 import (
+	"regexp"
 	"testing"
 
+	"github.com/lomik/go-carbon/persister"
+	whisper "github.com/lomik/go-whisper"
 	"github.com/raintank/metrictank/api/models"
 	"github.com/raintank/metrictank/consolidation"
 	"github.com/raintank/metrictank/mdata"
 )
 
-type alignCase struct {
-	reqs        []models.Req
-	aggSettings mdata.AggSettings
-	outReqs     []models.Req
-	outErr      error
-	now         uint32
+// testAlign verifies the aligment of the given requests, given the retentions (one or more patterns, one or more retentions each)
+func testAlign(reqs []models.Req, retentions [][]whisper.Retention, outReqs []models.Req, outErr error, now uint32, t *testing.T) {
+	var schemas []persister.Schema
+	for _, ret := range retentions {
+		schemas = append(schemas, persister.Schema{
+			Pattern:    regexp.MustCompile(".*"),
+			Retentions: whisper.Retentions(ret),
+		})
+	}
+
+	mdata.Schemas = persister.WhisperSchemas(schemas)
+	out, err := alignRequests(now, reqs)
+	if err != outErr {
+		t.Errorf("different err value expected: %v, got: %v", outErr, err)
+	}
+	if len(out) != len(outReqs) {
+		t.Errorf("different number of requests expected: %v, got: %v", len(outReqs), len(out))
+	} else {
+		for r, exp := range outReqs {
+			if !compareReqEqual(exp, out[r]) {
+				t.Errorf("request %d:\nexpected: %v\n     got: %v", r, exp.DebugString(), out[r].DebugString())
+			}
+		}
+	}
 }
 
-func TestAlignRequests(t *testing.T) {
-	input := []alignCase{
-		{
-			// a basic case with uniform raw intervals
-			[]models.Req{
-				reqRaw("a", 0, 30, 800, 60, consolidation.Avg),
-				reqRaw("b", 0, 30, 800, 60, consolidation.Avg),
-			},
-			mdata.AggSettings{
-				1000, // need TTL of 1000 to read back from ts=0 if now=1000
-				[]mdata.AggSetting{},
+// 2 series requested with equal raw intervals. req 0-30. now 1200. one archive of ttl=1200 does it
+func TestAlignRequestsBasic(t *testing.T) {
+	testAlign([]models.Req{
+		reqRaw("a", 0, 30, 800, 60, consolidation.Avg, 0, 0),
+		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 0, 0),
+	},
+		[][]whisper.Retention{
+			{
+				whisper.NewRetentionMT(60, 1200, 0, 0, true),
 			},
-			[]models.Req{
-				reqOut("a", 0, 30, 800, 60, consolidation.Avg, 0, 60, 60, 1),
-				reqOut("b", 0, 30, 800, 60, consolidation.Avg, 0, 60, 60, 1),
-			},
-			nil,
-			1000,
 		},
-		{
-			// real example seen with alerting queries
-			// need to consolidate to bring them to the same step
-			// because raw intervals are not uniform
-			[]models.Req{
-				reqRaw("a", 0, 30, 800, 10, consolidation.Avg),
-				reqRaw("b", 0, 30, 800, 60, consolidation.Avg),
+		[]models.Req{
+			reqOut("a", 0, 30, 800, 60, consolidation.Avg, 0, 0, 0, 60, 1200, 60, 1),
+			reqOut("b", 0, 30, 800, 60, consolidation.Avg, 0, 0, 0, 60, 1200, 60, 1),
+		},
+		nil,
+		1200,
+		t,
+	)
+}
+
+// 2 series requested with equal raw intervals from different schemas. req 0-30. now 1200. their archives of ttl=1200 do it
+func TestAlignRequestsBasicDiff(t *testing.T) {
+	testAlign([]models.Req{
+		reqRaw("a", 0, 30, 800, 60, consolidation.Avg, 0, 0),
+		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
+	},
+		[][]whisper.Retention{
+			{
+				whisper.NewRetentionMT(60, 1200, 0, 0, true),
 			},
-			mdata.AggSettings{
-				1000, // need TTL of 1000 to read back from ts=0 if now=1000
-				[]mdata.AggSetting{},
+			{
+				whisper.NewRetentionMT(60, 1200, 0, 0, true),
 			},
-			[]models.Req{
-				reqOut("a", 0, 30, 800, 10, consolidation.Avg, 0, 10, 60, 6),
-				reqOut("b", 0, 30, 800, 60, consolidation.Avg, 0, 60, 60, 1),
+		},
+		[]models.Req{
+			reqOut("a", 0, 30, 800, 60, consolidation.Avg, 0, 0, 0, 60, 1200, 60, 1),
+			reqOut("b", 0, 30, 800, 60, consolidation.Avg, 1, 0, 0, 60, 1200, 60, 1),
+		},
+		nil,
+		1200,
+		t,
+	)
+}
+
+// 2 series requested with different raw intervals from different schemas. req 0-30. now 1200. their archives of ttl=1200 do it, but needs normalizing
+// (real example seen with alerting queries)
+func TestAlignRequestsAlerting(t *testing.T) {
+	testAlign([]models.Req{
+		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
+		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
+	},
+		[][]whisper.Retention{{
+			whisper.NewRetentionMT(10, 1200, 0, 0, true),
+		}, {
+			whisper.NewRetentionMT(60, 1200, 0, 0, true),
+		},
+		},
+		[]models.Req{
+			reqOut("a", 0, 30, 800, 10, consolidation.Avg, 0, 0, 0, 10, 1200, 60, 6),
+			reqOut("b", 0, 30, 800, 60, consolidation.Avg, 1, 0, 0, 60, 1200, 60, 1),
+		},
+		nil,
+		1200,
+		t,
+	)
+}
+
+// 2 series requested with different raw intervals from different schemas. req 0-30. now 1200. neither has long enough archive. no rollups, so best effort from raw
+func TestAlignRequestsBasicBestEffort(t *testing.T) {
+	testAlign([]models.Req{
+		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
+		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
+	},
+		[][]whisper.Retention{
+			{
+				whisper.NewRetentionMT(10, 800, 0, 0, true),
+			}, {
+				whisper.NewRetentionMT(60, 1100, 0, 0, true),
 			},
-			nil,
-			1000,
 		},
-		{
-			// same but now raw TTL is not long enough but we don't have a rollup so best effort from raw
-			[]models.Req{
-				reqRaw("a", 0, 30, 800, 10, consolidation.Avg),
-				reqRaw("b", 0, 30, 800, 60, consolidation.Avg),
+		[]models.Req{
+			reqOut("a", 0, 30, 800, 10, consolidation.Avg, 0, 0, 0, 10, 800, 60, 6),
+			reqOut("b", 0, 30, 800, 60, consolidation.Avg, 1, 0, 0, 60, 1100, 60, 1),
+		},
+		nil,
+		1200,
+		t,
+	)
+}
+
+// 2 series requested with different raw intervals from different schemas. req 0-30. now 1200. one has short raw. other has short raw + good rollup
+func TestAlignRequestsHalfGood(t *testing.T) {
+	testAlign([]models.Req{
+		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
+		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
+	},
+		[][]whisper.Retention{
+			{
+				whisper.NewRetentionMT(10, 800, 0, 0, true),
+			}, {
+				whisper.NewRetentionMT(60, 1100, 0, 0, true),
+				whisper.NewRetentionMT(120, 1200, 0, 0, true),
 			},
-			mdata.AggSettings{
-				800,
-				[]mdata.AggSetting{},
+		},
+		[]models.Req{
+			reqOut("a", 0, 30, 800, 10, consolidation.Avg, 0, 0, 0, 10, 800, 120, 12),
+			reqOut("b", 0, 30, 800, 60, consolidation.Avg, 1, 0, 1, 120, 1200, 120, 1),
+		},
+		nil,
+		1200,
+		t,
+	)
+}
+
+// 2 series requested with different raw intervals from different schemas. req 0-30. now 1200. both have short raw + good rollup
+func TestAlignRequestsGoodRollup(t *testing.T) {
+	testAlign([]models.Req{
+		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
+		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
+	},
+		[][]whisper.Retention{
+			{
+				whisper.NewRetentionMT(10, 1199, 0, 0, true), // just not long enough
+				whisper.NewRetentionMT(120, 1200, 600, 2, true),
 			},
-			[]models.Req{
-				reqOut("a", 0, 30, 800, 10, consolidation.Avg, 0, 10, 60, 6),
-				reqOut("b", 0, 30, 800, 60, consolidation.Avg, 0, 60, 60, 1),
+			{
+				whisper.NewRetentionMT(60, 1199, 0, 0, true), // just not long enough
+				whisper.NewRetentionMT(120, 1200, 600, 2, true),
 			},
-			nil,
-			1000,
 		},
-		{
-			// now raw TTL is not long and we have a rollup we can use instead
-			[]models.Req{
-				reqRaw("a", 0, 30, 800, 10, consolidation.Avg),
-				reqRaw("b", 0, 30, 800, 60, consolidation.Avg),
-			},
-			mdata.AggSettings{
-				999, // just not long enough
-				[]mdata.AggSetting{
-					mdata.NewAggSetting(120, 600, 2, 1000, true),
-				},
+		[]models.Req{
+			reqOut("a", 0, 30, 800, 10, consolidation.Avg, 0, 0, 1, 120, 1200, 120, 1),
+			reqOut("b", 0, 30, 800, 60, consolidation.Avg, 1, 0, 1, 120, 1200, 120, 1),
+		},
+		nil,
+		1200,
+		t,
+	)
+}
+
+// 2 series requested with different raw intervals, and rollup intervals from different schemas. req 0-30. now 1200. both have short raw + good rollup
+func TestAlignRequestsDiffGoodRollup(t *testing.T) {
+	testAlign([]models.Req{
+		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
+		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
+	},
+		[][]whisper.Retention{
+			{
+				whisper.NewRetentionMT(10, 1199, 0, 0, true), // just not long enough
+				whisper.NewRetentionMT(100, 1200, 600, 2, true),
 			},
-			[]models.Req{
-				reqOut("a", 0, 30, 800, 10, consolidation.Avg, 1, 120, 120, 1),
-				reqOut("b", 0, 30, 800, 60, consolidation.Avg, 1, 120, 120, 1),
+			{
+				whisper.NewRetentionMT(60, 1199, 0, 0, true), // just not long enough
+				whisper.NewRetentionMT(600, 1200, 600, 2, true),
 			},
-			nil,
-			1000,
 		},
-		{
-			// now raw TTL is not long and we have a rollup we can use instead, at same interval as one of the raws
-			// i suppose our engine could be a bit smarter and see for the 2nd one which it has more in memory of.
-			[]models.Req{
-				reqRaw("a", 0, 30, 800, 10, consolidation.Avg),
-				reqRaw("b", 0, 30, 800, 60, consolidation.Avg),
-			},
-			mdata.AggSettings{
-				999, // just not long enough
-				[]mdata.AggSetting{
-					mdata.NewAggSetting(60, 600, 2, 1000, true),
-				},
+		[]models.Req{
+			reqOut("a", 0, 30, 800, 10, consolidation.Avg, 0, 0, 1, 100, 1200, 600, 6),
+			reqOut("b", 0, 30, 800, 60, consolidation.Avg, 1, 0, 1, 600, 1200, 600, 1),
+		},
+		nil,
+		1200,
+		t,
+	)
+}
+
+// now raw is short and we have a rollup we can use instead, at same interval as one of the raws
+func TestAlignRequestsWeird(t *testing.T) {
+	testAlign([]models.Req{
+		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
+		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
+	},
+		[][]whisper.Retention{
+			{
+				whisper.NewRetentionMT(10, 1199, 0, 0, true),
+				whisper.NewRetentionMT(60, 1200, 600, 2, true),
 			},
-			[]models.Req{
-				reqOut("a", 0, 30, 800, 10, consolidation.Avg, 1, 60, 60, 1),
-				reqOut("b", 0, 30, 800, 60, consolidation.Avg, 1, 60, 60, 1),
+			{
+				whisper.NewRetentionMT(60, 1200, 0, 0, true),
 			},
-			nil,
-			1000,
 		},
-		{
-			// now TTL of first rollup is *just* enough
-			[]models.Req{
-				reqRaw("a", 0, 30, 800, 10, consolidation.Avg),
-				reqRaw("b", 0, 30, 800, 60, consolidation.Avg),
-			},
-			mdata.AggSettings{
-				900, // just not long enough
-				[]mdata.AggSetting{
-					mdata.NewAggSetting(120, 600, 2, 1000, true),
-				},
+		[]models.Req{
+			reqOut("a", 0, 30, 800, 10, consolidation.Avg, 0, 0, 1, 60, 1200, 60, 1),
+			reqOut("b", 0, 30, 800, 60, consolidation.Avg, 1, 0, 0, 60, 1200, 60, 1),
+		},
+		nil,
+		1200,
+		t,
+	)
+}
+
+// now TTL of first rollup is *just* enough
+func TestAlignRequestsWeird2(t *testing.T) {
+	testAlign([]models.Req{
+		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
+		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
+	},
+		[][]whisper.Retention{
+			{
+				whisper.NewRetentionMT(10, 1100, 0, 0, true), // just not long enough
+				whisper.NewRetentionMT(120, 1200, 600, 2, true),
 			},
-			[]models.Req{
-				reqOut("a", 0, 30, 800, 10, consolidation.Avg, 1, 120, 120, 1),
-				reqOut("b", 0, 30, 800, 60, consolidation.Avg, 1, 120, 120, 1),
+			{
+				whisper.NewRetentionMT(60, 1100, 0, 0, true), // just not long enough
+				whisper.NewRetentionMT(120, 1200, 600, 2, true),
 			},
-			nil,
-			1000,
 		},
-		{
-			// now TTL of first rollup is not enough but we have no other choice but to use it
-			[]models.Req{
-				reqRaw("a", 0, 30, 800, 10, consolidation.Avg),
-				reqRaw("b", 0, 30, 800, 60, consolidation.Avg),
-			},
-			mdata.AggSettings{
-				900, // just not long enough
-				[]mdata.AggSetting{
-					mdata.NewAggSetting(120, 600, 2, 999, true),
-				},
+		[]models.Req{
+			reqOut("a", 0, 30, 800, 10, consolidation.Avg, 0, 0, 1, 120, 1200, 120, 1),
+			reqOut("b", 0, 30, 800, 60, consolidation.Avg, 1, 0, 1, 120, 1200, 120, 1),
+		},
+		nil,
+		1200,
+		t,
+	)
+}
+
+// now TTL of first rollup is not enough but we have no other choice but to use it
+func TestAlignRequestsNoOtherChoice(t *testing.T) {
+	testAlign([]models.Req{
+		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
+		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
+	},
+		[][]whisper.Retention{
+			{
+				whisper.NewRetentionMT(10, 1100, 0, 0, true),
+				whisper.NewRetentionMT(120, 1199, 600, 2, true),
 			},
-			[]models.Req{
-				reqOut("a", 0, 30, 800, 10, consolidation.Avg, 1, 120, 120, 1),
-				reqOut("b", 0, 30, 800, 60, consolidation.Avg, 1, 120, 120, 1),
+			{
+				whisper.NewRetentionMT(60, 1100, 0, 0, true),
+				whisper.NewRetentionMT(120, 1199, 600, 2, true),
 			},
-			nil,
-			1000,
 		},
-		{
-			// now TTL of first rollup is not enough and we have a 3rd band to use
-			[]models.Req{
-				reqRaw("a", 0, 30, 800, 10, consolidation.Avg),
-				reqRaw("b", 0, 30, 800, 60, consolidation.Avg),
-			},
-			mdata.AggSettings{
-				900, // just not long enough
-				[]mdata.AggSetting{
-					mdata.NewAggSetting(120, 600, 2, 999, true),
-					mdata.NewAggSetting(240, 600, 2, 1000, true),
-				},
+		[]models.Req{
+			reqOut("a", 0, 30, 800, 10, consolidation.Avg, 0, 0, 1, 120, 1199, 120, 1),
+			reqOut("b", 0, 30, 800, 60, consolidation.Avg, 1, 0, 1, 120, 1199, 120, 1),
+		},
+		nil,
+		1200,
+		t,
+	)
+}
+
+// now TTL of first rollup is not enough and we have a 3rd band to use
+func TestAlignRequests3rdBand(t *testing.T) {
+	testAlign([]models.Req{
+		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
+		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
+	},
+		[][]whisper.Retention{
+			{
+				whisper.NewRetentionMT(1, 1100, 0, 0, true),
+				whisper.NewRetentionMT(120, 1199, 600, 2, true),
+				whisper.NewRetentionMT(240, 1200, 600, 2, true),
 			},
-			[]models.Req{
-				reqOut("a", 0, 30, 800, 10, consolidation.Avg, 2, 240, 240, 1),
-				reqOut("b", 0, 30, 800, 60, consolidation.Avg, 2, 240, 240, 1),
+			{
+				whisper.NewRetentionMT(60, 1100, 0, 0, true),
+				whisper.NewRetentionMT(240, 1200, 600, 2, true),
 			},
-			nil,
-			1000,
 		},
-		{
-			// now TTL of raw/first rollup is not enough but the two rollups are disabled, so must use raw
-			[]models.Req{
-				reqRaw("a", 0, 30, 800, 10, consolidation.Avg),
-				reqRaw("b", 0, 30, 800, 60, consolidation.Avg),
+		[]models.Req{
+			reqOut("a", 0, 30, 800, 10, consolidation.Avg, 0, 0, 2, 240, 1200, 240, 1),
+			reqOut("b", 0, 30, 800, 60, consolidation.Avg, 1, 0, 1, 240, 1200, 240, 1),
+		},
+		nil,
+		1200,
+		t,
+	)
+}
+
+// now TTL of raw/first rollup is not enough but the two rollups are disabled, so must use raw
+func TestAlignRequests2RollupsDisabled(t *testing.T) {
+	testAlign([]models.Req{
+		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
+		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
+	},
+		[][]whisper.Retention{
+			{
+				whisper.NewRetentionMT(10, 1100, 0, 0, true), // just not long enough
+				whisper.NewRetentionMT(120, 1199, 600, 2, false),
+				whisper.NewRetentionMT(240, 1200, 600, 2, false),
 			},
-			mdata.AggSettings{
-				900, // just not long enough
-				[]mdata.AggSetting{
-					mdata.NewAggSetting(120, 600, 2, 999, false),
-					mdata.NewAggSetting(240, 600, 2, 1000, false),
-				},
+			{
+				whisper.NewRetentionMT(60, 1100, 0, 0, true), // just not long enough
+				whisper.NewRetentionMT(240, 1200, 600, 2, false),
 			},
-			[]models.Req{
-				reqOut("a", 0, 30, 800, 10, consolidation.Avg, 0, 10, 60, 6),
-				reqOut("b", 0, 30, 800, 60, consolidation.Avg, 0, 60, 60, 1),
+		},
+		[]models.Req{
+			reqOut("a", 0, 30, 800, 10, consolidation.Avg, 0, 0, 0, 10, 1100, 60, 6),
+			reqOut("b", 0, 30, 800, 60, consolidation.Avg, 1, 0, 0, 60, 1100, 60, 1),
+		},
+		nil,
+		1200,
+		t,
+	)
+}
+func TestAlignRequestsHuh(t *testing.T) {
+	testAlign([]models.Req{
+		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
+		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
+	},
+		[][]whisper.Retention{
+			{
+				whisper.NewRetentionMT(1, 1000, 0, 0, true),
+				whisper.NewRetentionMT(120, 1080, 600, 2, true),
+				whisper.NewRetentionMT(240, 1200, 600, 2, false),
+			},
+			{
+				whisper.NewRetentionMT(60, 1100, 0, 0, true),
+				whisper.NewRetentionMT(240, 1200, 600, 2, false),
 			},
-			nil,
-			1000,
 		},
-	}
-	for i, ac := range input {
-		out, err := alignRequests(ac.now, ac.reqs, ac.aggSettings)
-		if err != ac.outErr {
-			t.Errorf("different err value for testcase %d  expected: %v, got: %v", i, ac.outErr, err)
-		}
-		if len(out) != len(ac.outReqs) {
-			t.Errorf("different number of requests for testcase %d  expected: %v, got: %v", i, len(ac.outReqs), len(out))
-		} else {
-			for r, exp := range ac.outReqs {
-				if !compareReqEqual(exp, out[r]) {
-					t.Errorf("testcase %d, request %d:\nexpected: %v\n     got: %v", i, r, exp.DebugString(), out[r].DebugString())
-				}
-			}
-		}
-	}
+		[]models.Req{
+			reqOut("a", 0, 30, 800, 10, consolidation.Avg, 0, 0, 1, 120, 1080, 120, 1),
+			reqOut("b", 0, 30, 800, 60, consolidation.Avg, 1, 0, 0, 60, 1100, 120, 2),
+		},
+		nil,
+		1200,
+		t,
+	)
 }
 
 var result []models.Req
@@ -211,21 +365,45 @@ var result []models.Req
 func BenchmarkAlignRequests(b *testing.B) {
 	var res []models.Req
 	reqs := []models.Req{
-		reqRaw("a", 0, 3600*24*7, 1000, 10, consolidation.Avg),
-		reqRaw("b", 0, 3600*24*7, 1000, 30, consolidation.Avg),
-		reqRaw("c", 0, 3600*24*7, 1000, 60, consolidation.Avg),
+		reqRaw("a", 0, 3600*24*7, 1000, 10, consolidation.Avg, 0, 0),
+		reqRaw("b", 0, 3600*24*7, 1000, 30, consolidation.Avg, 1, 0),
+		reqRaw("c", 0, 3600*24*7, 1000, 60, consolidation.Avg, 2, 0),
 	}
-	aggSettings := mdata.AggSettings{
-		35 * 24 * 60 * 60,
-		[]mdata.AggSetting{
-			mdata.NewAggSetting(600, 21600, 1, 2*30*24*3600, true),
-			mdata.NewAggSetting(7200, 21600, 1, 6*30*24*3600, true),
-			mdata.NewAggSetting(21600, 21600, 1, 2*365*24*3600, true),
+	mdata.Schemas = persister.WhisperSchemas([]persister.Schema{
+		{
+			Pattern: regexp.MustCompile("a"),
+			Retentions: whisper.Retentions(
+				[]whisper.Retention{
+					whisper.NewRetentionMT(10, 35*24*3600, 0, 0, true),
+					whisper.NewRetentionMT(600, 60*24*3600, 0, 0, true),
+					whisper.NewRetentionMT(7200, 180*24*3600, 0, 0, true),
+					whisper.NewRetentionMT(21600, 2*365*24*3600, 0, 0, true),
+				}),
 		},
-	}
+		{
+			Pattern: regexp.MustCompile("b"),
+			Retentions: whisper.Retentions(
+				[]whisper.Retention{
+					whisper.NewRetentionMT(30, 35*24*3600, 0, 0, true),
+					whisper.NewRetentionMT(600, 60*24*3600, 0, 0, true),
+					whisper.NewRetentionMT(7200, 180*24*3600, 0, 0, true),
+					whisper.NewRetentionMT(21600, 2*365*24*3600, 0, 0, true),
+				}),
+		},
+		{
+			Pattern: regexp.MustCompile(".*"),
+			Retentions: whisper.Retentions(
+				[]whisper.Retention{
+					whisper.NewRetentionMT(60, 35*24*3600, 0, 0, true),
+					whisper.NewRetentionMT(600, 60*24*3600, 0, 0, true),
+					whisper.NewRetentionMT(7200, 180*24*3600, 0, 0, true),
+					whisper.NewRetentionMT(21600, 2*365*24*3600, 0, 0, true),
+				}),
+		},
+	})
 
 	for n := 0; n < b.N; n++ {
-		res, _ = alignRequests(14*24*3600, reqs, aggSettings)
+		res, _ = alignRequests(14*24*3600, reqs)
 	}
 	result = res
 }
diff --git a/idx/cassandra/cassandra_test.go b/idx/cassandra/cassandra_test.go
index 9afbb94de8..50af017612 100644
--- a/idx/cassandra/cassandra_test.go
+++ b/idx/cassandra/cassandra_test.go
@@ -81,7 +81,7 @@ func TestGetAddKey(t *testing.T) {
 		orgId := series[0].OrgId
 		Convey(fmt.Sprintf("When indexing metrics for orgId %d", orgId), t, func() {
 			for _, s := range series {
-				ix.AddOrUpdate(s, 1)
+				ix.AddOrUpdate(s, 1, 0, 0)
 			}
 			Convey(fmt.Sprintf("Then listing metrics for OrgId %d", orgId), func() {
 				defs := ix.List(orgId)
@@ -99,7 +99,7 @@ func TestGetAddKey(t *testing.T) {
 		for _, series := range org1Series {
 			series.Interval = 60
 			series.SetId()
-			ix.AddOrUpdate(series, 1)
+			ix.AddOrUpdate(series, 1, 0, 0)
 		}
 		Convey("then listing metrics", func() {
 			defs := ix.List(1)
@@ -112,19 +112,19 @@ func TestFind(t *testing.T) {
 	ix := New()
 	ix.Init()
 	for _, s := range getMetricData(-1, 2, 5, 10, "metric.demo") {
-		ix.AddOrUpdate(s, 1)
+		ix.AddOrUpdate(s, 1, 0, 0)
 	}
 	for _, s := range getMetricData(1, 2, 5, 10, "metric.demo") {
-		ix.AddOrUpdate(s, 1)
+		ix.AddOrUpdate(s, 1, 0, 0)
 	}
 	for _, s := range getMetricData(1, 1, 5, 10, "foo.demo") {
-		ix.AddOrUpdate(s, 1)
+		ix.AddOrUpdate(s, 1, 0, 0)
 		s.Interval = 60
 		s.SetId()
-		ix.AddOrUpdate(s, 1)
+		ix.AddOrUpdate(s, 1, 0, 0)
 	}
 	for _, s := range getMetricData(2, 2, 5, 10, "metric.foo") {
-		ix.AddOrUpdate(s, 1)
+		ix.AddOrUpdate(s, 1, 0, 0)
 	}
 
 	Convey("When listing root nodes", t, func() {
@@ -248,7 +248,7 @@ func insertDefs(ix idx.MetricIndex, i int) {
 			OrgId:    1,
 		}
 		data.SetId()
-		ix.AddOrUpdate(data, 1)
+		ix.AddOrUpdate(data, 1, 0, 0)
 	}
 }
 
diff --git a/idx/memory/memory_find_test.go b/idx/memory/memory_find_test.go
index c1bef662af..bf41dea760 100644
--- a/idx/memory/memory_find_test.go
+++ b/idx/memory/memory_find_test.go
@@ -63,7 +63,7 @@ func Init() {
 			OrgId:    1,
 		}
 		data.SetId()
-		ix.AddOrUpdate(data, 1)
+		ix.AddOrUpdate(data, 1, 0, 0)
 	}
 	for _, series := range diskMetrics(5, 1000, 0, 10, "collectd") {
 		data = &schema.MetricData{
@@ -73,7 +73,7 @@ func Init() {
 			OrgId:    1,
 		}
 		data.SetId()
-		ix.AddOrUpdate(data, 1)
+		ix.AddOrUpdate(data, 1, 0, 0)
 	}
 	// orgId has 1,680,000 series
 
@@ -85,7 +85,7 @@ func Init() {
 			OrgId:    2,
 		}
 		data.SetId()
-		ix.AddOrUpdate(data, 1)
+		ix.AddOrUpdate(data, 1, 0, 0)
 	}
 	for _, series := range diskMetrics(5, 100, 950, 10, "collectd") {
 		data = &schema.MetricData{
@@ -95,7 +95,7 @@ func Init() {
 			OrgId:    2,
 		}
 		data.SetId()
-		ix.AddOrUpdate(data, 1)
+		ix.AddOrUpdate(data, 1, 0, 0)
 	}
 	//orgId 2 has 168,000 mertics
 
diff --git a/idx/memory/memory_test.go b/idx/memory/memory_test.go
index 0f77103f89..fd108a255c 100644
--- a/idx/memory/memory_test.go
+++ b/idx/memory/memory_test.go
@@ -68,7 +68,7 @@ func TestGetAddKey(t *testing.T) {
 		orgId := series[0].OrgId
 		Convey(fmt.Sprintf("When indexing metrics for orgId %d", orgId), t, func() {
 			for _, s := range series {
-				ix.AddOrUpdate(s, 1)
+				ix.AddOrUpdate(s, 1, 0, 0)
 			}
 			Convey(fmt.Sprintf("Then listing metrics for OrgId %d", orgId), func() {
 				defs := ix.List(orgId)
@@ -86,7 +86,7 @@ func TestGetAddKey(t *testing.T) {
 		for _, series := range org1Series {
 			series.Interval = 60
 			series.SetId()
-			ix.AddOrUpdate(series, 1)
+			ix.AddOrUpdate(series, 1, 0, 0)
 		}
 		Convey("then listing metrics", func() {
 			defs := ix.List(1)
@@ -100,23 +100,23 @@ func TestFind(t *testing.T) {
 	ix.Init()
 	for _, s := range getMetricData(-1, 2, 5, 10, "metric.demo") {
 		s.Time = 10 * 86400
-		ix.AddOrUpdate(s, 1)
+		ix.AddOrUpdate(s, 1, 0, 0)
 	}
 	for _, s := range getMetricData(1, 2, 5, 10, "metric.demo") {
 		s.Time = 10 * 86400
-		ix.AddOrUpdate(s, 1)
+		ix.AddOrUpdate(s, 1, 0, 0)
 	}
 	for _, s := range getMetricData(1, 1, 5, 10, "foo.demo") {
 		s.Time = 1 * 86400
-		ix.AddOrUpdate(s, 1)
+		ix.AddOrUpdate(s, 1, 0, 0)
 		s.Time = 2 * 86400
 		s.Interval = 60
 		s.SetId()
-		ix.AddOrUpdate(s, 1)
+		ix.AddOrUpdate(s, 1, 0, 0)
 	}
 	for _, s := range getMetricData(2, 2, 5, 10, "metric.foo") {
 		s.Time = 1 * 86400
-		ix.AddOrUpdate(s, 1)
+		ix.AddOrUpdate(s, 1, 0, 0)
 	}
 
 	Convey("When listing root nodes", t, func() {
@@ -226,10 +226,10 @@ func TestDelete(t *testing.T) {
 	org1Series := getMetricData(1, 2, 5, 10, "metric.org1")
 
 	for _, s := range publicSeries {
-		ix.AddOrUpdate(s, 1)
+		ix.AddOrUpdate(s, 1, 0, 0)
 	}
 	for _, s := range org1Series {
-		ix.AddOrUpdate(s, 1)
+		ix.AddOrUpdate(s, 1, 0, 0)
 	}
 	Convey("when deleting exact path", t, func() {
 		defs, err := ix.Delete(1, org1Series[0].Name)
@@ -307,10 +307,10 @@ func TestMixedBranchLeaf(t *testing.T) {
 
 	Convey("when adding the first metric", t, func() {
 
-		ix.AddOrUpdate(first, 1)
+		ix.AddOrUpdate(first, 1, 0, 0)
 		Convey("we should be able to add a leaf under another leaf", func() {
 
-			ix.AddOrUpdate(second, 1)
+			ix.AddOrUpdate(second, 1, 0, 0)
 			_, ok := ix.Get(second.Id)
 			So(ok, ShouldEqual, true)
 			defs := ix.List(1)
@@ -318,7 +318,7 @@ func TestMixedBranchLeaf(t *testing.T) {
 		})
 		Convey("we should be able to add a leaf that collides with an existing branch", func() {
 
-			ix.AddOrUpdate(third, 1)
+			ix.AddOrUpdate(third, 1, 0, 0)
 			_, ok := ix.Get(third.Id)
 			So(ok, ShouldEqual, true)
 			defs := ix.List(1)
@@ -364,7 +364,7 @@ func TestMixedBranchLeafDelete(t *testing.T) {
 	}
 	for _, s := range series {
 		s.SetId()
-		ix.AddOrUpdate(s, 1)
+		ix.AddOrUpdate(s, 1, 0, 0)
 	}
 	Convey("when deleting mixed leaf/branch", t, func() {
 		defs, err := ix.Delete(1, "a.b.c")
@@ -424,7 +424,7 @@ func TestPrune(t *testing.T) {
 			Time:     1,
 		}
 		d.SetId()
-		ix.AddOrUpdate(d, 1)
+		ix.AddOrUpdate(d, 1, 0, 0)
 	}
 	//new series
 	for _, s := range getSeriesNames(2, 5, "metric.foo") {
@@ -436,7 +436,7 @@ func TestPrune(t *testing.T) {
 			Time:     10,
 		}
 		d.SetId()
-		ix.AddOrUpdate(d, 1)
+		ix.AddOrUpdate(d, 1, 0, 0)
 	}
 	Convey("after populating index", t, func() {
 		defs := ix.List(-1)
@@ -461,7 +461,7 @@ func TestPrune(t *testing.T) {
 		newDef.Interval = 30
 		newDef.LastUpdate = 100
 		newDef.SetId()
-		ix.AddOrUpdateDef(&newDef)
+		ix.AddOrUpdateDef(&newDef, 0, 0)
 		Convey("When purging old series", func() {
 			purged, err := ix.Prune(1, time.Unix(12, 0))
 			So(err, ShouldBeNil)
@@ -491,6 +491,6 @@ func BenchmarkIndexing(b *testing.B) {
 			OrgId:    1,
 		}
 		data.SetId()
-		ix.AddOrUpdate(data, 1)
+		ix.AddOrUpdate(data, 1, 0, 0)
 	}
 }
diff --git a/input/carbon/carbon_test.go b/input/carbon/carbon_test.go
deleted file mode 100644
index eab962bc20..0000000000
--- a/input/carbon/carbon_test.go
+++ /dev/null
@@ -1,111 +0,0 @@
-package carbon
-
-import (
-	"fmt"
-	"net"
-	"regexp"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/benbjohnson/clock"
-	"github.com/lomik/go-carbon/persister"
-	"github.com/raintank/metrictank/cluster"
-	"github.com/raintank/metrictank/idx/memory"
-	"github.com/raintank/metrictank/input"
-	"github.com/raintank/metrictank/mdata"
-	"github.com/raintank/metrictank/mdata/cache"
-	"github.com/raintank/metrictank/usage"
-	"gopkg.in/raintank/schema.v1"
-)
-
-func Test_HandleMessage(t *testing.T) {
-	cluster.Init("default", "test", time.Now(), "http", 6060)
-	store := mdata.NewDevnullStore()
-	aggmetrics := mdata.NewAggMetrics(store, &cache.MockCache{}, 600, 10, 800, 8000, 10000, 0, make([]mdata.AggSetting, 0))
-	metricIndex := memory.New()
-	metricIndex.Init()
-	usage := usage.New(300, aggmetrics, metricIndex, clock.New())
-	Enabled = true
-	addr = "localhost:2003"
-	var err error
-	s := persister.Schema{
-		Name:         "default",
-		RetentionStr: "1s:1d",
-		Pattern:      regexp.MustCompile(".*"),
-	}
-	s.Retentions, err = persister.ParseRetentionDefs(s.RetentionStr)
-	if err != nil {
-		panic(err)
-	}
-
-	schemas = persister.WhisperSchemas{s}
-	c := New()
-	// note: we could better create a mock handler that tracks Process calls
-	// rather then having to rely on the real one and index.
-	c.Start(input.NewDefaultHandler(aggmetrics, metricIndex, usage, "carbon"))
-
-	allMetrics := make(map[string]int)
-	var mu sync.Mutex
-	var wg sync.WaitGroup
-	for i := 0; i < 5; i++ {
-		wg.Add(1)
-		go func(i int, t *testing.T) {
-			metrics := test_handleMessage(i, t)
-			mu.Lock()
-			for mId, id := range metrics {
-				allMetrics[mId] = id
-			}
-			mu.Unlock()
-			wg.Done()
-		}(i, t)
-	}
-	wg.Wait()
-	c.Stop()
-	defs := metricIndex.List(-1)
-	if len(defs) != len(allMetrics) {
-		t.Fatalf("query for org -1 should result in %d distinct metrics. not %d", len(allMetrics), len(defs))
-	}
-
-	for _, d := range defs {
-		id := allMetrics[d.Id]
-		if d.Name != fmt.Sprintf("some.id.%d", id) {
-			t.Fatalf("incorrect name for %s : %s, expected %s", d.Id, d.Name, fmt.Sprintf("some.id.%d", id))
-		}
-		if d.OrgId != 1 {
-			t.Fatalf("incorrect OrgId for %s : %d", d.Id, d.OrgId)
-		}
-	}
-}
-
-func test_handleMessage(worker int, t *testing.T) map[string]int {
-	conn, _ := net.Dial("tcp", "127.0.0.1:2003")
-	defer conn.Close()
-	metrics := make(map[string]int)
-	for m := 0; m < 4; m++ {
-		id := (worker + 1) * (m + 1)
-		t.Logf("worker %d metric %d -> adding metric with id %d and orgid %d", worker, m, id, 1)
-		md := &schema.MetricData{
-			Name:     fmt.Sprintf("some.id.%d", id),
-			Metric:   fmt.Sprintf("some.id.%d", id),
-			Interval: 1,
-			Value:    1234.567,
-			Unit:     "unknown",
-			Time:     int64(id),
-			Mtype:    "gauge",
-			Tags:     []string{},
-			OrgId:    1, // admin org
-		}
-		md.SetId()
-		metrics[md.Id] = id
-		t.Logf("%s = %s", md.Id, md.Name)
-		_, err := fmt.Fprintf(conn, fmt.Sprintf("%s %f %d\n", md.Name, md.Value, md.Time))
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-	// as soon as this function ends, the server will close the socket.  We need to sleep here
-	// to ensure that the packets have time to be processed by the kernel and passed to the server.
-	time.Sleep(time.Millisecond * 100)
-	return metrics
-}
diff --git a/input/input_test.go b/input/input_test.go
index aa6e92b83e..e973fbf16b 100644
--- a/input/input_test.go
+++ b/input/input_test.go
@@ -1,11 +1,11 @@
 package input
 
 import (
-	"fmt"
 	"testing"
 	"time"
 
 	"github.com/benbjohnson/clock"
+	whisper "github.com/lomik/go-whisper"
 	"github.com/raintank/metrictank/cluster"
 	"github.com/raintank/metrictank/idx/memory"
 	"github.com/raintank/metrictank/mdata"
@@ -14,81 +14,15 @@ import (
 	"gopkg.in/raintank/schema.v1"
 )
 
-func Test_Process(t *testing.T) {
-	cluster.Init("default", "test", time.Now(), "http", 6060)
-	store := mdata.NewDevnullStore()
-	aggmetrics := mdata.NewAggMetrics(store, &cache.MockCache{}, 600, 10, 800, 8000, 10000, 0, make([]mdata.AggSetting, 0))
-	metricIndex := memory.New()
-	metricIndex.Init()
-	usage := usage.New(300, aggmetrics, metricIndex, clock.New())
-	in := NewDefaultHandler(aggmetrics, metricIndex, usage, "TestProcess")
-
-	allMetrics := make(map[string]int)
-	for i := 0; i < 5; i++ {
-		metrics := test_Process(i, &in, t)
-		for mId, id := range metrics {
-			allMetrics[mId] = id
-		}
-	}
-	defs := metricIndex.List(-1)
-	if len(defs) != 13 {
-		t.Fatalf("query for org -1 should result in 13 distinct metrics. not %d", len(defs))
-	}
-
-	for _, d := range defs {
-		id := allMetrics[d.Id]
-		if d.Name != fmt.Sprintf("some.id.%d", id) {
-			t.Fatalf("incorrect name for %s : %s", d.Id, d.Name)
-		}
-		if d.OrgId != id {
-			t.Fatalf("incorrect OrgId for %s : %d", d.Id, d.OrgId)
-		}
-		if d.Tags[0] != fmt.Sprintf("%d", id) {
-			t.Fatalf("incorrect tags for %s : %s", d.Id, d.Tags)
-		}
-	}
-
-	defs = metricIndex.List(2)
-	if len(defs) != 1 {
-		t.Fatalf("len of defs should be exactly 1. got defs with len %d: %v", len(defs), defs)
-	}
-	d := defs[0]
-	if d.OrgId != 2 {
-		t.Fatalf("incorrect metricdef returned: %v", d)
-	}
-}
-
-func test_Process(worker int, in Handler, t *testing.T) map[string]int {
-	var metric *schema.MetricData
-	metrics := make(map[string]int)
-	for m := 0; m < 4; m++ {
-		id := (worker + 1) * (m + 1)
-		t.Logf("worker %d metric %d -> adding metric with id and orgid %d", worker, m, id)
-
-		metric = &schema.MetricData{
-			Id:       "",
-			OrgId:    id,
-			Name:     fmt.Sprintf("some.id.%d", id),
-			Metric:   fmt.Sprintf("some.id.%d", id),
-			Interval: 60,
-			Value:    1234.567,
-			Unit:     "ms",
-			Time:     int64(id),
-			Mtype:    "gauge",
-			Tags:     []string{fmt.Sprintf("%d", id)},
-		}
-		metric.SetId()
-		metrics[metric.Id] = id
-		in.Process(metric, 1)
-	}
-	return metrics
-}
-
 func BenchmarkProcess(b *testing.B) {
 	cluster.Init("default", "test", time.Now(), "http", 6060)
 
 	store := mdata.NewDevnullStore()
-	aggmetrics := mdata.NewAggMetrics(store, &cache.MockCache{}, 600, 10, 800, 8000, 10000, 0, make([]mdata.AggSetting, 0))
+
+	mdata.SetSingleSchema(whisper.NewRetentionMT(10, 10000, 600, 10, true))
+	mdata.SetOnlyDefaultAgg(whisper.Average, whisper.Min, whisper.Max)
+
+	aggmetrics := mdata.NewAggMetrics(store, &cache.MockCache{}, 800, 8000, 0)
 	metricIndex := memory.New()
 	metricIndex.Init()
 	usage := usage.New(300, aggmetrics, metricIndex, clock.New())
diff --git a/input/kafkamdm/kafkamdm_test.go b/input/kafkamdm/kafkamdm_test.go
deleted file mode 100644
index 320d865fa5..0000000000
--- a/input/kafkamdm/kafkamdm_test.go
+++ /dev/null
@@ -1,94 +0,0 @@
-package kafkamdm
-
-import (
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/benbjohnson/clock"
-	"github.com/raintank/metrictank/cluster"
-	"github.com/raintank/metrictank/idx/memory"
-	"github.com/raintank/metrictank/input"
-	"github.com/raintank/metrictank/mdata"
-	"github.com/raintank/metrictank/mdata/cache"
-	"github.com/raintank/metrictank/usage"
-
-	"gopkg.in/raintank/schema.v1"
-)
-
-func Test_HandleMessage(t *testing.T) {
-	cluster.Init("default", "test", time.Now(), "http", 6060)
-	store := mdata.NewDevnullStore()
-	aggmetrics := mdata.NewAggMetrics(store, &cache.MockCache{}, 600, 10, 800, 8000, 10000, 0, make([]mdata.AggSetting, 0))
-	metricIndex := memory.New()
-	metricIndex.Init()
-	usage := usage.New(300, aggmetrics, metricIndex, clock.New())
-	k := KafkaMdm{}
-	k.Start(input.NewDefaultHandler(aggmetrics, metricIndex, usage, "test"))
-
-	allMetrics := make(map[string]int)
-	for i := 0; i < 5; i++ {
-		metrics := test_handleMessage(i, &k, t)
-		for mId, id := range metrics {
-			allMetrics[mId] = id
-		}
-	}
-	defs := metricIndex.List(-1)
-	if len(defs) != 13 {
-		t.Fatalf("query for org -1 should result in 13 distinct metrics. not %d", len(defs))
-	}
-
-	for _, d := range defs {
-		id := allMetrics[d.Id]
-		if d.Name != fmt.Sprintf("some.id.%d", id) {
-			t.Fatalf("incorrect name for %s : %s", d.Id, d.Name)
-		}
-		if d.OrgId != id {
-			t.Fatalf("incorrect OrgId for %s : %d", d.Id, d.OrgId)
-		}
-		if d.Tags[0] != fmt.Sprintf("%d", id) {
-			t.Fatalf("incorrect tags for %s : %s", d.Id, d.Tags)
-		}
-	}
-
-	defs = metricIndex.List(2)
-	if len(defs) != 1 {
-		t.Fatalf("len of defs should be exactly 1. got defs with len %d: %v", len(defs), defs)
-	}
-	d := defs[0]
-	if d.OrgId != 2 {
-		t.Fatalf("incorrect metricdef returned: %v", d)
-	}
-}
-
-func test_handleMessage(worker int, k *KafkaMdm, t *testing.T) map[string]int {
-	var metric *schema.MetricData
-	metrics := make(map[string]int)
-	for m := 0; m < 4; m++ {
-		id := (worker + 1) * (m + 1)
-		t.Logf("worker %d metric %d -> adding metric with id and orgid %d", worker, m, id)
-
-		metric = &schema.MetricData{
-			Id:       "",
-			OrgId:    id,
-			Name:     fmt.Sprintf("some.id.%d", id),
-			Metric:   fmt.Sprintf("some.id.%d", id),
-			Interval: 60,
-			Value:    1234.567,
-			Unit:     "ms",
-			Time:     int64(id),
-			Mtype:    "gauge",
-			Tags:     []string{fmt.Sprintf("%d", id)},
-		}
-		metric.SetId()
-		metrics[metric.Id] = id
-		var data []byte
-		var err error
-		data, err = metric.MarshalMsg(data[:])
-		if err != nil {
-			t.Fatal(err.Error())
-		}
-		k.handleMsg(data, 1)
-	}
-	return metrics
-}
diff --git a/mdata/aggmetric_test.go b/mdata/aggmetric_test.go
index 06d43ff34e..91ba2311be 100644
--- a/mdata/aggmetric_test.go
+++ b/mdata/aggmetric_test.go
@@ -5,6 +5,7 @@ import (
 	"testing"
 	"time"
 
+	whisper "github.com/lomik/go-whisper"
 	"github.com/raintank/metrictank/cluster"
 	"github.com/raintank/metrictank/mdata/cache"
 )
@@ -93,7 +94,8 @@ func testMetricPersistOptionalPrimary(t *testing.T, primary bool) {
 	mockCache.CacheIfHotCb = func() { calledCb <- true }
 
 	numChunks, chunkAddCount, chunkSpan := uint32(5), uint32(10), uint32(300)
-	agg := NewAggMetric(dnstore, &mockCache, "foo", chunkSpan, numChunks, 1, []AggSetting{}...)
+	ret := []*whisper.Retention{whisper.NewRetentionMT(1, 1, chunkSpan, numChunks, true)}
+	agg := NewAggMetric(dnstore, &mockCache, "foo", ret, nil)
 
 	for ts := chunkSpan; ts <= chunkSpan*chunkAddCount; ts += chunkSpan {
 		agg.Add(ts, 1)
@@ -128,7 +130,8 @@ func testMetricPersistOptionalPrimary(t *testing.T, primary bool) {
 func TestAggMetric(t *testing.T) {
 	cluster.Init("default", "test", time.Now(), "http", 6060)
 
-	c := NewChecker(t, NewAggMetric(dnstore, &cache.MockCache{}, "foo", 100, 5, 1, []AggSetting{}...))
+	ret := []*whisper.Retention{whisper.NewRetentionMT(1, 1, 100, 5, true)}
+	c := NewChecker(t, NewAggMetric(dnstore, &cache.MockCache{}, "foo", ret, nil))
 
 	// basic case, single range
 	c.Add(101, 101)
@@ -207,131 +210,114 @@ func TestAggMetric(t *testing.T) {
 func BenchmarkAggMetrics1000Metrics1Day(b *testing.B) {
 	cluster.Init("default", "test", time.Now(), "http", 6060)
 	// we will store 10s metrics in 5 chunks of 2 hours
-	// aggragate them in 5min buckets, stored in 1 chunk of 24hours
-	chunkSpan := uint32(2 * 3600)
-	numChunks := uint32(5)
+	// aggregate them in 5min buckets, stored in 1 chunk of 24hours
+	SetOnlyDefaultAgg(whisper.Average, whisper.Min, whisper.Max)
+	SetSingleSchema(
+		whisper.NewRetentionMT(1, 84600, 2*3600, 5, true),
+		whisper.NewRetentionMT(300, 30*84600, 24*3600, 1, true),
+	)
 	chunkMaxStale := uint32(3600)
 	metricMaxStale := uint32(21600)
-	ttl := uint32(84600)
-	aggSettings := []AggSetting{
-		{
-			Span:      uint32(300),
-			ChunkSpan: uint32(24 * 3600),
-			NumChunks: uint32(1),
-		},
-	}
 
 	keys := make([]string, 1000)
 	for i := 0; i < 1000; i++ {
 		keys[i] = fmt.Sprintf("hello.this.is.a.test.key.%d", i)
 	}
 
-	metrics := NewAggMetrics(dnstore, &cache.MockCache{}, chunkSpan, numChunks, chunkMaxStale, metricMaxStale, ttl, 0, aggSettings)
+	metrics := NewAggMetrics(dnstore, &cache.MockCache{}, chunkMaxStale, metricMaxStale, 0)
 
 	maxT := 3600 * 24 * uint32(b.N) // b.N in days
 	for t := uint32(1); t < maxT; t += 10 {
 		for metricI := 0; metricI < 1000; metricI++ {
 			k := keys[metricI]
-			m := metrics.GetOrCreate(k, k)
+			m := metrics.GetOrCreate(k, k, 0, 0)
 			m.Add(t, float64(t))
 		}
 	}
 }
 
 func BenchmarkAggMetrics1kSeries2Chunks1kQueueSize(b *testing.B) {
-	chunkSpan := uint32(600)
-	numChunks := uint32(5)
 	chunkMaxStale := uint32(3600)
 	metricMaxStale := uint32(21600)
 
-	cluster.Init("default", "test", time.Now(), "http", 6060)
+	SetOnlyDefaultAgg(whisper.Average, whisper.Min, whisper.Max)
+	SetSingleSchema(
+		whisper.NewRetentionMT(1, 84600, 600, 5, true),
+		whisper.NewRetentionMT(300, 84600, 24*3600, 2, true),
+	)
 
-	ttl := uint32(84600)
-	aggSettings := []AggSetting{
-		{
-			Span:      uint32(300),
-			ChunkSpan: uint32(24 * 3600),
-			NumChunks: uint32(2),
-		},
-	}
+	cluster.Init("default", "test", time.Now(), "http", 6060)
 
 	keys := make([]string, 1000)
 	for i := 0; i < 1000; i++ {
 		keys[i] = fmt.Sprintf("hello.this.is.a.test.key.%d", i)
 	}
 
-	metrics := NewAggMetrics(dnstore, &cache.MockCache{}, chunkSpan, numChunks, chunkMaxStale, metricMaxStale, ttl, 0, aggSettings)
+	metrics := NewAggMetrics(dnstore, &cache.MockCache{}, chunkMaxStale, metricMaxStale, 0)
 
 	maxT := uint32(1200)
 	for t := uint32(1); t < maxT; t += 10 {
 		for metricI := 0; metricI < 1000; metricI++ {
-			m := metrics.GetOrCreate(keys[metricI])
+			k := keys[metricI]
+			m := metrics.GetOrCreate(k, k, 0, 0)
 			m.Add(t, float64(t))
 		}
 	}
 }
 
 func BenchmarkAggMetrics10kSeries2Chunks10kQueueSize(b *testing.B) {
-	chunkSpan := uint32(600)
-	numChunks := uint32(5)
 	chunkMaxStale := uint32(3600)
 	metricMaxStale := uint32(21600)
 
-	cluster.Init("default", "test", time.Now(), "http", 6060)
+	SetOnlyDefaultAgg(whisper.Average, whisper.Min, whisper.Max)
+	SetSingleSchema(
+		whisper.NewRetentionMT(1, 84600, 600, 5, true),
+		whisper.NewRetentionMT(300, 84600, 24*3600, 2, true),
+	)
 
-	ttl := uint32(84600)
-	aggSettings := []AggSetting{
-		{
-			Span:      uint32(300),
-			ChunkSpan: uint32(24 * 3600),
-			NumChunks: uint32(2),
-		},
-	}
+	cluster.Init("default", "test", time.Now(), "http", 6060)
 
 	keys := make([]string, 10000)
 	for i := 0; i < 10000; i++ {
 		keys[i] = fmt.Sprintf("hello.this.is.a.test.key.%d", i)
 	}
 
-	metrics := NewAggMetrics(dnstore, &cache.MockCache{}, chunkSpan, numChunks, chunkMaxStale, metricMaxStale, ttl, 0, aggSettings)
+	metrics := NewAggMetrics(dnstore, &cache.MockCache{}, chunkMaxStale, metricMaxStale, 0)
 
 	maxT := uint32(1200)
 	for t := uint32(1); t < maxT; t += 10 {
 		for metricI := 0; metricI < 10000; metricI++ {
-			m := metrics.GetOrCreate(keys[metricI])
+			k := keys[metricI]
+			m := metrics.GetOrCreate(k, k, 0, 0)
 			m.Add(t, float64(t))
 		}
 	}
 }
 
 func BenchmarkAggMetrics100kSeries2Chunks100kQueueSize(b *testing.B) {
-	chunkSpan := uint32(600)
-	numChunks := uint32(5)
 	chunkMaxStale := uint32(3600)
 	metricMaxStale := uint32(21600)
 
-	cluster.Init("default", "test", time.Now(), "http", 6060)
+	SetOnlyDefaultAgg(whisper.Average, whisper.Min, whisper.Max)
+	SetSingleSchema(
+		whisper.NewRetentionMT(1, 84600, 600, 5, true),
+		whisper.NewRetentionMT(300, 84600, 24*3600, 2, true),
+	)
 
-	ttl := uint32(84600)
-	aggSettings := []AggSetting{
-		{
-			Span:      uint32(300),
-			ChunkSpan: uint32(24 * 3600),
-			NumChunks: uint32(2),
-		},
-	}
+	cluster.Init("default", "test", time.Now(), "http", 6060)
 
 	keys := make([]string, 100000)
 	for i := 0; i < 100000; i++ {
 		keys[i] = fmt.Sprintf("hello.this.is.a.test.key.%d", i)
 	}
 
-	metrics := NewAggMetrics(dnstore, &cache.MockCache{}, chunkSpan, numChunks, chunkMaxStale, metricMaxStale, ttl, 0, aggSettings)
+	metrics := NewAggMetrics(dnstore, &cache.MockCache{}, chunkMaxStale, metricMaxStale, 0)
 
 	maxT := uint32(1200)
 	for t := uint32(1); t < maxT; t += 10 {
 		for metricI := 0; metricI < 100000; metricI++ {
-			m := metrics.GetOrCreate(keys[metricI])
+			k := keys[metricI]
+			m := metrics.GetOrCreate(k, k, 0, 0)
 			m.Add(t, float64(t))
 		}
 	}
diff --git a/mdata/aggregator_test.go b/mdata/aggregator_test.go
index 389daeb766..d6645287f2 100644
--- a/mdata/aggregator_test.go
+++ b/mdata/aggregator_test.go
@@ -1,11 +1,13 @@
 package mdata
 
 import (
+	"testing"
+	"time"
+
+	whisper "github.com/lomik/go-whisper"
 	"github.com/raintank/metrictank/cluster"
 	"github.com/raintank/metrictank/mdata/cache"
 	"gopkg.in/raintank/schema.v1"
-	"testing"
-	"time"
 )
 
 type testcase struct {
@@ -63,13 +65,15 @@ func TestAggregator(t *testing.T) {
 		}
 		cluster.Manager.SetPrimary(false)
 	}
-	agg := NewAggregator(dnstore, &cache.MockCache{}, "test", 60, 120, 10, 86400)
+	ret := whisper.NewRetentionMT(60, 86400, 120, 10, true)
+	aggs := AllAggregations()
+	agg := NewAggregator(dnstore, &cache.MockCache{}, "test", *ret, aggs)
 	agg.Add(100, 123.4)
 	agg.Add(110, 5)
 	expected := []schema.Point{}
 	compare("simple-min-unfinished", agg.minMetric, expected)
 
-	agg = NewAggregator(dnstore, &cache.MockCache{}, "test", 60, 120, 10, 86400)
+	agg = NewAggregator(dnstore, &cache.MockCache{}, "test", *ret, aggs)
 	agg.Add(100, 123.4)
 	agg.Add(110, 5)
 	agg.Add(130, 130)
@@ -78,7 +82,7 @@ func TestAggregator(t *testing.T) {
 	}
 	compare("simple-min-one-block", agg.minMetric, expected)
 
-	agg = NewAggregator(dnstore, &cache.MockCache{}, "test", 60, 120, 10, 86400)
+	agg = NewAggregator(dnstore, &cache.MockCache{}, "test", *ret, aggs)
 	agg.Add(100, 123.4)
 	agg.Add(110, 5)
 	agg.Add(120, 4)
@@ -87,7 +91,7 @@ func TestAggregator(t *testing.T) {
 	}
 	compare("simple-min-one-block-done-cause-last-point-just-right", agg.minMetric, expected)
 
-	agg = NewAggregator(dnstore, &cache.MockCache{}, "test", 60, 120, 10, 86400)
+	agg = NewAggregator(dnstore, &cache.MockCache{}, "test", *ret, aggs)
 	agg.Add(100, 123.4)
 	agg.Add(110, 5)
 	agg.Add(150, 1.123)
@@ -98,7 +102,7 @@ func TestAggregator(t *testing.T) {
 	}
 	compare("simple-min-two-blocks-done-cause-last-point-just-right", agg.minMetric, expected)
 
-	agg = NewAggregator(dnstore, &cache.MockCache{}, "test", 60, 120, 10, 86400)
+	agg = NewAggregator(dnstore, &cache.MockCache{}, "test", *ret, aggs)
 	agg.Add(100, 123.4)
 	agg.Add(110, 5)
 	agg.Add(190, 2451.123)
diff --git a/mdata/init.go b/mdata/init.go
index d35c447a0d..7daf014574 100644
--- a/mdata/init.go
+++ b/mdata/init.go
@@ -43,8 +43,9 @@ var (
 	// metric tank.gc_metric is the number of times the metrics GC is about to inspect a metric (series)
 	gcMetric = stats.NewCounter32("tank.gc_metric")
 
-	schemas      persister.WhisperSchemas
-	aggregations *persister.WhisperAggregation
+	// set either via ConfigProcess or from the unit tests. other code should not touch
+	Schemas      persister.WhisperSchemas
+	Aggregations *persister.WhisperAggregation
 
 	schemasFile = "/etc/raintank/storage-schemas.conf"
 	aggFile     = "/etc/raintank/storage-aggregation.conf"
@@ -52,12 +53,12 @@ var (
 
 func ConfigProcess() {
 	var err error
-	schemas, err = persister.ReadWhisperSchemas(schemasFile)
+	Schemas, err = persister.ReadWhisperSchemas(schemasFile)
 	if err != nil {
 		log.Fatalf("can't read schemas file %q: %s", schemasFile, err.Error())
 	}
 	var defaultFound bool
-	for _, schema := range schemas {
+	for _, schema := range Schemas {
 		if schema.Pattern.String() == ".*" {
 			defaultFound = true
 		}
@@ -70,7 +71,7 @@ func ConfigProcess() {
 		// but we definitely need to always be able to determine which interval to use
 		log.Fatal(4, "storage-conf does not have a default '.*' pattern")
 	}
-	aggregations, err = persister.ReadWhisperAggregation(aggFile)
+	Aggregations, err = persister.ReadWhisperAggregation(aggFile)
 	if err != nil {
 		log.Fatalf("can't read storage-aggregation file %q: %s", aggFile, err.Error())
 	}
diff --git a/mdata/schema.go b/mdata/schema.go
index b5b198dc5b..440805d449 100644
--- a/mdata/schema.go
+++ b/mdata/schema.go
@@ -1,6 +1,8 @@
 package mdata
 
 import (
+	"regexp"
+
 	"github.com/lomik/go-carbon/persister"
 	"github.com/lomik/go-whisper"
 	"github.com/raintank/metrictank/util"
@@ -9,35 +11,35 @@ import (
 // MatchSchema returns the schema for the given metric key, and the index of the schema (to efficiently reference it)
 // it will always find the schema because we made sure there is a catchall '.*' pattern
 func MatchSchema(key string) (uint16, persister.Schema) {
-	i, schema, _ := schemas.Match(key)
+	i, schema, _ := Schemas.Match(key)
 	return i, schema
 }
 
 // MatchAgg returns the aggregation definition for the given metric key, and the index of it (to efficiently reference it)
 // i may be 1 more than the last defined by user, in which case it's the default.
 func MatchAgg(key string) (uint16, *persister.WhisperAggregationItem) {
-	i, agg := aggregations.Match(key)
+	i, agg := Aggregations.Match(key)
 	return i, agg
 }
 
 // caller must assure i is valid
 func GetRetentions(i uint16) whisper.Retentions {
-	return schemas[i].Retentions
+	return Schemas[i].Retentions
 }
 
 // caller must assure i is valid
 // note the special case
 func GetAgg(i uint16) persister.WhisperAggregationItem {
-	if i+1 > uint16(len(aggregations.Data)) {
-		return *aggregations.Default
+	if i+1 > uint16(len(Aggregations.Data)) {
+		return *Aggregations.Default
 	}
-	return *aggregations.Data[i]
+	return *Aggregations.Data[i]
 }
 
 // TTLs returns a slice of all TTL's seen amongst all archives of all schemas
 func TTLs() []uint32 {
 	ttls := make(map[uint32]struct{})
-	for _, s := range schemas {
+	for _, s := range Schemas {
 		for _, r := range s.Retentions {
 			ttls[uint32(r.MaxRetention())] = struct{}{}
 		}
@@ -52,10 +54,36 @@ func TTLs() []uint32 {
 // MaxChunkSpan returns the largest chunkspan seen amongst all archives of all schemas
 func MaxChunkSpan() uint32 {
 	max := uint32(0)
-	for _, s := range schemas {
+	for _, s := range Schemas {
 		for _, r := range s.Retentions {
 			max = util.Max(max, r.ChunkSpan)
 		}
 	}
 	return max
 }
+
+func CommonAggregations() persister.WhisperAggregationItem {
+	return persister.WhisperAggregationItem{
+		AggregationMethod: []whisper.AggregationMethod{whisper.Average, whisper.Min, whisper.Max},
+	}
+}
+
+func AllAggregations() persister.WhisperAggregationItem {
+	return persister.WhisperAggregationItem{
+		AggregationMethod: []whisper.AggregationMethod{whisper.Average, whisper.Min, whisper.Max, whisper.Sum, whisper.Last},
+	}
+}
+
+func SetSingleSchema(ret ...whisper.Retention) {
+	Schemas = persister.WhisperSchemas{
+		{
+			Pattern:    regexp.MustCompile(".*"),
+			Retentions: whisper.Retentions(ret),
+		},
+	}
+}
+
+func SetOnlyDefaultAgg(met ...whisper.AggregationMethod) {
+	Aggregations = persister.NewWhisperAggregation()
+	Aggregations.Default.AggregationMethod = met
+}
diff --git a/usage/usage_test.go b/usage/usage_test.go
index b16a09a8d4..e736245dc2 100644
--- a/usage/usage_test.go
+++ b/usage/usage_test.go
@@ -6,6 +6,8 @@ import (
 	"time"
 
 	"github.com/benbjohnson/clock"
+	"github.com/lomik/go-carbon/persister"
+	whisper "github.com/lomik/go-whisper"
 	"github.com/raintank/metrictank/consolidation"
 	"github.com/raintank/metrictank/idx/memory"
 	"github.com/raintank/metrictank/mdata"
@@ -30,7 +32,7 @@ func (f *FakeAggMetrics) Get(key string) (mdata.Metric, bool) {
 	f.Unlock()
 	return m, ok
 }
-func (f *FakeAggMetrics) GetOrCreate(key, name string) mdata.Metric {
+func (f *FakeAggMetrics) GetOrCreate(key, name string, schemaI, aggI uint16) mdata.Metric {
 	f.Lock()
 	m, ok := f.Metrics[key]
 	if !ok {
@@ -40,10 +42,15 @@ func (f *FakeAggMetrics) GetOrCreate(key, name string) mdata.Metric {
 	f.Unlock()
 	return m
 }
-func (f *FakeAggMetrics) AggSettings() mdata.AggSettings {
-	return mdata.AggSettings{
-		35 * 24 * 60 * 60,
-		[]mdata.AggSetting{},
+func (f *FakeAggMetrics) Schemas() persister.WhisperSchemas {
+	return persister.WhisperSchemas{
+		{
+			Retentions: whisper.Retentions(
+				[]whisper.Retention{
+					whisper.NewRetentionMT(1, 35*24*3600, 600, 5, true),
+				},
+			),
+		},
 	}
 }
 
diff --git a/vendor/github.com/lomik/go-whisper/whisper.go b/vendor/github.com/lomik/go-whisper/whisper.go
index 1ff4d4667a..794664e77e 100644
--- a/vendor/github.com/lomik/go-whisper/whisper.go
+++ b/vendor/github.com/lomik/go-whisper/whisper.go
@@ -822,6 +822,16 @@ func NewRetention(secondsPerPoint, numberOfPoints int) Retention {
 	}
 }
 
+func NewRetentionMT(secondsPerPoint int, ttl, chunkSpan, numChunks uint32, ready bool) *Retention {
+	return &Retention{
+		secondsPerPoint: secondsPerPoint,
+		numberOfPoints:  int(ttl) / secondsPerPoint,
+		ChunkSpan:       chunkSpan,
+		NumChunks:       numChunks,
+		Ready:           ready,
+	}
+}
+
 type Retentions []*Retention
 
 func (r Retentions) Len() int {

From 59e70b0eb3c86c4a88dd00cd968df83ac1ce29da Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Wed, 1 Mar 2017 21:47:24 +0100
Subject: [PATCH 08/29] go-carbon: stop the pointer abuse

we can simplify code if we just work with the data normally
---
 mdata/aggmetric.go                            |  2 +-
 mdata/aggmetric_test.go                       |  4 +--
 mdata/aggregator.go                           | 10 +++----
 mdata/aggregator_test.go                      | 10 +++----
 mdata/init.go                                 |  2 +-
 mdata/schema.go                               |  9 +++----
 .../lomik/go-carbon/persister/whisper.go      |  4 ---
 .../persister/whisper_aggregation.go          | 26 +++++++++----------
 .../go-carbon/persister/whisper_schema.go     |  5 ++--
 vendor/github.com/lomik/go-whisper/whisper.go | 18 ++++++-------
 10 files changed, 42 insertions(+), 48 deletions(-)

diff --git a/mdata/aggmetric.go b/mdata/aggmetric.go
index 66edb0b144..68faf4774e 100644
--- a/mdata/aggmetric.go
+++ b/mdata/aggmetric.go
@@ -61,7 +61,7 @@ func NewAggMetric(store Store, cachePusher cache.CachePusher, key string, retent
 		lastWrite: uint32(time.Now().Unix()),
 	}
 	for _, ret := range retentions[1:] {
-		m.aggregators = append(m.aggregators, NewAggregator(store, cachePusher, key, *ret, *agg))
+		m.aggregators = append(m.aggregators, NewAggregator(store, cachePusher, key, ret, *agg))
 	}
 
 	return &m
diff --git a/mdata/aggmetric_test.go b/mdata/aggmetric_test.go
index 91ba2311be..dc9c81e801 100644
--- a/mdata/aggmetric_test.go
+++ b/mdata/aggmetric_test.go
@@ -94,7 +94,7 @@ func testMetricPersistOptionalPrimary(t *testing.T, primary bool) {
 	mockCache.CacheIfHotCb = func() { calledCb <- true }
 
 	numChunks, chunkAddCount, chunkSpan := uint32(5), uint32(10), uint32(300)
-	ret := []*whisper.Retention{whisper.NewRetentionMT(1, 1, chunkSpan, numChunks, true)}
+	ret := []whisper.Retention{whisper.NewRetentionMT(1, 1, chunkSpan, numChunks, true)}
 	agg := NewAggMetric(dnstore, &mockCache, "foo", ret, nil)
 
 	for ts := chunkSpan; ts <= chunkSpan*chunkAddCount; ts += chunkSpan {
@@ -130,7 +130,7 @@ func testMetricPersistOptionalPrimary(t *testing.T, primary bool) {
 func TestAggMetric(t *testing.T) {
 	cluster.Init("default", "test", time.Now(), "http", 6060)
 
-	ret := []*whisper.Retention{whisper.NewRetentionMT(1, 1, 100, 5, true)}
+	ret := []whisper.Retention{whisper.NewRetentionMT(1, 1, 100, 5, true)}
 	c := NewChecker(t, NewAggMetric(dnstore, &cache.MockCache{}, "foo", ret, nil))
 
 	// basic case, single range
diff --git a/mdata/aggregator.go b/mdata/aggregator.go
index de2153935a..6462cfd8ef 100644
--- a/mdata/aggregator.go
+++ b/mdata/aggregator.go
@@ -46,14 +46,14 @@ func NewAggregator(store Store, cachePusher cache.CachePusher, key string, ret w
 		switch agg {
 		case whisper.Average:
 			if aggregator.sumMetric == nil {
-				aggregator.sumMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_sum_%d", key, span), whisper.Retentions{&ret}, nil)
+				aggregator.sumMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_sum_%d", key, span), whisper.Retentions{ret}, nil)
 			}
 			if aggregator.cntMetric == nil {
-				aggregator.cntMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_cnt_%d", key, span), whisper.Retentions{&ret}, nil)
+				aggregator.cntMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_cnt_%d", key, span), whisper.Retentions{ret}, nil)
 			}
 		case whisper.Sum:
 			if aggregator.sumMetric == nil {
-				aggregator.sumMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_sum_%d", key, span), whisper.Retentions{&ret}, nil)
+				aggregator.sumMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_sum_%d", key, span), whisper.Retentions{ret}, nil)
 			}
 		case whisper.Last:
 			if aggregator.lstMetric == nil {
@@ -61,11 +61,11 @@ func NewAggregator(store Store, cachePusher cache.CachePusher, key string, ret w
 			}
 		case whisper.Max:
 			if aggregator.maxMetric == nil {
-				aggregator.maxMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_max_%d", key, span), whisper.Retentions{&ret}, nil)
+				aggregator.maxMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_max_%d", key, span), whisper.Retentions{ret}, nil)
 			}
 		case whisper.Min:
 			if aggregator.minMetric == nil {
-				aggregator.minMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_min_%d", key, span), whisper.Retentions{&ret}, nil)
+				aggregator.minMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_min_%d", key, span), whisper.Retentions{ret}, nil)
 			}
 		}
 	}
diff --git a/mdata/aggregator_test.go b/mdata/aggregator_test.go
index d6645287f2..f282339c6c 100644
--- a/mdata/aggregator_test.go
+++ b/mdata/aggregator_test.go
@@ -67,13 +67,13 @@ func TestAggregator(t *testing.T) {
 	}
 	ret := whisper.NewRetentionMT(60, 86400, 120, 10, true)
 	aggs := AllAggregations()
-	agg := NewAggregator(dnstore, &cache.MockCache{}, "test", *ret, aggs)
+	agg := NewAggregator(dnstore, &cache.MockCache{}, "test", ret, aggs)
 	agg.Add(100, 123.4)
 	agg.Add(110, 5)
 	expected := []schema.Point{}
 	compare("simple-min-unfinished", agg.minMetric, expected)
 
-	agg = NewAggregator(dnstore, &cache.MockCache{}, "test", *ret, aggs)
+	agg = NewAggregator(dnstore, &cache.MockCache{}, "test", ret, aggs)
 	agg.Add(100, 123.4)
 	agg.Add(110, 5)
 	agg.Add(130, 130)
@@ -82,7 +82,7 @@ func TestAggregator(t *testing.T) {
 	}
 	compare("simple-min-one-block", agg.minMetric, expected)
 
-	agg = NewAggregator(dnstore, &cache.MockCache{}, "test", *ret, aggs)
+	agg = NewAggregator(dnstore, &cache.MockCache{}, "test", ret, aggs)
 	agg.Add(100, 123.4)
 	agg.Add(110, 5)
 	agg.Add(120, 4)
@@ -91,7 +91,7 @@ func TestAggregator(t *testing.T) {
 	}
 	compare("simple-min-one-block-done-cause-last-point-just-right", agg.minMetric, expected)
 
-	agg = NewAggregator(dnstore, &cache.MockCache{}, "test", *ret, aggs)
+	agg = NewAggregator(dnstore, &cache.MockCache{}, "test", ret, aggs)
 	agg.Add(100, 123.4)
 	agg.Add(110, 5)
 	agg.Add(150, 1.123)
@@ -102,7 +102,7 @@ func TestAggregator(t *testing.T) {
 	}
 	compare("simple-min-two-blocks-done-cause-last-point-just-right", agg.minMetric, expected)
 
-	agg = NewAggregator(dnstore, &cache.MockCache{}, "test", *ret, aggs)
+	agg = NewAggregator(dnstore, &cache.MockCache{}, "test", ret, aggs)
 	agg.Add(100, 123.4)
 	agg.Add(110, 5)
 	agg.Add(190, 2451.123)
diff --git a/mdata/init.go b/mdata/init.go
index 7daf014574..cbee2ec876 100644
--- a/mdata/init.go
+++ b/mdata/init.go
@@ -45,7 +45,7 @@ var (
 
 	// set either via ConfigProcess or from the unit tests. other code should not touch
 	Schemas      persister.WhisperSchemas
-	Aggregations *persister.WhisperAggregation
+	Aggregations persister.WhisperAggregation
 
 	schemasFile = "/etc/raintank/storage-schemas.conf"
 	aggFile     = "/etc/raintank/storage-aggregation.conf"
diff --git a/mdata/schema.go b/mdata/schema.go
index 440805d449..b8ce0461ec 100644
--- a/mdata/schema.go
+++ b/mdata/schema.go
@@ -17,9 +17,8 @@ func MatchSchema(key string) (uint16, persister.Schema) {
 
 // MatchAgg returns the aggregation definition for the given metric key, and the index of it (to efficiently reference it)
 // i may be 1 more than the last defined by user, in which case it's the default.
-func MatchAgg(key string) (uint16, *persister.WhisperAggregationItem) {
-	i, agg := Aggregations.Match(key)
-	return i, agg
+func MatchAgg(key string) (uint16, persister.WhisperAggregationItem) {
+	return Aggregations.Match(key)
 }
 
 // caller must assure i is valid
@@ -31,9 +30,9 @@ func GetRetentions(i uint16) whisper.Retentions {
 // note the special case
 func GetAgg(i uint16) persister.WhisperAggregationItem {
 	if i+1 > uint16(len(Aggregations.Data)) {
-		return *Aggregations.Default
+		return Aggregations.Default
 	}
-	return *Aggregations.Data[i]
+	return Aggregations.Data[i]
 }
 
 // TTLs returns a slice of all TTL's seen amongst all archives of all schemas
diff --git a/vendor/github.com/lomik/go-carbon/persister/whisper.go b/vendor/github.com/lomik/go-carbon/persister/whisper.go
index 932d291556..31262969a0 100644
--- a/vendor/github.com/lomik/go-carbon/persister/whisper.go
+++ b/vendor/github.com/lomik/go-carbon/persister/whisper.go
@@ -111,10 +111,6 @@ func store(p *Whisper, values *points.Points) {
 		}
 
 		_, aggr := p.aggregation.Match(values.Metric)
-		if aggr == nil {
-			logrus.Errorf("[persister] No storage aggregation defined for %s", values.Metric)
-			return
-		}
 
 		logrus.WithFields(logrus.Fields{
 			"retention":    schema.RetentionStr,
diff --git a/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go b/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
index cd88bb2ed4..e2f1a5c8fb 100644
--- a/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
+++ b/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
@@ -25,15 +25,15 @@ type WhisperAggregationItem struct {
 
 // WhisperAggregation ...
 type WhisperAggregation struct {
-	Data    []*WhisperAggregationItem
-	Default *WhisperAggregationItem
+	Data    []WhisperAggregationItem
+	Default WhisperAggregationItem
 }
 
 // NewWhisperAggregation create instance of WhisperAggregation
-func NewWhisperAggregation() *WhisperAggregation {
-	return &WhisperAggregation{
-		Data: make([]*WhisperAggregationItem, 0),
-		Default: &WhisperAggregationItem{
+func NewWhisperAggregation() WhisperAggregation {
+	return WhisperAggregation{
+		Data: make([]WhisperAggregationItem, 0),
+		Default: WhisperAggregationItem{
 			name:                 "default",
 			pattern:              nil,
 			XFilesFactor:         0.5,
@@ -44,21 +44,21 @@ func NewWhisperAggregation() *WhisperAggregation {
 }
 
 // ReadWhisperAggregation ...
-func ReadWhisperAggregation(file string) (*WhisperAggregation, error) {
+func ReadWhisperAggregation(file string) (WhisperAggregation, error) {
 	config, err := configparser.Read(file)
 	if err != nil {
-		return nil, err
+		return WhisperAggregation{}, err
 	}
 	// pp.Println(config)
 	sections, err := config.AllSections()
 	if err != nil {
-		return nil, err
+		return WhisperAggregation{}, err
 	}
 
 	result := NewWhisperAggregation()
 
 	for _, s := range sections {
-		item := &WhisperAggregationItem{}
+		item := WhisperAggregationItem{}
 		// this is mildly stupid, but I don't feel like forking
 		// configparser just for this
 		item.name =
@@ -71,14 +71,14 @@ func ReadWhisperAggregation(file string) (*WhisperAggregation, error) {
 		if err != nil {
 			logrus.Errorf("[persister] Failed to parse pattern '%s'for [%s]: %s",
 				s.ValueOf("pattern"), item.name, err.Error())
-			return nil, err
+			return WhisperAggregation{}, err
 		}
 
 		item.XFilesFactor, err = strconv.ParseFloat(s.ValueOf("xFilesFactor"), 64)
 		if err != nil {
 			logrus.Errorf("failed to parse xFilesFactor '%s' in %s: %s",
 				s.ValueOf("xFilesFactor"), item.name, err.Error())
-			return nil, err
+			return WhisperAggregation{}, err
 		}
 
 		item.aggregationMethodStr = s.ValueOf("aggregationMethod")
@@ -112,7 +112,7 @@ func ReadWhisperAggregation(file string) (*WhisperAggregation, error) {
 }
 
 // Match find schema for metric
-func (a *WhisperAggregation) Match(metric string) (uint16, *WhisperAggregationItem) {
+func (a *WhisperAggregation) Match(metric string) (uint16, WhisperAggregationItem) {
 	for i, s := range a.Data {
 		if s.pattern.MatchString(metric) {
 			return uint16(i), s
diff --git a/vendor/github.com/lomik/go-carbon/persister/whisper_schema.go b/vendor/github.com/lomik/go-carbon/persister/whisper_schema.go
index e1369ec553..be060792ee 100644
--- a/vendor/github.com/lomik/go-carbon/persister/whisper_schema.go
+++ b/vendor/github.com/lomik/go-carbon/persister/whisper_schema.go
@@ -105,11 +105,10 @@ func ParseRetentionDefs(retentionDefs string) (whisper.Retentions, error) {
 		val1, err1 := strconv.ParseInt(parts[0], 10, 0)
 		val2, err2 := strconv.ParseInt(parts[1], 10, 0)
 
-		var retention *whisper.Retention
+		var retention whisper.Retention
 		var err error
 		if err1 == nil && err2 == nil {
-			ret := whisper.NewRetention(int(val1), int(val2))
-			retention = &ret
+			retention = whisper.NewRetention(int(val1), int(val2))
 		} else {
 			// try new format
 			retention, err = whisper.ParseRetentionDef(retentionDef)
diff --git a/vendor/github.com/lomik/go-whisper/whisper.go b/vendor/github.com/lomik/go-whisper/whisper.go
index 794664e77e..bf389c0a4a 100644
--- a/vendor/github.com/lomik/go-whisper/whisper.go
+++ b/vendor/github.com/lomik/go-whisper/whisper.go
@@ -94,22 +94,22 @@ func parseRetentionPart(retentionPart string) (int, error) {
 
   See: http://graphite.readthedocs.org/en/1.0/config-carbon.html#storage-schemas-conf
 */
-func ParseRetentionDef(retentionDef string) (*Retention, error) {
+func ParseRetentionDef(retentionDef string) (Retention, error) {
 	parts := strings.Split(retentionDef, ":")
 	if len(parts) < 2 {
-		return nil, fmt.Errorf("Not enough parts in retentionDef [%v]", retentionDef)
+		return Retention{}, fmt.Errorf("Not enough parts in retentionDef [%v]", retentionDef)
 	}
 	precision, err := parseRetentionPart(parts[0])
 	if err != nil {
-		return nil, fmt.Errorf("Failed to parse precision: %v", err)
+		return Retention{}, fmt.Errorf("Failed to parse precision: %v", err)
 	}
 
 	ttl, err := parseRetentionPart(parts[1])
 	if err != nil {
-		return nil, fmt.Errorf("Failed to parse points: %v", err)
+		return Retention{}, fmt.Errorf("Failed to parse points: %v", err)
 	}
 
-	return &Retention{
+	return Retention{
 		secondsPerPoint: precision,
 		numberOfPoints:  ttl / precision}, err
 }
@@ -193,7 +193,7 @@ func CreateWithOptions(path string, retentions Retentions, aggregationMethod Agg
 	offset := MetadataSize + (ArchiveInfoSize * len(retentions))
 	whisper.archives = make([]*archiveInfo, 0, len(retentions))
 	for _, retention := range retentions {
-		whisper.archives = append(whisper.archives, &archiveInfo{*retention, offset})
+		whisper.archives = append(whisper.archives, &archiveInfo{retention, offset})
 		offset += retention.Size()
 	}
 
@@ -822,8 +822,8 @@ func NewRetention(secondsPerPoint, numberOfPoints int) Retention {
 	}
 }
 
-func NewRetentionMT(secondsPerPoint int, ttl, chunkSpan, numChunks uint32, ready bool) *Retention {
-	return &Retention{
+func NewRetentionMT(secondsPerPoint int, ttl, chunkSpan, numChunks uint32, ready bool) Retention {
+	return Retention{
 		secondsPerPoint: secondsPerPoint,
 		numberOfPoints:  int(ttl) / secondsPerPoint,
 		ChunkSpan:       chunkSpan,
@@ -832,7 +832,7 @@ func NewRetentionMT(secondsPerPoint int, ttl, chunkSpan, numChunks uint32, ready
 	}
 }
 
-type Retentions []*Retention
+type Retentions []Retention
 
 func (r Retentions) Len() int {
 	return len(r)

From 6e92dc4c4add52e0d82888286179315ed9490819 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Fri, 3 Mar 2017 10:16:57 +0100
Subject: [PATCH 09/29] add storage-aggregation.conf to our various deployments

so that MT is ready to go via docker or packages
(they already had storage-schemas.conf)
---
 scripts/Dockerfile                      | 1 +
 scripts/config/storage-aggregation.conf | 4 ++++
 scripts/package.sh                      | 5 +++++
 3 files changed, 10 insertions(+)
 create mode 100644 scripts/config/storage-aggregation.conf

diff --git a/scripts/Dockerfile b/scripts/Dockerfile
index c574695f79..06f9aa54e7 100644
--- a/scripts/Dockerfile
+++ b/scripts/Dockerfile
@@ -4,6 +4,7 @@ MAINTAINER Anthony Woods awoods@raintank.io
 RUN mkdir -p /etc/raintank
 COPY config/metrictank-docker.ini /etc/raintank/metrictank.ini
 COPY config/storage-schemas.conf /etc/raintank/storage-schemas.conf
+COPY config/storage-aggregation.conf /etc/raintank/storage-aggregation.conf
 
 COPY build/* /usr/bin/
 
diff --git a/scripts/config/storage-aggregation.conf b/scripts/config/storage-aggregation.conf
new file mode 100644
index 0000000000..251d252590
--- /dev/null
+++ b/scripts/config/storage-aggregation.conf
@@ -0,0 +1,4 @@
+[default]
+pattern = .*
+xFilesFactor = 0.1
+aggregationMethod = avg,min,max
diff --git a/scripts/package.sh b/scripts/package.sh
index 8cf6fcfdc8..f8d8b5eaeb 100755
--- a/scripts/package.sh
+++ b/scripts/package.sh
@@ -17,6 +17,7 @@ mkdir -p ${BUILD}/etc/raintank
 
 cp ${BASE}/config/metrictank-package.ini ${BUILD}/etc/raintank/metrictank.ini
 cp ${BASE}/config/storage-schemas.conf ${BUILD}/etc/raintank/
+cp ${BASE}/config/storage-aggregation.conf ${BUILD}/etc/raintank/
 cp ${BUILD_ROOT}/{metrictank,mt-*} ${BUILD}/usr/sbin/
 
 PACKAGE_NAME="${BUILD}/metrictank-${VERSION}_${ARCH}.deb"
@@ -37,6 +38,7 @@ mkdir -p ${BUILD}/etc/raintank
 
 cp ${BASE}/config/metrictank-package.ini ${BUILD}/etc/raintank/metrictank.ini
 cp ${BASE}/config/storage-schemas.conf ${BUILD}/etc/raintank/
+cp ${BASE}/config/storage-aggregation.conf ${BUILD}/etc/raintank/
 cp ${BUILD_ROOT}/metrictank ${BUILD}/usr/sbin/
 
 PACKAGE_NAME="${BUILD}/metrictank-${VERSION}_${ARCH}.deb"
@@ -57,6 +59,7 @@ mkdir -p ${BUILD}/var/run/raintank
 
 cp ${BASE}/config/metrictank-package.ini ${BUILD}/etc/raintank/metrictank.ini
 cp ${BASE}/config/storage-schemas.conf ${BUILD}/etc/raintank/
+cp ${BASE}/config/storage-aggregation.conf ${BUILD}/etc/raintank/
 cp ${BUILD_ROOT}/metrictank ${BUILD}/usr/sbin/
 cp ${BASE}/config/systemd/metrictank.service $BUILD/lib/systemd/system/
 
@@ -76,6 +79,7 @@ mkdir -p ${BUILD}/var/run/raintank
 
 cp ${BASE}/config/metrictank-package.ini ${BUILD}/etc/raintank/metrictank.ini
 cp ${BASE}/config/storage-schemas.conf ${BUILD}/etc/raintank/
+cp ${BASE}/config/storage-aggregation.conf ${BUILD}/etc/raintank/
 cp ${BUILD_ROOT}/metrictank ${BUILD}/usr/sbin/
 cp ${BASE}/config/systemd/metrictank.service $BUILD/lib/systemd/system/
 
@@ -95,6 +99,7 @@ mkdir -p ${BUILD}/etc/raintank
 
 cp ${BASE}/config/metrictank-package.ini ${BUILD}/etc/raintank/metrictank.ini
 cp ${BASE}/config/storage-schemas.conf ${BUILD}/etc/raintank/
+cp ${BASE}/config/storage-aggregation.conf ${BUILD}/etc/raintank/
 cp ${BUILD_ROOT}/metrictank ${BUILD}/usr/sbin/
 cp ${BASE}/config/upstart-0.6.5/metrictank.conf $BUILD/etc/init
 

From b36e4ee671eaba5e05cfa8d52b8b738a91a60047 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Fri, 3 Mar 2017 10:21:56 +0100
Subject: [PATCH 10/29] use /etc/metrictank instead of /etc/raintank config
 directory

/etc/raintank was silly to begin with.
Config directories should be named after the software, not after
the company.  Especially now we're getting rid of the raintank name
---
 cmd/mt-kafka-mdm-sniff/main.go                |  2 +-
 cmd/mt-split-metrics-by-ttl/main.go           |  2 +-
 cmd/mt-store-cat/main.go                      |  4 +-
 cmd/mt-update-ttl/main.go                     |  2 +-
 docker/docker-cluster/docker-compose.yml      |  8 ++--
 docker/docker-cluster/metrictank.ini          |  4 +-
 .../docker-compose.yml                        |  6 +--
 .../metrictank.ini                            |  4 +-
 docker/docker-dev/docker-compose.yml          |  6 +--
 docs/config.md                                |  4 +-
 docs/installation-deb.md                      |  4 +-
 docs/installation-rpm.md                      |  4 +-
 docs/installation-source.md                   |  2 +-
 docs/tools.md                                 | 12 ++---
 idx/cassandra/cassandra.go                    |  2 +-
 mdata/init.go                                 |  4 +-
 metrictank-sample.ini                         |  4 +-
 metrictank.go                                 |  4 +-
 scripts/Dockerfile                            | 10 ++---
 scripts/config/metrictank-docker.ini          |  4 +-
 scripts/config/metrictank-package.ini         |  4 +-
 scripts/config/systemd/metrictank.service     |  2 +-
 scripts/config/sysvinit/init.d/metrictank     |  2 +-
 scripts/config/upstart-0.6.5/metrictank.conf  |  2 +-
 scripts/config/upstart/metrictank             |  2 +-
 scripts/package.sh                            | 44 +++++++++----------
 26 files changed, 74 insertions(+), 74 deletions(-)

diff --git a/cmd/mt-kafka-mdm-sniff/main.go b/cmd/mt-kafka-mdm-sniff/main.go
index 214978842e..887f9c6057 100644
--- a/cmd/mt-kafka-mdm-sniff/main.go
+++ b/cmd/mt-kafka-mdm-sniff/main.go
@@ -20,7 +20,7 @@ import (
 )
 
 var (
-	confFile = flag.String("config", "/etc/raintank/metrictank.ini", "configuration file path")
+	confFile = flag.String("config", "/etc/metrictank/metrictank.ini", "configuration file path")
 	format   = flag.String("format", "{{.Part}} {{.OrgId}} {{.Id}} {{.Name}} {{.Metric}} {{.Interval}} {{.Value}} {{.Time}} {{.Unit}} {{.Mtype}} {{.Tags}}", "template to render the data with")
 	prefix   = flag.String("prefix", "", "only show metrics that have this prefix")
 	substr   = flag.String("substr", "", "only show metrics that have this substring")
diff --git a/cmd/mt-split-metrics-by-ttl/main.go b/cmd/mt-split-metrics-by-ttl/main.go
index 1557ce4d5c..93811aff73 100644
--- a/cmd/mt-split-metrics-by-ttl/main.go
+++ b/cmd/mt-split-metrics-by-ttl/main.go
@@ -25,7 +25,7 @@ var (
 	cqlProtocolVersion           = flag.Int("cql-protocol-version", 4, "cql protocol version to use")
 
 	cassandraSSL              = flag.Bool("cassandra-ssl", false, "enable SSL connection to cassandra")
-	cassandraCaPath           = flag.String("cassandra-ca-path", "/etc/raintank/ca.pem", "cassandra CA certificate path when using SSL")
+	cassandraCaPath           = flag.String("cassandra-ca-path", "/etc/metrictank/ca.pem", "cassandra CA certificate path when using SSL")
 	cassandraHostVerification = flag.Bool("cassandra-host-verification", true, "host (hostname and server cert) verification when using SSL")
 
 	cassandraAuth     = flag.Bool("cassandra-auth", false, "enable cassandra authentication")
diff --git a/cmd/mt-store-cat/main.go b/cmd/mt-store-cat/main.go
index 53418f6b3c..7bf420cc99 100644
--- a/cmd/mt-store-cat/main.go
+++ b/cmd/mt-store-cat/main.go
@@ -25,7 +25,7 @@ var (
 
 	// flags from metrictank.go, globals
 	showVersion = flag.Bool("version", false, "print version string")
-	confFile    = flag.String("config", "/etc/raintank/metrictank.ini", "configuration file path")
+	confFile    = flag.String("config", "/etc/metrictank/metrictank.ini", "configuration file path")
 
 	// flags from metrictank.go, Cassandra
 	cassandraAddrs               = flag.String("cassandra-addrs", "localhost", "cassandra host (may be given multiple times as comma-separated list)")
@@ -41,7 +41,7 @@ var (
 	cqlProtocolVersion = flag.Int("cql-protocol-version", 4, "cql protocol version to use")
 
 	cassandraSSL              = flag.Bool("cassandra-ssl", false, "enable SSL connection to cassandra")
-	cassandraCaPath           = flag.String("cassandra-ca-path", "/etc/raintank/ca.pem", "cassandra CA certificate path when using SSL")
+	cassandraCaPath           = flag.String("cassandra-ca-path", "/etc/metrictank/ca.pem", "cassandra CA certificate path when using SSL")
 	cassandraHostVerification = flag.Bool("cassandra-host-verification", true, "host (hostname and server cert) verification when using SSL")
 
 	cassandraAuth     = flag.Bool("cassandra-auth", false, "enable cassandra authentication")
diff --git a/cmd/mt-update-ttl/main.go b/cmd/mt-update-ttl/main.go
index 6ac54341d8..048cf09a72 100644
--- a/cmd/mt-update-ttl/main.go
+++ b/cmd/mt-update-ttl/main.go
@@ -23,7 +23,7 @@ var (
 	cqlProtocolVersion           = flag.Int("cql-protocol-version", 4, "cql protocol version to use")
 
 	cassandraSSL              = flag.Bool("cassandra-ssl", false, "enable SSL connection to cassandra")
-	cassandraCaPath           = flag.String("cassandra-ca-path", "/etc/raintank/ca.pem", "cassandra CA certificate path when using SSL")
+	cassandraCaPath           = flag.String("cassandra-ca-path", "/etc/metrictank/ca.pem", "cassandra CA certificate path when using SSL")
 	cassandraHostVerification = flag.Bool("cassandra-host-verification", true, "host (hostname and server cert) verification when using SSL")
 
 	cassandraAuth     = flag.Bool("cassandra-auth", false, "enable cassandra authentication")
diff --git a/docker/docker-cluster/docker-compose.yml b/docker/docker-cluster/docker-compose.yml
index fea86e7999..9f54aee5ac 100644
--- a/docker/docker-cluster/docker-compose.yml
+++ b/docker/docker-cluster/docker-compose.yml
@@ -8,7 +8,7 @@ services:
      - "6060:6060"
     volumes:
       - ../../build/metrictank:/usr/bin/metrictank
-      - ./metrictank.ini:/etc/raintank/metrictank.ini
+      - ./metrictank.ini:/etc/metrictank/metrictank.ini
     environment:
       WAIT_HOSTS: kafka:9092,cassandra:9042
       WAIT_TIMEOUT: 30
@@ -29,7 +29,7 @@ services:
       - "6061:6060"
     volumes:
       - ../../build/metrictank:/usr/bin/metrictank
-      - ./metrictank.ini:/etc/raintank/metrictank.ini
+      - ./metrictank.ini:/etc/metrictank/metrictank.ini
     environment:
       WAIT_HOSTS: kafka:9092,cassandra:9042,metrictank0:6060
       WAIT_TIMEOUT: 30
@@ -52,7 +52,7 @@ services:
       - "6062:6060"
     volumes:
       - ../../build/metrictank:/usr/bin/metrictank
-      - ./metrictank.ini:/etc/raintank/metrictank.ini
+      - ./metrictank.ini:/etc/metrictank/metrictank.ini
     environment:
       WAIT_HOSTS: kafka:9092,cassandra:9042,metrictank0:6060
       WAIT_TIMEOUT: 30
@@ -75,7 +75,7 @@ services:
       - "6063:6060"
     volumes:
       - ../../build/metrictank:/usr/bin/metrictank
-      - ./metrictank.ini:/etc/raintank/metrictank.ini
+      - ./metrictank.ini:/etc/metrictank/metrictank.ini
     environment:
       WAIT_HOSTS: kafka:9092,cassandra:9042,metrictank0:6060
       WAIT_TIMEOUT: 30
diff --git a/docker/docker-cluster/metrictank.ini b/docker/docker-cluster/metrictank.ini
index a5270a5c4f..a7f17396bf 100644
--- a/docker/docker-cluster/metrictank.ini
+++ b/docker/docker-cluster/metrictank.ini
@@ -57,7 +57,7 @@ cql-protocol-version = 4
 # enable SSL connection to cassandra
 cassandra-ssl = false
 # cassandra CA certficate path when using SSL
-cassandra-ca-path = /etc/raintank/ca.pem
+cassandra-ca-path = /etc/metrictank/ca.pem
 # host (hostname and server cert) verification when using SSL
 cassandra-host-verification = true
 # enable cassandra user authentication
@@ -253,7 +253,7 @@ update-fuzzyness = 0.5
 # enable SSL connection to cassandra
 ssl = false
 # cassandra CA certficate path when using SSL
-ca-path = /etc/raintank/ca.pem
+ca-path = /etc/metrictank/ca.pem
 # host (hostname and server cert) verification when using SSL
 host-verification = true
 # enable cassandra user authentication
diff --git a/docker/docker-dev-custom-cfg-kafka/docker-compose.yml b/docker/docker-dev-custom-cfg-kafka/docker-compose.yml
index 46b1b01d33..e3a62b8a51 100644
--- a/docker/docker-dev-custom-cfg-kafka/docker-compose.yml
+++ b/docker/docker-dev-custom-cfg-kafka/docker-compose.yml
@@ -9,9 +9,9 @@ services:
      - "2003:2003"
     volumes:
       - ../../build/metrictank:/usr/bin/metrictank
-      - ./metrictank.ini:/etc/raintank/metrictank.ini
-      - ./storage-schemas.conf:/etc/raintank/storage-schemas.conf
-      - ./storage-aggregation.conf:/etc/raintank/storage-aggregation.conf
+      - ./metrictank.ini:/etc/metrictank/metrictank.ini
+      - ./storage-schemas.conf:/etc/metrictank/storage-schemas.conf
+      - ./storage-aggregation.conf:/etc/metrictank/storage-aggregation.conf
     environment:
      WAIT_HOSTS: cassandra:9042,kafka:9092
      WAIT_TIMEOUT: 60
diff --git a/docker/docker-dev-custom-cfg-kafka/metrictank.ini b/docker/docker-dev-custom-cfg-kafka/metrictank.ini
index 45ebe3f061..a78d1bad0b 100644
--- a/docker/docker-dev-custom-cfg-kafka/metrictank.ini
+++ b/docker/docker-dev-custom-cfg-kafka/metrictank.ini
@@ -57,7 +57,7 @@ cql-protocol-version = 4
 # enable SSL connection to cassandra
 cassandra-ssl = false
 # cassandra CA certficate path when using SSL
-cassandra-ca-path = /etc/raintank/ca.pem
+cassandra-ca-path = /etc/metrictank/ca.pem
 # host (hostname and server cert) verification when using SSL
 cassandra-host-verification = true
 # enable cassandra user authentication
@@ -253,7 +253,7 @@ update-fuzzyness = 0.5
 # enable SSL connection to cassandra
 ssl = false
 # cassandra CA certficate path when using SSL
-ca-path = /etc/raintank/ca.pem
+ca-path = /etc/metrictank/ca.pem
 # host (hostname and server cert) verification when using SSL
 host-verification = true
 # enable cassandra user authentication
diff --git a/docker/docker-dev/docker-compose.yml b/docker/docker-dev/docker-compose.yml
index 53b39b9843..ec03e0a765 100644
--- a/docker/docker-dev/docker-compose.yml
+++ b/docker/docker-dev/docker-compose.yml
@@ -9,9 +9,9 @@ services:
      - "2003:2003"
     volumes:
       - ../../build/metrictank:/usr/bin/metrictank
-      - ../../scripts/config/metrictank-docker.ini:/etc/raintank/metrictank.ini
-      - ../../scripts/config/storage-schemas.conf:/etc/raintank/storage-schemas.conf
-      - ../../scripts/config/storage-aggregation.conf:/etc/raintank/storage-aggregation.conf
+      - ../../scripts/config/metrictank-docker.ini:/etc/metrictank/metrictank.ini
+      - ../../scripts/config/storage-schemas.conf:/etc/metrictank/storage-schemas.conf
+      - ../../scripts/config/storage-aggregation.conf:/etc/metrictank/storage-aggregation.conf
     environment:
      WAIT_HOSTS: cassandra:9042
      WAIT_TIMEOUT: 60
diff --git a/docs/config.md b/docs/config.md
index a2416f0009..81a82ec293 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -82,7 +82,7 @@ cql-protocol-version = 4
 # enable SSL connection to cassandra
 cassandra-ssl = false
 # cassandra CA certficate path when using SSL
-cassandra-ca-path = /etc/raintank/ca.pem
+cassandra-ca-path = /etc/metrictank/ca.pem
 # host (hostname and server cert) verification when using SSL
 cassandra-host-verification = true
 # enable cassandra user authentication
@@ -298,7 +298,7 @@ update-fuzzyness = 0.5
 # enable SSL connection to cassandra
 ssl = false
 # cassandra CA certficate path when using SSL
-ca-path = /etc/raintank/ca.pem
+ca-path = /etc/metrictank/ca.pem
 # host (hostname and server cert) verification when using SSL
 host-verification = true
 # enable cassandra user authentication
diff --git a/docs/installation-deb.md b/docs/installation-deb.md
index e5a78dcc03..d0bb481605 100644
--- a/docs/installation-deb.md
+++ b/docs/installation-deb.md
@@ -235,14 +235,14 @@ The log - if you need it - lives at /opt/kafka/logs/server.log
 
 ## Configuration
 
-Now edit the file at `/etc/raintank/metrictank.ini`.  It should be commented enough to guide you through the various options.
+Now edit the file at `/etc/metrictank/metrictank.ini`.  It should be commented enough to guide you through the various options.
 
 You may have to adjust `statsd-addr`, `cassandra-addrs`, `cassandra-idx`'s `hosts` option and `kafka-mdm-in`'s `brokers` option if you run
 any of these services on different locations then the localhost defaults.
 
 Out of the box, one input is enabled: the [Carbon line input](https://github.com/raintank/metrictank/blob/master/docs/inputs.md#carbon)
 It uses a default storage-schemas to coalesce every incoming metric into 1 second resolution.  You may want to fine tune this for your needs
-at `/etc/raintank/storage-schemas.conf`. (or simply what you already use in a pre-existing Graphite install).
+at `/etc/metrictank/storage-schemas.conf`. (or simply what you already use in a pre-existing Graphite install).
 See the input plugin documentation referenced above for more details.
 
 If you want to use Kafka, you should enable the Kafka-mdm input plugin.
diff --git a/docs/installation-rpm.md b/docs/installation-rpm.md
index e053f33d71..a7c3f9ce02 100644
--- a/docs/installation-rpm.md
+++ b/docs/installation-rpm.md
@@ -235,14 +235,14 @@ The log - if you need it - lives at /opt/kafka/logs/server.log
 
 ## Configuration
 
-Now edit the file at `/etc/raintank/metrictank.ini`.  It should be commented enough to guide you through the various options.
+Now edit the file at `/etc/metrictank/metrictank.ini`.  It should be commented enough to guide you through the various options.
 
 You may have to adjust `statsd-addr`, `cassandra-addrs`, `cassandra-idx`'s `hosts` option and `kafka-mdm-in`'s `brokers` option if you run
 any of these services on different locations then the localhost defaults.
 
 Out of the box, one input is enabled: the [Carbon line input](https://github.com/raintank/metrictank/blob/master/docs/inputs.md#carbon)
 It uses a default storage-schemas to coalesce every incoming metric into 1 second resolution.  You may want to fine tune this for your needs
-at `/etc/raintank/storage-schemas.conf`. (or simply what you already use in a pre-existing Graphite install).
+at `/etc/metrictank/storage-schemas.conf`. (or simply what you already use in a pre-existing Graphite install).
 See the input plugin documentation referenced above for more details.
 
 If you want to use Kafka, you should enable the Kafka-mdm input plugin.
diff --git a/docs/installation-source.md b/docs/installation-source.md
index da9bfcdbcc..e95d6e28d7 100644
--- a/docs/installation-source.md
+++ b/docs/installation-source.md
@@ -25,7 +25,7 @@ You may want to make the `GOPATH` setting persistent, by putting that export lin
 go get github.com/raintank/metrictank
 ```
 
-Take the file from `go/src/github.com/raintank/metrictank/metrictank-sample.ini`, put it in `/etc/raintank/metrictank.ini` and make any changes.
+Take the file from `go/src/github.com/raintank/metrictank/metrictank-sample.ini`, put it in `/etc/metrictank/metrictank.ini` and make any changes.
 
 ## Run it!
 
diff --git a/docs/tools.md b/docs/tools.md
index 4fbd83ecb0..9fdc88c1ec 100644
--- a/docs/tools.md
+++ b/docs/tools.md
@@ -45,7 +45,7 @@ cass config flags:
   -auth
     	enable cassandra user authentication
   -ca-path string
-    	cassandra CA certficate path when using SSL (default "/etc/raintank/ca.pem")
+    	cassandra CA certficate path when using SSL (default "/etc/metrictank/ca.pem")
   -consistency string
     	write consistency (any|one|two|three|quorum|all|local_quorum|each_quorum|local_one (default "one")
   -enabled
@@ -151,7 +151,7 @@ Inspects what's flowing through kafka (in mdm format) and reports it to you
 Flags:
 
   -config string
-    	configuration file path (default "/etc/raintank/metrictank.ini")
+    	configuration file path (default "/etc/metrictank/metrictank.ini")
   -format string
     	template to render the data with (default "{{.Part}} {{.OrgId}} {{.Id}} {{.Name}} {{.Metric}} {{.Interval}} {{.Value}} {{.Time}} {{.Unit}} {{.Mtype}} {{.Tags}}")
 ```
@@ -200,7 +200,7 @@ Flags:
   -cassandra-auth
     	enable cassandra authentication
   -cassandra-ca-path string
-    	cassandra CA certificate path when using SSL (default "/etc/raintank/ca.pem")
+    	cassandra CA certificate path when using SSL (default "/etc/metrictank/ca.pem")
   -cassandra-consistency string
     	write consistency (any|one|two|three|quorum|all|local_quorum|each_quorum|local_one (default "one")
   -cassandra-host-selection-policy string
@@ -246,7 +246,7 @@ Flags:
   -cassandra-auth
     	enable cassandra authentication
   -cassandra-ca-path string
-    	cassandra CA certificate path when using SSL (default "/etc/raintank/ca.pem")
+    	cassandra CA certificate path when using SSL (default "/etc/metrictank/ca.pem")
   -cassandra-consistency string
     	write consistency (any|one|two|three|quorum|all|local_quorum|each_quorum|local_one (default "one")
   -cassandra-host-selection-policy string
@@ -270,7 +270,7 @@ Flags:
   -cassandra-username string
     	username for authentication (default "cassandra")
   -config string
-    	configuration file path (default "/etc/raintank/metrictank.ini")
+    	configuration file path (default "/etc/metrictank/metrictank.ini")
   -cql-protocol-version int
     	cql protocol version to use (default 4)
   -fix int
@@ -305,7 +305,7 @@ Flags:
   -cassandra-auth
     	enable cassandra authentication
   -cassandra-ca-path string
-    	cassandra CA certificate path when using SSL (default "/etc/raintank/ca.pem")
+    	cassandra CA certificate path when using SSL (default "/etc/metrictank/ca.pem")
   -cassandra-concurrency int
     	max number of concurrent reads to cassandra. (default 20)
   -cassandra-consistency string
diff --git a/idx/cassandra/cassandra.go b/idx/cassandra/cassandra.go
index b8d9392b05..180af2be36 100644
--- a/idx/cassandra/cassandra.go
+++ b/idx/cassandra/cassandra.go
@@ -98,7 +98,7 @@ func ConfigSetup() *flag.FlagSet {
 	casIdx.IntVar(&protoVer, "protocol-version", 4, "cql protocol version to use")
 
 	casIdx.BoolVar(&ssl, "ssl", false, "enable SSL connection to cassandra")
-	casIdx.StringVar(&capath, "ca-path", "/etc/raintank/ca.pem", "cassandra CA certficate path when using SSL")
+	casIdx.StringVar(&capath, "ca-path", "/etc/metrictank/ca.pem", "cassandra CA certficate path when using SSL")
 	casIdx.BoolVar(&hostverification, "host-verification", true, "host (hostname and server cert) verification when using SSL")
 
 	casIdx.BoolVar(&auth, "auth", false, "enable cassandra user authentication")
diff --git a/mdata/init.go b/mdata/init.go
index cbee2ec876..19944bd9c4 100644
--- a/mdata/init.go
+++ b/mdata/init.go
@@ -47,8 +47,8 @@ var (
 	Schemas      persister.WhisperSchemas
 	Aggregations persister.WhisperAggregation
 
-	schemasFile = "/etc/raintank/storage-schemas.conf"
-	aggFile     = "/etc/raintank/storage-aggregation.conf"
+	schemasFile = "/etc/metrictank/storage-schemas.conf"
+	aggFile     = "/etc/metrictank/storage-aggregation.conf"
 )
 
 func ConfigProcess() {
diff --git a/metrictank-sample.ini b/metrictank-sample.ini
index 19baebca4a..a7da998b39 100644
--- a/metrictank-sample.ini
+++ b/metrictank-sample.ini
@@ -60,7 +60,7 @@ cql-protocol-version = 4
 # enable SSL connection to cassandra
 cassandra-ssl = false
 # cassandra CA certficate path when using SSL
-cassandra-ca-path = /etc/raintank/ca.pem
+cassandra-ca-path = /etc/metrictank/ca.pem
 # host (hostname and server cert) verification when using SSL
 cassandra-host-verification = true
 # enable cassandra user authentication
@@ -256,7 +256,7 @@ update-fuzzyness = 0.5
 # enable SSL connection to cassandra
 ssl = false
 # cassandra CA certficate path when using SSL
-ca-path = /etc/raintank/ca.pem
+ca-path = /etc/metrictank/ca.pem
 # host (hostname and server cert) verification when using SSL
 host-verification = true
 # enable cassandra user authentication
diff --git a/metrictank.go b/metrictank.go
index bb6eaa640b..fa7b360ee5 100644
--- a/metrictank.go
+++ b/metrictank.go
@@ -50,7 +50,7 @@ var (
 	// Misc:
 	instance    = flag.String("instance", "default", "instance identifier. must be unique. used in clustering messages, for naming queue consumers and emitted metrics")
 	showVersion = flag.Bool("version", false, "print version string")
-	confFile    = flag.String("config", "/etc/raintank/metrictank.ini", "configuration file path")
+	confFile    = flag.String("config", "/etc/metrictank/metrictank.ini", "configuration file path")
 
 	accountingPeriodStr = flag.String("accounting-period", "5min", "accounting period to track per-org usage metrics")
 
@@ -75,7 +75,7 @@ var (
 	cqlProtocolVersion           = flag.Int("cql-protocol-version", 4, "cql protocol version to use")
 
 	cassandraSSL              = flag.Bool("cassandra-ssl", false, "enable SSL connection to cassandra")
-	cassandraCaPath           = flag.String("cassandra-ca-path", "/etc/raintank/ca.pem", "cassandra CA certificate path when using SSL")
+	cassandraCaPath           = flag.String("cassandra-ca-path", "/etc/metrictank/ca.pem", "cassandra CA certificate path when using SSL")
 	cassandraHostVerification = flag.Bool("cassandra-host-verification", true, "host (hostname and server cert) verification when using SSL")
 
 	cassandraAuth     = flag.Bool("cassandra-auth", false, "enable cassandra authentication")
diff --git a/scripts/Dockerfile b/scripts/Dockerfile
index 06f9aa54e7..421588110c 100644
--- a/scripts/Dockerfile
+++ b/scripts/Dockerfile
@@ -1,10 +1,10 @@
 FROM alpine
 MAINTAINER Anthony Woods awoods@raintank.io
 
-RUN mkdir -p /etc/raintank
-COPY config/metrictank-docker.ini /etc/raintank/metrictank.ini
-COPY config/storage-schemas.conf /etc/raintank/storage-schemas.conf
-COPY config/storage-aggregation.conf /etc/raintank/storage-aggregation.conf
+RUN mkdir -p /etc/metrictank
+COPY config/metrictank-docker.ini /etc/metrictank/metrictank.ini
+COPY config/storage-schemas.conf /etc/metrictank/storage-schemas.conf
+COPY config/storage-aggregation.conf /etc/metrictank/storage-aggregation.conf
 
 COPY build/* /usr/bin/
 
@@ -13,4 +13,4 @@ COPY wait_for_endpoint.sh /usr/bin/wait_for_endpoint.sh
 EXPOSE 6060
 
 ENTRYPOINT ["/usr/bin/wait_for_endpoint.sh"]
-CMD ["/usr/bin/metrictank", "-config=/etc/raintank/metrictank.ini"]
+CMD ["/usr/bin/metrictank", "-config=/etc/metrictank/metrictank.ini"]
diff --git a/scripts/config/metrictank-docker.ini b/scripts/config/metrictank-docker.ini
index a6939c7f97..6ea88f39f6 100644
--- a/scripts/config/metrictank-docker.ini
+++ b/scripts/config/metrictank-docker.ini
@@ -57,7 +57,7 @@ cql-protocol-version = 4
 # enable SSL connection to cassandra
 cassandra-ssl = false
 # cassandra CA certficate path when using SSL
-cassandra-ca-path = /etc/raintank/ca.pem
+cassandra-ca-path = /etc/metrictank/ca.pem
 # host (hostname and server cert) verification when using SSL
 cassandra-host-verification = true
 # enable cassandra user authentication
@@ -253,7 +253,7 @@ update-fuzzyness = 0.5
 # enable SSL connection to cassandra
 ssl = false
 # cassandra CA certficate path when using SSL
-ca-path = /etc/raintank/ca.pem
+ca-path = /etc/metrictank/ca.pem
 # host (hostname and server cert) verification when using SSL
 host-verification = true
 # enable cassandra user authentication
diff --git a/scripts/config/metrictank-package.ini b/scripts/config/metrictank-package.ini
index 7476f76300..5d3cb0fb5c 100644
--- a/scripts/config/metrictank-package.ini
+++ b/scripts/config/metrictank-package.ini
@@ -57,7 +57,7 @@ cql-protocol-version = 4
 # enable SSL connection to cassandra
 cassandra-ssl = false
 # cassandra CA certficate path when using SSL
-cassandra-ca-path = /etc/raintank/ca.pem
+cassandra-ca-path = /etc/metrictank/ca.pem
 # host (hostname and server cert) verification when using SSL
 cassandra-host-verification = true
 # enable cassandra user authentication
@@ -253,7 +253,7 @@ update-fuzzyness = 0.5
 # enable SSL connection to cassandra
 ssl = false
 # cassandra CA certficate path when using SSL
-ca-path = /etc/raintank/ca.pem
+ca-path = /etc/metrictank/ca.pem
 # host (hostname and server cert) verification when using SSL
 host-verification = true
 # enable cassandra user authentication
diff --git a/scripts/config/systemd/metrictank.service b/scripts/config/systemd/metrictank.service
index 95a1523f75..d523def627 100644
--- a/scripts/config/systemd/metrictank.service
+++ b/scripts/config/systemd/metrictank.service
@@ -10,7 +10,7 @@ Group=root
 Type=simple
 Restart=on-failure
 WorkingDirectory=/var/run/raintank
-ExecStart=/usr/sbin/metrictank -config=/etc/raintank/metrictank.ini
+ExecStart=/usr/sbin/metrictank -config=/etc/metrictank/metrictank.ini
 LimitNOFILE=102400
 TimeoutStopSec=60
 
diff --git a/scripts/config/sysvinit/init.d/metrictank b/scripts/config/sysvinit/init.d/metrictank
index 21c5808d6c..cbc9576564 100755
--- a/scripts/config/sysvinit/init.d/metrictank
+++ b/scripts/config/sysvinit/init.d/metrictank
@@ -20,7 +20,7 @@ export PATH
 
 name=metrictank
 program=/usr/sbin/metrictank
-args=-config\\\=/etc/raintank/metrictank.ini
+args=-config\\\=/etc/metrictank/metrictank.ini
 pidfile="/var/run/$name.pid"
 
 [ -r /etc/default/$name ] && . /etc/default/$name
diff --git a/scripts/config/upstart-0.6.5/metrictank.conf b/scripts/config/upstart-0.6.5/metrictank.conf
index 19c191e068..908953c871 100644
--- a/scripts/config/upstart-0.6.5/metrictank.conf
+++ b/scripts/config/upstart-0.6.5/metrictank.conf
@@ -22,4 +22,4 @@ limit nofile 102400 102400
 #limit stack <softlimit> <hardlimit>
 
 
-exec chroot --userspec root:root / /usr/sbin/metrictank "-config=/etc/raintank/metrictank.ini"
+exec chroot --userspec root:root / /usr/sbin/metrictank "-config=/etc/metrictank/metrictank.ini"
diff --git a/scripts/config/upstart/metrictank b/scripts/config/upstart/metrictank
index 1e9c932ecd..2595ab4724 100644
--- a/scripts/config/upstart/metrictank
+++ b/scripts/config/upstart/metrictank
@@ -26,4 +26,4 @@ setgid root
 console log # log stdout/stderr to /var/log/upstart/
 
 
-exec /usr/sbin/metrictank "-config=/etc/raintank/metrictank.ini"
+exec /usr/sbin/metrictank "-config=/etc/metrictank/metrictank.ini"
diff --git a/scripts/package.sh b/scripts/package.sh
index f8d8b5eaeb..bf30a8030e 100755
--- a/scripts/package.sh
+++ b/scripts/package.sh
@@ -13,11 +13,11 @@ VERSION=$(git describe --long --always)
 ## debian wheezy
 BUILD=${BUILD_ROOT}/sysvinit
 mkdir -p ${BUILD}/usr/sbin
-mkdir -p ${BUILD}/etc/raintank
+mkdir -p ${BUILD}/etc/metrictank
 
-cp ${BASE}/config/metrictank-package.ini ${BUILD}/etc/raintank/metrictank.ini
-cp ${BASE}/config/storage-schemas.conf ${BUILD}/etc/raintank/
-cp ${BASE}/config/storage-aggregation.conf ${BUILD}/etc/raintank/
+cp ${BASE}/config/metrictank-package.ini ${BUILD}/etc/metrictank/metrictank.ini
+cp ${BASE}/config/storage-schemas.conf ${BUILD}/etc/metrictank/
+cp ${BASE}/config/storage-aggregation.conf ${BUILD}/etc/metrictank/
 cp ${BUILD_ROOT}/{metrictank,mt-*} ${BUILD}/usr/sbin/
 
 PACKAGE_NAME="${BUILD}/metrictank-${VERSION}_${ARCH}.deb"
@@ -34,11 +34,11 @@ BUILD=${BUILD_ROOT}/upstart
 
 mkdir -p ${BUILD}/usr/sbin
 mkdir -p ${BUILD}/etc/init
-mkdir -p ${BUILD}/etc/raintank
+mkdir -p ${BUILD}/etc/metrictank
 
-cp ${BASE}/config/metrictank-package.ini ${BUILD}/etc/raintank/metrictank.ini
-cp ${BASE}/config/storage-schemas.conf ${BUILD}/etc/raintank/
-cp ${BASE}/config/storage-aggregation.conf ${BUILD}/etc/raintank/
+cp ${BASE}/config/metrictank-package.ini ${BUILD}/etc/metrictank/metrictank.ini
+cp ${BASE}/config/storage-schemas.conf ${BUILD}/etc/metrictank/
+cp ${BASE}/config/storage-aggregation.conf ${BUILD}/etc/metrictank/
 cp ${BUILD_ROOT}/metrictank ${BUILD}/usr/sbin/
 
 PACKAGE_NAME="${BUILD}/metrictank-${VERSION}_${ARCH}.deb"
@@ -54,19 +54,19 @@ BUILD=${BUILD_ROOT}/systemd
 
 mkdir -p ${BUILD}/usr/sbin
 mkdir -p ${BUILD}/lib/systemd/system/
-mkdir -p ${BUILD}/etc/raintank
+mkdir -p ${BUILD}/etc/metrictank
 mkdir -p ${BUILD}/var/run/raintank
 
-cp ${BASE}/config/metrictank-package.ini ${BUILD}/etc/raintank/metrictank.ini
-cp ${BASE}/config/storage-schemas.conf ${BUILD}/etc/raintank/
-cp ${BASE}/config/storage-aggregation.conf ${BUILD}/etc/raintank/
+cp ${BASE}/config/metrictank-package.ini ${BUILD}/etc/metrictank/metrictank.ini
+cp ${BASE}/config/storage-schemas.conf ${BUILD}/etc/metrictank/
+cp ${BASE}/config/storage-aggregation.conf ${BUILD}/etc/metrictank/
 cp ${BUILD_ROOT}/metrictank ${BUILD}/usr/sbin/
 cp ${BASE}/config/systemd/metrictank.service $BUILD/lib/systemd/system/
 
 PACKAGE_NAME="${BUILD}/metrictank-${VERSION}_${ARCH}.deb"
 fpm -s dir -t deb \
   -v ${VERSION} -n metrictank -a ${ARCH} --description "metrictank, the gorilla-inspired timeseries database backend for graphite" \
-  --config-files /etc/raintank/ \
+  --config-files /etc/metrictank/ \
   -m "Raintank Inc. <hello@raintank.io>" --vendor "raintank.io" \
   --license "Apache2.0" -C ${BUILD} -p ${PACKAGE_NAME} .
 
@@ -74,19 +74,19 @@ BUILD=${BUILD_ROOT}/systemd-centos7
 
 mkdir -p ${BUILD}/usr/sbin
 mkdir -p ${BUILD}/lib/systemd/system/
-mkdir -p ${BUILD}/etc/raintank
+mkdir -p ${BUILD}/etc/metrictank
 mkdir -p ${BUILD}/var/run/raintank
 
-cp ${BASE}/config/metrictank-package.ini ${BUILD}/etc/raintank/metrictank.ini
-cp ${BASE}/config/storage-schemas.conf ${BUILD}/etc/raintank/
-cp ${BASE}/config/storage-aggregation.conf ${BUILD}/etc/raintank/
+cp ${BASE}/config/metrictank-package.ini ${BUILD}/etc/metrictank/metrictank.ini
+cp ${BASE}/config/storage-schemas.conf ${BUILD}/etc/metrictank/
+cp ${BASE}/config/storage-aggregation.conf ${BUILD}/etc/metrictank/
 cp ${BUILD_ROOT}/metrictank ${BUILD}/usr/sbin/
 cp ${BASE}/config/systemd/metrictank.service $BUILD/lib/systemd/system/
 
 PACKAGE_NAME="${BUILD}/metrictank-${VERSION}.el7.${ARCH}.rpm"
 fpm -s dir -t rpm \
   -v ${VERSION} -n metrictank -a ${ARCH} --description "metrictank, the gorilla-inspired timeseries database backend for graphite" \
-  --config-files /etc/raintank/ \
+  --config-files /etc/metrictank/ \
   -m "Raintank Inc. <hello@raintank.io>" --vendor "raintank.io" \
   --license "Apache2.0" -C ${BUILD} -p ${PACKAGE_NAME} .
 
@@ -95,11 +95,11 @@ BUILD=${BUILD_ROOT}/upstart-0.6.5
 
 mkdir -p ${BUILD}/usr/sbin
 mkdir -p ${BUILD}/etc/init
-mkdir -p ${BUILD}/etc/raintank
+mkdir -p ${BUILD}/etc/metrictank
 
-cp ${BASE}/config/metrictank-package.ini ${BUILD}/etc/raintank/metrictank.ini
-cp ${BASE}/config/storage-schemas.conf ${BUILD}/etc/raintank/
-cp ${BASE}/config/storage-aggregation.conf ${BUILD}/etc/raintank/
+cp ${BASE}/config/metrictank-package.ini ${BUILD}/etc/metrictank/metrictank.ini
+cp ${BASE}/config/storage-schemas.conf ${BUILD}/etc/metrictank/
+cp ${BASE}/config/storage-aggregation.conf ${BUILD}/etc/metrictank/
 cp ${BUILD_ROOT}/metrictank ${BUILD}/usr/sbin/
 cp ${BASE}/config/upstart-0.6.5/metrictank.conf $BUILD/etc/init
 

From d451b5d63255c741c249f0e302fd3a242fd87830 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Fri, 3 Mar 2017 10:39:30 +0100
Subject: [PATCH 11/29] don't whine if build directory already exists

---
 scripts/build_docker.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh
index a6cecf2c12..080c761833 100755
--- a/scripts/build_docker.sh
+++ b/scripts/build_docker.sh
@@ -7,7 +7,7 @@ cd ${DIR}
 
 VERSION=`git describe --always`
 
-mkdir build
+mkdir -p build
 cp ../build/metrictank build/
 
 docker build -t raintank/metrictank .

From 6e364f97e1e55128cc2afc8bc07dad2b7dda6c36 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Fri, 3 Mar 2017 13:53:04 +0100
Subject: [PATCH 12/29] be more graphite-like and document it

---
 mdata/init.go                           | 44 +++++++++++++++++--------
 scripts/config/storage-aggregation.conf | 13 ++++++++
 scripts/config/storage-schemas.conf     | 43 ++++++++++++++++++++++++
 3 files changed, 87 insertions(+), 13 deletions(-)

diff --git a/mdata/init.go b/mdata/init.go
index 19944bd9c4..f29cbe8aeb 100644
--- a/mdata/init.go
+++ b/mdata/init.go
@@ -4,9 +4,12 @@
 package mdata
 
 import (
+	"io/ioutil"
 	"log"
+	"regexp"
 
 	"github.com/lomik/go-carbon/persister"
+	whisper "github.com/lomik/go-whisper"
 	"github.com/raintank/metrictank/stats"
 )
 
@@ -53,26 +56,41 @@ var (
 
 func ConfigProcess() {
 	var err error
+
+	// === read storage-schemas.conf ===
+
+	// graphite behavior: abort on any config reading errors, but skip any rules that have problems.
+	// at the end, add a default schema of 7 days of minutely data.
+	// we are stricter and don't tolerate any errors, that seems in the user's best interest.
+
 	Schemas, err = persister.ReadWhisperSchemas(schemasFile)
 	if err != nil {
 		log.Fatalf("can't read schemas file %q: %s", schemasFile, err.Error())
 	}
-	var defaultFound bool
-	for _, schema := range Schemas {
-		if schema.Pattern.String() == ".*" {
-			defaultFound = true
-		}
-		if len(schema.Retentions) == 0 {
-			log.Fatal(4, "retention setting cannot be empty")
-		}
-	}
-	if !defaultFound {
-		// good graphite health (not sure what graphite does if there's no .*)
-		// but we definitely need to always be able to determine which interval to use
-		log.Fatal(4, "storage-conf does not have a default '.*' pattern")
+	Schemas = append(Schemas, persister.Schema{
+		Name:       "default",
+		Pattern:    regexp.MustCompile(""),
+		Retentions: whisper.Retentions([]whisper.Retention{whisper.NewRetentionMT(60, 3600*24*7, 120*60, 2, true)}),
+	})
+
+	// === read storage-aggregation.conf ===
+
+	// graphite behavior:
+	// continue if file can't be read. (e.g. file is optional) but quit if other error reading config
+	// always add a default rule with xFilesFactor None and aggregationMethod None
+	// (which get interpreted by whisper as 0.5 and avg) at the end.
+
+	// since we can't distinguish errors reading vs parsing, we'll just try a read separately first
+	_, err = ioutil.ReadFile(aggFile)
+	if err != nil {
+		// this will add the default case we need
+		Aggregations = persister.NewWhisperAggregation()
+		return
 	}
+	// note: in this case, the library will include the default setting
 	Aggregations, err = persister.ReadWhisperAggregation(aggFile)
 	if err != nil {
 		log.Fatalf("can't read storage-aggregation file %q: %s", aggFile, err.Error())
 	}
+
 }
diff --git a/scripts/config/storage-aggregation.conf b/scripts/config/storage-aggregation.conf
index 251d252590..f3c0947509 100644
--- a/scripts/config/storage-aggregation.conf
+++ b/scripts/config/storage-aggregation.conf
@@ -1,3 +1,16 @@
+# This config file controls which summaries are created (using which consolidation functions) for your lower-precision archives, as defined in storage-schemas.conf
+# It is an extension of http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-aggregation-conf
+# Note:
+# * This file is optional. If it is not present, we will use avg for everything
+# * Anything not matched also uses avg for everything
+# * xFilesFactor is not honored yet.  What it is in graphite is a floating point number between 0 and 1 specifying what fraction of the previous retention level's slots must have non-null values in order to aggregate to a non-null value. The default is 0.5.
+# * aggregationMethod specifies the functions used to aggregate values for the next retention level. Legal methods are avg/average, sum, min, max, and last. The default is average.
+# Unlike Graphite, you can specify multiple, as it is often handy to have different summaries available depending on what analysis you need to do.
+# When using multiple, the first one is used for reading.  In the future, we will add capabilities to select the different archives for reading.
+# * the settings configured when metrictank starts are what is applied. So you can enable or disable archives by restarting metrictank.
+#
+# see https://github.com/raintank/metrictank/blob/master/docs/consolidation.md for related info.
+
 [default]
 pattern = .*
 xFilesFactor = 0.1
diff --git a/scripts/config/storage-schemas.conf b/scripts/config/storage-schemas.conf
index da6b38fa3c..9ea8e44e03 100644
--- a/scripts/config/storage-schemas.conf
+++ b/scripts/config/storage-schemas.conf
@@ -1,3 +1,46 @@
+# This config file sets up your retention rules.
+# It is an extension of http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-schemas-conf
+# Note:
+# * You can have 0 to N sections
+# * The first match wins, starting from the top. If no match found, we default to single archive of minutely points, retained for 7 days.
+# * The patterns are unanchored regular expressions, add '^' or '$' to match the beginning or end of a pattern.
+# * When running a cluster of metrictank instances, all instances should have the same agg-settings.
+# * Unlike whisper (graphite), the config doesn't stick: if you restart metrictank with updated settings, then those
+# will be applied. The configured rollups will be saved by primary nodes and served in responses if they are ready.
+# (note in particular that if you remove archives here, we will no longer read from them)
+# * Retentions must be specified in order of increasing interval and retention
+# 
+# A given rule is made up of 3 lines: the name, regex pattern and retentions.
+# The retentions line can specify multiple retention definitions. You need one or more, space separated.
+#
+# There are 2 formats:
+# 1) 'series-interval:count-of-datapoints' legacy and not easy to read
+# 2) 'series-interval:retention[:chunkspan:numchunks:ready]'
+#
+#Frequencies and histories are specified using the following suffixes:
+#
+#s - second
+#m - minute
+#h - hour
+#d - day
+#y - year
+#
+# The final 3 fields are specific to metrictank and if unspecified, use sane defaults.
+# See https://github.com/raintank/metrictank/blob/master/docs/memory-server.md for more details
+#
+# chunkspan: duration of chunks. e.g. 10min, 30min, 1h, 90min...
+# must be valid value as described here https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans
+# Defaults to a the smallest chunkspan that can hold at least 100 points.
+#
+# chunkspan: number of raw chunks to keep in in-memory ring buffer
+# See https://github.com/raintank/metrictank/blob/master/docs/memory-server.md for details and trade-offs, especially when compared to chunk-cache
+# which may be a more effective method to cache data and alleviate workload for cassandra.
+# Defaults to 2
+#
+# ready: whether the archive is ready for querying.  This is useful if you recently introduced a new archive, but it's still being populated
+# so you rather query other archives, even if they don't have the retention to serve your queries
+# Defaults to true
+
 [default]
 pattern = .*
 retentions = 1s:35d:10min:7

From 036e0995e1533a1268d180572ab6feaf98d6b1b3 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Fri, 3 Mar 2017 15:21:08 +0100
Subject: [PATCH 13/29] update config-to-doc.sh script to publish new config
 files

---
 docs/config.md           | 90 +++++++++++++++++++++++++++++++++++++---
 scripts/config-to-doc.sh | 88 ++++++++++++++++++++++++++-------------
 2 files changed, 145 insertions(+), 33 deletions(-)

diff --git a/docs/config.md b/docs/config.md
index 81a82ec293..d6e967dacb 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -1,9 +1,12 @@
 # Config
 
-Metrictank comes with an [example config file](https://github.com/raintank/metrictank/blob/master/metrictank-sample.ini)
-which is well documented, and for your convenience, it is replicated below.  
+Metrictank comes with an [example main config file](https://github.com/raintank/metrictank/blob/master/metrictank-sample.ini),
+a [storage-schemas.conf file](https://github.com/raintank/metrictank/blob/master/scripts/config/storage-schemas.conf) and
+a [storage-aggregations.conf file](https://github.com/raintank/metrictank/blob/master/scripts/config/storage-aggregation.conf)
 
-Config values can also be set, or overridden via environment variables.
+The files themselves are well documented, but for your convenience, they are replicated below.  
+
+Config values for the main ini config file can also be set, or overridden via environment variables.
 They require the 'MT_' prefix.  Any delimiter is represented as an underscore.
 Settings within section names in the config just require you to prefix the section header.
 
@@ -15,11 +18,12 @@ MT_CASSANDRA_WRITE_CONCURRENCY: 10        # MT_<setting_name>
 MT_KAFKA_MDM_IN_DATA_DIR: /your/data/dir  # MT_<section_title>_<setting_name>
 ```
 
-This file is generated by [config-to-doc](https://github.com/raintank/metrictank/blob/master/scripts/config-to-doc.sh)
-
 ---
 
 
+# Metrictank.ini
+
+
 sample config for metrictank  
 the defaults here match the default behavior.  
 ## misc ##
@@ -315,3 +319,79 @@ password = cassandra
 [memory-idx]
 enabled = false
 ```
+
+# storage-schemas.conf
+
+```
+# This config file sets up your retention rules.
+# It is an extension of http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-schemas-conf
+# Note:
+# * You can have 0 to N sections
+# * The first match wins, starting from the top. If no match found, we default to single archive of minutely points, retained for 7 days.
+# * The patterns are unanchored regular expressions, add '^' or '$' to match the beginning or end of a pattern.
+# * When running a cluster of metrictank instances, all instances should have the same agg-settings.
+# * Unlike whisper (graphite), the config doesn't stick: if you restart metrictank with updated settings, then those
+# will be applied. The configured rollups will be saved by primary nodes and served in responses if they are ready.
+# (note in particular that if you remove archives here, we will no longer read from them)
+# * Retentions must be specified in order of increasing interval and retention
+# 
+# A given rule is made up of 3 lines: the name, regex pattern and retentions.
+# The retentions line can specify multiple retention definitions. You need one or more, space separated.
+#
+# There are 2 formats:
+# 1) 'series-interval:count-of-datapoints' legacy and not easy to read
+# 2) 'series-interval:retention[:chunkspan:numchunks:ready]'
+#
+#Frequencies and histories are specified using the following suffixes:
+#
+#s - second
+#m - minute
+#h - hour
+#d - day
+#y - year
+#
+# The final 3 fields are specific to metrictank and if unspecified, use sane defaults.
+# See https://github.com/raintank/metrictank/blob/master/docs/memory-server.md for more details
+#
+# chunkspan: duration of chunks. e.g. 10min, 30min, 1h, 90min...
+# must be valid value as described here https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans
+# Defaults to a the smallest chunkspan that can hold at least 100 points.
+#
+# chunkspan: number of raw chunks to keep in in-memory ring buffer
+# See https://github.com/raintank/metrictank/blob/master/docs/memory-server.md for details and trade-offs, especially when compared to chunk-cache
+# which may be a more effective method to cache data and alleviate workload for cassandra.
+# Defaults to 2
+#
+# ready: whether the archive is ready for querying.  This is useful if you recently introduced a new archive, but it's still being populated
+# so you rather query other archives, even if they don't have the retention to serve your queries
+# Defaults to true
+
+[default]
+pattern = .*
+retentions = 1s:35d:10min:7
+```
+
+# storage-aggregation.conf
+
+```
+# This config file controls which summaries are created (using which consolidation functions) for your lower-precision archives, as defined in storage-schemas.conf
+# It is an extension of http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-aggregation-conf
+# Note:
+# * This file is optional. If it is not present, we will use avg for everything
+# * Anything not matched also uses avg for everything
+# * xFilesFactor is not honored yet.  What it is in graphite is a floating point number between 0 and 1 specifying what fraction of the previous retention level's slots must have non-null values in order to aggregate to a non-null value. The default is 0.5.
+# * aggregationMethod specifies the functions used to aggregate values for the next retention level. Legal methods are avg/average, sum, min, max, and last. The default is average.
+# Unlike Graphite, you can specify multiple, as it is often handy to have different summaries available depending on what analysis you need to do.
+# When using multiple, the first one is used for reading.  In the future, we will add capabilities to select the different archives for reading.
+# * the settings configured when metrictank starts are what is applied. So you can enable or disable archives by restarting metrictank.
+#
+# see https://github.com/raintank/metrictank/blob/master/docs/consolidation.md for related info.
+
+[default]
+pattern = .*
+xFilesFactor = 0.1
+aggregationMethod = avg,min,max
+```
+
+This file is generated by [config-to-doc](https://github.com/raintank/metrictank/blob/master/scripts/config-to-doc.sh)
+
diff --git a/scripts/config-to-doc.sh b/scripts/config-to-doc.sh
index 0fa621ec3f..5282a724a3 100755
--- a/scripts/config-to-doc.sh
+++ b/scripts/config-to-doc.sh
@@ -4,33 +4,8 @@
 # headers like h3 and h2 are printed as-is, and the lines between them (config items and their comments)
 # are wrapped in ``` blocks. t=code means we're in such a block.
 
-cat << EOF
-# Config
-
-Metrictank comes with an [example config file](https://github.com/raintank/metrictank/blob/master/metrictank-sample.ini)
-which is well documented, and for your convenience, it is replicated below.  
-
-Config values can also be set, or overridden via environment variables.
-They require the 'MT_' prefix.  Any delimiter is represented as an underscore.
-Settings within section names in the config just require you to prefix the section header.
-
-Examples:
-
-\`\`\`
-MT_LOG_LEVEL: 1                           # MT_<setting_name>
-MT_CASSANDRA_WRITE_CONCURRENCY: 10        # MT_<setting_name>
-MT_KAFKA_MDM_IN_DATA_DIR: /your/data/dir  # MT_<section_title>_<setting_name>
-\`\`\`
-
-This file is generated by [config-to-doc](https://github.com/raintank/metrictank/blob/master/scripts/config-to-doc.sh)
-
----
-
-
-EOF
-
-
-
+function process() {
+	local filename="$1"
 t=
 while read line; do
 	# skip empty lines
@@ -66,9 +41,66 @@ while read line; do
 			echo "$line"
 		fi
 	fi
-done < metrictank-sample.ini
+done < "$filename"
 
 # finish of pending code block
 if [ "$t" == code ]; then
 	echo '```'
 fi
+}
+
+cat << EOF
+# Config
+
+Metrictank comes with an [example main config file](https://github.com/raintank/metrictank/blob/master/metrictank-sample.ini),
+a [storage-schemas.conf file](https://github.com/raintank/metrictank/blob/master/scripts/config/storage-schemas.conf) and
+a [storage-aggregations.conf file](https://github.com/raintank/metrictank/blob/master/scripts/config/storage-aggregation.conf)
+
+The files themselves are well documented, but for your convenience, they are replicated below.  
+
+Config values for the main ini config file can also be set, or overridden via environment variables.
+They require the 'MT_' prefix.  Any delimiter is represented as an underscore.
+Settings within section names in the config just require you to prefix the section header.
+
+Examples:
+
+\`\`\`
+MT_LOG_LEVEL: 1                           # MT_<setting_name>
+MT_CASSANDRA_WRITE_CONCURRENCY: 10        # MT_<setting_name>
+MT_KAFKA_MDM_IN_DATA_DIR: /your/data/dir  # MT_<section_title>_<setting_name>
+\`\`\`
+
+---
+
+
+# Metrictank.ini
+
+
+EOF
+
+process metrictank-sample.ini
+
+cat << EOF
+
+# storage-schemas.conf
+
+\`\`\`
+EOF
+cat scripts/config/storage-schemas.conf
+
+cat << EOF
+\`\`\`
+
+# storage-aggregation.conf
+
+\`\`\`
+EOF
+
+cat scripts/config/storage-aggregation.conf
+
+cat << EOF
+\`\`\`
+
+This file is generated by [config-to-doc](https://github.com/raintank/metrictank/blob/master/scripts/config-to-doc.sh)
+
+EOF

From 0613804a2b28bd0edfdaaa40e10dbb2ac5f1e9a0 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Mon, 6 Mar 2017 23:25:26 +0100
Subject: [PATCH 14/29] better output

---
 api/models/request.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/api/models/request.go b/api/models/request.go
index 5dc007364a..8296b624c0 100644
--- a/api/models/request.go
+++ b/api/models/request.go
@@ -53,6 +53,6 @@ func (r Req) String() string {
 }
 
 func (r Req) DebugString() string {
-	return fmt.Sprintf("%s %d - %d . points <= %d. %s - archive %d, rawInt %d, archInt %d, outInt %d, aggNum %d",
-		r.Key, r.From, r.To, r.MaxPoints, r.Consolidator, r.Archive, r.RawInterval, r.ArchInterval, r.OutInterval, r.AggNum)
+	return fmt.Sprintf("Req key=%s %d - %d maxPoints=%d rawInt=%d cons=%s schemaI=%d aggI=%d archive=%d archInt=%d ttl=%d outInt=%d aggNum=%d",
+		r.Key, r.From, r.To, r.MaxPoints, r.RawInterval, r.Consolidator, r.SchemaI, r.AggI, r.Archive, r.ArchInterval, r.TTL, r.OutInterval, r.AggNum)
 }

From eaf10ad74f2e71d832fed783105ad5b72da85fc3 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Wed, 8 Mar 2017 11:59:18 +0100
Subject: [PATCH 15/29] make name a public property

useful for debugging
---
 .../lomik/go-carbon/persister/whisper.go           |  2 +-
 .../go-carbon/persister/whisper_aggregation.go     | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/vendor/github.com/lomik/go-carbon/persister/whisper.go b/vendor/github.com/lomik/go-carbon/persister/whisper.go
index 31262969a0..e32b33c79d 100644
--- a/vendor/github.com/lomik/go-carbon/persister/whisper.go
+++ b/vendor/github.com/lomik/go-carbon/persister/whisper.go
@@ -115,7 +115,7 @@ func store(p *Whisper, values *points.Points) {
 		logrus.WithFields(logrus.Fields{
 			"retention":    schema.RetentionStr,
 			"schema":       schema.Name,
-			"aggregation":  aggr.name,
+			"aggregation":  aggr.Name,
 			"xFilesFactor": aggr.XFilesFactor,
 			"method":       aggr.aggregationMethodStr,
 		}).Debugf("[persister] Creating %s", path)
diff --git a/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go b/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
index e2f1a5c8fb..8888139b71 100644
--- a/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
+++ b/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
@@ -16,7 +16,7 @@ import (
 )
 
 type WhisperAggregationItem struct {
-	name                 string
+	Name                 string
 	pattern              *regexp.Regexp
 	XFilesFactor         float64
 	aggregationMethodStr string
@@ -34,7 +34,7 @@ func NewWhisperAggregation() WhisperAggregation {
 	return WhisperAggregation{
 		Data: make([]WhisperAggregationItem, 0),
 		Default: WhisperAggregationItem{
-			name:                 "default",
+			Name:                 "default",
 			pattern:              nil,
 			XFilesFactor:         0.5,
 			aggregationMethodStr: "average",
@@ -61,23 +61,23 @@ func ReadWhisperAggregation(file string) (WhisperAggregation, error) {
 		item := WhisperAggregationItem{}
 		// this is mildly stupid, but I don't feel like forking
 		// configparser just for this
-		item.name =
+		item.Name =
 			strings.Trim(strings.SplitN(s.String(), "\n", 2)[0], " []")
-		if item.name == "" || strings.HasPrefix(item.name, "#") {
+		if item.Name == "" || strings.HasPrefix(item.Name, "#") {
 			continue
 		}
 
 		item.pattern, err = regexp.Compile(s.ValueOf("pattern"))
 		if err != nil {
 			logrus.Errorf("[persister] Failed to parse pattern '%s'for [%s]: %s",
-				s.ValueOf("pattern"), item.name, err.Error())
+				s.ValueOf("pattern"), item.Name, err.Error())
 			return WhisperAggregation{}, err
 		}
 
 		item.XFilesFactor, err = strconv.ParseFloat(s.ValueOf("xFilesFactor"), 64)
 		if err != nil {
 			logrus.Errorf("failed to parse xFilesFactor '%s' in %s: %s",
-				s.ValueOf("xFilesFactor"), item.name, err.Error())
+				s.ValueOf("xFilesFactor"), item.Name, err.Error())
 			return WhisperAggregation{}, err
 		}
 
@@ -102,7 +102,7 @@ func ReadWhisperAggregation(file string) (WhisperAggregation, error) {
 		}
 
 		logrus.Debugf("[persister] Adding aggregation [%s] pattern = %s aggregationMethod = %s xFilesFactor = %f",
-			item.name, s.ValueOf("pattern"),
+			item.Name, s.ValueOf("pattern"),
 			item.aggregationMethodStr, item.XFilesFactor)
 
 		result.Data = append(result.Data, item)

From 839ff60abdf22dc2a34727d0f260b75aeb56b53b Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Wed, 8 Mar 2017 11:59:45 +0100
Subject: [PATCH 16/29] fix : set aggregation/schema correctly

---
 idx/memory/memory.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/idx/memory/memory.go b/idx/memory/memory.go
index 9c560b3187..fd4723fb1a 100644
--- a/idx/memory/memory.go
+++ b/idx/memory/memory.go
@@ -277,6 +277,8 @@ func (m *MemoryIdx) Find(orgId int, pattern string, from int64) ([]idx.Node, err
 				Path:        n.Path,
 				Leaf:        n.Leaf(),
 				HasChildren: n.HasChildren(),
+				SchemaI:     n.SchemaI,
+				AggI:        n.AggI,
 			}
 			if idxNode.Leaf {
 				idxNode.Defs = make([]schema.MetricDefinition, 0, len(n.Defs))

From c10137a2c97539f2f2937ec7fbd762fa7bd49bb0 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Wed, 8 Mar 2017 13:29:22 +0100
Subject: [PATCH 17/29] support custom output formats

---
 cmd/mt-index-cat/main.go    | 23 ++++++++++++++---------
 cmd/mt-index-cat/out/out.go | 13 +++++++++++++
 2 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/cmd/mt-index-cat/main.go b/cmd/mt-index-cat/main.go
index 4f9c7bae9b..718c878380 100644
--- a/cmd/mt-index-cat/main.go
+++ b/cmd/mt-index-cat/main.go
@@ -59,7 +59,8 @@ func main() {
 		fmt.Printf("cass config flags:\n\n")
 		cassFlags.PrintDefaults()
 		fmt.Println()
-		fmt.Printf("output: %v\n\n\n", strings.Join(outputs, "|"))
+		fmt.Printf("output: either presets like %v\n", strings.Join(outputs, "|"))
+		fmt.Printf("output: or custom templates like '{{.Id}} {{.OrgId}} {{.Name}} {{.Metric}} {{.Interval}} {{.Unit}} {{.Mtype}} {{.Tags}} {{.LastUpdate}} {{.Partition}}'\n\n\n")
 		fmt.Println("EXAMPLES:")
 		fmt.Println("mt-index-cat -from 60min cass -hosts cassandra:9042 list")
 	}
@@ -74,16 +75,20 @@ func main() {
 		os.Exit(-1)
 	}
 
-	last := os.Args[len(os.Args)-1]
+	format := os.Args[len(os.Args)-1]
 	var found bool
-	for _, output := range outputs {
-		if last == output {
-			found = true
-			break
+	if strings.Contains(format, "{{") {
+		found = true
+	} else {
+		for _, output := range outputs {
+			if format == output {
+				found = true
+				break
+			}
 		}
 	}
 	if !found {
-		log.Printf("invalid output %q", last)
+		log.Printf("invalid output %q", format)
 		flag.Usage()
 		os.Exit(-1)
 	}
@@ -105,7 +110,7 @@ func main() {
 
 	var show func(d schema.MetricDefinition)
 
-	switch os.Args[len(os.Args)-1] {
+	switch format {
 	case "dump":
 		show = out.Dump
 	case "list":
@@ -115,7 +120,7 @@ func main() {
 	case "vegeta-render-patterns":
 		show = out.GetVegetaRenderPattern(addr, from)
 	default:
-		panic("this should never happen. we already validated the output type")
+		show = out.Template(format)
 	}
 
 	idx := cassandra.New()
diff --git a/cmd/mt-index-cat/out/out.go b/cmd/mt-index-cat/out/out.go
index d5919d5b1b..f23c126d4b 100644
--- a/cmd/mt-index-cat/out/out.go
+++ b/cmd/mt-index-cat/out/out.go
@@ -3,7 +3,9 @@ package out
 import (
 	"fmt"
 	"math/rand"
+	"os"
 	"strings"
+	"text/template"
 
 	"github.com/davecgh/go-spew/spew"
 
@@ -49,3 +51,14 @@ func pattern(in string) string {
 	// mode 3: do nothing :)
 	return in
 }
+
+func Template(format string) func(d schema.MetricDefinition) {
+	tpl := template.Must(template.New("format").Parse(format + "\n"))
+
+	return func(d schema.MetricDefinition) {
+		err := tpl.Execute(os.Stdout, d)
+		if err != nil {
+			panic(err)
+		}
+	}
+}

From a3ea153894c9b180cded8f7ac21d85d3888c1e2f Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Wed, 8 Mar 2017 15:44:10 +0100
Subject: [PATCH 18/29] clarify `from` parameter

---
 cmd/mt-index-cat/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/mt-index-cat/main.go b/cmd/mt-index-cat/main.go
index 718c878380..82245cb580 100644
--- a/cmd/mt-index-cat/main.go
+++ b/cmd/mt-index-cat/main.go
@@ -35,7 +35,7 @@ func main() {
 	globalFlags.StringVar(&addr, "addr", "http://localhost:6060", "graphite/metrictank address")
 	globalFlags.StringVar(&prefix, "prefix", "", "only show metrics that have this prefix")
 	globalFlags.StringVar(&substr, "substr", "", "only show metrics that have this substring")
-	globalFlags.StringVar(&from, "from", "30min", "from. eg '30min', '5h', '14d', etc. or a unix timestamp")
+	globalFlags.StringVar(&from, "from", "30min", "for vegeta outputs, will generate requests for data starting from now minus... eg '30min', '5h', '14d', etc. or a unix timestamp")
 	globalFlags.StringVar(&maxAge, "max-age", "6h30min", "max age (last update diff with now) of metricdefs.  use 0 to disable")
 	globalFlags.IntVar(&limit, "limit", 0, "only show this many metrics.  use 0 to disable")
 	globalFlags.BoolVar(&verbose, "verbose", false, "print stats to stderr")

From 164a9c3d5c32f19059d021fba5443319719c9dd5 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Wed, 8 Mar 2017 22:29:19 +0100
Subject: [PATCH 19/29] simplify a bit

---
 api/query_engine.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/api/query_engine.go b/api/query_engine.go
index 6e80f45df2..d1b9950aa6 100644
--- a/api/query_engine.go
+++ b/api/query_engine.go
@@ -98,9 +98,9 @@ func alignRequests(now uint32, reqs []models.Req) ([]models.Req, error) {
 				if interval == archInterval && ret.Ready {
 					// we're in luck. this will be more efficient than runtime consolidation
 					req.Archive = req.Archive + 1 + i
-					req.ArchInterval = uint32(ret.SecondsPerPoint())
+					req.ArchInterval = archInterval
 					req.TTL = uint32(ret.MaxRetention())
-					req.OutInterval = req.ArchInterval
+					req.OutInterval = archInterval
 					req.AggNum = 1
 					break
 				}

From 7cd7aeadc3f47a8200758f0a68b0815b844aed01 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Thu, 9 Mar 2017 10:44:43 +0100
Subject: [PATCH 20/29] back schemas and aggs matching with a cache

before this, all regex matching dominated the cpu profile.
With this, cpu usage reduced by easily 5x
Though we still have:
      flat                     cumulative
70ms  0.86% 69.79%      770ms  9.49%
github.com/raintank/metrictank/mdata/matchcache.(*Cache).Get

due to the map locking

We could further optimize this, probably, by changing the
idx.AddOrUpdate signature to returning SchemaI and AggI, instead
of requiring it as input as @replay suggested.
This way we only have to match if it wasn't in the index already.
However this requires more intensive changes to the index than
I'm comfortable with right now (DefById only has the metricdef, not the
properties, we could add them but then we need to adjust how we work
with DefById everywhere and do we still need to store the properties in the tree, etc)
I rather re-address this when the need is clearer and we have time to
give this the attention it deserves.
---
 input/input_test.go            |  1 +
 mdata/init.go                  |  9 +++++
 mdata/matchcache/matchcache.go | 63 ++++++++++++++++++++++++++++++++++
 mdata/schema.go                | 13 +++++--
 4 files changed, 83 insertions(+), 3 deletions(-)
 create mode 100644 mdata/matchcache/matchcache.go

diff --git a/input/input_test.go b/input/input_test.go
index e973fbf16b..298914504a 100644
--- a/input/input_test.go
+++ b/input/input_test.go
@@ -21,6 +21,7 @@ func BenchmarkProcess(b *testing.B) {
 
 	mdata.SetSingleSchema(whisper.NewRetentionMT(10, 10000, 600, 10, true))
 	mdata.SetOnlyDefaultAgg(whisper.Average, whisper.Min, whisper.Max)
+	mdata.Cache(time.Minute, time.Hour)
 
 	aggmetrics := mdata.NewAggMetrics(store, &cache.MockCache{}, 800, 8000, 0)
 	metricIndex := memory.New()
diff --git a/mdata/init.go b/mdata/init.go
index f29cbe8aeb..461ed61994 100644
--- a/mdata/init.go
+++ b/mdata/init.go
@@ -7,9 +7,11 @@ import (
 	"io/ioutil"
 	"log"
 	"regexp"
+	"time"
 
 	"github.com/lomik/go-carbon/persister"
 	whisper "github.com/lomik/go-whisper"
+	"github.com/raintank/metrictank/mdata/matchcache"
 	"github.com/raintank/metrictank/stats"
 )
 
@@ -49,6 +51,8 @@ var (
 	// set either via ConfigProcess or from the unit tests. other code should not touch
 	Schemas      persister.WhisperSchemas
 	Aggregations persister.WhisperAggregation
+	schemasCache *matchcache.Cache
+	aggsCache    *matchcache.Cache
 
 	schemasFile = "/etc/metrictank/storage-schemas.conf"
 	aggFile     = "/etc/metrictank/storage-aggregation.conf"
@@ -92,5 +96,10 @@ func ConfigProcess() {
 	if err != nil {
 		log.Fatalf("can't read storage-aggregation file %q: %s", aggFile, err.Error())
 	}
+	Cache(time.Hour, time.Hour)
+}
 
+func Cache(cleanInterval, expireAfter time.Duration) {
+	schemasCache = matchcache.New(cleanInterval, expireAfter)
+	aggsCache = matchcache.New(cleanInterval, expireAfter)
 }
diff --git a/mdata/matchcache/matchcache.go b/mdata/matchcache/matchcache.go
new file mode 100644
index 0000000000..3805d32a23
--- /dev/null
+++ b/mdata/matchcache/matchcache.go
@@ -0,0 +1,63 @@
+package matchcache
+
+import (
+	"sync"
+	"time"
+)
+
+// Cache caches key to uint16 lookups (for schemas and aggregations)
+// when it cleans the cache it locks up the entire cache
+// this is a tradeoff we can make for simplicity, since this sits on the ingestion
+// path, where occasional stalls are ok.
+type Cache struct {
+	sync.Mutex
+	data map[string]Item
+
+	cleanInterval time.Duration
+	expireAfter   time.Duration
+}
+
+type Item struct {
+	val  uint16
+	seen int64
+}
+
+func New(cleanInterval, expireAfter time.Duration) *Cache {
+	m := &Cache{
+		data:          make(map[string]Item),
+		cleanInterval: cleanInterval,
+		expireAfter:   expireAfter,
+	}
+	go m.maintain()
+	return m
+}
+
+type AddFunc func(key string) uint16
+
+// if not in cache, will atomically add it using the provided function
+func (m *Cache) Get(key string, fn AddFunc) uint16 {
+	m.Lock()
+	item, ok := m.data[key]
+	if !ok {
+		item.val = fn(key)
+	}
+	item.seen = time.Now().Unix()
+	m.data[key] = item
+	m.Unlock()
+	return item.val
+}
+
+func (m *Cache) maintain() {
+	ticker := time.NewTicker(m.cleanInterval)
+	diff := int64(m.expireAfter.Seconds())
+	for now := range ticker.C {
+		nowUnix := now.Unix()
+		m.Lock()
+		for key, item := range m.data {
+			if nowUnix-item.seen > diff {
+				delete(m.data, key)
+			}
+		}
+		m.Unlock()
+	}
+}
diff --git a/mdata/schema.go b/mdata/schema.go
index b8ce0461ec..3f28a42a0e 100644
--- a/mdata/schema.go
+++ b/mdata/schema.go
@@ -11,14 +11,21 @@ import (
 // MatchSchema returns the schema for the given metric key, and the index of the schema (to efficiently reference it)
 // it will always find the schema because we made sure there is a catchall '.*' pattern
 func MatchSchema(key string) (uint16, persister.Schema) {
-	i, schema, _ := Schemas.Match(key)
-	return i, schema
+	i := schemasCache.Get(key, func(key string) uint16 {
+		i, _, _ := Schemas.Match(key)
+		return i
+	})
+	return i, Schemas[i]
 }
 
 // MatchAgg returns the aggregation definition for the given metric key, and the index of it (to efficiently reference it)
 // i may be 1 more than the last defined by user, in which case it's the default.
 func MatchAgg(key string) (uint16, persister.WhisperAggregationItem) {
-	return Aggregations.Match(key)
+	i := aggsCache.Get(key, func(key string) uint16 {
+		i, _ := Aggregations.Match(key)
+		return i
+	})
+	return i, GetAgg(i)
 }
 
 // caller must assure i is valid

From 747578926a48e883a4831ba3cf844a191f2e4718 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Thu, 9 Mar 2017 11:36:33 +0100
Subject: [PATCH 21/29] split up input Processing tests to compare same metrics
 vs new ones

on my laptop:

MASTER:
BenchmarkProcess-8     2000000         718 ns/op
PASS
ok    github.com/raintank/metrictank/input  2.796s

HEAD:
go test -run='^$' -bench=.
BenchmarkProcessUniqueMetrics-8      1000000        2388 ns/op
BenchmarkProcessSameMetric-8         2000000         885 ns/op
PASS
ok    github.com/raintank/metrictank/input  6.081s

So we're a bit slower but carbon input should be a lot faster
(for which we don't have benchmarks) since it used to do regex matching
all the time
---
 input/input_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 45 insertions(+), 3 deletions(-)

diff --git a/input/input_test.go b/input/input_test.go
index 298914504a..3bd157f5f8 100644
--- a/input/input_test.go
+++ b/input/input_test.go
@@ -1,6 +1,7 @@
 package input
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
@@ -14,7 +15,7 @@ import (
 	"gopkg.in/raintank/schema.v1"
 )
 
-func BenchmarkProcess(b *testing.B) {
+func BenchmarkProcessUniqueMetrics(b *testing.B) {
 	cluster.Init("default", "test", time.Now(), "http", 6060)
 
 	store := mdata.NewDevnullStore()
@@ -32,11 +33,52 @@ func BenchmarkProcess(b *testing.B) {
 	// timestamps start at 10 and go up from there. (we can't use 0, see AggMetric.Add())
 	datas := make([]*schema.MetricData, b.N)
 	for i := 0; i < b.N; i++ {
+		name := fmt.Sprintf("fake.metric.%d", i)
 		metric := &schema.MetricData{
 			Id:       "some.id.of.a.metric",
 			OrgId:    500,
-			Name:     "some.id",
-			Metric:   "metric",
+			Name:     name,
+			Metric:   name,
+			Interval: 10,
+			Value:    1234.567,
+			Unit:     "ms",
+			Time:     int64((i + 1) * 10),
+			Mtype:    "gauge",
+			Tags:     []string{"some_tag", "ok"},
+		}
+		datas[i] = metric
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		in.Process(datas[i], 1)
+	}
+}
+
+func BenchmarkProcessSameMetric(b *testing.B) {
+	cluster.Init("default", "test", time.Now(), "http", 6060)
+
+	store := mdata.NewDevnullStore()
+
+	mdata.SetSingleSchema(whisper.NewRetentionMT(10, 10000, 600, 10, true))
+	mdata.SetOnlyDefaultAgg(whisper.Average, whisper.Min, whisper.Max)
+	mdata.Cache(time.Minute, time.Hour)
+
+	aggmetrics := mdata.NewAggMetrics(store, &cache.MockCache{}, 800, 8000, 0)
+	metricIndex := memory.New()
+	metricIndex.Init()
+	usage := usage.New(300, aggmetrics, metricIndex, clock.New())
+	in := NewDefaultHandler(aggmetrics, metricIndex, usage, "BenchmarkProcess")
+
+	// timestamps start at 10 and go up from there. (we can't use 0, see AggMetric.Add())
+	datas := make([]*schema.MetricData, b.N)
+	for i := 0; i < b.N; i++ {
+		name := "fake.metric.same"
+		metric := &schema.MetricData{
+			Id:       "some.id.of.a.metric",
+			OrgId:    500,
+			Name:     name,
+			Metric:   name,
 			Interval: 10,
 			Value:    1234.567,
 			Unit:     "ms",

From 800b43046bfa5ee2295631b0191d05922042a7db Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Thu, 9 Mar 2017 14:58:51 +0100
Subject: [PATCH 22/29] replace the already-quite-patched lomik libraries with
 our own version

much cleaner code if I may say so.
also: consistently apply a simpler "add default to catch all remaining
metrics" default mechanism.
---
 api/dataprocessor_test.go                     |    6 +-
 api/graphite.go                               |    2 +-
 api/query_engine.go                           |    8 +-
 api/query_engine_test.go                      |  165 ++-
 conf/aggregations.go                          |  112 ++
 conf/init.go                                  |   11 +
 conf/method.go                                |   11 +
 conf/retention.go                             |  173 +++
 conf/schemas.go                               |  107 ++
 input/carbon/carbon.go                        |    2 +-
 input/input_test.go                           |   10 +-
 mdata/aggmetric.go                            |    5 +-
 mdata/aggmetric_test.go                       |   30 +-
 mdata/aggmetrics.go                           |    4 +-
 mdata/aggregator.go                           |   29 +-
 mdata/aggregator_test.go                      |    9 +-
 mdata/init.go                                 |   31 +-
 mdata/schema.go                               |   85 +-
 usage/usage_test.go                           |   13 -
 .../hydrogen18/stalecucumber/LICENSE          |   10 -
 .../hydrogen18/stalecucumber/README.md        |   65 --
 .../hydrogen18/stalecucumber/fuzz.go          |   14 -
 .../hydrogen18/stalecucumber/helpers.go       |  201 ----
 .../hydrogen18/stalecucumber/jump_list.go     |   22 -
 .../hydrogen18/stalecucumber/opcodes.go       |   55 -
 .../stalecucumber/pickle_machine.go           |  598 ----------
 .../hydrogen18/stalecucumber/pickle_writer.go |  514 --------
 .../stalecucumber/populate_jump_list.go       |   58 -
 .../hydrogen18/stalecucumber/protocol_0.go    |  723 ------------
 .../hydrogen18/stalecucumber/protocol_1.go    |  523 ---------
 .../hydrogen18/stalecucumber/protocol_2.go    |  322 -----
 .../hydrogen18/stalecucumber/python_types.go  |   22 -
 .../hydrogen18/stalecucumber/tuple.go         |    7 -
 .../hydrogen18/stalecucumber/unpack.go        |  329 ------
 vendor/github.com/lomik/go-carbon/LICENSE.md  |   21 -
 .../lomik/go-carbon/helper/stoppable.go       |   87 --
 .../lomik/go-carbon/persister/whisper.go      |  346 ------
 .../persister/whisper_aggregation.go          |  122 --
 .../go-carbon/persister/whisper_schema.go     |  228 ----
 .../lomik/go-carbon/points/points.go          |  204 ----
 .../github.com/lomik/go-whisper/LICENCE.txt   |   28 -
 vendor/github.com/lomik/go-whisper/README.md  |   46 -
 vendor/github.com/lomik/go-whisper/whisper.go | 1038 -----------------
 .../lomik/go-whisper/whisper_test.py          |   46 -
 vendor/vendor.json                            |   30 -
 45 files changed, 589 insertions(+), 5883 deletions(-)
 create mode 100644 conf/aggregations.go
 create mode 100644 conf/init.go
 create mode 100644 conf/method.go
 create mode 100644 conf/retention.go
 create mode 100644 conf/schemas.go
 delete mode 100644 vendor/github.com/hydrogen18/stalecucumber/LICENSE
 delete mode 100644 vendor/github.com/hydrogen18/stalecucumber/README.md
 delete mode 100644 vendor/github.com/hydrogen18/stalecucumber/fuzz.go
 delete mode 100644 vendor/github.com/hydrogen18/stalecucumber/helpers.go
 delete mode 100644 vendor/github.com/hydrogen18/stalecucumber/jump_list.go
 delete mode 100644 vendor/github.com/hydrogen18/stalecucumber/opcodes.go
 delete mode 100644 vendor/github.com/hydrogen18/stalecucumber/pickle_machine.go
 delete mode 100644 vendor/github.com/hydrogen18/stalecucumber/pickle_writer.go
 delete mode 100644 vendor/github.com/hydrogen18/stalecucumber/populate_jump_list.go
 delete mode 100644 vendor/github.com/hydrogen18/stalecucumber/protocol_0.go
 delete mode 100644 vendor/github.com/hydrogen18/stalecucumber/protocol_1.go
 delete mode 100644 vendor/github.com/hydrogen18/stalecucumber/protocol_2.go
 delete mode 100644 vendor/github.com/hydrogen18/stalecucumber/python_types.go
 delete mode 100644 vendor/github.com/hydrogen18/stalecucumber/tuple.go
 delete mode 100644 vendor/github.com/hydrogen18/stalecucumber/unpack.go
 delete mode 100644 vendor/github.com/lomik/go-carbon/LICENSE.md
 delete mode 100644 vendor/github.com/lomik/go-carbon/helper/stoppable.go
 delete mode 100644 vendor/github.com/lomik/go-carbon/persister/whisper.go
 delete mode 100644 vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
 delete mode 100644 vendor/github.com/lomik/go-carbon/persister/whisper_schema.go
 delete mode 100644 vendor/github.com/lomik/go-carbon/points/points.go
 delete mode 100644 vendor/github.com/lomik/go-whisper/LICENCE.txt
 delete mode 100644 vendor/github.com/lomik/go-whisper/README.md
 delete mode 100644 vendor/github.com/lomik/go-whisper/whisper.go
 delete mode 100644 vendor/github.com/lomik/go-whisper/whisper_test.py

diff --git a/api/dataprocessor_test.go b/api/dataprocessor_test.go
index 6a9341e500..2c77daff81 100644
--- a/api/dataprocessor_test.go
+++ b/api/dataprocessor_test.go
@@ -8,9 +8,9 @@ import (
 	"testing"
 	"time"
 
-	whisper "github.com/lomik/go-whisper"
 	"github.com/raintank/metrictank/api/models"
 	"github.com/raintank/metrictank/cluster"
+	"github.com/raintank/metrictank/conf"
 	"github.com/raintank/metrictank/consolidation"
 	"github.com/raintank/metrictank/mdata"
 	"github.com/raintank/metrictank/mdata/cache"
@@ -568,8 +568,8 @@ func TestGetSeriesFixed(t *testing.T) {
 	cluster.Init("default", "test", time.Now(), "http", 6060)
 	store := mdata.NewDevnullStore()
 
-	mdata.SetOnlyDefaultAgg(whisper.Average, whisper.Min, whisper.Max)
-	mdata.SetSingleSchema(whisper.NewRetentionMT(10, 100, 600, 10, true))
+	mdata.SetSingleAgg(conf.Avg, conf.Min, conf.Max)
+	mdata.SetSingleSchema(conf.NewRetentionMT(10, 100, 600, 10, true))
 
 	metrics := mdata.NewAggMetrics(store, &cache.MockCache{}, 0, 0, 0)
 	srv, _ := NewServer()
diff --git a/api/graphite.go b/api/graphite.go
index 0047c2b7c1..24c9c25ffe 100644
--- a/api/graphite.go
+++ b/api/graphite.go
@@ -255,7 +255,7 @@ func (s *Server) renderMetrics(ctx *middleware.Context, request models.GraphiteR
 			def := locdef.def
 			// set consolidator that will be used to normalize raw data before feeding into processing functions
 			// not to be confused with runtime consolidation which happens in the graphite api, after all processing.
-			fn := mdata.GetAgg(locdef.AggI).AggregationMethod[0]
+			fn := mdata.Aggregations.Get(locdef.AggI).AggregationMethod[0]
 			consolidator := consolidation.Consolidator(fn) // we use the same number assignments so we can cast them
 			// target is like foo.bar or foo.* or consolidateBy(foo.*,'sum')
 			// pattern is like foo.bar or foo.*
diff --git a/api/query_engine.go b/api/query_engine.go
index d1b9950aa6..cdb74b481e 100644
--- a/api/query_engine.go
+++ b/api/query_engine.go
@@ -40,7 +40,7 @@ func alignRequests(now uint32, reqs []models.Req) ([]models.Req, error) {
 
 	for i := range reqs {
 		req := &reqs[i]
-		retentions := mdata.GetRetentions(req.SchemaI)
+		retentions := mdata.Schemas.Get(req.SchemaI).Retentions
 		req.Archive = -1
 
 		for i, ret := range retentions {
@@ -51,7 +51,7 @@ func alignRequests(now uint32, reqs []models.Req) ([]models.Req, error) {
 
 			req.Archive = i
 			req.TTL = uint32(ret.MaxRetention())
-			req.ArchInterval = uint32(ret.SecondsPerPoint())
+			req.ArchInterval = uint32(ret.SecondsPerPoint)
 
 			if now-req.TTL <= req.From {
 				break
@@ -92,9 +92,9 @@ func alignRequests(now uint32, reqs []models.Req) ([]models.Req, error) {
 			// we have to deliver an interval higher than what we originally came up with
 
 			// let's see first if we can deliver it via lower-res rollup archives, if we have any
-			retentions := mdata.GetRetentions(req.SchemaI)
+			retentions := mdata.Schemas.Get(req.SchemaI).Retentions
 			for i, ret := range retentions[req.Archive+1:] {
-				archInterval := uint32(ret.SecondsPerPoint())
+				archInterval := uint32(ret.SecondsPerPoint)
 				if interval == archInterval && ret.Ready {
 					// we're in luck. this will be more efficient than runtime consolidation
 					req.Archive = req.Archive + 1 + i
diff --git a/api/query_engine_test.go b/api/query_engine_test.go
index 962e6c936d..429e717f91 100644
--- a/api/query_engine_test.go
+++ b/api/query_engine_test.go
@@ -4,24 +4,23 @@ import (
 	"regexp"
 	"testing"
 
-	"github.com/lomik/go-carbon/persister"
-	whisper "github.com/lomik/go-whisper"
 	"github.com/raintank/metrictank/api/models"
+	"github.com/raintank/metrictank/conf"
 	"github.com/raintank/metrictank/consolidation"
 	"github.com/raintank/metrictank/mdata"
 )
 
 // testAlign verifies the aligment of the given requests, given the retentions (one or more patterns, one or more retentions each)
-func testAlign(reqs []models.Req, retentions [][]whisper.Retention, outReqs []models.Req, outErr error, now uint32, t *testing.T) {
-	var schemas []persister.Schema
+func testAlign(reqs []models.Req, retentions [][]conf.Retention, outReqs []models.Req, outErr error, now uint32, t *testing.T) {
+	var schemas []conf.Schema
 	for _, ret := range retentions {
-		schemas = append(schemas, persister.Schema{
+		schemas = append(schemas, conf.Schema{
 			Pattern:    regexp.MustCompile(".*"),
-			Retentions: whisper.Retentions(ret),
+			Retentions: conf.Retentions(ret),
 		})
 	}
 
-	mdata.Schemas = persister.WhisperSchemas(schemas)
+	mdata.Schemas = conf.Schemas(schemas)
 	out, err := alignRequests(now, reqs)
 	if err != outErr {
 		t.Errorf("different err value expected: %v, got: %v", outErr, err)
@@ -43,9 +42,9 @@ func TestAlignRequestsBasic(t *testing.T) {
 		reqRaw("a", 0, 30, 800, 60, consolidation.Avg, 0, 0),
 		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 0, 0),
 	},
-		[][]whisper.Retention{
+		[][]conf.Retention{
 			{
-				whisper.NewRetentionMT(60, 1200, 0, 0, true),
+				conf.NewRetentionMT(60, 1200, 0, 0, true),
 			},
 		},
 		[]models.Req{
@@ -64,12 +63,12 @@ func TestAlignRequestsBasicDiff(t *testing.T) {
 		reqRaw("a", 0, 30, 800, 60, consolidation.Avg, 0, 0),
 		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
 	},
-		[][]whisper.Retention{
+		[][]conf.Retention{
 			{
-				whisper.NewRetentionMT(60, 1200, 0, 0, true),
+				conf.NewRetentionMT(60, 1200, 0, 0, true),
 			},
 			{
-				whisper.NewRetentionMT(60, 1200, 0, 0, true),
+				conf.NewRetentionMT(60, 1200, 0, 0, true),
 			},
 		},
 		[]models.Req{
@@ -89,10 +88,10 @@ func TestAlignRequestsAlerting(t *testing.T) {
 		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
 		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
 	},
-		[][]whisper.Retention{{
-			whisper.NewRetentionMT(10, 1200, 0, 0, true),
+		[][]conf.Retention{{
+			conf.NewRetentionMT(10, 1200, 0, 0, true),
 		}, {
-			whisper.NewRetentionMT(60, 1200, 0, 0, true),
+			conf.NewRetentionMT(60, 1200, 0, 0, true),
 		},
 		},
 		[]models.Req{
@@ -111,11 +110,11 @@ func TestAlignRequestsBasicBestEffort(t *testing.T) {
 		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
 		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
 	},
-		[][]whisper.Retention{
+		[][]conf.Retention{
 			{
-				whisper.NewRetentionMT(10, 800, 0, 0, true),
+				conf.NewRetentionMT(10, 800, 0, 0, true),
 			}, {
-				whisper.NewRetentionMT(60, 1100, 0, 0, true),
+				conf.NewRetentionMT(60, 1100, 0, 0, true),
 			},
 		},
 		[]models.Req{
@@ -134,12 +133,12 @@ func TestAlignRequestsHalfGood(t *testing.T) {
 		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
 		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
 	},
-		[][]whisper.Retention{
+		[][]conf.Retention{
 			{
-				whisper.NewRetentionMT(10, 800, 0, 0, true),
+				conf.NewRetentionMT(10, 800, 0, 0, true),
 			}, {
-				whisper.NewRetentionMT(60, 1100, 0, 0, true),
-				whisper.NewRetentionMT(120, 1200, 0, 0, true),
+				conf.NewRetentionMT(60, 1100, 0, 0, true),
+				conf.NewRetentionMT(120, 1200, 0, 0, true),
 			},
 		},
 		[]models.Req{
@@ -158,14 +157,14 @@ func TestAlignRequestsGoodRollup(t *testing.T) {
 		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
 		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
 	},
-		[][]whisper.Retention{
+		[][]conf.Retention{
 			{
-				whisper.NewRetentionMT(10, 1199, 0, 0, true), // just not long enough
-				whisper.NewRetentionMT(120, 1200, 600, 2, true),
+				conf.NewRetentionMT(10, 1199, 0, 0, true), // just not long enough
+				conf.NewRetentionMT(120, 1200, 600, 2, true),
 			},
 			{
-				whisper.NewRetentionMT(60, 1199, 0, 0, true), // just not long enough
-				whisper.NewRetentionMT(120, 1200, 600, 2, true),
+				conf.NewRetentionMT(60, 1199, 0, 0, true), // just not long enough
+				conf.NewRetentionMT(120, 1200, 600, 2, true),
 			},
 		},
 		[]models.Req{
@@ -184,14 +183,14 @@ func TestAlignRequestsDiffGoodRollup(t *testing.T) {
 		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
 		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
 	},
-		[][]whisper.Retention{
+		[][]conf.Retention{
 			{
-				whisper.NewRetentionMT(10, 1199, 0, 0, true), // just not long enough
-				whisper.NewRetentionMT(100, 1200, 600, 2, true),
+				conf.NewRetentionMT(10, 1199, 0, 0, true), // just not long enough
+				conf.NewRetentionMT(100, 1200, 600, 2, true),
 			},
 			{
-				whisper.NewRetentionMT(60, 1199, 0, 0, true), // just not long enough
-				whisper.NewRetentionMT(600, 1200, 600, 2, true),
+				conf.NewRetentionMT(60, 1199, 0, 0, true), // just not long enough
+				conf.NewRetentionMT(600, 1200, 600, 2, true),
 			},
 		},
 		[]models.Req{
@@ -210,13 +209,13 @@ func TestAlignRequestsWeird(t *testing.T) {
 		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
 		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
 	},
-		[][]whisper.Retention{
+		[][]conf.Retention{
 			{
-				whisper.NewRetentionMT(10, 1199, 0, 0, true),
-				whisper.NewRetentionMT(60, 1200, 600, 2, true),
+				conf.NewRetentionMT(10, 1199, 0, 0, true),
+				conf.NewRetentionMT(60, 1200, 600, 2, true),
 			},
 			{
-				whisper.NewRetentionMT(60, 1200, 0, 0, true),
+				conf.NewRetentionMT(60, 1200, 0, 0, true),
 			},
 		},
 		[]models.Req{
@@ -235,14 +234,14 @@ func TestAlignRequestsWeird2(t *testing.T) {
 		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
 		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
 	},
-		[][]whisper.Retention{
+		[][]conf.Retention{
 			{
-				whisper.NewRetentionMT(10, 1100, 0, 0, true), // just not long enough
-				whisper.NewRetentionMT(120, 1200, 600, 2, true),
+				conf.NewRetentionMT(10, 1100, 0, 0, true), // just not long enough
+				conf.NewRetentionMT(120, 1200, 600, 2, true),
 			},
 			{
-				whisper.NewRetentionMT(60, 1100, 0, 0, true), // just not long enough
-				whisper.NewRetentionMT(120, 1200, 600, 2, true),
+				conf.NewRetentionMT(60, 1100, 0, 0, true), // just not long enough
+				conf.NewRetentionMT(120, 1200, 600, 2, true),
 			},
 		},
 		[]models.Req{
@@ -261,14 +260,14 @@ func TestAlignRequestsNoOtherChoice(t *testing.T) {
 		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
 		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
 	},
-		[][]whisper.Retention{
+		[][]conf.Retention{
 			{
-				whisper.NewRetentionMT(10, 1100, 0, 0, true),
-				whisper.NewRetentionMT(120, 1199, 600, 2, true),
+				conf.NewRetentionMT(10, 1100, 0, 0, true),
+				conf.NewRetentionMT(120, 1199, 600, 2, true),
 			},
 			{
-				whisper.NewRetentionMT(60, 1100, 0, 0, true),
-				whisper.NewRetentionMT(120, 1199, 600, 2, true),
+				conf.NewRetentionMT(60, 1100, 0, 0, true),
+				conf.NewRetentionMT(120, 1199, 600, 2, true),
 			},
 		},
 		[]models.Req{
@@ -287,15 +286,15 @@ func TestAlignRequests3rdBand(t *testing.T) {
 		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
 		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
 	},
-		[][]whisper.Retention{
+		[][]conf.Retention{
 			{
-				whisper.NewRetentionMT(1, 1100, 0, 0, true),
-				whisper.NewRetentionMT(120, 1199, 600, 2, true),
-				whisper.NewRetentionMT(240, 1200, 600, 2, true),
+				conf.NewRetentionMT(1, 1100, 0, 0, true),
+				conf.NewRetentionMT(120, 1199, 600, 2, true),
+				conf.NewRetentionMT(240, 1200, 600, 2, true),
 			},
 			{
-				whisper.NewRetentionMT(60, 1100, 0, 0, true),
-				whisper.NewRetentionMT(240, 1200, 600, 2, true),
+				conf.NewRetentionMT(60, 1100, 0, 0, true),
+				conf.NewRetentionMT(240, 1200, 600, 2, true),
 			},
 		},
 		[]models.Req{
@@ -314,15 +313,15 @@ func TestAlignRequests2RollupsDisabled(t *testing.T) {
 		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
 		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
 	},
-		[][]whisper.Retention{
+		[][]conf.Retention{
 			{
-				whisper.NewRetentionMT(10, 1100, 0, 0, true), // just not long enough
-				whisper.NewRetentionMT(120, 1199, 600, 2, false),
-				whisper.NewRetentionMT(240, 1200, 600, 2, false),
+				conf.NewRetentionMT(10, 1100, 0, 0, true), // just not long enough
+				conf.NewRetentionMT(120, 1199, 600, 2, false),
+				conf.NewRetentionMT(240, 1200, 600, 2, false),
 			},
 			{
-				whisper.NewRetentionMT(60, 1100, 0, 0, true), // just not long enough
-				whisper.NewRetentionMT(240, 1200, 600, 2, false),
+				conf.NewRetentionMT(60, 1100, 0, 0, true), // just not long enough
+				conf.NewRetentionMT(240, 1200, 600, 2, false),
 			},
 		},
 		[]models.Req{
@@ -339,15 +338,15 @@ func TestAlignRequestsHuh(t *testing.T) {
 		reqRaw("a", 0, 30, 800, 10, consolidation.Avg, 0, 0),
 		reqRaw("b", 0, 30, 800, 60, consolidation.Avg, 1, 0),
 	},
-		[][]whisper.Retention{
+		[][]conf.Retention{
 			{
-				whisper.NewRetentionMT(1, 1000, 0, 0, true),
-				whisper.NewRetentionMT(120, 1080, 600, 2, true),
-				whisper.NewRetentionMT(240, 1200, 600, 2, false),
+				conf.NewRetentionMT(1, 1000, 0, 0, true),
+				conf.NewRetentionMT(120, 1080, 600, 2, true),
+				conf.NewRetentionMT(240, 1200, 600, 2, false),
 			},
 			{
-				whisper.NewRetentionMT(60, 1100, 0, 0, true),
-				whisper.NewRetentionMT(240, 1200, 600, 2, false),
+				conf.NewRetentionMT(60, 1100, 0, 0, true),
+				conf.NewRetentionMT(240, 1200, 600, 2, false),
 			},
 		},
 		[]models.Req{
@@ -369,35 +368,35 @@ func BenchmarkAlignRequests(b *testing.B) {
 		reqRaw("b", 0, 3600*24*7, 1000, 30, consolidation.Avg, 1, 0),
 		reqRaw("c", 0, 3600*24*7, 1000, 60, consolidation.Avg, 2, 0),
 	}
-	mdata.Schemas = persister.WhisperSchemas([]persister.Schema{
+	mdata.Schemas = conf.Schemas([]conf.Schema{
 		{
 			Pattern: regexp.MustCompile("a"),
-			Retentions: whisper.Retentions(
-				[]whisper.Retention{
-					whisper.NewRetentionMT(10, 35*24*3600, 0, 0, true),
-					whisper.NewRetentionMT(600, 60*24*3600, 0, 0, true),
-					whisper.NewRetentionMT(7200, 180*24*3600, 0, 0, true),
-					whisper.NewRetentionMT(21600, 2*365*24*3600, 0, 0, true),
+			Retentions: conf.Retentions(
+				[]conf.Retention{
+					conf.NewRetentionMT(10, 35*24*3600, 0, 0, true),
+					conf.NewRetentionMT(600, 60*24*3600, 0, 0, true),
+					conf.NewRetentionMT(7200, 180*24*3600, 0, 0, true),
+					conf.NewRetentionMT(21600, 2*365*24*3600, 0, 0, true),
 				}),
 		},
 		{
 			Pattern: regexp.MustCompile("b"),
-			Retentions: whisper.Retentions(
-				[]whisper.Retention{
-					whisper.NewRetentionMT(30, 35*24*3600, 0, 0, true),
-					whisper.NewRetentionMT(600, 60*24*3600, 0, 0, true),
-					whisper.NewRetentionMT(7200, 180*24*3600, 0, 0, true),
-					whisper.NewRetentionMT(21600, 2*365*24*3600, 0, 0, true),
+			Retentions: conf.Retentions(
+				[]conf.Retention{
+					conf.NewRetentionMT(30, 35*24*3600, 0, 0, true),
+					conf.NewRetentionMT(600, 60*24*3600, 0, 0, true),
+					conf.NewRetentionMT(7200, 180*24*3600, 0, 0, true),
+					conf.NewRetentionMT(21600, 2*365*24*3600, 0, 0, true),
 				}),
 		},
 		{
 			Pattern: regexp.MustCompile(".*"),
-			Retentions: whisper.Retentions(
-				[]whisper.Retention{
-					whisper.NewRetentionMT(60, 35*24*3600, 0, 0, true),
-					whisper.NewRetentionMT(600, 60*24*3600, 0, 0, true),
-					whisper.NewRetentionMT(7200, 180*24*3600, 0, 0, true),
-					whisper.NewRetentionMT(21600, 2*365*24*3600, 0, 0, true),
+			Retentions: conf.Retentions(
+				[]conf.Retention{
+					conf.NewRetentionMT(60, 35*24*3600, 0, 0, true),
+					conf.NewRetentionMT(600, 60*24*3600, 0, 0, true),
+					conf.NewRetentionMT(7200, 180*24*3600, 0, 0, true),
+					conf.NewRetentionMT(21600, 2*365*24*3600, 0, 0, true),
 				}),
 		},
 	})
diff --git a/conf/aggregations.go b/conf/aggregations.go
new file mode 100644
index 0000000000..4f891e63b7
--- /dev/null
+++ b/conf/aggregations.go
@@ -0,0 +1,112 @@
+package conf
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/alyu/configparser"
+)
+
+var DefaultAggregation = Aggregation{
+	Name:              "default",
+	Pattern:           nil,
+	XFilesFactor:      0.5,
+	AggregationMethod: []Method{Avg},
+}
+
+// Aggregations holds the aggregation definitions
+type Aggregations struct {
+	Data []Aggregation
+}
+
+type Aggregation struct {
+	Name              string
+	Pattern           *regexp.Regexp
+	XFilesFactor      float64
+	AggregationMethod []Method
+}
+
+// NewAggregations create instance of Aggregations
+func NewAggregations() Aggregations {
+	return Aggregations{
+		Data: make([]Aggregation, 0),
+	}
+}
+
+// ReadAggregations returns the defined aggregations from a storage-aggregations.conf file
+// and adds the default
+func ReadAggregations(file string) (Aggregations, error) {
+	config, err := configparser.Read(file)
+	if err != nil {
+		return Aggregations{}, err
+	}
+	sections, err := config.AllSections()
+	if err != nil {
+		return Aggregations{}, err
+	}
+
+	result := NewAggregations()
+
+	for _, s := range sections {
+		item := Aggregation{}
+		item.Name = strings.Trim(strings.SplitN(s.String(), "\n", 2)[0], " []")
+		if item.Name == "" || strings.HasPrefix(item.Name, "#") {
+			continue
+		}
+
+		item.Pattern, err = regexp.Compile(s.ValueOf("pattern"))
+		if err != nil {
+			return Aggregations{}, fmt.Errorf("[%s]: failed to parse pattern %q: %s", item.Name, s.ValueOf("pattern"), err.Error())
+		}
+
+		item.XFilesFactor, err = strconv.ParseFloat(s.ValueOf("xFilesFactor"), 64)
+		if err != nil {
+			return Aggregations{}, fmt.Errorf("[%s]: failed to parse xFilesFactor %q: %s", item.Name, s.ValueOf("xFilesFactor"), err.Error())
+		}
+
+		aggregationMethodStr := s.ValueOf("aggregationMethod")
+		methodStrs := strings.Split(aggregationMethodStr, ",")
+		for _, methodStr := range methodStrs {
+			switch methodStr {
+			case "average", "avg":
+				item.AggregationMethod = append(item.AggregationMethod, Avg)
+			case "sum":
+				item.AggregationMethod = append(item.AggregationMethod, Sum)
+			case "last":
+				item.AggregationMethod = append(item.AggregationMethod, Lst)
+			case "max":
+				item.AggregationMethod = append(item.AggregationMethod, Max)
+			case "min":
+				item.AggregationMethod = append(item.AggregationMethod, Min)
+			default:
+				return result, fmt.Errorf("[%s]: unknown aggregation method %q", item.Name, methodStr)
+			}
+		}
+
+		result.Data = append(result.Data, item)
+	}
+
+	return result, nil
+}
+
+// Match returns the correct aggregation setting for the given metric
+// it can always find a valid setting, because there's a default catch all
+// also returns the index of the setting, to efficiently reference it
+func (a Aggregations) Match(metric string) (uint16, Aggregation) {
+	for i, s := range a.Data {
+		if s.Pattern.MatchString(metric) {
+			return uint16(i), s
+		}
+	}
+	return uint16(len(a.Data)), DefaultAggregation
+}
+
+// Get returns the aggregation setting corresponding to the given index
+func (a Aggregations) Get(i uint16) Aggregation {
+	if i+1 > uint16(len(a.Data)) {
+		return DefaultAggregation
+	}
+	return a.Data[i]
+}
diff --git a/conf/init.go b/conf/init.go
new file mode 100644
index 0000000000..2d04c94dc0
--- /dev/null
+++ b/conf/init.go
@@ -0,0 +1,11 @@
+// Package conf reads config data from two of carbon's config files
+// * storage-schemas.conf (old and new retention format)
+// see https://graphite.readthedocs.io/en/0.9.9/config-carbon.html#storage-schemas-conf
+// * storage-aggregation.conf
+// see http://graphite.readthedocs.io/en/latest/config-carbon.html#storage-aggregation-conf
+//
+// it also adds defaults (the same ones as graphite),
+// so that even if nothing is matched in the user provided schemas or aggregations,
+// a setting is *always* found
+// uses some modified snippets from github.com/lomik/go-carbon and github.com/lomik/go-whisper
+package conf
diff --git a/conf/method.go b/conf/method.go
new file mode 100644
index 0000000000..90a08093fa
--- /dev/null
+++ b/conf/method.go
@@ -0,0 +1,11 @@
+package conf
+
+type Method int
+
+const (
+	Avg Method = iota + 1
+	Sum
+	Lst
+	Max
+	Min
+)
diff --git a/conf/retention.go b/conf/retention.go
new file mode 100644
index 0000000000..450d9a342c
--- /dev/null
+++ b/conf/retention.go
@@ -0,0 +1,173 @@
+package conf
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/raintank/dur"
+	"github.com/raintank/metrictank/mdata/chunk"
+)
+
+const Month_sec = 60 * 60 * 24 * 28
+
+type Retentions []Retention
+
+// Validate assures the retentions are sane.  As the whisper source code says:
+// An ArchiveList must:
+// 1. Have at least one archive config. Example: (60, 86400)
+// 2. No archive may be a duplicate of another.
+// 3. Higher precision archives' precision must evenly divide all lower precision archives' precision.
+// 4. Lower precision archives must cover larger time intervals than higher precision archives.
+// 5. Each archive must have at least enough points to consolidate to the next archive
+func (rets Retentions) Validate() error {
+	if len(rets) == 0 {
+		return fmt.Errorf("No retentions")
+	}
+	for i := 1; i < len(rets); i++ {
+		prev := rets[i-1]
+		ret := rets[i]
+
+		if prev.SecondsPerPoint >= ret.SecondsPerPoint {
+			return fmt.Errorf("retention must have lower resolution than prior retention")
+		}
+
+		if ret.SecondsPerPoint%prev.SecondsPerPoint != 0 {
+			return fmt.Errorf("lower resolution retentions must be evenly divisible by higher resolution retentions (%d does not divide by %d)", ret.SecondsPerPoint, prev.SecondsPerPoint)
+		}
+
+		if prev.MaxRetention() >= ret.MaxRetention() {
+			return fmt.Errorf("lower resolution archives must have longer retention than higher resolution archives")
+		}
+
+		if prev.NumberOfPoints < (ret.SecondsPerPoint / prev.SecondsPerPoint) {
+			return fmt.Errorf("Each archive must have at least enough points to consolidate to the next archive (archive%v consolidates %v of archive%v's points but it has only %v total points)", i, ret.SecondsPerPoint/prev.SecondsPerPoint, i-1, prev.NumberOfPoints)
+		}
+	}
+	return nil
+}
+
+/*
+  A retention level.
+
+  Retention levels describe a given archive in the database. How detailed it is and how far back
+  it records.
+*/
+type Retention struct {
+	SecondsPerPoint int    // interval in seconds
+	NumberOfPoints  int    // ~ttl
+	ChunkSpan       uint32 // duration of chunk of aggregated metric for storage, controls how many aggregated points go into 1 chunk
+	NumChunks       uint32 // number of chunks to keep in memory. remember, for a query from now until 3 months ago, we will end up querying the memory server as well.
+	Ready           bool   // ready for reads?
+}
+
+func (r Retention) MaxRetention() int {
+	return r.SecondsPerPoint * r.NumberOfPoints
+}
+
+func NewRetention(secondsPerPoint, numberOfPoints int) Retention {
+	return Retention{
+		SecondsPerPoint: secondsPerPoint,
+		NumberOfPoints:  numberOfPoints,
+	}
+}
+
+func NewRetentionMT(secondsPerPoint int, ttl, chunkSpan, numChunks uint32, ready bool) Retention {
+	return Retention{
+		SecondsPerPoint: secondsPerPoint,
+		NumberOfPoints:  int(ttl) / secondsPerPoint,
+		ChunkSpan:       chunkSpan,
+		NumChunks:       numChunks,
+		Ready:           ready,
+	}
+}
+
+// ParseRetentions parses retention definitions into a Retentions structure
+func ParseRetentions(defs string) (Retentions, error) {
+	retentions := make(Retentions, 0)
+	for _, def := range strings.Split(defs, ",") {
+		def = strings.TrimSpace(def)
+		parts := strings.Split(def, ":")
+		if len(parts) < 2 || len(parts) > 5 {
+			return nil, fmt.Errorf("bad retentions spec %q", def)
+		}
+
+		// try old format
+		val1, err1 := strconv.ParseInt(parts[0], 10, 0)
+		val2, err2 := strconv.ParseInt(parts[1], 10, 0)
+
+		var retention Retention
+		var err error
+		if err1 == nil && err2 == nil {
+			retention = NewRetention(int(val1), int(val2))
+		} else {
+			// try new format
+			retention, err = ParseRetentionNew(def)
+			if err != nil {
+				return nil, err
+			}
+		}
+		if len(parts) >= 3 {
+			retention.ChunkSpan, err = dur.ParseUNsec(parts[2])
+			if err != nil {
+				return nil, err
+			}
+			if (Month_sec % retention.ChunkSpan) != 0 {
+				return nil, errors.New("chunkSpan must fit without remainders into month_sec (28*24*60*60)")
+			}
+			_, ok := chunk.RevChunkSpans[retention.ChunkSpan]
+			if !ok {
+				return nil, fmt.Errorf("chunkSpan %s is not a valid value (https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans)", parts[2])
+			}
+		} else {
+			// default to a valid chunkspan that can hold at least 100 points, or select the largest one otherwise.
+			approxSpan := uint32(retention.SecondsPerPoint * 100)
+			var span uint32
+			for _, span = range chunk.ChunkSpans {
+				if span >= approxSpan {
+					break
+				}
+			}
+			retention.ChunkSpan = span
+		}
+		retention.NumChunks = 2
+		if len(parts) >= 4 {
+			i, err := strconv.Atoi(parts[3])
+			if err != nil {
+				return nil, err
+			}
+			retention.NumChunks = uint32(i)
+		}
+		retention.Ready = true
+		if len(parts) == 5 {
+			retention.Ready, err = strconv.ParseBool(parts[4])
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		retentions = append(retentions, retention)
+	}
+	return retentions, retentions.Validate()
+}
+
+func ParseRetentionNew(def string) (Retention, error) {
+	parts := strings.Split(def, ":")
+	if len(parts) < 2 {
+		return Retention{}, fmt.Errorf("Not enough parts in retentionDef %q", def)
+	}
+	interval, err := dur.ParseUsec(parts[0])
+	if err != nil {
+		return Retention{}, fmt.Errorf("Failed to parse interval in %q: %s", def, err)
+	}
+
+	ttl, err := dur.ParseUsec(parts[1])
+	if err != nil {
+		return Retention{}, fmt.Errorf("Failed to parse TTL in %q: %s", def, err)
+	}
+
+	return Retention{
+		SecondsPerPoint: int(interval),
+		NumberOfPoints:  int(ttl / interval)}, err
+}
diff --git a/conf/schemas.go b/conf/schemas.go
new file mode 100644
index 0000000000..2d27f14b5a
--- /dev/null
+++ b/conf/schemas.go
@@ -0,0 +1,107 @@
+package conf
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/alyu/configparser"
+)
+
+var DefaultSchema = Schema{
+	Name:       "default",
+	Pattern:    nil,
+	Retentions: Retentions([]Retention{NewRetentionMT(60, 3600*24*7, 120*60, 2, true)}),
+}
+
+// Schemas contains schema settings
+type Schemas []Schema
+
+func (s Schemas) Len() int           { return len(s) }
+func (s Schemas) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+func (s Schemas) Less(i, j int) bool { return s[i].Priority >= s[j].Priority }
+
+// Schema represents one schema setting
+type Schema struct {
+	Name       string
+	Pattern    *regexp.Regexp
+	Retentions Retentions
+	Priority   int64
+}
+
+func NewSchemas() []Schema {
+	return nil
+}
+
+// ReadSchemas reads and parses a storage-schemas.conf file and returns a sorted schemas structure
+// see https://graphite.readthedocs.io/en/0.9.9/config-carbon.html#storage-schemas-conf
+func ReadSchemas(file string) (Schemas, error) {
+	config, err := configparser.Read(file)
+	if err != nil {
+		return nil, err
+	}
+
+	sections, err := config.AllSections()
+	if err != nil {
+		return nil, err
+	}
+
+	var schemas Schemas
+
+	for i, sec := range sections {
+		schema := Schema{}
+		schema.Name = strings.Trim(strings.SplitN(sec.String(), "\n", 2)[0], " []")
+		if schema.Name == "" || strings.HasPrefix(schema.Name, "#") {
+			continue
+		}
+
+		if sec.ValueOf("pattern") == "" {
+			return nil, fmt.Errorf("[%s]: empty pattern", schema.Name)
+		}
+		schema.Pattern, err = regexp.Compile(sec.ValueOf("pattern"))
+		if err != nil {
+			return nil, fmt.Errorf("[%s]: failed to parse pattern %q: %s", schema.Name, sec.ValueOf("pattern"), err.Error())
+		}
+
+		schema.Retentions, err = ParseRetentions(sec.ValueOf("retentions"))
+		if err != nil {
+			return nil, fmt.Errorf("[%s]: failed to parse retentions %q: %s", schema.Name, sec.ValueOf("retentions"), err.Error())
+		}
+
+		p := int64(0)
+		if sec.ValueOf("priority") != "" {
+			p, err = strconv.ParseInt(sec.ValueOf("priority"), 10, 0)
+			if err != nil {
+				return nil, fmt.Errorf("Failed to parse priority %q for [%s]: %s", sec.ValueOf("priority"), schema.Name, err)
+			}
+		}
+		schema.Priority = int64(p)<<32 - int64(i) // to sort records with same priority by position in file
+
+		schemas = append(schemas, schema)
+	}
+
+	sort.Sort(schemas)
+	return schemas, nil
+}
+
+// Match returns the correct schema setting for the given metric
+// it can always find a valid setting, because there's a default catch all
+// also returns the index of the setting, to efficiently reference it
+func (s Schemas) Match(metric string) (uint16, Schema) {
+	for i, schema := range s {
+		if schema.Pattern.MatchString(metric) {
+			return uint16(i), schema
+		}
+	}
+	return uint16(len(s)), DefaultSchema
+}
+
+// Get returns the schema setting corresponding to the given index
+func (s Schemas) Get(i uint16) Schema {
+	if i+1 > uint16(len(s)) {
+		return DefaultSchema
+	}
+	return s[i]
+}
diff --git a/input/carbon/carbon.go b/input/carbon/carbon.go
index 45e69e561f..3961c597a9 100644
--- a/input/carbon/carbon.go
+++ b/input/carbon/carbon.go
@@ -173,7 +173,7 @@ func (c *Carbon) handle(conn net.Conn) {
 		}
 		name := string(key)
 		_, s := mdata.MatchSchema(name) // note: also called by metrictank DefaultHandler.Process. maybe can be optimized
-		interval := s.Retentions[0].SecondsPerPoint()
+		interval := s.Retentions[0].SecondsPerPoint
 		md := &schema.MetricData{
 			Name:     name,
 			Metric:   name,
diff --git a/input/input_test.go b/input/input_test.go
index 3bd157f5f8..43cfb12d61 100644
--- a/input/input_test.go
+++ b/input/input_test.go
@@ -6,8 +6,8 @@ import (
 	"time"
 
 	"github.com/benbjohnson/clock"
-	whisper "github.com/lomik/go-whisper"
 	"github.com/raintank/metrictank/cluster"
+	"github.com/raintank/metrictank/conf"
 	"github.com/raintank/metrictank/idx/memory"
 	"github.com/raintank/metrictank/mdata"
 	"github.com/raintank/metrictank/mdata/cache"
@@ -20,8 +20,8 @@ func BenchmarkProcessUniqueMetrics(b *testing.B) {
 
 	store := mdata.NewDevnullStore()
 
-	mdata.SetSingleSchema(whisper.NewRetentionMT(10, 10000, 600, 10, true))
-	mdata.SetOnlyDefaultAgg(whisper.Average, whisper.Min, whisper.Max)
+	mdata.SetSingleSchema(conf.NewRetentionMT(10, 10000, 600, 10, true))
+	mdata.SetSingleAgg(conf.Avg, conf.Min, conf.Max)
 	mdata.Cache(time.Minute, time.Hour)
 
 	aggmetrics := mdata.NewAggMetrics(store, &cache.MockCache{}, 800, 8000, 0)
@@ -60,8 +60,8 @@ func BenchmarkProcessSameMetric(b *testing.B) {
 
 	store := mdata.NewDevnullStore()
 
-	mdata.SetSingleSchema(whisper.NewRetentionMT(10, 10000, 600, 10, true))
-	mdata.SetOnlyDefaultAgg(whisper.Average, whisper.Min, whisper.Max)
+	mdata.SetSingleSchema(conf.NewRetentionMT(10, 10000, 600, 10, true))
+	mdata.SetSingleAgg(conf.Avg, conf.Min, conf.Max)
 	mdata.Cache(time.Minute, time.Hour)
 
 	aggmetrics := mdata.NewAggMetrics(store, &cache.MockCache{}, 800, 8000, 0)
diff --git a/mdata/aggmetric.go b/mdata/aggmetric.go
index 68faf4774e..bbea922394 100644
--- a/mdata/aggmetric.go
+++ b/mdata/aggmetric.go
@@ -6,9 +6,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/lomik/go-carbon/persister"
-	whisper "github.com/lomik/go-whisper"
 	"github.com/raintank/metrictank/cluster"
+	"github.com/raintank/metrictank/conf"
 	"github.com/raintank/metrictank/consolidation"
 	"github.com/raintank/metrictank/mdata/cache"
 	"github.com/raintank/metrictank/mdata/chunk"
@@ -43,7 +42,7 @@ type AggMetric struct {
 // it optionally also creates aggregations with the given settings
 // the 0th retention is the native archive of this metric. if there's several others, we create aggregators, using agg.
 // it's the callers responsability to make sure agg is not nil in that case!
-func NewAggMetric(store Store, cachePusher cache.CachePusher, key string, retentions whisper.Retentions, agg *persister.WhisperAggregationItem) *AggMetric {
+func NewAggMetric(store Store, cachePusher cache.CachePusher, key string, retentions conf.Retentions, agg *conf.Aggregation) *AggMetric {
 
 	// note: during parsing of retentions, we assure there's at least 1.
 	ret := retentions[0]
diff --git a/mdata/aggmetric_test.go b/mdata/aggmetric_test.go
index dc9c81e801..af24d177fe 100644
--- a/mdata/aggmetric_test.go
+++ b/mdata/aggmetric_test.go
@@ -5,8 +5,8 @@ import (
 	"testing"
 	"time"
 
-	whisper "github.com/lomik/go-whisper"
 	"github.com/raintank/metrictank/cluster"
+	"github.com/raintank/metrictank/conf"
 	"github.com/raintank/metrictank/mdata/cache"
 )
 
@@ -94,7 +94,7 @@ func testMetricPersistOptionalPrimary(t *testing.T, primary bool) {
 	mockCache.CacheIfHotCb = func() { calledCb <- true }
 
 	numChunks, chunkAddCount, chunkSpan := uint32(5), uint32(10), uint32(300)
-	ret := []whisper.Retention{whisper.NewRetentionMT(1, 1, chunkSpan, numChunks, true)}
+	ret := []conf.Retention{conf.NewRetentionMT(1, 1, chunkSpan, numChunks, true)}
 	agg := NewAggMetric(dnstore, &mockCache, "foo", ret, nil)
 
 	for ts := chunkSpan; ts <= chunkSpan*chunkAddCount; ts += chunkSpan {
@@ -130,7 +130,7 @@ func testMetricPersistOptionalPrimary(t *testing.T, primary bool) {
 func TestAggMetric(t *testing.T) {
 	cluster.Init("default", "test", time.Now(), "http", 6060)
 
-	ret := []whisper.Retention{whisper.NewRetentionMT(1, 1, 100, 5, true)}
+	ret := []conf.Retention{conf.NewRetentionMT(1, 1, 100, 5, true)}
 	c := NewChecker(t, NewAggMetric(dnstore, &cache.MockCache{}, "foo", ret, nil))
 
 	// basic case, single range
@@ -211,10 +211,10 @@ func BenchmarkAggMetrics1000Metrics1Day(b *testing.B) {
 	cluster.Init("default", "test", time.Now(), "http", 6060)
 	// we will store 10s metrics in 5 chunks of 2 hours
 	// aggregate them in 5min buckets, stored in 1 chunk of 24hours
-	SetOnlyDefaultAgg(whisper.Average, whisper.Min, whisper.Max)
+	SetSingleAgg(conf.Avg, conf.Min, conf.Max)
 	SetSingleSchema(
-		whisper.NewRetentionMT(1, 84600, 2*3600, 5, true),
-		whisper.NewRetentionMT(300, 30*84600, 24*3600, 1, true),
+		conf.NewRetentionMT(1, 84600, 2*3600, 5, true),
+		conf.NewRetentionMT(300, 30*84600, 24*3600, 1, true),
 	)
 	chunkMaxStale := uint32(3600)
 	metricMaxStale := uint32(21600)
@@ -240,10 +240,10 @@ func BenchmarkAggMetrics1kSeries2Chunks1kQueueSize(b *testing.B) {
 	chunkMaxStale := uint32(3600)
 	metricMaxStale := uint32(21600)
 
-	SetOnlyDefaultAgg(whisper.Average, whisper.Min, whisper.Max)
+	SetSingleAgg(conf.Avg, conf.Min, conf.Max)
 	SetSingleSchema(
-		whisper.NewRetentionMT(1, 84600, 600, 5, true),
-		whisper.NewRetentionMT(300, 84600, 24*3600, 2, true),
+		conf.NewRetentionMT(1, 84600, 600, 5, true),
+		conf.NewRetentionMT(300, 84600, 24*3600, 2, true),
 	)
 
 	cluster.Init("default", "test", time.Now(), "http", 6060)
@@ -269,10 +269,10 @@ func BenchmarkAggMetrics10kSeries2Chunks10kQueueSize(b *testing.B) {
 	chunkMaxStale := uint32(3600)
 	metricMaxStale := uint32(21600)
 
-	SetOnlyDefaultAgg(whisper.Average, whisper.Min, whisper.Max)
+	SetSingleAgg(conf.Avg, conf.Min, conf.Max)
 	SetSingleSchema(
-		whisper.NewRetentionMT(1, 84600, 600, 5, true),
-		whisper.NewRetentionMT(300, 84600, 24*3600, 2, true),
+		conf.NewRetentionMT(1, 84600, 600, 5, true),
+		conf.NewRetentionMT(300, 84600, 24*3600, 2, true),
 	)
 
 	cluster.Init("default", "test", time.Now(), "http", 6060)
@@ -298,10 +298,10 @@ func BenchmarkAggMetrics100kSeries2Chunks100kQueueSize(b *testing.B) {
 	chunkMaxStale := uint32(3600)
 	metricMaxStale := uint32(21600)
 
-	SetOnlyDefaultAgg(whisper.Average, whisper.Min, whisper.Max)
+	SetSingleAgg(conf.Avg, conf.Min, conf.Max)
 	SetSingleSchema(
-		whisper.NewRetentionMT(1, 84600, 600, 5, true),
-		whisper.NewRetentionMT(300, 84600, 24*3600, 2, true),
+		conf.NewRetentionMT(1, 84600, 600, 5, true),
+		conf.NewRetentionMT(300, 84600, 24*3600, 2, true),
 	)
 
 	cluster.Init("default", "test", time.Now(), "http", 6060)
diff --git a/mdata/aggmetrics.go b/mdata/aggmetrics.go
index b33ce34425..60fb74fc07 100644
--- a/mdata/aggmetrics.go
+++ b/mdata/aggmetrics.go
@@ -83,8 +83,8 @@ func (ms *AggMetrics) GetOrCreate(key, name string, schemaI, aggI uint16) Metric
 	ms.Lock()
 	m, ok := ms.Metrics[key]
 	if !ok {
-		agg := GetAgg(aggI)
-		m = NewAggMetric(ms.store, ms.cachePusher, key, GetRetentions(schemaI), &agg)
+		agg := Aggregations.Get(aggI)
+		m = NewAggMetric(ms.store, ms.cachePusher, key, Schemas.Get(schemaI).Retentions, &agg)
 		ms.Metrics[key] = m
 		metricsActive.Set(len(ms.Metrics))
 	}
diff --git a/mdata/aggregator.go b/mdata/aggregator.go
index 6462cfd8ef..2398b8b323 100644
--- a/mdata/aggregator.go
+++ b/mdata/aggregator.go
@@ -3,8 +3,7 @@ package mdata
 import (
 	"fmt"
 
-	"github.com/lomik/go-carbon/persister"
-	whisper "github.com/lomik/go-whisper"
+	"github.com/raintank/metrictank/conf"
 	"github.com/raintank/metrictank/mdata/cache"
 )
 
@@ -32,11 +31,11 @@ type Aggregator struct {
 	lstMetric       *AggMetric
 }
 
-func NewAggregator(store Store, cachePusher cache.CachePusher, key string, ret whisper.Retention, agg persister.WhisperAggregationItem) *Aggregator {
+func NewAggregator(store Store, cachePusher cache.CachePusher, key string, ret conf.Retention, agg conf.Aggregation) *Aggregator {
 	if len(agg.AggregationMethod) == 0 {
 		panic("NewAggregator called without aggregations. this should never happen")
 	}
-	span := uint32(ret.SecondsPerPoint())
+	span := uint32(ret.SecondsPerPoint)
 	aggregator := &Aggregator{
 		key:  key,
 		span: span,
@@ -44,28 +43,28 @@ func NewAggregator(store Store, cachePusher cache.CachePusher, key string, ret w
 	}
 	for _, agg := range agg.AggregationMethod {
 		switch agg {
-		case whisper.Average:
+		case conf.Avg:
 			if aggregator.sumMetric == nil {
-				aggregator.sumMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_sum_%d", key, span), whisper.Retentions{ret}, nil)
+				aggregator.sumMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_sum_%d", key, span), conf.Retentions{ret}, nil)
 			}
 			if aggregator.cntMetric == nil {
-				aggregator.cntMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_cnt_%d", key, span), whisper.Retentions{ret}, nil)
+				aggregator.cntMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_cnt_%d", key, span), conf.Retentions{ret}, nil)
 			}
-		case whisper.Sum:
+		case conf.Sum:
 			if aggregator.sumMetric == nil {
-				aggregator.sumMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_sum_%d", key, span), whisper.Retentions{ret}, nil)
+				aggregator.sumMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_sum_%d", key, span), conf.Retentions{ret}, nil)
 			}
-		case whisper.Last:
+		case conf.Lst:
 			if aggregator.lstMetric == nil {
-				aggregator.lstMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_lst_%d", key, span), whisper.Retentions{ret}, nil)
+				aggregator.lstMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_lst_%d", key, span), conf.Retentions{ret}, nil)
 			}
-		case whisper.Max:
+		case conf.Max:
 			if aggregator.maxMetric == nil {
-				aggregator.maxMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_max_%d", key, span), whisper.Retentions{ret}, nil)
+				aggregator.maxMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_max_%d", key, span), conf.Retentions{ret}, nil)
 			}
-		case whisper.Min:
+		case conf.Min:
 			if aggregator.minMetric == nil {
-				aggregator.minMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_min_%d", key, span), whisper.Retentions{ret}, nil)
+				aggregator.minMetric = NewAggMetric(store, cachePusher, fmt.Sprintf("%s_min_%d", key, span), conf.Retentions{ret}, nil)
 			}
 		}
 	}
diff --git a/mdata/aggregator_test.go b/mdata/aggregator_test.go
index f282339c6c..5afc89247b 100644
--- a/mdata/aggregator_test.go
+++ b/mdata/aggregator_test.go
@@ -4,8 +4,8 @@ import (
 	"testing"
 	"time"
 
-	whisper "github.com/lomik/go-whisper"
 	"github.com/raintank/metrictank/cluster"
+	"github.com/raintank/metrictank/conf"
 	"github.com/raintank/metrictank/mdata/cache"
 	"gopkg.in/raintank/schema.v1"
 )
@@ -65,8 +65,11 @@ func TestAggregator(t *testing.T) {
 		}
 		cluster.Manager.SetPrimary(false)
 	}
-	ret := whisper.NewRetentionMT(60, 86400, 120, 10, true)
-	aggs := AllAggregations()
+	ret := conf.NewRetentionMT(60, 86400, 120, 10, true)
+	aggs := conf.Aggregation{
+		AggregationMethod: []conf.Method{conf.Avg, conf.Min, conf.Max, conf.Sum, conf.Lst},
+	}
+
 	agg := NewAggregator(dnstore, &cache.MockCache{}, "test", ret, aggs)
 	agg.Add(100, 123.4)
 	agg.Add(110, 5)
diff --git a/mdata/init.go b/mdata/init.go
index 461ed61994..388fba630e 100644
--- a/mdata/init.go
+++ b/mdata/init.go
@@ -6,11 +6,9 @@ package mdata
 import (
 	"io/ioutil"
 	"log"
-	"regexp"
 	"time"
 
-	"github.com/lomik/go-carbon/persister"
-	whisper "github.com/lomik/go-whisper"
+	"github.com/raintank/metrictank/conf"
 	"github.com/raintank/metrictank/mdata/matchcache"
 	"github.com/raintank/metrictank/stats"
 )
@@ -49,8 +47,8 @@ var (
 	gcMetric = stats.NewCounter32("tank.gc_metric")
 
 	// set either via ConfigProcess or from the unit tests. other code should not touch
-	Schemas      persister.WhisperSchemas
-	Aggregations persister.WhisperAggregation
+	Schemas      conf.Schemas
+	Aggregations conf.Aggregations
 	schemasCache *matchcache.Cache
 	aggsCache    *matchcache.Cache
 
@@ -67,15 +65,10 @@ func ConfigProcess() {
 	// at the end, add a default schema of 7 days of minutely data.
 	// we are stricter and don't tolerate any errors, that seems in the user's best interest.
 
-	Schemas, err = persister.ReadWhisperSchemas(schemasFile)
+	Schemas, err = conf.ReadSchemas(schemasFile)
 	if err != nil {
 		log.Fatalf("can't read schemas file %q: %s", schemasFile, err.Error())
 	}
-	Schemas = append(Schemas, persister.Schema{
-		Name:       "default",
-		Pattern:    regexp.MustCompile(""),
-		Retentions: whisper.Retentions([]whisper.Retention{whisper.NewRetentionMT(60, 3600*24*7, 120*60, 2, true)}),
-	})
 
 	// === read storage-aggregation.conf ===
 
@@ -86,15 +79,13 @@ func ConfigProcess() {
 
 	// since we can't distinguish errors reading vs parsing, we'll just try a read separately first
 	_, err = ioutil.ReadFile(aggFile)
-	if err != nil {
-		// this will add the default case we need
-		Aggregations = persister.NewWhisperAggregation()
-		return
-	}
-	// note: in this case, the library will include the default setting
-	Aggregations, err = persister.ReadWhisperAggregation(aggFile)
-	if err != nil {
-		log.Fatalf("can't read storage-aggregation file %q: %s", aggFile, err.Error())
+	if err == nil {
+		Aggregations, err = conf.ReadAggregations(aggFile)
+		if err != nil {
+			log.Fatalf("can't read storage-aggregation file %q: %s", aggFile, err.Error())
+		}
+	} else {
+		Aggregations = conf.NewAggregations()
 	}
 	Cache(time.Hour, time.Hour)
 }
diff --git a/mdata/schema.go b/mdata/schema.go
index 3f28a42a0e..78311b5096 100644
--- a/mdata/schema.go
+++ b/mdata/schema.go
@@ -1,47 +1,10 @@
 package mdata
 
 import (
-	"regexp"
-
-	"github.com/lomik/go-carbon/persister"
-	"github.com/lomik/go-whisper"
+	"github.com/raintank/metrictank/conf"
 	"github.com/raintank/metrictank/util"
 )
 
-// MatchSchema returns the schema for the given metric key, and the index of the schema (to efficiently reference it)
-// it will always find the schema because we made sure there is a catchall '.*' pattern
-func MatchSchema(key string) (uint16, persister.Schema) {
-	i := schemasCache.Get(key, func(key string) uint16 {
-		i, _, _ := Schemas.Match(key)
-		return i
-	})
-	return i, Schemas[i]
-}
-
-// MatchAgg returns the aggregation definition for the given metric key, and the index of it (to efficiently reference it)
-// i may be 1 more than the last defined by user, in which case it's the default.
-func MatchAgg(key string) (uint16, persister.WhisperAggregationItem) {
-	i := aggsCache.Get(key, func(key string) uint16 {
-		i, _ := Aggregations.Match(key)
-		return i
-	})
-	return i, GetAgg(i)
-}
-
-// caller must assure i is valid
-func GetRetentions(i uint16) whisper.Retentions {
-	return Schemas[i].Retentions
-}
-
-// caller must assure i is valid
-// note the special case
-func GetAgg(i uint16) persister.WhisperAggregationItem {
-	if i+1 > uint16(len(Aggregations.Data)) {
-		return Aggregations.Default
-	}
-	return Aggregations.Data[i]
-}
-
 // TTLs returns a slice of all TTL's seen amongst all archives of all schemas
 func TTLs() []uint32 {
 	ttls := make(map[uint32]struct{})
@@ -50,6 +13,9 @@ func TTLs() []uint32 {
 			ttls[uint32(r.MaxRetention())] = struct{}{}
 		}
 	}
+	for _, r := range conf.DefaultSchema.Retentions {
+		ttls[uint32(r.MaxRetention())] = struct{}{}
+	}
 	var ttlSlice []uint32
 	for ttl := range ttls {
 		ttlSlice = append(ttlSlice, ttl)
@@ -65,31 +31,38 @@ func MaxChunkSpan() uint32 {
 			max = util.Max(max, r.ChunkSpan)
 		}
 	}
+	for _, r := range conf.DefaultSchema.Retentions {
+		max = util.Max(max, r.ChunkSpan)
+	}
 	return max
 }
 
-func CommonAggregations() persister.WhisperAggregationItem {
-	return persister.WhisperAggregationItem{
-		AggregationMethod: []whisper.AggregationMethod{whisper.Average, whisper.Min, whisper.Max},
-	}
+// MatchSchema returns the schema for the given metric key, and the index of the schema (to efficiently reference it)
+// it will always find the schema because Schemas has a catchall default
+func MatchSchema(key string) (uint16, conf.Schema) {
+	i := schemasCache.Get(key, func(key string) uint16 {
+		i, _ := Schemas.Match(key)
+		return i
+	})
+	return i, Schemas.Get(i)
 }
 
-func AllAggregations() persister.WhisperAggregationItem {
-	return persister.WhisperAggregationItem{
-		AggregationMethod: []whisper.AggregationMethod{whisper.Average, whisper.Min, whisper.Max, whisper.Sum, whisper.Last},
-	}
+// MatchAgg returns the aggregation definition for the given metric key, and the index of it (to efficiently reference it)
+// it will always find the aggregation definition because Aggregations has a catchall default
+func MatchAgg(key string) (uint16, conf.Aggregation) {
+	i := aggsCache.Get(key, func(key string) uint16 {
+		i, _ := Aggregations.Match(key)
+		return i
+	})
+	return i, Aggregations.Get(i)
 }
 
-func SetSingleSchema(ret ...whisper.Retention) {
-	Schemas = persister.WhisperSchemas{
-		{
-			Pattern:    regexp.MustCompile(".*"),
-			Retentions: whisper.Retentions(ret),
-		},
-	}
+func SetSingleSchema(ret ...conf.Retention) {
+	Schemas = conf.NewSchemas()
+	conf.DefaultSchema.Retentions = conf.Retentions(ret)
 }
 
-func SetOnlyDefaultAgg(met ...whisper.AggregationMethod) {
-	Aggregations = persister.NewWhisperAggregation()
-	Aggregations.Default.AggregationMethod = met
+func SetSingleAgg(met ...conf.Method) {
+	Aggregations = conf.NewAggregations()
+	conf.DefaultAggregation.AggregationMethod = met
 }
diff --git a/usage/usage_test.go b/usage/usage_test.go
index e736245dc2..9fd4d53e0b 100644
--- a/usage/usage_test.go
+++ b/usage/usage_test.go
@@ -6,8 +6,6 @@ import (
 	"time"
 
 	"github.com/benbjohnson/clock"
-	"github.com/lomik/go-carbon/persister"
-	whisper "github.com/lomik/go-whisper"
 	"github.com/raintank/metrictank/consolidation"
 	"github.com/raintank/metrictank/idx/memory"
 	"github.com/raintank/metrictank/mdata"
@@ -42,17 +40,6 @@ func (f *FakeAggMetrics) GetOrCreate(key, name string, schemaI, aggI uint16) mda
 	f.Unlock()
 	return m
 }
-func (f *FakeAggMetrics) Schemas() persister.WhisperSchemas {
-	return persister.WhisperSchemas{
-		{
-			Retentions: whisper.Retentions(
-				[]whisper.Retention{
-					whisper.NewRetentionMT(1, 35*24*3600, 600, 5, true),
-				},
-			),
-		},
-	}
-}
 
 type FakeAggMetric struct {
 	key     string
diff --git a/vendor/github.com/hydrogen18/stalecucumber/LICENSE b/vendor/github.com/hydrogen18/stalecucumber/LICENSE
deleted file mode 100644
index efcb2414a3..0000000000
--- a/vendor/github.com/hydrogen18/stalecucumber/LICENSE
+++ /dev/null
@@ -1,10 +0,0 @@
-Copyright (c) 2014, Eric Urban
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/vendor/github.com/hydrogen18/stalecucumber/README.md b/vendor/github.com/hydrogen18/stalecucumber/README.md
deleted file mode 100644
index 06951ea818..0000000000
--- a/vendor/github.com/hydrogen18/stalecucumber/README.md
+++ /dev/null
@@ -1,65 +0,0 @@
-#stalecucumber
-
-This package reads and writes pickled data. The format is the same
-as the Python "pickle" module.
-
-Protocols 0,1,2 are implemented. These are the versions written by the Python
-2.x series. Python 3 defines newer protocol versions, but can write the older
-protocol versions so they are readable by this package.
-
-[Full documentation is available here.](https://godoc.org/github.com/hydrogen18/stalecucumber)
-
-##TLDR
-
-Read a pickled string or unicode object
-```
-pickle.dumps("foobar")
----
-var somePickledData io.Reader
-mystring, err := stalecucumber.String(stalecucumber.Unpickle(somePickledData))
-````
-
-Read a pickled integer
-```
-pickle.dumps(42)
----
-var somePickledData io.Reader
-myint64, err := stalecucumber.Int(stalecucumber.Unpickle(somePickledData))
-```
-
-Read a pickled list of numbers into a structure
-```
-pickle.dumps([8,8,2005])
----
-var somePickledData io.Reader
-numbers := make([]int64,0)
-
-err := stalecucumber.UnpackInto(&numbers).From(stalecucumber.Unpickle(somePickledData))
-```
-
-Read a pickled dictionary into a structure
-```
-pickle.dumps({"apple":1,"banana":2,"cat":"hello","Dog":42.0})
----
-var somePickledData io.Reader
-mystruct := struct{
-	Apple int
-	Banana uint
-	Cat string
-	Dog float32}{}
-
-err := stalecucumber.UnpackInto(&mystruct).From(stalecucumber.Unpickle(somePickledData))
-```
-
-Pickle a struct
-
-```
-buf := new(bytes.Buffer)
-mystruct := struct{
-		Apple int
-		Banana uint
-		Cat string
-		Dog float32}{}
-
-err := stalecucumber.NewPickler(buf).Pickle(mystruct)
-```
diff --git a/vendor/github.com/hydrogen18/stalecucumber/fuzz.go b/vendor/github.com/hydrogen18/stalecucumber/fuzz.go
deleted file mode 100644
index e710631449..0000000000
--- a/vendor/github.com/hydrogen18/stalecucumber/fuzz.go
+++ /dev/null
@@ -1,14 +0,0 @@
-// +build gofuzz
-
-package stalecucumber
-
-import (
-	"bytes"
-)
-
-func Fuzz(data []byte) int {
-	if _, err := Unpickle(bytes.NewReader(data)); err != nil {
-		return 0
-	}
-	return 1
-}
diff --git a/vendor/github.com/hydrogen18/stalecucumber/helpers.go b/vendor/github.com/hydrogen18/stalecucumber/helpers.go
deleted file mode 100644
index a133ed2b46..0000000000
--- a/vendor/github.com/hydrogen18/stalecucumber/helpers.go
+++ /dev/null
@@ -1,201 +0,0 @@
-package stalecucumber
-
-import "fmt"
-import "math/big"
-
-/*
-This type is returned whenever a helper cannot convert the result of
-Unpickle into the desired type.
-*/
-type WrongTypeError struct {
-	Result  interface{}
-	Request string
-}
-
-func (wte WrongTypeError) Error() string {
-	return fmt.Sprintf("Unpickling returned type %T which cannot be converted to %s", wte.Result, wte.Request)
-}
-
-func newWrongTypeError(result interface{}, request interface{}) error {
-	return WrongTypeError{Result: result, Request: fmt.Sprintf("%T", request)}
-}
-
-/*
-This helper attempts to convert the return value of Unpickle into a string.
-
-If Unpickle returns an error that error is returned immediately.
-
-If the value cannot be converted an error is returned.
-*/
-func String(v interface{}, err error) (string, error) {
-	if err != nil {
-		return "", err
-	}
-	vs, ok := v.(string)
-	if ok {
-		return vs, nil
-	}
-
-	return "", newWrongTypeError(v, vs)
-}
-
-/*
-This helper attempts to convert the return value of Unpickle into a int64.
-
-If Unpickle returns an error that error is returned immediately.
-
-If the value cannot be converted an error is returned.
-*/
-func Int(v interface{}, err error) (int64, error) {
-	if err != nil {
-		return 0, err
-	}
-	vi, ok := v.(int64)
-	if ok {
-		return vi, nil
-	}
-
-	vbi, ok := v.(*big.Int)
-	if ok {
-		if vbi.BitLen() <= 63 {
-			return vbi.Int64(), nil
-		}
-	}
-
-	return 0, newWrongTypeError(v, vi)
-
-}
-
-/*
-This helper attempts to convert the return value of Unpickle into a bool.
-
-If Unpickle returns an error that error is returned immediately.
-
-If the value cannot be converted an error is returned.
-*/
-func Bool(v interface{}, err error) (bool, error) {
-	if err != nil {
-		return false, err
-	}
-	vb, ok := v.(bool)
-	if ok {
-		return vb, nil
-	}
-
-	return false, newWrongTypeError(v, vb)
-
-}
-
-/*
-This helper attempts to convert the return value of Unpickle into a *big.Int.
-
-If Unpickle returns an error that error is returned immediately.
-
-If the value cannot be converted an error is returned.
-*/
-func Big(v interface{}, err error) (*big.Int, error) {
-	if err != nil {
-		return nil, err
-	}
-
-	vb, ok := v.(*big.Int)
-	if ok {
-		return vb, nil
-	}
-
-	return nil, newWrongTypeError(v, vb)
-
-}
-
-/*
-This helper attempts to convert the return value of Unpickle into a float64.
-
-If Unpickle returns an error that error is returned immediately.
-
-If the value cannot be converted an error is returned.
-*/
-func Float(v interface{}, err error) (float64, error) {
-	if err != nil {
-		return 0.0, err
-	}
-
-	vf, ok := v.(float64)
-	if ok {
-		return vf, nil
-	}
-
-	return 0.0, newWrongTypeError(v, vf)
-}
-
-/*
-This helper attempts to convert the return value of Unpickle into a []interface{}.
-
-If Unpickle returns an error that error is returned immediately.
-
-If the value cannot be converted an error is returned.
-*/
-func ListOrTuple(v interface{}, err error) ([]interface{}, error) {
-	if err != nil {
-		return nil, err
-	}
-
-	vl, ok := v.([]interface{})
-	if ok {
-		return vl, nil
-	}
-
-	return nil, newWrongTypeError(v, vl)
-
-}
-
-/*
-This helper attempts to convert the return value of Unpickle into a map[interface{}]interface{}.
-
-If Unpickle returns an error that error is returned immediately.
-
-If the value cannot be converted an error is returned.
-*/
-func Dict(v interface{}, err error) (map[interface{}]interface{}, error) {
-	if err != nil {
-		return nil, err
-	}
-
-	vd, ok := v.(map[interface{}]interface{})
-	if ok {
-		return vd, nil
-	}
-
-	return nil, newWrongTypeError(v, vd)
-}
-
-/*
-This helper attempts to convert the return value of Unpickle into a map[string]interface{}.
-
-If Unpickle returns an error that error is returned immediately.
-
-If the value cannot be converted an error is returned.
-*/
-func DictString(v interface{}, err error) (map[string]interface{}, error) {
-	var src map[interface{}]interface{}
-	src, err = Dict(v, err)
-	if err != nil {
-		return nil, err
-	}
-
-	return tryDictToDictString(src)
-}
-
-func tryDictToDictString(src map[interface{}]interface{}) (map[string]interface{}, error) {
-	dst := make(map[string]interface{}, len(src))
-
-	for k, v := range src {
-		kstr, ok := k.(string)
-		if !ok {
-			return nil, newWrongTypeError(src, dst)
-		}
-		dst[kstr] = v
-	}
-
-	return dst, nil
-
-}
diff --git a/vendor/github.com/hydrogen18/stalecucumber/jump_list.go b/vendor/github.com/hydrogen18/stalecucumber/jump_list.go
deleted file mode 100644
index 6f3df294e6..0000000000
--- a/vendor/github.com/hydrogen18/stalecucumber/jump_list.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package stalecucumber
-
-import "errors"
-
-type opcodeFunc func(*PickleMachine) error
-
-var ErrOpcodeInvalid = errors.New("Opcode is invalid")
-
-func (pm *PickleMachine) Opcode_Invalid() error {
-	return ErrOpcodeInvalid
-}
-
-type opcodeJumpList [256]opcodeFunc
-
-func buildEmptyJumpList() opcodeJumpList {
-	jl := opcodeJumpList{}
-
-	for i := range jl {
-		jl[i] = (*PickleMachine).Opcode_Invalid
-	}
-	return jl
-}
diff --git a/vendor/github.com/hydrogen18/stalecucumber/opcodes.go b/vendor/github.com/hydrogen18/stalecucumber/opcodes.go
deleted file mode 100644
index 5e34d10d68..0000000000
--- a/vendor/github.com/hydrogen18/stalecucumber/opcodes.go
+++ /dev/null
@@ -1,55 +0,0 @@
-package stalecucumber
-
-const OPCODE_INT = 0x49
-const OPCODE_LONG = 0x4c
-const OPCODE_STRING = 0x53
-const OPCODE_NONE = 0x4e
-const OPCODE_UNICODE = 0x56
-const OPCODE_FLOAT = 0x46
-const OPCODE_APPEND = 0x61
-const OPCODE_LIST = 0x6c
-const OPCODE_TUPLE = 0x74
-const OPCODE_DICT = 0x64
-const OPCODE_SETITEM = 0x73
-const OPCODE_POP = 0x30
-const OPCODE_DUP = 0x32
-const OPCODE_MARK = 0x28
-const OPCODE_GET = 0x67
-const OPCODE_PUT = 0x70
-const OPCODE_GLOBAL = 0x63
-const OPCODE_REDUCE = 0x52
-const OPCODE_BUILD = 0x62
-const OPCODE_INST = 0x69
-const OPCODE_STOP = 0x2e
-const OPCODE_PERSID = 0x50
-const OPCODE_BININT = 0x4a
-const OPCODE_BININT1 = 0x4b
-const OPCODE_BININT2 = 0x4d
-const OPCODE_BINSTRING = 0x54
-const OPCODE_SHORT_BINSTRING = 0x55
-const OPCODE_BINUNICODE = 0x58
-const OPCODE_BINFLOAT = 0x47
-const OPCODE_EMPTY_LIST = 0x5d
-const OPCODE_APPENDS = 0x65
-const OPCODE_EMPTY_TUPLE = 0x29
-const OPCODE_EMPTY_DICT = 0x7d
-const OPCODE_SETITEMS = 0x75
-const OPCODE_POP_MARK = 0x31
-const OPCODE_BINGET = 0x68
-const OPCODE_LONG_BINGET = 0x6a
-const OPCODE_BINPUT = 0x71
-const OPCODE_LONG_BINPUT = 0x72
-const OPCODE_OBJ = 0x6f
-const OPCODE_BINPERSID = 0x51
-const OPCODE_LONG1 = 0x8a
-const OPCODE_LONG4 = 0x8b
-const OPCODE_NEWTRUE = 0x88
-const OPCODE_NEWFALSE = 0x89
-const OPCODE_TUPLE1 = 0x85
-const OPCODE_TUPLE2 = 0x86
-const OPCODE_TUPLE3 = 0x87
-const OPCODE_EXT1 = 0x82
-const OPCODE_EXT2 = 0x83
-const OPCODE_EXT4 = 0x84
-const OPCODE_NEWOBJ = 0x81
-const OPCODE_PROTO = 0x80
diff --git a/vendor/github.com/hydrogen18/stalecucumber/pickle_machine.go b/vendor/github.com/hydrogen18/stalecucumber/pickle_machine.go
deleted file mode 100644
index b302608d19..0000000000
--- a/vendor/github.com/hydrogen18/stalecucumber/pickle_machine.go
+++ /dev/null
@@ -1,598 +0,0 @@
-/*
-This package reads and writes pickled data. The format is the same
-as the Python "pickle" module.
-
-Protocols 0,1,2 are implemented. These are the versions written by the Python
-2.x series. Python 3 defines newer protocol versions, but can write the older
-protocol versions so they are readable by this package.
-
-To read data, see stalecucumber.Unpickle.
-
-To write data, see stalecucumber.NewPickler.
-
-TLDR
-
-Read a pickled string or unicode object
-	pickle.dumps("foobar")
-	---
-	var somePickledData io.Reader
-	mystring, err := stalecucumber.String(stalecucumber.Unpickle(somePickledData))
-
-Read a pickled integer
-	pickle.dumps(42)
-	---
-	var somePickledData io.Reader
-	myint64, err := stalecucumber.Int(stalecucumber.Unpickle(somePickledData))
-
-Read a pickled list of numbers into a structure
-	pickle.dumps([8,8,2005])
-	---
-	var somePickledData io.Reader
-	numbers := make([]int64,0)
-
-	err := stalecucumber.UnpackInto(&numbers).From(stalecucumber.Unpickle(somePickledData))
-
-Read a pickled dictionary into a structure
-	pickle.dumps({"apple":1,"banana":2,"cat":"hello","Dog":42.0})
-	---
-	var somePickledData io.Reader
-	mystruct := struct{
-		Apple int
-		Banana uint
-		Cat string
-		Dog float32}{}
-
-	err := stalecucumber.UnpackInto(&mystruct).From(stalecucumber.Unpickle(somePickledData))
-
-Pickle a struct
-
-	buf := new(bytes.Buffer)
-	mystruct := struct{
-			Apple int
-			Banana uint
-			Cat string
-			Dog float32}{}
-
-	err := stalecucumber.NewPickler(buf).Pickle(mystruct)
-
-
-
-Recursive objects
-
-You can pickle recursive objects like so
-
-	a = {}
-	a["self"] = a
-	pickle.dumps(a)
-
-Python's pickler is intelligent enough not to emit an infinite data structure
-when a recursive object is pickled.
-
-I recommend against pickling recursive objects in the first place, but this
-library handles unpickling them without a problem. The result of unpickling
-the above is map[interface{}]interface{} with a key "a" that contains
-a reference to itself.
-
-Attempting to unpack the result of the above python code into a structure
-with UnpackInto would either fail or recurse forever.
-
-Protocol Performance
-
-If the version of Python you are using supports protocol version 1 or 2,
-you should always specify that protocol version. By default the "pickle"
-and "cPickle" modules in Python write using protocol 0. Protocol 0
-requires much more space to represent the same values and is much
-slower to parse.
-
-Unsupported Opcodes
-
-The pickle format is incredibly flexible and as a result has some
-features that are impractical or unimportant when implementing a reader in
-another language.
-
-Each set of opcodes is listed below by protocol version with the impact.
-
-Protocol 0
-
-	GLOBAL
-
-This opcode is equivalent to calling "import foo; foo.bar" in python. It is
-generated whenever an object instance, class definition, or method definition
-is serialized. As long as the pickled data does not contain an instance
-of a python class or a reference to a python callable this opcode is not
-emitted by the "pickle" module.
-
-A few examples of what will definitely cause this opcode to be emitted
-
-	pickle.dumps(range) #Pickling the range function
-	pickle.dumps(Exception()) #Pickling an instance of a python class
-
-This opcode will be partially supported in a future revision to this package
-that allows the unpickling of instances of Python classes.
-
-	REDUCE
-	BUILD
-	INST
-
-These opcodes are used in recreating pickled python objects. That is currently
-not supported by this package.
-
-These opcodes will be supported in a future revision to this package
-that allows the unpickling of instances of Python classes.
-
-	PERSID
-
-This opcode is used to reference concrete definitions of objects between
-a pickler and an unpickler by an ID number. The pickle protocol doesn't define
-what a persistent ID means.
-
-This opcode is unlikely to ever be supported by this package.
-
-Protocol 1
-
-	OBJ
-
-This opcodes is used in recreating pickled python objects. That is currently
-not supported by this package.
-
-This opcode will supported in a future revision to this package
-that allows the unpickling of instances of Python classes.
-
-
-	BINPERSID
-
-This opcode is equivalent to PERSID in protocol 0 and won't be supported
-for the same reason.
-
-Protocol 2
-
-	NEWOBJ
-
-This opcodes is used in recreating pickled python objects. That is currently
-not supported by this package.
-
-This opcode will supported in a future revision to this package
-that allows the unpickling of instances of Python classes.
-
-	EXT1
-	EXT2
-	EXT4
-
-These opcodes allow using a registry
-of popular objects that are pickled by name, typically classes.
-It is envisioned that through a global negotiation and
-registration process, third parties can set up a mapping between
-ints and object names.
-
-These opcodes are unlikely to ever be supported by this package.
-
-*/
-package stalecucumber
-
-import "errors"
-import "io"
-import "bytes"
-import "encoding/binary"
-import "fmt"
-
-var ErrOpcodeStopped = errors.New("STOP opcode found")
-var ErrStackTooSmall = errors.New("Stack is too small to perform requested operation")
-var ErrInputTruncated = errors.New("Input to the pickle machine was truncated")
-var ErrOpcodeNotImplemented = errors.New("Input encountered opcode that is not implemented")
-var ErrNoResult = errors.New("Input did not place a value onto the stack")
-var ErrMarkNotFound = errors.New("Mark could not be found on the stack")
-
-/*
-Unpickle a value from a reader. This function takes a reader and
-attempts to read a complete pickle program from it. This is normally
-the output of the function "pickle.dump" from Python.
-
-The returned type is interface{} because unpickling can generate any type. Use
-a helper function to convert to another type without an additional type check.
-
-This function returns an error if
-the reader fails, the pickled data is invalid, or if the pickled data contains
-an unsupported opcode. See unsupported opcodes in the documentation of
-this package for more information.
-
-Type Conversions
-
-Types conversion Python types to Go types is performed as followed
-	int -> int64
-	string -> string
-	unicode -> string
-	float -> float64
-	long -> big.Int from the "math/big" package
-	lists -> []interface{}
-	tuples -> []interface{}
-	dict -> map[interface{}]interface{}
-
-The following values are converted from Python to the Go types
-	True & False -> bool
-	None -> stalecucumber.PickleNone, sets pointers to nil
-
-Helper Functions
-
-The following helper functions were inspired by the github.com/garyburd/redigo
-package. Each function takes the result of Unpickle as its arguments. If unpickle
-fails it does nothing and returns that error. Otherwise it attempts to
-convert to the appropriate type. If type conversion fails it returns an error
-
-	String - string from Python string or unicode
-	Int - int64 from Python int or long
-	Bool - bool from Python True or False
-	Big - *big.Int from Python long
-	ListOrTuple - []interface{} from Python Tuple or List
-	Float - float64 from Python float
-	Dict - map[interface{}]interface{} from Python dictionary
-	DictString -
-		map[string]interface{} from Python dictionary.
-		Keys must all be of type unicode or string.
-
-Unpacking into structures
-
-If the pickled object is a python dictionary that has only unicode and string
-objects for keys, that object can be unpickled into a struct in Go by using
-the "UnpackInto" function. The "From" receiver on the return value accepts
-the result of "Unpickle" as its actual parameters.
-
-The keys of the python dictionary are assigned to fields in a structure.
-Structures may specify the tag "pickle" on fields. The value of this tag is taken
-as the key name of the Python dictionary value to place in this field. If no
-field has a matching "pickle" tag the fields are looked up by name. If
-the first character of the key is not uppercase, it is uppercased. If a field
-matching that name is found, the value in the python dictionary is unpacked
-into the value of the field within the structure.
-
-A list of python dictionaries can be unpickled into a slice of structures in
-Go.
-
-A homogeneous list of python values can be unpickled into a slice in
-Go with the appropriate element type.
-
-A nested python dictionary is unpickled into nested structures in Go. If a
-field is of type map[interface{}]interface{} it is of course unpacked into that
-as well.
-
-By default UnpackInto skips any missing fields and fails if a field's
-type is not compatible with the object's type.
-
-This behavior can be changed by setting "AllowMissingFields" and
-"AllowMismatchedFields" on the return value of UnpackInto before calling
-From.
-
-*/
-func Unpickle(reader io.Reader) (interface{}, error) {
-	var pm PickleMachine
-	pm.buf = &bytes.Buffer{}
-	pm.Reader = reader
-	pm.lastMark = -1
-	//Pre allocate a small stack
-	pm.Stack = make([]interface{}, 0, 16)
-
-	err := (&pm).execute()
-	if err != ErrOpcodeStopped {
-		return nil, pm.error(err)
-	}
-
-	if len(pm.Stack) == 0 {
-		return nil, ErrNoResult
-	}
-
-	return pm.Stack[0], nil
-
-}
-
-var jumpList = buildEmptyJumpList()
-
-func init() {
-	populateJumpList(&jumpList)
-}
-
-/*
-This type is returned whenever Unpickle encounters an error in pickled data.
-*/
-type PickleMachineError struct {
-	Err       error
-	StackSize int
-	MemoSize  int
-	Opcode    uint8
-}
-
-/*
-This struct is current exposed but not useful. It is likely to be hidden
-in the near future.
-*/
-type PickleMachine struct {
-	Stack  []interface{}
-	Memo   []interface{}
-	Reader io.Reader
-
-	currentOpcode uint8
-	buf           *bytes.Buffer
-	lastMark      int
-
-	memoBuffer               [16]memoBufferElement
-	memoBufferMaxDestination int64
-	memoBufferIndex          int
-}
-
-type memoBufferElement struct {
-	Destination int64
-	V           interface{}
-}
-
-func (pme PickleMachineError) Error() string {
-	return fmt.Sprintf("Pickle Machine failed on opcode:0x%x. Stack size:%d. Memo size:%d. Cause:%v", pme.Opcode, pme.StackSize, pme.MemoSize, pme.Err)
-}
-
-func (pm *PickleMachine) error(src error) error {
-	return PickleMachineError{
-		StackSize: len(pm.Stack),
-		MemoSize:  len(pm.Memo),
-		Err:       src,
-		Opcode:    pm.currentOpcode,
-	}
-}
-
-func (pm *PickleMachine) execute() error {
-	for {
-		err := binary.Read(pm.Reader, binary.BigEndian, &pm.currentOpcode)
-		if err != nil {
-			return err
-		}
-
-		err = jumpList[int(pm.currentOpcode)](pm)
-
-		if err != nil {
-			return err
-		}
-	}
-}
-
-func (pm *PickleMachine) flushMemoBuffer(vIndex int64, v interface{}) {
-	//Extend the memo until it is large enough
-	if pm.memoBufferMaxDestination >= int64(len(pm.Memo)) {
-		replacement := make([]interface{}, pm.memoBufferMaxDestination<<1)
-		copy(replacement, pm.Memo)
-		pm.Memo = replacement
-	}
-
-	//If a value was passed into this function, write it into the memo
-	//as well
-	if vIndex != -1 {
-		pm.Memo[vIndex] = v
-	}
-
-	//Write the contents of the buffer into the memo
-	//in the same order as the puts were issued
-	for i := 0; i != pm.memoBufferIndex; i++ {
-		buffered := pm.memoBuffer[i]
-		pm.Memo[buffered.Destination] = buffered.V
-	}
-
-	//Reset the buffer
-	pm.memoBufferIndex = 0
-	pm.memoBufferMaxDestination = 0
-
-	return
-}
-
-func (pm *PickleMachine) storeMemo(index int64, v interface{}) error {
-	if index < 0 {
-		return fmt.Errorf("Requested to write to invalid memo index:%v", index)
-	}
-
-	//If there is space in the memo presently, then store it
-	//and it is done.
-	if index < int64(len(pm.Memo)) {
-		pm.Memo[index] = v
-		return nil
-	}
-
-	//Update the maximum index in the buffer if need be
-	if index > pm.memoBufferMaxDestination {
-		pm.memoBufferMaxDestination = index
-	}
-
-	//If the buffer is not full write into it
-	if pm.memoBufferIndex != len(pm.memoBuffer) {
-		pm.memoBuffer[pm.memoBufferIndex].V = v
-		pm.memoBuffer[pm.memoBufferIndex].Destination = index
-		pm.memoBufferIndex++
-	} else {
-		//If the buffer is full flush it now
-		pm.flushMemoBuffer(index, v)
-	}
-
-	return nil
-}
-
-func (pm *PickleMachine) readFromMemo(index int64) (interface{}, error) {
-	if index < 0 {
-		return nil, fmt.Errorf("Requested to read from negative memo index %d", index)
-
-	}
-
-	//Test to see if the value is outside the current length of the memo
-	if index >= int64(len(pm.Memo)) {
-		pm.flushMemoBuffer(-1, nil)
-		if index >= int64(len(pm.Memo)) {
-			return nil, fmt.Errorf("Requested to read from invalid memo index %d", index)
-		}
-	}
-
-	//Grab the value
-	retval := pm.Memo[index]
-
-	//If nil then flush the memo buffer to see if it is within it
-	if retval == nil {
-		pm.flushMemoBuffer(-1, nil)
-		//Grab the value again after the flush
-		retval = pm.Memo[index]
-		//If still nil, then this is a read from an invalid position
-		if retval == nil {
-			return nil, fmt.Errorf("Requested to read from invalid memo index %d", index)
-		}
-	}
-
-	return retval, nil
-}
-
-func (pm *PickleMachine) push(v interface{}) {
-	pm.Stack = append(pm.Stack, v)
-}
-
-func (pm *PickleMachine) pop() (interface{}, error) {
-	l := len(pm.Stack)
-	if l == 0 {
-		return nil, ErrStackTooSmall
-	}
-
-	l--
-	top := pm.Stack[l]
-	pm.Stack = pm.Stack[:l]
-	return top, nil
-}
-
-func (pm *PickleMachine) readFromStack(offset int) (interface{}, error) {
-	return pm.readFromStackAt(len(pm.Stack) - 1 - offset)
-}
-
-func (pm *PickleMachine) readFromStackAt(position int) (interface{}, error) {
-
-	if position < 0 {
-		return nil, fmt.Errorf("Request to read from invalid stack position %d", position)
-	}
-
-	return pm.Stack[position], nil
-
-}
-
-func (pm *PickleMachine) readIntFromStack(offset int) (int64, error) {
-	v, err := pm.readFromStack(offset)
-	if err != nil {
-		return 0, err
-	}
-
-	vi, ok := v.(int64)
-	if !ok {
-		return 0, fmt.Errorf("Type %T was requested from stack but found %v(%T)", vi, v, v)
-	}
-
-	return vi, nil
-}
-
-func (pm *PickleMachine) popAfterIndex(index int) {
-	//Input to this function must be sane, no checking is done
-
-	/**
-	if len(pm.Stack)-1 < index {
-		return ErrStackTooSmall
-	}**/
-
-	pm.Stack = pm.Stack[0:index]
-}
-
-func (pm *PickleMachine) findMark() (int, error) {
-	if pm.lastMark != -1 {
-		mark := pm.lastMark
-		pm.lastMark = -1
-		if mark < len(pm.Stack) {
-			if _, ok := pm.Stack[mark].(PickleMark); ok {
-				return mark, nil
-			}
-		}
-	}
-
-	for i := len(pm.Stack) - 1; i != -1; i-- {
-		if _, ok := pm.Stack[i].(PickleMark); ok {
-			return i, nil
-		}
-	}
-	return -1, ErrMarkNotFound
-}
-
-func (pm *PickleMachine) readFixedLengthRaw(l int64) ([]byte, error) {
-
-	pm.buf.Reset()
-	_, err := io.CopyN(pm.buf, pm.Reader, l)
-	if err != nil {
-		return nil, err
-	}
-
-	return pm.buf.Bytes(), nil
-}
-
-func (pm *PickleMachine) readFixedLengthString(l int64) (string, error) {
-
-	//Avoid getting "<nil>"
-	if l == 0 {
-		return "", nil
-	}
-
-	pm.buf.Reset()
-	_, err := io.CopyN(pm.buf, pm.Reader, l)
-	if err != nil {
-		return "", err
-	}
-	return pm.buf.String(), nil
-}
-
-func (pm *PickleMachine) readBytes() ([]byte, error) {
-	//This is slow and protocol 0 only
-	pm.buf.Reset()
-	for {
-		var v [1]byte
-		n, err := pm.Reader.Read(v[:])
-		if n != 1 {
-			return nil, ErrInputTruncated
-		}
-		if err != nil {
-			return nil, err
-		}
-
-		if v[0] == '\n' {
-			break
-		}
-		pm.buf.WriteByte(v[0])
-	}
-
-	return pm.buf.Bytes(), nil
-}
-
-func (pm *PickleMachine) readString() (string, error) {
-	//This is slow and protocol 0 only
-	pm.buf.Reset()
-	for {
-		var v [1]byte
-		n, err := pm.Reader.Read(v[:])
-		if n != 1 {
-			return "", ErrInputTruncated
-		}
-		if err != nil {
-			return "", err
-		}
-
-		if v[0] == '\n' {
-			break
-		}
-		pm.buf.WriteByte(v[0])
-	}
-
-	//Avoid getting "<nil>"
-	if pm.buf.Len() == 0 {
-		return "", nil
-	}
-	return pm.buf.String(), nil
-}
-
-func (pm *PickleMachine) readBinaryInto(dst interface{}, bigEndian bool) error {
-	var bo binary.ByteOrder
-	if bigEndian {
-		bo = binary.BigEndian
-	} else {
-		bo = binary.LittleEndian
-	}
-	return binary.Read(pm.Reader, bo, dst)
-}
diff --git a/vendor/github.com/hydrogen18/stalecucumber/pickle_writer.go b/vendor/github.com/hydrogen18/stalecucumber/pickle_writer.go
deleted file mode 100644
index 1fd316d021..0000000000
--- a/vendor/github.com/hydrogen18/stalecucumber/pickle_writer.go
+++ /dev/null
@@ -1,514 +0,0 @@
-package stalecucumber
-
-import "io"
-import "reflect"
-import "errors"
-import "encoding/binary"
-import "fmt"
-import "math/big"
-
-type pickleProxy interface {
-	WriteTo(io.Writer) (int, error)
-}
-
-type Pickler struct {
-	W io.Writer
-
-	program []pickleProxy
-}
-
-/*
-This type is used to pickle data.Picklers are created
-by calling NewPickler. Each call to Pickle writes a
-complete pickle program to the underlying io.Writer object.
-
-Its safe to assign W to other values in between calls to Pickle.
-
-Failures return the underlying error or an instance of PicklingError.
-
-Data is always written using Pickle Protocol 2. This format is
-compatible with Python 2.3 and all newer version.
-
-Type Conversions
-
-Type conversion from Go types to Python types is as follows
-
-	uint8,uint16,int8,int16,int32 -> Python int
-	int,int64,uint,uint64 -> Python int if it fits, otherwise Python Long
-	string -> Python unicode
-	slices, arrays -> Python list
-	maps -> Python dict
-	bool -> Python True and False
-	big.Int -> Python Long
-	struct -> Python dict
-
-Structs are pickled using their field names unless a tag is present on the
-field specifying the name. For example
-
-	type MyType struct {
-		FirstField int
-		SecondField int `pickle:"meow"`
-	}
-
-This struct would be pickled into a dictionary with two keys: "FirstField"
-and "meow".
-
-Embedded structs are marshalled as a nested dictionary. Exported types
-are never pickled.
-
-Pickling Tuples
-
-There is no equivalent type to Python's tuples in Go. You may not need
-to use tuples at all. For example, consider the following Python code
-
-
-	a, b, c = pickle.load(data_in)
-
-This code tries to set to the variables "a", "b", and "c" from the result
-of unpickling. In this case it does not matter if the source type
-is a Python list or a Python tuple.
-
-If you really need to write tuples, call NewTuple and pass the data
-in as the arguments. This special type exists to inform stalecucumber.Pickle that
-a tuple should be pickled.
-
-*/
-func NewPickler(writer io.Writer) *Pickler {
-	retval := &Pickler{}
-	retval.W = writer
-	return retval
-}
-
-func (p *Pickler) Pickle(v interface{}) (int, error) {
-	if p.program != nil {
-		p.program = p.program[0:0]
-	}
-
-	err := p.dump(v)
-	if err != nil {
-		return 0, err
-	}
-
-	return p.writeProgram()
-}
-
-var programStart = []uint8{OPCODE_PROTO, 0x2}
-var programEnd = []uint8{OPCODE_STOP}
-
-func (p *Pickler) writeProgram() (n int, err error) {
-	n, err = p.W.Write(programStart)
-	if err != nil {
-		return
-	}
-	var m int
-	for _, proxy := range p.program {
-		m, err = proxy.WriteTo(p.W)
-		if err != nil {
-			return
-		}
-
-		n += m
-	}
-
-	m, err = p.W.Write(programEnd)
-	if err != nil {
-		return
-	}
-	n += m
-
-	return
-}
-
-const BININT_MAX = (1 << 31) - 1
-const BININT_MIN = 0 - BININT_MAX
-
-var ErrTypeNotPickleable = errors.New("Can't pickle this type")
-var ErrEmptyInterfaceNotPickleable = errors.New("The empty interface is not pickleable")
-
-type PicklingError struct {
-	V   interface{}
-	Err error
-}
-
-func (pe PicklingError) Error() string {
-	return fmt.Sprintf("Failed pickling (%T)%v:%v", pe.V, pe.V, pe.Err)
-}
-
-func (p *Pickler) dump(input interface{}) error {
-	if input == nil {
-		p.pushOpcode(OPCODE_NONE)
-		return nil
-	}
-
-	switch input := input.(type) {
-	case int:
-		if input <= BININT_MAX && input >= BININT_MIN {
-			p.dumpInt(int64(input))
-			return nil
-		}
-		p.dumpIntAsLong(int64(input))
-		return nil
-	case int64:
-		if input <= BININT_MAX && input >= BININT_MIN {
-			p.dumpInt(input)
-			return nil
-		}
-		p.dumpIntAsLong(input)
-		return nil
-	case int8:
-		p.dumpInt(int64(input))
-		return nil
-	case int16:
-		p.dumpInt(int64(input))
-		return nil
-	case int32:
-		p.dumpInt(int64(input))
-		return nil
-
-	case uint8:
-		p.dumpInt(int64(input))
-		return nil
-	case uint16:
-		p.dumpInt(int64(input))
-		return nil
-
-	case uint32:
-		if input <= BININT_MAX {
-			p.dumpInt(int64(input))
-			return nil
-		}
-		p.dumpUintAsLong(uint64(input))
-		return nil
-
-	case uint:
-		if input <= BININT_MAX {
-			p.dumpInt(int64(input))
-			return nil
-		}
-		p.dumpUintAsLong(uint64(input))
-		return nil
-	case uint64:
-		if input <= BININT_MAX {
-			p.dumpInt(int64(input))
-			return nil
-		}
-		p.dumpUintAsLong(input)
-		return nil
-	case float32:
-		p.dumpFloat(float64(input))
-		return nil
-	case float64:
-		p.dumpFloat(input)
-		return nil
-	case string:
-		p.dumpString(input)
-		return nil
-	case bool:
-		p.dumpBool(input)
-		return nil
-	case big.Int:
-		p.dumpBigInt(input)
-		return nil
-	case PickleNone:
-		p.pushOpcode(OPCODE_NONE)
-		return nil
-	case PickleTuple:
-		l := len(input)
-		switch l {
-		case 0:
-			p.pushOpcode(OPCODE_EMPTY_TUPLE)
-			return nil
-		case 1, 2, 3:
-		default:
-			p.pushOpcode(OPCODE_MARK)
-		}
-
-		for _, v := range input {
-			err := p.dump(v)
-			if err != nil {
-				return err
-			}
-		}
-
-		switch l {
-		case 1:
-			p.pushOpcode(OPCODE_TUPLE1)
-		case 2:
-			p.pushOpcode(OPCODE_TUPLE2)
-		case 3:
-			p.pushOpcode(OPCODE_TUPLE3)
-		default:
-			p.pushOpcode(OPCODE_TUPLE)
-		}
-
-		return nil
-	}
-
-	v := reflect.ValueOf(input)
-	vKind := v.Kind()
-
-	switch vKind {
-	//Check for pointers. They can't be
-	//meaningfully written as a pickle unless nil. Dereference
-	//and recurse.
-	case reflect.Ptr:
-		if v.IsNil() {
-			p.pushOpcode(OPCODE_NONE)
-			return nil
-		}
-		return p.dump(v.Elem().Interface())
-	case reflect.Map:
-		p.pushOpcode(OPCODE_EMPTY_DICT)
-		p.pushOpcode(OPCODE_MARK)
-
-		keys := v.MapKeys()
-		for _, key := range keys {
-			err := p.dump(key.Interface())
-			if err != nil {
-				return err
-			}
-			val := v.MapIndex(key)
-			err = p.dump(val.Interface())
-			if err != nil {
-				return err
-			}
-		}
-		p.pushOpcode(OPCODE_SETITEMS)
-		return nil
-	case reflect.Slice, reflect.Array:
-		p.pushOpcode(OPCODE_EMPTY_LIST)
-		p.pushOpcode(OPCODE_MARK)
-		for i := 0; i != v.Len(); i++ {
-			element := v.Index(i)
-			p.dump(element.Interface())
-		}
-		p.pushOpcode(OPCODE_APPENDS)
-		return nil
-	case reflect.Struct:
-		return p.dumpStruct(input)
-	}
-
-	return PicklingError{V: input, Err: ErrTypeNotPickleable}
-}
-
-func (p *Pickler) dumpBool(v bool) {
-	if v {
-		p.pushOpcode(OPCODE_NEWTRUE)
-	} else {
-		p.pushOpcode(OPCODE_NEWFALSE)
-	}
-}
-
-func (p *Pickler) dumpStruct(input interface{}) error {
-	vType := reflect.TypeOf(input)
-	v := reflect.ValueOf(input)
-
-	p.pushOpcode(OPCODE_EMPTY_DICT)
-	p.pushOpcode(OPCODE_MARK)
-
-	for i := 0; i != v.NumField(); i++ {
-		field := vType.Field(i)
-		//Never attempt to write
-		//unexported names
-		if len(field.PkgPath) != 0 {
-			continue
-		}
-
-		//Prefer the tagged name of the
-		//field, fall back to fields actual name
-		fieldKey := field.Tag.Get(PICKLE_TAG)
-		if len(fieldKey) == 0 {
-			fieldKey = field.Name
-		}
-		p.dumpString(fieldKey)
-
-		fieldValue := v.Field(i)
-		err := p.dump(fieldValue.Interface())
-		if err != nil {
-			return err
-		}
-
-	}
-	p.pushOpcode(OPCODE_SETITEMS)
-	return nil
-}
-
-func (p *Pickler) pushProxy(proxy pickleProxy) {
-	p.program = append(p.program, proxy)
-}
-
-func (p *Pickler) dumpFloat(v float64) {
-	p.pushProxy(floatProxy(v))
-}
-
-type opcodeProxy uint8
-
-func (proxy opcodeProxy) WriteTo(w io.Writer) (int, error) {
-	return w.Write([]byte{byte(proxy)})
-}
-
-func (p *Pickler) pushOpcode(code uint8) {
-	p.pushProxy(opcodeProxy(code))
-}
-
-type bigIntProxy struct {
-	v *big.Int
-}
-
-var zeroPad = []byte{0}
-var maxPad = []byte{0xff}
-
-func (proxy bigIntProxy) WriteTo(w io.Writer) (int, error) {
-	var negative = proxy.v.Sign() == -1
-	var raw []byte
-	if negative {
-		offset := big.NewInt(1)
-
-		bitLen := uint(proxy.v.BitLen())
-		remainder := bitLen % 8
-		bitLen += 8 - remainder
-
-		offset.Lsh(offset, bitLen)
-
-		offset.Add(proxy.v, offset)
-
-		raw = offset.Bytes()
-	} else {
-		raw = proxy.v.Bytes()
-	}
-
-	var pad []byte
-	var padL int
-	var highBitSet = (raw[0] & 0x80) != 0
-
-	if negative && !highBitSet {
-		pad = maxPad
-		padL = 1
-	} else if !negative && highBitSet {
-		pad = zeroPad
-		padL = 1
-	}
-
-	l := len(raw)
-	var header interface{}
-	if l < 256 {
-		header = struct {
-			Opcode uint8
-			Length uint8
-		}{
-			OPCODE_LONG1,
-			uint8(l + padL),
-		}
-	} else {
-		header = struct {
-			Opcode uint8
-			Length uint32
-		}{
-			OPCODE_LONG4,
-			uint32(l + padL),
-		}
-	}
-
-	err := binary.Write(w, binary.LittleEndian, header)
-	if err != nil {
-		return 0, err
-	}
-
-	n := binary.Size(header)
-	n += l
-	n += padL
-
-	reversed := make([]byte, l)
-
-	for i, v := range raw {
-		reversed[l-i-1] = v
-	}
-
-	_, err = w.Write(reversed)
-	if err != nil {
-		return n, err
-	}
-
-	_, err = w.Write(pad)
-
-	return n, err
-}
-
-func (p *Pickler) dumpIntAsLong(v int64) {
-	p.pushProxy(bigIntProxy{big.NewInt(v)})
-}
-
-func (p *Pickler) dumpBigInt(v big.Int) {
-	p.pushProxy(bigIntProxy{&v}) //Note that this is a shallow copy
-}
-
-func (p *Pickler) dumpUintAsLong(v uint64) {
-	w := big.NewInt(0)
-	w.SetUint64(v)
-	p.pushProxy(bigIntProxy{w})
-}
-
-type floatProxy float64
-
-func (proxy floatProxy) WriteTo(w io.Writer) (int, error) {
-	data := struct {
-		Opcode uint8
-		V      float64
-	}{
-		OPCODE_BINFLOAT,
-		float64(proxy),
-	}
-
-	return binary.Size(data), binary.Write(w, binary.BigEndian, data)
-}
-
-type intProxy int32
-
-func (proxy intProxy) WriteTo(w io.Writer) (int, error) {
-	data := struct {
-		Opcode uint8
-		V      int32
-	}{
-		OPCODE_BININT,
-		int32(proxy),
-	}
-
-	return binary.Size(data), binary.Write(w, binary.LittleEndian, data)
-}
-
-func (p *Pickler) dumpInt(v int64) {
-	p.pushProxy(intProxy(v))
-}
-
-type stringProxy string
-
-func (proxy stringProxy) V() interface{} {
-	return proxy
-}
-
-func (proxy stringProxy) WriteTo(w io.Writer) (int, error) {
-	header := struct {
-		Opcode uint8
-		Length int32
-	}{
-		OPCODE_BINUNICODE,
-		int32(len(proxy)),
-	}
-	err := binary.Write(w, binary.LittleEndian, header)
-	if err != nil {
-		return 0, err
-	}
-	n := binary.Size(header)
-
-	m, err := io.WriteString(w, string(proxy))
-	if err != nil {
-		return 0, err
-	}
-
-	return n + m, nil
-
-}
-
-func (p *Pickler) dumpString(v string) {
-	p.pushProxy(stringProxy(v))
-}
diff --git a/vendor/github.com/hydrogen18/stalecucumber/populate_jump_list.go b/vendor/github.com/hydrogen18/stalecucumber/populate_jump_list.go
deleted file mode 100644
index a9067ffae9..0000000000
--- a/vendor/github.com/hydrogen18/stalecucumber/populate_jump_list.go
+++ /dev/null
@@ -1,58 +0,0 @@
-package stalecucumber
-
-func populateJumpList(jl *opcodeJumpList) {
-	jl[OPCODE_INT] = (*PickleMachine).opcode_INT
-	jl[OPCODE_LONG] = (*PickleMachine).opcode_LONG
-	jl[OPCODE_STRING] = (*PickleMachine).opcode_STRING
-	jl[OPCODE_NONE] = (*PickleMachine).opcode_NONE
-	jl[OPCODE_UNICODE] = (*PickleMachine).opcode_UNICODE
-	jl[OPCODE_FLOAT] = (*PickleMachine).opcode_FLOAT
-	jl[OPCODE_APPEND] = (*PickleMachine).opcode_APPEND
-	jl[OPCODE_LIST] = (*PickleMachine).opcode_LIST
-	jl[OPCODE_TUPLE] = (*PickleMachine).opcode_TUPLE
-	jl[OPCODE_DICT] = (*PickleMachine).opcode_DICT
-	jl[OPCODE_SETITEM] = (*PickleMachine).opcode_SETITEM
-	jl[OPCODE_POP] = (*PickleMachine).opcode_POP
-	jl[OPCODE_DUP] = (*PickleMachine).opcode_DUP
-	jl[OPCODE_MARK] = (*PickleMachine).opcode_MARK
-	jl[OPCODE_GET] = (*PickleMachine).opcode_GET
-	jl[OPCODE_PUT] = (*PickleMachine).opcode_PUT
-	jl[OPCODE_GLOBAL] = (*PickleMachine).opcode_GLOBAL
-	jl[OPCODE_REDUCE] = (*PickleMachine).opcode_REDUCE
-	jl[OPCODE_BUILD] = (*PickleMachine).opcode_BUILD
-	jl[OPCODE_INST] = (*PickleMachine).opcode_INST
-	jl[OPCODE_STOP] = (*PickleMachine).opcode_STOP
-	jl[OPCODE_PERSID] = (*PickleMachine).opcode_PERSID
-	jl[OPCODE_BININT] = (*PickleMachine).opcode_BININT
-	jl[OPCODE_BININT1] = (*PickleMachine).opcode_BININT1
-	jl[OPCODE_BININT2] = (*PickleMachine).opcode_BININT2
-	jl[OPCODE_BINSTRING] = (*PickleMachine).opcode_BINSTRING
-	jl[OPCODE_SHORT_BINSTRING] = (*PickleMachine).opcode_SHORT_BINSTRING
-	jl[OPCODE_BINUNICODE] = (*PickleMachine).opcode_BINUNICODE
-	jl[OPCODE_BINFLOAT] = (*PickleMachine).opcode_BINFLOAT
-	jl[OPCODE_EMPTY_LIST] = (*PickleMachine).opcode_EMPTY_LIST
-	jl[OPCODE_APPENDS] = (*PickleMachine).opcode_APPENDS
-	jl[OPCODE_EMPTY_TUPLE] = (*PickleMachine).opcode_EMPTY_TUPLE
-	jl[OPCODE_EMPTY_DICT] = (*PickleMachine).opcode_EMPTY_DICT
-	jl[OPCODE_SETITEMS] = (*PickleMachine).opcode_SETITEMS
-	jl[OPCODE_POP_MARK] = (*PickleMachine).opcode_POP_MARK
-	jl[OPCODE_BINGET] = (*PickleMachine).opcode_BINGET
-	jl[OPCODE_LONG_BINGET] = (*PickleMachine).opcode_LONG_BINGET
-	jl[OPCODE_BINPUT] = (*PickleMachine).opcode_BINPUT
-	jl[OPCODE_LONG_BINPUT] = (*PickleMachine).opcode_LONG_BINPUT
-	jl[OPCODE_OBJ] = (*PickleMachine).opcode_OBJ
-	jl[OPCODE_BINPERSID] = (*PickleMachine).opcode_BINPERSID
-	jl[OPCODE_LONG1] = (*PickleMachine).opcode_LONG1
-	jl[OPCODE_LONG4] = (*PickleMachine).opcode_LONG4
-	jl[OPCODE_NEWTRUE] = (*PickleMachine).opcode_NEWTRUE
-	jl[OPCODE_NEWFALSE] = (*PickleMachine).opcode_NEWFALSE
-	jl[OPCODE_TUPLE1] = (*PickleMachine).opcode_TUPLE1
-	jl[OPCODE_TUPLE2] = (*PickleMachine).opcode_TUPLE2
-	jl[OPCODE_TUPLE3] = (*PickleMachine).opcode_TUPLE3
-	jl[OPCODE_EXT1] = (*PickleMachine).opcode_EXT1
-	jl[OPCODE_EXT2] = (*PickleMachine).opcode_EXT2
-	jl[OPCODE_EXT4] = (*PickleMachine).opcode_EXT4
-	jl[OPCODE_NEWOBJ] = (*PickleMachine).opcode_NEWOBJ
-	jl[OPCODE_PROTO] = (*PickleMachine).opcode_PROTO
-}
-
diff --git a/vendor/github.com/hydrogen18/stalecucumber/protocol_0.go b/vendor/github.com/hydrogen18/stalecucumber/protocol_0.go
deleted file mode 100644
index e77dcfd47f..0000000000
--- a/vendor/github.com/hydrogen18/stalecucumber/protocol_0.go
+++ /dev/null
@@ -1,723 +0,0 @@
-package stalecucumber
-
-import "strconv"
-import "fmt"
-import "math/big"
-import "errors"
-
-//import "unicode/utf8"
-import "unicode/utf16"
-
-/**
-Opcode: INT
-Push an integer or bool.
-
-      The argument is a newline-terminated decimal literal string.
-
-      The intent may have been that this always fit in a short Python int,
-      but INT can be generated in pickles written on a 64-bit box that
-      require a Python long on a 32-bit box.  The difference between this
-      and LONG then is that INT skips a trailing 'L', and produces a short
-      int whenever possible.
-
-      Another difference is due to that, when bool was introduced as a
-      distinct type in 2.3, builtin names True and False were also added to
-      2.2.2, mapping to ints 1 and 0.  For compatibility in both directions,
-      True gets pickled as INT + "I01\n", and False as INT + "I00\n".
-      Leading zeroes are never produced for a genuine integer.  The 2.3
-      (and later) unpicklers special-case these and return bool instead;
-      earlier unpicklers ignore the leading "0" and return the int.
-      **
-Stack before: []
-Stack after: [int_or_bool]
-**/
-func (pm *PickleMachine) opcode_INT() error {
-	str, err := pm.readString()
-	if err != nil {
-		return err
-	}
-
-	//check for boolean sentinels
-	if len(str) == 2 {
-		switch str {
-		case "01":
-			pm.push(true)
-			return nil
-		case "00":
-			pm.push(false)
-			return nil
-		default:
-		}
-	}
-
-	n, err := strconv.ParseInt(str, 10, 64)
-	if err != nil {
-		return err
-	}
-
-	pm.push(n)
-	return nil
-}
-
-/**
-Opcode: LONG
-Push a long integer.
-
-      The same as INT, except that the literal ends with 'L', and always
-      unpickles to a Python long.  There doesn't seem a real purpose to the
-      trailing 'L'.
-
-      Note that LONG takes time quadratic in the number of digits when
-      unpickling (this is simply due to the nature of decimal->binary
-      conversion).  Proto 2 added linear-time (in C; still quadratic-time
-      in Python) LONG1 and LONG4 opcodes.
-      **
-Stack before: []
-Stack after: [long]
-**/
-func (pm *PickleMachine) opcode_LONG() error {
-	i := new(big.Int)
-	str, err := pm.readString()
-	if err != nil {
-		return err
-	}
-	if len(str) == 0 {
-		return fmt.Errorf("String for LONG opcode cannot be zero length")
-	}
-
-	last := str[len(str)-1]
-	if last != 'L' {
-		return fmt.Errorf("String for LONG opcode must end with %q not %q", "L", last)
-	}
-	v := str[:len(str)-1]
-	_, err = fmt.Sscan(v, i)
-	if err != nil {
-		return err
-	}
-	pm.push(i)
-	return nil
-}
-
-/**
-Opcode: STRING
-Push a Python string object.
-
-      The argument is a repr-style string, with bracketing quote characters,
-      and perhaps embedded escapes.  The argument extends until the next
-      newline character.
-      **
-Stack before: []
-Stack after: [str]
-**/
-
-var unquoteInputs = []byte{0x27, 0x22, 0x0}
-
-func (pm *PickleMachine) opcode_STRING() error {
-	str, err := pm.readString()
-	if err != nil {
-		return err
-	}
-
-	//For whatever reason, the string is quoted. So the first and last character
-	//should always be the single quote
-	if len(str) < 2 {
-		return fmt.Errorf("For STRING opcode, argument has invalid length %d", len(str))
-	}
-
-	if str[0] != '\'' || str[len(str)-1] != '\'' {
-		return fmt.Errorf("For STRING opcode, argument has poorly formed value %q", str)
-	}
-
-	v := str[1 : len(str)-1]
-
-	f := make([]rune, 0, len(v))
-
-	for len(v) != 0 {
-		var vr rune
-		var replacement string
-		for _, i := range unquoteInputs {
-			vr, _, replacement, err = strconv.UnquoteChar(v, i)
-			if err == nil {
-				break
-			}
-		}
-
-		if err != nil {
-			c := v[0]
-			return fmt.Errorf("Read thus far %q. Failed to unquote character %c error:%v", string(f), c, err)
-		}
-		v = replacement
-
-		f = append(f, vr)
-	}
-
-	pm.push(string(f))
-	return nil
-}
-
-/**
-Opcode: NONE
-Push None on the stack.**
-Stack before: []
-Stack after: [None]
-**/
-func (pm *PickleMachine) opcode_NONE() error {
-	pm.push(PickleNone{})
-	return nil
-}
-
-/**
-Opcode: UNICODE
-Push a Python Unicode string object.
-
-      The argument is a raw-unicode-escape encoding of a Unicode string,
-      and so may contain embedded escape sequences.  The argument extends
-      until the next newline character.
-      **
-Stack before: []
-Stack after: [unicode]
-**/
-func (pm *PickleMachine) opcode_UNICODE() error {
-	str, err := pm.readBytes()
-	if err != nil {
-		return err
-	}
-
-	f := make([]rune, 0, len(str))
-
-	var total int
-	var consumed int
-	total = len(str)
-	for total != consumed {
-		h := str[consumed]
-
-		//Python 'raw-unicode-escape' doesnt
-		//escape extended ascii
-		if h > 127 {
-			ea := utf16.Decode([]uint16{uint16(h)})
-			f = append(f, ea...)
-			consumed += 1
-			continue
-		}
-
-		//Multibyte unicode points are escaped
-		//so use "UnquoteChar" to handle those
-		var vr rune
-		for _, i := range unquoteInputs {
-			pre := string(str[consumed:])
-			var post string
-			vr, _, post, err = strconv.UnquoteChar(pre, i)
-			if err == nil {
-				consumed += len(pre) - len(post)
-				break
-			}
-
-		}
-
-		if err != nil {
-			c := str[0]
-			return fmt.Errorf("Read thus far %q. Failed to unquote character %c error:%v", string(f), c, err)
-		}
-
-		f = append(f, vr)
-	}
-
-	pm.push(string(f))
-
-	return nil
-}
-
-/**
-Opcode: FLOAT
-Newline-terminated decimal float literal.
-
-      The argument is repr(a_float), and in general requires 17 significant
-      digits for roundtrip conversion to be an identity (this is so for
-      IEEE-754 double precision values, which is what Python float maps to
-      on most boxes).
-
-      In general, FLOAT cannot be used to transport infinities, NaNs, or
-      minus zero across boxes (or even on a single box, if the platform C
-      library can't read the strings it produces for such things -- Windows
-      is like that), but may do less damage than BINFLOAT on boxes with
-      greater precision or dynamic range than IEEE-754 double.
-      **
-Stack before: []
-Stack after: [float]
-**/
-func (pm *PickleMachine) opcode_FLOAT() error {
-	str, err := pm.readString()
-	if err != nil {
-		return err
-	}
-	var v float64
-	_, err = fmt.Sscanf(str, "%f", &v)
-	if err != nil {
-		return err
-	}
-	pm.push(v)
-	return nil
-}
-
-/**
-Opcode: APPEND
-Append an object to a list.
-
-      Stack before:  ... pylist anyobject
-      Stack after:   ... pylist+[anyobject]
-
-      although pylist is really extended in-place.
-      **
-Stack before: [list, any]
-Stack after: [list]
-**/
-func (pm *PickleMachine) opcode_APPEND() error {
-	v, err := pm.pop()
-	if err != nil {
-		return err
-	}
-
-	listI, err := pm.pop()
-	if err != nil {
-		return err
-	}
-
-	list, ok := listI.([]interface{})
-	if !ok {
-		fmt.Errorf("Second item on top of stack must be of %T not %T", list, listI)
-	}
-	list = append(list, v)
-	pm.push(list)
-	return nil
-}
-
-/**
-Opcode: LIST
-Build a list out of the topmost stack slice, after markobject.
-
-      All the stack entries following the topmost markobject are placed into
-      a single Python list, which single list object replaces all of the
-      stack from the topmost markobject onward.  For example,
-
-      Stack before: ... markobject 1 2 3 'abc'
-      Stack after:  ... [1, 2, 3, 'abc']
-      **
-Stack before: [mark, stackslice]
-Stack after: [list]
-**/
-func (pm *PickleMachine) opcode_LIST() error {
-	markIndex, err := pm.findMark()
-	if err != nil {
-		return err
-	}
-	v := make([]interface{}, 0)
-	for i := markIndex + 1; i != len(pm.Stack); i++ {
-		v = append(v, pm.Stack[i])
-	}
-
-	//Pop the values off the stack
-	pm.popAfterIndex(markIndex)
-
-	pm.push(v)
-	return nil
-}
-
-/**
-Opcode: TUPLE
-Build a tuple out of the topmost stack slice, after markobject.
-
-      All the stack entries following the topmost markobject are placed into
-      a single Python tuple, which single tuple object replaces all of the
-      stack from the topmost markobject onward.  For example,
-
-      Stack before: ... markobject 1 2 3 'abc'
-      Stack after:  ... (1, 2, 3, 'abc')
-      **
-Stack before: [mark, stackslice]
-Stack after: [tuple]
-**/
-func (pm *PickleMachine) opcode_TUPLE() error {
-	return pm.opcode_LIST()
-}
-
-/**
-Opcode: DICT
-Build a dict out of the topmost stack slice, after markobject.
-
-      All the stack entries following the topmost markobject are placed into
-      a single Python dict, which single dict object replaces all of the
-      stack from the topmost markobject onward.  The stack slice alternates
-      key, value, key, value, ....  For example,
-
-      Stack before: ... markobject 1 2 3 'abc'
-      Stack after:  ... {1: 2, 3: 'abc'}
-      **
-Stack before: [mark, stackslice]
-Stack after: [dict]
-**/
-func (pm *PickleMachine) opcode_DICT() (err error) {
-	defer func() {
-		if r := recover(); r != nil {
-			switch x := r.(type) {
-			case string:
-				err = errors.New(x)
-			case error:
-				err = x
-			default:
-				err = errors.New("Unknown panic")
-			}
-		}
-	}()
-	markIndex, err := pm.findMark()
-	if err != nil {
-		return err
-	}
-
-	v := make(map[interface{}]interface{})
-	var key interface{}
-	for i := markIndex + 1; i != len(pm.Stack); i++ {
-		if key == nil {
-			key = pm.Stack[i]
-		} else {
-			v[key] = pm.Stack[i]
-			key = nil
-		}
-	}
-	if key != nil {
-		return fmt.Errorf("For opcode DICT stack after mark contained an odd number of items, this is not valid")
-	}
-	pm.popAfterIndex(markIndex)
-
-	pm.push(v)
-	return nil
-}
-
-/**
-Opcode: SETITEM
-Add a key+value pair to an existing dict.
-
-      Stack before:  ... pydict key value
-      Stack after:   ... pydict
-
-      where pydict has been modified via pydict[key] = value.
-      **
-Stack before: [dict, any, any]
-Stack after: [dict]
-**/
-func (pm *PickleMachine) opcode_SETITEM() (err error) {
-	defer func() {
-		if r := recover(); r != nil {
-			switch x := r.(type) {
-			case string:
-				err = errors.New(x)
-			case error:
-				err = x
-			default:
-				err = errors.New("Unknown panic")
-			}
-		}
-	}()
-	v, err := pm.pop()
-	if err != nil {
-		return err
-	}
-
-	k, err := pm.pop()
-	if err != nil {
-		return err
-	}
-
-	dictI, err := pm.pop()
-	if err != nil {
-		return err
-	}
-
-	dict, ok := dictI.(map[interface{}]interface{})
-	if !ok {
-		return fmt.Errorf("For opcode SETITEM stack item 2 from top must be of type %T not %T", dict, dictI)
-	}
-
-	dict[k] = v
-	pm.push(dict)
-
-	return nil
-}
-
-/**
-Opcode: POP
-Discard the top stack item, shrinking the stack by one item.**
-Stack before: [any]
-Stack after: []
-**/
-func (pm *PickleMachine) opcode_POP() error {
-	_, err := pm.pop()
-	return err
-
-}
-
-/**
-Opcode: DUP
-Push the top stack item onto the stack again, duplicating it.**
-Stack before: [any]
-Stack after: [any, any]
-**/
-func (pm *PickleMachine) opcode_DUP() error {
-	return ErrOpcodeNotImplemented
-}
-
-/**
-Opcode: MARK
-Push markobject onto the stack.
-
-      markobject is a unique object, used by other opcodes to identify a
-      region of the stack containing a variable number of objects for them
-      to work on.  See markobject.doc for more detail.
-      **
-Stack before: []
-Stack after: [mark]
-**/
-func (pm *PickleMachine) opcode_MARK() error {
-	pm.lastMark = len(pm.Stack)
-	pm.push(PickleMark{})
-	return nil
-}
-
-/**
-Opcode: GET
-Read an object from the memo and push it on the stack.
-
-      The index of the memo object to push is given by the newline-terminated
-      decimal string following.  BINGET and LONG_BINGET are space-optimized
-      versions.
-      **
-Stack before: []
-Stack after: [any]
-**/
-func (pm *PickleMachine) opcode_GET() error {
-	str, err := pm.readString()
-	if err != nil {
-		return err
-	}
-
-	index, err := strconv.Atoi(str)
-	if err != nil {
-		return err
-	}
-
-	v, err := pm.readFromMemo(int64(index))
-	if err != nil {
-		return err
-	}
-
-	//TODO test if the object we are about to push is mutable
-	//if so it needs to be somehow deep copied first
-	pm.push(v)
-	return nil
-}
-
-/**
-Opcode: PUT
-Store the stack top into the memo.  The stack is not popped.
-
-      The index of the memo location to write into is given by the newline-
-      terminated decimal string following.  BINPUT and LONG_BINPUT are
-      space-optimized versions.
-      **
-Stack before: []
-Stack after: []
-**/
-func (pm *PickleMachine) opcode_PUT() error {
-	if len(pm.Stack) < 1 {
-		return ErrStackTooSmall
-	}
-
-	str, err := pm.readString()
-	if err != nil {
-		return err
-	}
-
-	idx, err := strconv.Atoi(str)
-	if err != nil {
-		return err
-	}
-
-	pm.storeMemo(int64(idx), pm.Stack[len(pm.Stack)-1])
-
-	return nil
-}
-
-/**
-Opcode: GLOBAL
-Push a global object (module.attr) on the stack.
-
-      Two newline-terminated strings follow the GLOBAL opcode.  The first is
-      taken as a module name, and the second as a class name.  The class
-      object module.class is pushed on the stack.  More accurately, the
-      object returned by self.find_class(module, class) is pushed on the
-      stack, so unpickling subclasses can override this form of lookup.
-      **
-Stack before: []
-Stack after: [any]
-**/
-func (pm *PickleMachine) opcode_GLOBAL() error {
-	//TODO push an object that represents the result of this operation
-	return ErrOpcodeNotImplemented
-}
-
-/**
-Opcode: REDUCE
-Push an object built from a callable and an argument tuple.
-
-      The opcode is named to remind of the __reduce__() method.
-
-      Stack before: ... callable pytuple
-      Stack after:  ... callable(*pytuple)
-
-      The callable and the argument tuple are the first two items returned
-      by a __reduce__ method.  Applying the callable to the argtuple is
-      supposed to reproduce the original object, or at least get it started.
-      If the __reduce__ method returns a 3-tuple, the last component is an
-      argument to be passed to the object's __setstate__, and then the REDUCE
-      opcode is followed by code to create setstate's argument, and then a
-      BUILD opcode to apply  __setstate__ to that argument.
-
-      If type(callable) is not ClassType, REDUCE complains unless the
-      callable has been registered with the copy_reg module's
-      safe_constructors dict, or the callable has a magic
-      '__safe_for_unpickling__' attribute with a true value.  I'm not sure
-      why it does this, but I've sure seen this complaint often enough when
-      I didn't want to <wink>.
-      **
-Stack before: [any, any]
-Stack after: [any]
-**/
-func (pm *PickleMachine) opcode_REDUCE() error {
-	//TODO push an object that represents the result result of this operation
-	return ErrOpcodeNotImplemented
-}
-
-/**
-Opcode: BUILD
-Finish building an object, via __setstate__ or dict update.
-
-      Stack before: ... anyobject argument
-      Stack after:  ... anyobject
-
-      where anyobject may have been mutated, as follows:
-
-      If the object has a __setstate__ method,
-
-          anyobject.__setstate__(argument)
-
-      is called.
-
-      Else the argument must be a dict, the object must have a __dict__, and
-      the object is updated via
-
-          anyobject.__dict__.update(argument)
-
-      This may raise RuntimeError in restricted execution mode (which
-      disallows access to __dict__ directly); in that case, the object
-      is updated instead via
-
-          for k, v in argument.items():
-              anyobject[k] = v
-      **
-Stack before: [any, any]
-Stack after: [any]
-**/
-func (pm *PickleMachine) opcode_BUILD() error {
-	return ErrOpcodeNotImplemented
-}
-
-/**
-Opcode: INST
-Build a class instance.
-
-      This is the protocol 0 version of protocol 1's OBJ opcode.
-      INST is followed by two newline-terminated strings, giving a
-      module and class name, just as for the GLOBAL opcode (and see
-      GLOBAL for more details about that).  self.find_class(module, name)
-      is used to get a class object.
-
-      In addition, all the objects on the stack following the topmost
-      markobject are gathered into a tuple and popped (along with the
-      topmost markobject), just as for the TUPLE opcode.
-
-      Now it gets complicated.  If all of these are true:
-
-        + The argtuple is empty (markobject was at the top of the stack
-          at the start).
-
-        + It's an old-style class object (the type of the class object is
-          ClassType).
-
-        + The class object does not have a __getinitargs__ attribute.
-
-      then we want to create an old-style class instance without invoking
-      its __init__() method (pickle has waffled on this over the years; not
-      calling __init__() is current wisdom).  In this case, an instance of
-      an old-style dummy class is created, and then we try to rebind its
-      __class__ attribute to the desired class object.  If this succeeds,
-      the new instance object is pushed on the stack, and we're done.  In
-      restricted execution mode it can fail (assignment to __class__ is
-      disallowed), and I'm not really sure what happens then -- it looks
-      like the code ends up calling the class object's __init__ anyway,
-      via falling into the next case.
-
-      Else (the argtuple is not empty, it's not an old-style class object,
-      or the class object does have a __getinitargs__ attribute), the code
-      first insists that the class object have a __safe_for_unpickling__
-      attribute.  Unlike as for the __safe_for_unpickling__ check in REDUCE,
-      it doesn't matter whether this attribute has a true or false value, it
-      only matters whether it exists (XXX this is a bug; cPickle
-      requires the attribute to be true).  If __safe_for_unpickling__
-      doesn't exist, UnpicklingError is raised.
-
-      Else (the class object does have a __safe_for_unpickling__ attr),
-      the class object obtained from INST's arguments is applied to the
-      argtuple obtained from the stack, and the resulting instance object
-      is pushed on the stack.
-
-      NOTE:  checks for __safe_for_unpickling__ went away in Python 2.3.
-      **
-Stack before: [mark, stackslice]
-Stack after: [any]
-**/
-func (pm *PickleMachine) opcode_INST() error {
-	return ErrOpcodeNotImplemented
-}
-
-/**
-Opcode: STOP
-Stop the unpickling machine.
-
-      Every pickle ends with this opcode.  The object at the top of the stack
-      is popped, and that's the result of unpickling.  The stack should be
-      empty then.
-      **
-Stack before: [any]
-Stack after: []
-**/
-func (pm *PickleMachine) opcode_STOP() error {
-	return ErrOpcodeStopped
-}
-
-/**
-Opcode: PERSID
-Push an object identified by a persistent ID.
-
-      The pickle module doesn't define what a persistent ID means.  PERSID's
-      argument is a newline-terminated str-style (no embedded escapes, no
-      bracketing quote characters) string, which *is* "the persistent ID".
-      The unpickler passes this string to self.persistent_load().  Whatever
-      object that returns is pushed on the stack.  There is no implementation
-      of persistent_load() in Python's unpickler:  it must be supplied by an
-      unpickler subclass.
-      **
-Stack before: []
-Stack after: [any]
-**/
-func (pm *PickleMachine) opcode_PERSID() error {
-	return ErrOpcodeNotImplemented
-}
diff --git a/vendor/github.com/hydrogen18/stalecucumber/protocol_1.go b/vendor/github.com/hydrogen18/stalecucumber/protocol_1.go
deleted file mode 100644
index 80fb5267a6..0000000000
--- a/vendor/github.com/hydrogen18/stalecucumber/protocol_1.go
+++ /dev/null
@@ -1,523 +0,0 @@
-package stalecucumber
-
-import "fmt"
-import "errors"
-
-/**
-Opcode: BININT (0x4a)
-Push a four-byte signed integer.
-
-      This handles the full range of Python (short) integers on a 32-bit
-      box, directly as binary bytes (1 for the opcode and 4 for the integer).
-      If the integer is non-negative and fits in 1 or 2 bytes, pickling via
-      BININT1 or BININT2 saves space.
-      **
-Stack before: []
-Stack after: [int]
-**/
-func (pm *PickleMachine) opcode_BININT() error {
-	var v int32
-	err := pm.readBinaryInto(&v, false)
-	if err != nil {
-		return err
-	}
-
-	pm.push(int64(v))
-	return nil
-}
-
-/**
-Opcode: BININT1 (0x4b)
-Push a one-byte unsigned integer.
-
-      This is a space optimization for pickling very small non-negative ints,
-      in range(256).
-      **
-Stack before: []
-Stack after: [int]
-**/
-func (pm *PickleMachine) opcode_BININT1() error {
-	var v uint8
-	err := pm.readBinaryInto(&v, false)
-	if err != nil {
-		return err
-	}
-	pm.push(int64(v))
-	return nil
-}
-
-/**
-Opcode: BININT2 (0x4d)
-Push a two-byte unsigned integer.
-
-      This is a space optimization for pickling small positive ints, in
-      range(256, 2**16).  Integers in range(256) can also be pickled via
-      BININT2, but BININT1 instead saves a byte.
-      **
-Stack before: []
-Stack after: [int]
-**/
-func (pm *PickleMachine) opcode_BININT2() error {
-	var v uint16
-	err := pm.readBinaryInto(&v, false)
-	if err != nil {
-		return err
-	}
-	pm.push(int64(v))
-	return nil
-
-}
-
-/**
-Opcode: BINSTRING (0x54)
-Push a Python string object.
-
-      There are two arguments:  the first is a 4-byte little-endian signed int
-      giving the number of bytes in the string, and the second is that many
-      bytes, which are taken literally as the string content.
-      **
-Stack before: []
-Stack after: [str]
-**/
-func (pm *PickleMachine) opcode_BINSTRING() error {
-	var strlen int32
-	err := pm.readBinaryInto(&strlen, false)
-	if err != nil {
-		return err
-	}
-
-	if strlen < 0 {
-		return fmt.Errorf("BINSTRING specified negative string length of %d", strlen)
-	}
-
-	str, err := pm.readFixedLengthString(int64(strlen))
-	if err != nil {
-		return err
-	}
-	pm.push(str)
-	return nil
-}
-
-/**
-Opcode: SHORT_BINSTRING (0x55)
-Push a Python string object.
-
-      There are two arguments:  the first is a 1-byte unsigned int giving
-      the number of bytes in the string, and the second is that many bytes,
-      which are taken literally as the string content.
-      **
-Stack before: []
-Stack after: [str]
-**/
-func (pm *PickleMachine) opcode_SHORT_BINSTRING() error {
-	var strlen uint8
-	err := pm.readBinaryInto(&strlen, false)
-	if err != nil {
-		return err
-	}
-
-	if strlen < 0 {
-		return fmt.Errorf("SHORT_BINSTRING specified negative string length of %d", strlen)
-	}
-
-	str, err := pm.readFixedLengthString(int64(strlen))
-	if err != nil {
-		return err
-	}
-	pm.push(str)
-	return nil
-}
-
-/**
-Opcode: BINUNICODE (0x58)
-Push a Python Unicode string object.
-
-      There are two arguments:  the first is a 4-byte little-endian signed int
-      giving the number of bytes in the string.  The second is that many
-      bytes, and is the UTF-8 encoding of the Unicode string.
-      **
-Stack before: []
-Stack after: [unicode]
-**/
-func (pm *PickleMachine) opcode_BINUNICODE() error {
-	var l int32
-	err := pm.readBinaryInto(&l, false)
-	if err != nil {
-		return err
-	}
-
-	str, err := pm.readFixedLengthString(int64(l))
-	if err != nil {
-		return err
-	}
-
-	pm.push(str)
-	return nil
-}
-
-/**
-Opcode: BINFLOAT (0x47)
-Float stored in binary form, with 8 bytes of data.
-
-      This generally requires less than half the space of FLOAT encoding.
-      In general, BINFLOAT cannot be used to transport infinities, NaNs, or
-      minus zero, raises an exception if the exponent exceeds the range of
-      an IEEE-754 double, and retains no more than 53 bits of precision (if
-      there are more than that, "add a half and chop" rounding is used to
-      cut it back to 53 significant bits).
-      **
-Stack before: []
-Stack after: [float]
-**/
-func (pm *PickleMachine) opcode_BINFLOAT() error {
-	var v float64
-	err := pm.readBinaryInto(&v, true)
-	if err != nil {
-		return err
-	}
-
-	pm.push(v)
-	return nil
-
-}
-
-/**
-Opcode: EMPTY_LIST (0x5d)
-Push an empty list.**
-Stack before: []
-Stack after: [list]
-**/
-func (pm *PickleMachine) opcode_EMPTY_LIST() error {
-	v := make([]interface{}, 0)
-	pm.push(v)
-	return nil
-}
-
-/**
-Opcode: APPENDS (0x65)
-Extend a list by a slice of stack objects.
-
-      Stack before:  ... pylist markobject stackslice
-      Stack after:   ... pylist+stackslice
-
-      although pylist is really extended in-place.
-      **
-Stack before: [list, mark, stackslice]
-Stack after: [list]
-**/
-func (pm *PickleMachine) opcode_APPENDS() error {
-	markIndex, err := pm.findMark()
-	if err != nil {
-		return err
-	}
-
-	pyListI, err := pm.readFromStackAt(markIndex - 1)
-	if err != nil {
-		return err
-	}
-
-	pyList, ok := pyListI.([]interface{})
-	if !ok {
-		return fmt.Errorf("APPENDS expected type %T but got (%v)%T", pyList, pyListI, pyListI)
-	}
-
-	pyList = append(pyList, pm.Stack[markIndex+1:]...)
-	pm.popAfterIndex(markIndex - 1)
-
-	/**
-	if err != nil {
-		return err
-	}**/
-
-	pm.push(pyList)
-	return nil
-}
-
-/**
-Opcode: EMPTY_TUPLE (0x29)
-Push an empty tuple.**
-Stack before: []
-Stack after: [tuple]
-**/
-func (pm *PickleMachine) opcode_EMPTY_TUPLE() error {
-	return pm.opcode_EMPTY_LIST()
-}
-
-/**
-Opcode: EMPTY_DICT (0x7d)
-Push an empty dict.**
-Stack before: []
-Stack after: [dict]
-**/
-func (pm *PickleMachine) opcode_EMPTY_DICT() error {
-	pm.push(make(map[interface{}]interface{}))
-	return nil
-}
-
-/**
-Opcode: SETITEMS (0x75)
-Add an arbitrary number of key+value pairs to an existing dict.
-
-      The slice of the stack following the topmost markobject is taken as
-      an alternating sequence of keys and values, added to the dict
-      immediately under the topmost markobject.  Everything at and after the
-      topmost markobject is popped, leaving the mutated dict at the top
-      of the stack.
-
-      Stack before:  ... pydict markobject key_1 value_1 ... key_n value_n
-      Stack after:   ... pydict
-
-      where pydict has been modified via pydict[key_i] = value_i for i in
-      1, 2, ..., n, and in that order.
-      **
-Stack before: [dict, mark, stackslice]
-Stack after: [dict]
-**/
-func (pm *PickleMachine) opcode_SETITEMS() (err error) {
-	defer func() {
-		if r := recover(); r != nil {
-			switch x := r.(type) {
-			case string:
-				err = errors.New(x)
-			case error:
-				err = x
-			default:
-				err = errors.New("Unknown panic")
-			}
-		}
-	}()
-	markIndex, err := pm.findMark()
-	if err != nil {
-		return err
-	}
-
-	vI, err := pm.readFromStackAt(markIndex - 1)
-	if err != nil {
-		return err
-	}
-
-	v, ok := vI.(map[interface{}]interface{})
-	if !ok {
-		return fmt.Errorf("Opcode SETITEMS expected type %T on stack but found %v(%T)", v, vI, vI)
-	}
-
-	if ((len(pm.Stack) - markIndex + 1) % 2) != 0 {
-		return fmt.Errorf("Found odd number of items on stack after mark:%d", len(pm.Stack)-markIndex+1)
-	}
-
-	for i := markIndex + 1; i != len(pm.Stack); i++ {
-		key := pm.Stack[i]
-		i++
-		v[key] = pm.Stack[i]
-	}
-
-	pm.popAfterIndex(markIndex)
-
-	return nil
-}
-
-/**
-Opcode: POP_MARK (0x31)
-Pop all the stack objects at and above the topmost markobject.
-
-      When an opcode using a variable number of stack objects is done,
-      POP_MARK is used to remove those objects, and to remove the markobject
-      that delimited their starting position on the stack.
-      **
-Stack before: [mark, stackslice]
-Stack after: []
-**/
-func (pm *PickleMachine) opcode_POP_MARK() error {
-	markIndex, err := pm.findMark()
-	if err != nil {
-		return nil
-	}
-	pm.popAfterIndex(markIndex)
-	return nil
-}
-
-/**
-Opcode: BINGET (0x68)
-Read an object from the memo and push it on the stack.
-
-      The index of the memo object to push is given by the 1-byte unsigned
-      integer following.
-      **
-Stack before: []
-Stack after: [any]
-**/
-func (pm *PickleMachine) opcode_BINGET() (err error) {
-	defer func() {
-		if r := recover(); r != nil {
-			switch x := r.(type) {
-			case string:
-				err = errors.New(x)
-			case error:
-				err = x
-			default:
-				err = errors.New("Unknown panic")
-			}
-		}
-	}()
-	var index uint8
-	err = pm.readBinaryInto(&index, false)
-	if err != nil {
-		return err
-	}
-
-	v, err := pm.readFromMemo(int64(index))
-	if err != nil {
-		return err
-	}
-
-	//TODO test if the object we are about to push is mutable
-	//if so it needs to be somehow deep copied first
-	pm.push(v)
-
-	return nil
-}
-
-/**
-Opcode: LONG_BINGET (0x6a)
-Read an object from the memo and push it on the stack.
-
-      The index of the memo object to push is given by the 4-byte signed
-      little-endian integer following.
-      **
-Stack before: []
-Stack after: [any]
-**/
-func (pm *PickleMachine) opcode_LONG_BINGET() (err error) {
-	defer func() {
-		if r := recover(); r != nil {
-			switch x := r.(type) {
-			case string:
-				err = errors.New(x)
-			case error:
-				err = x
-			default:
-				err = errors.New("Unknown panic")
-			}
-		}
-	}()
-	var index int32
-	err = pm.readBinaryInto(&index, false)
-	if err != nil {
-		return err
-	}
-
-	v, err := pm.readFromMemo(int64(index))
-	if err != nil {
-		return err
-	}
-
-	//TODO test if the object we are about to push is mutable
-	//if so it needs to be somehow deep copied first
-	pm.push(v)
-	return nil
-}
-
-/**
-Opcode: BINPUT (0x71)
-Store the stack top into the memo.  The stack is not popped.
-
-      The index of the memo location to write into is given by the 1-byte
-      unsigned integer following.
-      **
-Stack before: []
-Stack after: []
-**/
-func (pm *PickleMachine) opcode_BINPUT() error {
-	v, err := pm.readFromStack(0)
-	if err != nil {
-		return err
-	}
-
-	var index uint8
-	err = pm.readBinaryInto(&index, false)
-	if err != nil {
-		return err
-	}
-
-	pm.storeMemo(int64(index), v)
-	return nil
-}
-
-/**
-Opcode: LONG_BINPUT (0x72)
-Store the stack top into the memo.  The stack is not popped.
-
-      The index of the memo location to write into is given by the 4-byte
-      signed little-endian integer following.
-      **
-Stack before: []
-Stack after: []
-**/
-func (pm *PickleMachine) opcode_LONG_BINPUT() error {
-	var index int32
-	err := pm.readBinaryInto(&index, false)
-	if err != nil {
-		return err
-	}
-
-	v, err := pm.readFromStack(0)
-	if err != nil {
-		return err
-	}
-	err = pm.storeMemo(int64(index), v)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-/**
-Opcode: OBJ (0x6f)
-Build a class instance.
-
-      This is the protocol 1 version of protocol 0's INST opcode, and is
-      very much like it.  The major difference is that the class object
-      is taken off the stack, allowing it to be retrieved from the memo
-      repeatedly if several instances of the same class are created.  This
-      can be much more efficient (in both time and space) than repeatedly
-      embedding the module and class names in INST opcodes.
-
-      Unlike INST, OBJ takes no arguments from the opcode stream.  Instead
-      the class object is taken off the stack, immediately above the
-      topmost markobject:
-
-      Stack before: ... markobject classobject stackslice
-      Stack after:  ... new_instance_object
-
-      As for INST, the remainder of the stack above the markobject is
-      gathered into an argument tuple, and then the logic seems identical,
-      except that no __safe_for_unpickling__ check is done (XXX this is
-      a bug; cPickle does test __safe_for_unpickling__).  See INST for
-      the gory details.
-
-      NOTE:  In Python 2.3, INST and OBJ are identical except for how they
-      get the class object.  That was always the intent; the implementations
-      had diverged for accidental reasons.
-      **
-Stack before: [mark, any, stackslice]
-Stack after: [any]
-**/
-func (pm *PickleMachine) opcode_OBJ() error {
-	return ErrOpcodeNotImplemented
-}
-
-/**
-Opcode: BINPERSID (0x51)
-Push an object identified by a persistent ID.
-
-      Like PERSID, except the persistent ID is popped off the stack (instead
-      of being a string embedded in the opcode bytestream).  The persistent
-      ID is passed to self.persistent_load(), and whatever object that
-      returns is pushed on the stack.  See PERSID for more detail.
-      **
-Stack before: [any]
-Stack after: [any]
-**/
-func (pm *PickleMachine) opcode_BINPERSID() error {
-	return ErrOpcodeNotImplemented
-}
diff --git a/vendor/github.com/hydrogen18/stalecucumber/protocol_2.go b/vendor/github.com/hydrogen18/stalecucumber/protocol_2.go
deleted file mode 100644
index 583dfe88e2..0000000000
--- a/vendor/github.com/hydrogen18/stalecucumber/protocol_2.go
+++ /dev/null
@@ -1,322 +0,0 @@
-package stalecucumber
-
-import "fmt"
-import "math/big"
-
-/**
-Opcode: LONG1 (0x8a)
-Long integer using one-byte length.
-	An arbitrary length integer encoded as a bytestring.
-	A single byte following the opcode indicates the length
-	of the bytestring
-	If the string length is zero, then the value is zero.
-	Otherwise the bytestring is 256-complement representation
-	of an integer in reverse.
-      **
-Stack before: []
-Stack after: [long]
-**/
-func (pm *PickleMachine) opcode_LONG1() error {
-	var l uint8
-	err := pm.readBinaryInto(&l, false)
-	if err != nil {
-		return err
-	}
-
-	if l == 0 {
-		pm.push(big.NewInt(0))
-		return nil
-	}
-
-	reversedData, err := pm.readFixedLengthRaw(int64(l))
-	if err != nil {
-		return err
-	}
-
-	//For no obvious reason, the python pickler
-	//always reverses the bytes. Reverse it here
-	var data [256]byte
-	{
-		var j int
-		for i := len(reversedData) - 1; i != -1; i-- {
-			data[j] = reversedData[i]
-			j++
-		}
-	}
-
-	v := new(big.Int)
-	v.SetBytes(data[:l])
-
-	invertIfNegative(data[0], v, int(l))
-
-	pm.push(v)
-
-	return nil
-
-}
-
-func invertIfNegative(first byte, v *big.Int, l int) {
-	var negative bool
-	//Check for negative number.
-	negative = (0x80 & first) != 0x0
-
-	if negative {
-		offset := big.NewInt(1)
-		offset.Lsh(offset, uint(l*8))
-		v.Sub(v, offset)
-	}
-
-}
-
-/**
-Opcode: LONG4 (0x8b)
-Long integer using found-byte length.
-
-    An arbitrary length integer encoded as a bytestring.
-	A four-byte signed little-endian integer
-	following the opcode indicates the length  of the bytestring.
-	If the string length is zero, then the value is zero.
-	Otherwise the bytestring is 256-complement representation
-	of an integer in reverse.**
-Stack before: []
-Stack after: [long]
-**/
-func (pm *PickleMachine) opcode_LONG4() error {
-	var l uint32
-	err := pm.readBinaryInto(&l, false)
-	if err != nil {
-		return err
-	}
-
-	if l == 0 {
-		pm.push(big.NewInt(0))
-		return nil
-	}
-
-	reversedData, err := pm.readFixedLengthRaw(int64(l))
-	if err != nil {
-		return err
-	}
-
-	//For no obvious reason, the python pickler
-	//always reverses the bytes. Reverse it here
-	data := make([]byte, len(reversedData))
-	{
-		var j int
-		for i := len(reversedData) - 1; i != -1; i-- {
-			data[j] = reversedData[i]
-			j++
-		}
-	}
-
-	v := new(big.Int)
-	v.SetBytes(data[:l])
-
-	invertIfNegative(data[0], v, len(data))
-
-	pm.push(v)
-
-	return nil
-
-}
-
-/**
-Opcode: NEWTRUE (0x88)
-True.
-
-      Push True onto the stack.**
-Stack before: []
-Stack after: [bool]
-**/
-func (pm *PickleMachine) opcode_NEWTRUE() error {
-	pm.push(true)
-	return nil
-}
-
-/**
-Opcode: NEWFALSE (0x89)
-True.
-
-      Push False onto the stack.**
-Stack before: []
-Stack after: [bool]
-**/
-func (pm *PickleMachine) opcode_NEWFALSE() error {
-	pm.push(false)
-	return nil
-}
-
-/**
-Opcode: TUPLE1 (0x85)
-Build a one-tuple out of the topmost item on the stack.
-
-      This code pops one value off the stack and pushes a tuple of
-      length 1 whose one item is that value back onto it.  In other
-      words:
-
-          stack[-1] = tuple(stack[-1:])
-      **
-Stack before: [any]
-Stack after: [tuple]
-**/
-func (pm *PickleMachine) opcode_TUPLE1() error {
-	v, err := pm.pop()
-	if err != nil {
-		return err
-	}
-
-	pm.push([]interface{}{v})
-	return nil
-}
-
-/**
-Opcode: TUPLE2 (0x86)
-Build a two-tuple out of the top two items on the stack.
-
-      This code pops two values off the stack and pushes a tuple of
-      length 2 whose items are those values back onto it.  In other
-      words:
-
-          stack[-2:] = [tuple(stack[-2:])]
-      **
-Stack before: [any, any]
-Stack after: [tuple]
-**/
-func (pm *PickleMachine) opcode_TUPLE2() error {
-	v := make([]interface{}, 2)
-	var err error
-	v[1], err = pm.pop()
-	if err != nil {
-		return err
-	}
-	v[0], err = pm.pop()
-	if err != nil {
-		return err
-	}
-
-	pm.push(v)
-	return nil
-}
-
-/**
-Opcode: TUPLE3 (0x87)
-Build a three-tuple out of the top three items on the stack.
-
-      This code pops three values off the stack and pushes a tuple of
-      length 3 whose items are those values back onto it.  In other
-      words:
-
-          stack[-3:] = [tuple(stack[-3:])]
-      **
-Stack before: [any, any, any]
-Stack after: [tuple]
-**/
-func (pm *PickleMachine) opcode_TUPLE3() error {
-	v := make([]interface{}, 3)
-	var err error
-	v[2], err = pm.pop()
-	if err != nil {
-		return err
-	}
-	v[1], err = pm.pop()
-	if err != nil {
-		return err
-	}
-	v[0], err = pm.pop()
-	if err != nil {
-		return err
-	}
-
-	pm.push(v)
-	return nil
-}
-
-/**
-Opcode: EXT1 (0x82)
-Extension code.
-
-      This code and the similar EXT2 and EXT4 allow using a registry
-      of popular objects that are pickled by name, typically classes.
-      It is envisioned that through a global negotiation and
-      registration process, third parties can set up a mapping between
-      ints and object names.
-
-      In order to guarantee pickle interchangeability, the extension
-      code registry ought to be global, although a range of codes may
-      be reserved for private use.
-
-      EXT1 has a 1-byte integer argument.  This is used to index into the
-      extension registry, and the object at that index is pushed on the stack.
-      **
-Stack before: []
-Stack after: [any]
-**/
-func (pm *PickleMachine) opcode_EXT1() error {
-	return ErrOpcodeNotImplemented
-}
-
-/**
-Opcode: EXT2 (0x83)
-Extension code.
-
-      See EXT1.  EXT2 has a two-byte integer argument.
-      **
-Stack before: []
-Stack after: [any]
-**/
-func (pm *PickleMachine) opcode_EXT2() error {
-	return ErrOpcodeNotImplemented
-}
-
-/**
-Opcode: EXT4 (0x84)
-Extension code.
-
-      See EXT1.  EXT4 has a four-byte integer argument.
-      **
-Stack before: []
-Stack after: [any]
-**/
-func (pm *PickleMachine) opcode_EXT4() error {
-	return ErrOpcodeNotImplemented
-}
-
-/**
-Opcode: NEWOBJ (0x81)
-Build an object instance.
-
-      The stack before should be thought of as containing a class
-      object followed by an argument tuple (the tuple being the stack
-      top).  Call these cls and args.  They are popped off the stack,
-      and the value returned by cls.__new__(cls, *args) is pushed back
-      onto the stack.
-      **
-Stack before: [any, any]
-Stack after: [any]
-**/
-func (pm *PickleMachine) opcode_NEWOBJ() error {
-	return ErrOpcodeNotImplemented
-}
-
-/**
-Opcode: PROTO (0x80)
-Protocol version indicator.
-
-      For protocol 2 and above, a pickle must start with this opcode.
-      The argument is the protocol version, an int in range(2, 256).
-      **
-Stack before: []
-Stack after: []
-**/
-func (pm *PickleMachine) opcode_PROTO() error {
-	var version int8
-	err := pm.readBinaryInto(&version, false)
-	if err != nil {
-		return err
-	}
-	if version != 2 {
-		return fmt.Errorf("Unsupported version #%d detected", version)
-	}
-
-	return nil
-}
diff --git a/vendor/github.com/hydrogen18/stalecucumber/python_types.go b/vendor/github.com/hydrogen18/stalecucumber/python_types.go
deleted file mode 100644
index 5213b2e18d..0000000000
--- a/vendor/github.com/hydrogen18/stalecucumber/python_types.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package stalecucumber
-
-/*
-This type is used internally to represent a concept known as a mark
-on the Pickle Machine's stack. Oddly formed pickled data could return
-this value as the result of Unpickle. In normal usage this type
-is needed only internally.
-*/
-type PickleMark struct{}
-
-func (_ PickleMark) String() string {
-	return "PickleMachine Mark"
-}
-
-/*
-This type is used to represent the Python object "None"
-*/
-type PickleNone struct{}
-
-func (_ PickleNone) String() string {
-	return "Python None"
-}
diff --git a/vendor/github.com/hydrogen18/stalecucumber/tuple.go b/vendor/github.com/hydrogen18/stalecucumber/tuple.go
deleted file mode 100644
index 037c2f66c0..0000000000
--- a/vendor/github.com/hydrogen18/stalecucumber/tuple.go
+++ /dev/null
@@ -1,7 +0,0 @@
-package stalecucumber
-
-type PickleTuple []interface{}
-
-func NewTuple(v ...interface{}) PickleTuple {
-	return PickleTuple(v)
-}
diff --git a/vendor/github.com/hydrogen18/stalecucumber/unpack.go b/vendor/github.com/hydrogen18/stalecucumber/unpack.go
deleted file mode 100644
index 64d962a674..0000000000
--- a/vendor/github.com/hydrogen18/stalecucumber/unpack.go
+++ /dev/null
@@ -1,329 +0,0 @@
-package stalecucumber
-
-import "reflect"
-import "fmt"
-import "errors"
-import "strings"
-import "math/big"
-
-const PICKLE_TAG = "pickle"
-
-type UnpackingError struct {
-	Source      interface{}
-	Destination reflect.Value
-	Err         error
-}
-
-/*
-This type is returned when a call to From() fails.
-Setting "AllowMissingFields" and "AllowMismatchedFields"
-on the result of "UnpackInto" controls if this error is
-returned or not.
-*/
-func (ue UnpackingError) Error() string {
-	var dv string
-	var dt string
-	k := ue.Destination.Kind()
-	switch k {
-	case reflect.Ptr:
-		dt = fmt.Sprintf("%s", ue.Destination.Type().Elem())
-		if !ue.Destination.IsNil() {
-			dv = fmt.Sprintf("%v", ue.Destination.Elem().Interface())
-		} else {
-			dv = "nil"
-
-		}
-	case reflect.Invalid:
-		dv = "invalid"
-		dt = dv
-	default:
-		dv = fmt.Sprintf("%v", ue.Destination.Interface())
-		dt = fmt.Sprintf("%s", ue.Destination.Type())
-	}
-
-	return fmt.Sprintf("Error unpacking %v(%T) into %s(%s):%v",
-		ue.Source,
-		ue.Source,
-		dv,
-		dt,
-		ue.Err)
-}
-
-var ErrNilPointer = errors.New("Destination cannot be a nil pointer")
-var ErrNotPointer = errors.New("Destination must be a pointer type")
-var ErrTargetTypeNotPointer = errors.New("Target type must be a pointer to unpack this value")
-var ErrTargetTypeOverflow = errors.New("Value overflows target type")
-var ErrTargetTypeMismatch = errors.New("Target type does not match source type")
-
-type unpacker struct {
-	dest                  reflect.Value
-	AllowMissingFields    bool
-	AllowMismatchedFields bool
-}
-
-func UnpackInto(dest interface{}) unpacker {
-	return unpacker{dest: reflect.ValueOf(dest),
-		AllowMissingFields:    true,
-		AllowMismatchedFields: false}
-}
-
-func (u unpacker) From(srcI interface{}, err error) error {
-	//Check if an error occurred
-	if err != nil {
-		return err
-	}
-
-	return u.from(srcI)
-}
-
-func (u unpacker) from(srcI interface{}) error {
-
-	//Get the value of the destination
-	v := u.dest
-
-	//The destination must always be a pointer
-	if v.Kind() != reflect.Ptr {
-		return UnpackingError{Source: srcI,
-			Destination: u.dest,
-			Err:         ErrNotPointer}
-	}
-
-	//The destination can never be nil
-	if v.IsNil() {
-		return UnpackingError{Source: srcI,
-			Destination: u.dest,
-			Err:         ErrNilPointer}
-
-	}
-
-	//Indirect the destination. This gets the actual
-	//value pointed at
-	vIndirect := v
-	for vIndirect.Kind() == reflect.Ptr {
-		if vIndirect.IsNil() {
-			vIndirect.Set(reflect.New(vIndirect.Type().Elem()))
-		}
-		vIndirect = vIndirect.Elem()
-	}
-
-	//Check the input against known types
-	switch s := srcI.(type) {
-	default:
-		return UnpackingError{Source: srcI,
-			Destination: u.dest,
-			Err:         errors.New("Unknown source type")}
-	case PickleNone:
-		vElem := v.Elem()
-		for vElem.Kind() == reflect.Ptr {
-			next := vElem.Elem()
-			if next.Kind() == reflect.Ptr {
-				vElem = next
-				continue
-			}
-
-			if vElem.CanSet() {
-				vElem.Set(reflect.Zero(vElem.Type()))
-				return nil
-			}
-		}
-
-		return UnpackingError{Source: srcI,
-			Destination: u.dest,
-			Err:         ErrTargetTypeNotPointer}
-
-	case int64:
-		switch vIndirect.Kind() {
-		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int64, reflect.Int32:
-			if vIndirect.OverflowInt(s) {
-				return UnpackingError{Source: srcI,
-					Destination: u.dest,
-					Err:         ErrTargetTypeOverflow}
-			}
-			vIndirect.SetInt(s)
-			return nil
-		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
-			if s < 0 || vIndirect.OverflowUint(uint64(s)) {
-				return UnpackingError{Source: srcI,
-					Destination: u.dest,
-					Err:         ErrTargetTypeOverflow}
-			}
-
-			vIndirect.SetUint(uint64(s))
-			return nil
-		}
-
-		dstBig, ok := vIndirect.Addr().Interface().(*big.Int)
-		if ok {
-			dstBig.SetInt64(s)
-			return nil
-		}
-
-	case string:
-		switch vIndirect.Kind() {
-		case reflect.String:
-			vIndirect.SetString(s)
-			return nil
-		}
-	case bool:
-		switch vIndirect.Kind() {
-		case reflect.Bool:
-			vIndirect.SetBool(s)
-			return nil
-		}
-	case float64:
-		switch vIndirect.Kind() {
-		case reflect.Float32, reflect.Float64:
-			vIndirect.SetFloat(s)
-			return nil
-		}
-	case *big.Int:
-		dstBig, ok := vIndirect.Addr().Interface().(*big.Int)
-		if ok {
-			dstBig.Set(s)
-			return nil
-		}
-
-		if vi, err := Int(srcI, nil); err == nil {
-			return unpacker{dest: v,
-				AllowMismatchedFields: u.AllowMismatchedFields,
-				AllowMissingFields:    u.AllowMissingFields}.From(vi, nil)
-		}
-
-	case []interface{}:
-		//Check that the destination is a slice
-		if vIndirect.Kind() != reflect.Slice {
-			return UnpackingError{Source: s,
-				Destination: u.dest,
-				Err:         fmt.Errorf("Cannot unpack slice into destination")}
-		}
-		//Check for exact type match
-		if vIndirect.Type().Elem().Kind() == reflect.Interface {
-			vIndirect.Set(reflect.ValueOf(s))
-			return nil
-		}
-
-		//Build the value using reflection
-		var replacement reflect.Value
-		if vIndirect.IsNil() || vIndirect.Len() < len(s) {
-			replacement = reflect.MakeSlice(vIndirect.Type(),
-				len(s), len(s))
-		} else {
-			replacement = vIndirect.Slice(0, len(s))
-		}
-
-		for i, srcV := range s {
-			dstV := replacement.Index(i)
-
-			//Recurse to set the value
-			err := unpacker{dest: dstV.Addr(),
-				AllowMissingFields:    u.AllowMissingFields,
-				AllowMismatchedFields: u.AllowMismatchedFields}.
-				From(srcV, nil)
-			if err != nil {
-				return err
-			}
-		}
-		vIndirect.Set(replacement)
-		return nil
-
-	case map[interface{}]interface{}:
-		//Check to see if the field is exactly
-		//of the type
-		if vIndirect.Kind() == reflect.Map {
-			dstT := vIndirect.Type()
-			if dstT.Key().Kind() == reflect.Interface &&
-				dstT.Elem().Kind() == reflect.Interface {
-				vIndirect.Set(reflect.ValueOf(s))
-				return nil
-			}
-		}
-
-		var src map[string]interface{}
-		var err error
-		src, err = DictString(srcI, err)
-		if err != nil {
-			return UnpackingError{Source: srcI,
-				Destination: u.dest,
-				Err:         fmt.Errorf("Cannot unpack source into struct")}
-		}
-
-		if vIndirect.Kind() != reflect.Struct {
-			return UnpackingError{Source: src,
-				Destination: u.dest,
-				Err:         fmt.Errorf("Cannot unpack into %v", v.Kind().String())}
-
-		}
-
-		var fieldByTag map[string]int
-		vIndirectType := reflect.TypeOf(vIndirect.Interface())
-		numFields := vIndirectType.NumField()
-
-		for i := 0; i != numFields; i++ {
-			fv := vIndirectType.Field(i)
-			tag := fv.Tag.Get(PICKLE_TAG)
-
-			if len(tag) != 0 {
-				if fieldByTag == nil {
-					fieldByTag = make(map[string]int)
-				}
-				fieldByTag[tag] = i
-			}
-		}
-
-		for k, kv := range src {
-			var fv reflect.Value
-
-			if fieldIndex, ok := fieldByTag[k]; ok {
-				fv = vIndirect.Field(fieldIndex)
-			} else {
-				//Try the name verbatim. This catches
-				//embedded fields as well
-				fv = vIndirect.FieldByName(k)
-
-				if !fv.IsValid() {
-					//Capitalize the first character. Structs
-					//do not export fields with a lower case
-					//first character
-					capk := strings.ToUpper(k[0:1]) + k[1:]
-					fv = vIndirect.FieldByName(capk)
-				}
-			}
-
-			if !fv.IsValid() || !fv.CanSet() {
-				if !u.AllowMissingFields {
-					return UnpackingError{Source: src,
-						Destination: u.dest,
-						Err:         fmt.Errorf("Cannot find field for key %q", k)}
-				}
-				continue
-			}
-
-			err := unpacker{dest: fv.Addr(),
-				AllowMismatchedFields: u.AllowMismatchedFields,
-				AllowMissingFields:    u.AllowMissingFields}.from(kv)
-
-			if err != nil {
-				if u.AllowMismatchedFields {
-					if unpackingError, ok := err.(UnpackingError); ok {
-						switch unpackingError.Err {
-						case ErrTargetTypeOverflow,
-							ErrTargetTypeNotPointer,
-							ErrTargetTypeMismatch:
-
-							fv.Set(reflect.Zero(fv.Type()))
-							continue
-						}
-
-					}
-				}
-				return err
-			}
-		}
-
-		return nil
-	}
-
-	return UnpackingError{Source: srcI,
-		Destination: u.dest,
-		Err:         ErrTargetTypeMismatch}
-}
diff --git a/vendor/github.com/lomik/go-carbon/LICENSE.md b/vendor/github.com/lomik/go-carbon/LICENSE.md
deleted file mode 100644
index 948425f08d..0000000000
--- a/vendor/github.com/lomik/go-carbon/LICENSE.md
+++ /dev/null
@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2015 Lomonosov Roman
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
diff --git a/vendor/github.com/lomik/go-carbon/helper/stoppable.go b/vendor/github.com/lomik/go-carbon/helper/stoppable.go
deleted file mode 100644
index 2d4a7d3c03..0000000000
--- a/vendor/github.com/lomik/go-carbon/helper/stoppable.go
+++ /dev/null
@@ -1,87 +0,0 @@
-package helper
-
-import "sync"
-
-type StoppableInterface interface {
-}
-
-// Stoppable is abstract class with Start/Stop methods
-type Stoppable struct {
-	sync.RWMutex
-	exit     chan bool
-	wg       sync.WaitGroup
-	Go       func(callable func(exit chan bool))
-	WithExit func(callable func(exit chan bool))
-}
-
-// Start ...
-func (s *Stoppable) Start() {
-	s.StartFunc(func() error { return nil })
-}
-
-// Stop ...
-func (s *Stoppable) Stop() {
-	s.StopFunc(func() {})
-}
-
-// StartFunc ...
-func (s *Stoppable) StartFunc(startProcedure func() error) error {
-	s.Lock()
-	defer s.Unlock()
-
-	// already started
-	if s.exit != nil {
-		return nil
-	}
-
-	exit := make(chan bool)
-	s.exit = exit
-	s.Go = func(callable func(exit chan bool)) {
-		s.wg.Add(1)
-		go func() {
-			callable(exit)
-			s.wg.Done()
-		}()
-	}
-	s.WithExit = func(callable func(exit chan bool)) {
-		callable(exit)
-	}
-
-	err := startProcedure()
-
-	// stop all if start failed
-	if err != nil {
-		s.doStop(func() {})
-	}
-
-	return err
-}
-
-func (s *Stoppable) doStop(callable func()) {
-	// already stopped
-	if s.exit == nil {
-		return
-	}
-
-	close(s.exit)
-	callable()
-	s.wg.Wait()
-	s.exit = nil
-	s.Go = func(callable func(exit chan bool)) {}
-	s.WithExit = func(callable func(exit chan bool)) {
-		callable(nil)
-	}
-}
-
-// StopFunc ...
-func (s *Stoppable) StopFunc(callable func()) {
-	s.Lock()
-	defer s.Unlock()
-
-	// already stopped
-	if s.exit == nil {
-		return
-	}
-
-	s.doStop(callable)
-}
diff --git a/vendor/github.com/lomik/go-carbon/persister/whisper.go b/vendor/github.com/lomik/go-carbon/persister/whisper.go
deleted file mode 100644
index e32b33c79d..0000000000
--- a/vendor/github.com/lomik/go-carbon/persister/whisper.go
+++ /dev/null
@@ -1,346 +0,0 @@
-package persister
-
-import (
-	"fmt"
-	"hash/crc32"
-	"os"
-	"path/filepath"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/lomik/go-whisper"
-
-	"github.com/lomik/go-carbon/helper"
-	"github.com/lomik/go-carbon/points"
-)
-
-// Whisper write data to *.wsp files
-type Whisper struct {
-	helper.Stoppable
-	updateOperations    uint32
-	commitedPoints      uint32
-	in                  chan *points.Points
-	confirm             chan *points.Points
-	schemas             WhisperSchemas
-	aggregation         *WhisperAggregation
-	metricInterval      time.Duration // checkpoint interval
-	workersCount        int
-	rootPath            string
-	graphPrefix         string
-	created             uint32 // counter
-	sparse              bool
-	maxUpdatesPerSecond int
-	mockStore           func(p *Whisper, values *points.Points)
-}
-
-// NewWhisper create instance of Whisper
-func NewWhisper(rootPath string, schemas WhisperSchemas, aggregation *WhisperAggregation, in chan *points.Points, confirm chan *points.Points) *Whisper {
-	return &Whisper{
-		in:                  in,
-		confirm:             confirm,
-		schemas:             schemas,
-		aggregation:         aggregation,
-		metricInterval:      time.Minute,
-		workersCount:        1,
-		rootPath:            rootPath,
-		maxUpdatesPerSecond: 0,
-	}
-}
-
-// SetGraphPrefix for internal cache metrics
-func (p *Whisper) SetGraphPrefix(prefix string) {
-	p.graphPrefix = prefix
-}
-
-// SetMaxUpdatesPerSecond enable throttling
-func (p *Whisper) SetMaxUpdatesPerSecond(maxUpdatesPerSecond int) {
-	p.maxUpdatesPerSecond = maxUpdatesPerSecond
-}
-
-// GetMaxUpdatesPerSecond returns current throttling speed
-func (p *Whisper) GetMaxUpdatesPerSecond() int {
-	return p.maxUpdatesPerSecond
-}
-
-// SetWorkers count
-func (p *Whisper) SetWorkers(count int) {
-	p.workersCount = count
-}
-
-// SetSparse creation
-func (p *Whisper) SetSparse(sparse bool) {
-	p.sparse = sparse
-}
-
-// SetMetricInterval sets doChekpoint interval
-func (p *Whisper) SetMetricInterval(interval time.Duration) {
-	p.metricInterval = interval
-}
-
-// Stat sends internal statistics to cache
-func (p *Whisper) Stat(metric string, value float64) {
-	p.in <- points.OnePoint(
-		fmt.Sprintf("%spersister.%s", p.graphPrefix, metric),
-		value,
-		time.Now().Unix(),
-	)
-}
-
-func store(p *Whisper, values *points.Points) {
-	path := filepath.Join(p.rootPath, strings.Replace(values.Metric, ".", "/", -1)+".wsp")
-
-	if p.confirm != nil {
-		defer func() { p.confirm <- values }()
-	}
-
-	w, err := whisper.Open(path)
-	if err != nil {
-		// create new whisper if file not exists
-		if !os.IsNotExist(err) {
-			logrus.Errorf("[persister] Failed to open whisper file %s: %s", path, err.Error())
-			return
-		}
-
-		_, schema, ok := p.schemas.Match(values.Metric)
-		if !ok {
-			logrus.Errorf("[persister] No storage schema defined for %s", values.Metric)
-			return
-		}
-
-		_, aggr := p.aggregation.Match(values.Metric)
-
-		logrus.WithFields(logrus.Fields{
-			"retention":    schema.RetentionStr,
-			"schema":       schema.Name,
-			"aggregation":  aggr.Name,
-			"xFilesFactor": aggr.XFilesFactor,
-			"method":       aggr.aggregationMethodStr,
-		}).Debugf("[persister] Creating %s", path)
-
-		if err = os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil {
-			logrus.Error(err)
-			return
-		}
-
-		w, err = whisper.CreateWithOptions(path, schema.Retentions, aggr.AggregationMethod[0], float32(aggr.XFilesFactor), &whisper.Options{
-			Sparse: p.sparse,
-		})
-		if err != nil {
-			logrus.Errorf("[persister] Failed to create new whisper file %s: %s", path, err.Error())
-			return
-		}
-
-		atomic.AddUint32(&p.created, 1)
-	}
-
-	points := make([]*whisper.TimeSeriesPoint, len(values.Data))
-	for i, r := range values.Data {
-		points[i] = &whisper.TimeSeriesPoint{Time: int(r.Timestamp), Value: r.Value}
-	}
-
-	atomic.AddUint32(&p.commitedPoints, uint32(len(values.Data)))
-	atomic.AddUint32(&p.updateOperations, 1)
-
-	defer w.Close()
-
-	defer func() {
-		if r := recover(); r != nil {
-			logrus.Errorf("[persister] UpdateMany %s recovered: %s", path, r)
-		}
-	}()
-	w.UpdateMany(points)
-}
-
-func (p *Whisper) worker(in chan *points.Points, exit chan bool) {
-	storeFunc := store
-	if p.mockStore != nil {
-		storeFunc = p.mockStore
-	}
-
-LOOP:
-	for {
-		select {
-		case <-exit:
-			break LOOP
-		case values, ok := <-in:
-			if !ok {
-				break LOOP
-			}
-			storeFunc(p, values)
-		}
-	}
-}
-
-func (p *Whisper) shuffler(in chan *points.Points, out [](chan *points.Points), exit chan bool) {
-	workers := uint32(len(out))
-
-LOOP:
-	for {
-		select {
-		case <-exit:
-			break LOOP
-		case values, ok := <-in:
-			if !ok {
-				break LOOP
-			}
-			index := crc32.ChecksumIEEE([]byte(values.Metric)) % workers
-			out[index] <- values
-		}
-	}
-
-	for _, ch := range out {
-		close(ch)
-	}
-}
-
-// save stat
-func (p *Whisper) doCheckpoint() {
-	updateOperations := atomic.LoadUint32(&p.updateOperations)
-	commitedPoints := atomic.LoadUint32(&p.commitedPoints)
-	atomic.AddUint32(&p.updateOperations, -updateOperations)
-	atomic.AddUint32(&p.commitedPoints, -commitedPoints)
-
-	created := atomic.LoadUint32(&p.created)
-	atomic.AddUint32(&p.created, -created)
-
-	logrus.WithFields(logrus.Fields{
-		"updateOperations": int(updateOperations),
-		"commitedPoints":   int(commitedPoints),
-		"created":          int(created),
-	}).Info("[persister] doCheckpoint()")
-
-	p.Stat("updateOperations", float64(updateOperations))
-	p.Stat("commitedPoints", float64(commitedPoints))
-	if updateOperations > 0 {
-		p.Stat("pointsPerUpdate", float64(commitedPoints)/float64(updateOperations))
-	} else {
-		p.Stat("pointsPerUpdate", 0.0)
-	}
-
-	p.Stat("created", float64(created))
-
-}
-
-// stat timer
-func (p *Whisper) statWorker(exit chan bool) {
-	ticker := time.NewTicker(p.metricInterval)
-	defer ticker.Stop()
-
-LOOP:
-	for {
-		select {
-		case <-exit:
-			break LOOP
-		case <-ticker.C:
-			go p.doCheckpoint()
-		}
-	}
-}
-
-func throttleChan(in chan *points.Points, ratePerSec int, exit chan bool) chan *points.Points {
-	out := make(chan *points.Points, cap(in))
-
-	delimeter := ratePerSec
-	chunk := 1
-
-	if ratePerSec > 1000 {
-		minRemainder := ratePerSec
-
-		for i := 100; i < 1000; i++ {
-			if ratePerSec%i < minRemainder {
-				delimeter = i
-				minRemainder = ratePerSec % delimeter
-			}
-		}
-
-		chunk = ratePerSec / delimeter
-	}
-
-	step := time.Duration(1e9/delimeter) * time.Nanosecond
-
-	var onceClose sync.Once
-
-	throttleWorker := func() {
-		var p *points.Points
-		var ok bool
-
-		defer onceClose.Do(func() { close(out) })
-
-		// start flight
-		throttleTicker := time.NewTicker(step)
-		defer throttleTicker.Stop()
-
-	LOOP:
-		for {
-			select {
-			case <-throttleTicker.C:
-				for i := 0; i < chunk; i++ {
-					select {
-					case p, ok = <-in:
-						if !ok {
-							break LOOP
-						}
-					case <-exit:
-						break LOOP
-					}
-					out <- p
-				}
-			case <-exit:
-				break LOOP
-			}
-		}
-	}
-
-	go throttleWorker()
-
-	return out
-}
-
-// Start worker
-func (p *Whisper) Start() error {
-
-	return p.StartFunc(func() error {
-
-		p.Go(func(exitChan chan bool) {
-			p.statWorker(exitChan)
-		})
-
-		p.WithExit(func(exitChan chan bool) {
-
-			inChan := p.in
-
-			readerExit := exitChan
-
-			if p.maxUpdatesPerSecond > 0 {
-				inChan = throttleChan(inChan, p.maxUpdatesPerSecond, exitChan)
-				readerExit = nil // read all before channel is closed
-			}
-
-			if p.workersCount <= 1 { // solo worker
-				p.Go(func(e chan bool) {
-					p.worker(inChan, readerExit)
-				})
-			} else {
-				var channels [](chan *points.Points)
-
-				for i := 0; i < p.workersCount; i++ {
-					ch := make(chan *points.Points, 32)
-					channels = append(channels, ch)
-					p.Go(func(e chan bool) {
-						p.worker(ch, nil)
-					})
-				}
-
-				p.Go(func(e chan bool) {
-					p.shuffler(inChan, channels, readerExit)
-				})
-			}
-
-		})
-
-		return nil
-	})
-}
diff --git a/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go b/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
deleted file mode 100644
index 8888139b71..0000000000
--- a/vendor/github.com/lomik/go-carbon/persister/whisper_aggregation.go
+++ /dev/null
@@ -1,122 +0,0 @@
-package persister
-
-/*
-Schemas read code from https://github.com/grobian/carbonwriter/
-*/
-
-import (
-	"fmt"
-	"regexp"
-	"strconv"
-	"strings"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/alyu/configparser"
-	"github.com/lomik/go-whisper"
-)
-
-type WhisperAggregationItem struct {
-	Name                 string
-	pattern              *regexp.Regexp
-	XFilesFactor         float64
-	aggregationMethodStr string
-	AggregationMethod    []whisper.AggregationMethod
-}
-
-// WhisperAggregation ...
-type WhisperAggregation struct {
-	Data    []WhisperAggregationItem
-	Default WhisperAggregationItem
-}
-
-// NewWhisperAggregation create instance of WhisperAggregation
-func NewWhisperAggregation() WhisperAggregation {
-	return WhisperAggregation{
-		Data: make([]WhisperAggregationItem, 0),
-		Default: WhisperAggregationItem{
-			Name:                 "default",
-			pattern:              nil,
-			XFilesFactor:         0.5,
-			aggregationMethodStr: "average",
-			AggregationMethod:    []whisper.AggregationMethod{whisper.Average},
-		},
-	}
-}
-
-// ReadWhisperAggregation ...
-func ReadWhisperAggregation(file string) (WhisperAggregation, error) {
-	config, err := configparser.Read(file)
-	if err != nil {
-		return WhisperAggregation{}, err
-	}
-	// pp.Println(config)
-	sections, err := config.AllSections()
-	if err != nil {
-		return WhisperAggregation{}, err
-	}
-
-	result := NewWhisperAggregation()
-
-	for _, s := range sections {
-		item := WhisperAggregationItem{}
-		// this is mildly stupid, but I don't feel like forking
-		// configparser just for this
-		item.Name =
-			strings.Trim(strings.SplitN(s.String(), "\n", 2)[0], " []")
-		if item.Name == "" || strings.HasPrefix(item.Name, "#") {
-			continue
-		}
-
-		item.pattern, err = regexp.Compile(s.ValueOf("pattern"))
-		if err != nil {
-			logrus.Errorf("[persister] Failed to parse pattern '%s'for [%s]: %s",
-				s.ValueOf("pattern"), item.Name, err.Error())
-			return WhisperAggregation{}, err
-		}
-
-		item.XFilesFactor, err = strconv.ParseFloat(s.ValueOf("xFilesFactor"), 64)
-		if err != nil {
-			logrus.Errorf("failed to parse xFilesFactor '%s' in %s: %s",
-				s.ValueOf("xFilesFactor"), item.Name, err.Error())
-			return WhisperAggregation{}, err
-		}
-
-		item.aggregationMethodStr = s.ValueOf("aggregationMethod")
-		methodStrs := strings.Split(item.aggregationMethodStr, ",")
-		for _, methodStr := range methodStrs {
-			switch methodStr {
-			case "average", "avg":
-				item.AggregationMethod = append(item.AggregationMethod, whisper.Average)
-			case "sum":
-				item.AggregationMethod = append(item.AggregationMethod, whisper.Sum)
-			case "last":
-				item.AggregationMethod = append(item.AggregationMethod, whisper.Last)
-			case "max":
-				item.AggregationMethod = append(item.AggregationMethod, whisper.Max)
-			case "min":
-				item.AggregationMethod = append(item.AggregationMethod, whisper.Min)
-			default:
-				logrus.Errorf("unknown aggregation method '%s'", methodStr)
-				return result, fmt.Errorf("unknown aggregation method %q", methodStr)
-			}
-		}
-
-		logrus.Debugf("[persister] Adding aggregation [%s] pattern = %s aggregationMethod = %s xFilesFactor = %f",
-			item.Name, s.ValueOf("pattern"),
-			item.aggregationMethodStr, item.XFilesFactor)
-
-		result.Data = append(result.Data, item)
-	}
-
-	return result, nil
-}
-
-// Match find schema for metric
-func (a *WhisperAggregation) Match(metric string) (uint16, WhisperAggregationItem) {
-	for i, s := range a.Data {
-		if s.pattern.MatchString(metric) {
-			return uint16(i), s
-		}
-	}
-	return uint16(len(a.Data)), a.Default // default has the index of one more than last of what's in a.Data
-}
diff --git a/vendor/github.com/lomik/go-carbon/persister/whisper_schema.go b/vendor/github.com/lomik/go-carbon/persister/whisper_schema.go
deleted file mode 100644
index be060792ee..0000000000
--- a/vendor/github.com/lomik/go-carbon/persister/whisper_schema.go
+++ /dev/null
@@ -1,228 +0,0 @@
-// this is a parser for graphite's storage-schemas.conf
-// it supports old and new retention format
-// see https://graphite.readthedocs.io/en/0.9.9/config-carbon.html#storage-schemas-conf
-// based on https://github.com/grobian/carbonwriter but with some improvements
-package persister
-
-import (
-	"errors"
-	"fmt"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-
-	"github.com/alyu/configparser"
-	"github.com/lomik/go-whisper"
-	"github.com/raintank/dur"
-	"github.com/raintank/metrictank/mdata/chunk"
-)
-
-const Month_sec = 60 * 60 * 24 * 28
-
-var ChunkSpans = [32]uint32{
-	1,
-	5,
-	10,
-	15,
-	20,
-	30,
-	60,        // 1m
-	90,        // 1.5m
-	2 * 60,    // 2m
-	3 * 60,    // 3m
-	5 * 60,    // 5m
-	10 * 60,   // 10m
-	15 * 60,   // 15m
-	20 * 60,   // 20m
-	30 * 60,   // 30m
-	45 * 60,   // 45m
-	3600,      // 1h
-	90 * 60,   // 1.5h
-	2 * 3600,  // 2h
-	150 * 60,  // 2.5h
-	3 * 3600,  // 3h
-	4 * 3600,  // 4h
-	5 * 3600,  // 5h
-	6 * 3600,  // 6h
-	7 * 3600,  // 7h
-	8 * 3600,  // 8h
-	9 * 3600,  // 9h
-	10 * 3600, // 10h
-	12 * 3600, // 12h
-	15 * 3600, // 15h
-	18 * 3600, // 18h
-	24 * 3600, // 24h
-}
-
-type SpanCode uint8
-
-var RevChunkSpans = make(map[uint32]SpanCode, len(ChunkSpans))
-
-func init() {
-	for k, v := range ChunkSpans {
-		RevChunkSpans[v] = SpanCode(k)
-	}
-}
-
-// Schema represents one schema setting
-type Schema struct {
-	Name         string
-	Pattern      *regexp.Regexp
-	RetentionStr string
-	Retentions   whisper.Retentions
-	Priority     int64
-}
-
-// WhisperSchemas contains schema settings
-type WhisperSchemas []Schema
-
-func (s WhisperSchemas) Len() int           { return len(s) }
-func (s WhisperSchemas) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
-func (s WhisperSchemas) Less(i, j int) bool { return s[i].Priority >= s[j].Priority }
-
-// Match finds the schema for metric or returns false if none found
-func (s WhisperSchemas) Match(metric string) (uint16, Schema, bool) {
-	for i, schema := range s {
-		if schema.Pattern.MatchString(metric) {
-			return uint16(i), schema, true
-		}
-	}
-	return 0, Schema{}, false
-}
-
-// ParseRetentionDefs parses retention definitions into a Retentions structure
-func ParseRetentionDefs(retentionDefs string) (whisper.Retentions, error) {
-	retentions := make(whisper.Retentions, 0)
-	for _, retentionDef := range strings.Split(retentionDefs, ",") {
-		retentionDef = strings.TrimSpace(retentionDef)
-		parts := strings.Split(retentionDef, ":")
-		if len(parts) < 2 || len(parts) > 5 {
-			return nil, fmt.Errorf("bad retentions spec %q", retentionDef)
-		}
-
-		// try old format
-		val1, err1 := strconv.ParseInt(parts[0], 10, 0)
-		val2, err2 := strconv.ParseInt(parts[1], 10, 0)
-
-		var retention whisper.Retention
-		var err error
-		if err1 == nil && err2 == nil {
-			retention = whisper.NewRetention(int(val1), int(val2))
-		} else {
-			// try new format
-			retention, err = whisper.ParseRetentionDef(retentionDef)
-			if err != nil {
-				return nil, err
-			}
-		}
-		if len(parts) >= 3 {
-			retention.ChunkSpan, err = dur.ParseUNsec(parts[2])
-			if err != nil {
-				return nil, err
-			}
-			if (Month_sec % retention.ChunkSpan) != 0 {
-				return nil, errors.New("chunkSpan must fit without remainders into month_sec (28*24*60*60)")
-			}
-			_, ok := chunk.RevChunkSpans[retention.ChunkSpan]
-			if !ok {
-				return nil, fmt.Errorf("chunkSpan %s is not a valid value (https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans)", parts[2])
-			}
-		} else {
-			// default to a valid chunkspan that can hold at least 100 points, or select the largest one otherwise.
-			approxSpan := uint32(retention.SecondsPerPoint() * 100)
-			var span uint32
-			for _, span = range ChunkSpans {
-				if span >= approxSpan {
-					break
-				}
-			}
-			retention.ChunkSpan = span
-		}
-		retention.NumChunks = 2
-		if len(parts) >= 4 {
-			i, err := strconv.Atoi(parts[3])
-			if err != nil {
-				return nil, err
-			}
-			retention.NumChunks = uint32(i)
-		}
-		retention.Ready = true
-		if len(parts) == 5 {
-			retention.Ready, err = strconv.ParseBool(parts[4])
-			if err != nil {
-				return nil, err
-			}
-		}
-
-		retentions = append(retentions, retention)
-	}
-	prevInterval := 0
-	for _, r := range retentions {
-		if r.SecondsPerPoint() <= prevInterval {
-			// for users' own sanity, and also so we can reference to archive 0, 1, 2 and it means what you expect
-			return nil, errors.New("aggregation archives must be in ascending order")
-		}
-		prevInterval = r.SecondsPerPoint()
-	}
-	return retentions, nil
-}
-
-// ReadWhisperSchemas reads and parses a storage-schemas.conf file and returns a sorted
-// schemas structure
-// see https://graphite.readthedocs.io/en/0.9.9/config-carbon.html#storage-schemas-conf
-func ReadWhisperSchemas(file string) (WhisperSchemas, error) {
-	config, err := configparser.Read(file)
-	if err != nil {
-		return nil, err
-	}
-
-	sections, err := config.AllSections()
-	if err != nil {
-		return nil, err
-	}
-
-	var schemas WhisperSchemas
-
-	for i, sec := range sections {
-		schema := Schema{}
-		schema.Name =
-			strings.Trim(strings.SplitN(sec.String(), "\n", 2)[0], " []")
-		if schema.Name == "" || strings.HasPrefix(schema.Name, "#") {
-			continue
-		}
-
-		patternStr := sec.ValueOf("pattern")
-		if patternStr == "" {
-			return nil, fmt.Errorf("[persister] Empty pattern for [%s]", schema.Name)
-		}
-		schema.Pattern, err = regexp.Compile(patternStr)
-		if err != nil {
-			return nil, fmt.Errorf("[persister] Failed to parse pattern %q for [%s]: %s",
-				sec.ValueOf("pattern"), schema.Name, err.Error())
-		}
-		schema.RetentionStr = sec.ValueOf("retentions")
-		schema.Retentions, err = ParseRetentionDefs(schema.RetentionStr)
-
-		if err != nil {
-			return nil, fmt.Errorf("[persister] Failed to parse retentions %q for [%s]: %s",
-				sec.ValueOf("retentions"), schema.Name, err.Error())
-		}
-
-		priorityStr := sec.ValueOf("priority")
-
-		p := int64(0)
-		if priorityStr != "" {
-			p, err = strconv.ParseInt(priorityStr, 10, 0)
-			if err != nil {
-				return nil, fmt.Errorf("[persister] Failed to parse priority %q for [%s]: %s", priorityStr, schema.Name, err)
-			}
-		}
-		schema.Priority = int64(p)<<32 - int64(i) // to sort records with same priority by position in file
-
-		schemas = append(schemas, schema)
-	}
-
-	sort.Sort(schemas)
-	return schemas, nil
-}
diff --git a/vendor/github.com/lomik/go-carbon/points/points.go b/vendor/github.com/lomik/go-carbon/points/points.go
deleted file mode 100644
index 2d257bc458..0000000000
--- a/vendor/github.com/lomik/go-carbon/points/points.go
+++ /dev/null
@@ -1,204 +0,0 @@
-package points
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"math"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/hydrogen18/stalecucumber"
-)
-
-// Point value/time pair
-type Point struct {
-	Value     float64
-	Timestamp int64
-}
-
-// Points from carbon clients
-type Points struct {
-	Metric string
-	Data   []*Point
-}
-
-// New creates new instance of Points
-func New() *Points {
-	return &Points{}
-}
-
-// OnePoint create Points instance with single point
-func OnePoint(metric string, value float64, timestamp int64) *Points {
-	return &Points{
-		Metric: metric,
-		Data: []*Point{
-			&Point{
-				Value:     value,
-				Timestamp: timestamp,
-			},
-		},
-	}
-}
-
-// NowPoint create OnePoint with now timestamp
-func NowPoint(metric string, value float64) *Points {
-	return OnePoint(metric, value, time.Now().Unix())
-}
-
-// Copy returns copy of object
-func (p *Points) Copy() *Points {
-	return &Points{
-		Metric: p.Metric,
-		Data:   p.Data,
-	}
-}
-
-// ParseText parse text protocol Point
-//  host.Point.value 42 1422641531\n
-func ParseText(line string) (*Points, error) {
-
-	row := strings.Split(strings.Trim(line, "\n \t\r"), " ")
-	if len(row) != 3 {
-		return nil, fmt.Errorf("bad message: %#v", line)
-	}
-
-	// 0x2e == ".". Or use split? @TODO: benchmark
-	// if strings.Contains(row[0], "..") || row[0][0] == 0x2e || row[0][len(row)-1] == 0x2e {
-	// 	return nil, fmt.Errorf("bad message: %#v", line)
-	// }
-
-	value, err := strconv.ParseFloat(row[1], 64)
-
-	if err != nil || math.IsNaN(value) {
-		return nil, fmt.Errorf("bad message: %#v", line)
-	}
-
-	tsf, err := strconv.ParseFloat(row[2], 64)
-
-	if err != nil || math.IsNaN(tsf) {
-		return nil, fmt.Errorf("bad message: %#v", line)
-	}
-
-	// 315522000 == "1980-01-01 00:00:00"
-	// if tsf < 315532800 {
-	// 	return nil, fmt.Errorf("bad message: %#v", line)
-	// }
-
-	// 4102444800 = "2100-01-01 00:00:00"
-	// Hello people from the future
-	// if tsf > 4102444800 {
-	// 	return nil, fmt.Errorf("bad message: %#v", line)
-	// }
-
-	return OnePoint(row[0], value, int64(tsf)), nil
-}
-
-// ParsePickle ...
-func ParsePickle(pkt []byte) ([]*Points, error) {
-	result, err := stalecucumber.Unpickle(bytes.NewReader(pkt))
-
-	list, err := stalecucumber.ListOrTuple(result, err)
-	if err != nil {
-		return nil, err
-	}
-
-	msgs := []*Points{}
-	for i := 0; i < len(list); i++ {
-		metric, err := stalecucumber.ListOrTuple(list[i], nil)
-		if err != nil {
-			return nil, err
-		}
-
-		if len(metric) < 2 {
-			return nil, errors.New("Unexpected array length while unpickling metric")
-		}
-
-		name, err := stalecucumber.String(metric[0], nil)
-		if err != nil {
-			return nil, err
-		}
-
-		msg := New()
-		msg.Metric = name
-
-		for j := 1; j < len(metric); j++ {
-			v, err := stalecucumber.ListOrTuple(metric[j], nil)
-			if err != nil {
-				return nil, err
-			}
-			if len(v) != 2 {
-				return nil, errors.New("Unexpected array length while unpickling data point")
-			}
-			timestamp, err := stalecucumber.Int(v[0], nil)
-			if err != nil {
-				timestampFloat, err := stalecucumber.Float(v[0], nil)
-				if err != nil {
-					return nil, err
-				}
-				timestamp = int64(timestampFloat)
-			}
-			if timestamp > math.MaxUint32 || timestamp < 0 {
-				err = errors.New("Unexpected value for timestamp, cannot be cast to uint32")
-				return nil, err
-			}
-
-			value, err := stalecucumber.Float(v[1], nil)
-			if err != nil {
-				valueInt, err := stalecucumber.Int(v[1], nil)
-				if err != nil {
-					return nil, err
-				}
-				value = float64(valueInt)
-			}
-
-			msg.Add(value, timestamp)
-		}
-		msgs = append(msgs, msg)
-	}
-	return msgs, nil
-}
-
-// Append point
-func (p *Points) Append(onePoint *Point) *Points {
-	p.Data = append(p.Data, onePoint)
-	return p
-}
-
-// Add value/timestamp pair to points
-func (p *Points) Add(value float64, timestamp int64) *Points {
-	p.Data = append(p.Data, &Point{
-		Value:     value,
-		Timestamp: timestamp,
-	})
-	return p
-}
-
-// Eq points check
-func (p *Points) Eq(other *Points) bool {
-	if other == nil {
-		return false
-	}
-	if p.Metric != other.Metric {
-		return false
-	}
-	if p.Data == nil && other.Data == nil {
-		return true
-	}
-	if (p.Data == nil || other.Data == nil) && (p.Data != nil || other.Data != nil) {
-		return false
-	}
-	if len(p.Data) != len(other.Data) {
-		return false
-	}
-	for i := 0; i < len(p.Data); i++ {
-		if p.Data[i].Value != other.Data[i].Value {
-			return false
-		}
-		if p.Data[i].Timestamp != other.Data[i].Timestamp {
-			return false
-		}
-	}
-	return true
-}
diff --git a/vendor/github.com/lomik/go-whisper/LICENCE.txt b/vendor/github.com/lomik/go-whisper/LICENCE.txt
deleted file mode 100644
index 9d5f47a14e..0000000000
--- a/vendor/github.com/lomik/go-whisper/LICENCE.txt
+++ /dev/null
@@ -1,28 +0,0 @@
-Copyright (c) 2012 Rob Young. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
diff --git a/vendor/github.com/lomik/go-whisper/README.md b/vendor/github.com/lomik/go-whisper/README.md
deleted file mode 100644
index d81be88685..0000000000
--- a/vendor/github.com/lomik/go-whisper/README.md
+++ /dev/null
@@ -1,46 +0,0 @@
-# Go Whisper
-
-[![Build Status](https://travis-ci.org/robyoung/go-whisper.png?branch=master)](https://travis-ci.org/robyoung/go-whisper?branch=master)
-
-Go Whisper is a [Go](http://golang.org/) implementation of the [Whisper](https://github.com/graphite-project/whisper) database, which is part of the [Graphite Project](http://graphite.wikidot.com/). 
-
-To create a new whisper database you must define it's retention levels (see: [storage schemas](http://graphite.readthedocs.org/en/1.0/config-carbon.html#storage-schemas-conf)), aggregation method and the xFilesFactor. The xFilesFactor specifies the fraction of data points in a propagation interval that must have known values for a propagation to occur.
-
-## Examples
-
-Create a new whisper database in "/tmp/test.wsp" with two retention levels (1 second for 1 day and 1 hour for 5 weeks), it will sum values when propagating them to the next retention level, and it requires half the values of the first retention level to be set before they are propagated.
-```go
-retentions, err := whisper.ParseRetentionDefs("1s:1d,1h:5w")
-if err == nil {
-  wsp, err := whisper.Create("/tmp/test.wsp", retentions, whisper.Sum, 0.5)
-}
-```
-
-Alternatively you can open an existing whisper database.
-```go
-wsp, err := whisper.Open("/tmp/test.wsp")
-```
-
-Once you have a whisper database you can set values at given time points. This sets the time point 1 hour ago to 12345.678.
-```go
-wsp.Update(12345.678, time.Now().Add(time.ParseDuration("-1h")).Unix())
-```
-
-And you can retrieve time series from it. This example fetches a time series for the last 1 hour and then iterates through it's points.
-```go
-series, err := wsp.Fetch(time.Now().Add(time.ParseDuration("-1h")).Unix(), time.Now().Unix())
-if err != nil {
-  // handle
-}
-for _, point := range series.Points() {
-  fmt.Println(point.Time, point.Value)
-}
-```
-
-## Thread Safety
-
-This implementation is *not* thread safe. Writing to a database concurrently will cause bad things to happen. It is up to the user to manage this in their application as they need to.
-
-## Licence
-
-Go Whisper is licenced under a BSD Licence.
diff --git a/vendor/github.com/lomik/go-whisper/whisper.go b/vendor/github.com/lomik/go-whisper/whisper.go
deleted file mode 100644
index bf389c0a4a..0000000000
--- a/vendor/github.com/lomik/go-whisper/whisper.go
+++ /dev/null
@@ -1,1038 +0,0 @@
-/*
-	Package whisper implements Graphite's Whisper database format
-*/
-package whisper
-
-import (
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"math"
-	"os"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-	"time"
-)
-
-const (
-	IntSize         = 4
-	FloatSize       = 4
-	Float64Size     = 8
-	PointSize       = 12
-	MetadataSize    = 16
-	ArchiveInfoSize = 12
-)
-
-const (
-	Seconds = 1
-	Minutes = 60
-	Hours   = 3600
-	Days    = 86400
-	Weeks   = 86400 * 7
-	Years   = 86400 * 365
-)
-
-type AggregationMethod int
-
-const (
-	Average AggregationMethod = iota + 1
-	Sum
-	Last
-	Max
-	Min
-)
-
-type Options struct {
-	Sparse bool
-}
-
-func unitMultiplier(s string) (int, error) {
-	switch {
-	case strings.HasPrefix(s, "s"):
-		return Seconds, nil
-	case strings.HasPrefix(s, "m"):
-		return Minutes, nil
-	case strings.HasPrefix(s, "h"):
-		return Hours, nil
-	case strings.HasPrefix(s, "d"):
-		return Days, nil
-	case strings.HasPrefix(s, "w"):
-		return Weeks, nil
-	case strings.HasPrefix(s, "y"):
-		return Years, nil
-	}
-	return 0, fmt.Errorf("Invalid unit multiplier [%v]", s)
-}
-
-var retentionRegexp *regexp.Regexp = regexp.MustCompile("^(\\d+)([smhdwy]+)$")
-
-func parseRetentionPart(retentionPart string) (int, error) {
-	part, err := strconv.ParseInt(retentionPart, 10, 32)
-	if err == nil {
-		return int(part), nil
-	}
-	if !retentionRegexp.MatchString(retentionPart) {
-		return 0, fmt.Errorf("%v", retentionPart)
-	}
-	matches := retentionRegexp.FindStringSubmatch(retentionPart)
-	value, err := strconv.ParseInt(matches[1], 10, 32)
-	if err != nil {
-		panic(fmt.Sprintf("Regex on %v is borked, %v cannot be parsed as int", retentionPart, matches[1]))
-	}
-	multiplier, err := unitMultiplier(matches[2])
-	return multiplier * int(value), err
-}
-
-/*
-  Parse a retention definition as you would find in the storage-schemas.conf of a Carbon install.
-  Note that this only parses a single retention definition, if you have multiple definitions (separated by a comma)
-  you will have to split them yourself.
-
-  ParseRetentionDef("10s:14d") Retention{10, 120960}
-
-  See: http://graphite.readthedocs.org/en/1.0/config-carbon.html#storage-schemas-conf
-*/
-func ParseRetentionDef(retentionDef string) (Retention, error) {
-	parts := strings.Split(retentionDef, ":")
-	if len(parts) < 2 {
-		return Retention{}, fmt.Errorf("Not enough parts in retentionDef [%v]", retentionDef)
-	}
-	precision, err := parseRetentionPart(parts[0])
-	if err != nil {
-		return Retention{}, fmt.Errorf("Failed to parse precision: %v", err)
-	}
-
-	ttl, err := parseRetentionPart(parts[1])
-	if err != nil {
-		return Retention{}, fmt.Errorf("Failed to parse points: %v", err)
-	}
-
-	return Retention{
-		secondsPerPoint: precision,
-		numberOfPoints:  ttl / precision}, err
-}
-
-func ParseRetentionDefs(retentionDefs string) (Retentions, error) {
-	retentions := make(Retentions, 0)
-	for _, retentionDef := range strings.Split(retentionDefs, ",") {
-		retention, err := ParseRetentionDef(retentionDef)
-		if err != nil {
-			return nil, err
-		}
-		retentions = append(retentions, retention)
-	}
-	return retentions, nil
-}
-
-/*
-	Represents a Whisper database file.
-*/
-type Whisper struct {
-	file *os.File
-
-	// Metadata
-	aggregationMethod AggregationMethod
-	maxRetention      int
-	xFilesFactor      float32
-	archives          []*archiveInfo
-}
-
-// Wrappers for whisper.file operations
-func (whisper *Whisper) fileWriteAt(b []byte, off int64) error {
-	_, err := whisper.file.WriteAt(b, off)
-	return err
-}
-
-// Wrappers for file.ReadAt operations
-func (whisper *Whisper) fileReadAt(b []byte, off int64) error {
-	_, err := whisper.file.ReadAt(b, off)
-	return err
-}
-
-/*
-	Create a new Whisper database file and write it's header.
-*/
-func Create(path string, retentions Retentions, aggregationMethod AggregationMethod, xFilesFactor float32) (whisper *Whisper, err error) {
-	return CreateWithOptions(path, retentions, aggregationMethod, xFilesFactor, &Options{
-		Sparse: false,
-	})
-}
-
-// CreateWithOptions is more customizable create function
-func CreateWithOptions(path string, retentions Retentions, aggregationMethod AggregationMethod, xFilesFactor float32, options *Options) (whisper *Whisper, err error) {
-	if options == nil {
-		options = &Options{}
-	}
-	sort.Sort(retentionsByPrecision{retentions})
-	if err = validateRetentions(retentions); err != nil {
-		return nil, err
-	}
-	_, err = os.Stat(path)
-	if err == nil {
-		return nil, os.ErrExist
-	}
-	file, err := os.Create(path)
-	if err != nil {
-		return nil, err
-	}
-	whisper = new(Whisper)
-
-	// Set the metadata
-	whisper.file = file
-	whisper.aggregationMethod = aggregationMethod
-	whisper.xFilesFactor = xFilesFactor
-	for _, retention := range retentions {
-		if retention.MaxRetention() > whisper.maxRetention {
-			whisper.maxRetention = retention.MaxRetention()
-		}
-	}
-
-	// Set the archive info
-	offset := MetadataSize + (ArchiveInfoSize * len(retentions))
-	whisper.archives = make([]*archiveInfo, 0, len(retentions))
-	for _, retention := range retentions {
-		whisper.archives = append(whisper.archives, &archiveInfo{retention, offset})
-		offset += retention.Size()
-	}
-
-	err = whisper.writeHeader()
-	if err != nil {
-		return nil, err
-	}
-
-	// pre-allocate file size, fallocate proved slower
-	if options.Sparse {
-		if _, err = whisper.file.Seek(int64(whisper.Size()-1), 0); err != nil {
-			return nil, err
-		}
-		if _, err = whisper.file.Write([]byte{0}); err != nil {
-			return nil, err
-		}
-	} else {
-		remaining := whisper.Size() - whisper.MetadataSize()
-		chunkSize := 16384
-		zeros := make([]byte, chunkSize)
-		for remaining > chunkSize {
-			if _, err = whisper.file.Write(zeros); err != nil {
-				return nil, err
-			}
-			remaining -= chunkSize
-		}
-		if _, err = whisper.file.Write(zeros[:remaining]); err != nil {
-			return nil, err
-		}
-	}
-	// whisper.file.Sync()
-
-	return whisper, nil
-}
-
-func validateRetentions(retentions Retentions) error {
-	if len(retentions) == 0 {
-		return fmt.Errorf("No retentions")
-	}
-	for i, retention := range retentions {
-		if i == len(retentions)-1 {
-			break
-		}
-
-		nextRetention := retentions[i+1]
-		if !(retention.secondsPerPoint < nextRetention.secondsPerPoint) {
-			return fmt.Errorf("A Whisper database may not be configured having two archives with the same precision (archive%v: %v, archive%v: %v)", i, retention, i+1, nextRetention)
-		}
-
-		if mod(nextRetention.secondsPerPoint, retention.secondsPerPoint) != 0 {
-			return fmt.Errorf("Higher precision archives' precision must evenly divide all lower precision archives' precision (archive%v: %v, archive%v: %v)", i, retention.secondsPerPoint, i+1, nextRetention.secondsPerPoint)
-		}
-
-		if retention.MaxRetention() >= nextRetention.MaxRetention() {
-			return fmt.Errorf("Lower precision archives must cover larger time intervals than higher precision archives (archive%v: %v seconds, archive%v: %v seconds)", i, retention.MaxRetention(), i+1, nextRetention.MaxRetention())
-		}
-
-		if retention.numberOfPoints < (nextRetention.secondsPerPoint / retention.secondsPerPoint) {
-			return fmt.Errorf("Each archive must have at least enough points to consolidate to the next archive (archive%v consolidates %v of archive%v's points but it has only %v total points)", i+1, nextRetention.secondsPerPoint/retention.secondsPerPoint, i, retention.numberOfPoints)
-		}
-	}
-	return nil
-}
-
-/*
-  Open an existing Whisper database and read it's header
-*/
-func Open(path string) (whisper *Whisper, err error) {
-	file, err := os.OpenFile(path, os.O_RDWR, 0666)
-	if err != nil {
-		return
-	}
-
-	defer func() {
-		if err != nil {
-			whisper = nil
-			file.Close()
-		}
-	}()
-
-	whisper = new(Whisper)
-	whisper.file = file
-
-	offset := 0
-
-	// read the metadata
-	b := make([]byte, MetadataSize)
-	readed, err := file.Read(b)
-
-	if err != nil {
-		err = fmt.Errorf("Unable to read header: %s", err.Error())
-		return
-	}
-	if readed != MetadataSize {
-		err = fmt.Errorf("Unable to read header: EOF")
-		return
-	}
-
-	a := unpackInt(b[offset : offset+IntSize])
-	if a > 1024 { // support very old format. File starts with lastUpdate and has only average aggregation method
-		whisper.aggregationMethod = Average
-	} else {
-		whisper.aggregationMethod = AggregationMethod(a)
-	}
-	offset += IntSize
-	whisper.maxRetention = unpackInt(b[offset : offset+IntSize])
-	offset += IntSize
-	whisper.xFilesFactor = unpackFloat32(b[offset : offset+FloatSize])
-	offset += FloatSize
-	archiveCount := unpackInt(b[offset : offset+IntSize])
-	offset += IntSize
-
-	// read the archive info
-	b = make([]byte, ArchiveInfoSize)
-
-	whisper.archives = make([]*archiveInfo, 0)
-	for i := 0; i < archiveCount; i++ {
-		readed, err = file.Read(b)
-		if err != nil || readed != ArchiveInfoSize {
-			err = fmt.Errorf("Unable to read archive %d metadata", i)
-			return
-		}
-		whisper.archives = append(whisper.archives, unpackArchiveInfo(b))
-	}
-
-	return whisper, nil
-}
-
-func (whisper *Whisper) writeHeader() (err error) {
-	b := make([]byte, whisper.MetadataSize())
-	i := 0
-	i += packInt(b, int(whisper.aggregationMethod), i)
-	i += packInt(b, whisper.maxRetention, i)
-	i += packFloat32(b, whisper.xFilesFactor, i)
-	i += packInt(b, len(whisper.archives), i)
-	for _, archive := range whisper.archives {
-		i += packInt(b, archive.offset, i)
-		i += packInt(b, archive.secondsPerPoint, i)
-		i += packInt(b, archive.numberOfPoints, i)
-	}
-	_, err = whisper.file.Write(b)
-
-	return err
-}
-
-/*
-  Close the whisper file
-*/
-func (whisper *Whisper) Close() {
-	whisper.file.Close()
-}
-
-/*
-  Calculate the total number of bytes the Whisper file should be according to the metadata.
-*/
-func (whisper *Whisper) Size() int {
-	size := whisper.MetadataSize()
-	for _, archive := range whisper.archives {
-		size += archive.Size()
-	}
-	return size
-}
-
-/*
-  Calculate the number of bytes the metadata section will be.
-*/
-func (whisper *Whisper) MetadataSize() int {
-	return MetadataSize + (ArchiveInfoSize * len(whisper.archives))
-}
-
-/* Return aggregation method */
-func (whisper *Whisper) AggregationMethod() string {
-	aggr := "unknown"
-	switch whisper.aggregationMethod {
-	case Average:
-		aggr = "Average"
-	case Sum:
-		aggr = "Sum"
-	case Last:
-		aggr = "Last"
-	case Max:
-		aggr = "Max"
-	case Min:
-		aggr = "Min"
-	}
-	return aggr
-}
-
-/* Return max retention in seconds */
-func (whisper *Whisper) MaxRetention() int {
-	return whisper.maxRetention
-}
-
-/* Return xFilesFactor */
-func (whisper *Whisper) XFilesFactor() float32 {
-	return whisper.xFilesFactor
-}
-
-/* Return retentions */
-func (whisper *Whisper) Retentions() []Retention {
-	ret := make([]Retention, 0, 4)
-	for _, archive := range whisper.archives {
-		ret = append(ret, archive.Retention)
-	}
-
-	return ret
-}
-
-/*
-  Update a value in the database.
-
-  If the timestamp is in the future or outside of the maximum retention it will
-  fail immediately.
-*/
-func (whisper *Whisper) Update(value float64, timestamp int) (err error) {
-	// recover panics and return as error
-	defer func() {
-		if e := recover(); e != nil {
-			err = errors.New(e.(string))
-		}
-	}()
-
-	diff := int(time.Now().Unix()) - timestamp
-	if !(diff < whisper.maxRetention && diff >= 0) {
-		return fmt.Errorf("Timestamp not covered by any archives in this database")
-	}
-	var archive *archiveInfo
-	var lowerArchives []*archiveInfo
-	var i int
-	for i, archive = range whisper.archives {
-		if archive.MaxRetention() < diff {
-			continue
-		}
-		lowerArchives = whisper.archives[i+1:] // TODO: investigate just returning the positions
-		break
-	}
-
-	myInterval := timestamp - mod(timestamp, archive.secondsPerPoint)
-	point := dataPoint{myInterval, value}
-
-	_, err = whisper.file.WriteAt(point.Bytes(), whisper.getPointOffset(myInterval, archive))
-	if err != nil {
-		return err
-	}
-
-	higher := archive
-	for _, lower := range lowerArchives {
-		propagated, err := whisper.propagate(myInterval, higher, lower)
-		if err != nil {
-			return err
-		} else if !propagated {
-			break
-		}
-		higher = lower
-	}
-
-	return nil
-}
-
-func reversePoints(points []*TimeSeriesPoint) {
-	size := len(points)
-	end := size / 2
-
-	for i := 0; i < end; i++ {
-		points[i], points[size-i-1] = points[size-i-1], points[i]
-	}
-}
-
-func (whisper *Whisper) UpdateMany(points []*TimeSeriesPoint) (err error) {
-	// recover panics and return as error
-	defer func() {
-		if e := recover(); e != nil {
-			err = errors.New(e.(string))
-		}
-	}()
-
-	// sort the points, newest first
-	reversePoints(points)
-	sort.Stable(timeSeriesPointsNewestFirst{points})
-
-	now := int(time.Now().Unix()) // TODO: danger of 2030 something overflow
-
-	var currentPoints []*TimeSeriesPoint
-	for _, archive := range whisper.archives {
-		currentPoints, points = extractPoints(points, now, archive.MaxRetention())
-		if len(currentPoints) == 0 {
-			continue
-		}
-		// reverse currentPoints
-		reversePoints(currentPoints)
-		err = whisper.archiveUpdateMany(archive, currentPoints)
-		if err != nil {
-			return
-		}
-
-		if len(points) == 0 { // nothing left to do
-			break
-		}
-	}
-	return
-}
-
-func (whisper *Whisper) archiveUpdateMany(archive *archiveInfo, points []*TimeSeriesPoint) error {
-	alignedPoints := alignPoints(archive, points)
-	intervals, packedBlocks := packSequences(archive, alignedPoints)
-
-	baseInterval := whisper.getBaseInterval(archive)
-	if baseInterval == 0 {
-		baseInterval = intervals[0]
-	}
-
-	for i := range intervals {
-		myOffset := archive.PointOffset(baseInterval, intervals[i])
-		bytesBeyond := int(myOffset-archive.End()) + len(packedBlocks[i])
-		if bytesBeyond > 0 {
-			pos := len(packedBlocks[i]) - bytesBeyond
-			err := whisper.fileWriteAt(packedBlocks[i][:pos], myOffset)
-			if err != nil {
-				return err
-			}
-			err = whisper.fileWriteAt(packedBlocks[i][pos:], archive.Offset())
-			if err != nil {
-				return err
-			}
-		} else {
-			err := whisper.fileWriteAt(packedBlocks[i], myOffset)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	higher := archive
-	lowerArchives := whisper.lowerArchives(archive)
-
-	for _, lower := range lowerArchives {
-		seen := make(map[int]bool)
-		propagateFurther := false
-		for _, point := range alignedPoints {
-			interval := point.interval - mod(point.interval, lower.secondsPerPoint)
-			if !seen[interval] {
-				if propagated, err := whisper.propagate(interval, higher, lower); err != nil {
-					panic("Failed to propagate")
-				} else if propagated {
-					propagateFurther = true
-				}
-			}
-		}
-		if !propagateFurther {
-			break
-		}
-		higher = lower
-	}
-	return nil
-}
-
-func extractPoints(points []*TimeSeriesPoint, now int, maxRetention int) (currentPoints []*TimeSeriesPoint, remainingPoints []*TimeSeriesPoint) {
-	maxAge := now - maxRetention
-	for i, point := range points {
-		if point.Time < maxAge {
-			if i > 0 {
-				return points[:i-1], points[i-1:]
-			} else {
-				return []*TimeSeriesPoint{}, points
-			}
-		}
-	}
-	return points, remainingPoints
-}
-
-func alignPoints(archive *archiveInfo, points []*TimeSeriesPoint) []dataPoint {
-	alignedPoints := make([]dataPoint, 0, len(points))
-	positions := make(map[int]int)
-	for _, point := range points {
-		dPoint := dataPoint{point.Time - mod(point.Time, archive.secondsPerPoint), point.Value}
-		if p, ok := positions[dPoint.interval]; ok {
-			alignedPoints[p] = dPoint
-		} else {
-			alignedPoints = append(alignedPoints, dPoint)
-			positions[dPoint.interval] = len(alignedPoints) - 1
-		}
-	}
-	return alignedPoints
-}
-
-func packSequences(archive *archiveInfo, points []dataPoint) (intervals []int, packedBlocks [][]byte) {
-	intervals = make([]int, 0)
-	packedBlocks = make([][]byte, 0)
-	for i, point := range points {
-		if i == 0 || point.interval != intervals[len(intervals)-1]+archive.secondsPerPoint {
-			intervals = append(intervals, point.interval)
-			packedBlocks = append(packedBlocks, point.Bytes())
-		} else {
-			packedBlocks[len(packedBlocks)-1] = append(packedBlocks[len(packedBlocks)-1], point.Bytes()...)
-		}
-	}
-	return
-}
-
-/*
-	Calculate the offset for a given interval in an archive
-
-	This method retrieves the baseInterval and the
-*/
-func (whisper *Whisper) getPointOffset(start int, archive *archiveInfo) int64 {
-	baseInterval := whisper.getBaseInterval(archive)
-	if baseInterval == 0 {
-		return archive.Offset()
-	}
-	return archive.PointOffset(baseInterval, start)
-}
-
-func (whisper *Whisper) getBaseInterval(archive *archiveInfo) int {
-	baseInterval, err := whisper.readInt(archive.Offset())
-	if err != nil {
-		panic("Failed to read baseInterval")
-	}
-	return baseInterval
-}
-
-func (whisper *Whisper) lowerArchives(archive *archiveInfo) (lowerArchives []*archiveInfo) {
-	for i, lower := range whisper.archives {
-		if lower.secondsPerPoint > archive.secondsPerPoint {
-			return whisper.archives[i:]
-		}
-	}
-	return
-}
-
-func (whisper *Whisper) propagate(timestamp int, higher, lower *archiveInfo) (bool, error) {
-	lowerIntervalStart := timestamp - mod(timestamp, lower.secondsPerPoint)
-
-	higherFirstOffset := whisper.getPointOffset(lowerIntervalStart, higher)
-
-	// TODO: extract all this series extraction stuff
-	higherPoints := lower.secondsPerPoint / higher.secondsPerPoint
-	higherSize := higherPoints * PointSize
-	relativeFirstOffset := higherFirstOffset - higher.Offset()
-	relativeLastOffset := int64(mod(int(relativeFirstOffset+int64(higherSize)), higher.Size()))
-	higherLastOffset := relativeLastOffset + higher.Offset()
-
-	series, err := whisper.readSeries(higherFirstOffset, higherLastOffset, higher)
-	if err != nil {
-		return false, err
-	}
-
-	// and finally we construct a list of values
-	knownValues := make([]float64, 0, len(series))
-	currentInterval := lowerIntervalStart
-
-	for _, dPoint := range series {
-		if dPoint.interval == currentInterval {
-			knownValues = append(knownValues, dPoint.value)
-		}
-		currentInterval += higher.secondsPerPoint
-	}
-
-	// propagate aggregateValue to propagate from neighborValues if we have enough known points
-	if len(knownValues) == 0 {
-		return false, nil
-	}
-	knownPercent := float32(len(knownValues)) / float32(len(series))
-	if knownPercent < whisper.xFilesFactor { // check we have enough data points to propagate a value
-		return false, nil
-	} else {
-		aggregateValue := aggregate(whisper.aggregationMethod, knownValues)
-		point := dataPoint{lowerIntervalStart, aggregateValue}
-		if _, err := whisper.file.WriteAt(point.Bytes(), whisper.getPointOffset(lowerIntervalStart, lower)); err != nil {
-			return false, err
-		}
-	}
-	return true, nil
-}
-
-func (whisper *Whisper) readSeries(start, end int64, archive *archiveInfo) ([]dataPoint, error) {
-	var b []byte
-	if start < end {
-		b = make([]byte, end-start)
-		err := whisper.fileReadAt(b, start)
-		if err != nil {
-			return nil, err
-		}
-	} else {
-		b = make([]byte, archive.End()-start)
-		err := whisper.fileReadAt(b, start)
-		if err != nil {
-			return nil, err
-		}
-		b2 := make([]byte, end-archive.Offset())
-		err = whisper.fileReadAt(b2, archive.Offset())
-		if err != nil {
-			return nil, err
-		}
-		b = append(b, b2...)
-	}
-	return unpackDataPoints(b), nil
-}
-
-/*
-  Calculate the starting time for a whisper db.
-*/
-func (whisper *Whisper) StartTime() int {
-	now := int(time.Now().Unix()) // TODO: danger of 2030 something overflow
-	return now - whisper.maxRetention
-}
-
-/*
-  Fetch a TimeSeries for a given time span from the file.
-*/
-func (whisper *Whisper) Fetch(fromTime, untilTime int) (timeSeries *TimeSeries, err error) {
-	now := int(time.Now().Unix()) // TODO: danger of 2030 something overflow
-	if fromTime > untilTime {
-		return nil, fmt.Errorf("Invalid time interval: from time '%d' is after until time '%d'", fromTime, untilTime)
-	}
-	oldestTime := whisper.StartTime()
-	// range is in the future
-	if fromTime > now {
-		return nil, nil
-	}
-	// range is beyond retention
-	if untilTime < oldestTime {
-		return nil, nil
-	}
-	if fromTime < oldestTime {
-		fromTime = oldestTime
-	}
-	if untilTime > now {
-		untilTime = now
-	}
-
-	// TODO: improve this algorithm it's ugly
-	diff := now - fromTime
-	var archive *archiveInfo
-	for _, archive = range whisper.archives {
-		if archive.MaxRetention() >= diff {
-			break
-		}
-	}
-
-	fromInterval := archive.Interval(fromTime)
-	untilInterval := archive.Interval(untilTime)
-	baseInterval := whisper.getBaseInterval(archive)
-
-	if baseInterval == 0 {
-		step := archive.secondsPerPoint
-		points := (untilInterval - fromInterval) / step
-		values := make([]float64, points)
-		for i := range values {
-			values[i] = math.NaN()
-		}
-		return &TimeSeries{fromInterval, untilInterval, step, values}, nil
-	}
-
-	fromOffset := archive.PointOffset(baseInterval, fromInterval)
-	untilOffset := archive.PointOffset(baseInterval, untilInterval)
-
-	series, err := whisper.readSeries(fromOffset, untilOffset, archive)
-	if err != nil {
-		return nil, err
-	}
-
-	values := make([]float64, len(series))
-	for i := range values {
-		values[i] = math.NaN()
-	}
-	currentInterval := fromInterval
-	step := archive.secondsPerPoint
-
-	for i, dPoint := range series {
-		if dPoint.interval == currentInterval {
-			values[i] = dPoint.value
-		}
-		currentInterval += step
-	}
-
-	return &TimeSeries{fromInterval, untilInterval, step, values}, nil
-}
-
-func (whisper *Whisper) readInt(offset int64) (int, error) {
-	// TODO: make errors better
-	b := make([]byte, IntSize)
-	_, err := whisper.file.ReadAt(b, offset)
-	if err != nil {
-		return 0, err
-	}
-
-	return unpackInt(b), nil
-}
-
-/*
-  A retention level.
-
-  Retention levels describe a given archive in the database. How detailed it is and how far back
-  it records.
-*/
-type Retention struct {
-	secondsPerPoint int    // interval in seconds
-	numberOfPoints  int    // ~ttl
-	ChunkSpan       uint32 // duration of chunk of aggregated metric for storage, controls how many aggregated points go into 1 chunk
-	NumChunks       uint32 // number of chunks to keep in memory. remember, for a query from now until 3 months ago, we will end up querying the memory server as well.
-	Ready           bool   // ready for reads?
-}
-
-// note missing, but can be gotten via MaxRetention: TTL       uint32 // how many seconds to keep the chunk in cassandra
-
-func (retention *Retention) MaxRetention() int {
-	return retention.secondsPerPoint * retention.numberOfPoints
-}
-
-func (retention *Retention) Size() int {
-	return retention.numberOfPoints * PointSize
-}
-
-func (retention *Retention) SecondsPerPoint() int {
-	return retention.secondsPerPoint
-}
-
-func (retention *Retention) NumberOfPoints() int {
-	return retention.numberOfPoints
-}
-
-func NewRetention(secondsPerPoint, numberOfPoints int) Retention {
-	return Retention{
-		secondsPerPoint: secondsPerPoint,
-		numberOfPoints:  numberOfPoints,
-	}
-}
-
-func NewRetentionMT(secondsPerPoint int, ttl, chunkSpan, numChunks uint32, ready bool) Retention {
-	return Retention{
-		secondsPerPoint: secondsPerPoint,
-		numberOfPoints:  int(ttl) / secondsPerPoint,
-		ChunkSpan:       chunkSpan,
-		NumChunks:       numChunks,
-		Ready:           ready,
-	}
-}
-
-type Retentions []Retention
-
-func (r Retentions) Len() int {
-	return len(r)
-}
-
-func (r Retentions) Swap(i, j int) {
-	r[i], r[j] = r[j], r[i]
-}
-
-type retentionsByPrecision struct{ Retentions }
-
-func (r retentionsByPrecision) Less(i, j int) bool {
-	return r.Retentions[i].secondsPerPoint < r.Retentions[j].secondsPerPoint
-}
-
-/*
-  Describes a time series in a file.
-
-  The only addition this type has over a Retention is the offset at which it exists within the
-  whisper file.
-*/
-type archiveInfo struct {
-	Retention
-	offset int
-}
-
-func (archive *archiveInfo) Offset() int64 {
-	return int64(archive.offset)
-}
-
-func (archive *archiveInfo) PointOffset(baseInterval, interval int) int64 {
-	timeDistance := interval - baseInterval
-	pointDistance := timeDistance / archive.secondsPerPoint
-	byteDistance := pointDistance * PointSize
-	myOffset := archive.Offset() + int64(mod(byteDistance, archive.Size()))
-
-	return myOffset
-}
-
-func (archive *archiveInfo) End() int64 {
-	return archive.Offset() + int64(archive.Size())
-}
-
-func (archive *archiveInfo) Interval(time int) int {
-	return time - mod(time, archive.secondsPerPoint) + archive.secondsPerPoint
-}
-
-type TimeSeries struct {
-	fromTime  int
-	untilTime int
-	step      int
-	values    []float64
-}
-
-func (ts *TimeSeries) FromTime() int {
-	return ts.fromTime
-}
-
-func (ts *TimeSeries) UntilTime() int {
-	return ts.untilTime
-}
-
-func (ts *TimeSeries) Step() int {
-	return ts.step
-}
-
-func (ts *TimeSeries) Values() []float64 {
-	return ts.values
-}
-
-func (ts *TimeSeries) Points() []TimeSeriesPoint {
-	points := make([]TimeSeriesPoint, len(ts.values))
-	for i, value := range ts.values {
-		points[i] = TimeSeriesPoint{Time: ts.fromTime + ts.step*i, Value: value}
-	}
-	return points
-}
-
-func (ts *TimeSeries) String() string {
-	return fmt.Sprintf("TimeSeries{'%v' '%-v' %v %v}", time.Unix(int64(ts.fromTime), 0), time.Unix(int64(ts.untilTime), 0), ts.step, ts.values)
-}
-
-type TimeSeriesPoint struct {
-	Time  int
-	Value float64
-}
-
-type timeSeriesPoints []*TimeSeriesPoint
-
-func (p timeSeriesPoints) Len() int {
-	return len(p)
-}
-
-func (p timeSeriesPoints) Swap(i, j int) {
-	p[i], p[j] = p[j], p[i]
-}
-
-type timeSeriesPointsNewestFirst struct {
-	timeSeriesPoints
-}
-
-func (p timeSeriesPointsNewestFirst) Less(i, j int) bool {
-	return p.timeSeriesPoints[i].Time > p.timeSeriesPoints[j].Time
-}
-
-type dataPoint struct {
-	interval int
-	value    float64
-}
-
-func (point *dataPoint) Bytes() []byte {
-	b := make([]byte, PointSize)
-	packInt(b, point.interval, 0)
-	packFloat64(b, point.value, IntSize)
-	return b
-}
-
-func sum(values []float64) float64 {
-	result := 0.0
-	for _, value := range values {
-		result += value
-	}
-	return result
-}
-
-func aggregate(method AggregationMethod, knownValues []float64) float64 {
-	switch method {
-	case Average:
-		return sum(knownValues) / float64(len(knownValues))
-	case Sum:
-		return sum(knownValues)
-	case Last:
-		return knownValues[len(knownValues)-1]
-	case Max:
-		max := knownValues[0]
-		for _, value := range knownValues {
-			if value > max {
-				max = value
-			}
-		}
-		return max
-	case Min:
-		min := knownValues[0]
-		for _, value := range knownValues {
-			if value < min {
-				min = value
-			}
-		}
-		return min
-	}
-	panic("Invalid aggregation method")
-}
-
-func packInt(b []byte, v, i int) int {
-	binary.BigEndian.PutUint32(b[i:i+IntSize], uint32(v))
-	return IntSize
-}
-
-func packFloat32(b []byte, v float32, i int) int {
-	binary.BigEndian.PutUint32(b[i:i+FloatSize], math.Float32bits(v))
-	return FloatSize
-}
-
-func packFloat64(b []byte, v float64, i int) int {
-	binary.BigEndian.PutUint64(b[i:i+Float64Size], math.Float64bits(v))
-	return Float64Size
-}
-
-func unpackInt(b []byte) int {
-	return int(binary.BigEndian.Uint32(b))
-}
-
-func unpackFloat32(b []byte) float32 {
-	return math.Float32frombits(binary.BigEndian.Uint32(b))
-}
-
-func unpackFloat64(b []byte) float64 {
-	return math.Float64frombits(binary.BigEndian.Uint64(b))
-}
-
-func unpackArchiveInfo(b []byte) *archiveInfo {
-	return &archiveInfo{Retention{secondsPerPoint: unpackInt(b[IntSize : IntSize*2]), numberOfPoints: unpackInt(b[IntSize*2 : IntSize*3])}, unpackInt(b[:IntSize])}
-}
-
-func unpackDataPoint(b []byte) dataPoint {
-	return dataPoint{unpackInt(b[0:IntSize]), unpackFloat64(b[IntSize:PointSize])}
-}
-
-func unpackDataPoints(b []byte) (series []dataPoint) {
-	series = make([]dataPoint, 0, len(b)/PointSize)
-	for i := 0; i < len(b); i += PointSize {
-		series = append(series, unpackDataPoint(b[i:i+PointSize]))
-	}
-	return
-}
-
-/*
-	Implementation of modulo that works like Python
-	Thanks @timmow for this
-*/
-func mod(a, b int) int {
-	return a - (b * int(math.Floor(float64(a)/float64(b))))
-}
diff --git a/vendor/github.com/lomik/go-whisper/whisper_test.py b/vendor/github.com/lomik/go-whisper/whisper_test.py
deleted file mode 100644
index 8757d70a69..0000000000
--- a/vendor/github.com/lomik/go-whisper/whisper_test.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import whisper
-import os
-import time
-
-def set_up_create():
-	path = "/tmp/whisper-testing.wsp"
-	try:
-		os.remove(path)
-	except:
-		pass
-	archive_list = [[1,300], [60,30], [300,12]]
-	def tear_down():
-		os.remove(path)
-	
-	return path, archive_list, tear_down
-
-def benchmark_create_update_fetch():
-	path, archive_list, tear_down = set_up_create()
-	# start timer
-	start_time = time.clock()
-	for i in range(100):
-		whisper.create(path, archive_list)
-
-		seconds_ago = 3500
-		current_value = 0.5
-		increment = 0.2
-		now = time.time()
-		# file_update closes the file so we have to reopen every time
-		for i in range(seconds_ago):
-			whisper.update(path, current_value, now - seconds_ago + i)
-			current_value += increment
-
-		from_time = now - seconds_ago
-		until_time = from_time + 1000
-
-		whisper.fetch(path, from_time, until_time)
-		tear_down()
-
-	# end timer
-	end_time = time.clock()
-	elapsed_time = end_time - start_time
-
-	print "Executed 100 iterations in %ss (%i ns/op)" % (elapsed_time, (elapsed_time * 1000 * 1000 * 1000) / 100)
-
-if __name__ == "__main__":
-	benchmark_create_update_fetch()
diff --git a/vendor/vendor.json b/vendor/vendor.json
index ae2afbe026..b18505d608 100644
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -206,12 +206,6 @@
 			"revision": "9800c50ab79c002353852a9b1095e9591b161513",
 			"revisionTime": "2016-12-13T23:44:46Z"
 		},
-		{
-			"checksumSHA1": "IJBXyoVO+uyqCrCYiNO/djnlcNk=",
-			"path": "github.com/hydrogen18/stalecucumber",
-			"revision": "9b38526d4bdf8e197c31344777fc28f7f48d250d",
-			"revisionTime": "2015-11-02T14:43:22Z"
-		},
 		{
 			"checksumSHA1": "tewA7jXVGCw1zb5mA0BDecWi4iQ=",
 			"path": "github.com/jtolds/gls",
@@ -248,30 +242,6 @@
 			"revision": "6834731faf32e62a2dd809d99fb24d1e4ae5a92d",
 			"revisionTime": "2016-01-12T14:50:11Z"
 		},
-		{
-			"checksumSHA1": "wxF2QM5J2Q5EleD8R43dpsD8fWI=",
-			"path": "github.com/lomik/go-carbon/helper",
-			"revision": "12b15c57fff585e38a965de347c4f8ecad1f9321",
-			"revisionTime": "2016-05-05T22:09:32Z"
-		},
-		{
-			"checksumSHA1": "eXXSyNEcJ1/cd9433icwB6SME4I=",
-			"path": "github.com/lomik/go-carbon/persister",
-			"revision": "12b15c57fff585e38a965de347c4f8ecad1f9321",
-			"revisionTime": "2016-05-05T22:09:32Z"
-		},
-		{
-			"checksumSHA1": "QhIV0ZGviP8XU4Nd8CxDD6j6cqc=",
-			"path": "github.com/lomik/go-carbon/points",
-			"revision": "12b15c57fff585e38a965de347c4f8ecad1f9321",
-			"revisionTime": "2016-05-05T22:09:32Z"
-		},
-		{
-			"checksumSHA1": "FWzBDjjSx/OaoGNNK/MTamWEr8M=",
-			"path": "github.com/lomik/go-whisper",
-			"revision": "537ebed0a597c2262dfe1b2eaa1ac68efd5804fb",
-			"revisionTime": "2016-05-05T07:06:08Z"
-		},
 		{
 			"checksumSHA1": "03SwtyHKqhFk9IT7ZboF4I1S5Bw=",
 			"path": "github.com/mattbaird/elastigo/lib",

From b61de1be6ea952914f70e30616cd200b103419b9 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Thu, 9 Mar 2017 22:49:05 +0100
Subject: [PATCH 23/29] hit/miss stats for matchcache

---
 mdata/init.go                  |  4 ++--
 mdata/matchcache/matchcache.go | 14 ++++++++++++--
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/mdata/init.go b/mdata/init.go
index 388fba630e..054ff32a9c 100644
--- a/mdata/init.go
+++ b/mdata/init.go
@@ -91,6 +91,6 @@ func ConfigProcess() {
 }
 
 func Cache(cleanInterval, expireAfter time.Duration) {
-	schemasCache = matchcache.New(cleanInterval, expireAfter)
-	aggsCache = matchcache.New(cleanInterval, expireAfter)
+	schemasCache = matchcache.New("schemas", cleanInterval, expireAfter)
+	aggsCache = matchcache.New("aggs", cleanInterval, expireAfter)
 }
diff --git a/mdata/matchcache/matchcache.go b/mdata/matchcache/matchcache.go
index 3805d32a23..62663db7a7 100644
--- a/mdata/matchcache/matchcache.go
+++ b/mdata/matchcache/matchcache.go
@@ -1,8 +1,11 @@
 package matchcache
 
 import (
+	"fmt"
 	"sync"
 	"time"
+
+	"github.com/raintank/metrictank/stats"
 )
 
 // Cache caches key to uint16 lookups (for schemas and aggregations)
@@ -15,6 +18,8 @@ type Cache struct {
 
 	cleanInterval time.Duration
 	expireAfter   time.Duration
+	hits          *stats.Counter32
+	miss          *stats.Counter32
 }
 
 type Item struct {
@@ -22,11 +27,13 @@ type Item struct {
 	seen int64
 }
 
-func New(cleanInterval, expireAfter time.Duration) *Cache {
+func New(name string, cleanInterval, expireAfter time.Duration) *Cache {
 	m := &Cache{
 		data:          make(map[string]Item),
 		cleanInterval: cleanInterval,
 		expireAfter:   expireAfter,
+		hits:          stats.NewCounter32(fmt.Sprintf("idx.matchcache.%s.ops.hit", name)),
+		miss:          stats.NewCounter32(fmt.Sprintf("idx.matchcache.%s.ops.miss", name)),
 	}
 	go m.maintain()
 	return m
@@ -38,7 +45,10 @@ type AddFunc func(key string) uint16
 func (m *Cache) Get(key string, fn AddFunc) uint16 {
 	m.Lock()
 	item, ok := m.data[key]
-	if !ok {
+	if ok {
+		m.hits.Inc()
+	} else {
+		m.miss.Inc()
 		item.val = fn(key)
 	}
 	item.seen = time.Now().Unix()

From 22bc4cae8545253f7c97e069417b8f586a3f4aa0 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Thu, 9 Mar 2017 22:49:28 +0100
Subject: [PATCH 24/29] Revert "hit/miss stats for matchcache"

This reverts commit 3bae47fc22680c07ff5a9764061b77333a66fc42.

They consume about 1% cpu (flat and cumul) under heavy ingest load
and they basically just confirm it works exactly like it should.

But the commit is here in case someone ever wants to re-apply it.
---
 mdata/init.go                  |  4 ++--
 mdata/matchcache/matchcache.go | 14 ++------------
 2 files changed, 4 insertions(+), 14 deletions(-)

diff --git a/mdata/init.go b/mdata/init.go
index 054ff32a9c..388fba630e 100644
--- a/mdata/init.go
+++ b/mdata/init.go
@@ -91,6 +91,6 @@ func ConfigProcess() {
 }
 
 func Cache(cleanInterval, expireAfter time.Duration) {
-	schemasCache = matchcache.New("schemas", cleanInterval, expireAfter)
-	aggsCache = matchcache.New("aggs", cleanInterval, expireAfter)
+	schemasCache = matchcache.New(cleanInterval, expireAfter)
+	aggsCache = matchcache.New(cleanInterval, expireAfter)
 }
diff --git a/mdata/matchcache/matchcache.go b/mdata/matchcache/matchcache.go
index 62663db7a7..3805d32a23 100644
--- a/mdata/matchcache/matchcache.go
+++ b/mdata/matchcache/matchcache.go
@@ -1,11 +1,8 @@
 package matchcache
 
 import (
-	"fmt"
 	"sync"
 	"time"
-
-	"github.com/raintank/metrictank/stats"
 )
 
 // Cache caches key to uint16 lookups (for schemas and aggregations)
@@ -18,8 +15,6 @@ type Cache struct {
 
 	cleanInterval time.Duration
 	expireAfter   time.Duration
-	hits          *stats.Counter32
-	miss          *stats.Counter32
 }
 
 type Item struct {
@@ -27,13 +22,11 @@ type Item struct {
 	seen int64
 }
 
-func New(name string, cleanInterval, expireAfter time.Duration) *Cache {
+func New(cleanInterval, expireAfter time.Duration) *Cache {
 	m := &Cache{
 		data:          make(map[string]Item),
 		cleanInterval: cleanInterval,
 		expireAfter:   expireAfter,
-		hits:          stats.NewCounter32(fmt.Sprintf("idx.matchcache.%s.ops.hit", name)),
-		miss:          stats.NewCounter32(fmt.Sprintf("idx.matchcache.%s.ops.miss", name)),
 	}
 	go m.maintain()
 	return m
@@ -45,10 +38,7 @@ type AddFunc func(key string) uint16
 func (m *Cache) Get(key string, fn AddFunc) uint16 {
 	m.Lock()
 	item, ok := m.data[key]
-	if ok {
-		m.hits.Inc()
-	} else {
-		m.miss.Inc()
+	if !ok {
 		item.val = fn(key)
 	}
 	item.seen = time.Now().Unix()

From fb2f6b1d5688a4d078df15bf2b65190862a2aaa6 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Fri, 10 Mar 2017 13:32:47 +0100
Subject: [PATCH 25/29] add a mode to mt-store-cat to see a full dump

prints how many chunks per series in each table, with TTL's
optionally filter down by metric prefix
---
 cmd/mt-store-cat/full.go   |  96 ++++++++++++++
 cmd/mt-store-cat/main.go   | 257 +++++++------------------------------
 cmd/mt-store-cat/series.go | 194 ++++++++++++++++++++++++++++
 mdata/store_cassandra.go   |  34 ++---
 4 files changed, 351 insertions(+), 230 deletions(-)
 create mode 100644 cmd/mt-store-cat/full.go
 create mode 100644 cmd/mt-store-cat/series.go

diff --git a/cmd/mt-store-cat/full.go b/cmd/mt-store-cat/full.go
new file mode 100644
index 0000000000..5a9c2ae1c6
--- /dev/null
+++ b/cmd/mt-store-cat/full.go
@@ -0,0 +1,96 @@
+package main
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/raintank/metrictank/mdata"
+)
+
+type byTTL []string
+
+func (t byTTL) Len() int      { return len(t) }
+func (t byTTL) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
+func (t byTTL) Less(i, j int) bool {
+	iTTL, _ := strconv.Atoi(strings.Split(t[i], "_")[1])
+	jTTL, _ := strconv.Atoi(strings.Split(t[j], "_")[1])
+	return iTTL < jTTL
+}
+
+func Dump(keyspace, prefix string, store *mdata.CassandraStore, roundTTL int) error {
+	targets := make(map[string]string)
+
+	if prefix == "" {
+		fmt.Println("# Looking for ALL metrics")
+	} else {
+		fmt.Println("# Looking for these metrics:")
+		iter := store.Session.Query("select id,metric from metric_idx").Iter()
+		var id, metric string
+		for iter.Scan(&id, &metric) {
+			if strings.HasPrefix(metric, prefix) {
+				fmt.Println(id, metric)
+				targets[id] = metric
+			}
+		}
+		err := iter.Close()
+		if err != nil {
+			log.Error(3, "cassandra query error. %s", err)
+			return err
+		}
+	}
+
+	fmt.Printf("# Keyspace %q contents:\n", keyspace)
+
+	meta, err := store.Session.KeyspaceMetadata(keyspace)
+	if err != nil {
+		return err
+	}
+	var tables []string
+	for tbl := range meta.Tables {
+		if tbl == "metric_idx" || !strings.HasPrefix(tbl, "metric_") {
+			continue
+		}
+		tables = append(tables, tbl)
+	}
+
+	sort.Sort(byTTL(tables))
+	for _, tbl := range tables {
+		query := fmt.Sprintf("select key, ttl(data) from %s", tbl)
+		//	ret := c.session.Query(query, row_key, t0, data).Exec()
+		fmt.Println("## Table", tbl)
+		iter := store.Session.Query(query).Iter()
+		var row, prevRow string
+		var ttl, prevTTL, cnt int
+		for iter.Scan(&row, &ttl) {
+			rowParts := strings.Split(row, "_")
+			if prefix != "" {
+				_, ok := targets[rowParts[0]]
+				if !ok {
+					continue
+				}
+			}
+			ttl = ttl / roundTTL
+			if ttl == prevTTL && row == prevRow {
+				cnt += 1
+			} else {
+				if prevRow != "" && prevTTL != 0 {
+					fmt.Println(prevRow, prevTTL, cnt)
+				}
+				cnt = 0
+				prevTTL = ttl
+				prevRow = row
+			}
+		}
+		if cnt != 0 {
+			fmt.Println(prevRow, prevTTL, cnt)
+		}
+		err := iter.Close()
+		if err != nil {
+			log.Error(3, "cassandra query error. %s", err)
+		}
+	}
+	return nil
+}
diff --git a/cmd/mt-store-cat/main.go b/cmd/mt-store-cat/main.go
index 7bf420cc99..b84679ce96 100644
--- a/cmd/mt-store-cat/main.go
+++ b/cmd/mt-store-cat/main.go
@@ -3,19 +3,16 @@ package main
 import (
 	"flag"
 	"fmt"
-	"math"
 	"os"
 	"runtime"
+	"strconv"
 	"time"
 
 	log "github.com/Sirupsen/logrus"
 
 	"github.com/raintank/dur"
-	"github.com/raintank/metrictank/api"
 	"github.com/raintank/metrictank/mdata"
-	"github.com/raintank/metrictank/mdata/chunk"
 	"github.com/rakyll/globalconf"
-	"gopkg.in/raintank/schema.v1"
 )
 
 const tsFormat = "2006-01-02 15:04:05"
@@ -57,137 +54,6 @@ var (
 	printTs      = flag.Bool("print-ts", false, "print time stamps instead of formatted dates")
 )
 
-func printNormal(igens []chunk.IterGen, from, to uint32) {
-	fmt.Println("number of chunks:", len(igens))
-	for i, ig := range igens {
-		fmt.Printf("## chunk %d (span %d)\n", i, ig.Span)
-		iter, err := ig.Get()
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "chunk %d itergen.Get: %s", i, err)
-			continue
-		}
-		for iter.Next() {
-			ts, val := iter.Values()
-			printRecord(ts, val, ts >= from && ts < to, math.IsNaN(val))
-		}
-	}
-}
-
-func printPointsNormal(points []schema.Point, from, to uint32) {
-	fmt.Println("number of points:", len(points))
-	for _, p := range points {
-		printRecord(p.Ts, p.Val, p.Ts >= from && p.Ts < to, math.IsNaN(p.Val))
-	}
-}
-
-func printRecord(ts uint32, val float64, in, nan bool) {
-	printTime := func(ts uint32) string {
-		if *printTs {
-			return fmt.Sprintf("%d", ts)
-		} else {
-			return time.Unix(int64(ts), 0).Format(tsFormat)
-		}
-	}
-	if in {
-		if nan {
-			fmt.Println("> ", printTime(ts), "NAN")
-		} else {
-			fmt.Println("> ", printTime(ts), val)
-		}
-	} else {
-		if nan {
-			fmt.Println("- ", printTime(ts), "NAN")
-		} else {
-			fmt.Println("- ", printTime(ts), val)
-		}
-	}
-}
-
-func printSummary(igens []chunk.IterGen, from, to uint32) {
-
-	var count int
-	first := true
-	var prevIn, prevNaN bool
-	var ts uint32
-	var val float64
-
-	var followup = func(count int, in, nan bool) {
-		fmt.Printf("... and %d more of in_range=%t nan=%t ...\n", count, in, nan)
-	}
-
-	for i, ig := range igens {
-		iter, err := ig.Get()
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "chunk %d itergen.Get: %s", i, err)
-			continue
-		}
-		for iter.Next() {
-			ts, val = iter.Values()
-
-			nan := math.IsNaN(val)
-			in := (ts >= from && ts < to)
-
-			if first {
-				printRecord(ts, val, in, nan)
-			} else if nan == prevNaN && in == prevIn {
-				count++
-			} else {
-				followup(count, prevIn, prevNaN)
-				printRecord(ts, val, in, nan)
-				count = 0
-			}
-
-			prevNaN = nan
-			prevIn = in
-			first = false
-		}
-	}
-	if count > 0 {
-		followup(count, prevIn, prevNaN)
-		fmt.Println("last value was:")
-		printRecord(ts, val, prevIn, prevNaN)
-	}
-}
-
-func printPointsSummary(points []schema.Point, from, to uint32) {
-
-	var count int
-	first := true
-	var prevIn, prevNaN bool
-	var ts uint32
-	var val float64
-
-	var followup = func(count int, in, nan bool) {
-		fmt.Printf("... and %d more of in_range=%t nan=%t ...\n", count, in, nan)
-	}
-
-	for _, p := range points {
-		ts, val = p.Ts, p.Val
-
-		nan := math.IsNaN(val)
-		in := (ts >= from && ts < to)
-
-		if first {
-			printRecord(ts, val, in, nan)
-		} else if nan == prevNaN && in == prevIn {
-			count++
-		} else {
-			followup(count, prevIn, prevNaN)
-			printRecord(ts, val, in, nan)
-			count = 0
-		}
-
-		prevNaN = nan
-		prevIn = in
-		first = false
-	}
-	if count > 0 {
-		followup(count, prevIn, prevNaN)
-		fmt.Println("last value was:")
-		printRecord(ts, val, prevIn, prevNaN)
-	}
-}
-
 func main() {
 	flag.Usage = func() {
 		fmt.Println("mt-store-cat")
@@ -197,6 +63,7 @@ func main() {
 		fmt.Println("Usage:")
 		fmt.Printf("	mt-store-cat [flags] <normal|summary> id <metric-id> <ttl>\n")
 		fmt.Printf("	mt-store-cat [flags] <normal|summary> query <org-id> <graphite query> (not supported yet)\n")
+		fmt.Printf("	mt-store-cat [flags] <normal|summary> full [roundTTL. defaults to 3600] [prefix match]\n")
 		fmt.Println("Flags:")
 		flag.PrintDefaults()
 		fmt.Println("Notes:")
@@ -208,8 +75,7 @@ func main() {
 		fmt.Printf("mt-store-cat (built with %s, git hash %s)\n", runtime.Version(), GitHash)
 		return
 	}
-
-	if flag.NArg() < 4 {
+	if flag.NArg() < 2 {
 		flag.Usage()
 		os.Exit(-1)
 	}
@@ -217,13 +83,37 @@ func main() {
 	selector := flag.Arg(1)
 	var id string
 	var ttl uint32
+	var roundTTL = 3600
+	var prefix string
 	// var query string
 	// var org int
 
 	switch selector {
 	case "id":
+		if flag.NArg() < 4 {
+			flag.Usage()
+			os.Exit(-1)
+		}
+
 		id = flag.Arg(2)
 		ttl = dur.MustParseUNsec("ttl", flag.Arg(3))
+	case "full":
+		if flag.NArg() >= 3 {
+			var err error
+			roundTTL, err = strconv.Atoi(flag.Arg(2))
+			if err != nil {
+				flag.Usage()
+				os.Exit(-1)
+			}
+		}
+		if flag.NArg() == 4 {
+			prefix = flag.Arg(3)
+		}
+		if flag.NArg() > 4 {
+			flag.Usage()
+			os.Exit(-1)
+		}
+
 	case "query":
 		//		if flag.NArg() < 4 {
 		//			flag.Usage()
@@ -242,21 +132,6 @@ func main() {
 		os.Exit(-1)
 	}
 
-	now := time.Now()
-
-	defaultFrom := uint32(now.Add(-time.Duration(24) * time.Hour).Unix())
-	defaultTo := uint32(now.Add(time.Duration(1) * time.Second).Unix())
-
-	fromUnix, err := dur.ParseTSpec(*from, now, defaultFrom)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	toUnix, err := dur.ParseTSpec(*to, now, defaultTo)
-	if err != nil {
-		log.Fatal(err)
-	}
-
 	// Only try and parse the conf file if it exists
 	path := ""
 	if _, err := os.Stat(*confFile); err == nil {
@@ -278,77 +153,33 @@ func main() {
 		return
 	}
 
+	mode := flag.Arg(0)
+	if mode != "normal" && mode != "summary" {
+		panic("unsupported mode " + mode)
+	}
+
 	store, err := mdata.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, *cassandraReadConcurrency, *cassandraReadConcurrency, *cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, *windowFactor, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, []uint32{ttl})
 	if err != nil {
 		log.Fatal(4, "failed to initialize cassandra. %s", err)
 	}
 
-	// if we're gonna mimic MT, then it would be:
-	/*
-		target, consolidateBy, err := parseTarget(query)
-		consolidator, err := consolidation.GetConsolidator(&def, parsedTargets[target])
-		if err != nil {
-		}
-		query := strings.Replace(queryForTarget[target], target, def.Name, -1)
-		reqs = append(reqs, models.NewReq(def.Id, query, fromUnix, toUnix, request.MaxDataPoints, uint32(def.Interval), consolidator))
-		reqs, err = alignRequests(reqs, s.MemoryStore.AggSettings())
-		points, interval, err := s.getTarget(req)
-		// ...
-		merged := mergeSeries(out)
-	*/
-
-	mode := flag.Arg(0)
-
-	if *fix != 0 {
-		points := getSeries(id, ttl, fromUnix, toUnix, uint32(*fix), store)
-
-		switch mode {
-		case "normal":
-			printPointsNormal(points, fromUnix, toUnix)
-		case "summary":
-			printPointsSummary(points, fromUnix, toUnix)
-		default:
-			panic("unsupported mode")
-		}
-	} else {
+	switch selector {
+	case "id":
+		now := time.Now()
+		defaultFrom := uint32(now.Add(-time.Duration(24) * time.Hour).Unix())
+		defaultTo := uint32(now.Add(time.Duration(1) * time.Second).Unix())
 
-		igens, err := store.Search(id, ttl, fromUnix, toUnix)
+		fromUnix, err := dur.ParseTSpec(*from, now, defaultFrom)
 		if err != nil {
-			panic(err)
-		}
-
-		switch mode {
-		case "normal":
-			printNormal(igens, fromUnix, toUnix)
-		case "summary":
-			printSummary(igens, fromUnix, toUnix)
-		default:
-			panic("unsupported mode")
+			log.Fatal(err)
 		}
-	}
-
-}
-
-func getSeries(id string, ttl, fromUnix, toUnix, interval uint32, store mdata.Store) []schema.Point {
-	itgens, err := store.Search(id, ttl, fromUnix, toUnix)
-	if err != nil {
-		panic(err)
-	}
-
-	var points []schema.Point
 
-	for i, itgen := range itgens {
-		iter, err := itgen.Get()
+		toUnix, err := dur.ParseTSpec(*to, now, defaultTo)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "chunk %d itergen.Get: %s", i, err)
-			continue
-		}
-		for iter.Next() {
-			ts, val := iter.Values()
-			if ts >= fromUnix && ts < toUnix {
-				points = append(points, schema.Point{Val: val, Ts: ts})
-			}
+			log.Fatal(err)
 		}
+		catId(id, ttl, fromUnix, toUnix, uint32(*fix), mode, store)
+	case "full":
+		Dump(*cassandraKeyspace, prefix, store, roundTTL)
 	}
-	return api.Fix(points, fromUnix, toUnix, interval)
 }
diff --git a/cmd/mt-store-cat/series.go b/cmd/mt-store-cat/series.go
new file mode 100644
index 0000000000..5cd61f2737
--- /dev/null
+++ b/cmd/mt-store-cat/series.go
@@ -0,0 +1,194 @@
+package main
+
+import (
+	"fmt"
+	"math"
+	"os"
+	"time"
+
+	"github.com/raintank/metrictank/api"
+	"github.com/raintank/metrictank/mdata"
+	"github.com/raintank/metrictank/mdata/chunk"
+	"gopkg.in/raintank/schema.v1"
+)
+
+// mode must be normal or summary
+func catId(id string, ttl, fromUnix, toUnix, fix uint32, mode string, store mdata.Store) {
+	if fix != 0 {
+		points := getSeries(id, ttl, fromUnix, toUnix, fix, store)
+
+		switch mode {
+		case "normal":
+			printPointsNormal(points, fromUnix, toUnix)
+		case "summary":
+			printPointsSummary(points, fromUnix, toUnix)
+		}
+	} else {
+		igens, err := store.Search(id, ttl, fromUnix, toUnix)
+		if err != nil {
+			panic(err)
+		}
+
+		switch mode {
+		case "normal":
+			printNormal(igens, fromUnix, toUnix)
+		case "summary":
+			printSummary(igens, fromUnix, toUnix)
+		}
+	}
+}
+
+func getSeries(id string, ttl, fromUnix, toUnix, interval uint32, store mdata.Store) []schema.Point {
+	itgens, err := store.Search(id, ttl, fromUnix, toUnix)
+	if err != nil {
+		panic(err)
+	}
+
+	var points []schema.Point
+
+	for i, itgen := range itgens {
+		iter, err := itgen.Get()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "chunk %d itergen.Get: %s", i, err)
+			continue
+		}
+		for iter.Next() {
+			ts, val := iter.Values()
+			if ts >= fromUnix && ts < toUnix {
+				points = append(points, schema.Point{Val: val, Ts: ts})
+			}
+		}
+	}
+	return api.Fix(points, fromUnix, toUnix, interval)
+}
+
+func printNormal(igens []chunk.IterGen, from, to uint32) {
+	fmt.Println("number of chunks:", len(igens))
+	for i, ig := range igens {
+		fmt.Printf("## chunk %d (span %d)\n", i, ig.Span)
+		iter, err := ig.Get()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "chunk %d itergen.Get: %s", i, err)
+			continue
+		}
+		for iter.Next() {
+			ts, val := iter.Values()
+			printRecord(ts, val, ts >= from && ts < to, math.IsNaN(val))
+		}
+	}
+}
+
+func printPointsNormal(points []schema.Point, from, to uint32) {
+	fmt.Println("number of points:", len(points))
+	for _, p := range points {
+		printRecord(p.Ts, p.Val, p.Ts >= from && p.Ts < to, math.IsNaN(p.Val))
+	}
+}
+
+func printRecord(ts uint32, val float64, in, nan bool) {
+	printTime := func(ts uint32) string {
+		if *printTs {
+			return fmt.Sprintf("%d", ts)
+		} else {
+			return time.Unix(int64(ts), 0).Format(tsFormat)
+		}
+	}
+	if in {
+		if nan {
+			fmt.Println("> ", printTime(ts), "NAN")
+		} else {
+			fmt.Println("> ", printTime(ts), val)
+		}
+	} else {
+		if nan {
+			fmt.Println("- ", printTime(ts), "NAN")
+		} else {
+			fmt.Println("- ", printTime(ts), val)
+		}
+	}
+}
+
+func printSummary(igens []chunk.IterGen, from, to uint32) {
+
+	var count int
+	first := true
+	var prevIn, prevNaN bool
+	var ts uint32
+	var val float64
+
+	var followup = func(count int, in, nan bool) {
+		fmt.Printf("... and %d more of in_range=%t nan=%t ...\n", count, in, nan)
+	}
+
+	for i, ig := range igens {
+		iter, err := ig.Get()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "chunk %d itergen.Get: %s", i, err)
+			continue
+		}
+		for iter.Next() {
+			ts, val = iter.Values()
+
+			nan := math.IsNaN(val)
+			in := (ts >= from && ts < to)
+
+			if first {
+				printRecord(ts, val, in, nan)
+			} else if nan == prevNaN && in == prevIn {
+				count++
+			} else {
+				followup(count, prevIn, prevNaN)
+				printRecord(ts, val, in, nan)
+				count = 0
+			}
+
+			prevNaN = nan
+			prevIn = in
+			first = false
+		}
+	}
+	if count > 0 {
+		followup(count, prevIn, prevNaN)
+		fmt.Println("last value was:")
+		printRecord(ts, val, prevIn, prevNaN)
+	}
+}
+
+func printPointsSummary(points []schema.Point, from, to uint32) {
+
+	var count int
+	first := true
+	var prevIn, prevNaN bool
+	var ts uint32
+	var val float64
+
+	var followup = func(count int, in, nan bool) {
+		fmt.Printf("... and %d more of in_range=%t nan=%t ...\n", count, in, nan)
+	}
+
+	for _, p := range points {
+		ts, val = p.Ts, p.Val
+
+		nan := math.IsNaN(val)
+		in := (ts >= from && ts < to)
+
+		if first {
+			printRecord(ts, val, in, nan)
+		} else if nan == prevNaN && in == prevIn {
+			count++
+		} else {
+			followup(count, prevIn, prevNaN)
+			printRecord(ts, val, in, nan)
+			count = 0
+		}
+
+		prevNaN = nan
+		prevIn = in
+		first = false
+	}
+	if count > 0 {
+		followup(count, prevIn, prevNaN)
+		fmt.Println("last value was:")
+		printRecord(ts, val, prevIn, prevNaN)
+	}
+}
diff --git a/mdata/store_cassandra.go b/mdata/store_cassandra.go
index 5d3da95214..88021c47c3 100644
--- a/mdata/store_cassandra.go
+++ b/mdata/store_cassandra.go
@@ -81,8 +81,8 @@ type ttlTable struct {
 	windowSize uint32
 }
 
-type cassandraStore struct {
-	session     *gocql.Session
+type CassandraStore struct {
+	Session     *gocql.Session
 	writeQueues []chan *ChunkWriteRequest
 	readQueue   chan *ChunkReadRequest
 	ttlTables   TTLTables
@@ -153,7 +153,7 @@ func GetTTLTables(ttls []uint32, windowFactor int, nameFormat string) TTLTables
 	return tables
 }
 
-func NewCassandraStore(addrs, keyspace, consistency, CaPath, Username, Password, hostSelectionPolicy string, timeout, readers, writers, readqsize, writeqsize, retries, protoVer, windowFactor int, ssl, auth, hostVerification bool, ttls []uint32) (*cassandraStore, error) {
+func NewCassandraStore(addrs, keyspace, consistency, CaPath, Username, Password, hostSelectionPolicy string, timeout, readers, writers, readqsize, writeqsize, retries, protoVer, windowFactor int, ssl, auth, hostVerification bool, ttls []uint32) (*CassandraStore, error) {
 
 	stats.NewGauge32("store.cassandra.write_queue.size").Set(writeqsize)
 	stats.NewGauge32("store.cassandra.num_writers").Set(writers)
@@ -233,8 +233,8 @@ func NewCassandraStore(addrs, keyspace, consistency, CaPath, Username, Password,
 		return nil, err
 	}
 	log.Debug("CS: created session to %s keysp %s cons %v with policy %s timeout %d readers %d writers %d readq %d writeq %d retries %d proto %d ssl %t auth %t hostverif %t", addrs, keyspace, consistency, hostSelectionPolicy, timeout, readers, writers, readqsize, writeqsize, retries, protoVer, ssl, auth, hostVerification)
-	c := &cassandraStore{
-		session:     session,
+	c := &CassandraStore{
+		Session:     session,
 		writeQueues: make([]chan *ChunkWriteRequest, writers),
 		readQueue:   make(chan *ChunkReadRequest, readqsize),
 		ttlTables:   ttlTables,
@@ -252,7 +252,7 @@ func NewCassandraStore(addrs, keyspace, consistency, CaPath, Username, Password,
 
 	return c, err
 }
-func (c *cassandraStore) Add(cwr *ChunkWriteRequest) {
+func (c *CassandraStore) Add(cwr *ChunkWriteRequest) {
 	sum := 0
 	for _, char := range cwr.key {
 		sum += int(char)
@@ -263,7 +263,7 @@ func (c *cassandraStore) Add(cwr *ChunkWriteRequest) {
 
 /* process writeQueue.
  */
-func (c *cassandraStore) processWriteQueue(queue chan *ChunkWriteRequest, meter *stats.Range32) {
+func (c *CassandraStore) processWriteQueue(queue chan *ChunkWriteRequest, meter *stats.Range32) {
 	tick := time.Tick(time.Duration(1) * time.Second)
 	for {
 		select {
@@ -305,7 +305,7 @@ func (c *cassandraStore) processWriteQueue(queue chan *ChunkWriteRequest, meter
 	}
 }
 
-func (c *cassandraStore) GetTableNames() []string {
+func (c *CassandraStore) GetTableNames() []string {
 	names := make([]string, 0)
 	for _, table := range c.ttlTables {
 		names = append(names, table.Table)
@@ -313,7 +313,7 @@ func (c *cassandraStore) GetTableNames() []string {
 	return names
 }
 
-func (c *cassandraStore) getTable(ttl uint32) (string, error) {
+func (c *CassandraStore) getTable(ttl uint32) (string, error) {
 	entry, ok := c.ttlTables[ttl]
 	if !ok {
 		return "", errTableNotFound
@@ -326,9 +326,9 @@ func (c *cassandraStore) getTable(ttl uint32) (string, error) {
 // key: is the metric_id
 // ts: is the start of the aggregated time range.
 // data: is the payload as bytes.
-func (c *cassandraStore) insertChunk(key string, t0, ttl uint32, data []byte) error {
+func (c *CassandraStore) insertChunk(key string, t0, ttl uint32, data []byte) error {
 	// for unit tests
-	if c.session == nil {
+	if c.Session == nil {
 		return nil
 	}
 
@@ -340,7 +340,7 @@ func (c *cassandraStore) insertChunk(key string, t0, ttl uint32, data []byte) er
 	query := fmt.Sprintf("INSERT INTO %s (key, ts, data) values(?,?,?) USING TTL %d", table, ttl)
 	row_key := fmt.Sprintf("%s_%d", key, t0/Month_sec) // "month number" based on unix timestamp (rounded down)
 	pre := time.Now()
-	ret := c.session.Query(query, row_key, t0, data).Exec()
+	ret := c.Session.Query(query, row_key, t0, data).Exec()
 	cassPutExecDuration.Value(time.Now().Sub(pre))
 	return ret
 }
@@ -356,11 +356,11 @@ func (o asc) Len() int           { return len(o) }
 func (o asc) Swap(i, j int)      { o[i], o[j] = o[j], o[i] }
 func (o asc) Less(i, j int) bool { return o[i].sortKey < o[j].sortKey }
 
-func (c *cassandraStore) processReadQueue() {
+func (c *CassandraStore) processReadQueue() {
 	for crr := range c.readQueue {
 		cassGetWaitDuration.Value(time.Since(crr.timestamp))
 		pre := time.Now()
-		iter := outcome{crr.month, crr.sortKey, c.session.Query(crr.q, crr.p...).Iter()}
+		iter := outcome{crr.month, crr.sortKey, c.Session.Query(crr.q, crr.p...).Iter()}
 		cassGetExecDuration.Value(time.Since(pre))
 		crr.out <- iter
 	}
@@ -368,7 +368,7 @@ func (c *cassandraStore) processReadQueue() {
 
 // Basic search of cassandra.
 // start inclusive, end exclusive
-func (c *cassandraStore) Search(key string, ttl, start, end uint32) ([]chunk.IterGen, error) {
+func (c *CassandraStore) Search(key string, ttl, start, end uint32) ([]chunk.IterGen, error) {
 	itgens := make([]chunk.IterGen, 0)
 	if start > end {
 		return itgens, errStartBeforeEnd
@@ -476,6 +476,6 @@ func (c *cassandraStore) Search(key string, ttl, start, end uint32) ([]chunk.Ite
 	return itgens, nil
 }
 
-func (c *cassandraStore) Stop() {
-	c.session.Close()
+func (c *CassandraStore) Stop() {
+	c.Session.Close()
 }

From f912baa8f2b49a55c19ce12031734f0f892532e5 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Thu, 16 Mar 2017 16:06:58 +0100
Subject: [PATCH 26/29] bugfix: make sure MemoryIdx also updates Partition

users who have their index set to memory idx had a bug where
the partition was never initially set properly, nor changed properly
on updates.
---
 idx/memory/memory.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/idx/memory/memory.go b/idx/memory/memory.go
index fd4723fb1a..d218610837 100644
--- a/idx/memory/memory.go
+++ b/idx/memory/memory.go
@@ -109,12 +109,14 @@ func (m *MemoryIdx) AddOrUpdate(data *schema.MetricData, partition int32, schema
 	if ok {
 		log.Debug("metricDef with id %s already in index.", data.Id)
 		existing.LastUpdate = data.Time
+		existing.Partition = partition
 		statUpdate.Inc()
 		statUpdateDuration.Value(time.Since(pre))
 		return
 	}
 
 	def := schema.MetricDefinitionFromMetricData(data)
+	def.Partition = partition
 	m.add(def, schemaI, aggI)
 	statMetricsActive.Inc()
 	statAddDuration.Value(time.Since(pre))

From 1673644ffab5b18c0bb0d58094aee74cf7dc4615 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Wed, 15 Mar 2017 19:06:27 +0100
Subject: [PATCH 27/29] fixups based on woodsaj's feedback and stuff

* remove matchcache, just use index instead
* s/schemaI/schemaId/ and s/aggI/aggId/
* track schemaId and aggId per metricdef, not per node
  since interval of a node can change over time and we want
  to support agg/schema settings based on interval as well.
* make AddOrUpdate return the archive added, so you get access
  to schemaId and aggId "for free".

also:
* remove AddOrUpdateDef because it's trivial to use AddOrUpdate instead.
  there's no need for AddOrUpdateDef's "update even if exists" behavior.
  Except for updating LastUpdate which we still do.
---
 api/dataprocessor_test.go       |   8 +-
 api/graphite.go                 |  27 ++-
 api/models/cluster_gen.go       | 189 ++++++++++---------
 api/models/graphite.go          |   4 +-
 api/models/graphite_test.go     |  29 ++-
 api/models/request.go           |  14 +-
 api/models/series_gen.go        |  98 +++++-----
 api/query_engine.go             |   4 +-
 api/response/fast_json_test.go  |  13 +-
 api/response/json_test.go       |  14 +-
 idx/cassandra/cassandra.go      |  33 ++--
 idx/cassandra/cassandra_test.go |  16 +-
 idx/idx.go                      |  41 ++--
 idx/idx_gen.go                  | 322 ++++++++++++++++++++++++++++----
 idx/idx_gen_test.go             | 113 +++++++++++
 idx/memory/memory.go            |  76 +++-----
 idx/memory/memory_find_test.go  |   8 +-
 idx/memory/memory_test.go       |  47 ++---
 input/carbon/carbon.go          |   2 +-
 input/input.go                  |   7 +-
 input/input_test.go             |   2 -
 mdata/aggmetrics.go             |   6 +-
 mdata/ifaces.go                 |   2 +-
 mdata/init.go                   |  10 -
 mdata/matchcache/matchcache.go  |  63 -------
 mdata/notifier.go               |  12 +-
 mdata/schema.go                 |  12 +-
 usage/usage.go                  |   8 +-
 usage/usage_test.go             |   2 +-
 29 files changed, 726 insertions(+), 456 deletions(-)
 delete mode 100644 mdata/matchcache/matchcache.go

diff --git a/api/dataprocessor_test.go b/api/dataprocessor_test.go
index 2c77daff81..1c5475a8fc 100644
--- a/api/dataprocessor_test.go
+++ b/api/dataprocessor_test.go
@@ -603,12 +603,12 @@ func TestGetSeriesFixed(t *testing.T) {
 	}
 }
 
-func reqRaw(key string, from, to, maxPoints, rawInterval uint32, consolidator consolidation.Consolidator, schemaI, aggI uint16) models.Req {
-	req := models.NewReq(key, key, from, to, maxPoints, rawInterval, consolidator, cluster.Manager.ThisNode(), schemaI, aggI)
+func reqRaw(key string, from, to, maxPoints, rawInterval uint32, consolidator consolidation.Consolidator, schemaId, aggId uint16) models.Req {
+	req := models.NewReq(key, key, from, to, maxPoints, rawInterval, consolidator, cluster.Manager.ThisNode(), schemaId, aggId)
 	return req
 }
-func reqOut(key string, from, to, maxPoints, rawInterval uint32, consolidator consolidation.Consolidator, schemaI, aggI uint16, archive int, archInterval, ttl, outInterval, aggNum uint32) models.Req {
-	req := models.NewReq(key, key, from, to, maxPoints, rawInterval, consolidator, cluster.Manager.ThisNode(), schemaI, aggI)
+func reqOut(key string, from, to, maxPoints, rawInterval uint32, consolidator consolidation.Consolidator, schemaId, aggId uint16, archive int, archInterval, ttl, outInterval, aggNum uint32) models.Req {
+	req := models.NewReq(key, key, from, to, maxPoints, rawInterval, consolidator, cluster.Manager.ThisNode(), schemaId, aggId)
 	req.Archive = archive
 	req.ArchInterval = archInterval
 	req.TTL = ttl
diff --git a/api/graphite.go b/api/graphite.go
index 24c9c25ffe..c7045de1e3 100644
--- a/api/graphite.go
+++ b/api/graphite.go
@@ -18,7 +18,6 @@ import (
 	"github.com/raintank/metrictank/mdata"
 	"github.com/raintank/metrictank/stats"
 	"github.com/raintank/worldping-api/pkg/log"
-	"gopkg.in/raintank/schema.v1"
 )
 
 var MissingOrgHeaderErr = errors.New("orgId not set in headers")
@@ -212,10 +211,8 @@ func (s *Server) renderMetrics(ctx *middleware.Context, request models.GraphiteR
 
 	patterns := make([]string, 0)
 	type locatedDef struct {
-		def     schema.MetricDefinition
-		node    cluster.Node
-		SchemaI uint16
-		AggI    uint16
+		def  idx.Archive
+		node cluster.Node
 	}
 
 	//locatedDefs[<pattern>][<def.id>]locatedDef
@@ -245,24 +242,24 @@ func (s *Server) renderMetrics(ctx *middleware.Context, request models.GraphiteR
 				continue
 			}
 			for _, def := range metric.Defs {
-				locatedDefs[s.Pattern][def.Id] = locatedDef{def, s.Node, metric.SchemaI, metric.AggI}
+				locatedDefs[s.Pattern][def.Id] = locatedDef{def, s.Node}
 			}
 		}
 	}
 
 	for pattern, ldefs := range locatedDefs {
 		for _, locdef := range ldefs {
-			def := locdef.def
+			archive := locdef.def
 			// set consolidator that will be used to normalize raw data before feeding into processing functions
 			// not to be confused with runtime consolidation which happens in the graphite api, after all processing.
-			fn := mdata.Aggregations.Get(locdef.AggI).AggregationMethod[0]
+			fn := mdata.Aggregations.Get(archive.AggId).AggregationMethod[0]
 			consolidator := consolidation.Consolidator(fn) // we use the same number assignments so we can cast them
 			// target is like foo.bar or foo.* or consolidateBy(foo.*,'sum')
 			// pattern is like foo.bar or foo.*
 			// def.Name is like foo.concretebar
 			// so we want target to contain the concrete graphite name, potentially wrapped with consolidateBy().
-			target := strings.Replace(targetForPattern[pattern], pattern, def.Name, -1)
-			reqs = append(reqs, models.NewReq(def.Id, target, fromUnix, toUnix, request.MaxDataPoints, uint32(def.Interval), consolidator, locdef.node, locdef.SchemaI, locdef.AggI))
+			target := strings.Replace(targetForPattern[pattern], pattern, archive.Name, -1)
+			reqs = append(reqs, models.NewReq(archive.Id, target, fromUnix, toUnix, request.MaxDataPoints, uint32(archive.Interval), consolidator, locdef.node, archive.SchemaId, archive.AggId))
 		}
 	}
 
@@ -350,20 +347,20 @@ func (s *Server) metricsFind(ctx *middleware.Context, request models.GraphiteFin
 	response.Write(ctx, response.NewJson(200, b, request.Jsonp))
 }
 
-func (s *Server) listLocal(orgId int) []schema.MetricDefinition {
+func (s *Server) listLocal(orgId int) []idx.Archive {
 	return s.MetricIndex.List(orgId)
 }
 
-func (s *Server) listRemote(orgId int, peer cluster.Node) ([]schema.MetricDefinition, error) {
+func (s *Server) listRemote(orgId int, peer cluster.Node) ([]idx.Archive, error) {
 	log.Debug("HTTP IndexJson() querying %s/index/list for %d", peer.Name, orgId)
 	buf, err := peer.Post("/index/list", models.IndexList{OrgId: orgId})
 	if err != nil {
 		log.Error(4, "HTTP IndexJson() error querying %s/index/list: %q", peer.Name, err)
 		return nil, err
 	}
-	result := make([]schema.MetricDefinition, 0)
+	result := make([]idx.Archive, 0)
 	for len(buf) != 0 {
-		var def schema.MetricDefinition
+		var def idx.Archive
 		buf, err = def.UnmarshalMsg(buf)
 		if err != nil {
 			log.Error(3, "HTTP IndexJson() error unmarshaling body from %s/index/list: %q", peer.Name, err)
@@ -377,7 +374,7 @@ func (s *Server) listRemote(orgId int, peer cluster.Node) ([]schema.MetricDefini
 func (s *Server) metricsIndex(ctx *middleware.Context) {
 	peers := cluster.MembersForQuery()
 	errors := make([]error, 0)
-	series := make([]schema.MetricDefinition, 0)
+	series := make([]idx.Archive, 0)
 	seenDefs := make(map[string]struct{})
 	var mu sync.Mutex
 	var wg sync.WaitGroup
diff --git a/api/models/cluster_gen.go b/api/models/cluster_gen.go
index 7d1d0f6bc2..130fa04fd3 100644
--- a/api/models/cluster_gen.go
+++ b/api/models/cluster_gen.go
@@ -13,31 +13,31 @@ import (
 func (z *GetDataResp) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
 	_ = field
-	var isz uint32
-	isz, err = dc.ReadMapHeader()
+	var zbzg uint32
+	zbzg, err = dc.ReadMapHeader()
 	if err != nil {
 		return
 	}
-	for isz > 0 {
-		isz--
+	for zbzg > 0 {
+		zbzg--
 		field, err = dc.ReadMapKeyPtr()
 		if err != nil {
 			return
 		}
 		switch msgp.UnsafeString(field) {
 		case "Series":
-			var xsz uint32
-			xsz, err = dc.ReadArrayHeader()
+			var zbai uint32
+			zbai, err = dc.ReadArrayHeader()
 			if err != nil {
 				return
 			}
-			if cap(z.Series) >= int(xsz) {
-				z.Series = z.Series[:xsz]
+			if cap(z.Series) >= int(zbai) {
+				z.Series = (z.Series)[:zbai]
 			} else {
-				z.Series = make([]Series, xsz)
+				z.Series = make([]Series, zbai)
 			}
-			for xvk := range z.Series {
-				err = z.Series[xvk].DecodeMsg(dc)
+			for zxvk := range z.Series {
+				err = z.Series[zxvk].DecodeMsg(dc)
 				if err != nil {
 					return
 				}
@@ -64,8 +64,8 @@ func (z *GetDataResp) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	for xvk := range z.Series {
-		err = z.Series[xvk].EncodeMsg(en)
+	for zxvk := range z.Series {
+		err = z.Series[zxvk].EncodeMsg(en)
 		if err != nil {
 			return
 		}
@@ -80,8 +80,8 @@ func (z *GetDataResp) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "Series"
 	o = append(o, 0x81, 0xa6, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73)
 	o = msgp.AppendArrayHeader(o, uint32(len(z.Series)))
-	for xvk := range z.Series {
-		o, err = z.Series[xvk].MarshalMsg(o)
+	for zxvk := range z.Series {
+		o, err = z.Series[zxvk].MarshalMsg(o)
 		if err != nil {
 			return
 		}
@@ -93,31 +93,31 @@ func (z *GetDataResp) MarshalMsg(b []byte) (o []byte, err error) {
 func (z *GetDataResp) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var field []byte
 	_ = field
-	var isz uint32
-	isz, bts, err = msgp.ReadMapHeaderBytes(bts)
+	var zcmr uint32
+	zcmr, bts, err = msgp.ReadMapHeaderBytes(bts)
 	if err != nil {
 		return
 	}
-	for isz > 0 {
-		isz--
+	for zcmr > 0 {
+		zcmr--
 		field, bts, err = msgp.ReadMapKeyZC(bts)
 		if err != nil {
 			return
 		}
 		switch msgp.UnsafeString(field) {
 		case "Series":
-			var xsz uint32
-			xsz, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			var zajw uint32
+			zajw, bts, err = msgp.ReadArrayHeaderBytes(bts)
 			if err != nil {
 				return
 			}
-			if cap(z.Series) >= int(xsz) {
-				z.Series = z.Series[:xsz]
+			if cap(z.Series) >= int(zajw) {
+				z.Series = (z.Series)[:zajw]
 			} else {
-				z.Series = make([]Series, xsz)
+				z.Series = make([]Series, zajw)
 			}
-			for xvk := range z.Series {
-				bts, err = z.Series[xvk].UnmarshalMsg(bts)
+			for zxvk := range z.Series {
+				bts, err = z.Series[zxvk].UnmarshalMsg(bts)
 				if err != nil {
 					return
 				}
@@ -133,10 +133,11 @@ func (z *GetDataResp) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *GetDataResp) Msgsize() (s int) {
 	s = 1 + 7 + msgp.ArrayHeaderSize
-	for xvk := range z.Series {
-		s += z.Series[xvk].Msgsize()
+	for zxvk := range z.Series {
+		s += z.Series[zxvk].Msgsize()
 	}
 	return
 }
@@ -145,56 +146,56 @@ func (z *GetDataResp) Msgsize() (s int) {
 func (z *IndexFindResp) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
 	_ = field
-	var isz uint32
-	isz, err = dc.ReadMapHeader()
+	var zxhx uint32
+	zxhx, err = dc.ReadMapHeader()
 	if err != nil {
 		return
 	}
-	for isz > 0 {
-		isz--
+	for zxhx > 0 {
+		zxhx--
 		field, err = dc.ReadMapKeyPtr()
 		if err != nil {
 			return
 		}
 		switch msgp.UnsafeString(field) {
 		case "Nodes":
-			var msz uint32
-			msz, err = dc.ReadMapHeader()
+			var zlqf uint32
+			zlqf, err = dc.ReadMapHeader()
 			if err != nil {
 				return
 			}
-			if z.Nodes == nil && msz > 0 {
-				z.Nodes = make(map[string][]idx.Node, msz)
+			if z.Nodes == nil && zlqf > 0 {
+				z.Nodes = make(map[string][]idx.Node, zlqf)
 			} else if len(z.Nodes) > 0 {
 				for key, _ := range z.Nodes {
 					delete(z.Nodes, key)
 				}
 			}
-			for msz > 0 {
-				msz--
-				var bzg string
-				var bai []idx.Node
-				bzg, err = dc.ReadString()
+			for zlqf > 0 {
+				zlqf--
+				var zwht string
+				var zhct []idx.Node
+				zwht, err = dc.ReadString()
 				if err != nil {
 					return
 				}
-				var xsz uint32
-				xsz, err = dc.ReadArrayHeader()
+				var zdaf uint32
+				zdaf, err = dc.ReadArrayHeader()
 				if err != nil {
 					return
 				}
-				if cap(bai) >= int(xsz) {
-					bai = bai[:xsz]
+				if cap(zhct) >= int(zdaf) {
+					zhct = (zhct)[:zdaf]
 				} else {
-					bai = make([]idx.Node, xsz)
+					zhct = make([]idx.Node, zdaf)
 				}
-				for cmr := range bai {
-					err = bai[cmr].DecodeMsg(dc)
+				for zcua := range zhct {
+					err = zhct[zcua].DecodeMsg(dc)
 					if err != nil {
 						return
 					}
 				}
-				z.Nodes[bzg] = bai
+				z.Nodes[zwht] = zhct
 			}
 		default:
 			err = dc.Skip()
@@ -218,17 +219,17 @@ func (z *IndexFindResp) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	for bzg, bai := range z.Nodes {
-		err = en.WriteString(bzg)
+	for zwht, zhct := range z.Nodes {
+		err = en.WriteString(zwht)
 		if err != nil {
 			return
 		}
-		err = en.WriteArrayHeader(uint32(len(bai)))
+		err = en.WriteArrayHeader(uint32(len(zhct)))
 		if err != nil {
 			return
 		}
-		for cmr := range bai {
-			err = bai[cmr].EncodeMsg(en)
+		for zcua := range zhct {
+			err = zhct[zcua].EncodeMsg(en)
 			if err != nil {
 				return
 			}
@@ -244,11 +245,11 @@ func (z *IndexFindResp) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "Nodes"
 	o = append(o, 0x81, 0xa5, 0x4e, 0x6f, 0x64, 0x65, 0x73)
 	o = msgp.AppendMapHeader(o, uint32(len(z.Nodes)))
-	for bzg, bai := range z.Nodes {
-		o = msgp.AppendString(o, bzg)
-		o = msgp.AppendArrayHeader(o, uint32(len(bai)))
-		for cmr := range bai {
-			o, err = bai[cmr].MarshalMsg(o)
+	for zwht, zhct := range z.Nodes {
+		o = msgp.AppendString(o, zwht)
+		o = msgp.AppendArrayHeader(o, uint32(len(zhct)))
+		for zcua := range zhct {
+			o, err = zhct[zcua].MarshalMsg(o)
 			if err != nil {
 				return
 			}
@@ -261,56 +262,56 @@ func (z *IndexFindResp) MarshalMsg(b []byte) (o []byte, err error) {
 func (z *IndexFindResp) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var field []byte
 	_ = field
-	var isz uint32
-	isz, bts, err = msgp.ReadMapHeaderBytes(bts)
+	var zpks uint32
+	zpks, bts, err = msgp.ReadMapHeaderBytes(bts)
 	if err != nil {
 		return
 	}
-	for isz > 0 {
-		isz--
+	for zpks > 0 {
+		zpks--
 		field, bts, err = msgp.ReadMapKeyZC(bts)
 		if err != nil {
 			return
 		}
 		switch msgp.UnsafeString(field) {
 		case "Nodes":
-			var msz uint32
-			msz, bts, err = msgp.ReadMapHeaderBytes(bts)
+			var zjfb uint32
+			zjfb, bts, err = msgp.ReadMapHeaderBytes(bts)
 			if err != nil {
 				return
 			}
-			if z.Nodes == nil && msz > 0 {
-				z.Nodes = make(map[string][]idx.Node, msz)
+			if z.Nodes == nil && zjfb > 0 {
+				z.Nodes = make(map[string][]idx.Node, zjfb)
 			} else if len(z.Nodes) > 0 {
 				for key, _ := range z.Nodes {
 					delete(z.Nodes, key)
 				}
 			}
-			for msz > 0 {
-				var bzg string
-				var bai []idx.Node
-				msz--
-				bzg, bts, err = msgp.ReadStringBytes(bts)
+			for zjfb > 0 {
+				var zwht string
+				var zhct []idx.Node
+				zjfb--
+				zwht, bts, err = msgp.ReadStringBytes(bts)
 				if err != nil {
 					return
 				}
-				var xsz uint32
-				xsz, bts, err = msgp.ReadArrayHeaderBytes(bts)
+				var zcxo uint32
+				zcxo, bts, err = msgp.ReadArrayHeaderBytes(bts)
 				if err != nil {
 					return
 				}
-				if cap(bai) >= int(xsz) {
-					bai = bai[:xsz]
+				if cap(zhct) >= int(zcxo) {
+					zhct = (zhct)[:zcxo]
 				} else {
-					bai = make([]idx.Node, xsz)
+					zhct = make([]idx.Node, zcxo)
 				}
-				for cmr := range bai {
-					bts, err = bai[cmr].UnmarshalMsg(bts)
+				for zcua := range zhct {
+					bts, err = zhct[zcua].UnmarshalMsg(bts)
 					if err != nil {
 						return
 					}
 				}
-				z.Nodes[bzg] = bai
+				z.Nodes[zwht] = zhct
 			}
 		default:
 			bts, err = msgp.Skip(bts)
@@ -323,14 +324,15 @@ func (z *IndexFindResp) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *IndexFindResp) Msgsize() (s int) {
 	s = 1 + 6 + msgp.MapHeaderSize
 	if z.Nodes != nil {
-		for bzg, bai := range z.Nodes {
-			_ = bai
-			s += msgp.StringPrefixSize + len(bzg) + msgp.ArrayHeaderSize
-			for cmr := range bai {
-				s += bai[cmr].Msgsize()
+		for zwht, zhct := range z.Nodes {
+			_ = zhct
+			s += msgp.StringPrefixSize + len(zwht) + msgp.ArrayHeaderSize
+			for zcua := range zhct {
+				s += zhct[zcua].Msgsize()
 			}
 		}
 	}
@@ -341,13 +343,13 @@ func (z *IndexFindResp) Msgsize() (s int) {
 func (z *MetricsDeleteResp) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
 	_ = field
-	var isz uint32
-	isz, err = dc.ReadMapHeader()
+	var zeff uint32
+	zeff, err = dc.ReadMapHeader()
 	if err != nil {
 		return
 	}
-	for isz > 0 {
-		isz--
+	for zeff > 0 {
+		zeff--
 		field, err = dc.ReadMapKeyPtr()
 		if err != nil {
 			return
@@ -397,13 +399,13 @@ func (z MetricsDeleteResp) MarshalMsg(b []byte) (o []byte, err error) {
 func (z *MetricsDeleteResp) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var field []byte
 	_ = field
-	var isz uint32
-	isz, bts, err = msgp.ReadMapHeaderBytes(bts)
+	var zrsw uint32
+	zrsw, bts, err = msgp.ReadMapHeaderBytes(bts)
 	if err != nil {
 		return
 	}
-	for isz > 0 {
-		isz--
+	for zrsw > 0 {
+		zrsw--
 		field, bts, err = msgp.ReadMapKeyZC(bts)
 		if err != nil {
 			return
@@ -425,6 +427,7 @@ func (z *MetricsDeleteResp) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z MetricsDeleteResp) Msgsize() (s int) {
 	s = 1 + 12 + msgp.IntSize
 	return
diff --git a/api/models/graphite.go b/api/models/graphite.go
index 699426c08a..c7a6b0cdc6 100644
--- a/api/models/graphite.go
+++ b/api/models/graphite.go
@@ -5,8 +5,8 @@ import (
 	"strconv"
 
 	"github.com/go-macaron/binding"
+	"github.com/raintank/metrictank/idx"
 	"gopkg.in/macaron.v1"
-	"gopkg.in/raintank/schema.v1"
 )
 
 type GraphiteRender struct {
@@ -51,7 +51,7 @@ type MetricsDelete struct {
 	Query string `json:"query" form:"query" binding:"Required"`
 }
 
-type MetricNames []schema.MetricDefinition
+type MetricNames []idx.Archive
 
 func (defs MetricNames) MarshalJSONFast(b []byte) ([]byte, error) {
 	seen := make(map[string]struct{})
diff --git a/api/models/graphite_test.go b/api/models/graphite_test.go
index a88e80125a..5f3341ac81 100644
--- a/api/models/graphite_test.go
+++ b/api/models/graphite_test.go
@@ -4,42 +4,35 @@ import (
 	"encoding/json"
 	"testing"
 
-	"gopkg.in/raintank/schema.v1"
+	"github.com/raintank/metrictank/idx"
 )
 
 func TestGraphiteNames(t *testing.T) {
+
 	cases := []struct {
-		in  []schema.MetricDefinition
+		in  []idx.Archive
 		out string
 	}{
 		{
-			in:  []schema.MetricDefinition{},
+			in:  []idx.Archive{},
 			out: `[]`,
 		},
 		{
-			in: []schema.MetricDefinition{
-				{
-					Name: "foo",
-				},
+			in: []idx.Archive{
+				idx.NewArchiveBare("foo"),
 			},
 			out: `["foo"]`,
 		},
 		{
-			in: []schema.MetricDefinition{
-				{
-					Name: "foo",
-				},
-				{
-					Name: "bar",
-				},
+			in: []idx.Archive{
+				idx.NewArchiveBare("foo"),
+				idx.NewArchiveBare("bar"),
 			},
 			out: `["bar","foo"]`,
 		},
 		{
-			in: []schema.MetricDefinition{
-				{
-					Name: `a\b`,
-				},
+			in: []idx.Archive{
+				idx.NewArchiveBare(`a\b`),
 			},
 			out: `["a\\b"]`,
 		},
diff --git a/api/models/request.go b/api/models/request.go
index 8296b624c0..959a73c733 100644
--- a/api/models/request.go
+++ b/api/models/request.go
@@ -17,8 +17,8 @@ type Req struct {
 	RawInterval  uint32                     `json:"rawInterval"`  // the interval of the raw metric before any consolidation
 	Consolidator consolidation.Consolidator `json:"consolidator"` // consolidation method for rollup archive and normalization. (not runtime consolidation)
 	Node         cluster.Node               `json:"-"`
-	SchemaI      uint16                     `json:"schemaI"`
-	AggI         uint16                     `json:"aggI"`
+	SchemaId     uint16                     `json:"schemaId"`
+	AggId        uint16                     `json:"aggId"`
 
 	// these fields need some more coordination and are typically set later
 	Archive      int    `json:"archive"`      // 0 means original data, 1 means first agg level, 2 means 2nd, etc.
@@ -28,7 +28,7 @@ type Req struct {
 	AggNum       uint32 `json:"aggNum"`       // how many points to consolidate together at runtime, after fetching from the archive
 }
 
-func NewReq(key, target string, from, to, maxPoints, rawInterval uint32, consolidator consolidation.Consolidator, node cluster.Node, schemaI, aggI uint16) Req {
+func NewReq(key, target string, from, to, maxPoints, rawInterval uint32, consolidator consolidation.Consolidator, node cluster.Node, schemaId, aggId uint16) Req {
 	return Req{
 		key,
 		target,
@@ -38,8 +38,8 @@ func NewReq(key, target string, from, to, maxPoints, rawInterval uint32, consoli
 		rawInterval,
 		consolidator,
 		node,
-		schemaI,
-		aggI,
+		schemaId,
+		aggId,
 		-1, // this is supposed to be updated still!
 		0,  // this is supposed to be updated still
 		0,  // this is supposed to be updated still
@@ -53,6 +53,6 @@ func (r Req) String() string {
 }
 
 func (r Req) DebugString() string {
-	return fmt.Sprintf("Req key=%s %d - %d maxPoints=%d rawInt=%d cons=%s schemaI=%d aggI=%d archive=%d archInt=%d ttl=%d outInt=%d aggNum=%d",
-		r.Key, r.From, r.To, r.MaxPoints, r.RawInterval, r.Consolidator, r.SchemaI, r.AggI, r.Archive, r.ArchInterval, r.TTL, r.OutInterval, r.AggNum)
+	return fmt.Sprintf("Req key=%s %d - %d maxPoints=%d rawInt=%d cons=%s schemaId=%d aggId=%d archive=%d archInt=%d ttl=%d outInt=%d aggNum=%d",
+		r.Key, r.From, r.To, r.MaxPoints, r.RawInterval, r.Consolidator, r.SchemaId, r.AggId, r.Archive, r.ArchInterval, r.TTL, r.OutInterval, r.AggNum)
 }
diff --git a/api/models/series_gen.go b/api/models/series_gen.go
index bc3e6e2293..de3c0a2e31 100644
--- a/api/models/series_gen.go
+++ b/api/models/series_gen.go
@@ -13,13 +13,13 @@ import (
 func (z *Series) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
 	_ = field
-	var isz uint32
-	isz, err = dc.ReadMapHeader()
+	var zbzg uint32
+	zbzg, err = dc.ReadMapHeader()
 	if err != nil {
 		return
 	}
-	for isz > 0 {
-		isz--
+	for zbzg > 0 {
+		zbzg--
 		field, err = dc.ReadMapKeyPtr()
 		if err != nil {
 			return
@@ -31,18 +31,18 @@ func (z *Series) DecodeMsg(dc *msgp.Reader) (err error) {
 				return
 			}
 		case "Datapoints":
-			var xsz uint32
-			xsz, err = dc.ReadArrayHeader()
+			var zbai uint32
+			zbai, err = dc.ReadArrayHeader()
 			if err != nil {
 				return
 			}
-			if cap(z.Datapoints) >= int(xsz) {
-				z.Datapoints = z.Datapoints[:xsz]
+			if cap(z.Datapoints) >= int(zbai) {
+				z.Datapoints = (z.Datapoints)[:zbai]
 			} else {
-				z.Datapoints = make([]schema.Point, xsz)
+				z.Datapoints = make([]schema.Point, zbai)
 			}
-			for xvk := range z.Datapoints {
-				err = z.Datapoints[xvk].DecodeMsg(dc)
+			for zxvk := range z.Datapoints {
+				err = z.Datapoints[zxvk].DecodeMsg(dc)
 				if err != nil {
 					return
 				}
@@ -83,8 +83,8 @@ func (z *Series) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	for xvk := range z.Datapoints {
-		err = z.Datapoints[xvk].EncodeMsg(en)
+	for zxvk := range z.Datapoints {
+		err = z.Datapoints[zxvk].EncodeMsg(en)
 		if err != nil {
 			return
 		}
@@ -111,8 +111,8 @@ func (z *Series) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "Datapoints"
 	o = append(o, 0xaa, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73)
 	o = msgp.AppendArrayHeader(o, uint32(len(z.Datapoints)))
-	for xvk := range z.Datapoints {
-		o, err = z.Datapoints[xvk].MarshalMsg(o)
+	for zxvk := range z.Datapoints {
+		o, err = z.Datapoints[zxvk].MarshalMsg(o)
 		if err != nil {
 			return
 		}
@@ -127,13 +127,13 @@ func (z *Series) MarshalMsg(b []byte) (o []byte, err error) {
 func (z *Series) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var field []byte
 	_ = field
-	var isz uint32
-	isz, bts, err = msgp.ReadMapHeaderBytes(bts)
+	var zcmr uint32
+	zcmr, bts, err = msgp.ReadMapHeaderBytes(bts)
 	if err != nil {
 		return
 	}
-	for isz > 0 {
-		isz--
+	for zcmr > 0 {
+		zcmr--
 		field, bts, err = msgp.ReadMapKeyZC(bts)
 		if err != nil {
 			return
@@ -145,18 +145,18 @@ func (z *Series) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				return
 			}
 		case "Datapoints":
-			var xsz uint32
-			xsz, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			var zajw uint32
+			zajw, bts, err = msgp.ReadArrayHeaderBytes(bts)
 			if err != nil {
 				return
 			}
-			if cap(z.Datapoints) >= int(xsz) {
-				z.Datapoints = z.Datapoints[:xsz]
+			if cap(z.Datapoints) >= int(zajw) {
+				z.Datapoints = (z.Datapoints)[:zajw]
 			} else {
-				z.Datapoints = make([]schema.Point, xsz)
+				z.Datapoints = make([]schema.Point, zajw)
 			}
-			for xvk := range z.Datapoints {
-				bts, err = z.Datapoints[xvk].UnmarshalMsg(bts)
+			for zxvk := range z.Datapoints {
+				bts, err = z.Datapoints[zxvk].UnmarshalMsg(bts)
 				if err != nil {
 					return
 				}
@@ -177,10 +177,11 @@ func (z *Series) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *Series) Msgsize() (s int) {
 	s = 1 + 7 + msgp.StringPrefixSize + len(z.Target) + 11 + msgp.ArrayHeaderSize
-	for xvk := range z.Datapoints {
-		s += z.Datapoints[xvk].Msgsize()
+	for zxvk := range z.Datapoints {
+		s += z.Datapoints[zxvk].Msgsize()
 	}
 	s += 9 + msgp.Uint32Size
 	return
@@ -188,18 +189,18 @@ func (z *Series) Msgsize() (s int) {
 
 // DecodeMsg implements msgp.Decodable
 func (z *SeriesByTarget) DecodeMsg(dc *msgp.Reader) (err error) {
-	var xsz uint32
-	xsz, err = dc.ReadArrayHeader()
+	var zcua uint32
+	zcua, err = dc.ReadArrayHeader()
 	if err != nil {
 		return
 	}
-	if cap((*z)) >= int(xsz) {
-		(*z) = (*z)[:xsz]
+	if cap((*z)) >= int(zcua) {
+		(*z) = (*z)[:zcua]
 	} else {
-		(*z) = make(SeriesByTarget, xsz)
+		(*z) = make(SeriesByTarget, zcua)
 	}
-	for bai := range *z {
-		err = (*z)[bai].DecodeMsg(dc)
+	for zhct := range *z {
+		err = (*z)[zhct].DecodeMsg(dc)
 		if err != nil {
 			return
 		}
@@ -213,8 +214,8 @@ func (z SeriesByTarget) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	for cmr := range z {
-		err = z[cmr].EncodeMsg(en)
+	for zxhx := range z {
+		err = z[zxhx].EncodeMsg(en)
 		if err != nil {
 			return
 		}
@@ -226,8 +227,8 @@ func (z SeriesByTarget) EncodeMsg(en *msgp.Writer) (err error) {
 func (z SeriesByTarget) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	o = msgp.AppendArrayHeader(o, uint32(len(z)))
-	for cmr := range z {
-		o, err = z[cmr].MarshalMsg(o)
+	for zxhx := range z {
+		o, err = z[zxhx].MarshalMsg(o)
 		if err != nil {
 			return
 		}
@@ -237,18 +238,18 @@ func (z SeriesByTarget) MarshalMsg(b []byte) (o []byte, err error) {
 
 // UnmarshalMsg implements msgp.Unmarshaler
 func (z *SeriesByTarget) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var xsz uint32
-	xsz, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	var zdaf uint32
+	zdaf, bts, err = msgp.ReadArrayHeaderBytes(bts)
 	if err != nil {
 		return
 	}
-	if cap((*z)) >= int(xsz) {
-		(*z) = (*z)[:xsz]
+	if cap((*z)) >= int(zdaf) {
+		(*z) = (*z)[:zdaf]
 	} else {
-		(*z) = make(SeriesByTarget, xsz)
+		(*z) = make(SeriesByTarget, zdaf)
 	}
-	for ajw := range *z {
-		bts, err = (*z)[ajw].UnmarshalMsg(bts)
+	for zlqf := range *z {
+		bts, err = (*z)[zlqf].UnmarshalMsg(bts)
 		if err != nil {
 			return
 		}
@@ -257,10 +258,11 @@ func (z *SeriesByTarget) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z SeriesByTarget) Msgsize() (s int) {
 	s = msgp.ArrayHeaderSize
-	for wht := range z {
-		s += z[wht].Msgsize()
+	for zpks := range z {
+		s += z[zpks].Msgsize()
 	}
 	return
 }
diff --git a/api/query_engine.go b/api/query_engine.go
index cdb74b481e..4a902693ab 100644
--- a/api/query_engine.go
+++ b/api/query_engine.go
@@ -40,7 +40,7 @@ func alignRequests(now uint32, reqs []models.Req) ([]models.Req, error) {
 
 	for i := range reqs {
 		req := &reqs[i]
-		retentions := mdata.Schemas.Get(req.SchemaI).Retentions
+		retentions := mdata.Schemas.Get(req.SchemaId).Retentions
 		req.Archive = -1
 
 		for i, ret := range retentions {
@@ -92,7 +92,7 @@ func alignRequests(now uint32, reqs []models.Req) ([]models.Req, error) {
 			// we have to deliver an interval higher than what we originally came up with
 
 			// let's see first if we can deliver it via lower-res rollup archives, if we have any
-			retentions := mdata.Schemas.Get(req.SchemaI).Retentions
+			retentions := mdata.Schemas.Get(req.SchemaId).Retentions
 			for i, ret := range retentions[req.Archive+1:] {
 				archInterval := uint32(ret.SecondsPerPoint)
 				if interval == archInterval && ret.Ready {
diff --git a/api/response/fast_json_test.go b/api/response/fast_json_test.go
index 282f6df981..6fad1e85a7 100644
--- a/api/response/fast_json_test.go
+++ b/api/response/fast_json_test.go
@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/raintank/metrictank/api/models"
+	"github.com/raintank/metrictank/idx"
 	"gopkg.in/raintank/schema.v1"
 )
 
@@ -124,11 +125,9 @@ func BenchmarkHttpRespFastJsonNulls(b *testing.B) {
 }
 
 func BenchmarkHttpRespFastJson1MMetricNames(b *testing.B) {
-	series := make([]schema.MetricDefinition, 1000000)
+	series := make([]idx.Archive, 1000000)
 	for i := 0; i < 1000000; i++ {
-		series[i] = schema.MetricDefinition{
-			Name: fmt.Sprintf("this.is.the.name.of.a.random-graphite-series.%d", i),
-		}
+		series[i] = idx.NewArchiveBare(fmt.Sprintf("this.is.the.name.of.a.random-graphite-series.%d", i))
 	}
 	b.ResetTimer()
 	var resp *FastJson
@@ -140,11 +139,9 @@ func BenchmarkHttpRespFastJson1MMetricNames(b *testing.B) {
 }
 
 func BenchmarkHttpRespFastJson1MMetricNamesNeedEscaping(b *testing.B) {
-	series := make([]schema.MetricDefinition, 1000000)
+	series := make([]idx.Archive, 1000000)
 	for i := 0; i < 1000000; i++ {
-		series[i] = schema.MetricDefinition{
-			Name: fmt.Sprintf(`this.is.the.name.of.\.random\graphite\series.%d`, i),
-		}
+		series[i] = idx.NewArchiveBare(fmt.Sprintf(`this.is.the.name.of.\.random\graphite\series.%d`, i))
 	}
 	b.ResetTimer()
 	var resp *FastJson
diff --git a/api/response/json_test.go b/api/response/json_test.go
index ff4670bbb9..8e38b2fcb6 100644
--- a/api/response/json_test.go
+++ b/api/response/json_test.go
@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/raintank/metrictank/api/models"
+	"github.com/raintank/metrictank/idx"
 	"gopkg.in/raintank/schema.v1"
 )
 
@@ -126,12 +127,11 @@ func BenchmarkHttpRespJsonNulls(b *testing.B) {
 }
 
 func BenchmarkHttpRespJson1MMetricNames(b *testing.B) {
-	series := make([]schema.MetricDefinition, 1000000)
+	series := make([]idx.Archive, 1000000)
 	for i := 0; i < 1000000; i++ {
-		series[i] = schema.MetricDefinition{
-			Name: fmt.Sprintf("this.is.the.name.of.a.random-graphite-series.%d", i),
-		}
+		series[i] = idx.NewArchiveBare(fmt.Sprintf("this.is.the.name.of.a.random-graphite-series.%d", i))
 	}
+
 	b.ResetTimer()
 	var resp *Json
 	for n := 0; n < b.N; n++ {
@@ -142,11 +142,9 @@ func BenchmarkHttpRespJson1MMetricNames(b *testing.B) {
 }
 
 func BenchmarkHttpRespJson1MMetricNamesNeedEscaping(b *testing.B) {
-	series := make([]schema.MetricDefinition, 1000000)
+	series := make([]idx.Archive, 1000000)
 	for i := 0; i < 1000000; i++ {
-		series[i] = schema.MetricDefinition{
-			Name: fmt.Sprintf(`this.is.the.name.of.\.random\graphite\series.%d`, i),
-		}
+		series[i] = idx.NewArchiveBare(fmt.Sprintf(`this.is.the.name.of.\.random\graphite\series.%d`, i))
 	}
 	b.ResetTimer()
 	var resp *Json
diff --git a/idx/cassandra/cassandra.go b/idx/cassandra/cassandra.go
index 180af2be36..63fe58a644 100644
--- a/idx/cassandra/cassandra.go
+++ b/idx/cassandra/cassandra.go
@@ -11,6 +11,7 @@ import (
 	"github.com/gocql/gocql"
 	"github.com/raintank/metrictank/cassandra"
 	"github.com/raintank/metrictank/cluster"
+	"github.com/raintank/metrictank/idx"
 	"github.com/raintank/metrictank/idx/memory"
 	"github.com/raintank/metrictank/stats"
 	"github.com/raintank/worldping-api/pkg/log"
@@ -221,7 +222,7 @@ func (c *CasIdx) Stop() {
 	c.session.Close()
 }
 
-func (c *CasIdx) AddOrUpdate(data *schema.MetricData, partition int32, schemaI, aggI uint16) {
+func (c *CasIdx) AddOrUpdate(data *schema.MetricData, partition int32) idx.Archive {
 	pre := time.Now()
 	existing, inMemory := c.MemoryIdx.Get(data.Id)
 
@@ -230,13 +231,12 @@ func (c *CasIdx) AddOrUpdate(data *schema.MetricData, partition int32, schemaI,
 			oldest := time.Now().Add(-1 * updateInterval).Add(-1 * time.Duration(rand.Int63n(updateInterval.Nanoseconds()*int64(updateFuzzyness*100)/100)))
 			if existing.LastUpdate < oldest.Unix() {
 				log.Debug("cassandra-idx def hasn't been seen for a while, updating index.")
-				def := schema.MetricDefinitionFromMetricData(data)
-				def.Partition = partition
-				c.MemoryIdx.AddOrUpdateDef(def, schemaI, aggI)
-				c.writeQueue <- writeReq{recvTime: time.Now(), def: def}
+				archive := c.MemoryIdx.AddOrUpdate(data, partition)
+				c.writeQueue <- writeReq{recvTime: time.Now(), def: &archive.MetricDefinition}
 				statUpdateDuration.Value(time.Since(pre))
+				return archive
 			}
-			return
+			return existing
 		}
 		// the partition of the metric has changed. So we need to delete
 		// the current metricDef from cassandra.  We do this is a separate
@@ -246,18 +246,15 @@ func (c *CasIdx) AddOrUpdate(data *schema.MetricData, partition int32, schemaI,
 				log.Error(3, err.Error())
 			}
 		}()
-		def := schema.MetricDefinitionFromMetricData(data)
-		def.Partition = partition
-		c.MemoryIdx.AddOrUpdateDef(def, schemaI, aggI)
-		c.writeQueue <- writeReq{recvTime: time.Now(), def: def}
+		archive := c.MemoryIdx.AddOrUpdate(data, partition)
+		c.writeQueue <- writeReq{recvTime: time.Now(), def: &archive.MetricDefinition}
 		statUpdateDuration.Value(time.Since(pre))
-		return
+		return archive
 	}
-	def := schema.MetricDefinitionFromMetricData(data)
-	def.Partition = partition
-	c.MemoryIdx.AddOrUpdateDef(def, schemaI, aggI)
-	c.writeQueue <- writeReq{recvTime: time.Now(), def: def}
+	archive := c.MemoryIdx.AddOrUpdate(data, partition)
+	c.writeQueue <- writeReq{recvTime: time.Now(), def: &archive.MetricDefinition}
 	statAddDuration.Value(time.Since(pre))
+	return archive
 }
 
 func (c *CasIdx) rebuildIndex() {
@@ -361,7 +358,7 @@ func (c *CasIdx) processWriteQueue() {
 	c.wg.Done()
 }
 
-func (c *CasIdx) Delete(orgId int, pattern string) ([]schema.MetricDefinition, error) {
+func (c *CasIdx) Delete(orgId int, pattern string) ([]idx.Archive, error) {
 	pre := time.Now()
 	defs, err := c.MemoryIdx.Delete(orgId, pattern)
 	if err != nil {
@@ -377,7 +374,7 @@ func (c *CasIdx) Delete(orgId int, pattern string) ([]schema.MetricDefinition, e
 	return defs, err
 }
 
-func (c *CasIdx) deleteDef(def *schema.MetricDefinition) error {
+func (c *CasIdx) deleteDef(def *idx.Archive) error {
 	pre := time.Now()
 	attempts := 0
 	for attempts < 5 {
@@ -397,7 +394,7 @@ func (c *CasIdx) deleteDef(def *schema.MetricDefinition) error {
 	return fmt.Errorf("unable to delete metricDef %s from index after %d attempts.", def.Id, attempts)
 }
 
-func (c *CasIdx) Prune(orgId int, oldest time.Time) ([]schema.MetricDefinition, error) {
+func (c *CasIdx) Prune(orgId int, oldest time.Time) ([]idx.Archive, error) {
 	pre := time.Now()
 	pruned, err := c.MemoryIdx.Prune(orgId, oldest)
 	// if an error was encountered then pruned is probably a partial list of metricDefs
diff --git a/idx/cassandra/cassandra_test.go b/idx/cassandra/cassandra_test.go
index 50af017612..9afbb94de8 100644
--- a/idx/cassandra/cassandra_test.go
+++ b/idx/cassandra/cassandra_test.go
@@ -81,7 +81,7 @@ func TestGetAddKey(t *testing.T) {
 		orgId := series[0].OrgId
 		Convey(fmt.Sprintf("When indexing metrics for orgId %d", orgId), t, func() {
 			for _, s := range series {
-				ix.AddOrUpdate(s, 1, 0, 0)
+				ix.AddOrUpdate(s, 1)
 			}
 			Convey(fmt.Sprintf("Then listing metrics for OrgId %d", orgId), func() {
 				defs := ix.List(orgId)
@@ -99,7 +99,7 @@ func TestGetAddKey(t *testing.T) {
 		for _, series := range org1Series {
 			series.Interval = 60
 			series.SetId()
-			ix.AddOrUpdate(series, 1, 0, 0)
+			ix.AddOrUpdate(series, 1)
 		}
 		Convey("then listing metrics", func() {
 			defs := ix.List(1)
@@ -112,19 +112,19 @@ func TestFind(t *testing.T) {
 	ix := New()
 	ix.Init()
 	for _, s := range getMetricData(-1, 2, 5, 10, "metric.demo") {
-		ix.AddOrUpdate(s, 1, 0, 0)
+		ix.AddOrUpdate(s, 1)
 	}
 	for _, s := range getMetricData(1, 2, 5, 10, "metric.demo") {
-		ix.AddOrUpdate(s, 1, 0, 0)
+		ix.AddOrUpdate(s, 1)
 	}
 	for _, s := range getMetricData(1, 1, 5, 10, "foo.demo") {
-		ix.AddOrUpdate(s, 1, 0, 0)
+		ix.AddOrUpdate(s, 1)
 		s.Interval = 60
 		s.SetId()
-		ix.AddOrUpdate(s, 1, 0, 0)
+		ix.AddOrUpdate(s, 1)
 	}
 	for _, s := range getMetricData(2, 2, 5, 10, "metric.foo") {
-		ix.AddOrUpdate(s, 1, 0, 0)
+		ix.AddOrUpdate(s, 1)
 	}
 
 	Convey("When listing root nodes", t, func() {
@@ -248,7 +248,7 @@ func insertDefs(ix idx.MetricIndex, i int) {
 			OrgId:    1,
 		}
 		data.SetId()
-		ix.AddOrUpdate(data, 1, 0, 0)
+		ix.AddOrUpdate(data, 1)
 	}
 }
 
diff --git a/idx/idx.go b/idx/idx.go
index 69360ef5df..07b4181284 100644
--- a/idx/idx.go
+++ b/idx/idx.go
@@ -18,10 +18,25 @@ var (
 type Node struct {
 	Path        string
 	Leaf        bool
-	Defs        []schema.MetricDefinition
+	Defs        []Archive
 	HasChildren bool
-	SchemaI     uint16 // index in mdata.schemas (not persisted)
-	AggI        uint16 // index in mdata.aggregations (not persisted)
+}
+
+type Archive struct {
+	schema.MetricDefinition
+	SchemaId uint16 // index in mdata.schemas (not persisted)
+	AggId    uint16 // index in mdata.aggregations (not persisted)
+}
+
+// used primarily by tests, for convenience
+func NewArchiveBare(name string) Archive {
+	return Archive{
+		schema.MetricDefinition{
+			Name: name,
+		},
+		0,
+		0,
+	}
 }
 
 /*
@@ -50,15 +65,15 @@ Interface
 * Stop():
  This will be called when metrictank is shutting down.
 
-* AddOrUpdate(*schema.MetricData, int32, uint16, uint16):
+* AddOrUpdate(*schema.MetricData, int32) Archive:
   Every metric received will result in a call to this method to ensure the
   metric has been added to the index. The method is passed the metricData
   payload and the partition id of the metric
 
-* Get(string) (schema.MetricDefinition, bool):
+* Get(string) (Archive, bool):
   This method should return the MetricDefintion with the passed Id.
 
-* List(int) []schema.MetricDefinition:
+* List(int) []Archive:
   This method should return all MetricDefinitions for the passed OrgId.  If the
   passed OrgId is "-1", then all metricDefinitions across all organisations
   should be returned.
@@ -71,14 +86,14 @@ Interface
   And the unix stimestamp is used to ignore series that have been stale since
   the timestamp.
 
-* Delete(int, string) ([]schema.MetricDefinition, error):
+* Delete(int, string) ([]Archive, error):
   This method is used for deleting items from the index. The method is passed
   an OrgId and a query pattern.  If the pattern matches a branch node, then
   all leaf nodes on that branch should also be deleted. So if the pattern is
   "*", all items in the index should be deleted.  A copy of all of the
   metricDefinitions deleted are returned.
 
-* Prune(int, time.Time) ([]schema.MetricDefinition, error):
+* Prune(int, time.Time) ([]Archive, error):
   This method should delete all metrics from the index for the passed org where
   the last time the metric was seen is older then the passed timestamp. If the org
   passed is -1, then the all orgs should be examined for stale metrics to be deleted.
@@ -88,10 +103,10 @@ Interface
 type MetricIndex interface {
 	Init() error
 	Stop()
-	AddOrUpdate(*schema.MetricData, int32, uint16, uint16)
-	Get(string) (schema.MetricDefinition, bool)
-	Delete(int, string) ([]schema.MetricDefinition, error)
+	AddOrUpdate(*schema.MetricData, int32) Archive
+	Get(string) (Archive, bool)
+	Delete(int, string) ([]Archive, error)
 	Find(int, string, int64) ([]Node, error)
-	List(int) []schema.MetricDefinition
-	Prune(int, time.Time) ([]schema.MetricDefinition, error)
+	List(int) []Archive
+	Prune(int, time.Time) ([]Archive, error)
 }
diff --git a/idx/idx_gen.go b/idx/idx_gen.go
index eca6087737..60c47cfdfb 100644
--- a/idx/idx_gen.go
+++ b/idx/idx_gen.go
@@ -4,22 +4,160 @@ package idx
 // MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)
 // DO NOT EDIT
 
-import (
-	"github.com/tinylib/msgp/msgp"
-	"gopkg.in/raintank/schema.v1"
-)
+import "github.com/tinylib/msgp/msgp"
 
 // DecodeMsg implements msgp.Decodable
-func (z *Node) DecodeMsg(dc *msgp.Reader) (err error) {
+func (z *Archive) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zxvk uint32
+	zxvk, err = dc.ReadMapHeader()
+	if err != nil {
+		return
+	}
+	for zxvk > 0 {
+		zxvk--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "MetricDefinition":
+			err = z.MetricDefinition.DecodeMsg(dc)
+			if err != nil {
+				return
+			}
+		case "SchemaId":
+			z.SchemaId, err = dc.ReadUint16()
+			if err != nil {
+				return
+			}
+		case "AggId":
+			z.AggId, err = dc.ReadUint16()
+			if err != nil {
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *Archive) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 3
+	// write "MetricDefinition"
+	err = en.Append(0x83, 0xb0, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e)
+	if err != nil {
+		return err
+	}
+	err = z.MetricDefinition.EncodeMsg(en)
+	if err != nil {
+		return
+	}
+	// write "SchemaId"
+	err = en.Append(0xa8, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x49, 0x64)
+	if err != nil {
+		return err
+	}
+	err = en.WriteUint16(z.SchemaId)
+	if err != nil {
+		return
+	}
+	// write "AggId"
+	err = en.Append(0xa5, 0x41, 0x67, 0x67, 0x49, 0x64)
+	if err != nil {
+		return err
+	}
+	err = en.WriteUint16(z.AggId)
+	if err != nil {
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *Archive) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 3
+	// string "MetricDefinition"
+	o = append(o, 0x83, 0xb0, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e)
+	o, err = z.MetricDefinition.MarshalMsg(o)
+	if err != nil {
+		return
+	}
+	// string "SchemaId"
+	o = append(o, 0xa8, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x49, 0x64)
+	o = msgp.AppendUint16(o, z.SchemaId)
+	// string "AggId"
+	o = append(o, 0xa5, 0x41, 0x67, 0x67, 0x49, 0x64)
+	o = msgp.AppendUint16(o, z.AggId)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *Archive) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var field []byte
 	_ = field
 	var zbzg uint32
-	zbzg, err = dc.ReadMapHeader()
+	zbzg, bts, err = msgp.ReadMapHeaderBytes(bts)
 	if err != nil {
 		return
 	}
 	for zbzg > 0 {
 		zbzg--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "MetricDefinition":
+			bts, err = z.MetricDefinition.UnmarshalMsg(bts)
+			if err != nil {
+				return
+			}
+		case "SchemaId":
+			z.SchemaId, bts, err = msgp.ReadUint16Bytes(bts)
+			if err != nil {
+				return
+			}
+		case "AggId":
+			z.AggId, bts, err = msgp.ReadUint16Bytes(bts)
+			if err != nil {
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *Archive) Msgsize() (s int) {
+	s = 1 + 17 + z.MetricDefinition.Msgsize() + 9 + msgp.Uint16Size + 6 + msgp.Uint16Size
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *Node) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zcmr uint32
+	zcmr, err = dc.ReadMapHeader()
+	if err != nil {
+		return
+	}
+	for zcmr > 0 {
+		zcmr--
 		field, err = dc.ReadMapKeyPtr()
 		if err != nil {
 			return
@@ -36,21 +174,56 @@ func (z *Node) DecodeMsg(dc *msgp.Reader) (err error) {
 				return
 			}
 		case "Defs":
-			var zbai uint32
-			zbai, err = dc.ReadArrayHeader()
+			var zajw uint32
+			zajw, err = dc.ReadArrayHeader()
 			if err != nil {
 				return
 			}
-			if cap(z.Defs) >= int(zbai) {
-				z.Defs = (z.Defs)[:zbai]
+			if cap(z.Defs) >= int(zajw) {
+				z.Defs = (z.Defs)[:zajw]
 			} else {
-				z.Defs = make([]schema.MetricDefinition, zbai)
+				z.Defs = make([]Archive, zajw)
 			}
-			for zxvk := range z.Defs {
-				err = z.Defs[zxvk].DecodeMsg(dc)
+			for zbai := range z.Defs {
+				var zwht uint32
+				zwht, err = dc.ReadMapHeader()
 				if err != nil {
 					return
 				}
+				for zwht > 0 {
+					zwht--
+					field, err = dc.ReadMapKeyPtr()
+					if err != nil {
+						return
+					}
+					switch msgp.UnsafeString(field) {
+					case "MetricDefinition":
+						err = z.Defs[zbai].MetricDefinition.DecodeMsg(dc)
+						if err != nil {
+							return
+						}
+					case "SchemaId":
+						z.Defs[zbai].SchemaId, err = dc.ReadUint16()
+						if err != nil {
+							return
+						}
+					case "AggId":
+						z.Defs[zbai].AggId, err = dc.ReadUint16()
+						if err != nil {
+							return
+						}
+					default:
+						err = dc.Skip()
+						if err != nil {
+							return
+						}
+					}
+				}
+			}
+		case "HasChildren":
+			z.HasChildren, err = dc.ReadBool()
+			if err != nil {
+				return
 			}
 		default:
 			err = dc.Skip()
@@ -64,9 +237,9 @@ func (z *Node) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *Node) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 3
+	// map header, size 4
 	// write "Path"
-	err = en.Append(0x83, 0xa4, 0x50, 0x61, 0x74, 0x68)
+	err = en.Append(0x84, 0xa4, 0x50, 0x61, 0x74, 0x68)
 	if err != nil {
 		return err
 	}
@@ -92,21 +265,54 @@ func (z *Node) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	for zxvk := range z.Defs {
-		err = z.Defs[zxvk].EncodeMsg(en)
+	for zbai := range z.Defs {
+		// map header, size 3
+		// write "MetricDefinition"
+		err = en.Append(0x83, 0xb0, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e)
+		if err != nil {
+			return err
+		}
+		err = z.Defs[zbai].MetricDefinition.EncodeMsg(en)
+		if err != nil {
+			return
+		}
+		// write "SchemaId"
+		err = en.Append(0xa8, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x49, 0x64)
+		if err != nil {
+			return err
+		}
+		err = en.WriteUint16(z.Defs[zbai].SchemaId)
+		if err != nil {
+			return
+		}
+		// write "AggId"
+		err = en.Append(0xa5, 0x41, 0x67, 0x67, 0x49, 0x64)
+		if err != nil {
+			return err
+		}
+		err = en.WriteUint16(z.Defs[zbai].AggId)
 		if err != nil {
 			return
 		}
 	}
+	// write "HasChildren"
+	err = en.Append(0xab, 0x48, 0x61, 0x73, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e)
+	if err != nil {
+		return err
+	}
+	err = en.WriteBool(z.HasChildren)
+	if err != nil {
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *Node) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 3
+	// map header, size 4
 	// string "Path"
-	o = append(o, 0x83, 0xa4, 0x50, 0x61, 0x74, 0x68)
+	o = append(o, 0x84, 0xa4, 0x50, 0x61, 0x74, 0x68)
 	o = msgp.AppendString(o, z.Path)
 	// string "Leaf"
 	o = append(o, 0xa4, 0x4c, 0x65, 0x61, 0x66)
@@ -114,12 +320,24 @@ func (z *Node) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "Defs"
 	o = append(o, 0xa4, 0x44, 0x65, 0x66, 0x73)
 	o = msgp.AppendArrayHeader(o, uint32(len(z.Defs)))
-	for zxvk := range z.Defs {
-		o, err = z.Defs[zxvk].MarshalMsg(o)
+	for zbai := range z.Defs {
+		// map header, size 3
+		// string "MetricDefinition"
+		o = append(o, 0x83, 0xb0, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e)
+		o, err = z.Defs[zbai].MetricDefinition.MarshalMsg(o)
 		if err != nil {
 			return
 		}
+		// string "SchemaId"
+		o = append(o, 0xa8, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x49, 0x64)
+		o = msgp.AppendUint16(o, z.Defs[zbai].SchemaId)
+		// string "AggId"
+		o = append(o, 0xa5, 0x41, 0x67, 0x67, 0x49, 0x64)
+		o = msgp.AppendUint16(o, z.Defs[zbai].AggId)
 	}
+	// string "HasChildren"
+	o = append(o, 0xab, 0x48, 0x61, 0x73, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e)
+	o = msgp.AppendBool(o, z.HasChildren)
 	return
 }
 
@@ -127,13 +345,13 @@ func (z *Node) MarshalMsg(b []byte) (o []byte, err error) {
 func (z *Node) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var field []byte
 	_ = field
-	var zcmr uint32
-	zcmr, bts, err = msgp.ReadMapHeaderBytes(bts)
+	var zhct uint32
+	zhct, bts, err = msgp.ReadMapHeaderBytes(bts)
 	if err != nil {
 		return
 	}
-	for zcmr > 0 {
-		zcmr--
+	for zhct > 0 {
+		zhct--
 		field, bts, err = msgp.ReadMapKeyZC(bts)
 		if err != nil {
 			return
@@ -150,21 +368,56 @@ func (z *Node) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				return
 			}
 		case "Defs":
-			var zajw uint32
-			zajw, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			var zcua uint32
+			zcua, bts, err = msgp.ReadArrayHeaderBytes(bts)
 			if err != nil {
 				return
 			}
-			if cap(z.Defs) >= int(zajw) {
-				z.Defs = (z.Defs)[:zajw]
+			if cap(z.Defs) >= int(zcua) {
+				z.Defs = (z.Defs)[:zcua]
 			} else {
-				z.Defs = make([]schema.MetricDefinition, zajw)
+				z.Defs = make([]Archive, zcua)
 			}
-			for zxvk := range z.Defs {
-				bts, err = z.Defs[zxvk].UnmarshalMsg(bts)
+			for zbai := range z.Defs {
+				var zxhx uint32
+				zxhx, bts, err = msgp.ReadMapHeaderBytes(bts)
 				if err != nil {
 					return
 				}
+				for zxhx > 0 {
+					zxhx--
+					field, bts, err = msgp.ReadMapKeyZC(bts)
+					if err != nil {
+						return
+					}
+					switch msgp.UnsafeString(field) {
+					case "MetricDefinition":
+						bts, err = z.Defs[zbai].MetricDefinition.UnmarshalMsg(bts)
+						if err != nil {
+							return
+						}
+					case "SchemaId":
+						z.Defs[zbai].SchemaId, bts, err = msgp.ReadUint16Bytes(bts)
+						if err != nil {
+							return
+						}
+					case "AggId":
+						z.Defs[zbai].AggId, bts, err = msgp.ReadUint16Bytes(bts)
+						if err != nil {
+							return
+						}
+					default:
+						bts, err = msgp.Skip(bts)
+						if err != nil {
+							return
+						}
+					}
+				}
+			}
+		case "HasChildren":
+			z.HasChildren, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				return
 			}
 		default:
 			bts, err = msgp.Skip(bts)
@@ -180,8 +433,9 @@ func (z *Node) UnmarshalMsg(bts []byte) (o []byte, err error) {
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *Node) Msgsize() (s int) {
 	s = 1 + 5 + msgp.StringPrefixSize + len(z.Path) + 5 + msgp.BoolSize + 5 + msgp.ArrayHeaderSize
-	for zxvk := range z.Defs {
-		s += z.Defs[zxvk].Msgsize()
+	for zbai := range z.Defs {
+		s += 1 + 17 + z.Defs[zbai].MetricDefinition.Msgsize() + 9 + msgp.Uint16Size + 6 + msgp.Uint16Size
 	}
+	s += 12 + msgp.BoolSize
 	return
 }
diff --git a/idx/idx_gen_test.go b/idx/idx_gen_test.go
index 092a65dab8..375d9b508d 100644
--- a/idx/idx_gen_test.go
+++ b/idx/idx_gen_test.go
@@ -11,6 +11,119 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
+func TestMarshalUnmarshalArchive(t *testing.T) {
+	v := Archive{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgArchive(b *testing.B) {
+	v := Archive{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgArchive(b *testing.B) {
+	v := Archive{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalArchive(b *testing.B) {
+	v := Archive{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeArchive(t *testing.T) {
+	v := Archive{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Logf("WARNING: Msgsize() for %v is inaccurate", v)
+	}
+
+	vn := Archive{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeArchive(b *testing.B) {
+	v := Archive{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeArchive(b *testing.B) {
+	v := Archive{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalNode(t *testing.T) {
 	v := Node{}
 	bts, err := v.MarshalMsg(nil)
diff --git a/idx/memory/memory.go b/idx/memory/memory.go
index d218610837..f4b55dfc31 100644
--- a/idx/memory/memory.go
+++ b/idx/memory/memory.go
@@ -59,9 +59,6 @@ type Node struct {
 	Path     string
 	Children []string
 	Defs     []string
-	SchemaI  uint16 // index in mdata.schemas (not persisted)
-	AggI     uint16 // index in mdata.aggregations (not persisted)
-
 }
 
 func (n *Node) HasChildren() bool {
@@ -82,13 +79,13 @@ func (n *Node) String() string {
 // Implements the the "MetricIndex" interface
 type MemoryIdx struct {
 	sync.RWMutex
-	DefById map[string]*schema.MetricDefinition
+	DefById map[string]*idx.Archive
 	Tree    map[int]*Tree
 }
 
 func New() *MemoryIdx {
 	return &MemoryIdx{
-		DefById: make(map[string]*schema.MetricDefinition),
+		DefById: make(map[string]*idx.Archive),
 		Tree:    make(map[int]*Tree),
 	}
 }
@@ -101,7 +98,7 @@ func (m *MemoryIdx) Stop() {
 	return
 }
 
-func (m *MemoryIdx) AddOrUpdate(data *schema.MetricData, partition int32, schemaI, aggI uint16) {
+func (m *MemoryIdx) AddOrUpdate(data *schema.MetricData, partition int32) idx.Archive {
 	pre := time.Now()
 	m.Lock()
 	defer m.Unlock()
@@ -112,14 +109,15 @@ func (m *MemoryIdx) AddOrUpdate(data *schema.MetricData, partition int32, schema
 		existing.Partition = partition
 		statUpdate.Inc()
 		statUpdateDuration.Value(time.Since(pre))
-		return
+		return *existing
 	}
 
 	def := schema.MetricDefinitionFromMetricData(data)
 	def.Partition = partition
-	m.add(def, schemaI, aggI)
+	archive := m.add(def)
 	statMetricsActive.Inc()
 	statAddDuration.Value(time.Since(pre))
+	return archive
 }
 
 // Used to rebuild the index from an existing set of metricDefinitions.
@@ -133,10 +131,7 @@ func (m *MemoryIdx) Load(defs []schema.MetricDefinition) int {
 		if _, ok := m.DefById[def.Id]; ok {
 			continue
 		}
-		schemaI, _ := mdata.MatchSchema(def.Name)
-		aggI, _ := mdata.MatchAgg(def.Name)
-
-		m.add(def, schemaI, aggI)
+		m.add(def)
 		num++
 		statMetricsActive.Inc()
 		statAddDuration.Value(time.Since(pre))
@@ -145,24 +140,16 @@ func (m *MemoryIdx) Load(defs []schema.MetricDefinition) int {
 	return num
 }
 
-func (m *MemoryIdx) AddOrUpdateDef(def *schema.MetricDefinition, schemaI, aggI uint16) {
-	pre := time.Now()
-	m.Lock()
-	defer m.Unlock()
-	if _, ok := m.DefById[def.Id]; ok {
-		log.Debug("memory-idx: metricDef with id %s already in index.", def.Id)
-		m.DefById[def.Id] = def
-		statUpdate.Inc()
-		statUpdateDuration.Value(time.Since(pre))
-		return
+func (m *MemoryIdx) add(def *schema.MetricDefinition) idx.Archive {
+	path := def.Name
+	schemaId, _ := mdata.MatchSchema(def.Name)
+	aggId, _ := mdata.MatchAgg(def.Name)
+	archive := &idx.Archive{
+		*def,
+		schemaId,
+		aggId,
 	}
-	m.add(def, schemaI, aggI)
-	statMetricsActive.Inc()
-	statAddDuration.Value(time.Since(pre))
-}
 
-func (m *MemoryIdx) add(def *schema.MetricDefinition, schemaI, aggI uint16) {
-	path := def.Name
 	//first check to see if a tree has been created for this OrgId
 	tree, ok := m.Tree[def.OrgId]
 	if !ok || len(tree.Items) == 0 {
@@ -183,9 +170,9 @@ func (m *MemoryIdx) add(def *schema.MetricDefinition, schemaI, aggI uint16) {
 		if node, ok := tree.Items[path]; ok {
 			log.Debug("memory-idx: existing index entry for %s. Adding %s to Defs list", path, def.Id)
 			node.Defs = append(node.Defs, def.Id)
-			m.DefById[def.Id] = def
+			m.DefById[def.Id] = archive
 			statAdd.Inc()
-			return
+			return *archive
 		}
 	}
 	// now walk backwards through the node path to find the first branch which exists that
@@ -236,14 +223,13 @@ func (m *MemoryIdx) add(def *schema.MetricDefinition, schemaI, aggI uint16) {
 		Path:     path,
 		Children: []string{},
 		Defs:     []string{def.Id},
-		SchemaI:  schemaI,
-		AggI:     aggI,
 	}
-	m.DefById[def.Id] = def
+	m.DefById[def.Id] = archive
 	statAdd.Inc()
+	return *archive
 }
 
-func (m *MemoryIdx) Get(id string) (schema.MetricDefinition, bool) {
+func (m *MemoryIdx) Get(id string) (idx.Archive, bool) {
 	pre := time.Now()
 	m.RLock()
 	defer m.RUnlock()
@@ -252,7 +238,7 @@ func (m *MemoryIdx) Get(id string) (schema.MetricDefinition, bool) {
 	if ok {
 		return *def, ok
 	}
-	return schema.MetricDefinition{}, ok
+	return idx.Archive{}, ok
 }
 
 func (m *MemoryIdx) Find(orgId int, pattern string, from int64) ([]idx.Node, error) {
@@ -279,11 +265,9 @@ func (m *MemoryIdx) Find(orgId int, pattern string, from int64) ([]idx.Node, err
 				Path:        n.Path,
 				Leaf:        n.Leaf(),
 				HasChildren: n.HasChildren(),
-				SchemaI:     n.SchemaI,
-				AggI:        n.AggI,
 			}
 			if idxNode.Leaf {
-				idxNode.Defs = make([]schema.MetricDefinition, 0, len(n.Defs))
+				idxNode.Defs = make([]idx.Archive, 0, len(n.Defs))
 				for _, id := range n.Defs {
 					def := m.DefById[id]
 					if from != 0 && def.LastUpdate < from {
@@ -429,7 +413,7 @@ func match(pattern string, candidates []string) ([]string, error) {
 	return results, nil
 }
 
-func (m *MemoryIdx) List(orgId int) []schema.MetricDefinition {
+func (m *MemoryIdx) List(orgId int) []idx.Archive {
 	pre := time.Now()
 	m.RLock()
 	defer m.RUnlock()
@@ -443,7 +427,7 @@ func (m *MemoryIdx) List(orgId int) []schema.MetricDefinition {
 			i++
 		}
 	}
-	defs := make([]schema.MetricDefinition, 0)
+	defs := make([]idx.Archive, 0)
 	for _, org := range orgs {
 		tree, ok := m.Tree[org]
 		if !ok {
@@ -463,8 +447,8 @@ func (m *MemoryIdx) List(orgId int) []schema.MetricDefinition {
 	return defs
 }
 
-func (m *MemoryIdx) Delete(orgId int, pattern string) ([]schema.MetricDefinition, error) {
-	var deletedDefs []schema.MetricDefinition
+func (m *MemoryIdx) Delete(orgId int, pattern string) ([]idx.Archive, error) {
+	var deletedDefs []idx.Archive
 	pre := time.Now()
 	m.Lock()
 	defer m.Unlock()
@@ -482,9 +466,9 @@ func (m *MemoryIdx) Delete(orgId int, pattern string) ([]schema.MetricDefinition
 	return deletedDefs, nil
 }
 
-func (m *MemoryIdx) delete(orgId int, n *Node) []schema.MetricDefinition {
+func (m *MemoryIdx) delete(orgId int, n *Node) []idx.Archive {
 	tree := m.Tree[orgId]
-	deletedDefs := make([]schema.MetricDefinition, 0)
+	deletedDefs := make([]idx.Archive, 0)
 	if n.HasChildren() {
 		log.Debug("memory-idx: deleting branch %s", n.Path)
 		// walk up the tree to find all leaf nodes and delete them.
@@ -562,9 +546,9 @@ func (m *MemoryIdx) delete(orgId int, n *Node) []schema.MetricDefinition {
 }
 
 // delete series from the index if they have not been seen since "oldest"
-func (m *MemoryIdx) Prune(orgId int, oldest time.Time) ([]schema.MetricDefinition, error) {
+func (m *MemoryIdx) Prune(orgId int, oldest time.Time) ([]idx.Archive, error) {
 	oldestUnix := oldest.Unix()
-	pruned := make([]schema.MetricDefinition, 0)
+	var pruned []idx.Archive
 	pre := time.Now()
 	m.RLock()
 	orgs := []int{orgId}
diff --git a/idx/memory/memory_find_test.go b/idx/memory/memory_find_test.go
index bf41dea760..c1bef662af 100644
--- a/idx/memory/memory_find_test.go
+++ b/idx/memory/memory_find_test.go
@@ -63,7 +63,7 @@ func Init() {
 			OrgId:    1,
 		}
 		data.SetId()
-		ix.AddOrUpdate(data, 1, 0, 0)
+		ix.AddOrUpdate(data, 1)
 	}
 	for _, series := range diskMetrics(5, 1000, 0, 10, "collectd") {
 		data = &schema.MetricData{
@@ -73,7 +73,7 @@ func Init() {
 			OrgId:    1,
 		}
 		data.SetId()
-		ix.AddOrUpdate(data, 1, 0, 0)
+		ix.AddOrUpdate(data, 1)
 	}
 	// orgId has 1,680,000 series
 
@@ -85,7 +85,7 @@ func Init() {
 			OrgId:    2,
 		}
 		data.SetId()
-		ix.AddOrUpdate(data, 1, 0, 0)
+		ix.AddOrUpdate(data, 1)
 	}
 	for _, series := range diskMetrics(5, 100, 950, 10, "collectd") {
 		data = &schema.MetricData{
@@ -95,7 +95,7 @@ func Init() {
 			OrgId:    2,
 		}
 		data.SetId()
-		ix.AddOrUpdate(data, 1, 0, 0)
+		ix.AddOrUpdate(data, 1)
 	}
 	//orgId 2 has 168,000 mertics
 
diff --git a/idx/memory/memory_test.go b/idx/memory/memory_test.go
index fd108a255c..625e5cfd7b 100644
--- a/idx/memory/memory_test.go
+++ b/idx/memory/memory_test.go
@@ -68,7 +68,7 @@ func TestGetAddKey(t *testing.T) {
 		orgId := series[0].OrgId
 		Convey(fmt.Sprintf("When indexing metrics for orgId %d", orgId), t, func() {
 			for _, s := range series {
-				ix.AddOrUpdate(s, 1, 0, 0)
+				ix.AddOrUpdate(s, 1)
 			}
 			Convey(fmt.Sprintf("Then listing metrics for OrgId %d", orgId), func() {
 				defs := ix.List(orgId)
@@ -86,7 +86,7 @@ func TestGetAddKey(t *testing.T) {
 		for _, series := range org1Series {
 			series.Interval = 60
 			series.SetId()
-			ix.AddOrUpdate(series, 1, 0, 0)
+			ix.AddOrUpdate(series, 1)
 		}
 		Convey("then listing metrics", func() {
 			defs := ix.List(1)
@@ -100,23 +100,23 @@ func TestFind(t *testing.T) {
 	ix.Init()
 	for _, s := range getMetricData(-1, 2, 5, 10, "metric.demo") {
 		s.Time = 10 * 86400
-		ix.AddOrUpdate(s, 1, 0, 0)
+		ix.AddOrUpdate(s, 1)
 	}
 	for _, s := range getMetricData(1, 2, 5, 10, "metric.demo") {
 		s.Time = 10 * 86400
-		ix.AddOrUpdate(s, 1, 0, 0)
+		ix.AddOrUpdate(s, 1)
 	}
 	for _, s := range getMetricData(1, 1, 5, 10, "foo.demo") {
 		s.Time = 1 * 86400
-		ix.AddOrUpdate(s, 1, 0, 0)
+		ix.AddOrUpdate(s, 1)
 		s.Time = 2 * 86400
 		s.Interval = 60
 		s.SetId()
-		ix.AddOrUpdate(s, 1, 0, 0)
+		ix.AddOrUpdate(s, 1)
 	}
 	for _, s := range getMetricData(2, 2, 5, 10, "metric.foo") {
 		s.Time = 1 * 86400
-		ix.AddOrUpdate(s, 1, 0, 0)
+		ix.AddOrUpdate(s, 1)
 	}
 
 	Convey("When listing root nodes", t, func() {
@@ -226,10 +226,10 @@ func TestDelete(t *testing.T) {
 	org1Series := getMetricData(1, 2, 5, 10, "metric.org1")
 
 	for _, s := range publicSeries {
-		ix.AddOrUpdate(s, 1, 0, 0)
+		ix.AddOrUpdate(s, 1)
 	}
 	for _, s := range org1Series {
-		ix.AddOrUpdate(s, 1, 0, 0)
+		ix.AddOrUpdate(s, 1)
 	}
 	Convey("when deleting exact path", t, func() {
 		defs, err := ix.Delete(1, org1Series[0].Name)
@@ -307,10 +307,10 @@ func TestMixedBranchLeaf(t *testing.T) {
 
 	Convey("when adding the first metric", t, func() {
 
-		ix.AddOrUpdate(first, 1, 0, 0)
+		ix.AddOrUpdate(first, 1)
 		Convey("we should be able to add a leaf under another leaf", func() {
 
-			ix.AddOrUpdate(second, 1, 0, 0)
+			ix.AddOrUpdate(second, 1)
 			_, ok := ix.Get(second.Id)
 			So(ok, ShouldEqual, true)
 			defs := ix.List(1)
@@ -318,7 +318,7 @@ func TestMixedBranchLeaf(t *testing.T) {
 		})
 		Convey("we should be able to add a leaf that collides with an existing branch", func() {
 
-			ix.AddOrUpdate(third, 1, 0, 0)
+			ix.AddOrUpdate(third, 1)
 			_, ok := ix.Get(third.Id)
 			So(ok, ShouldEqual, true)
 			defs := ix.List(1)
@@ -364,7 +364,7 @@ func TestMixedBranchLeafDelete(t *testing.T) {
 	}
 	for _, s := range series {
 		s.SetId()
-		ix.AddOrUpdate(s, 1, 0, 0)
+		ix.AddOrUpdate(s, 1)
 	}
 	Convey("when deleting mixed leaf/branch", t, func() {
 		defs, err := ix.Delete(1, "a.b.c")
@@ -424,7 +424,7 @@ func TestPrune(t *testing.T) {
 			Time:     1,
 		}
 		d.SetId()
-		ix.AddOrUpdate(d, 1, 0, 0)
+		ix.AddOrUpdate(d, 1)
 	}
 	//new series
 	for _, s := range getSeriesNames(2, 5, "metric.foo") {
@@ -436,7 +436,7 @@ func TestPrune(t *testing.T) {
 			Time:     10,
 		}
 		d.SetId()
-		ix.AddOrUpdate(d, 1, 0, 0)
+		ix.AddOrUpdate(d, 1)
 	}
 	Convey("after populating index", t, func() {
 		defs := ix.List(-1)
@@ -457,11 +457,16 @@ func TestPrune(t *testing.T) {
 	Convey("after purge", t, func() {
 		defs := ix.List(-1)
 		So(defs, ShouldHaveLength, 5)
-		newDef := defs[0]
-		newDef.Interval = 30
-		newDef.LastUpdate = 100
-		newDef.SetId()
-		ix.AddOrUpdateDef(&newDef, 0, 0)
+		data := &schema.MetricData{
+			Name:     defs[0].Name,
+			Metric:   defs[0].Metric,
+			Id:       defs[0].Id,
+			OrgId:    1,
+			Interval: 30,
+			Time:     100,
+		}
+		data.SetId()
+		ix.AddOrUpdate(data, 0)
 		Convey("When purging old series", func() {
 			purged, err := ix.Prune(1, time.Unix(12, 0))
 			So(err, ShouldBeNil)
@@ -491,6 +496,6 @@ func BenchmarkIndexing(b *testing.B) {
 			OrgId:    1,
 		}
 		data.SetId()
-		ix.AddOrUpdate(data, 1, 0, 0)
+		ix.AddOrUpdate(data, 1)
 	}
 }
diff --git a/input/carbon/carbon.go b/input/carbon/carbon.go
index 3961c597a9..9b6dfbc381 100644
--- a/input/carbon/carbon.go
+++ b/input/carbon/carbon.go
@@ -172,7 +172,7 @@ func (c *Carbon) handle(conn net.Conn) {
 			continue
 		}
 		name := string(key)
-		_, s := mdata.MatchSchema(name) // note: also called by metrictank DefaultHandler.Process. maybe can be optimized
+		_, s := mdata.MatchSchema(name) // note: also called in idx.AddOrUpdate via metrictank DefaultHandler.Process. maybe can be optimized
 		interval := s.Retentions[0].SecondsPerPoint
 		md := &schema.MetricData{
 			Name:     name,
diff --git a/input/input.go b/input/input.go
index 3f32812f2a..4769657b54 100644
--- a/input/input.go
+++ b/input/input.go
@@ -67,15 +67,12 @@ func (in DefaultHandler) Process(metric *schema.MetricData, partition int32) {
 		return
 	}
 
-	schemaI, _ := mdata.MatchSchema(metric.Name)
-	aggI, _ := mdata.MatchAgg(metric.Name)
-
 	pre := time.Now()
-	in.metricIndex.AddOrUpdate(metric, partition, schemaI, aggI)
+	archive := in.metricIndex.AddOrUpdate(metric, partition)
 	in.pressureIdx.Add(int(time.Since(pre).Nanoseconds()))
 
 	pre = time.Now()
-	m := in.metrics.GetOrCreate(metric.Id, metric.Name, schemaI, aggI)
+	m := in.metrics.GetOrCreate(metric.Id, metric.Name, archive.SchemaId, archive.AggId)
 	m.Add(uint32(metric.Time), metric.Value)
 	if in.usage != nil {
 		in.usage.Add(metric.OrgId, metric.Id)
diff --git a/input/input_test.go b/input/input_test.go
index 43cfb12d61..0cb41487ae 100644
--- a/input/input_test.go
+++ b/input/input_test.go
@@ -22,7 +22,6 @@ func BenchmarkProcessUniqueMetrics(b *testing.B) {
 
 	mdata.SetSingleSchema(conf.NewRetentionMT(10, 10000, 600, 10, true))
 	mdata.SetSingleAgg(conf.Avg, conf.Min, conf.Max)
-	mdata.Cache(time.Minute, time.Hour)
 
 	aggmetrics := mdata.NewAggMetrics(store, &cache.MockCache{}, 800, 8000, 0)
 	metricIndex := memory.New()
@@ -62,7 +61,6 @@ func BenchmarkProcessSameMetric(b *testing.B) {
 
 	mdata.SetSingleSchema(conf.NewRetentionMT(10, 10000, 600, 10, true))
 	mdata.SetSingleAgg(conf.Avg, conf.Min, conf.Max)
-	mdata.Cache(time.Minute, time.Hour)
 
 	aggmetrics := mdata.NewAggMetrics(store, &cache.MockCache{}, 800, 8000, 0)
 	metricIndex := memory.New()
diff --git a/mdata/aggmetrics.go b/mdata/aggmetrics.go
index 60fb74fc07..ff10448982 100644
--- a/mdata/aggmetrics.go
+++ b/mdata/aggmetrics.go
@@ -79,12 +79,12 @@ func (ms *AggMetrics) Get(key string) (Metric, bool) {
 	return m, ok
 }
 
-func (ms *AggMetrics) GetOrCreate(key, name string, schemaI, aggI uint16) Metric {
+func (ms *AggMetrics) GetOrCreate(key, name string, schemaId, aggId uint16) Metric {
 	ms.Lock()
 	m, ok := ms.Metrics[key]
 	if !ok {
-		agg := Aggregations.Get(aggI)
-		m = NewAggMetric(ms.store, ms.cachePusher, key, Schemas.Get(schemaI).Retentions, &agg)
+		agg := Aggregations.Get(aggId)
+		m = NewAggMetric(ms.store, ms.cachePusher, key, Schemas.Get(schemaId).Retentions, &agg)
 		ms.Metrics[key] = m
 		metricsActive.Set(len(ms.Metrics))
 	}
diff --git a/mdata/ifaces.go b/mdata/ifaces.go
index cd5ec2080a..823124f9f5 100644
--- a/mdata/ifaces.go
+++ b/mdata/ifaces.go
@@ -7,7 +7,7 @@ import (
 
 type Metrics interface {
 	Get(key string) (Metric, bool)
-	GetOrCreate(key, name string, schemaI, aggI uint16) Metric
+	GetOrCreate(key, name string, schemaId, aggId uint16) Metric
 }
 
 type Metric interface {
diff --git a/mdata/init.go b/mdata/init.go
index 388fba630e..586d29016c 100644
--- a/mdata/init.go
+++ b/mdata/init.go
@@ -6,10 +6,8 @@ package mdata
 import (
 	"io/ioutil"
 	"log"
-	"time"
 
 	"github.com/raintank/metrictank/conf"
-	"github.com/raintank/metrictank/mdata/matchcache"
 	"github.com/raintank/metrictank/stats"
 )
 
@@ -49,8 +47,6 @@ var (
 	// set either via ConfigProcess or from the unit tests. other code should not touch
 	Schemas      conf.Schemas
 	Aggregations conf.Aggregations
-	schemasCache *matchcache.Cache
-	aggsCache    *matchcache.Cache
 
 	schemasFile = "/etc/metrictank/storage-schemas.conf"
 	aggFile     = "/etc/metrictank/storage-aggregation.conf"
@@ -87,10 +83,4 @@ func ConfigProcess() {
 	} else {
 		Aggregations = conf.NewAggregations()
 	}
-	Cache(time.Hour, time.Hour)
-}
-
-func Cache(cleanInterval, expireAfter time.Duration) {
-	schemasCache = matchcache.New(cleanInterval, expireAfter)
-	aggsCache = matchcache.New(cleanInterval, expireAfter)
 }
diff --git a/mdata/matchcache/matchcache.go b/mdata/matchcache/matchcache.go
deleted file mode 100644
index 3805d32a23..0000000000
--- a/mdata/matchcache/matchcache.go
+++ /dev/null
@@ -1,63 +0,0 @@
-package matchcache
-
-import (
-	"sync"
-	"time"
-)
-
-// Cache caches key to uint16 lookups (for schemas and aggregations)
-// when it cleans the cache it locks up the entire cache
-// this is a tradeoff we can make for simplicity, since this sits on the ingestion
-// path, where occasional stalls are ok.
-type Cache struct {
-	sync.Mutex
-	data map[string]Item
-
-	cleanInterval time.Duration
-	expireAfter   time.Duration
-}
-
-type Item struct {
-	val  uint16
-	seen int64
-}
-
-func New(cleanInterval, expireAfter time.Duration) *Cache {
-	m := &Cache{
-		data:          make(map[string]Item),
-		cleanInterval: cleanInterval,
-		expireAfter:   expireAfter,
-	}
-	go m.maintain()
-	return m
-}
-
-type AddFunc func(key string) uint16
-
-// if not in cache, will atomically add it using the provided function
-func (m *Cache) Get(key string, fn AddFunc) uint16 {
-	m.Lock()
-	item, ok := m.data[key]
-	if !ok {
-		item.val = fn(key)
-	}
-	item.seen = time.Now().Unix()
-	m.data[key] = item
-	m.Unlock()
-	return item.val
-}
-
-func (m *Cache) maintain() {
-	ticker := time.NewTicker(m.cleanInterval)
-	diff := int64(m.expireAfter.Seconds())
-	for now := range ticker.C {
-		nowUnix := now.Unix()
-		m.Lock()
-		for key, item := range m.data {
-			if nowUnix-item.seen > diff {
-				delete(m.data, key)
-			}
-		}
-		m.Unlock()
-	}
-}
diff --git a/mdata/notifier.go b/mdata/notifier.go
index 38cebef3cf..e7b343b8a0 100644
--- a/mdata/notifier.go
+++ b/mdata/notifier.go
@@ -85,9 +85,9 @@ func (cl Notifier) Handle(data []byte) {
 				}
 			}
 			if cl.CreateMissingMetrics {
-				schemaI, _ := MatchSchema(c.Name)
-				aggI, _ := MatchAgg(c.Name)
-				agg := cl.Metrics.GetOrCreate(key[0], c.Name, schemaI, aggI)
+				schemaId, _ := MatchSchema(c.Name)
+				aggId, _ := MatchAgg(c.Name)
+				agg := cl.Metrics.GetOrCreate(key[0], c.Name, schemaId, aggId)
 				if len(key) == 3 {
 					agg.(*AggMetric).SyncAggregatedChunkSaveState(c.T0, consolidator, uint32(aggSpan))
 				} else {
@@ -112,9 +112,9 @@ func (cl Notifier) Handle(data []byte) {
 		messagesReceived.Add(1)
 		// get metric
 		if cl.CreateMissingMetrics {
-			schemaI, _ := MatchSchema(ms.Name)
-			aggI, _ := MatchAgg(ms.Name)
-			agg := cl.Metrics.GetOrCreate(ms.Key, ms.Name, schemaI, aggI)
+			schemaId, _ := MatchSchema(ms.Name)
+			aggId, _ := MatchAgg(ms.Name)
+			agg := cl.Metrics.GetOrCreate(ms.Key, ms.Name, schemaId, aggId)
 			agg.(*AggMetric).SyncChunkSaveState(ms.T0)
 		} else if agg, ok := cl.Metrics.Get(ms.Key); ok {
 			agg.(*AggMetric).SyncChunkSaveState(ms.T0)
diff --git a/mdata/schema.go b/mdata/schema.go
index 78311b5096..18e63c588f 100644
--- a/mdata/schema.go
+++ b/mdata/schema.go
@@ -40,21 +40,13 @@ func MaxChunkSpan() uint32 {
 // MatchSchema returns the schema for the given metric key, and the index of the schema (to efficiently reference it)
 // it will always find the schema because Schemas has a catchall default
 func MatchSchema(key string) (uint16, conf.Schema) {
-	i := schemasCache.Get(key, func(key string) uint16 {
-		i, _ := Schemas.Match(key)
-		return i
-	})
-	return i, Schemas.Get(i)
+	return Schemas.Match(key)
 }
 
 // MatchAgg returns the aggregation definition for the given metric key, and the index of it (to efficiently reference it)
 // it will always find the aggregation definition because Aggregations has a catchall default
 func MatchAgg(key string) (uint16, conf.Aggregation) {
-	i := aggsCache.Get(key, func(key string) uint16 {
-		i, _ := Aggregations.Match(key)
-		return i
-	})
-	return i, Aggregations.Get(i)
+	return Aggregations.Match(key)
 }
 
 func SetSingleSchema(ret ...conf.Retention) {
diff --git a/usage/usage.go b/usage/usage.go
index 0701a08226..8b92d441b5 100644
--- a/usage/usage.go
+++ b/usage/usage.go
@@ -101,13 +101,11 @@ func (u *Usage) Report() {
 		met.Value = val
 		met.SetId()
 
-		schemaI, _ := mdata.MatchSchema(name)
-		aggI, _ := mdata.MatchAgg(name)
+		//TODO: how to set the partition of the metric?  We probably just need to publish the metric to our Input Plugin
+		archive := metricIndex.AddOrUpdate(met, 0)
 
-		m := metrics.GetOrCreate(met.Id, met.Metric, schemaI, aggI)
+		m := metrics.GetOrCreate(met.Id, met.Metric, archive.SchemaId, archive.AggId)
 		m.Add(uint32(met.Time), met.Value)
-		//TODO: how to set the partition of the metric?  We probably just need to publish the metric to our Input Plugin
-		metricIndex.AddOrUpdate(met, 0, schemaI, aggI)
 	}
 	for {
 		ticker := tick()
diff --git a/usage/usage_test.go b/usage/usage_test.go
index 9fd4d53e0b..090d6476df 100644
--- a/usage/usage_test.go
+++ b/usage/usage_test.go
@@ -30,7 +30,7 @@ func (f *FakeAggMetrics) Get(key string) (mdata.Metric, bool) {
 	f.Unlock()
 	return m, ok
 }
-func (f *FakeAggMetrics) GetOrCreate(key, name string, schemaI, aggI uint16) mdata.Metric {
+func (f *FakeAggMetrics) GetOrCreate(key, name string, schemaId, aggId uint16) mdata.Metric {
 	f.Lock()
 	m, ok := f.Metrics[key]
 	if !ok {

From 28cc30baff47aaf2002c90b0688558511ef26e54 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Fri, 17 Mar 2017 17:24:55 +0100
Subject: [PATCH 28/29] carbon-in: instead of matching every point, use
 idx.Find

this is marginal performance improvement (tested with 50kHz fakemetrics
workload)

before, 87% of cpu is used by Carbon.handle which spends
        3.88/13.14s=29.5% of its time getting the interval
after, 79.22% and 3.13/13.04=24%

BEFORE:
(pprof) top30 -cum
5.51s of 13.14s total (41.93%)
Dropped 144 nodes (cum <= 0.07s)
Showing top 30 nodes out of 150 (cum >= 0.51s)
      flat  flat%   sum%        cum   cum%
         0     0%     0%     12.81s 97.49%  runtime.goexit
     0.08s  0.61%  1.14%      3.88s 29.53%  github.com/raintank/metrictank/conf.Schemas.Match
         0     0%  1.14%      3.88s 29.53%  github.com/raintank/metrictank/mdata.MatchSchema
     0.02s  0.15%  1.29%      3.79s 28.84%  regexp.(*Regexp).MatchString
     0.06s  0.46%  1.75%      3.77s 28.69%  regexp.(*Regexp).doMatch
     0.03s  0.23%  1.98%      3.71s 28.23%  regexp.(*Regexp).doExecute
     0.19s  1.45%  3.42%      3.36s 25.57%  regexp.(*machine).backtrack
     0.03s  0.23%  3.65%      2.77s 21.08%  github.com/raintank/metrictank/input.(*DefaultHandler).Process
     1.56s 11.87% 15.53%      2.77s 21.08%  regexp.(*machine).tryBacktrack
     0.01s 0.076% 15.60%      2.73s 20.78%  github.com/raintank/metrictank/input.DefaultHandler.Process
     0.02s  0.15% 15.75%      2.36s 17.96%  github.com/raintank/metrictank/vendor/github.com/metrics20/go-metrics20/carbon20.ValidatePacket
     0.01s 0.076% 15.83%      1.89s 14.38%  github.com/raintank/metrictank/vendor/gopkg.in/raintank/schema%2ev1.(*MetricData).SetId
         0     0% 15.83%      1.86s 14.16%  bytes.Fields
     0.77s  5.86% 21.69%      1.86s 14.16%  bytes.FieldsFunc
     0.01s 0.076% 21.77%      1.50s 11.42%  runtime.systemstack
     0.57s  4.34% 26.10%      1.17s  8.90%  runtime.scanobject
         0     0% 26.10%      1.13s  8.60%  runtime.gcBgMarkWorker
         0     0% 26.10%      1.12s  8.52%  runtime.gcBgMarkWorker.func2
     0.02s  0.15% 26.26%      1.12s  8.52%  runtime.gcDrain
     0.50s  3.81% 30.06%      1.06s  8.07%  runtime.mallocgc
     0.20s  1.52% 31.58%      0.97s  7.38%  github.com/raintank/metrictank/mdata.(*AggMetric).Add
     0.01s 0.076% 31.66%      0.89s  6.77%  github.com/raintank/metrictank/idx/cassandra.(*CasIdx).AddOrUpdate
         0     0% 31.66%      0.84s  6.39%  fmt.Sprintf
     0.04s   0.3% 31.96%      0.70s  5.33%  fmt.(*pp).doPrintf
     0.06s  0.46% 32.42%      0.67s  5.10%  runtime.newobject
     0.03s  0.23% 32.65%      0.66s  5.02%  fmt.(*pp).printArg
     0.59s  4.49% 37.14%      0.59s  4.49%  regexp.(*bitState).push
     0.12s  0.91% 38.05%      0.54s  4.11%  github.com/raintank/metrictank/idx/memory.(*MemoryIdx).Get
     0.51s  3.88% 41.93%      0.51s  3.88%  unicode/utf8.DecodeRune
(pprof) list Carbon.*handle
Total: 13.14s
ROUTINE ======================== github.com/raintank/metrictank/input/carbon.(*Carbon).handle in /home/dieter/go/src/github.com/raintank/metrictank/input/carbon/carbon.go
      70ms     11.44s (flat, cum) 87.06% of Total
         .          .    147:	}()
         .          .    148:	// TODO c.SetTimeout(60e9)
         .          .    149:	r := bufio.NewReaderSize(conn, 4096)
         .          .    150:	for {
         .          .    151:		// note that we don't support lines longer than 4096B. that seems very reasonable..
      10ms      140ms    152:		buf, _, err := r.ReadLine()
         .          .    153:
         .          .    154:		if nil != err {
         .          .    155:			select {
         .          .    156:			case <-c.quit:
         .          .    157:				// we are shutting down.
         .          .    158:				break
         .          .    159:			default:
         .          .    160:			}
         .          .    161:			if io.EOF != err {
         .          .    162:				log.Error(4, "carbon-in: Recv error: %s", err.Error())
         .          .    163:			}
         .          .    164:			break
         .          .    165:		}
         .          .    166:
         .          .    167:		// no validation for m2.0 to provide a grace period in adopting new clients
         .      2.36s    168:		key, val, ts, err := carbon20.ValidatePacket(buf, carbon20.MediumLegacy, carbon20.NoneM20)
         .          .    169:		if err != nil {
         .          .    170:			metricsDecodeErr.Inc()
         .          .    171:			log.Error(4, "carbon-in: invalid metric: %s", err.Error())
         .          .    172:			continue
         .          .    173:		}
      10ms       80ms    174:		name := string(key)
         .      3.88s    175:		_, s := mdata.MatchSchema(name) // note: also called in idx.AddOrUpdate via metrictank DefaultHandler.Process. maybe can be optimized
         .          .    176:		interval := s.Retentions[0].SecondsPerPoint
         .          .    177:		md := &schema.MetricData{
         .          .    178:			Name:     name,
         .          .    179:			Metric:   name,
         .          .    180:			Interval: interval,
         .          .    181:			Value:    val,
         .          .    182:			Unit:     "unknown",
         .          .    183:			Time:     int64(ts),
         .          .    184:			Mtype:    "gauge",
         .       20ms    185:			Tags:     []string{},
      10ms      150ms    186:			OrgId:    1, // admin org
         .          .    187:		}
         .      1.89s    188:		md.SetId()
      30ms      140ms    189:		metricsPerMessage.ValueUint32(1)
      10ms      2.78s    190:		c.Handler.Process(md, int32(partitionId))
         .          .    191:	}
         .          .    192:	c.handlerWaitGroup.Done()
         .          .    193:}

AFTER:
(pprof) top30 -cum
5.39s of 13.04s total (41.33%)
Dropped 164 nodes (cum <= 0.07s)
Showing top 30 nodes out of 164 (cum >= 0.46s)
      flat  flat%   sum%        cum   cum%
         0     0%     0%     12.47s 95.63%  runtime.goexit
     0.12s  0.92%  0.92%     10.33s 79.22%  github.com/raintank/metrictank/input/carbon.(*Carbon).handle
         0     0%  0.92%      3.11s 23.85%  github.com/raintank/metrictank/input/carbon.(*IndexIntervalGetter).GetInterval
     0.03s  0.23%  1.15%      3.11s 23.85%  github.com/raintank/metrictank/input/carbon.IndexIntervalGetter.GetInterval
     0.50s  3.83%  4.98%      3.07s 23.54%  github.com/raintank/metrictank/idx/memory.(*MemoryIdx).Find
     0.02s  0.15%  5.14%      2.33s 17.87%  runtime.systemstack
         0     0%  5.14%      2.29s 17.56%  github.com/raintank/metrictank/input.(*DefaultHandler).Process
     0.04s  0.31%  5.44%      2.29s 17.56%  github.com/raintank/metrictank/input.DefaultHandler.Process
     0.03s  0.23%  5.67%      2.28s 17.48%  github.com/raintank/metrictank/vendor/gopkg.in/raintank/schema%2ev1.(*MetricData).SetId
     0.01s 0.077%  5.75%      2.08s 15.95%  github.com/raintank/metrictank/vendor/github.com/metrics20/go-metrics20/carbon20.ValidatePacket
     0.07s  0.54%  6.29%      1.68s 12.88%  github.com/raintank/metrictank/idx/memory.(*MemoryIdx).find
     0.90s  6.90% 13.19%      1.64s 12.58%  runtime.scanobject
         0     0% 13.19%      1.62s 12.42%  runtime.gcBgMarkWorker
     0.89s  6.83% 20.02%      1.62s 12.42%  runtime.mallocgc
         0     0% 20.02%      1.61s 12.35%  runtime.gcBgMarkWorker.func2
     0.02s  0.15% 20.17%      1.61s 12.35%  runtime.gcDrain
         0     0% 20.17%      1.57s 12.04%  bytes.Fields
     0.70s  5.37% 25.54%      1.57s 12.04%  bytes.FieldsFunc
     0.01s 0.077% 25.61%      0.88s  6.75%  fmt.Sprintf
     0.07s  0.54% 26.15%      0.88s  6.75%  runtime.newobject
     0.25s  1.92% 28.07%      0.83s  6.37%  github.com/raintank/metrictank/mdata.(*AggMetric).Add
     0.12s  0.92% 28.99%      0.80s  6.13%  fmt.(*pp).doPrintf
     0.50s  3.83% 32.82%      0.69s  5.29%  runtime.mapaccess2_faststr
     0.02s  0.15% 32.98%      0.68s  5.21%  fmt.(*pp).printArg
     0.03s  0.23% 33.21%      0.61s  4.68%  github.com/raintank/metrictank/idx/cassandra.(*CasIdx).AddOrUpdate
     0.05s  0.38% 33.59%      0.52s  3.99%  runtime.convT2E
     0.10s  0.77% 34.36%      0.51s  3.91%  runtime.makeslice
     0.13s     1% 35.35%      0.48s  3.68%  fmt.(*pp).printValue
     0.32s  2.45% 37.81%      0.47s  3.60%  runtime.mapassign
     0.46s  3.53% 41.33%      0.46s  3.53%  unicode/utf8.DecodeRune
(pprof) list Carbon.*handle
Total: 13.04s
ROUTINE ======================== github.com/raintank/metrictank/input/carbon.(*Carbon).handle in /home/dieter/go/src/github.com/raintank/metrictank/input/carbon/carbon.go
     120ms     10.33s (flat, cum) 79.22% of Total
         .          .    151:	}()
         .          .    152:	// TODO c.SetTimeout(60e9)
         .          .    153:	r := bufio.NewReaderSize(conn, 4096)
         .          .    154:	for {
         .          .    155:		// note that we don't support lines longer than 4096B. that seems very reasonable..
      40ms      190ms    156:		buf, _, err := r.ReadLine()
         .          .    157:
         .          .    158:		if nil != err {
         .          .    159:			select {
         .          .    160:			case <-c.quit:
         .          .    161:				// we are shutting down.
         .          .    162:				break
         .          .    163:			default:
         .          .    164:			}
         .          .    165:			if io.EOF != err {
         .          .    166:				log.Error(4, "carbon-in: Recv error: %s", err.Error())
         .          .    167:			}
         .          .    168:			break
         .          .    169:		}
         .          .    170:
         .          .    171:		// no validation for m2.0 to provide a grace period in adopting new clients
      20ms      2.10s    172:		key, val, ts, err := carbon20.ValidatePacket(buf, carbon20.MediumLegacy, carbon20.NoneM20)
         .          .    173:		if err != nil {
         .          .    174:			metricsDecodeErr.Inc()
         .          .    175:			log.Error(4, "carbon-in: invalid metric: %s", err.Error())
         .          .    176:			continue
         .          .    177:		}
      10ms       80ms    178:		name := string(key)
         .          .    179:		md := &schema.MetricData{
         .          .    180:			Name:     name,
         .          .    181:			Metric:   name,
      20ms      3.13s    182:			Interval: c.intervalGetter.GetInterval(name),
         .          .    183:			Value:    val,
         .          .    184:			Unit:     "unknown",
         .          .    185:			Time:     int64(ts),
         .          .    186:			Mtype:    "gauge",
         .          .    187:			Tags:     []string{},
         .       70ms    188:			OrgId:    1, // admin org
         .          .    189:		}
         .      2.28s    190:		md.SetId()
      30ms      190ms    191:		metricsPerMessage.ValueUint32(1)
         .      2.29s    192:		c.Handler.Process(md, int32(partitionId))
         .          .    193:	}
         .          .    194:	c.handlerWaitGroup.Done()
         .          .    195:}
---
 input/carbon/carbon.go         | 10 +++++----
 input/carbon/intervalgetter.go | 39 ++++++++++++++++++++++++++++++++++
 metrictank.go                  |  3 +++
 3 files changed, 48 insertions(+), 4 deletions(-)
 create mode 100644 input/carbon/intervalgetter.go

diff --git a/input/carbon/carbon.go b/input/carbon/carbon.go
index 9b6dfbc381..c9a38e8792 100644
--- a/input/carbon/carbon.go
+++ b/input/carbon/carbon.go
@@ -12,7 +12,6 @@ import (
 	"github.com/metrics20/go-metrics20/carbon20"
 	"github.com/raintank/metrictank/cluster"
 	"github.com/raintank/metrictank/input"
-	"github.com/raintank/metrictank/mdata"
 	"github.com/raintank/metrictank/stats"
 	"github.com/raintank/worldping-api/pkg/log"
 	"github.com/rakyll/globalconf"
@@ -33,6 +32,7 @@ type Carbon struct {
 	handlerWaitGroup sync.WaitGroup
 	quit             chan struct{}
 	connTrack        *ConnTrack
+	intervalGetter   IntervalGetter
 }
 
 type ConnTrack struct {
@@ -101,6 +101,10 @@ func New() *Carbon {
 	}
 }
 
+func (c *Carbon) IntervalGetter(i IntervalGetter) {
+	c.intervalGetter = i
+}
+
 func (c *Carbon) Start(handler input.Handler) {
 	c.Handler = handler
 	l, err := net.ListenTCP("tcp", c.addr)
@@ -172,12 +176,10 @@ func (c *Carbon) handle(conn net.Conn) {
 			continue
 		}
 		name := string(key)
-		_, s := mdata.MatchSchema(name) // note: also called in idx.AddOrUpdate via metrictank DefaultHandler.Process. maybe can be optimized
-		interval := s.Retentions[0].SecondsPerPoint
 		md := &schema.MetricData{
 			Name:     name,
 			Metric:   name,
-			Interval: interval,
+			Interval: c.intervalGetter.GetInterval(name),
 			Value:    val,
 			Unit:     "unknown",
 			Time:     int64(ts),
diff --git a/input/carbon/intervalgetter.go b/input/carbon/intervalgetter.go
new file mode 100644
index 0000000000..3255f72140
--- /dev/null
+++ b/input/carbon/intervalgetter.go
@@ -0,0 +1,39 @@
+package carbon
+
+import (
+	"github.com/raintank/metrictank/idx"
+	"github.com/raintank/metrictank/mdata"
+)
+
+//IntervalGetter is anything that can return the interval for the given path
+//we don't want the carbon plugin to directly talk to an index because the api
+//surface is too big and it would couple too tightly which is annoying in unit tests
+type IntervalGetter interface {
+	GetInterval(name string) int
+}
+
+type IndexIntervalGetter struct {
+	idx idx.MetricIndex
+}
+
+func NewIndexIntervalGetter(idx idx.MetricIndex) IntervalGetter {
+	return IndexIntervalGetter{idx}
+}
+
+func (i IndexIntervalGetter) GetInterval(name string) int {
+	nodes, err := i.idx.Find(1, name, 0)
+	if err != nil {
+		panic(err)
+	}
+	for _, n := range nodes {
+		// since the schema rules can't change at runtime and are the schemas are determined at runtime for new entries, they will be the same
+		// for any def/archive with the given name. so the first one we find is enough.
+		if n.Leaf {
+			return mdata.Schemas.Get(n.Defs[0].SchemaId).Retentions[0].SecondsPerPoint
+		}
+	}
+	// if it's the first time we're seeing this series, do the more expensive matching
+	// note that the index will also do this matching again first time it sees the metric
+	_, schema := mdata.MatchSchema(name)
+	return schema.Retentions[0].SecondsPerPoint
+}
diff --git a/metrictank.go b/metrictank.go
index fa7b360ee5..247a20ebf5 100644
--- a/metrictank.go
+++ b/metrictank.go
@@ -359,6 +359,9 @@ func main() {
 		Start our inputs
 	***********************************/
 	for _, plugin := range inputs {
+		if carbonPlugin, ok := plugin.(*inCarbon.Carbon); ok {
+			carbonPlugin.IntervalGetter(inCarbon.NewIndexIntervalGetter(metricIndex))
+		}
 		plugin.Start(input.NewDefaultHandler(metrics, metricIndex, usg, plugin.Name()))
 	}
 

From e6db43aeadeb49d558810b6b9f13f13782112726 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@raintank.io>
Date: Fri, 17 Mar 2017 18:37:49 +0100
Subject: [PATCH 29/29] add a quick path to get the archives under a path

and let carbon use it instead of the more expensive idx.Find()

Now Carbon.handle is at 66% of cpu of which 1.15/12.35=9.3% is spent
finding the interval

(pprof) top30 -cum
3.24s of 12.35s total (26.23%)
Dropped 184 nodes (cum <= 0.06s)
Showing top 30 nodes out of 146 (cum >= 0.75s)
      flat  flat%   sum%        cum   cum%
         0     0%     0%     11.12s 90.04%  runtime.goexit
     0.05s   0.4%   0.4%      8.22s 66.56%  github.com/raintank/metrictank/input/carbon.(*Carbon).handle
     0.03s  0.24%  0.65%      2.29s 18.54%  github.com/raintank/metrictank/vendor/github.com/metrics20/go-metrics20/carbon20.ValidatePacket
     0.01s 0.081%  0.73%      2.27s 18.38%  github.com/raintank/metrictank/input.(*DefaultHandler).Process
     0.04s  0.32%  1.05%      2.26s 18.30%  github.com/raintank/metrictank/input.DefaultHandler.Process
     0.05s   0.4%  1.46%         2s 16.19%  github.com/raintank/metrictank/vendor/gopkg.in/raintank/schema%2ev1.(*MetricData).SetId
     0.01s 0.081%  1.54%      1.87s 15.14%  runtime.systemstack
         0     0%  1.54%      1.79s 14.49%  bytes.Fields
     0.82s  6.64%  8.18%      1.79s 14.49%  bytes.FieldsFunc
     0.72s  5.83% 14.01%      1.43s 11.58%  runtime.scanobject
         0     0% 14.01%      1.39s 11.26%  runtime.gcBgMarkWorker
     0.75s  6.07% 20.08%      1.39s 11.26%  runtime.mallocgc
         0     0% 20.08%      1.37s 11.09%  runtime.gcBgMarkWorker.func2
         0     0% 20.08%      1.37s 11.09%  runtime.gcDrain
     0.03s  0.24% 20.32%      1.15s  9.31%  github.com/raintank/metrictank/input/carbon.(*IndexIntervalGetter).GetInterval
         0     0% 20.32%      1.12s  9.07%  github.com/raintank/metrictank/input/carbon.IndexIntervalGetter.GetInterval
     0.41s  3.32% 23.64%      1.09s  8.83%  github.com/raintank/metrictank/idx/memory.(*MemoryIdx).GetPath
     0.02s  0.16% 23.81%      1.07s  8.66%  github.com/raintank/metrictank/mdata.(*CassandraStore).processWriteQueue
     0.01s 0.081% 23.89%      0.97s  7.85%  github.com/raintank/metrictank/mdata.(*CassandraStore).insertChunk
     0.17s  1.38% 25.26%      0.91s  7.37%  github.com/raintank/metrictank/mdata.(*AggMetric).Add
         0     0% 25.26%      0.90s  7.29%  github.com/raintank/metrictank/vendor/github.com/gocql/gocql.(*Query).Exec
         0     0% 25.26%      0.90s  7.29%  github.com/raintank/metrictank/vendor/github.com/gocql/gocql.(*Query).Iter
         0     0% 25.26%      0.90s  7.29%  github.com/raintank/metrictank/vendor/github.com/gocql/gocql.(*Session).executeQuery
         0     0% 25.26%      0.90s  7.29%  github.com/raintank/metrictank/vendor/github.com/gocql/gocql.(*queryExecutor).executeQuery
         0     0% 25.26%      0.89s  7.21%  runtime.mcall
     0.03s  0.24% 25.51%      0.85s  6.88%  runtime.schedule
         0     0% 25.51%      0.80s  6.48%  fmt.Sprintf
     0.04s  0.32% 25.83%      0.80s  6.48%  runtime.findrunnable
     0.04s  0.32% 26.15%      0.80s  6.48%  runtime.newobject
     0.01s 0.081% 26.23%      0.75s  6.07%  github.com/raintank/metrictank/vendor/github.com/gocql/gocql.(*Conn).executeQuery
(pprof) list Carbon.*handle
Total: 12.35s
ROUTINE ======================== github.com/raintank/metrictank/input/carbon.(*Carbon).handle in /home/dieter/go/src/github.com/raintank/metrictank/input/carbon/carbon.go
      50ms      8.22s (flat, cum) 66.56% of Total
         .          .    151:	}()
         .          .    152:	// TODO c.SetTimeout(60e9)
         .          .    153:	r := bufio.NewReaderSize(conn, 4096)
         .          .    154:	for {
         .          .    155:		// note that we don't support lines longer than 4096B. that seems very reasonable..
      20ms      230ms    156:		buf, _, err := r.ReadLine()
         .          .    157:
         .          .    158:		if nil != err {
         .          .    159:			select {
         .          .    160:			case <-c.quit:
         .          .    161:				// we are shutting down.
         .          .    162:				break
         .          .    163:			default:
         .          .    164:			}
         .          .    165:			if io.EOF != err {
         .          .    166:				log.Error(4, "carbon-in: Recv error: %s", err.Error())
         .          .    167:			}
         .          .    168:			break
         .          .    169:		}
         .          .    170:
         .          .    171:		// no validation for m2.0 to provide a grace period in adopting new clients
         .      2.29s    172:		key, val, ts, err := carbon20.ValidatePacket(buf, carbon20.MediumLegacy, carbon20.NoneM20)
         .          .    173:		if err != nil {
         .          .    174:			metricsDecodeErr.Inc()
         .          .    175:			log.Error(4, "carbon-in: invalid metric: %s", err.Error())
         .          .    176:			continue
         .          .    177:		}
         .       40ms    178:		name := string(key)
         .          .    179:		md := &schema.MetricData{
         .          .    180:			Name:     name,
         .          .    181:			Metric:   name,
         .      1.15s    182:			Interval: c.intervalGetter.GetInterval(name),
         .          .    183:			Value:    val,
         .          .    184:			Unit:     "unknown",
         .          .    185:			Time:     int64(ts),
         .          .    186:			Mtype:    "gauge",
      20ms       30ms    187:			Tags:     []string{},
         .      130ms    188:			OrgId:    1, // admin org
         .          .    189:		}
         .         2s    190:		md.SetId()
         .       70ms    191:		metricsPerMessage.ValueUint32(1)
      10ms      2.28s    192:		c.Handler.Process(md, int32(partitionId))
         .          .    193:	}
         .          .    194:	c.handlerWaitGroup.Done()
         .          .    195:}
(pprof)
---
 idx/idx.go                     |  4 ++++
 idx/memory/memory.go           | 19 +++++++++++++++++++
 input/carbon/intervalgetter.go | 13 ++++---------
 3 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/idx/idx.go b/idx/idx.go
index 07b4181284..b83169d064 100644
--- a/idx/idx.go
+++ b/idx/idx.go
@@ -73,6 +73,9 @@ Interface
 * Get(string) (Archive, bool):
   This method should return the MetricDefintion with the passed Id.
 
+* GetPath(string) (Archive, bool) []Archive:
+  This method should return the archives under the given path
+
 * List(int) []Archive:
   This method should return all MetricDefinitions for the passed OrgId.  If the
   passed OrgId is "-1", then all metricDefinitions across all organisations
@@ -105,6 +108,7 @@ type MetricIndex interface {
 	Stop()
 	AddOrUpdate(*schema.MetricData, int32) Archive
 	Get(string) (Archive, bool)
+	GetPath(int, string) []Archive
 	Delete(int, string) ([]Archive, error)
 	Find(int, string, int64) ([]Node, error)
 	List(int) []Archive
diff --git a/idx/memory/memory.go b/idx/memory/memory.go
index f4b55dfc31..df39fa496c 100644
--- a/idx/memory/memory.go
+++ b/idx/memory/memory.go
@@ -241,6 +241,25 @@ func (m *MemoryIdx) Get(id string) (idx.Archive, bool) {
 	return idx.Archive{}, ok
 }
 
+// GetPath returns the node under the given org and path.
+// this is an alternative to Find for when you have a path, not a pattern, and want to lookup in a specific org tree only.
+func (m *MemoryIdx) GetPath(orgId int, path string) []idx.Archive {
+	tree, ok := m.Tree[orgId]
+	if !ok {
+		return nil
+	}
+	node := tree.Items[path]
+	if node == nil {
+		return nil
+	}
+	archives := make([]idx.Archive, len(node.Defs))
+	for i, def := range node.Defs {
+		archive := m.DefById[def]
+		archives[i] = *archive
+	}
+	return archives
+}
+
 func (m *MemoryIdx) Find(orgId int, pattern string, from int64) ([]idx.Node, error) {
 	pre := time.Now()
 	m.RLock()
diff --git a/input/carbon/intervalgetter.go b/input/carbon/intervalgetter.go
index 3255f72140..af3db48498 100644
--- a/input/carbon/intervalgetter.go
+++ b/input/carbon/intervalgetter.go
@@ -21,16 +21,11 @@ func NewIndexIntervalGetter(idx idx.MetricIndex) IntervalGetter {
 }
 
 func (i IndexIntervalGetter) GetInterval(name string) int {
-	nodes, err := i.idx.Find(1, name, 0)
-	if err != nil {
-		panic(err)
-	}
-	for _, n := range nodes {
+	archives := i.idx.GetPath(1, name)
+	for _, a := range archives {
 		// since the schema rules can't change at runtime and are the schemas are determined at runtime for new entries, they will be the same
-		// for any def/archive with the given name. so the first one we find is enough.
-		if n.Leaf {
-			return mdata.Schemas.Get(n.Defs[0].SchemaId).Retentions[0].SecondsPerPoint
-		}
+		// for any archive with the given name. so the first one we find is enough.
+		return a.Interval
 	}
 	// if it's the first time we're seeing this series, do the more expensive matching
 	// note that the index will also do this matching again first time it sees the metric