diff --git a/README.md b/README.md index 38cf8a659a..a0a3115661 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ and bugs to fix. It should be considered an *alpha* project. * no performance/availability isolation between tenants per instance. (only data isolation) * clustering is basic: statically defined peers, master promotions are manual, etc. See [clustering](https://github.com/raintank/metrictank/blob/master/docs/clustering.md) for more. -* no computation locality: we move the data from storage to processing code, which is both metrictank and graphite-api. +* minimum computation locality: we move the data from storage to processing code, which is both metrictank and graphite. * the datastructures can use performance engineering. [A Go GC issue may occassionally inflate response times](https://github.com/golang/go/issues/14812). * the native input protocol is inefficient. Should not send all metadata with each point. * we use metrics2.0 in native input protocol and indexes, but [barely do anything with it yet](https://github.com/raintank/metrictank/blob/master/docs/tags.md). @@ -37,8 +37,7 @@ Otherwise data loss of current chunks will be incurred. See [operations guide]( ## main features * 100% open source -* graphite is a first class citizen (note: currently requires a [fork of graphite-api](https://github.com/raintank/graphite-api/) - and the [graphite-metrictank](https://github.com/raintank/graphite-metrictank) plugin) +* graphite is a first class citizen. As of graphite-1.0.1, metrictank can be used as a graphite CLUSTER_SERVER. * accurate, flexible rollups by storing min/max/sum/count (which also gives us average). So we can do consolidation (combined runtime+archived) accurately and correctly, [unlike most other graphite backends like whisper](https://blog.raintank.io/25-graphite-grafana-and-statsd-gotchas/#runtime.consolidation) @@ -71,7 +70,7 @@ So we can do consolidation (combined runtime+archived) accurately and correctly, * [Consolidation](https://github.com/raintank/metrictank/blob/master/docs/consolidation.md) * [Multi-tenancy](https://github.com/raintank/metrictank/blob/master/docs/multi-tenancy.md) * [HTTP api](https://github.com/raintank/metrictank/blob/master/docs/http-api.md) -* [Graphite-api](https://github.com/raintank/metrictank/blob/master/docs/graphite-api.md) +* [Graphite](https://github.com/raintank/metrictank/blob/master/docs/graphite.md) * [Metadata](https://github.com/raintank/metrictank/blob/master/docs/metadata.md) * [Tags](https://github.com/raintank/metrictank/blob/master/docs/tags.md) * [Usage reporting](https://github.com/raintank/metrictank/blob/master/docs/usage-reporting.md) diff --git a/api/graphite.go b/api/graphite.go index ffd717c1d9..81de6b0f44 100644 --- a/api/graphite.go +++ b/api/graphite.go @@ -185,6 +185,10 @@ func (s *Server) renderMetrics(ctx *middleware.Context, request models.GraphiteR plan, err := expr.NewPlan(exprs, fromUnix, toUnix, request.MaxDataPoints, stable, nil) if err != nil { if fun, ok := err.(expr.ErrUnknownFunction); ok { + if request.NoProxy { + ctx.Error(http.StatusBadRequest, "localOnly requested, but the request cant be handled locally") + return + } ctx.Req.Request.Body = ctx.Body graphiteProxy.ServeHTTP(ctx.Resp, ctx.Req.Request) proxyStats.Miss(string(fun)) @@ -202,9 +206,12 @@ func (s *Server) renderMetrics(ctx *middleware.Context, request models.GraphiteR } sort.Sort(models.SeriesByTarget(out)) - if request.Format == "msgp" { + switch request.Format { + case "msgp": response.Write(ctx, response.NewMsgp(200, models.SeriesByTarget(out))) - } else { + case "pickle": + response.Write(ctx, response.NewPickle(200, models.SeriesPickleFormat(out))) + default: response.Write(ctx, response.NewFastJson(200, models.SeriesByTarget(out))) } plan.Clean() @@ -232,19 +239,14 @@ func (s *Server) metricsFind(ctx *middleware.Context, request models.GraphiteFin } } - var b interface{} switch request.Format { case "", "treejson", "json": - b, err = findTreejson(request.Query, nodes) + response.Write(ctx, response.NewJson(200, findTreejson(request.Query, nodes), request.Jsonp)) case "completer": - b, err = findCompleter(nodes) - } - - if err != nil { - response.Write(ctx, response.WrapError(err)) - return + response.Write(ctx, response.NewJson(200, findCompleter(nodes), request.Jsonp)) + case "pickle": + response.Write(ctx, response.NewPickle(200, findPickle(nodes, request))) } - response.Write(ctx, response.NewJson(200, b, request.Jsonp)) } func (s *Server) listLocal(orgId int) []idx.Archive { @@ -326,7 +328,7 @@ func (s *Server) metricsIndex(ctx *middleware.Context) { response.Write(ctx, response.NewFastJson(200, models.MetricNames(series))) } -func findCompleter(nodes []idx.Node) (models.SeriesCompleter, error) { +func findCompleter(nodes []idx.Node) models.SeriesCompleter { var result = models.NewSeriesCompleter() for _, g := range nodes { c := models.SeriesCompleterItem{ @@ -347,12 +349,24 @@ func findCompleter(nodes []idx.Node) (models.SeriesCompleter, error) { result.Add(c) } - return result, nil + return result +} + +func findPickle(nodes []idx.Node, request models.GraphiteFind) models.SeriesPickle { + result := make([]models.SeriesPickleItem, len(nodes)) + var intervals [][]int64 + if request.From != 0 && request.Until != 0 { + intervals = [][]int64{{request.From, request.Until}} + } + for i, g := range nodes { + result[i] = models.NewSeriesPickleItem(g.Path, g.Leaf, intervals) + } + return result } var treejsonContext = make(map[string]int) -func findTreejson(query string, nodes []idx.Node) (models.SeriesTree, error) { +func findTreejson(query string, nodes []idx.Node) models.SeriesTree { tree := models.NewSeriesTree() seen := make(map[string]struct{}) @@ -392,7 +406,7 @@ func findTreejson(query string, nodes []idx.Node) (models.SeriesTree, error) { } tree.Add(&t) } - return *tree, nil + return *tree } func (s *Server) metricsDelete(ctx *middleware.Context, req models.MetricsDelete) { diff --git a/api/models/graphite.go b/api/models/graphite.go index a2cfcf276b..adbeb121c3 100644 --- a/api/models/graphite.go +++ b/api/models/graphite.go @@ -15,7 +15,8 @@ type GraphiteRender struct { From string `json:"from" form:"from"` Until string `json:"until" form:"until"` To string `json:"to" form:"to"` - Format string `json:"format" form:"format" binding:"In(,json,msgp)"` + Format string `json:"format" form:"format" binding:"In(,json,msgp,pickle)"` + NoProxy bool `json:"local" form:"local"` //this is set to true by graphite-web when it passes request to cluster servers Process string `json:"process" form:"process" binding:"In(,none,stable,any);Default(stable)"` } @@ -44,7 +45,7 @@ type GraphiteFind struct { Query string `json:"query" form:"query" binding:"Required"` From int64 `json:"from" form:"from"` Until int64 `json:"until" form:"until"` - Format string `json:"format" form:"format" binding:"In(,completer,json,treejson)"` + Format string `json:"format" form:"format" binding:"In(,completer,json,treejson,pickle)"` Jsonp string `json:"jsonp" form:"jsonp"` } @@ -99,6 +100,22 @@ type SeriesCompleterItem struct { IsLeaf string `json:"is_leaf"` } +type SeriesPickle []SeriesPickleItem + +type SeriesPickleItem struct { + Path string `pickle:"path"` + IsLeaf bool `pickle:"isLeaf"` + Intervals [][]int64 `pickle:"intervals"` // list of (start,end) tuples +} + +func NewSeriesPickleItem(path string, isLeaf bool, intervals [][]int64) SeriesPickleItem { + return SeriesPickleItem{ + Path: path, + IsLeaf: isLeaf, + Intervals: intervals, + } +} + type SeriesTree []SeriesTreeItem func NewSeriesTree() *SeriesTree { diff --git a/api/models/series.go b/api/models/series.go index 111288f109..64b3de16c2 100644 --- a/api/models/series.go +++ b/api/models/series.go @@ -56,3 +56,31 @@ func (series SeriesByTarget) MarshalJSONFast(b []byte) ([]byte, error) { func (series SeriesByTarget) MarshalJSON() ([]byte, error) { return series.MarshalJSONFast(nil) } + +type SeriesForPickle struct { + Name string `pickle:"name"` + Start uint32 `pickle:"start"` + End uint32 `pickle:"end"` + Step uint32 `pickle:"step"` + Values []float64 `pickle:"values"` + PathExpression string `pickle:"pathExpression"` +} + +func SeriesPickleFormat(data []Series) []SeriesForPickle { + result := make([]SeriesForPickle, len(data)) + for i, s := range data { + datapoints := make([]float64, len(s.Datapoints)) + for i, p := range s.Datapoints { + datapoints[i] = p.Val + } + result[i] = SeriesForPickle{ + Name: s.Target, + Start: s.QueryFrom, + End: s.QueryTo, + Step: s.Interval, + Values: datapoints, + PathExpression: s.QueryPatt, + } + } + return result +} diff --git a/api/response/pickle.go b/api/response/pickle.go new file mode 100644 index 0000000000..8fbbe86006 --- /dev/null +++ b/api/response/pickle.go @@ -0,0 +1,39 @@ +package response + +import ( + "bytes" + pickle "github.com/kisielk/og-rek" +) + +type Pickle struct { + code int + body interface{} + buf []byte +} + +func NewPickle(code int, body interface{}) *Pickle { + return &Pickle{ + code: code, + body: body, + buf: BufferPool.Get(), + } +} + +func (r *Pickle) Code() int { + return r.code +} + +func (r *Pickle) Close() { + BufferPool.Put(r.buf) +} + +func (r *Pickle) Body() ([]byte, error) { + buffer := bytes.NewBuffer(r.buf) + encoder := pickle.NewEncoder(buffer) + err := encoder.Encode(r.body) + return buffer.Bytes(), err +} + +func (r *Pickle) Headers() (headers map[string]string) { + return map[string]string{"content-type": "application/pickle"} +} diff --git a/api/response/pickle_test.go b/api/response/pickle_test.go new file mode 100644 index 0000000000..dca0ffff44 --- /dev/null +++ b/api/response/pickle_test.go @@ -0,0 +1,138 @@ +package response + +import ( + "math" + "testing" + + "github.com/raintank/metrictank/api/models" + "gopkg.in/raintank/schema.v1" +) + +func BenchmarkHttpRespPickleEmptySeries(b *testing.B) { + data := []models.Series{ + { + Target: "an.empty.series", + Datapoints: make([]schema.Point, 0), + Interval: 10, + }, + } + var resp *Pickle + for n := 0; n < b.N; n++ { + resp = NewPickle(200, models.SeriesPickleFormat(data)) + resp.Body() + resp.Close() + } +} + +func BenchmarkHttpRespPickleEmptySeriesNeedsEscaping(b *testing.B) { + data := []models.Series{ + { + Target: `an.empty\series`, + Datapoints: make([]schema.Point, 0), + Interval: 10, + }, + } + var resp *Pickle + for n := 0; n < b.N; n++ { + resp = NewPickle(200, models.SeriesPickleFormat(data)) + resp.Body() + resp.Close() + } +} + +func BenchmarkHttpRespPickleIntegers(b *testing.B) { + points := make([]schema.Point, 1000, 1000) + baseTs := 1500000000 + for i := 0; i < 1000; i++ { + points[i] = schema.Point{Val: float64(10000 * i), Ts: uint32(baseTs + 10*i)} + } + data := []models.Series{ + { + Target: "some.metric.with.a-whole-bunch-of.integers", + Datapoints: points, + Interval: 10, + }, + } + b.SetBytes(int64(len(points) * 12)) + + b.ResetTimer() + var resp *Pickle + for n := 0; n < b.N; n++ { + resp = NewPickle(200, models.SeriesPickleFormat(data)) + resp.Body() + resp.Close() + } +} + +func BenchmarkHttpRespPickleFloats(b *testing.B) { + points := make([]schema.Point, 1000, 1000) + baseTs := 1500000000 + for i := 0; i < 1000; i++ { + points[i] = schema.Point{Val: 12.34 * float64(i), Ts: uint32(baseTs + 10*i)} + } + data := []models.Series{ + { + Target: "some.metric.with.a-whole-bunch-of.floats", + Datapoints: points, + Interval: 10, + }, + } + b.SetBytes(int64(len(points) * 12)) + + b.ResetTimer() + var resp *Pickle + for n := 0; n < b.N; n++ { + resp = NewPickle(200, models.SeriesPickleFormat(data)) + resp.Body() + resp.Close() + } +} + +func BenchmarkHttpRespPickleNulls(b *testing.B) { + points := make([]schema.Point, 1000, 1000) + baseTs := 1500000000 + for i := 0; i < 1000; i++ { + points[i] = schema.Point{Val: math.NaN(), Ts: uint32(baseTs + 10*i)} + } + data := []models.Series{ + { + Target: "some.metric.with.a-whole-bunch-of.nulls", + Datapoints: points, + Interval: 10, + }, + } + b.SetBytes(int64(len(points) * 12)) + + b.ResetTimer() + var resp *Pickle + for n := 0; n < b.N; n++ { + resp = NewPickle(200, models.SeriesPickleFormat(data)) + resp.Body() + resp.Close() + } +} + +var foo []models.SeriesForPickle + +func BenchmarkConvertSeriesToPickleFormat10k(b *testing.B) { + points := make([]schema.Point, 10000) + baseTs := 1500000000 + for i := 0; i < 10000; i++ { + points[i] = schema.Point{Val: 1.2345 * float64(i), Ts: uint32(baseTs + 10*i)} + } + data := []models.Series{ + { + Target: "foo", + Datapoints: points, + Interval: 10, + }, + } + b.SetBytes(int64(len(points) * 12)) + + b.ResetTimer() + var bar []models.SeriesForPickle + for n := 0; n < b.N; n++ { + bar = models.SeriesPickleFormat(data) + } + foo = bar +} diff --git a/api/response/response.go b/api/response/response.go index 6e194d9211..2fb02dbaf4 100644 --- a/api/response/response.go +++ b/api/response/response.go @@ -9,7 +9,7 @@ import ( var ErrMetricNotFound = errors.New("metric not found") -var BufferPool = util.NewBufferPool() // used by fastjson and mspg responses to serialize into +var BufferPool = util.NewBufferPool() // used by pickle, fastjson and msgp responses to serialize into func Write(w http.ResponseWriter, resp Response) { defer resp.Close() diff --git a/cmd/mt-explain/main.go b/cmd/mt-explain/main.go index ab8c6d737b..c2c3eeb12c 100644 --- a/cmd/mt-explain/main.go +++ b/cmd/mt-explain/main.go @@ -58,7 +58,7 @@ func main() { plan, err := expr.NewPlan(exps, fromUnix, toUnix, uint32(*mdp), *stable, nil) if err != nil { if fun, ok := err.(expr.ErrUnknownFunction); ok { - fmt.Printf("Unsupported function %q: must defer query to graphite-api\n", string(fun)) + fmt.Printf("Unsupported function %q: must defer query to graphite\n", string(fun)) plan.Dump(os.Stdout) return } diff --git a/docker/docker-cluster/datasources/graphite b/docker/docker-cluster/datasources/graphite new file mode 100644 index 0000000000..117e18eab4 --- /dev/null +++ b/docker/docker-cluster/datasources/graphite @@ -0,0 +1,7 @@ +{ + "name":"graphite", + "type":"graphite", + "url":"http://graphite", + "access":"proxy", + "isDefault":false +} diff --git a/docker/docker-cluster/datasources/metrictank b/docker/docker-cluster/datasources/metrictank index 25ebf9433e..b69830ff5b 100644 --- a/docker/docker-cluster/datasources/metrictank +++ b/docker/docker-cluster/datasources/metrictank @@ -1,7 +1,7 @@ { "name":"metrictank", "type":"graphite", - "url":"http://localhost:8080", - "access":"direct", + "url":"http://metrictank0:6060", + "access":"proxy", "isDefault":true } diff --git a/docker/docker-cluster/docker-compose.yml b/docker/docker-cluster/docker-compose.yml index 9f54aee5ac..cac1b4cd29 100644 --- a/docker/docker-cluster/docker-compose.yml +++ b/docker/docker-cluster/docker-compose.yml @@ -93,23 +93,24 @@ services: cassandra: hostname: cassandra - image: cassandra:3.0.8 + image: cassandra:3.9 environment: MAX_HEAP_SIZE: 1G HEAP_NEWSIZE: 256M ports: - "9042:9042" - graphite-api: - hostname: graphite-api - image: raintank/graphite-metrictank + graphite: + hostname: graphite + image: raintank/graphite-mt ports: - - "8080:8080" - links: - - metrictank0 - - statsdaemon - volumes: - - "./graphite-metrictank.yaml:/etc/graphite-metrictank/graphite-metrictank.yaml" + - "8080:80" + environment: + GRAPHITE_CLUSTER_SERVERS: metrictank0:6060 + GRAPHITE_STATSD_HOST: statsdaemon + SINGLE_TENANT: yes + WSGI_PROCESSES: 4 + WSGI_THREADS: 25 grafana: hostname: grafana diff --git a/docker/docker-cluster/graphite-metrictank.yaml b/docker/docker-cluster/graphite-metrictank.yaml deleted file mode 100644 index f87d85c007..0000000000 --- a/docker/docker-cluster/graphite-metrictank.yaml +++ /dev/null @@ -1,39 +0,0 @@ -finders: -- graphite_metrictank.RaintankFinder -functions: -- graphite_api.functions.SeriesFunctions -- graphite_api.functions.PieFunctions -statsd: - host: 'statsdaemon' - port: 8125 -logging: - version: 1 - handlers: - raw: - level: DEBUG - class: logging.StreamHandler - formatter: raw - loggers: - root: - handlers: - - raw - level: DEBUG - propagate: false - graphite_api: - handlers: - - raw - level: DEBUG - formatters: - default: - format: '%(asctime)s %(levelname)-8s %(name)-15s %(message)s' - datefmt: '%Y-%m-%d %H:%M:%S' - root: - level: ERROR -allowed_origins: - - localhost:3000 -raintank: - tank: - url: http://metrictank0:6060/ -search_index: /var/lib/graphite/index -time_zone: UTC - diff --git a/docker/docker-cluster/metrictank.ini b/docker/docker-cluster/metrictank.ini index 440a047b60..e42cce3c00 100644 --- a/docker/docker-cluster/metrictank.ini +++ b/docker/docker-cluster/metrictank.ini @@ -135,7 +135,7 @@ max-points-per-req-hard = 20000000 # require x-org-id authentication to auth as a specific org. otherwise orgId 1 is assumed multi-tenant = true # in case our /render endpoint does not support the requested processing, proxy the request to this graphite -fallback-graphite-addr = http://graphite-api:8080 +fallback-graphite-addr = http://graphite # only log incoming requests if their timerange is at least this duration. Use 0 to disable log-min-dur = 5min diff --git a/docker/docker-dev-custom-cfg-kafka/datasources/graphite b/docker/docker-dev-custom-cfg-kafka/datasources/graphite new file mode 100644 index 0000000000..117e18eab4 --- /dev/null +++ b/docker/docker-dev-custom-cfg-kafka/datasources/graphite @@ -0,0 +1,7 @@ +{ + "name":"graphite", + "type":"graphite", + "url":"http://graphite", + "access":"proxy", + "isDefault":false +} diff --git a/docker/docker-dev-custom-cfg-kafka/datasources/metrictank b/docker/docker-dev-custom-cfg-kafka/datasources/metrictank index 25ebf9433e..3fc34a69b6 100644 --- a/docker/docker-dev-custom-cfg-kafka/datasources/metrictank +++ b/docker/docker-dev-custom-cfg-kafka/datasources/metrictank @@ -1,7 +1,7 @@ { "name":"metrictank", "type":"graphite", - "url":"http://localhost:8080", - "access":"direct", + "url":"http://metrictank:6060", + "access":"proxy", "isDefault":true } diff --git a/docker/docker-dev-custom-cfg-kafka/docker-compose.yml b/docker/docker-dev-custom-cfg-kafka/docker-compose.yml index e3a62b8a51..d32882d147 100644 --- a/docker/docker-dev-custom-cfg-kafka/docker-compose.yml +++ b/docker/docker-dev-custom-cfg-kafka/docker-compose.yml @@ -33,23 +33,24 @@ services: cassandra: hostname: cassandra - image: cassandra:3.0.8 + image: cassandra:3.9 environment: MAX_HEAP_SIZE: 1G HEAP_NEWSIZE: 256M ports: - "9042:9042" - graphite-api: - hostname: graphite-api - image: raintank/graphite-metrictank + graphite: + hostname: graphite + image: raintank/graphite-mt ports: - - "8080:8080" - links: - - metrictank - - statsdaemon - volumes: - - "../graphite-metrictank.yaml:/etc/graphite-metrictank/graphite-metrictank.yaml" + - "8080:80" + environment: + GRAPHITE_CLUSTER_SERVERS: metrictank:6060 + GRAPHITE_STATSD_HOST: statsdaemon + SINGLE_TENANT: yes + WSGI_PROCESSES: 4 + WSGI_THREADS: 25 grafana: hostname: grafana diff --git a/docker/docker-dev-custom-cfg-kafka/metrictank.ini b/docker/docker-dev-custom-cfg-kafka/metrictank.ini index 8e07f60b81..70439d66af 100644 --- a/docker/docker-dev-custom-cfg-kafka/metrictank.ini +++ b/docker/docker-dev-custom-cfg-kafka/metrictank.ini @@ -135,7 +135,7 @@ max-points-per-req-hard = 20000000 # require x-org-id authentication to auth as a specific org. otherwise orgId 1 is assumed multi-tenant = true # in case our /render endpoint does not support the requested processing, proxy the request to this graphite -fallback-graphite-addr = http://graphite-api:8080 +fallback-graphite-addr = http://graphite # only log incoming requests if their timerange is at least this duration. Use 0 to disable log-min-dur = 5min diff --git a/docker/docker-dev-direct-single-tenant/datasources/mt-direct b/docker/docker-dev-direct-single-tenant/datasources/mt-direct deleted file mode 100644 index f52e415850..0000000000 --- a/docker/docker-dev-direct-single-tenant/datasources/mt-direct +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name":"mt-direct", - "type":"graphite", - "url":"http://localhost:6060", - "access":"direct", - "isDefault":false -} diff --git a/docker/docker-dev-direct-single-tenant/datasources/mt-graphite-api b/docker/docker-dev-direct-single-tenant/datasources/mt-graphite-api deleted file mode 100644 index 800e7872ab..0000000000 --- a/docker/docker-dev-direct-single-tenant/datasources/mt-graphite-api +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name":"mt-graphite-api", - "type":"graphite", - "url":"http://localhost:8080", - "access":"direct", - "isDefault":true -} diff --git a/docker/docker-dev-direct-single-tenant/docker-compose.yml b/docker/docker-dev-direct-single-tenant/docker-compose.yml deleted file mode 100644 index a92255d46f..0000000000 --- a/docker/docker-dev-direct-single-tenant/docker-compose.yml +++ /dev/null @@ -1,54 +0,0 @@ -version: '2' - -services: - metrictank: - hostname: metrictank - image: raintank/metrictank - ports: - - "6060:6060" - - "2003:2003" - volumes: - - ../../build/metrictank:/usr/bin/metrictank - - ../../scripts/config/metrictank-docker.ini:/etc/metrictank/metrictank.ini - - ../../scripts/config/storage-schemas.conf:/etc/metrictank/storage-schemas.conf - - ../../scripts/config/storage-aggregation.conf:/etc/metrictank/storage-aggregation.conf - environment: - WAIT_HOSTS: cassandra:9042 - WAIT_TIMEOUT: 60 - MT_HTTP_MULTI_TENANT: "false" - links: - - cassandra - - cassandra: - hostname: cassandra - image: cassandra:3.0.8 - environment: - MAX_HEAP_SIZE: 1G - HEAP_NEWSIZE: 256M - ports: - - "9042:9042" - - graphite-api: - hostname: graphite-api - image: raintank/graphite-metrictank - ports: - - "8080:8080" - links: - - metrictank - - statsdaemon - volumes: - - "../graphite-metrictank.yaml:/etc/graphite-metrictank/graphite-metrictank.yaml" - - grafana: - hostname: grafana - image: grafana/grafana - ports: - - "3000:3000" - - statsdaemon: - hostname: statsdaemon - image: raintank/statsdaemon - ports: - - "8125:8125/udp" - volumes: - - "../statsdaemon.ini:/etc/statsdaemon.ini" diff --git a/docker/docker-dev-direct-single-tenant/bench.sh b/docker/docker-dev/bench.sh similarity index 79% rename from docker/docker-dev-direct-single-tenant/bench.sh rename to docker/docker-dev/bench.sh index 8b98fb5e2f..300a55d090 100755 --- a/docker/docker-dev-direct-single-tenant/bench.sh +++ b/docker/docker-dev/bench.sh @@ -30,9 +30,8 @@ function run () { run '1A: MT simple series requests' 6060 '{{.Name}}' 1h 10 50s 10s run '1B: graphite simple series requests' 8080 '{{.Name}}' 1h 10 50s 10s -run '2A MT sumSeries(patterns.*) (no proxying)' 6060 'sumSeries({{.Name | pattern}})' 1h 250 25s 5s -run '2B same but lower load since graphite cant do much' 6060 'sumSeries({{.Name | pattern}})' 1h 5 25s 5s -run '2C graphite sumSeries(patterns.*)' 8080 'sumSeries({{.Name | pattern}})' 1h 5 50s 10s +run '2A MT sumSeries(patterns.*) (no proxying)' 6060 'sumSeries({{.Name | pattern}})' 1h 100 25s 5s +run '2B graphite sumSeries(patterns.*)' 8080 'sumSeries({{.Name | pattern}})' 1h 100 25s 5s -run '3A MT load needing proxying' 6060 'aliasByNode({{.Name}})' 1h 5 50s 10s -run '3B graphite directly to see proxying overhead' 8080 'aliasByNode({{.Name}})' 1h 5 50s 10s +run '3A MT load needing proxying' 6060 'aliasByNode({{.Name}})' 1h 100 50s 10s +run '3B graphite directly to see proxying overhead' 8080 'aliasByNode({{.Name}})' 1h 100 50s 10s diff --git a/docker/docker-dev/datasources/graphite b/docker/docker-dev/datasources/graphite new file mode 100644 index 0000000000..117e18eab4 --- /dev/null +++ b/docker/docker-dev/datasources/graphite @@ -0,0 +1,7 @@ +{ + "name":"graphite", + "type":"graphite", + "url":"http://graphite", + "access":"proxy", + "isDefault":false +} diff --git a/docker/docker-dev/datasources/metrictank b/docker/docker-dev/datasources/metrictank index 25ebf9433e..3fc34a69b6 100644 --- a/docker/docker-dev/datasources/metrictank +++ b/docker/docker-dev/datasources/metrictank @@ -1,7 +1,7 @@ { "name":"metrictank", "type":"graphite", - "url":"http://localhost:8080", - "access":"direct", + "url":"http://metrictank:6060", + "access":"proxy", "isDefault":true } diff --git a/docker/docker-dev/docker-compose.yml b/docker/docker-dev/docker-compose.yml index ec03e0a765..a8a6b7063a 100644 --- a/docker/docker-dev/docker-compose.yml +++ b/docker/docker-dev/docker-compose.yml @@ -15,6 +15,7 @@ services: environment: WAIT_HOSTS: cassandra:9042 WAIT_TIMEOUT: 60 + MT_HTTP_MULTI_TENANT: "false" links: - cassandra @@ -27,16 +28,16 @@ services: ports: - "9042:9042" - graphite-api: - hostname: graphite-api - image: raintank/graphite-metrictank + graphite: + hostname: graphite + image: raintank/graphite-mt ports: - - "8080:8080" - links: - - metrictank - - statsdaemon - volumes: - - "../graphite-metrictank.yaml:/etc/graphite-metrictank/graphite-metrictank.yaml" + - "8080:80" + environment: + GRAPHITE_CLUSTER_SERVERS: metrictank:6060 + GRAPHITE_STATSD_HOST: statsdaemon + WSGI_PROCESSES: 4 + WSGI_THREADS: 25 grafana: hostname: grafana @@ -50,4 +51,4 @@ services: ports: - "8125:8125/udp" volumes: - - "../statsdaemon.ini:/etc/statsdaemon.ini" + - "../statsdaemon.ini:/etc/statsdaemon.ini" \ No newline at end of file diff --git a/docker/docker-standard/datasources/graphite b/docker/docker-standard/datasources/graphite new file mode 100644 index 0000000000..20f232f951 --- /dev/null +++ b/docker/docker-standard/datasources/graphite @@ -0,0 +1,7 @@ +{ + "name":"graphite", + "type":"graphite", + "url":"http://graphite:80", + "access":"proxy", + "isDefault":false +} diff --git a/docker/docker-standard/datasources/metrictank b/docker/docker-standard/datasources/metrictank index 25ebf9433e..3fc34a69b6 100644 --- a/docker/docker-standard/datasources/metrictank +++ b/docker/docker-standard/datasources/metrictank @@ -1,7 +1,7 @@ { "name":"metrictank", "type":"graphite", - "url":"http://localhost:8080", - "access":"direct", + "url":"http://metrictank:6060", + "access":"proxy", "isDefault":true } diff --git a/docker/docker-standard/docker-compose.yml b/docker/docker-standard/docker-compose.yml index c26d7c8f44..f09e69e001 100644 --- a/docker/docker-standard/docker-compose.yml +++ b/docker/docker-standard/docker-compose.yml @@ -10,28 +10,29 @@ services: environment: WAIT_HOSTS: cassandra:9042 WAIT_TIMEOUT: 60 + MT_HTTP_MULTI_TENANT: "false" links: - cassandra cassandra: hostname: cassandra - image: cassandra:3.0.8 + image: cassandra:3.9 environment: MAX_HEAP_SIZE: 1G HEAP_NEWSIZE: 256M ports: - "9042:9042" - graphite-api: - hostname: graphite-api - image: raintank/graphite-metrictank + graphite: + hostname: graphite + image: raintank/graphite-mt ports: - - "8080:8080" - links: - - metrictank - - statsdaemon - volumes: - - "../graphite-metrictank.yaml:/etc/graphite-metrictank/graphite-metrictank.yaml" + - "8080:80" + environment: + GRAPHITE_CLUSTER_SERVERS: metrictank:6060 + GRAPHITE_STATSD_HOST: statsdaemon + WSGI_PROCESSES: 4 + WSGI_THREADS: 25 grafana: hostname: grafana @@ -45,4 +46,4 @@ services: ports: - "8125:8125/udp" volumes: - - "../statsdaemon.ini:/etc/statsdaemon.ini" + - "../statsdaemon.ini:/etc/statsdaemon.ini" \ No newline at end of file diff --git a/docker/extra/dashboards/graphite.json b/docker/extra/dashboards/graphite.json new file mode 100644 index 0000000000..782cbdf0bb --- /dev/null +++ b/docker/extra/dashboards/graphite.json @@ -0,0 +1,332 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": 12, + "links": [], + "refresh": "5s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "alias(sumSeries(stats.$environment.timers.view.graphite.metrics.views.find_view.*.count), 'find')" + }, + { + "refId": "B", + "target": "alias(sumSeries(stats.$environment.timers.view.graphite.render.views.renderView.*.count), 'render')", + "textEditor": false + }, + { + "refId": "C", + "target": "alias(sumSeries(stats.$environment.response.5[0-9][0-9]), 'errors')", + "textEditor": true + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Request counts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "alias(averageSeries(stats.$environment.timers.view.graphite.metrics.views.find_view.*.mean), 'mean')" + }, + { + "refId": "B", + "target": "alias(maxSeries(stats.$environment.timers.view.graphite.metrics.views.find_view.*.upper_90), 'upper_90')" + }, + { + "refId": "C", + "target": "alias(maxSeries(stats.$environment.timers.view.graphite.metrics.views.find_view.*.upper), 'upper')" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Find latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "alias(averageSeries(stats.$environment.timers.view.graphite.render.views.renderView.*.mean), 'mean')" + }, + { + "refId": "B", + "target": "alias(maxSeries(stats.$environment.timers.view.graphite.render.views.renderView.*.upper_90), 'upper_90')" + }, + { + "refId": "C", + "target": "alias(maxSeries(stats.$environment.timers.view.graphite.render.views.renderView.*.upper), 'upper')" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Render latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "graphite", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "docker-env", + "value": "docker-env" + }, + "datasource": "metrictank", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "environment", + "options": [], + "query": "stats.*", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Graphite", + "version": 1 +} \ No newline at end of file diff --git a/docker/extra/populate-grafana.sh b/docker/extra/populate-grafana.sh index 323868212b..09037dc59d 100755 --- a/docker/extra/populate-grafana.sh +++ b/docker/extra/populate-grafana.sh @@ -2,7 +2,7 @@ env=$1 -WAIT_HOSTS=localhost:3000 ../../scripts/wait_for_endpoint.sh +WAIT_HOSTS=localhost:3000 WAIT_TIMEOUT=120 ../../scripts/wait_for_endpoint.sh for file in $env/datasources/*; do echo "> adding datasources $file" diff --git a/docker/graphite-metrictank.yaml b/docker/graphite-metrictank.yaml deleted file mode 100644 index b9ff3261a0..0000000000 --- a/docker/graphite-metrictank.yaml +++ /dev/null @@ -1,39 +0,0 @@ -finders: -- graphite_metrictank.RaintankFinder -functions: -- graphite_api.functions.SeriesFunctions -- graphite_api.functions.PieFunctions -statsd: - host: 'statsdaemon' - port: 8125 -logging: - version: 1 - handlers: - raw: - level: INFO - class: logging.StreamHandler - formatter: raw - loggers: - root: - handlers: - - raw - level: INFO - propagate: false - graphite_api: - handlers: - - raw - level: INFO - formatters: - default: - format: '%(asctime)s %(levelname)-8s %(name)-15s %(message)s' - datefmt: '%Y-%m-%d %H:%M:%S' - root: - level: ERROR -allowed_origins: - - localhost:3000 -raintank: - tank: - url: http://metrictank:6060/ -search_index: /var/lib/graphite/index -time_zone: UTC - diff --git a/docs/assets/add-datasource-docker.png b/docs/assets/add-datasource-docker.png index 370fef2ada..454a714eca 100644 Binary files a/docs/assets/add-datasource-docker.png and b/docs/assets/add-datasource-docker.png differ diff --git a/docs/graphite-api.md b/docs/graphite.md similarity index 98% rename from docs/graphite-api.md rename to docs/graphite.md index f62e47801d..9187ef3bb8 100644 --- a/docs/graphite-api.md +++ b/docs/graphite.md @@ -1,4 +1,4 @@ -# Graphite-api +# Graphite Metrictank aims to be able to provide as much processing power as it can: we're in the process of implementing [Graphite's extensive processing api](http://graphite.readthedocs.io/en/latest/functions.html) into metrictank itself. diff --git a/docs/http-api.md b/docs/http-api.md index 12b75761a5..bfd2514601 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -88,8 +88,8 @@ curl -H "X-Org-Id: 12345" --data query=statsd.fakesite.counters.session_start.*. ## Graphite query api -This is the early beginning of a graphite-web/graphite-api replacement. It can return JSON or messagepack output -This section of the api is **very early stages**. Your best bet is to use graphite-api + graphite-metrictank in front of metrictank, for now. +This is the early beginning of a graphite-web replacement. It can return JSON, pickle or messagepack output +This section of the api is **very early stages**. Your best bet is to use graphite in front of metrictank, for now. ``` GET /render diff --git a/docs/installation-deb.md b/docs/installation-deb.md index d0bb481605..ba65cb4cb8 100644 --- a/docs/installation-deb.md +++ b/docs/installation-deb.md @@ -4,12 +4,10 @@ We'll go over these in more detail below. -* Cassandra. We run and recommend 3.0.8 . +* Cassandra. We run and recommend 3.8 or newer See [Cassandra](https://github.com/raintank/metrictank/blob/master/docs/cassandra.md) -* Our [graphite-raintank finder plugin](https://github.com/raintank/graphite-metrictank) - and our [graphite-api fork](https://github.com/raintank/graphite-api/) (installed as 1 component) - We're working toward simplifying this much more. -* Optional: [statsd](https://github.com/etsy/statsd) or something compatible with it. For instrumentation of graphite-api. +* The latest (1.0.1 or newer) version of [Graphite](http://graphite.readthedocs.io/en/latest/install.html) +* Optional: [statsd](https://github.com/etsy/statsd) or something compatible with it. For instrumentation of graphite. * Optional: Kafka, if you want to buffer data in case metrictank goes down. Kafka 0.10.0.1 is highly recommended. [more info](https://github.com/raintank/metrictank/blob/master/docs/kafka.md) @@ -25,8 +23,8 @@ instance or a cluster. Metrictank will also respond to queries: if the data is RAM, and older data is fetched from cassandra. This happens transparantly. Metrictank maintains an index of metrics metadata, for all series it sees. You can use an index entirely in memory, or backed by Cassandra for persistence. -You'll typically query metrictank by querying graphite-api which uses the graphite-metrictank plugin to talk -to metrictank. You can also query metrictank directly but this is experimental and too early for anything useful. +You can query metrictank directly (it has fast, but limited built-in processing and will fallback to graphite when needed) +or you can also just query graphite which will always use graphite's processing but use metrictank as a datastore. ## Get a machine with root access @@ -34,32 +32,7 @@ We recommend a server with at least 8GB RAM and a few CPU's. You need root access. All the commands shown assume you're root. -## Metrictank and graphite-metrictank - -### Short version - -You can enable our repository and install the packages like so: - -``` -curl -s https://packagecloud.io/install/repositories/raintank/raintank/script.deb.sh | bash -apt-get install metrictank graphite-metrictank -``` - -Then just start it: - -``` -systemctl start graphite-metrictank -``` - -Or: - -``` -service graphite-metrictank start -``` - -Logs - if you need them - will be at /var/log/graphite/graphite-metrictank.log - -### Long version +## Metrictank and graphite We automatically build rpms and debs on circleCi for all needed components whenever the build succeeds. These packages are pushed to packagecloud. @@ -67,7 +40,6 @@ These packages are pushed to packagecloud. You need to install these packages: * metrictank -* graphite-metrictank (includes both our graphite-api variant as well as the graphite-metrictank finder plugin) Releases are simply tagged versions like `0.5.1` ([releases](https://github.com/raintank/metrictank/releases)), whereas commits in master following a release will be named `version-commit-after` for example `0.5.1-20` for @@ -81,9 +53,39 @@ Supported distributions: * Debian 7 (wheezy), 8 (jessie) * Centos 6, 7 +### Install Metrictank +You can enable our repository and install the metrictank package like so: + +``` +curl -s https://packagecloud.io/install/repositories/raintank/raintank/script.deb.sh | bash +apt-get install metrictank +``` [more info](https://packagecloud.io/raintank/raintank/install) +### Install Graphite + +Install Graphite via your prefered method as detailed at http://graphite.readthedocs.io/en/latest/install.html +(We hope to provide Debian and Ubuntu packages in the near future.) + +Configure graphite with the following settings in local_settings.py +``` +CLUSTER_SERVERS = ['localhost:6060'] +REMOTE_EXCLUDE_LOCAL = False +USE_WORKER_POOL = True +POOL_WORKERS_PER_BACKEND = 8 +POOL_WORKERS = 1 +REMOTE_FIND_TIMEOUT = 30.0 +REMOTE_FETCH_TIMEOUT = 60.0 +REMOTE_RETRY_DELAY = 60.0 +MAX_FETCH_RETRIES = 2 +FIND_CACHE_DURATION = 300 +REMOTE_STORE_USE_POST = True +REMOTE_STORE_FORWARD_HEADERS = ["x-org-id"] +REMOTE_PREFETCH_DATA = True +STORAGE_FINDERS = () +``` + ## Set up cassandra Add the cassandra repository: @@ -115,7 +117,7 @@ The log - should you need it - is at /var/log/cassandra/system.log ## Set up statsd -You can optionally statsd or a statsd-compatible agent for instrumentation of graphite-api. +You can optionally statsd or a statsd-compatible agent for instrumentation of graphite and optionally any of your other applications. You can install the official [statsd](https://github.com/etsy/statsd) (see its installation instructions) or an alternative. We recommend [raintank/statsdaemon](https://github.com/raintank/statsdaemon). diff --git a/docs/installation-rpm.md b/docs/installation-rpm.md index a7c3f9ce02..c5bb417b53 100644 --- a/docs/installation-rpm.md +++ b/docs/installation-rpm.md @@ -4,12 +4,10 @@ We'll go over these in more detail below. -* Cassandra. We run and recommend 3.0.8 . +* Cassandra. We run and recommend 3.8 or newer. See [Cassandra](https://github.com/raintank/metrictank/blob/master/docs/cassandra.md) -* Our [graphite-raintank finder plugin](https://github.com/raintank/graphite-metrictank) - and our [graphite-api fork](https://github.com/raintank/graphite-api/) (installed as 1 component) - We're working toward simplifying this much more. -* Optional: [statsd](https://github.com/etsy/statsd) or something compatible with it. For instrumentation of graphite-api. +* The latest (1.0.1 or newer) version of [Graphite](http://graphite.readthedocs.io/en/latest/install.html) +* Optional: [statsd](https://github.com/etsy/statsd) or something compatible with it. For instrumentation of graphite. * Optional: Kafka, if you want to buffer data in case metrictank goes down. Kafka 0.10.0.1 is highly recommended. [more info](https://github.com/raintank/metrictank/blob/master/docs/kafka.md) @@ -25,8 +23,8 @@ instance or a cluster. Metrictank will also respond to queries: if the data is RAM, and older data is fetched from cassandra. This happens transparantly. Metrictank maintains an index of metrics metadata, for all series it sees. You can use an index entirely in memory, or backed by Cassandra for persistence. -You'll typically query metrictank by querying graphite-api which uses the graphite-metrictank plugin to talk -to metrictank. You can also query metrictank directly but this is experimental and too early for anything useful. +You can query metrictank directly (it has fast, but limited built-in processing and will fallback to graphite when needed) +or you can also just query graphite which will always use graphite's processing but use metrictank as a datastore. ## Get a machine with root access @@ -34,26 +32,7 @@ We recommend a server with at least 8GB RAM and a few CPU's. You need root access. All the commands shown assume you're root. -## Metrictank and graphite-metrictank - -### Short version - -You can enable our repository and install the packages like so: - -``` -curl -s https://packagecloud.io/install/repositories/raintank/raintank/script.rpm.sh | bash -yum install metrictank graphite-metrictank -``` - -Then just start it: - -``` -systemctl start graphite-metrictank -``` - -Logs - if you need them - will be at /var/log/graphite/graphite-metrictank.log - -### Long version +## Metrictank and graphite We automatically build rpms and debs on circleCi for all needed components whenever the build succeeds. These packages are pushed to packagecloud. @@ -61,7 +40,6 @@ These packages are pushed to packagecloud. You need to install these packages: * metrictank -* graphite-metrictank (includes both our graphite-api variant as well as the graphite-metrictank finder plugin) Releases are simply tagged versions like `0.5.1` ([releases](https://github.com/raintank/metrictank/releases)), whereas commits in master following a release will be named `version-commit-after` for example `0.5.1-20` for @@ -75,9 +53,39 @@ Supported distributions: * Debian 7 (wheezy), 8 (jessie) * Centos 6, 7 +### Install Metrictank +You can enable our repository and install the metrictank package like so: + +``` +curl -s https://packagecloud.io/install/repositories/raintank/raintank/script.rpm.sh | bash +apt-get install metrictank +``` [more info](https://packagecloud.io/raintank/raintank/install) +### Install Graphite + +Install Graphite via your prefered method as detailed at http://graphite.readthedocs.io/en/latest/install.html +(We hope to provide Debian and Ubuntu packages in the near future.) + +Configure graphite with the following settings in local_settings.py +``` +CLUSTER_SERVERS = ['localhost:6060'] +REMOTE_EXCLUDE_LOCAL = False +USE_WORKER_POOL = True +POOL_WORKERS_PER_BACKEND = 8 +POOL_WORKERS = 1 +REMOTE_FIND_TIMEOUT = 30.0 +REMOTE_FETCH_TIMEOUT = 60.0 +REMOTE_RETRY_DELAY = 60.0 +MAX_FETCH_RETRIES = 2 +FIND_CACHE_DURATION = 300 +REMOTE_STORE_USE_POST = True +REMOTE_STORE_FORWARD_HEADERS = ["x-org-id"] +REMOTE_PREFETCH_DATA = True +STORAGE_FINDERS = () +``` + ## Set up java Download and install the oracle java rpm: @@ -122,7 +130,7 @@ The log - should you need it - is at /var/log/cassandra/cassandra.log ## Set up statsd -You can optionally statsd or a statsd-compatible agent for instrumentation of graphite-api. +You can optionally statsd or a statsd-compatible agent for instrumentation of graphite and optionally any of your other applications. You can install the official [statsd](https://github.com/etsy/statsd) (see its installation instructions) or an alternative. We recommend [raintank/statsdaemon](https://github.com/raintank/statsdaemon). diff --git a/docs/installation-source.md b/docs/installation-source.md index e95d6e28d7..4a24335a76 100644 --- a/docs/installation-source.md +++ b/docs/installation-source.md @@ -35,34 +35,27 @@ Take the file from `go/src/github.com/raintank/metrictank/metrictank-sample.ini` Note that metrictank simply logs to stdout, and not to a file. -# Install graphite-raintank +# Install graphite This will be needed to query metrictank. -* Install the build dependencies. - * Under debian based distros, run `apt-get -y install python python-pip build-essential python-dev libffi-dev libcairo2-dev git` as root. - * For CentOS and other rpm-based distros, run `yum -y install python-setuptools python-devel gcc gcc-c++ make openssl-devel libffi-devel cairo-devel git; easy_install pip`. - * If neither of these instructions are relevant to you, figure out how your distribution or operating system refers to the above packages and install them. - -* Install `virtualenv`, if desired: `pip install virtualenv virtualenv-tools` - -* If you are installing graphite using `virtualenv`: - * `virtualenv /usr/share/python/graphite` - * Run all of the pip commands below as `/usr/share/python/graphite/bin/pip`. - -* Run these commands to install: - * `git clone https://github.com/raintank/graphite-metrictank.git` - * `pip install git+https://github.com/raintank/graphite-api.git` - * `pip install gunicorn==18.0` - * `pip install /path/to/graphite-metrictank` - * `pip install eventlet` - * `pip install git+https://github.com/woodsaj/pystatsd.git` - * `pip install Flask-Cache` - * `pip install python-memcached` - * `pip install blist` - * `find /usr/share/python/graphite ! -perm -a+r -exec chmod a+r {} \;` - * `cd /usr/share/python/graphite` - * `virtualenv-tools --update-path /usr/share/python/graphite` - * `mkdir -p /var/log/graphite` - -The easiest way to run graphite-api + graphite-metrictank when you've installed it from source is to find the appropriate startup script in the `pkg/` directory in the graphite-metrictank repo, the defaults file at `pkg/common/default/graphite-metrictank`, and the `graphite-metrictank.yaml` and `gunicorn_conf.py` config files in `pkg/common/graphite-metrictank`. **NB:** If you do not use `virtualenv` to install graphite-api and graphite-metrictank, you will need to modify the startup script to point at the system python and the gunicorn you installed (which will probably be at `/usr/local/bin/gunicorn`). +Install Graphite via your prefered method as detailed at http://graphite.readthedocs.io/en/latest/install.html +(We hope to provide Debian and Ubuntu packages in the near future.) + +Configure graphite with the following settings in local_settings.py +``` +CLUSTER_SERVERS = ['localhost:6060'] # update to match the host:port metrictank is running on. +REMOTE_EXCLUDE_LOCAL = False +USE_WORKER_POOL = True +POOL_WORKERS_PER_BACKEND = 8 +POOL_WORKERS = 1 +REMOTE_FIND_TIMEOUT = 30.0 +REMOTE_FETCH_TIMEOUT = 60.0 +REMOTE_RETRY_DELAY = 60.0 +MAX_FETCH_RETRIES = 2 +FIND_CACHE_DURATION = 300 +REMOTE_STORE_USE_POST = True +REMOTE_STORE_FORWARD_HEADERS = ["x-org-id"] +REMOTE_PREFETCH_DATA = True +STORAGE_FINDERS = () +``` diff --git a/docs/installation.md b/docs/installation.md index 8bf2328206..8ab323c5fc 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -11,5 +11,5 @@ In-depth guides that guide you you through setting things up component by compon There are also some alternative approaches, such as: * [Installation from source](https://github.com/raintank/metrictank/blob/master/docs/installation-source.md) -* [chef_metric_tank](https://github.com/raintank/chef_metric_tank) + diff --git a/docs/multi-tenancy.md b/docs/multi-tenancy.md index eeee00be36..b2cc20beba 100644 --- a/docs/multi-tenancy.md +++ b/docs/multi-tenancy.md @@ -6,7 +6,7 @@ A metrictank based stack is multi-tenant. Here's how it works: * Metrictank isolates data in storage based on the org-id, during ingestion as well as retrieval with the http api. * During ingestion, the org-id is set in the data coming in through kafka, or for carbon input plugin, is set to 1. * For retrieval, metrictank requires an x-org-id header. -* [graphite-api](https://github.com/raintank/graphite-api) and [graphite-metrictank](https://github.com/raintank/graphite-metrictank) pass the x-org-id header through to metrictank +* Requests sent to Graphite must include a "x-org-id" header. This header will be passed from graphite through to metrictank * For a secure setup, you must make sure these headers cannot be specified by users. You may need to run something in front to set the header correctly after authentication (e.g. [tsdb-gw](https://github.com/raintank/tsdb-gw) * orgs can only see the data that lives under their org-id, and also public data diff --git a/docs/quick-start-docker.md b/docs/quick-start-docker.md index d3dc36588a..e37e2fb250 100644 --- a/docs/quick-start-docker.md +++ b/docs/quick-start-docker.md @@ -31,7 +31,7 @@ The stack will listen on the following ports: * 2003 tcp (metrictank's carbon input) * 3000 tcp (grafana's http port) * 6060 tcp (metrictank's internal endpoint) -* 8080 tcp (the graphite api query endpoint) +* 8080 tcp (the graphite query endpoint) * 8125 udp (statsd endpoint) * 9042 tcp (cassandra) @@ -85,14 +85,15 @@ Add a new data source with with these settings: * Name: `metrictank` * Default: `yes` * Type: `Graphite` -* Url: `http://localhost:8080` -* Access: `direct` (not `proxy`) +* Url: `http://metrictank:6060` +* Access: `proxy` When you hit save, Grafana should succeed in talking to the data source. ![Add data source screenshot](https://raw.githubusercontent.com/raintank/metrictank/master/docs/assets/add-datasource-docker.png) -Note: it also works with `proxy` mode but then you have to enter `http://graphite-api:8080` as uri. +-Note: it also works with `direct` mode but then you have to enter `http://localhost:6060` as url. + Now let's see some data. If you go to `Dashboards`, `New` and add a new graph panel. In the metrics tab you should see a bunch of metrics already in the root hierarchy: @@ -102,7 +103,8 @@ In the metrics tab you should see a bunch of metrics already in the root hierarc * `metrictank.usage`: usage metrics reported by metrictank. See [Usage reporting](https://github.com/raintank/metrictank/blob/master/docs/usage-reporting.md) It may take a few minutes for the usage metrics to show up. -* `stats`: these are metrics coming from graphite-api, aggregated by statsdaemon and sent back to metrictank every second. +* `stats`: metrics aggregated by statsdaemon and sent into metrictank every second. Will only show up if something actually sends + metrics into statsdaemon (e.g. if graphite receives requests directly, you send stats to statsdaemon, etc) Note that metrictank is setup to track every metric on a 1-second granularity. If you wish to use it for less frequent metrics, diff --git a/scripts/config/metrictank-docker.ini b/scripts/config/metrictank-docker.ini index 1b7d94b7b5..ecc81a2afa 100644 --- a/scripts/config/metrictank-docker.ini +++ b/scripts/config/metrictank-docker.ini @@ -40,7 +40,7 @@ cassandra-consistency = one # tokenaware,hostpool-epsilon-greedy : prefer host that has the needed data, fallback to hostpool-epsilon-greedy. cassandra-host-selection-policy = tokenaware,hostpool-epsilon-greedy # cassandra timeout in milliseconds -cassandra-timeout = 1000 +cassandra-timeout = 10000 # max number of concurrent reads to cassandra cassandra-read-concurrency = 20 # max number of concurrent writes to cassandra @@ -135,7 +135,7 @@ max-points-per-req-hard = 20000000 # require x-org-id authentication to auth as a specific org. otherwise orgId 1 is assumed multi-tenant = true # in case our /render endpoint does not support the requested processing, proxy the request to this graphite -fallback-graphite-addr = http://graphite-api:8080 +fallback-graphite-addr = http://graphite # only log incoming requests if their timerange is at least this duration. Use 0 to disable log-min-dur = 5min @@ -250,7 +250,7 @@ protocol-version = 4 # write consistency (any|one|two|three|quorum|all|local_quorum|each_quorum|local_one consistency = one # cassandra request timeout -timeout = 1s +timeout = 10s # number of concurrent connections to cassandra num-conns = 10 # Max number of metricDefs allowed to be unwritten to cassandra diff --git a/vendor/github.com/kisielk/og-rek/LICENSE b/vendor/github.com/kisielk/og-rek/LICENSE new file mode 100644 index 0000000000..a2b16b5bd9 --- /dev/null +++ b/vendor/github.com/kisielk/og-rek/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2013 Kamil Kisiel + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/kisielk/og-rek/README.md b/vendor/github.com/kisielk/og-rek/README.md new file mode 100644 index 0000000000..566b48387e --- /dev/null +++ b/vendor/github.com/kisielk/og-rek/README.md @@ -0,0 +1,17 @@ +ogórek +====== +[![GoDoc](https://godoc.org/github.com/kisielk/og-rek?status.svg)](https://godoc.org/github.com/kisielk/og-rek) +[![Build Status](https://travis-ci.org/kisielk/og-rek.svg?branch=master)](https://travis-ci.org/kisielk/og-rek) + +ogórek is a Go library for encoding and decoding pickles. + +Fuzz Testing +------------ +Fuzz testing has been implemented for the decoder. To run fuzz tests do the following: + +``` +go get github.com/dvyukov/go-fuzz/go-fuzz +go get github.com/dvyukov/go-fuzz/go-fuzz-build +go-fuzz-build github.com/kisielk/og-rek +go-fuzz -bin=./ogórek-fuzz.zip -workdir=./fuzz +``` diff --git a/vendor/github.com/kisielk/og-rek/encode.go b/vendor/github.com/kisielk/og-rek/encode.go new file mode 100644 index 0000000000..95d1b67819 --- /dev/null +++ b/vendor/github.com/kisielk/og-rek/encode.go @@ -0,0 +1,384 @@ +package ogórek + +import ( + "encoding/binary" + "fmt" + "io" + "math" + "math/big" + "reflect" +) + +type TypeError struct { + typ string +} + +func (te *TypeError) Error() string { + return fmt.Sprintf("no support for type '%s'", te.typ) +} + +// An Encoder encodes Go data structures into pickle byte stream +type Encoder struct { + w io.Writer +} + +// NewEncoder returns a new Encoder struct with default values +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{w: w} +} + +// Encode writes the pickle encoding of v to w, the encoder's writer +func (e *Encoder) Encode(v interface{}) error { + rv := reflectValueOf(v) + err := e.encode(rv) + if err != nil { + return err + } + _, err = e.w.Write([]byte{opStop}) + return err +} + +func (e *Encoder) encode(rv reflect.Value) error { + + switch rk := rv.Kind(); rk { + + case reflect.Bool: + return e.encodeBool(rv.Bool()) + case reflect.Int, reflect.Int8, reflect.Int64, reflect.Int32, reflect.Int16: + return e.encodeInt(reflect.Int, rv.Int()) + case reflect.Uint8, reflect.Uint64, reflect.Uint, reflect.Uint32, reflect.Uint16: + return e.encodeInt(reflect.Uint, int64(rv.Uint())) + case reflect.String: + return e.encodeString(rv.String()) + case reflect.Array, reflect.Slice: + if rv.Type().Elem().Kind() == reflect.Uint8 { + return e.encodeBytes(rv.Bytes()) + } else if _, ok := rv.Interface().(Tuple); ok { + return e.encodeTuple(rv.Interface().(Tuple)) + } else { + return e.encodeArray(rv) + } + + case reflect.Map: + return e.encodeMap(rv) + + case reflect.Struct: + return e.encodeStruct(rv) + + case reflect.Float32, reflect.Float64: + return e.encodeFloat(float64(rv.Float())) + + case reflect.Interface: + // recurse until we get a concrete type + // could be optmized into a tail call + return e.encode(rv.Elem()) + + case reflect.Ptr: + + if rv.Elem().Kind() == reflect.Struct { + switch rv.Elem().Interface().(type) { + case None: + return e.encodeStruct(rv.Elem()) + } + } + + return e.encode(rv.Elem()) + + case reflect.Invalid: + _, err := e.w.Write([]byte{opNone}) + return err + default: + return &TypeError{typ: rk.String()} + } + + return nil +} + +func (e *Encoder) encodeTuple(t Tuple) error { + l := len(t) + + switch l { + case 0: + _, err := e.w.Write([]byte{opEmptyTuple}) + return err + + // TODO this are protocol 2 opcodes - check e.protocol before using them + //case 1: + //case 2: + //case 3: + } + + _, err := e.w.Write([]byte{opMark}) + if err != nil { + return err + } + + for i := 0; i < l; i++ { + err = e.encode(reflectValueOf(t[i])) + if err != nil { + return err + } + } + + _, err = e.w.Write([]byte{opTuple}) + return err +} + +func (e *Encoder) encodeArray(arr reflect.Value) error { + + l := arr.Len() + + _, err := e.w.Write([]byte{opEmptyList, opMark}) + if err != nil { + return err + } + + for i := 0; i < l; i++ { + v := arr.Index(i) + err = e.encode(v) + if err != nil { + return err + } + } + + _, err = e.w.Write([]byte{opAppends}) + return err +} + +func (e *Encoder) encodeBool(b bool) error { + var err error + + if b { + _, err = e.w.Write([]byte(opTrue)) + } else { + _, err = e.w.Write([]byte(opFalse)) + } + + return err +} + +func (e *Encoder) encodeBytes(byt []byte) error { + + l := len(byt) + + if l < 256 { + _, err := e.w.Write([]byte{opShortBinstring, byte(l)}) + if err != nil { + return err + } + } else { + _, err := e.w.Write([]byte{opBinstring}) + if err != nil { + return err + } + var b [4]byte + + binary.LittleEndian.PutUint32(b[:], uint32(l)) + _, err = e.w.Write(b[:]) + if err != nil { + return err + } + } + + _, err := e.w.Write(byt) + return err +} + +func (e *Encoder) encodeFloat(f float64) error { + var u uint64 + u = math.Float64bits(f) + + _, err := e.w.Write([]byte{opBinfloat}) + if err != nil { + return err + } + var b [8]byte + binary.BigEndian.PutUint64(b[:], uint64(u)) + + _, err = e.w.Write(b[:]) + return err +} + +func (e *Encoder) encodeInt(k reflect.Kind, i int64) error { + var err error + + // FIXME: need support for 64-bit ints + + switch { + case i > 0 && i < math.MaxUint8: + _, err = e.w.Write([]byte{opBinint1, byte(i)}) + case i > 0 && i < math.MaxUint16: + _, err = e.w.Write([]byte{opBinint2, byte(i), byte(i >> 8)}) + case i >= math.MinInt32 && i <= math.MaxInt32: + _, err = e.w.Write([]byte{opBinint}) + if err != nil { + return err + } + var b [4]byte + binary.LittleEndian.PutUint32(b[:], uint32(i)) + _, err = e.w.Write(b[:]) + default: // int64, but as a string :/ + _, err = e.w.Write([]byte{opInt}) + if err != nil { + return err + } + fmt.Fprintf(e.w, "%d\n", i) + } + + return err +} + +func (e *Encoder) encodeLong(b *big.Int) error { + // TODO if e.protocol >= 2 use opLong1 & opLong4 + _, err := fmt.Fprintf(e.w, "%c%dL\n", opLong, b) + return err +} + +func (e *Encoder) encodeMap(m reflect.Value) error { + + keys := m.MapKeys() + + l := len(keys) + + _, err := e.w.Write([]byte{opEmptyDict}) + if err != nil { + return err + } + + if l > 0 { + _, err := e.w.Write([]byte{opMark}) + if err != nil { + return err + } + + for _, k := range keys { + err = e.encode(k) + if err != nil { + return err + } + v := m.MapIndex(k) + + err = e.encode(v) + if err != nil { + return err + } + } + + _, err = e.w.Write([]byte{opSetitems}) + if err != nil { + return err + } + } + + return nil +} + +func (e *Encoder) encodeString(s string) error { + return e.encodeBytes([]byte(s)) +} + +func (e *Encoder) encodeCall(v *Call) error { + _, err := fmt.Fprintf(e.w, "%c%s\n%s\n", opGlobal, v.Callable.Module, v.Callable.Name) + if err != nil { + return err + } + err = e.encodeTuple(v.Args) + if err != nil { + return err + } + _, err = e.w.Write([]byte{opReduce}) + return err +} + +func (e *Encoder) encodeStruct(st reflect.Value) error { + + typ := st.Type() + + // first test if it's one of our internal python structs + switch v := st.Interface().(type) { + case None: + _, err := e.w.Write([]byte{opNone}) + return err + case Call: + return e.encodeCall(&v) + case big.Int: + return e.encodeLong(&v) + } + + structTags := getStructTags(st) + + _, err := e.w.Write([]byte{opEmptyDict, opMark}) + if err != nil { + return err + } + + if structTags != nil { + for f, i := range structTags { + err := e.encodeString(f) + if err != nil { + return err + } + + err = e.encode(st.Field(i)) + if err != nil { + return err + } + } + } else { + l := typ.NumField() + for i := 0; i < l; i++ { + fty := typ.Field(i) + if fty.PkgPath != "" { + continue // skip unexported names + } + + err := e.encodeString(fty.Name) + if err != nil { + return err + } + + err = e.encode(st.Field(i)) + if err != nil { + return err + } + } + } + + _, err = e.w.Write([]byte{opSetitems}) + return err +} + +func reflectValueOf(v interface{}) reflect.Value { + + rv, ok := v.(reflect.Value) + if !ok { + rv = reflect.ValueOf(v) + } + return rv +} + +func getStructTags(ptr reflect.Value) map[string]int { + if ptr.Kind() != reflect.Struct { + return nil + } + + m := make(map[string]int) + + t := ptr.Type() + + l := t.NumField() + numTags := 0 + for i := 0; i < l; i++ { + field := t.Field(i).Tag.Get("pickle") + if field != "" { + m[field] = i + numTags++ + } + } + + if numTags == 0 { + return nil + } + + return m +} diff --git a/vendor/github.com/kisielk/og-rek/fuzz.go b/vendor/github.com/kisielk/og-rek/fuzz.go new file mode 100644 index 0000000000..bc59cc0848 --- /dev/null +++ b/vendor/github.com/kisielk/og-rek/fuzz.go @@ -0,0 +1,17 @@ +// +build gofuzz + +package ogórek + +import ( + "bytes" +) + +func Fuzz(data []byte) int { + buf := bytes.NewBuffer(data) + dec := NewDecoder(buf) + _, err := dec.Decode() + if err != nil { + return 0 + } + return 1 +} diff --git a/vendor/github.com/kisielk/og-rek/ogorek.go b/vendor/github.com/kisielk/og-rek/ogorek.go new file mode 100644 index 0000000000..305e6ad9af --- /dev/null +++ b/vendor/github.com/kisielk/og-rek/ogorek.go @@ -0,0 +1,997 @@ +// Package ogórek is a library for decoding Python's pickle format. +// +// ogórek is Polish for "pickle". +package ogórek + +import ( + "bufio" + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "math/big" + "reflect" + "strconv" +) + +// Opcodes +const ( + opMark byte = '(' // push special markobject on stack + opStop = '.' // every pickle ends with STOP + opPop = '0' // discard topmost stack item + opPopMark = '1' // discard stack top through topmost markobject + opDup = '2' // duplicate top stack item + opFloat = 'F' // push float object; decimal string argument + opInt = 'I' // push integer or bool; decimal string argument + opBinint = 'J' // push four-byte signed int + opBinint1 = 'K' // push 1-byte unsigned int + opLong = 'L' // push long; decimal string argument + opBinint2 = 'M' // push 2-byte unsigned int + opNone = 'N' // push None + opPersid = 'P' // push persistent object; id is taken from string arg + opBinpersid = 'Q' // " " " ; " " " " stack + opReduce = 'R' // apply callable to argtuple, both on stack + opString = 'S' // push string; NL-terminated string argument + opBinstring = 'T' // push string; counted binary string argument + opShortBinstring = 'U' // " " ; " " " " < 256 bytes + opUnicode = 'V' // push Unicode string; raw-unicode-escaped"d argument + opBinunicode = 'X' // " " " ; counted UTF-8 string argument + opAppend = 'a' // append stack top to list below it + opBuild = 'b' // call __setstate__ or __dict__.update() + opGlobal = 'c' // push self.find_class(modname, name); 2 string args + opDict = 'd' // build a dict from stack items + opEmptyDict = '}' // push empty dict + opAppends = 'e' // extend list on stack by topmost stack slice + opGet = 'g' // push item from memo on stack; index is string arg + opBinget = 'h' // " " " " " " ; " " 1-byte arg + opInst = 'i' // build & push class instance + opLongBinget = 'j' // push item from memo on stack; index is 4-byte arg + opList = 'l' // build list from topmost stack items + opEmptyList = ']' // push empty list + opObj = 'o' // build & push class instance + opPut = 'p' // store stack top in memo; index is string arg + opBinput = 'q' // " " " " " ; " " 1-byte arg + opLongBinput = 'r' // " " " " " ; " " 4-byte arg + opSetitem = 's' // add key+value pair to dict + opTuple = 't' // build tuple from topmost stack items + opEmptyTuple = ')' // push empty tuple + opSetitems = 'u' // modify dict by adding topmost key+value pairs + opBinfloat = 'G' // push float; arg is 8-byte float encoding + + opTrue = "I01\n" // not an opcode; see INT docs in pickletools.py + opFalse = "I00\n" // not an opcode; see INT docs in pickletools.py + + // Protocol 2 + + opProto = '\x80' // identify pickle protocol + opNewobj = '\x81' // build object by applying cls.__new__ to argtuple + opExt1 = '\x82' // push object from extension registry; 1-byte index + opExt2 = '\x83' // ditto, but 2-byte index + opExt4 = '\x84' // ditto, but 4-byte index + opTuple1 = '\x85' // build 1-tuple from stack top + opTuple2 = '\x86' // build 2-tuple from two topmost stack items + opTuple3 = '\x87' // build 3-tuple from three topmost stack items + opNewtrue = '\x88' // push True + opNewfalse = '\x89' // push False + opLong1 = '\x8a' // push long from < 256 bytes + opLong4 = '\x8b' // push really big long + + // Protocol 4 + opShortBinUnicode = '\x8c' // push short string; UTF-8 length < 256 bytes + opMemoize = '\x94' // store top of the stack in memo + opFrame = '\x95' // indicate the beginning of a new frame +) + +var errNotImplemented = errors.New("unimplemented opcode") +var ErrInvalidPickleVersion = errors.New("invalid pickle version") +var errNoMarker = errors.New("no marker in stack") +var errStackUnderflow = errors.New("pickle: stack underflow") + +type OpcodeError struct { + Key byte + Pos int +} + +func (e OpcodeError) Error() string { + return fmt.Sprintf("Unknown opcode %d (%c) at position %d: %q", e.Key, e.Key, e.Pos, e.Key) +} + +// special marker +type mark struct{} + +// None is a representation of Python's None. +type None struct{} + +// Tuple is a representation of Python's tuple. +type Tuple []interface{} + +// Decoder is a decoder for pickle streams. +type Decoder struct { + r *bufio.Reader + stack []interface{} + memo map[string]interface{} + + // a reusable buffer that can be used by the various decoding functions + // functions using this should call buf.Reset to clear the old contents + buf bytes.Buffer + + // reusable buffer for readLine + line []byte +} + +// NewDecoder constructs a new Decoder which will decode the pickle stream in r. +func NewDecoder(r io.Reader) *Decoder { + reader := bufio.NewReader(r) + return &Decoder{r: reader, stack: make([]interface{}, 0), memo: make(map[string]interface{})} +} + +// Decode decodes the pickle stream and returns the result or an error. +func (d *Decoder) Decode() (interface{}, error) { + + insn := 0 +loop: + for { + key, err := d.r.ReadByte() + if err != nil { + if err == io.EOF && insn != 0 { + err = io.ErrUnexpectedEOF + } + return nil, err + } + + insn++ + + switch key { + case opMark: + d.mark() + case opStop: + break loop + case opPop: + _, err = d.pop() + case opPopMark: + d.popMark() + case opDup: + err = d.dup() + case opFloat: + err = d.loadFloat() + case opInt: + err = d.loadInt() + case opBinint: + err = d.loadBinInt() + case opBinint1: + err = d.loadBinInt1() + case opLong: + err = d.loadLong() + case opBinint2: + err = d.loadBinInt2() + case opNone: + err = d.loadNone() + case opPersid: + err = d.loadPersid() + case opBinpersid: + err = d.loadBinPersid() + case opReduce: + err = d.reduce() + case opString: + err = d.loadString() + case opBinstring: + err = d.loadBinString() + case opShortBinstring: + err = d.loadShortBinString() + case opUnicode: + err = d.loadUnicode() + case opBinunicode: + err = d.loadBinUnicode() + case opAppend: + err = d.loadAppend() + case opBuild: + err = d.build() + case opGlobal: + err = d.global() + case opDict: + err = d.loadDict() + case opEmptyDict: + err = d.loadEmptyDict() + case opAppends: + err = d.loadAppends() + case opGet: + err = d.get() + case opBinget: + err = d.binGet() + case opInst: + err = d.inst() + case opLong1: + err = d.loadLong1() + case opNewfalse: + err = d.loadBool(false) + case opNewtrue: + err = d.loadBool(true) + case opLongBinget: + err = d.longBinGet() + case opList: + err = d.loadList() + case opEmptyList: + d.push([]interface{}{}) + case opObj: + err = d.obj() + case opPut: + err = d.loadPut() + case opBinput: + err = d.binPut() + case opLongBinput: + err = d.longBinPut() + case opSetitem: + err = d.loadSetItem() + case opTuple: + err = d.loadTuple() + case opTuple1: + err = d.loadTuple1() + case opTuple2: + err = d.loadTuple2() + case opTuple3: + err = d.loadTuple3() + case opEmptyTuple: + d.push(Tuple{}) + case opSetitems: + err = d.loadSetItems() + case opBinfloat: + err = d.binFloat() + case opFrame: + err = d.loadFrame() + case opShortBinUnicode: + err = d.loadShortBinUnicode() + case opMemoize: + err = d.loadMemoize() + case opProto: + v, err := d.r.ReadByte() + if err == nil && v != 2 { + err = ErrInvalidPickleVersion + } + + default: + return nil, OpcodeError{key, insn} + } + + if err != nil { + if err == errNotImplemented { + return nil, OpcodeError{key, insn} + } + // EOF from individual opcode decoder is unexpected end of stream + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return nil, err + } + } + return d.pop() +} + +// readLine reads next line from pickle stream +// returned line is valid only till next call to readLine +func (d *Decoder) readLine() ([]byte, error) { + var ( + data []byte + isPrefix = true + err error + ) + d.line = d.line[:0] + for isPrefix { + data, isPrefix, err = d.r.ReadLine() + if err != nil { + return d.line, err + } + d.line = append(d.line, data...) + } + return d.line, nil +} + +// Push a marker +func (d *Decoder) mark() { + d.push(mark{}) +} + +// Return the position of the topmost marker +func (d *Decoder) marker() (int, error) { + m := mark{} + for k := len(d.stack) - 1; k >= 0; k-- { + if d.stack[k] == m { + return k, nil + } + } + return 0, errNoMarker +} + +// Append a new value +func (d *Decoder) push(v interface{}) { + d.stack = append(d.stack, v) +} + +// Pop a value +// The returned error is errStackUnderflow if decoder stack is empty +func (d *Decoder) pop() (interface{}, error) { + ln := len(d.stack) - 1 + if ln < 0 { + return nil, errStackUnderflow + } + v := d.stack[ln] + d.stack = d.stack[:ln] + return v, nil +} + +// Pop a value (when you know for sure decoder stack is not empty) +func (d *Decoder) xpop() interface{} { + v, err := d.pop() + if err != nil { + panic(err) + } + return v +} + +// Discard the stack through to the topmost marker +func (d *Decoder) popMark() error { + return errNotImplemented +} + +// Duplicate the top stack item +func (d *Decoder) dup() error { + if len(d.stack) < 1 { + return errStackUnderflow + } + d.stack = append(d.stack, d.stack[len(d.stack)-1]) + return nil +} + +// Push a float +func (d *Decoder) loadFloat() error { + line, err := d.readLine() + if err != nil { + return err + } + v, err := strconv.ParseFloat(string(line), 64) + if err != nil { + return err + } + d.push(interface{}(v)) + return nil +} + +// Push an int +func (d *Decoder) loadInt() error { + line, err := d.readLine() + if err != nil { + return err + } + + var val interface{} + + switch string(line) { + case opFalse[1:3]: + val = false + case opTrue[1:3]: + val = true + default: + i, err := strconv.ParseInt(string(line), 10, 64) + if err != nil { + return err + } + val = i + } + + d.push(val) + return nil +} + +// Push a four-byte signed int +func (d *Decoder) loadBinInt() error { + var b [4]byte + _, err := io.ReadFull(d.r, b[:]) + if err != nil { + return err + } + v := binary.LittleEndian.Uint32(b[:]) + d.push(int64(v)) + return nil +} + +// Push a 1-byte unsigned int +func (d *Decoder) loadBinInt1() error { + b, err := d.r.ReadByte() + if err != nil { + return err + } + d.push(int64(b)) + return nil +} + +// Push a long +func (d *Decoder) loadLong() error { + line, err := d.readLine() + if err != nil { + return err + } + l := len(line) + if l < 1 || line[l-1] != 'L' { + return io.ErrUnexpectedEOF + } + v := new(big.Int) + _, ok := v.SetString(string(line[:l-1]), 10) + if !ok { + return fmt.Errorf("pickle: loadLong: invalid string") + } + d.push(v) + return nil +} + +// Push a long1 +func (d *Decoder) loadLong1() error { + rawNum := []byte{} + b, err := d.r.ReadByte() + if err != nil { + return err + } + length, err := decodeLong(string(b)) + if err != nil { + return err + } + for i := 0; int64(i) < length.Int64(); i++ { + b2, err := d.r.ReadByte() + if err != nil { + return err + } + rawNum = append(rawNum, b2) + } + decodedNum, err := decodeLong(string(rawNum)) + d.push(decodedNum) + return nil +} + +// Push a 2-byte unsigned int +func (d *Decoder) loadBinInt2() error { + var b [2]byte + _, err := io.ReadFull(d.r, b[:]) + if err != nil { + return err + } + v := binary.LittleEndian.Uint16(b[:]) + d.push(int64(v)) + return nil +} + +// Push None +func (d *Decoder) loadNone() error { + d.push(None{}) + return nil +} + +// Push a persistent object id +func (d *Decoder) loadPersid() error { + return errNotImplemented +} + +// Push a persistent object id from items on the stack +func (d *Decoder) loadBinPersid() error { + return errNotImplemented +} + +type Call struct { + Callable Class + Args Tuple +} + +func (d *Decoder) reduce() error { + if len(d.stack) < 2 { + return errStackUnderflow + } + xargs := d.xpop() + xclass := d.xpop() + args, ok := xargs.(Tuple) + if !ok { + return fmt.Errorf("pickle: reduce: invalid args: %T", xargs) + } + class, ok := xclass.(Class) + if !ok { + return fmt.Errorf("pickle: reduce: invalid class: %T", xclass) + } + d.stack = append(d.stack, Call{Callable: class, Args: args}) + return nil +} + +func decodeStringEscape(b []byte) string { + // TODO + return string(b) +} + +// Push a string +func (d *Decoder) loadString() error { + line, err := d.readLine() + if err != nil { + return err + } + + if len(line) < 2 { + return io.ErrUnexpectedEOF + } + + var delim byte + switch line[0] { + case '\'': + delim = '\'' + case '"': + delim = '"' + default: + return fmt.Errorf("invalid string delimiter: %c", line[0]) + } + + if line[len(line)-1] != delim { + return io.ErrUnexpectedEOF + } + + d.push(decodeStringEscape(line[1 : len(line)-1])) + return nil +} + +func (d *Decoder) loadBinString() error { + var b [4]byte + _, err := io.ReadFull(d.r, b[:]) + if err != nil { + return err + } + v := binary.LittleEndian.Uint32(b[:]) + + d.buf.Reset() + d.buf.Grow(int(v)) + _, err = io.CopyN(&d.buf, d.r, int64(v)) + if err != nil { + return err + } + d.push(d.buf.String()) + return nil +} + +func (d *Decoder) loadShortBinString() error { + b, err := d.r.ReadByte() + if err != nil { + return err + } + + d.buf.Reset() + d.buf.Grow(int(b)) + _, err = io.CopyN(&d.buf, d.r, int64(b)) + if err != nil { + return err + } + d.push(d.buf.String()) + return nil +} + +func (d *Decoder) loadUnicode() error { + line, err := d.readLine() + + if err != nil { + return err + } + sline := string(line) + + d.buf.Reset() + d.buf.Grow(len(line)) // approximation + + for len(sline) > 0 { + var r rune + var err error + for len(sline) > 0 && sline[0] == '\'' { + d.buf.WriteByte(sline[0]) + sline = sline[1:] + } + if len(sline) == 0 { + break + } + r, _, sline, err = strconv.UnquoteChar(sline, '\'') + if err != nil { + return err + } + d.buf.WriteRune(r) + } + if len(sline) > 0 { + return fmt.Errorf("characters remaining after loadUnicode operation: %s", sline) + } + + d.push(d.buf.String()) + return nil +} + +func (d *Decoder) loadBinUnicode() error { + var length int32 + for i := 0; i < 4; i++ { + t, err := d.r.ReadByte() + if err != nil { + return err + } + length = length | (int32(t) << uint(8*i)) + } + rawB := []byte{} + for z := 0; int32(z) < length; z++ { + n, err := d.r.ReadByte() + if err != nil { + return err + } + rawB = append(rawB, n) + } + d.push(string(rawB)) + return nil +} + +func (d *Decoder) loadAppend() error { + if len(d.stack) < 2 { + return errStackUnderflow + } + v := d.xpop() + l := d.stack[len(d.stack)-1] + switch l.(type) { + case []interface{}: + l := l.([]interface{}) + d.stack[len(d.stack)-1] = append(l, v) + default: + return fmt.Errorf("pickle: loadAppend: expected a list, got %T", l) + } + return nil +} + +func (d *Decoder) build() error { + return errNotImplemented +} + +type Class struct { + Module, Name string +} + +func (d *Decoder) global() error { + module, err := d.readLine() + if err != nil { + return err + } + smodule := string(module) + name, err := d.readLine() + if err != nil { + return err + } + sname := string(name) + d.stack = append(d.stack, Class{Module: smodule, Name: sname}) + return nil +} + +func (d *Decoder) loadDict() error { + k, err := d.marker() + if err != nil { + return err + } + + m := make(map[interface{}]interface{}, 0) + items := d.stack[k+1:] + if len(items) % 2 != 0 { + return fmt.Errorf("pickle: loadDict: odd # of elements") + } + for i := 0; i < len(items); i += 2 { + key := items[i] + if !reflect.TypeOf(key).Comparable() { + return fmt.Errorf("pickle: loadDict: invalid key type %T", key) + } + m[key] = items[i+1] + } + d.stack = append(d.stack[:k], m) + return nil +} + +func (d *Decoder) loadEmptyDict() error { + m := make(map[interface{}]interface{}, 0) + d.push(m) + return nil +} + +func (d *Decoder) loadAppends() error { + k, err := d.marker() + if err != nil { + return err + } + if k < 1 { + return errStackUnderflow + } + + l := d.stack[k-1] + switch l.(type) { + case []interface{}: + l := l.([]interface{}) + for _, v := range d.stack[k+1 : len(d.stack)] { + l = append(l, v) + } + d.stack = append(d.stack[:k-1], l) + default: + return fmt.Errorf("pickle: loadAppends: expected a list, got %T", l) + } + return nil +} + +func (d *Decoder) get() error { + line, err := d.readLine() + if err != nil { + return err + } + v, ok := d.memo[string(line)] + if !ok { + return fmt.Errorf("pickle: memo: key error %q", line) + } + d.push(v) + return nil +} + +func (d *Decoder) binGet() error { + b, err := d.r.ReadByte() + if err != nil { + return err + } + + v, ok := d.memo[strconv.Itoa(int(b))] + if !ok { + return fmt.Errorf("pickle: memo: key error %d", b) + } + d.push(v) + return nil +} + +func (d *Decoder) inst() error { + return errNotImplemented +} + +func (d *Decoder) longBinGet() error { + var b [4]byte + _, err := io.ReadFull(d.r, b[:]) + if err != nil { + return err + } + v := binary.LittleEndian.Uint32(b[:]) + vv, ok := d.memo[strconv.Itoa(int(v))] + if !ok { + return fmt.Errorf("pickle: memo: key error %d", v) + } + d.push(vv) + return nil +} + +func (d *Decoder) loadBool(b bool) error { + d.push(b) + return nil +} + +func (d *Decoder) loadList() error { + k, err := d.marker() + if err != nil { + return err + } + + v := append([]interface{}{}, d.stack[k+1:]...) + d.stack = append(d.stack[:k], v) + return nil +} + +func (d *Decoder) loadTuple() error { + k, err := d.marker() + if err != nil { + return err + } + + v := append(Tuple{}, d.stack[k+1:]...) + d.stack = append(d.stack[:k], v) + return nil +} + +func (d *Decoder) loadTuple1() error { + if len(d.stack) < 1 { + return errStackUnderflow + } + k := len(d.stack) - 1 + v := append(Tuple{}, d.stack[k:]...) + d.stack = append(d.stack[:k], v) + return nil +} + +func (d *Decoder) loadTuple2() error { + if len(d.stack) < 2 { + return errStackUnderflow + } + k := len(d.stack) - 2 + v := append(Tuple{}, d.stack[k:]...) + d.stack = append(d.stack[:k], v) + return nil +} + +func (d *Decoder) loadTuple3() error { + if len(d.stack) < 3 { + return errStackUnderflow + } + k := len(d.stack) - 3 + v := append(Tuple{}, d.stack[k:]...) + d.stack = append(d.stack[:k], v) + return nil +} + +func (d *Decoder) obj() error { + return errNotImplemented +} + +func (d *Decoder) loadPut() error { + line, err := d.readLine() + if err != nil { + return err + } + if len(d.stack) < 1 { + return errStackUnderflow + } + d.memo[string(line)] = d.stack[len(d.stack)-1] + return nil +} + +func (d *Decoder) binPut() error { + if len(d.stack) < 1 { + return errStackUnderflow + } + b, err := d.r.ReadByte() + if err != nil { + return err + } + + d.memo[strconv.Itoa(int(b))] = d.stack[len(d.stack)-1] + return nil +} + +func (d *Decoder) longBinPut() error { + if len(d.stack) < 1 { + return errStackUnderflow + } + var b [4]byte + _, err := io.ReadFull(d.r, b[:]) + if err != nil { + return err + } + v := binary.LittleEndian.Uint32(b[:]) + d.memo[strconv.Itoa(int(v))] = d.stack[len(d.stack)-1] + return nil +} + +func (d *Decoder) loadSetItem() error { + if len(d.stack) < 3 { + return errStackUnderflow + } + v := d.xpop() + k := d.xpop() + m := d.stack[len(d.stack)-1] + switch m := m.(type) { + case map[interface{}]interface{}: + if !reflect.TypeOf(k).Comparable() { + return fmt.Errorf("pickle: loadSetItem: invalid key type %T", k) + } + m[k] = v + default: + return fmt.Errorf("pickle: loadSetItem: expected a map, got %T", m) + } + return nil +} + +func (d *Decoder) loadSetItems() error { + k, err := d.marker() + if err != nil { + return err + } + if k < 1 { + return errStackUnderflow + } + + l := d.stack[k-1] + switch m := l.(type) { + case map[interface{}]interface{}: + if (len(d.stack) - (k + 1)) % 2 != 0 { + return fmt.Errorf("pickle: loadSetItems: odd # of elements") + } + for i := k + 1; i < len(d.stack); i += 2 { + key := d.stack[i] + if !reflect.TypeOf(key).Comparable() { + return fmt.Errorf("pickle: loadSetItems: invalid key type %T", key) + } + m[d.stack[i]] = d.stack[i+1] + } + d.stack = append(d.stack[:k-1], m) + default: + return fmt.Errorf("pickle: loadSetItems: expected a map, got %T", m) + } + return nil +} + +func (d *Decoder) binFloat() error { + var b [8]byte + _, err := io.ReadFull(d.r, b[:]) + if err != nil { + return err + } + u := binary.BigEndian.Uint64(b[:]) + d.stack = append(d.stack, math.Float64frombits(u)) + return nil +} + +// loadFrame discards the framing opcode+information, this information is useful to do one large read (instead of many small reads) +// https://www.python.org/dev/peps/pep-3154/#framing +func (d *Decoder) loadFrame() error { + var b [8]byte + _, err := io.ReadFull(d.r, b[:]) + if err != nil { + return err + } + return nil +} + +func (d *Decoder) loadShortBinUnicode() error { + b, err := d.r.ReadByte() + if err != nil { + return err + } + + d.buf.Reset() + d.buf.Grow(int(b)) + _, err = io.CopyN(&d.buf, d.r, int64(b)) + if err != nil { + return err + } + d.push(d.buf.String()) + return nil +} + +func (d *Decoder) loadMemoize() error { + if len(d.stack) < 1 { + return errStackUnderflow + } + d.memo[strconv.Itoa(len(d.memo))] = d.stack[len(d.stack)-1] + return nil +} + +// decodeLong takes a byte array of 2's compliment little-endian binary words and converts them +// to a big integer +func decodeLong(data string) (*big.Int, error) { + decoded := big.NewInt(0) + var negative bool + switch x := len(data); { + case x < 1: + return decoded, nil + case x > 1: + if data[x-1] > 127 { + negative = true + } + for i := x - 1; i >= 0; i-- { + a := big.NewInt(int64(data[i])) + for n := i; n > 0; n-- { + a = a.Lsh(a, 8) + } + decoded = decoded.Add(a, decoded) + } + default: + if data[0] > 127 { + negative = true + } + decoded = big.NewInt(int64(data[0])) + } + + if negative { + // Subtract 1 from the number + one := big.NewInt(1) + decoded.Sub(decoded, one) + + // Flip the bits + bytes := decoded.Bytes() + for i := 0; i < len(bytes); i++ { + bytes[i] = ^bytes[i] + } + decoded.SetBytes(bytes) + + // Mark as negative now conversion has been completed + decoded.Neg(decoded) + } + return decoded, nil +} diff --git a/vendor/vendor.json b/vendor/vendor.json index b18505d608..e3fd134613 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -212,6 +212,18 @@ "revision": "8ddce2a84170772b95dd5d576c48d517b22cac63", "revisionTime": "2016-01-05T22:08:40Z" }, + { + "checksumSHA1": "6s2IAJ1smhtl7YePdwZZ1J2zqeA=", + "path": "github.com/kisielk/og-rek", + "revision": "ec792bc6e6aa06a6c490e8d292e15cca173c8bd3", + "revisionTime": "2017-04-05T22:37:46Z" + }, + { + "checksumSHA1": "o7abpsEIXBLz5n/khgI2QPRqSQA=", + "path": "github.com/kisielk/whisper-go/whisper", + "revision": "82e8091afdea241119c34a452fe24fcc2a0b962e", + "revisionTime": "2014-01-12T13:57:52Z" + }, { "checksumSHA1": "+CqJGh7NIDMnHgScq9sl9tPrnVM=", "path": "github.com/klauspost/compress/flate", @@ -230,12 +242,6 @@ "revision": "09cded8978dc9e80714c4d85b0322337b0a1e5e0", "revisionTime": "2016-03-02T07:53:16Z" }, - { - "checksumSHA1": "o7abpsEIXBLz5n/khgI2QPRqSQA=", - "path": "github.com/kisielk/whisper-go/whisper", - "revision": "82e8091afdea241119c34a452fe24fcc2a0b962e", - "revisionTime": "2014-01-12T13:57:52Z" - }, { "checksumSHA1": "7ttJJBMDGKL63tX23fNmW7r7NvQ=", "path": "github.com/klauspost/crc32",