kitgen implementation (#589)

* cmd/kitgen: parse

* First stymie

* Sketch developing

* Broken sketch.

* Further progress - still doesn't build

* Need to collect ast.Exprs, not strings

* Running downhill.

* sd: add Stop method to Instancer interface

Every implementation (modulo mock/test impls) already provided this
method, and so it makes sense to lift it to the interface definition.

Closes #566.

* Fruitful avenue. Committing for travel.

* Needs uniquify for varnames

* Track gauge values and support default tags for dogstatsd

Dogstatsd doesn't support sending deltas for gauges, so we must
maintain the raw gauge value ourselves.

* Service generation tests work

* add failing test for cloudwatch metrics, that are not reset

* Refactor cloudwatch: Reset().Walk() on every Send(), like influx impl does.

Note there is a breaking API change, as the cloudwatch object now has optional parameters.

* Tolerate that there may not be any lables, if the teststat.FillCounter() did not add any samples.

* Use Cloudwatch options in the struct itself, which is cleaner

* sd: fix TestDefaultEndpointer flake, hopefully

* util/conn: more detail for flaky test

* removed deprecated functions

changed `NewContext -> NewOutgoingContext` and `FromContext ->
FromIncomingContext` as described in
[metadta](https://github.com/grpc/grpc-go/blob/master/metadata/metadata.go)

* Functions extracted from inline codefile

* Cleaner/easier way for user to specify Cloudwatch metric percentiles.

* fix test to read quantile metrics with p prefix

* test cloudwatch.WithPercentiles()

* Handles anonymous fields for all parts of interface

* Handles underscore param names, produces compile-able code

* do not prefix metrics with 'p', just like it was previously.

* Fix for dogstatsd metrics with default tags and no labelValues

Signed-off-by: James Hamlin <james@goforward.com>

* Converted flat to a layout - proceeding to implement default

* Convenience function for formatting to a tree

* Fix spelling of deregisters

https://en.wiktionary.org/wiki/deregister

* Add basic auth middleware

* Default layout

* Need to handle mutating trees more effectively to do default layout.

* Basic Auth: optimize memory allocation.

* Fix typo

* cache required creds' slices

* Clean up comment

* Replacing idents successfully

* improve error handling and style

* Constructs import paths usefully

* Some debugging - transit

* Set time unit on metrics.Timer

* Changes as per code review

* fix missing comma in example histogram code

* update_deps.bash: handle detached HEAD better

* .travis.yml: go1.9 + tip exclusively

* circle.yml: go1.9 exclusively

* Selectify works - need to rearrange some idents now

* Updating golden masters so that they build

* Nearly 100% functionality

* Updated masters - all seems to work

* Now testing that everything builds

* Removing AST experiments

* Chopping up long sourcecontext.gog

* Tiny little notes

* auth/jwt: add claim factory to example

* auth/jwt: minor gofmt fixes

* fix typo in addcli

* Cleaning up some type assertion digging

* Remove dependency on juju

* Downstream usages of ratelimit package

* Recreating profilesvc issue

* Adding .ignore for rg

* flat layout works with defined types

* Default layout mostly works - one remaining selectify issue

* Debugging replaces - determined we need to do cloning

* Halfway through an edit - taking it home

* Works with new profilesvc testcases

* Try to fix Thrift failure (again) (#630)

* Empty commit to trigger CI

* examples/addsvc: rebuild with latest thrift

* cmd/kitgen: parse

* First stymie

* Sketch developing

* Broken sketch.

* Further progress - still doesn't build

* Need to collect ast.Exprs, not strings

* Running downhill.

* Fruitful avenue. Committing for travel.

* Needs uniquify for varnames

* Service generation tests work

* Functions extracted from inline codefile

* Handles anonymous fields for all parts of interface

* Handles underscore param names, produces compile-able code

* Converted flat to a layout - proceeding to implement default

* Convenience function for formatting to a tree

* Default layout

* Need to handle mutating trees more effectively to do default layout.

* Replacing idents successfully

* Constructs import paths usefully

* Some debugging - transit

* Selectify works - need to rearrange some idents now

* Updating golden masters so that they build

* Nearly 100% functionality

* Updated masters - all seems to work

* Now testing that everything builds

* Removing AST experiments

* Chopping up long sourcecontext.gog

* Tiny little notes

* Cleaning up some type assertion digging

* Recreating profilesvc issue

* Adding .ignore for rg

* flat layout works with defined types

* Default layout mostly works - one remaining selectify issue

* Debugging replaces - determined we need to do cloning

* Halfway through an edit - taking it home

* Works with new profilesvc testcases

cmd/kitgen/.ignore

@@ -0,0 +1 @@
+testdata/*/*/

cmd/kitgen/arg.go

@@ -0,0 +1,36 @@
+package main
+
+import "go/ast"
+
+type arg struct {
+	name, asField *ast.Ident
+	typ           ast.Expr
+}
+
+func (a arg) chooseName(scope *ast.Scope) *ast.Ident {
+	if a.name == nil || scope.Lookup(a.name.Name) != nil {
+		return inventName(a.typ, scope)
+	}
+	return a.name
+}
+
+func (a arg) field(scope *ast.Scope) *ast.Field {
+	return &ast.Field{
+		Names: []*ast.Ident{a.chooseName(scope)},
+		Type:  a.typ,
+	}
+}
+
+func (a arg) result() *ast.Field {
+	return &ast.Field{
+		Names: nil,
+		Type:  a.typ,
+	}
+}
+
+func (a arg) exported() *ast.Field {
+	return &ast.Field{
+		Names: []*ast.Ident{id(export(a.asField.Name))},
+		Type:  a.typ,
+	}
+}

cmd/kitgen/ast_helpers.go

@@ -0,0 +1,208 @@
+package main
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"strings"
+	"unicode"
+)
+
+func export(s string) string {
+	return strings.Title(s)
+}
+
+func unexport(s string) string {
+	first := true
+	return strings.Map(func(r rune) rune {
+		if first {
+			first = false
+			return unicode.ToLower(r)
+		}
+		return r
+	}, s)
+}
+
+func inventName(t ast.Expr, scope *ast.Scope) *ast.Ident {
+	n := baseName(t)
+	for try := 0; ; try++ {
+		nstr := pickName(n, try)
+		obj := ast.NewObj(ast.Var, nstr)
+		if alt := scope.Insert(obj); alt == nil {
+			return ast.NewIdent(nstr)
+		}
+	}
+}
+
+func baseName(t ast.Expr) string {
+	switch tt := t.(type) {
+	default:
+		panic(fmt.Sprintf("don't know how to choose a base name for %T (%[1]v)", tt))
+	case *ast.ArrayType:
+		return "slice"
+	case *ast.Ident:
+		return tt.Name
+	case *ast.SelectorExpr:
+		return tt.Sel.Name
+	}
+}
+
+func pickName(base string, idx int) string {
+	if idx == 0 {
+		switch base {
+		default:
+			return strings.Split(base, "")[0]
+		case "Context":
+			return "ctx"
+		case "error":
+			return "err"
+		}
+	}
+	return fmt.Sprintf("%s%d", base, idx)
+}
+
+func scopeWith(names ...string) *ast.Scope {
+	scope := ast.NewScope(nil)
+	for _, name := range names {
+		scope.Insert(ast.NewObj(ast.Var, name))
+	}
+	return scope
+}
+
+type visitFn func(ast.Node, func(ast.Node))
+
+func (fn visitFn) Visit(node ast.Node, r func(ast.Node)) Visitor {
+	fn(node, r)
+	return fn
+}
+
+func replaceIdent(src ast.Node, named string, with ast.Node) ast.Node {
+	r := visitFn(func(node ast.Node, replaceWith func(ast.Node)) {
+		switch id := node.(type) {
+		case *ast.Ident:
+			if id.Name == named {
+				replaceWith(with)
+			}
+		}
+	})
+	return WalkReplace(r, src)
+}
+
+func replaceLit(src ast.Node, from, to string) ast.Node {
+	r := visitFn(func(node ast.Node, replaceWith func(ast.Node)) {
+		switch lit := node.(type) {
+		case *ast.BasicLit:
+			if lit.Value == from {
+				replaceWith(&ast.BasicLit{Value: to})
+			}
+		}
+	})
+	return WalkReplace(r, src)
+}
+
+func fullAST() *ast.File {
+	full, err := ASTTemplates.Open("full.go")
+	if err != nil {
+		panic(err)
+	}
+	f, err := parser.ParseFile(token.NewFileSet(), "templates/full.go", full, parser.DeclarationErrors)
+	if err != nil {
+		panic(err)
+	}
+	return f
+}
+
+func fetchImports() []*ast.ImportSpec {
+	return fullAST().Imports
+}
+
+func fetchFuncDecl(name string) *ast.FuncDecl {
+	f := fullAST()
+	for _, decl := range f.Decls {
+		if f, ok := decl.(*ast.FuncDecl); ok && f.Name.Name == name {
+			return f
+		}
+	}
+	panic(fmt.Errorf("No function called %q in 'templates/full.go'", name))
+}
+
+func id(name string) *ast.Ident {
+	return ast.NewIdent(name)
+}
+
+func sel(ids ...*ast.Ident) ast.Expr {
+	switch len(ids) {
+	default:
+		return &ast.SelectorExpr{
+			X:   sel(ids[:len(ids)-1]...),
+			Sel: ids[len(ids)-1],
+		}
+	case 1:
+		return ids[0]
+	case 0:
+		panic("zero ids to sel()")
+	}
+}
+
+func typeField(t ast.Expr) *ast.Field {
+	return &ast.Field{Type: t}
+}
+
+func field(n *ast.Ident, t ast.Expr) *ast.Field {
+	return &ast.Field{
+		Names: []*ast.Ident{n},
+		Type:  t,
+	}
+}
+
+func fieldList(list ...*ast.Field) *ast.FieldList {
+	return &ast.FieldList{List: list}
+}
+
+func mappedFieldList(fn func(arg) *ast.Field, args ...arg) *ast.FieldList {
+	fl := &ast.FieldList{List: []*ast.Field{}}
+	for _, a := range args {
+		fl.List = append(fl.List, fn(a))
+	}
+	return fl
+}
+
+func blockStmt(stmts ...ast.Stmt) *ast.BlockStmt {
+	return &ast.BlockStmt{
+		List: stmts,
+	}
+}
+
+func structDecl(name *ast.Ident, fields *ast.FieldList) ast.Decl {
+	return typeDecl(&ast.TypeSpec{
+		Name: name,
+		Type: &ast.StructType{
+			Fields: fields,
+		},
+	})
+}
+
+func typeDecl(ts *ast.TypeSpec) ast.Decl {
+	return &ast.GenDecl{
+		Tok:   token.TYPE,
+		Specs: []ast.Spec{ts},
+	}
+}
+
+func pasteStmts(body *ast.BlockStmt, idx int, stmts []ast.Stmt) {
+	list := body.List
+	prefix := list[:idx]
+	suffix := make([]ast.Stmt, len(list)-idx-1)
+	copy(suffix, list[idx+1:])
+
+	body.List = append(append(prefix, stmts...), suffix...)
+}
+
+func importFor(is *ast.ImportSpec) *ast.GenDecl {
+	return &ast.GenDecl{Tok: token.IMPORT, Specs: []ast.Spec{is}}
+}
+
+func importSpec(path string) *ast.ImportSpec {
+	return &ast.ImportSpec{Path: &ast.BasicLit{Kind: token.STRING, Value: `"` + path + `"`}}
+}

cmd/kitgen/ast_templates.go

@@ -0,0 +1,11 @@
+// This file was automatically generated based on the contents of *.tmpl
+// If you need to update this file, change the contents of those files
+// (or add new ones) and run 'go generate'
+
+package main
+
+import "golang.org/x/tools/godoc/vfs/mapfs"
+
+var ASTTemplates = mapfs.New(map[string]string{
+	`full.go`: "package foo\n\nimport (\n	\"context\"\n	\"encoding/json\"\n	\"errors\"\n	\"net/http\"\n\n	\"github.com/go-kit/kit/endpoint\"\n	httptransport \"github.com/go-kit/kit/transport/http\"\n)\n\ntype ExampleService struct {\n}\n\ntype ExampleRequest struct {\n	I int\n	S string\n}\ntype ExampleResponse struct {\n	S   string\n	Err error\n}\n\ntype Endpoints struct {\n	ExampleEndpoint endpoint.Endpoint\n}\n\nfunc (f ExampleService) ExampleEndpoint(ctx context.Context, i int, s string) (string, error) {\n	panic(errors.New(\"not implemented\"))\n}\n\nfunc makeExampleEndpoint(f ExampleService) endpoint.Endpoint {\n	return func(ctx context.Context, request interface{}) (interface{}, error) {\n		req := request.(ExampleRequest)\n		s, err := f.ExampleEndpoint(ctx, req.I, req.S)\n		return ExampleResponse{S: s, Err: err}, nil\n	}\n}\n\nfunc inlineHandlerBuilder(m *http.ServeMux, endpoints Endpoints) {\n	m.Handle(\"/bar\", httptransport.NewServer(endpoints.ExampleEndpoint, DecodeExampleRequest, EncodeExampleResponse))\n}\n\nfunc NewHTTPHandler(endpoints Endpoints) http.Handler {\n	m := http.NewServeMux()\n	inlineHandlerBuilder(m, endpoints)\n	return m\n}\n\nfunc DecodeExampleRequest(_ context.Context, r *http.Request) (interface{}, error) {\n	var req ExampleRequest\n	err := json.NewDecoder(r.Body).Decode(&req)\n	return req, err\n}\n\nfunc EncodeExampleResponse(_ context.Context, w http.ResponseWriter, response interface{}) error {\n	w.Header().Set(\"Content-Type\", \"application/json; charset=utf-8\")\n	return json.NewEncoder(w).Encode(response)\n}\n",
+})

cmd/kitgen/deflayout.go

@@ -0,0 +1,63 @@
+package main
+
+import "path/filepath"
+
+type deflayout struct {
+	targetDir string
+}
+
+func (l deflayout) packagePath(sub string) string {
+	return filepath.Join(l.targetDir, sub)
+}
+
+func (l deflayout) transformAST(ctx *sourceContext) (files, error) {
+	out := make(outputTree)
+
+	endpoints := out.addFile("endpoints/endpoints.go", "endpoints")
+	http := out.addFile("http/http.go", "http")
+	service := out.addFile("service/service.go", "service")
+
+	addImports(endpoints, ctx)
+	addImports(http, ctx)
+	addImports(service, ctx)
+
+	for _, typ := range ctx.types {
+		addType(service, typ)
+	}
+
+	for _, iface := range ctx.interfaces { //only one...
+		addStubStruct(service, iface)
+
+		for _, meth := range iface.methods {
+			addMethod(service, iface, meth)
+			addRequestStruct(endpoints, meth)
+			addResponseStruct(endpoints, meth)
+			addEndpointMaker(endpoints, iface, meth)
+		}
+
+		addEndpointsStruct(endpoints, iface)
+		addHTTPHandler(http, iface)
+
+		for _, meth := range iface.methods {
+			addDecoder(http, meth)
+			addEncoder(http, meth)
+		}
+
+		for name := range out {
+			out[name] = selectify(out[name], "service", iface.stubName().Name, l.packagePath("service"))
+			for _, meth := range iface.methods {
+				out[name] = selectify(out[name], "endpoints", meth.requestStructName().Name, l.packagePath("endpoints"))
+			}
+		}
+	}
+
+	for name := range out {
+		out[name] = selectify(out[name], "endpoints", "Endpoints", l.packagePath("endpoints"))
+
+		for _, typ := range ctx.types {
+			out[name] = selectify(out[name], "service", typ.Name.Name, l.packagePath("service"))
+		}
+	}
+
+	return formatNodes(out)
+}

cmd/kitgen/flatlayout.go

@@ -0,0 +1,39 @@
+package main
+
+import "go/ast"
+
+type flat struct{}
+
+func (f flat) transformAST(ctx *sourceContext) (files, error) {
+	root := &ast.File{
+		Name:  ctx.pkg,
+		Decls: []ast.Decl{},
+	}
+
+	addImports(root, ctx)
+
+	for _, typ := range ctx.types {
+		addType(root, typ)
+	}
+
+	for _, iface := range ctx.interfaces { //only one...
+		addStubStruct(root, iface)
+
+		for _, meth := range iface.methods {
+			addMethod(root, iface, meth)
+			addRequestStruct(root, meth)
+			addResponseStruct(root, meth)
+			addEndpointMaker(root, iface, meth)
+		}
+
+		addEndpointsStruct(root, iface)
+		addHTTPHandler(root, iface)
+
+		for _, meth := range iface.methods {
+			addDecoder(root, meth)
+			addEncoder(root, meth)
+		}
+	}
+
+	return formatNodes(outputTree{"gokit.go": root})
+}