diff --git a/contract/context.go b/contract/context.go deleted file mode 100644 index a39db149..00000000 --- a/contract/context.go +++ /dev/null @@ -1,33 +0,0 @@ -package contract - -import "fmt" - -type contextKey string - -const ( - IpKey contextKey = "ip" // IP address - TenantKey contextKey = "tenant" // Tenant - TransportKey contextKey = "transport" // Transport, such as HTTP - RequestUrlKey contextKey = "requestUrl" // Request url -) - -// Tenant is interface representing a user or a consumer. -type Tenant interface { - // KV contains key values about this tenant. - KV() map[string]interface{} - // String should uniquely represent this user. It should be human friendly. - String() string -} - -// MapTenant is an demo Tenant implementation. Useful for testing. -type MapTenant map[string]interface{} - -// KV contains key values about this tenant. -func (d MapTenant) KV() map[string]interface{} { - return d -} - -// String should uniquely represent this user. It should be human friendly. -func (d MapTenant) String() string { - return fmt.Sprintf("%+v", map[string]interface{}(d)) -} diff --git a/contract/context_test.go b/contract/context_test.go deleted file mode 100644 index 06154950..00000000 --- a/contract/context_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package contract - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestContext(t *testing.T) { - tenant := MapTenant{} - assert.Equal(t, map[string]interface{}{}, tenant.KV()) - assert.Equal(t, "map[]", tenant.String()) -} diff --git a/ctxmeta/ctxmeta.go b/ctxmeta/ctxmeta.go new file mode 100644 index 00000000..61b91188 --- /dev/null +++ b/ctxmeta/ctxmeta.go @@ -0,0 +1,228 @@ +// Package ctxmeta provides a helper type for request-scoped metadata. This +// package is inspired by https://github.com/peterbourgon/ctxdata. (License: +// https://github.com/peterbourgon/ctxdata/blob/master/LICENSE) The original +// package doesn't support collecting different groups of contextual data. This +// forked version allows it. +package ctxmeta + +import ( + "context" + "errors" + "fmt" + "reflect" + + "github.com/DoNewsCode/core/contract" +) + +var _ contract.ConfigUnmarshaler = (*Baggage)(nil) + +// KeyVal combines a string key with its abstract value into a single tuple. +// It's used internally, and as a return type for Slice. +type KeyVal struct { + Key string + Val interface{} +} + +// ErrNoBaggage is returned by accessor methods when they're called on a nil +// pointer receiver. This typically means From was called on a context that +// didn't have Baggage injected into it previously via Inject. +var ErrNoBaggage = errors.New("no baggage in context") + +// ErrIncompatibleType is returned by Unmarshal if the value associated with a key +// isn't assignable to the provided target. +var ErrIncompatibleType = errors.New("incompatible type") + +// ErrNotFound is returned by Get or other accessors when the key isn't present. +var ErrNotFound = errors.New("key not found") + +// Baggage is an opaque type that can be injected into a context at e.g. the start +// of a request, updated with metadata over the course of the request, and then +// queried at the end of the request. +// +// When a new request arrives in your program, HTTP server, etc., use the New +// constructor with the incoming request's context to construct a new, empty +// Baggage. Use the returned context for all further operations on that request. +// Use the From helper function to retrieve a previously-injected Baggage from a +// context, and set or get metadata. At the end of the request, all metadata +// collected will be available from any point in the callstack. +type Baggage struct { + c chan []KeyVal +} + +// Unmarshal get the value at given path, and store it into the target variable. Target must +// be a pointer to an assignable type. Unmarshal will return ErrNotFound if the key +// is not found, and ErrIncompatibleType if the found value is not assignable to +// target. Unmarshal also implements contract.ConfigUnmarshaler. +func (b *Baggage) Unmarshal(path string, target interface{}) error { + val, err := b.Get(path) + if err != nil { + return err + } + + v := reflect.ValueOf(target) + t := v.Type() + if t.Kind() != reflect.Ptr || v.IsNil() { + return fmt.Errorf("target must be a non-nil pointer") + } + + targetType := t.Elem() + if !reflect.TypeOf(val).AssignableTo(targetType) { + return ErrIncompatibleType + } + + v.Elem().Set(reflect.ValueOf(val)) + return nil +} + +// Get the value associated with key, or return ErrNotFound. If this method is +// called on a nil Baggage pointer, it returns ErrNoBaggage. +func (b *Baggage) Get(key string) (value interface{}, err error) { + if b == nil { + return nil, ErrNoBaggage + } + + s := <-b.c + defer func() { b.c <- s }() + + for _, kv := range s { + if kv.Key == key { + return kv.Val, nil + } + } + + return nil, ErrNotFound +} + +// Set key to value. If key already exists, it will be overwritten. If this method +// is called on a nil Baggage pointer, it returns ErrNoBaggage. +func (b *Baggage) Set(key string, value interface{}) (err error) { + if b == nil { + return ErrNoBaggage + } + + s := <-b.c + defer func() { b.c <- s }() + + for i := range s { + if s[i].Key == key { + s[i].Val = value + s = append(s[:i], append(s[i+1:], s[i])...) + return nil + } + } + + s = append(s, KeyVal{key, value}) + + return nil +} + +// Update key to the value returned from the callback. If key doesn't exist, it +// returns ErrNotFound. If this method is called on a nil Baggage pointer, it +// returns ErrNoBaggage. +func (b *Baggage) Update(key string, callback func(value interface{}) interface{}) (err error) { + if b == nil { + return ErrNoBaggage + } + + s := <-b.c + defer func() { b.c <- s }() + + for i := range s { + if s[i].Key == key { + s[i].Val = callback(s[i].Val) + return nil + } + } + + return ErrNotFound +} + +// Delete key from baggage. If key doesn't exist, it returns ErrNotFound. If the +// MetadataSet is not associated with an initialized baggage, it returns +// ErrNoBaggage. +func (b *Baggage) Delete(key interface{}) (err error) { + if b == nil { + return ErrNoBaggage + } + s := <-b.c + defer func() { b.c <- s }() + + for i := range s { + if s[i].Key == key { + s = append(s[:i], s[i+1:]...) + return nil + } + } + + return ErrNotFound +} + +// Slice returns a slice of key/value pairs in the order in which they were set. +func (b *Baggage) Slice() []KeyVal { + s := <-b.c + defer func() { b.c <- s }() + + r := make([]KeyVal, len(s)) + copy(r, s) + return r +} + +// Map returns a map of key to value. +func (b *Baggage) Map() map[string]interface{} { + s := <-b.c + defer func() { b.c <- s }() + + mp := make(map[string]interface{}, len(s)) + for _, kv := range s { + mp[kv.Key] = kv.Val + } + return mp +} + +// MetadataSet is a group key to the contextual data stored the context. +// The data stored with different MetadataSet instances are not shared. +type MetadataSet struct { + key *struct{} +} + +// DefaultMetadata contains the default key for Baggage in the context. Use this if there +// is no need to categorize metadata, ie. put all data in one baggage. +var DefaultMetadata = MetadataSet{key: &struct{}{}} + +// New constructs a new set of metadata. This metadata can be used to retrieve a group of contextual data. +// The data stored with different MetadataSet instances are not shared. +func New() *MetadataSet { + return &MetadataSet{key: &struct{}{}} +} + +// Inject constructs a Baggage object and injects it into the provided context +// under the context key determined the metadata instance. Use the returned +// context for all further operations. The returned Baggage can be queried at any +// point for metadata collected over the life of the context. +func (m *MetadataSet) Inject(ctx context.Context) (*Baggage, context.Context) { + c := make(chan []KeyVal, 1) + c <- make([]KeyVal, 0, 32) + d := &Baggage{c: c} + return d, context.WithValue(ctx, m.key, d) +} + +// GetBaggage returns the Baggage stored in the context. +func (m *MetadataSet) GetBaggage(ctx context.Context) *Baggage { + if val, ok := ctx.Value(m.key).(*Baggage); ok { + return val + } + return nil +} + +// Inject constructs a Baggage object and injects it into the provided context +// under the default context key. Use the returned context for all further +// operations. The returned Data can be queried at any point for metadata +// collected over the life of the context. +func Inject(ctx context.Context) (*Baggage, context.Context) { + return DefaultMetadata.Inject(ctx) +} + +// GetBaggage returns the default Baggage stored in the context. +func GetBaggage(ctx context.Context) *Baggage { + return DefaultMetadata.GetBaggage(ctx) +} diff --git a/ctxmeta/ctxmeta_test.go b/ctxmeta/ctxmeta_test.go new file mode 100644 index 00000000..3a61f50c --- /dev/null +++ b/ctxmeta/ctxmeta_test.go @@ -0,0 +1,157 @@ +package ctxmeta + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestContextMeta_crud(t *testing.T) { + t.Parallel() + + ctx := context.Background() + metadata := New() + baggage, _ := metadata.Inject(ctx) + + baggage.Set("foo", "bar") + result, err := baggage.Get("foo") + assert.NoError(t, err) + assert.Equal(t, "bar", result) + + result = baggage.Slice() + assert.ElementsMatch(t, []KeyVal{{Key: "foo", Val: "bar"}}, result) + + resultMap := baggage.Map() + assert.Equal(t, "bar", resultMap["foo"]) + + var s string + baggage.Unmarshal("foo", &s) + assert.Equal(t, "bar", s) + + baggage.Update("foo", func(value interface{}) interface{} { return "baz" }) + result, err = baggage.Get("foo") + assert.NoError(t, err) + assert.Equal(t, "baz", result) + + baggage.Delete("foo") + _, err = baggage.Get("foo") + assert.ErrorIs(t, err, ErrNotFound) + +} + +func TestContextMeta_ErrNoBaggae(t *testing.T) { + t.Parallel() + + ctx := context.Background() + metadata := New() + baggage := metadata.GetBaggage(ctx) + + err := baggage.Set("foo", "bar") + assert.ErrorIs(t, err, ErrNoBaggage) + + _, err = baggage.Get("foo") + assert.ErrorIs(t, err, ErrNoBaggage) + + var s string + err = baggage.Unmarshal("foo", &s) + assert.ErrorIs(t, err, ErrNoBaggage) + + err = baggage.Update("foo", func(value interface{}) interface{} { return "baz" }) + assert.ErrorIs(t, err, ErrNoBaggage) + + err = baggage.Delete("foo") + assert.ErrorIs(t, err, ErrNoBaggage) +} + +func TestContextMeta_ErrNotFound(t *testing.T) { + t.Parallel() + + ctx := context.Background() + metadata := New() + baggage, _ := metadata.Inject(ctx) + + _, err := baggage.Get("foo") + assert.ErrorIs(t, err, ErrNotFound) + + var s string + err = baggage.Unmarshal("foo", &s) + assert.ErrorIs(t, err, ErrNotFound) + + err = baggage.Update("foo", func(value interface{}) interface{} { return "baz" }) + assert.ErrorIs(t, err, ErrNotFound) + + err = baggage.Delete("foo") + assert.ErrorIs(t, err, ErrNotFound) +} + +func TestContextMeta_ErrIncompatibleType(t *testing.T) { + t.Parallel() + + ctx := context.Background() + metadata := New() + baggage, _ := metadata.Inject(ctx) + + baggage.Set("foo", "bar") + + var s int + err := baggage.Unmarshal("foo", &s) + assert.ErrorIs(t, err, ErrIncompatibleType) +} + +func TestContextMeta_parallel(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + meta *MetadataSet + key string + value string + }{ + { + "first", + New(), + "foo", + "bar", + }, + { + "second", + New(), + "foo", + "baz", + }, + { + "default", + &DefaultMetadata, + "foo", + "qux", + }, + } + ctx := context.Background() + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + b, ctx := c.meta.Inject(ctx) + b.Set(c.key, c.value) + value, err := c.meta.GetBaggage(ctx).Get(c.key) + assert.NoError(t, err) + assert.Equal(t, c.value, value) + }) + } +} + +func TestMetadata_global(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baggage1, ctx := Inject(ctx) + baggage1.Set("hello", "world") + + baggage2 := GetBaggage(ctx) + world, _ := baggage2.Get("hello") + assert.Equal(t, "world", world) + + baggage3 := DefaultMetadata.GetBaggage(ctx) + world, _ = baggage3.Get("hello") + assert.Equal(t, "world", world) +} diff --git a/ctxmeta/example_middleware_test.go b/ctxmeta/example_middleware_test.go new file mode 100644 index 00000000..4861ff1d --- /dev/null +++ b/ctxmeta/example_middleware_test.go @@ -0,0 +1,57 @@ +package ctxmeta_test + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + + "github.com/DoNewsCode/core/ctxmeta" +) + +type Server struct{} + +func NewServer() *Server { + return &Server{} +} + +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + bag := ctxmeta.GetBaggage(r.Context()) + bag.Set("method", r.Method) + bag.Set("path", r.URL.Path) + bag.Set("content_length", r.ContentLength) + fmt.Fprintln(w, "OK") +} + +type Middleware struct { + next http.Handler +} + +func NewMiddleware(next http.Handler) *Middleware { + return &Middleware{next: next} +} + +func (mw *Middleware) ServeHTTP(w http.ResponseWriter, r *http.Request) { + bag, ctx := ctxmeta.Inject(r.Context()) + + defer func() { + for _, kv := range bag.Slice() { + fmt.Printf("%s: %v\n", kv.Key, kv.Val) + } + }() + + mw.next.ServeHTTP(w, r.WithContext(ctx)) +} + +func Example_middleware() { + server := NewServer() + middleware := NewMiddleware(server) + testserver := httptest.NewServer(middleware) + defer testserver.Close() + http.Post(testserver.URL+"/path", "text/plain; charset=utf-8", strings.NewReader("hello world")) + + // Output: + // method: POST + // path: /path + // content_length: 11 +} diff --git a/ctxmeta/example_test.go b/ctxmeta/example_test.go new file mode 100644 index 00000000..abacf93c --- /dev/null +++ b/ctxmeta/example_test.go @@ -0,0 +1,29 @@ +package ctxmeta_test + +import ( + "context" + "fmt" + "net/http" + + "github.com/DoNewsCode/core/ctxmeta" +) + +// This example demonstrates how to use Unmarshal to retrieve metadata into an +// arbitrary type. +func ExampleBaggage_Unmarshal() { + type DomainError struct { + Code int + Reason string + } + + bag, _ := ctxmeta.Inject(context.Background()) + derr := DomainError{Code: http.StatusTeapot, Reason: "Earl Gray exception"} + bag.Set("err", derr) + + if target := (DomainError{}); bag.Unmarshal("err", &target) == nil { + fmt.Printf("DomainError Code=%d Reason=%q\n", target.Code, target.Reason) + } + + // Output: + // DomainError Code=418 Reason="Earl Gray exception" +} diff --git a/go.mod b/go.mod index 584e15b2..5e7890b1 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,10 @@ module github.com/DoNewsCode/core -go 1.14 +go 1.15 require ( github.com/ClickHouse/clickhouse-go v1.4.5 // indirect github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect - github.com/Reasno/ifilter v0.1.2 github.com/aws/aws-sdk-go v1.38.68 github.com/fsnotify/fsnotify v1.4.9 github.com/gabriel-vasile/mimetype v1.1.2 diff --git a/go.sum b/go.sum index c8839d1f..27dfa7ea 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,6 @@ github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030I github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= -github.com/Reasno/ifilter v0.1.2 h1:k/rJAN7zb9sGWCdZA7Ur5EaU3FZkmwsxGDXkqwnWoP0= -github.com/Reasno/ifilter v0.1.2/go.mod h1:awLxvLyOE4W2tJER5mGl/Jw4+pGjUBIn9SmAIY6gqYc= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= diff --git a/logging/example_test.go b/logging/example_test.go index a90601bd..bfccb340 100644 --- a/logging/example_test.go +++ b/logging/example_test.go @@ -3,7 +3,7 @@ package logging_test import ( "context" - "github.com/DoNewsCode/core/contract" + "github.com/DoNewsCode/core/ctxmeta" "github.com/DoNewsCode/core/logging" "github.com/go-kit/kit/log/level" ) @@ -31,9 +31,10 @@ func ExampleWithLevel() { } func ExampleWithContext() { - ctx := context.WithValue(context.Background(), contract.IpKey, "127.0.0.1") - ctx = context.WithValue(ctx, contract.TransportKey, "http") - ctx = context.WithValue(ctx, contract.RequestUrlKey, "/example") + bag, ctx := ctxmeta.Inject(context.Background()) + bag.Set("clientIp", "127.0.0.1") + bag.Set("requestUrl", "/example") + bag.Set("transport", "http") logger := logging.NewLogger("json") ctxLogger := logging.WithContext(logger, ctx) ctxLogger.Log("foo", "bar") diff --git a/logging/log.go b/logging/log.go index e140dbb8..ffae5500 100644 --- a/logging/log.go +++ b/logging/log.go @@ -3,7 +3,7 @@ Package logging provides a kitlog compatible logger. This package is mostly a thin wrapper around kitlog (http://github.com/go-kit/kit/log). kitlog provides a minimalist, contextual, -fully composable logger. However it is too unopinionated, hence requiring some +fully composable logger. However, it is too unopinionated, hence requiring some efforts and coordination to set up a good practise. Integration @@ -23,6 +23,7 @@ import ( "os" "strings" + "github.com/DoNewsCode/core/ctxmeta" "github.com/opentracing/opentracing-go" "github.com/DoNewsCode/core/contract" @@ -97,34 +98,38 @@ func LevelFilter(levelCfg string) level.Option { type spanLogger struct { span opentracing.Span base log.Logger + kvs []interface{} } func (s spanLogger) Log(keyvals ...interface{}) error { - s.span.LogKV(keyvals...) - return s.base.Log(keyvals...) + s.kvs = append(s.kvs, keyvals...) + s.span.LogKV(s.kvs...) + return s.base.Log(s.kvs...) } -// WithContext decorates the log.Logger with information form context. If there is a opentracing span +// WithContext decorates the log.Logger with information form context. If there is an opentracing span // in the context, the span will receive the logger output as well. func WithContext(logger log.Logger, ctx context.Context) log.Logger { + var args []interface{} + + bag := ctxmeta.GetBaggage(ctx) + for _, kv := range bag.Slice() { + args = append(args, kv.Key, kv.Val) + } + span := opentracing.SpanFromContext(ctx) if span == nil { return withContext(logger, ctx) } - return spanLogger{span: span, base: withContext(logger, ctx)} + return spanLogger{span: span, base: logger, kvs: args} } func withContext(logger log.Logger, ctx context.Context) log.Logger { - transport, _ := ctx.Value(contract.TransportKey).(string) - requestUrl, _ := ctx.Value(contract.RequestUrlKey).(string) - ip, _ := ctx.Value(contract.IpKey).(string) - tenant, ok := ctx.Value(contract.TenantKey).(contract.Tenant) - if !ok { - tenant = contract.MapTenant{} - } - args := []interface{}{"transport", transport, "requestUrl", requestUrl, "clientIp", ip} - for k, v := range tenant.KV() { - args = append(args, k, v) + var args []interface{} + + bag := ctxmeta.GetBaggage(ctx) + for _, kv := range bag.Slice() { + args = append(args, kv.Key, kv.Val) } return log.With( @@ -196,7 +201,7 @@ func (l levelLogger) Err(args ...interface{}) { // WithLevel decorates the logger and returns a contract.LevelLogger. // // Note: Don't inject contract.LevelLogger to dependency consumers directly as -// this will weakens the powerful abstraction of log.Logger. Only inject +// this will weaken the powerful abstraction of log.Logger. Only inject // log.Logger, and converts log.Logger to contract.LevelLogger within the // boundary of dependency consumer if desired. func WithLevel(logger log.Logger) LevelLogger { diff --git a/logging/log_test.go b/logging/log_test.go index b132367f..870827cf 100644 --- a/logging/log_test.go +++ b/logging/log_test.go @@ -2,8 +2,10 @@ package logging import ( "bytes" + "context" "testing" + "github.com/DoNewsCode/core/ctxmeta" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/stretchr/testify/assert" @@ -37,3 +39,15 @@ func TestLevelFilter(t *testing.T) { func TestNewLogger(t *testing.T) { _ = NewLogger("logfmt") } + +func TestWithContext(t *testing.T) { + ctx := context.Background() + bag, ctx := ctxmeta.Inject(ctx) + bag.Set("foo", "bar") + + var buf bytes.Buffer + l := log.NewLogfmtLogger(&buf) + ll := WithContext(l, ctx) + ll.Log("baz", "qux") + assert.Contains(t, buf.String(), "foo=bar baz=qux") +} diff --git a/srvgrpc/trace.go b/srvgrpc/trace.go new file mode 100644 index 00000000..3eb25feb --- /dev/null +++ b/srvgrpc/trace.go @@ -0,0 +1,8 @@ +package srvgrpc + +import "github.com/opentracing-contrib/go-grpc" + +// Trace is an alias of otgrpc.OpenTracingServerInterceptor. It is recommended to use the trace +// implementation in github.com/opentracing-contrib/go-grpc. This alias serves +// as a pointer to it. +var Trace = otgrpc.OpenTracingServerInterceptor diff --git a/srvhttp/metrics.go b/srvhttp/metrics.go index 4e77050f..0bb3218e 100644 --- a/srvhttp/metrics.go +++ b/srvhttp/metrics.go @@ -18,7 +18,7 @@ func (m MetricsModule) ProvideHTTP(router *mux.Router) { router.PathPrefix("/metrics").Handler(promhttp.Handler()) } -// Metrics is a unary interceptor for standard library http package. It records the request duration in a histogram. +// Metrics is a middleware for standard library http package. It records the request duration in a histogram. func Metrics(metrics *RequestDurationSeconds) func(handler http.Handler) http.Handler { return func(handler http.Handler) http.Handler { return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { diff --git a/srvhttp/trace.go b/srvhttp/trace.go new file mode 100644 index 00000000..ad308d33 --- /dev/null +++ b/srvhttp/trace.go @@ -0,0 +1,8 @@ +package srvhttp + +import "github.com/opentracing-contrib/go-stdlib/nethttp" + +// Trace is an alias of nethttp.Middleware. It is recommended to use the trace +// implementation in github.com/opentracing-contrib/go-stdlib. This alias serves +// as a pointer to it. +var Trace = nethttp.Middleware