Skip to content

Commit

Permalink
rewritten dedupe algor and re-enabled it by default
Browse files Browse the repository at this point in the history
  • Loading branch information
@hedzr
hedzr committed Nov 17, 2024
1 parent 4807e1f commit 28c7321
Show file tree
Hide file tree
Showing 7 changed files with 228 additions and 213 deletions.
21 changes: 16 additions & 5 deletions bench/slog_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,19 @@ func Test2(t *testing.T) {
func TestUseJSON(t *testing.T) {
defer slogg.SaveFlagsAndMod(slogg.Lcaller)()

logger := slogg.New().SetWriter(os.Stdout).SetLevel(slogg.DebugLevel).SetJSONMode()
msg, attrs := getMessage(0), fakeLoggArgs()
logger.Info(msg, attrs...)
logger := slogg.New(slogg.With("int", 3)).
SetWriter(os.Stdout).
SetLevel(slogg.DebugLevel).
SetJSONMode()

ctx, msg, attrs := context.Background(), getMessage(0), fakeLoggArgs()
logger.InfoContext(ctx, msg, attrs...)

logger.SetJSONMode(false)
logger.Info(msg, attrs...)
logger.InfoContext(ctx, msg, attrs...)

logger.SetColorMode()
logger.Info(msg, attrs...)
logger.InfoContext(ctx, msg, attrs...)
}

func Test3(t *testing.T) {
Expand All @@ -40,3 +44,10 @@ func Test3(t *testing.T) {
SetWriter(os.Stdout).SetLevel(slogg.InfoLevel)
logger.InfoContext(ctx, msg)
}

func Test4(t *testing.T) {
ctx, msg, attrs := context.Background(), getMessage(0), fakeLoggArgs()
logger := newLoggTextMode().Set(attrs...).
SetWriter(os.Stdout).SetLevel(slogg.InfoLevel)
logger.InfoContext(ctx, msg)
}
63 changes: 61 additions & 2 deletions slog/attr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@ package slog

import (
"fmt"
"slices"
"time"

"github.com/hedzr/logg/slog/internal/strings"
)

func NewAttr(key string, val any) Attr { return &kvp{key, val} } // create an attribute
func NewAttrs(args ...any) Attrs { return buildUniqueAttrs(nil, args...) } //nolint:revive,lll // freeform args here, used by WithAttrs1. See also New or With for the usage.
func NewAttrs(args ...any) Attrs { return buildUniqueAttrs(args...) } //nolint:revive,lll // freeform args here, used by WithAttrs1. See also New or With for the usage.
func NewGroupedAttr(key string, as ...Attr) Attr { return &gkvp{key, as} } // similar with Group
func NewGroupedAttrEasy(key string, as ...any) Attr { return &gkvp{key: key, items: buildAttrs(as...)} } // synonym to Group

Expand Down Expand Up @@ -97,11 +98,70 @@ func (s Attrs) SerializeValueTo(pc *PrintCtx) {
_ = serializeAttrs(pc, s)
}

func dedupeSlice[S ~[]E, E any](x S, cmp func(a, b E) bool) S {
if len(x) == 0 {
return nil
}

j := 0
for i := 1; i < len(x); i++ {
if cmp(x[i], x[j]) {
x[j] = x[i]
continue
}
j++
// preserve the original data
// in[i], in[j] = in[j], in[i]
// only set what is required
x[j] = x[i]
}
result := x[:j+1]
return result
}

// serializeAttrs returns an error object if it's found in the given Attrs.
// The caller can do something with the object, For instance, printImpl
// will dump the error's stack trace if necessary.
func serializeAttrs(pc *PrintCtx, kvps Attrs) (err error) { //nolint:revive
prefix := pc.prefix
inGroupedMode := pc.inGroupedMode

if pc.dedupeAttrs {
slices.SortFunc(kvps, func(a, b Attr) int {
if a == nil {
if b == nil {
return 0
}
return -1
}
if b == nil {
return 1
}

k1, k2 := a.Key(), b.Key()
if k1 < k2 {
return -1
}
if k1 == k2 {
return 0
}
return 1
})

// sort.Slice(kvps, func(i, j int) bool {
// return kvps[i].Key() < kvps[j].Key()
// })
kvps = dedupeSlice(kvps, func(a, b Attr) bool {
if a == nil || b == nil {
if b == a {
return true
}
return false
}
return a.Key() == b.Key()
})
}

for _, v := range kvps {
if v == nil {
continue
Expand All @@ -114,7 +174,6 @@ func serializeAttrs(pc *PrintCtx, kvps Attrs) (err error) { //nolint:revive
ct.echoColorAndBg(pc, pc.clr, pc.bg)
}

inGroupedMode := pc.inGroupedMode
if !inGroupedMode {
_, inGroupedMode = v.(groupedValue)
}
Expand Down
175 changes: 57 additions & 118 deletions slog/entry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import (

"github.com/hedzr/is"
"github.com/hedzr/is/stringtool"
errorsv3 "gopkg.in/hedzr/errors.v3"
)

func newentry(parent *Entry, args ...any) *Entry {
Expand Down Expand Up @@ -56,7 +55,7 @@ func newentry(parent *Entry, args ...any) *Entry {

if l := len(todo); l > 0 {
s.attrs = make(Attrs, l)
argsToAttrs(&s.attrs, nil, todo...)
argsToAttrs(&s.attrs, todo...)
}

if namedAlways || parent != nil { // if not a detached logger (parent == nil means detached)
Expand Down Expand Up @@ -490,7 +489,7 @@ func (s *Entry) Set(args ...any) *Entry { // key1,val1,key2,val2,.... Of course,
// s.attrs = append(s.attrs, (*kvp)(nil))
// }
// }
argsToAttrs(&s.attrs, nil, args...)
argsToAttrs(&s.attrs, args...)
return s
}

Expand Down Expand Up @@ -985,29 +984,15 @@ func newFixedAttrs() (kvps Attrs) {
return kvps
}

func (s *Entry) parseArgs(ctx context.Context, kvps *Attrs, roughSize int, lvl Level, stackFrame uintptr, msg string, args ...any) {
// var roughSize = 4 + len(s.attrs) + len(args)
// // var key string
// // var keys = make(map[string]bool, roughSize)

var keys map[string]bool

// kvps = s.leadingTags(roughSize, lvl, stackFrame, msg)

func (s *Entry) collectArgs(ctx context.Context, kvps *Attrs, roughSize int, lvl Level, args ...any) {
if s.ctxKeysWanted() {
s.fromCtx(ctx, kvps)
}
if len(s.attrs) > 0 {
if dedupKeys && keys == nil { //nolint:cond
keys = make(map[string]bool, roughSize)
}
s.walkParentAttrs(ctx, lvl, s, keys, kvps)
s.walkParentAttrs(ctx, lvl, s, kvps)
}
if len(args) > 0 {
if dedupKeys && keys == nil {
keys = make(map[string]bool, roughSize)
}
argsToAttrs(kvps, keys, args...)
argsToAttrs(kvps, args...)
}

// if key != "" {
Expand All @@ -1021,15 +1006,11 @@ func (s *Entry) parseArgs(ctx context.Context, kvps *Attrs, roughSize int, lvl L
return
}

func (s *Entry) walkParentAttrs(ctx context.Context, lvl Level, e *Entry, keysKnown map[string]bool, kvps *Attrs) {
func (s *Entry) walkParentAttrs(ctx context.Context, lvl Level, e *Entry, kvps *Attrs) {
if e == nil {
return
}

// if keysKnown == nil {
// return e.attrs
// }

// try appending unique attributes and walk all parents

var roughlen = len(e.attrs)
Expand All @@ -1041,58 +1022,29 @@ func (s *Entry) walkParentAttrs(ctx context.Context, lvl Level, e *Entry, keysKn
roughlen = 8
}

// kvps = make([]Attr, 0, roughlen)

if IsAnyBitsSet(LattrsR) {
// lookup parents
if p := e.owner; p != nil {
s.walkParentAttrs(ctx, lvl, p, keysKnown, kvps)
s.walkParentAttrs(ctx, lvl, p, kvps)
}
}

if keysKnown != nil {
for _, attr := range e.attrs {
key := attr.Key()
if _, ok := keysKnown[key]; ok {
for ix, v := range *kvps {
if v.Key() == key {
(*kvps)[ix].SetValue(attr.Value())
}
}
} else {
*kvps = append(*kvps, attr)
keysKnown[key] = true
}
}
} else {
*kvps = append(*kvps, e.attrs...)
}
return
}

func (s *Entry) leadingTags(roughSize int, lvl Level, stackFrame uintptr, msg string) (kvps Attrs) {
// if s.useJSON || !s.useColor {
// kvps = make(Attrs, 0, roughSize+4) // pre-alloc slice spaces roughly
//
// // simulate logfmt format here, while non-JSON and non-colorful mode.
// // these key-value-pairs fit for serializing in JSON mode too.
// kvps = append(kvps,
// &kvp{timestampFieldName, time.Now()},
// &kvp{levelFieldName, lvl},
// &kvp{messageFieldName, msg})
//
// if IsAnyBitsSet(Lcaller) {
// source := getpcsource(stackFrame)
// kvps = append(kvps, source.toGroup())
// if keysKnown != nil {
// for _, attr := range e.attrs {
// key := attr.Key()
// if _, ok := keysKnown[key]; ok {
// for ix, v := range *kvps {
// if v.Key() == key {
// (*kvps)[ix].SetValue(attr.Value())
// }
// }
// } else {
// *kvps = append(*kvps, attr)
// keysKnown[key] = true
// }
// }
// }

// if roughSize > 0 {
// if roughSize > int(atomic.LoadInt32(&fixedSize)) && roughSize < maxFixedSize {
// atomic.StoreInt32(&fixedSize, int32(roughSize))
// }
// kvps = poolAttrs.Get().(Attrs)
// // kvps = make(Attrs, 0, roughSize) // pre-allocate slice spaces roughly
// } else {
*kvps = append(*kvps, e.attrs...)
// }
return
}
Expand All @@ -1102,6 +1054,11 @@ func (s *Entry) SetContextKeys(keys ...any) *Entry {
return s
}

func (s *Entry) ResetContextKeys(keys ...any) *Entry {
s.contextKeys = nil
return s
}

func (s *Entry) WithContextKeys(keys ...any) *Entry {
l := s.newChildLogger()
l.SetContextKeys(keys...)
Expand All @@ -1111,61 +1068,43 @@ func (s *Entry) WithContextKeys(keys ...any) *Entry {
func (s *Entry) ctxKeysWanted() bool { return len(s.contextKeys) > 0 }
func (s *Entry) ctxKeys() []any { return s.contextKeys }
func (s *Entry) fromCtx(ctx context.Context, kvps *Attrs) {
if dedupKeys {
mu := make(map[string]struct{})
for _, k := range s.ctxKeys() {
if v := ctx.Value(k); v != nil {
switch key := k.(type) {
case Stringer:
kk := key.String()
if _, ok := mu[kk]; !ok {
*kvps = append(*kvps, &kvp{kk, v})
mu[kk] = struct{}{}
}
case string:
if _, ok := mu[key]; !ok {
*kvps = append(*kvps, &kvp{key, v})
mu[key] = struct{}{}
}
}
}
}
} else {
for _, k := range s.ctxKeys() {
if v := ctx.Value(k); v != nil {
switch key := k.(type) {
case Stringer:
kk := key.String()
*kvps = append(*kvps, &kvp{kk, v})
case string:
*kvps = append(*kvps, &kvp{key, v})
}
// if dedupeKeys {
// // mu := make(map[string]struct{})
// for _, k := range s.ctxKeys() {
// if v := ctx.Value(k); v != nil {
// switch key := k.(type) {
// case Stringer:
// kk := key.String()
// if _, ok := keys[kk]; !ok {
// *kvps = append(*kvps, &kvp{kk, v})
// keys[kk] = true // struct{}{}
// }
// case string:
// if _, ok := keys[key]; !ok {
// *kvps = append(*kvps, &kvp{key, v})
// keys[key] = true // struct{}{}
// }
// }
// }
// }
// } else {
for _, k := range s.ctxKeys() {
if v := ctx.Value(k); v != nil {
switch key := k.(type) {
case Stringer:
kk := key.String()
*kvps = append(*kvps, &kvp{kk, v})
case string:
*kvps = append(*kvps, &kvp{key, v})
}
}
}
// }
return
}

const dedupKeys = false

// var poolHelper = Pool(
// newPrintCtx,
// func(ctx *PrintCtx, i int) []byte {
// return nil
// })
// ret = poolHelper(1)
//
// func(pc *PrintCtx, cb func(s *Entry, ctx context.Context, pc *PrintCtx) (ret []byte)) (ret []byte) {
// pc.set(s, lvl, timestamp, stackFrame, msg, kvps)
// // pc := newPrintCtx(s, lvl, timestamp, stackFrame, msg, kvps)
// return cb(ctx, pc)
// })

func (s *Entry) print(ctx context.Context, lvl Level, timestamp time.Time, stackFrame uintptr, msg string, kvps Attrs) {
pc := poolPrintCtx.Get().(*PrintCtx)
// defer func() { poolPrintCtx.Put(pc) }()

// pc := newPrintCtx(s, lvl, timestamp, stackFrame, msg, kvps)

// pc.set will truncate internal buffer and reset all states for
// this current session. So, don't worry about a reused buffer
Expand Down Expand Up @@ -1395,7 +1334,7 @@ func (s *Entry) logContext(ctx context.Context, lvl Level, stackFrame uintptr, m
// kvps = make(Attrs, 0, roughSize) // pre-allocate slice spaces roughly
// }

s.parseArgs(ctx, &kvps, roughSize, lvl, stackFrame, msg, args...)
s.collectArgs(ctx, &kvps, roughSize, lvl, args...)

now := time.Now()
s.print(ctx, lvl, now, stackFrame, msg, kvps)
Expand Down
Loading

0 comments on commit 28c7321

Please sign in to comment.