From 0d8b41d009daaba21b277bddd14d87c7b90dec2c Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Mon, 23 Aug 2021 11:52:46 -0600 Subject: [PATCH 01/14] Add tests for vulture pseudo-random methods --- cmd/tempo-vulture/main_test.go | 165 +++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 cmd/tempo-vulture/main_test.go diff --git a/cmd/tempo-vulture/main_test.go b/cmd/tempo-vulture/main_test.go new file mode 100644 index 00000000000..7c796e2a862 --- /dev/null +++ b/cmd/tempo-vulture/main_test.go @@ -0,0 +1,165 @@ +package main + +import ( + "testing" + "time" + + "github.com/grafana/tempo/pkg/tempopb" + v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1" + thrift "github.com/jaegertracing/jaeger/thrift-gen/jaeger" + "github.com/stretchr/testify/require" +) + +func TestHasMissingSpans(t *testing.T) { + cases := []struct { + trace *tempopb.Trace + expeted bool + }{ + { + &tempopb.Trace{ + Batches: []*v1.ResourceSpans{ + { + InstrumentationLibrarySpans: []*v1.InstrumentationLibrarySpans{ + { + Spans: []*v1.Span{ + { + ParentSpanId: []byte("01234"), + }, + }, + }, + }, + }, + }, + }, + true, + }, + { + &tempopb.Trace{ + Batches: []*v1.ResourceSpans{ + { + InstrumentationLibrarySpans: []*v1.InstrumentationLibrarySpans{ + { + Spans: []*v1.Span{ + { + SpanId: []byte("01234"), + }, + { + ParentSpanId: []byte("01234"), + }, + }, + }, + }, + }, + }, + }, + false, + }, + } + + for _, tc := range cases { + require.Equal(t, tc.expeted, hasMissingSpans(tc.trace)) + } +} + +func TestGenerateRandomInt(t *testing.T) { + cases := []struct { + min int64 + max int64 + result int64 + }{ + { + min: 1, + max: 5, + result: 3, + }, + { + min: 10, + max: 50, + result: 41, + }, + { + min: 1, + max: 3, + result: 2, + }, + } + + for _, tc := range cases { + result := generateRandomInt(tc.min, tc.max) + require.Equal(t, tc.result, result) + } +} + +func TestGenerateRandomString(t *testing.T) { + + strings := []string{ + "zgbaiCMRAjWwhTHc", + } + + for _, s := range strings { + result := generateRandomString() + require.Equal(t, s, result) + } +} + +func TestGenerateRandomTags(t *testing.T) { + expected := []*thrift.Tag{ + { + Key: "cXoEFfRsWxPLDnJOb", + VStr: stringPointer("uAxhxKQFDaFpLSjF"), + }, + { + Key: "eQYhYzRyWJjP", + VStr: stringPointer("sNVlgTeMaPEZQ"), + }, + } + result := generateRandomTags() + require.Equal(t, expected, result) +} + +func TestGenerateRandomLogs(t *testing.T) { + expected := []*thrift.Log{ + { + Timestamp: time.Now().Unix(), + Fields: []*thrift.Tag{ + { + Key: "tHsbZRjxAwnwekrBEmf", + VStr: stringPointer("fRFEgmotaF"), + }, + { + Key: "QZLCtTMt", + VStr: stringPointer("zdcEkXBAk"), + }, + { + Key: "AReKJy", + VStr: stringPointer("CoaNatyyiN"), + }, + { + Key: "ussVma", + VStr: stringPointer("XJrscctNswYNsG"), + }, + }, + }, + { + Timestamp: time.Now().Unix(), + Fields: []*thrift.Tag{ + { + Key: "GZsnwTKSmVoiG", + VStr: stringPointer("FZBsbOJiF"), + }, + { + Key: "VjaRzLNTXYeUCWKs", + VStr: stringPointer("OpbUOpEdKupdOMe"), + }, + { + Key: "LbtZsyMGeu", + VStr: stringPointer("bGyRAOmBTvKSJfjz"), + }, + }, + }, + } + result := generateRandomLogs() + require.Equal(t, expected, result) +} + +func stringPointer(s string) *string { return &s } From 18b5211f356389cf9a36b0de69681ee690d42a79 Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Mon, 23 Aug 2021 13:21:39 -0600 Subject: [PATCH 02/14] Supply a *rand.Rand to vulture functions and update tests --- cmd/tempo-vulture/main.go | 42 +++++++++---------- cmd/tempo-vulture/main_test.go | 75 +++++++++++++++++++++++----------- 2 files changed, 73 insertions(+), 44 deletions(-) diff --git a/cmd/tempo-vulture/main.go b/cmd/tempo-vulture/main.go index c0c8355a736..b896ba8d5d2 100644 --- a/cmd/tempo-vulture/main.go +++ b/cmd/tempo-vulture/main.go @@ -85,7 +85,7 @@ func main() { <-tickerWrite.C seed := (time.Now().Unix() / interval) * interval - rand.Seed(seed) + r := rand.New(rand.NewSource(seed)) traceIDHigh := rand.Int63() traceIDLow := rand.Int63() @@ -96,7 +96,7 @@ func main() { ) logger.Info("sending trace") - for i := int64(0); i < generateRandomInt(1, 100); i++ { + for i := int64(0); i < generateRandomInt(1, 100, r); i++ { ctx := user.InjectOrgID(context.Background(), tempoOrgID) ctx, err := user.InjectIntoGRPCRequest(ctx) if err != nil { @@ -104,7 +104,7 @@ func main() { metricErrorTotal.Inc() continue } - err = c.EmitBatch(ctx, makeThriftBatch(traceIDHigh, traceIDLow)) + err = c.EmitBatch(ctx, makeThriftBatch(traceIDHigh, traceIDLow, r)) if err != nil { logger.Error("error pushing batch to Tempo", zap.Error(err)) metricErrorTotal.Inc() @@ -127,7 +127,7 @@ func main() { } // pick past interval and re-generate trace - seed := (generateRandomInt(startTime, currentTime) / interval) * interval + seed := (generateRandomInt(startTime, currentTime, rand.New(rand.NewSource(1))) / interval) * interval rand.Seed(seed) hexID := fmt.Sprintf("%016x%016x", rand.Int63(), rand.Int63()) @@ -162,66 +162,66 @@ func newJaegerGRPCClient(endpoint string) (*jaeger_grpc.Reporter, error) { return jaeger_grpc.NewReporter(conn, nil, logger), err } -func generateRandomString() string { +func generateRandomString(r *rand.Rand) string { var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - s := make([]rune, generateRandomInt(5, 20)) + s := make([]rune, generateRandomInt(5, 20, r)) for i := range s { s[i] = letters[rand.Intn(len(letters))] } return string(s) } -func generateRandomTags() []*thrift.Tag { +func generateRandomTags(r *rand.Rand) []*thrift.Tag { var tags []*thrift.Tag - count := generateRandomInt(1, 5) + count := generateRandomInt(1, 5, r) for i := int64(0); i < count; i++ { - value := generateRandomString() + value := generateRandomString(r) tags = append(tags, &thrift.Tag{ - Key: generateRandomString(), + Key: generateRandomString(r), VStr: &value, }) } return tags } -func generateRandomLogs() []*thrift.Log { +func generateRandomLogs(r *rand.Rand) []*thrift.Log { var logs []*thrift.Log - count := generateRandomInt(1, 5) + count := generateRandomInt(1, 5, r) for i := int64(0); i < count; i++ { logs = append(logs, &thrift.Log{ Timestamp: time.Now().Unix(), - Fields: generateRandomTags(), + Fields: generateRandomTags(r), }) } return logs } -func makeThriftBatch(TraceIDHigh int64, TraceIDLow int64) *thrift.Batch { +func makeThriftBatch(TraceIDHigh int64, TraceIDLow int64, r *rand.Rand) *thrift.Batch { var spans []*thrift.Span - count := generateRandomInt(1, 5) + count := generateRandomInt(1, 5, r) for i := int64(0); i < count; i++ { spans = append(spans, &thrift.Span{ TraceIdLow: TraceIDLow, TraceIdHigh: TraceIDHigh, SpanId: rand.Int63(), ParentSpanId: 0, - OperationName: generateRandomString(), + OperationName: generateRandomString(r), References: nil, Flags: 0, StartTime: time.Now().Unix(), Duration: rand.Int63(), - Tags: generateRandomTags(), - Logs: generateRandomLogs(), + Tags: generateRandomTags(r), + Logs: generateRandomLogs(r), }) } return &thrift.Batch{Spans: spans} } -func generateRandomInt(min int64, max int64) int64 { - number := min + rand.Int63n(max-min) +func generateRandomInt(min int64, max int64, r *rand.Rand) int64 { + number := min + r.Int63n(max-min) if number == min { - return generateRandomInt(min, max) + return generateRandomInt(min, max, r) } return number } diff --git a/cmd/tempo-vulture/main_test.go b/cmd/tempo-vulture/main_test.go index 7c796e2a862..dbc88596517 100644 --- a/cmd/tempo-vulture/main_test.go +++ b/cmd/tempo-vulture/main_test.go @@ -1,6 +1,7 @@ package main import ( + "math/rand" "testing" "time" @@ -62,6 +63,8 @@ func TestHasMissingSpans(t *testing.T) { } func TestGenerateRandomInt(t *testing.T) { + r := rand.New(rand.NewSource(1)) + cases := []struct { min int64 max int64 @@ -85,58 +88,67 @@ func TestGenerateRandomInt(t *testing.T) { } for _, tc := range cases { - result := generateRandomInt(tc.min, tc.max) + result := generateRandomInt(tc.min, tc.max, r) require.Equal(t, tc.result, result) } } func TestGenerateRandomString(t *testing.T) { + r := rand.New(rand.NewSource(1)) strings := []string{ - "zgbaiCMRAjWwhTHc", + "XVlBzgbaiC", } for _, s := range strings { - result := generateRandomString() + result := generateRandomString(r) require.Equal(t, s, result) } } func TestGenerateRandomTags(t *testing.T) { + r := rand.New(rand.NewSource(1)) + expected := []*thrift.Tag{ { - Key: "cXoEFfRsWxPLDnJOb", - VStr: stringPointer("uAxhxKQFDaFpLSjF"), + Key: "hTHctcuAxhx", + VStr: stringPointer("MRAjWw"), + }, + { + Key: "FfRsWxP", + VStr: stringPointer("KQFDaFpLSjFbcXoE"), }, { - Key: "eQYhYzRyWJjP", - VStr: stringPointer("sNVlgTeMaPEZQ"), + Key: "lgTeMaPE", + VStr: stringPointer("LDnJObCsNV"), }, } - result := generateRandomTags() + result := generateRandomTags(r) require.Equal(t, expected, result) } func TestGenerateRandomLogs(t *testing.T) { + r := rand.New(rand.NewSource(1)) + expected := []*thrift.Log{ { Timestamp: time.Now().Unix(), Fields: []*thrift.Tag{ { - Key: "tHsbZRjxAwnwekrBEmf", - VStr: stringPointer("fRFEgmotaF"), + Key: "WJjPjzpfRFEgmota", + VStr: stringPointer("ZQleQYhYzRy"), }, { - Key: "QZLCtTMt", - VStr: stringPointer("zdcEkXBAk"), + Key: "RjxAwnwekr", + VStr: stringPointer("FetHsbZ"), }, { - Key: "AReKJy", - VStr: stringPointer("CoaNatyyiN"), + Key: "EkXBAkjQZLCtT", + VStr: stringPointer("BEmfdzdc"), }, { - Key: "ussVma", - VStr: stringPointer("XJrscctNswYNsG"), + Key: "eKJyiXJrscctNswYNsG", + VStr: stringPointer("MtTCoaNatyyiNKAR"), }, }, }, @@ -144,21 +156,38 @@ func TestGenerateRandomLogs(t *testing.T) { Timestamp: time.Now().Unix(), Fields: []*thrift.Tag{ { - Key: "GZsnwTKSmVoiG", - VStr: stringPointer("FZBsbOJiF"), + Key: "FZBsbOJiFQG", + VStr: stringPointer("RussVmaoz"), + }, + { + Key: "iGLOpbUOpEdKu", + VStr: stringPointer("ZsnwTKSmVo"), + }, + { + Key: "TXYeUC", + VStr: stringPointer("pdOMeRVjaRzLN"), + }, + { + Key: "mBTvKSJfjza", + VStr: stringPointer("WKsXbGyRAO"), }, + }, + }, + { + Timestamp: time.Now().Unix(), + Fields: []*thrift.Tag{ { - Key: "VjaRzLNTXYeUCWKs", - VStr: stringPointer("OpbUOpEdKupdOMe"), + Key: "DQiYCOhgHOv", + VStr: stringPointer("LbtZsyMGeuDtRzQM"), }, { - Key: "LbtZsyMGeu", - VStr: stringPointer("bGyRAOmBTvKSJfjz"), + Key: "fNjJhhjUVRuSqfgqVM", + VStr: stringPointer("gSeycJPJHYNu"), }, }, }, } - result := generateRandomLogs() + result := generateRandomLogs(r) require.Equal(t, expected, result) } From 4e42cc0290e02f6736f41883ebc5ee77a36c4f79 Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Tue, 24 Aug 2021 09:12:30 -0600 Subject: [PATCH 03/14] Adjust read handling to leverage time objects --- cmd/tempo-vulture/main.go | 93 ++++++++++++++++++++++++---------- cmd/tempo-vulture/main_test.go | 78 ++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 26 deletions(-) diff --git a/cmd/tempo-vulture/main.go b/cmd/tempo-vulture/main.go index b896ba8d5d2..58f6a64fc60 100644 --- a/cmd/tempo-vulture/main.go +++ b/cmd/tempo-vulture/main.go @@ -69,10 +69,10 @@ func main() { logger.Info("Tempo Vulture starting") - startTime := time.Now().Unix() + startTime := time.Now() tickerWrite := time.NewTicker(tempoWriteBackoffDuration) tickerRead := time.NewTicker(tempoReadBackoffDuration) - interval := int64(tempoWriteBackoffDuration / time.Second) + interval := tempoWriteBackoffDuration // Write go func() { @@ -81,32 +81,30 @@ func main() { panic(err) } - for { - <-tickerWrite.C + for now := range tickerWrite.C { + r := newRand(now) - seed := (time.Now().Unix() / interval) * interval - r := rand.New(rand.NewSource(seed)) - traceIDHigh := rand.Int63() - traceIDLow := rand.Int63() + traceIDHigh := r.Int63() + traceIDLow := r.Int63() - logger := logger.With( + log := logger.With( zap.String("org_id", tempoOrgID), zap.String("write_trace_id", fmt.Sprintf("%016x%016x", traceIDLow, traceIDHigh)), - zap.Int64("seed", seed), + zap.Int64("seed", now.Unix()), ) - logger.Info("sending trace") + log.Info("sending trace") for i := int64(0); i < generateRandomInt(1, 100, r); i++ { ctx := user.InjectOrgID(context.Background(), tempoOrgID) ctx, err := user.InjectIntoGRPCRequest(ctx) if err != nil { - logger.Error("error injecting org id", zap.Error(err)) + log.Error("error injecting org id", zap.Error(err)) metricErrorTotal.Inc() continue } err = c.EmitBatch(ctx, makeThriftBatch(traceIDHigh, traceIDLow, r)) if err != nil { - logger.Error("error pushing batch to Tempo", zap.Error(err)) + log.Error("error pushing batch to Tempo", zap.Error(err)) metricErrorTotal.Inc() continue } @@ -116,20 +114,17 @@ func main() { // Read go func() { - for { - <-tickerRead.C + for now := range tickerRead.C { - currentTime := time.Now().Unix() - - // don't query traces before retention - if (currentTime - startTime) > int64(tempoRetentionDuration/time.Second) { - startTime = currentTime - int64(tempoRetentionDuration/time.Second) - } + intervals := intervalsBetween(startTime, now, interval) + intervals = trimOutdatedIntervals(intervals, tempoRetentionDuration) // pick past interval and re-generate trace - seed := (generateRandomInt(startTime, currentTime, rand.New(rand.NewSource(1))) / interval) * interval - rand.Seed(seed) - hexID := fmt.Sprintf("%016x%016x", rand.Int63(), rand.Int63()) + pick := generateRandomInt(0, int64(len(intervals)), newRand(now)) + seed := intervals[pick] + + r := newRand(seed) + hexID := fmt.Sprintf("%016x%016x", r.Int63(), r.Int63()) // query the trace metrics, err := queryTempoAndAnalyze(tempoQueryURL, seed, hexID) @@ -148,12 +143,53 @@ func main() { log.Fatal(http.ListenAndServe(prometheusListenAddress, nil)) } +func intervalsBetween(start, stop time.Time, interval time.Duration) []time.Time { + if stop.Before(start) { + return nil + } + + intervals := []time.Time{start} + next := start.Add(interval) + + for next.Before(stop) { + intervals = append(intervals, next) + next = next.Add(interval) + } + + return intervals +} + +func trimOutdatedIntervals(intervals []time.Time, lookBehind time.Duration) []time.Time { + if len(intervals) == 0 { + return nil + } + + oldest := intervals[len(intervals)-1].Add(-lookBehind) + + for i, t := range intervals { + if t.Before(oldest) { + continue + } + + if t.After(oldest) { + return intervals[i:] + } + } + + return nil +} + func newJaegerGRPCClient(endpoint string) (*jaeger_grpc.Reporter, error) { // remove scheme and port u, err := url.Parse(endpoint) if err != nil { return nil, err } + + logger.Info("dialing grpc", + zap.String("endpoint", fmt.Sprintf("%s:14250", u.Host)), + ) + // new jaeger grpc exporter conn, err := grpc.Dial(u.Host+":14250", grpc.WithInsecure()) if err != nil { @@ -162,6 +198,10 @@ func newJaegerGRPCClient(endpoint string) (*jaeger_grpc.Reporter, error) { return jaeger_grpc.NewReporter(conn, nil, logger), err } +func newRand(t time.Time) *rand.Rand { + return rand.New(rand.NewSource(t.Unix())) +} + func generateRandomString(r *rand.Rand) string { var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") @@ -226,7 +266,7 @@ func generateRandomInt(min int64, max int64, r *rand.Rand) int64 { return number } -func queryTempoAndAnalyze(baseURL string, seed int64, traceID string) (traceMetrics, error) { +func queryTempoAndAnalyze(baseURL string, seed time.Time, traceID string) (traceMetrics, error) { tm := traceMetrics{ requested: 1, } @@ -234,7 +274,8 @@ func queryTempoAndAnalyze(baseURL string, seed int64, traceID string) (traceMetr logger := logger.With( zap.String("query_trace_id", traceID), zap.String("tempo_query_url", baseURL+"/api/traces/"+traceID), - zap.Int64("seed", seed), + zap.Int64("seed", seed.Unix()), + zap.Duration("ago", time.Since(seed)), ) logger.Info("querying Tempo") diff --git a/cmd/tempo-vulture/main_test.go b/cmd/tempo-vulture/main_test.go index dbc88596517..de0b9650bd0 100644 --- a/cmd/tempo-vulture/main_test.go +++ b/cmd/tempo-vulture/main_test.go @@ -191,4 +191,82 @@ func TestGenerateRandomLogs(t *testing.T) { require.Equal(t, expected, result) } +func TestIntervalsBetween(t *testing.T) { + now := time.Now() + cases := []struct { + start time.Time + stop time.Time + interval time.Duration + count int + }{ + { + start: now.Add(-1 * time.Minute), + stop: now, + interval: 11 * time.Second, + count: 6, + }, + { + start: now.Add(-1 * time.Hour), + stop: now, + interval: 33 * time.Second, + count: 110, + }, + } + + for _, tc := range cases { + result := intervalsBetween(tc.start, tc.stop, tc.interval) + require.Equal(t, tc.count, len(result)) + + if tc.count > 0 { + require.Equal(t, tc.start, result[0]) + require.True(t, result[len(result)-1].Before(tc.stop)) + } + } +} + +func TestTrimOutdatedIntervals(t *testing.T) { + now := time.Now() + + cases := []struct { + start time.Time + stop time.Time + interval time.Duration + retention time.Duration + count int + }{ + { + start: now.Add(-1 * time.Minute), + stop: now, + interval: 11 * time.Second, + count: 3, + retention: 30 * time.Second, + }, + { + start: now.Add(-1 * time.Hour), + stop: now, + interval: 33 * time.Second, + count: 110, + retention: 24 * time.Hour, + }, + { + start: now.Add(-25 * time.Hour), + stop: now, + interval: 33 * time.Second, + count: 2619, + retention: 24 * time.Hour, + }, + } + + for _, tc := range cases { + intervals := intervalsBetween(tc.start, tc.stop, tc.interval) + intervals = trimOutdatedIntervals(intervals, tc.retention) + + require.NotNil(t, intervals) + require.Equal(t, tc.count, len(intervals)) + + require.True(t, intervals[len(intervals)-1].Before(tc.stop)) + } + +} + func stringPointer(s string) *string { return &s } From 89a1e5629a486f8e7bd5c2dfe53bbcc32b117bbb Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Thu, 26 Aug 2021 12:30:38 -0600 Subject: [PATCH 04/14] Begin comparing traces --- CHANGELOG.md | 1 + cmd/tempo-vulture/main.go | 158 +++++++- cmd/tempo-vulture/main_test.go | 355 ++++++++++++++++-- cmd/tempo-vulture/testdata/trace.json | 1 + go.mod | 1 + go.sum | 2 + vendor/github.com/go-test/deep/.gitignore | 2 + vendor/github.com/go-test/deep/.travis.yml | 13 + vendor/github.com/go-test/deep/CHANGES.md | 42 +++ vendor/github.com/go-test/deep/LICENSE | 21 ++ vendor/github.com/go-test/deep/README.md | 51 +++ vendor/github.com/go-test/deep/deep.go | 417 +++++++++++++++++++++ vendor/github.com/go-test/deep/go.mod | 3 + vendor/github.com/go-test/deep/go.sum | 0 vendor/modules.txt | 3 + 15 files changed, 1023 insertions(+), 47 deletions(-) create mode 100644 cmd/tempo-vulture/testdata/trace.json create mode 100644 vendor/github.com/go-test/deep/.gitignore create mode 100644 vendor/github.com/go-test/deep/.travis.yml create mode 100644 vendor/github.com/go-test/deep/CHANGES.md create mode 100644 vendor/github.com/go-test/deep/LICENSE create mode 100644 vendor/github.com/go-test/deep/README.md create mode 100644 vendor/github.com/go-test/deep/deep.go create mode 100644 vendor/github.com/go-test/deep/go.mod create mode 100644 vendor/github.com/go-test/deep/go.sum diff --git a/CHANGELOG.md b/CHANGELOG.md index e8d8439009b..25681fe5d6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ * [ENHANCEMENT] Make s3 backend readError logic more robust [#905](https://github.com/grafana/tempo/pull/905) (@wei840222) * [ENHANCEMENT] Include additional detail when searching for traces [#916](https://github.com/grafana/tempo/pull/916) (@zalegrala) * [ENHANCEMENT] Add `gen index` and `gen bloom` commands to tempo-cli. [#903](https://github.com/grafana/tempo/pull/903) (@annanay25) +* [ENHANCEMENT] Implement trace comparison in Vulture [#904](https://github.com/grafana/tempo/pull/904) (@zalegrala) * [CHANGE] Renamed CLI flag from `--storage.trace.maintenance-cycle` to `--storage.trace.blocklist_poll`. This is a **breaking change** [#897](https://github.com/grafana/tempo/pull/897) (@mritunjaysharma394) ## v1.1.0 / 2021-08-26 diff --git a/cmd/tempo-vulture/main.go b/cmd/tempo-vulture/main.go index 58f6a64fc60..2925a715216 100644 --- a/cmd/tempo-vulture/main.go +++ b/cmd/tempo-vulture/main.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "encoding/hex" "flag" "fmt" "log" @@ -10,11 +11,18 @@ import ( "net/http" "net/url" "os" + "reflect" "time" + "github.com/go-test/deep" + "github.com/grafana/tempo/pkg/model" "github.com/grafana/tempo/pkg/tempopb" + v1common "github.com/grafana/tempo/pkg/tempopb/common/v1" + v1resource "github.com/grafana/tempo/pkg/tempopb/resource/v1" + v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1" "github.com/grafana/tempo/pkg/util" jaeger_grpc "github.com/jaegertracing/jaeger/cmd/agent/app/reporter/grpc" + "github.com/jaegertracing/jaeger/thrift-gen/jaeger" thrift "github.com/jaegertracing/jaeger/thrift-gen/jaeger" zaplogfmt "github.com/jsternberg/zap-logfmt" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -39,10 +47,11 @@ var ( ) type traceMetrics struct { - requested int - requestFailed int - notFound int - missingSpans int + requested int + requestFailed int + notFound int + missingSpans int + incorrectResult int } func init() { @@ -89,7 +98,7 @@ func main() { log := logger.With( zap.String("org_id", tempoOrgID), - zap.String("write_trace_id", fmt.Sprintf("%016x%016x", traceIDLow, traceIDHigh)), + zap.String("write_trace_id", fmt.Sprintf("%016x%016x", traceIDHigh, traceIDLow)), zap.Int64("seed", now.Unix()), ) log.Info("sending trace") @@ -102,7 +111,7 @@ func main() { metricErrorTotal.Inc() continue } - err = c.EmitBatch(ctx, makeThriftBatch(traceIDHigh, traceIDLow, r)) + err = c.EmitBatch(ctx, makeThriftBatch(traceIDHigh, traceIDLow, r, now)) if err != nil { log.Error("error pushing batch to Tempo", zap.Error(err)) metricErrorTotal.Inc() @@ -118,6 +127,7 @@ func main() { intervals := intervalsBetween(startTime, now, interval) intervals = trimOutdatedIntervals(intervals, tempoRetentionDuration) + startTime = intervals[0] // pick past interval and re-generate trace pick := generateRandomInt(0, int64(len(intervals)), newRand(now)) @@ -136,6 +146,7 @@ func main() { metricTracesErrors.WithLabelValues("requestfailed").Add(float64(metrics.requestFailed)) metricTracesErrors.WithLabelValues("notfound").Add(float64(metrics.notFound)) metricTracesErrors.WithLabelValues("missingspans").Add(float64(metrics.missingSpans)) + metricTracesErrors.WithLabelValues("incorrectresult").Add(float64(metrics.incorrectResult)) } }() @@ -159,12 +170,12 @@ func intervalsBetween(start, stop time.Time, interval time.Duration) []time.Time return intervals } -func trimOutdatedIntervals(intervals []time.Time, lookBehind time.Duration) []time.Time { +func trimOutdatedIntervals(intervals []time.Time, retention time.Duration) []time.Time { if len(intervals) == 0 { return nil } - oldest := intervals[len(intervals)-1].Add(-lookBehind) + oldest := intervals[len(intervals)-1].Add(-retention) for i, t := range intervals { if t.Before(oldest) { @@ -207,7 +218,7 @@ func generateRandomString(r *rand.Rand) string { s := make([]rune, generateRandomInt(5, 20, r)) for i := range s { - s[i] = letters[rand.Intn(len(letters))] + s[i] = letters[r.Intn(len(letters))] } return string(s) } @@ -225,39 +236,116 @@ func generateRandomTags(r *rand.Rand) []*thrift.Tag { return tags } -func generateRandomLogs(r *rand.Rand) []*thrift.Log { +func generateRandomLogs(r *rand.Rand, now time.Time) []*thrift.Log { var logs []*thrift.Log count := generateRandomInt(1, 5, r) for i := int64(0); i < count; i++ { logs = append(logs, &thrift.Log{ - Timestamp: time.Now().Unix(), + Timestamp: now.Unix(), Fields: generateRandomTags(r), }) } return logs } -func makeThriftBatch(TraceIDHigh int64, TraceIDLow int64, r *rand.Rand) *thrift.Batch { +func makeThriftBatch(TraceIDHigh int64, TraceIDLow int64, r *rand.Rand, now time.Time) *thrift.Batch { var spans []*thrift.Span count := generateRandomInt(1, 5, r) for i := int64(0); i < count; i++ { spans = append(spans, &thrift.Span{ TraceIdLow: TraceIDLow, TraceIdHigh: TraceIDHigh, - SpanId: rand.Int63(), + SpanId: r.Int63(), ParentSpanId: 0, OperationName: generateRandomString(r), References: nil, Flags: 0, - StartTime: time.Now().Unix(), - Duration: rand.Int63(), + StartTime: now.Unix(), + Duration: generateRandomInt(0, 100, r), Tags: generateRandomTags(r), - Logs: generateRandomLogs(r), + Logs: generateRandomLogs(r, now), }) } + return &thrift.Batch{Spans: spans} } +func jaegerBatchToPbTrace(batch *jaeger.Batch) *tempopb.Trace { + trace := &tempopb.Trace{ + Batches: []*v1.ResourceSpans{}, + } + libs := []*v1.InstrumentationLibrarySpans{ + { + InstrumentationLibrary: &v1common.InstrumentationLibrary{}, + }, + } + + for _, s := range batch.Spans { + traceIDHex := fmt.Sprintf("%016x%016x", s.TraceIdHigh, s.TraceIdLow) + traceID, err := util.HexStringToTraceID(traceIDHex) + if err != nil { + logger.Error(err.Error()) + } + + spanIDHex := fmt.Sprintf("%016x", s.SpanId) + spanID, err := hex.DecodeString(spanIDHex) + if err != nil { + logger.Error(err.Error()) + } + + startTime := time.Unix(s.StartTime, 0) + stopTime := startTime.Add(time.Duration(s.Duration) * time.Second) + + span := &v1.Span{ + TraceId: traceID, + SpanId: spanID, + Name: s.OperationName, + // TODO use time.UnixMilli() when upgraded to Go 1.17 + StartTimeUnixNano: uint64(startTime.UnixNano() / int64(time.Millisecond)), + EndTimeUnixNano: uint64(stopTime.UnixNano() / int64(time.Millisecond)), + Status: &v1.Status{}, + } + + for _, tag := range s.Tags { + span.Attributes = append(span.Attributes, &v1common.KeyValue{ + Key: tag.Key, + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: *tag.VStr}, + }, + }) + } + + for _, event := range s.Logs { + attrs := []*v1common.KeyValue{} + + for _, tag := range event.Fields { + attrs = append(attrs, &v1common.KeyValue{ + Key: tag.Key, + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: *tag.VStr}, + }, + }) + } + + t := time.Unix(event.Timestamp, 0) + + span.Events = append(span.Events, &v1.Span_Event{ + // TODO use time.UnixMilli() when upgraded to Go 1.17 + TimeUnixNano: uint64(t.UnixNano() / int64(time.Millisecond)), + Attributes: attrs, + }) + } + libs[0].Spans = append(libs[0].Spans, span) + } + + trace.Batches = append(trace.Batches, &v1.ResourceSpans{ + Resource: &v1resource.Resource{}, + InstrumentationLibrarySpans: libs, + }) + + return trace +} + func generateRandomInt(min int64, max int64, r *rand.Rand) int64 { number := min + r.Int63n(max-min) if number == min { @@ -301,9 +389,31 @@ func queryTempoAndAnalyze(baseURL string, seed time.Time, traceID string) (trace tm.missingSpans++ } + // Get the expected + expected := constructTraceFromEpoch(seed) + + match := equalTraces(expected, trace) + if !match { + tm.incorrectResult++ + if diff := deep.Equal(expected, trace); diff != nil { + for _, d := range diff { + logger.Error("incorrect result", + zap.String("expected -> response", d), + ) + } + } + } + return tm, nil } +func equalTraces(a, b *tempopb.Trace) bool { + model.SortTrace(a) + model.SortTrace(b) + + return reflect.DeepEqual(a, b) +} + func hasMissingSpans(t *tempopb.Trace) bool { // collect all parent span IDs linkedSpanIDs := make([][]byte, 0) @@ -340,3 +450,19 @@ func hasMissingSpans(t *tempopb.Trace) bool { return false } + +func constructTraceFromEpoch(epoch time.Time) *tempopb.Trace { + r := newRand(epoch) + traceIDHigh := r.Int63() + traceIDLow := r.Int63() + + trace := &tempopb.Trace{} + + for i := int64(0); i < generateRandomInt(1, 100, r); i++ { + batch := makeThriftBatch(traceIDHigh, traceIDLow, r, epoch) + result := jaegerBatchToPbTrace(batch) + trace.Batches = append(trace.Batches, result.Batches...) + } + + return trace +} diff --git a/cmd/tempo-vulture/main_test.go b/cmd/tempo-vulture/main_test.go index de0b9650bd0..ba478bda0a4 100644 --- a/cmd/tempo-vulture/main_test.go +++ b/cmd/tempo-vulture/main_test.go @@ -2,12 +2,18 @@ package main import ( "math/rand" + "os" "testing" "time" + "github.com/go-test/deep" + "github.com/gogo/protobuf/jsonpb" "github.com/grafana/tempo/pkg/tempopb" + v1common "github.com/grafana/tempo/pkg/tempopb/common/v1" v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1" + "github.com/jaegertracing/jaeger/thrift-gen/jaeger" thrift "github.com/jaegertracing/jaeger/thrift-gen/jaeger" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -97,7 +103,7 @@ func TestGenerateRandomString(t *testing.T) { r := rand.New(rand.NewSource(1)) strings := []string{ - "XVlBzgbaiC", + "VlBzgbaiCM", } for _, s := range strings { @@ -111,16 +117,16 @@ func TestGenerateRandomTags(t *testing.T) { expected := []*thrift.Tag{ { - Key: "hTHctcuAxhx", - VStr: stringPointer("MRAjWw"), + Key: "CMRAjWwhTHctcuAx", + VStr: stringPointer("lBzgba"), }, { - Key: "FfRsWxP", - VStr: stringPointer("KQFDaFpLSjFbcXoE"), + Key: "oEFfRsWxPLDnJOb", + VStr: stringPointer("xKQFDaFpLSjFbc"), }, { - Key: "lgTeMaPE", - VStr: stringPointer("LDnJObCsNV"), + Key: "eQYhYzRyWJjP", + VStr: stringPointer("sNVlgTeMaPEZQ"), }, } result := generateRandomTags(r) @@ -128,66 +134,76 @@ func TestGenerateRandomTags(t *testing.T) { } func TestGenerateRandomLogs(t *testing.T) { + now := time.Now() + r := rand.New(rand.NewSource(1)) expected := []*thrift.Log{ { - Timestamp: time.Now().Unix(), + Timestamp: now.Unix(), Fields: []*thrift.Tag{ { - Key: "WJjPjzpfRFEgmota", - VStr: stringPointer("ZQleQYhYzRy"), + Key: "whTHctcuAx", + VStr: stringPointer("BzgbaiCMRAj"), }, { - Key: "RjxAwnwekr", - VStr: stringPointer("FetHsbZ"), + Key: "oEFfRsWxPLDnJOb", + VStr: stringPointer("xKQFDaFpLSjFbc"), }, { - Key: "EkXBAkjQZLCtT", - VStr: stringPointer("BEmfdzdc"), + Key: "eQYhYzRyWJjP", + VStr: stringPointer("sNVlgTeMaPEZQ"), }, { - Key: "eKJyiXJrscctNswYNsG", - VStr: stringPointer("MtTCoaNatyyiNKAR"), + Key: "ZRjxAwnwekrBEmf", + VStr: stringPointer("zpfRFEgmotaFetHs"), }, }, }, { - Timestamp: time.Now().Unix(), + Timestamp: now.Unix(), Fields: []*thrift.Tag{ { - Key: "FZBsbOJiFQG", - VStr: stringPointer("RussVmaoz"), + Key: "TMtTCoaN", + VStr: stringPointer("cEkXBAkjQZLC"), }, { - Key: "iGLOpbUOpEdKu", - VStr: stringPointer("ZsnwTKSmVo"), + Key: "tNswYNsGRussVmaozFZ", + VStr: stringPointer("tyyiNKAReKJyiXJrsc"), }, { - Key: "TXYeUC", - VStr: stringPointer("pdOMeRVjaRzLN"), + Key: "GLOpbU", + VStr: stringPointer("sbOJiFQGZsnwTKSmVo"), }, { - Key: "mBTvKSJfjza", - VStr: stringPointer("WKsXbGyRAO"), + Key: "VjaRzLNTXYeUCWKs", + VStr: stringPointer("pEdKupdOMe"), }, }, }, { - Timestamp: time.Now().Unix(), + Timestamp: now.Unix(), Fields: []*thrift.Tag{ { - Key: "DQiYCOhgHOv", - VStr: stringPointer("LbtZsyMGeuDtRzQM"), + Key: "SJfjzaLbtZsyMGe", + VStr: stringPointer("yRAOmBTv"), + }, + { + Key: "HOvgSeycJPJHYNufN", + VStr: stringPointer("DtRzQMDQiYCOh"), }, { - Key: "fNjJhhjUVRuSqfgqVM", - VStr: stringPointer("gSeycJPJHYNu"), + Key: "qfgqVMkPYVkU", + VStr: stringPointer("JhhjUVRu"), + }, + { + Key: "rKCtzkjkZIvaBj", + VStr: stringPointer("UpiFvIZRgBmy"), }, }, }, } - result := generateRandomLogs(r) + result := generateRandomLogs(r, now) require.Equal(t, expected, result) } @@ -266,6 +282,283 @@ func TestTrimOutdatedIntervals(t *testing.T) { require.True(t, intervals[len(intervals)-1].Before(tc.stop)) } +} + +func TestEqualTraces(t *testing.T) { + now := time.Now() + + require.Equal(t, newRand(now).Int(), newRand(now).Int()) + + r1 := newRand(now) + batch1 := makeThriftBatch(r1.Int63(), r1.Int63(), r1, now) + pb1 := jaegerBatchToPbTrace(batch1) + + r2 := newRand(now) + batch2 := makeThriftBatch(r2.Int63(), r2.Int63(), r2, now) + pb2 := jaegerBatchToPbTrace(batch2) + + require.Equal(t, pb1, pb2) + require.True(t, equalTraces(pb1, pb2)) +} + +func TestJagerBatchToPbTrace(t *testing.T) { + nowish, err := time.Parse(time.RFC3339, "2021-08-25T10:15:31-06:00") + require.NoError(t, err) + r1 := newRand(nowish) + + cases := []struct { + batch *jaeger.Batch + expected *tempopb.Trace + }{ + { + batch: makeThriftBatch(r1.Int63(), r1.Int63(), r1, nowish), + expected: &tempopb.Trace{ + Batches: []*v1.ResourceSpans{ + { + InstrumentationLibrarySpans: []*v1.InstrumentationLibrarySpans{ + { + Spans: []*v1.Span{ + { + Name: "tgvrGx", + TraceId: []byte("0855b1342fd8f95914c1a43aae1b6c73"), + SpanId: []byte("39d2aec5556367e6"), + Events: []*v1.Span_Event{ + { + TimeUnixNano: 1629908131, + Attributes: []*v1common.KeyValue{ + { + Key: "fQKOABsE", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "rcEoMvnzLV"}, + }, + }, + { + Key: "IAeBiaSjF", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "YRmKJvUVSWD"}, + }, + }, + }, + }, + { + TimeUnixNano: 1629908131, + Attributes: []*v1common.KeyValue{ + { + Key: "hirzqJYPlSpH", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "GNowiq"}, + }, + }, + { + Key: "obDItNCcENJowMkHL", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "xmcFdTYhG"}, + }, + }, + }, + }, + { + TimeUnixNano: 1629908131, + Attributes: []*v1common.KeyValue{ + { + Key: "GrGxVkxyURMAFXv", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "dZiIwT"}, + }, + }, + { + Key: "HXfssLZ", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "njhpnYnxIaSiU"}, + }, + }, + { + Key: "kvuXiofamXq", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "bcxZfGtAvySamMkU"}, + }, + }, + { + Key: "AlSiwOjoJRVL", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "KiEJFRaxYF"}, + }, + }, + }, + }, + }, + Attributes: []*v1common.KeyValue{ + { + Key: "yZJMlOghq", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "paPjzXSe"}, + }, + }, + { + Key: "QDbZDAMYeQ", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "rXBYKuwWuCS"}, + }, + }, + }, + }, + { + Name: "BUZFbNNaAauNncS", + TraceId: []byte("0855b1342fd8f95914c1a43aae1b6c73"), + SpanId: []byte("3d278f63b44225b7"), + Events: []*v1.Span_Event{ + { + TimeUnixNano: 1629908131, + Attributes: []*v1common.KeyValue{ + { + Key: "KwxnCyqkJ", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "CfZmobCxYfqCkWR"}, + }, + }, + { + Key: "KQWDjfuUkSsLnxecMIC", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "zSgonMaGYEhS"}, + }, + }, + { + Key: "yOuqOyqCOvsdFGfW", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "ueDVCAMFiqMNPTr"}, + }, + }, + }, + }, + { + TimeUnixNano: 1629908131, + Attributes: []*v1common.KeyValue{ + { + Key: "vbgCDBMw", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "NnCdKhtWYsLq"}, + }, + }, + { + Key: "qborQobPng", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "jKsNTBoFbpvlaKkS"}, + }, + }, + { + Key: "FDRdKcKVtOAUvtKHnA", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "wrqriKCgwG"}, + }, + }, + { + Key: "IFSlfd", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "LbsCDuTrkW"}, + }, + }, + }, + }, + { + TimeUnixNano: 1629908131, + Attributes: []*v1common.KeyValue{ + { + Key: "iPSWrw", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "VOHOFrBpolYmHozHW"}, + }, + }, + { + Key: "HoUnCBfEAQMFGW", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "pncMfPLHwyqA"}, + }, + }, + }, + }, + }, + Attributes: []*v1common.KeyValue{ + { + Key: "XfdGaWAVs", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "NEoskqRbDSmtzKvZkp"}, + }, + }, + { + Key: "VLqUydNkPt", + Value: &v1common.AnyValue{ + Value: &v1common.AnyValue_StringValue{StringValue: "vPxSkJq"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, tc := range cases { + result := jaegerBatchToPbTrace(tc.batch) + if diff := deep.Equal(tc.expected, result); diff != nil { + t.Logf("expected: %+v", tc.expected) + t.Logf("result: %+v", result) + for _, d := range diff { + t.Error(d) + } + } + // require.Equal(t, tc.expected, result) + } +} + +func TestNewRand(t *testing.T) { + now := time.Now() + + r1 := newRand(now) + r2 := newRand(now) + r3 := newRand(now) + r4 := newRand(now) + + for _, x := range []*rand.Rand{r1, r2, r3, r4} { + x.Int63() + x.Int63() + x.Int63() + x.Int63() + generateRandomString(x) + generateRandomString(x) + generateRandomString(x) + generateRandomString(x) + generateRandomString(x) + } + + v := generateRandomString(r1) + for _, x := range []*rand.Rand{r2, r3, r4} { + require.Equal(t, v, generateRandomString(x)) + } +} + +func TestResponseFixture(t *testing.T) { + f, err := os.Open("testdata/trace.json") + require.NoError(t, err) + defer f.Close() + + response := &tempopb.Trace{} + err = jsonpb.Unmarshal(f, response) + require.NoError(t, err) + + seed := time.Unix(1630010049, 0) + expected := constructTraceFromEpoch(seed) + + assert.True(t, equalTraces(expected, response)) + + if diff := deep.Equal(expected, response); diff != nil { + for _, d := range diff { + t.Error(d) + } + } } diff --git a/cmd/tempo-vulture/testdata/trace.json b/cmd/tempo-vulture/testdata/trace.json new file mode 100644 index 00000000000..fda2de7f6d6 --- /dev/null +++ b/cmd/tempo-vulture/testdata/trace.json @@ -0,0 +1 @@ +{"batches":[{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"B52cYuiJF10=","name":"dXSIILHG","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010084000","attributes":[{"key":"YComJVvR","value":{"stringValue":"LiSeCjsOhSK"}},{"key":"nXZITjCuHTMmrkWIUN","value":{"stringValue":"YNjrChKeMgQiizwms"}},{"key":"qXPRlfeJAtnp","value":{"stringValue":"mFztHAvsquXlRvcl"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"zBbqtcyeqUbvOTA","value":{"stringValue":"gmdNyySNZdCpzWS"}},{"key":"FdTUPJYUMYpNRukN","value":{"stringValue":"YHBZKpUm"}},{"key":"mkWcCBbLzyj","value":{"stringValue":"tVjwJDMCeMkAj"}},{"key":"IriGfu","value":{"stringValue":"hclSVzeAk"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"XbsVqF","value":{"stringValue":"sOqwCOYdbLGdjl"}},{"key":"IcqPCjgxnrUdamAeQsf","value":{"stringValue":"RhuUDuAonjZYAdHz"}},{"key":"cVNvIcMuZjqivzrYZD","value":{"stringValue":"gzBluUlTTMFAJdJl"}},{"key":"rBdBTwBQRK","value":{"stringValue":"xMGpUqIFAfwAg"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"WMlNQsDlDkw=","name":"hyQLsgSJujQGDZqijWv","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010136000","attributes":[{"key":"qQTmMPw","value":{"stringValue":"ZFqMCqFK"}},{"key":"CSdtrFLoGUSv","value":{"stringValue":"qwnxLuZDwqYbG"}},{"key":"EzrZXducoHOhu","value":{"stringValue":"GtujMzPpdftAny"}},{"key":"imJuNWdgsCQejF","value":{"stringValue":"VQaSvYgziEpssgmzfB"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"vVSUmGOsiiqEcFYX","value":{"stringValue":"UMORYTNnqYpmIOMedA"}},{"key":"mpssrwckYSI","value":{"stringValue":"RMjIJJPHIk"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"VUAanGHvPBu","value":{"stringValue":"UJtJCASPYVbVka"}},{"key":"mCyaNwXswASBsKE","value":{"stringValue":"ZAmabkGedrWaWSgRHl"}},{"key":"yVBFRIHbP","value":{"stringValue":"NXwiazN"}},{"key":"NWkRwwtluVUHNfLc","value":{"stringValue":"xylxAygkROmY"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"nBngXmJxU","value":{"stringValue":"BAzsiJFmN"}},{"key":"iCLFwEoPJXeKoX","value":{"stringValue":"TzGJGB"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"jTvFSeAFPB","value":{"stringValue":"NoTAAmaoUQyUnIct"}},{"key":"VFRXqfmHITSpAFlvWk","value":{"stringValue":"RFjPuEFrFeMGHYWt"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"CQSClXtArh4=","name":"RdaJbyrJRBRd","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010071000","attributes":[{"key":"tVkrSzdI","value":{"stringValue":"bpxfaZedtxNmvHq"}},{"key":"VRPfphfp","value":{"stringValue":"LECQpHmvPxJKm"}},{"key":"AtcZyW","value":{"stringValue":"EvWlLQgoLlYMsVCwGG"}},{"key":"devnfpndUHi","value":{"stringValue":"UPhHzDADXs"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"QftMoHmlPunVaYP","value":{"stringValue":"hFIPzCvD"}},{"key":"qAirvJAQAPMsni","value":{"stringValue":"VrDkafQjoJBZKITVz"}},{"key":"yjWzKmSUCkXbHjauh","value":{"stringValue":"oMUPgpgklMeHPC"}},{"key":"HgFvIyuRBPT","value":{"stringValue":"wNDWqPiccSUsHy"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"bYWcWFMUnrFc","value":{"stringValue":"jPpvdbirVF"}},{"key":"rfvaLBFInDMp","value":{"stringValue":"MiINJdaEOmYQPT"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"Ie8lTXlA1F8=","name":"QaIbddwXWENcQZRxT","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010050000","attributes":[{"key":"jYPNPCunkgDGsnBrXK","value":{"stringValue":"cuhqrMaUqmvdfvQLyGy"}},{"key":"rAyPwuBuSlhwXZhMkeI","value":{"stringValue":"wrnMZeoxFqgARjAwSC"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"vwYWXyFodYXopPrypP","value":{"stringValue":"XWmOGpRKWxnrLY"}},{"key":"RBzHFTxFVfgivdWiTX","value":{"stringValue":"mSyhFUjmtJeWkfMWgI"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"yAfgJLkE","value":{"stringValue":"VAMzPXiEiUFh"}},{"key":"uaLXKJazqiDhP","value":{"stringValue":"iDrhqoZSeGZRQMusze"}},{"key":"txTKAwscNpOCzile","value":{"stringValue":"GhZwJQfKqYoFMb"}},{"key":"mhCauyTaVHOennCvm","value":{"stringValue":"cVvfQmqwIRERFnFwLiC"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"iVJikT","value":{"stringValue":"femOvXGIZoQGhjgpKsJ"}},{"key":"ywsjFWwEIUa","value":{"stringValue":"sfsKHqWEYdjdLQr"}},{"key":"BucYrgmwIQZgO","value":{"stringValue":"dfDPSiZOmHIsVqw"}},{"key":"jmpCPSw","value":{"stringValue":"cClcOCZEeYTobQ"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"C8w1NbaQWV8=","name":"CRtzZQyi","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010141000","attributes":[{"key":"sZvujGDwYgKPLL","value":{"stringValue":"fQCaIgIJYLQdwedZ"}},{"key":"ulNVBRxsiPbzlR","value":{"stringValue":"WhiIODkhnAgIDi"}},{"key":"sCtIDmiOewpOPb","value":{"stringValue":"aoSFRjkVuifYSF"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"UVsBIKnOmfirCioNFYX","value":{"stringValue":"pjrelGPPP"}},{"key":"LnHHcOAztxOXtbz","value":{"stringValue":"fNMHFKJIl"}},{"key":"NrAWfynHE","value":{"stringValue":"mqymPXHDQcd"}},{"key":"SCIBQtglWFH","value":{"stringValue":"WBhFfkkjdHZdKohxJ"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"VQaWHgFUcOxWj","value":{"stringValue":"jNjpwIYdVNPGNuB"}},{"key":"MDOVJhOGKqbQHkIcD","value":{"stringValue":"pjSKazuHzEAWDDjJ"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"NffbLD","value":{"stringValue":"WqAkMs"}},{"key":"vOskTL","value":{"stringValue":"orsxShrfECmmf"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"fkDc8yzldLc=","name":"kVqQrSUgdc","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010142000","attributes":[{"key":"MiXkyZqOt","value":{"stringValue":"sEeQcF"}},{"key":"YtcCaHNmHEHwc","value":{"stringValue":"aPxIzKgZLvqCwhC"}},{"key":"btyfggwUTvxCVQBCVV","value":{"stringValue":"BSiOWAgXRMi"}},{"key":"xZFXyoAJAjiUfwLHWq","value":{"stringValue":"LfNkqh"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"uQykWtqrTmGPGn","value":{"stringValue":"dsZzdhOSzUjjPYePX"}},{"key":"GgcPQCbMkiHnt","value":{"stringValue":"HEpbugqmBtbpP"}},{"key":"vIKSRpfHfHdmwpbuXtv","value":{"stringValue":"gcGLuwItVrhFMknr"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"eAzKTARGdRuGw","value":{"stringValue":"AfBEUsINzEcPgJOLyjm"}},{"key":"vrThHoVEipqf","value":{"stringValue":"FIFThsqxl"}},{"key":"IwfRDigObD","value":{"stringValue":"HSOGfzRqIPNBB"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"Gg5JjDKfvsg=","name":"kpmJvsfwVrx","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010129000","attributes":[{"key":"LgqtYKtZes","value":{"stringValue":"EvRBbDYhscZtwFuQ"}},{"key":"wzYOfvsnJikc","value":{"stringValue":"UPEgVsbOTGQSNxvYBy"}},{"key":"tCrMGdWxJGBdmsO","value":{"stringValue":"QCrYEfxX"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"evtqvUXMwYXm","value":{"stringValue":"xiqicllPHpNxzBuG"}},{"key":"DRmggxKduzSdYiSuXuR","value":{"stringValue":"pbrxaZQWy"}},{"key":"ELDBIpUiOTPyGJIy","value":{"stringValue":"YKgbIlhlvmJIslNODF"}},{"key":"SPSiNJOGEYwPgED","value":{"stringValue":"QDYtwnNSDahQRTYnIXf"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"KzUUQyIdlwAzwOh","value":{"stringValue":"jxvwsrkGjRjTDutmlBz"}},{"key":"tWiJGsiVpP","value":{"stringValue":"sShvrDETGn"}},{"key":"lZNXHWPZGhZYrQIfp","value":{"stringValue":"wMZIDZFoVvHgDxHC"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"zMVXgGngeYIrmawTSsn","value":{"stringValue":"vHpICdsivq"}},{"key":"rkvKKIdUr","value":{"stringValue":"keIfoTuqzdVaYdVu"}},{"key":"amMnEJwJk","value":{"stringValue":"kKDaKDQDBfntGip"}},{"key":"AVFKhownIOD","value":{"stringValue":"dYiiKhJPPJRgzQZl"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"ywvengUyoge","value":{"stringValue":"RQlQqkCOcKQg"}},{"key":"PVnLUwZjvqhTqHS","value":{"stringValue":"QHAMpzWQNbhg"}},{"key":"dtARzwsmkdaYKd","value":{"stringValue":"aQNZbsRolx"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"P+UE8bwUpK4=","name":"gCXHBx","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010091000","attributes":[{"key":"NZCRKBzKhFH","value":{"stringValue":"bCWEhcTrHhFgwYvfxSp"}},{"key":"buXdrVTq","value":{"stringValue":"ViwjTHCpsOTIZyigxbu"}},{"key":"cxsnpDrbOzCmtMzzh","value":{"stringValue":"RwYHSHsYxIAqlRgawHM"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"bwFnUHTt","value":{"stringValue":"WfPLuanhsAKpqAEad"}},{"key":"SmYPos","value":{"stringValue":"ioCTdsZAFROinDWH"}},{"key":"rKuDihtNLiDVMtJ","value":{"stringValue":"dpcvPI"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"lNOPpuX","value":{"stringValue":"KtEAmBWmmk"}},{"key":"nlFlSjOXI","value":{"stringValue":"ICysfmWymehSzKU"}},{"key":"gRrfERtq","value":{"stringValue":"ISLasi"}},{"key":"mcyYqyYjmo","value":{"stringValue":"GKBNNdRxElaB"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"XnYC8ZgG6cA=","name":"lQFwjsK","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010121000","attributes":[{"key":"kAhGGBugJAVKJuP","value":{"stringValue":"NBUMYIUdtOUqRPdXu"}},{"key":"qEdHMwGuEVHrWxFir","value":{"stringValue":"JdMLzzFVzDJYjTjxAYf"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"vNVBJdsWhxCWWWxcv","value":{"stringValue":"mDDDUQgAtymfygxUMb"}},{"key":"AbtYSxmyFb","value":{"stringValue":"UOeyYffq"}},{"key":"qAdFxSaZBUM","value":{"stringValue":"uHaGHSmA"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"FASMjFPCZMXDB","value":{"stringValue":"PTktOPgcpegaKAWaR"}},{"key":"VuGGKekpXaoGLAzuPNv","value":{"stringValue":"YqbWdoxUoBDzRetZXjG"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"dqGIBGv1rXk=","name":"JVIsaXWdhAWeWbdCVe","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010101000","attributes":[{"key":"shkuxzuLXoNKubj","value":{"stringValue":"hwzwRUSBSKd"}},{"key":"fmgWAEqFHEsdeuNTtn","value":{"stringValue":"TOfOgvbtBPx"}},{"key":"BboKZLlvwr","value":{"stringValue":"oNPGmHrmhWwoku"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"GrAxmeQXxnvjeLQSZfl","value":{"stringValue":"hnpfKucKClOErWvSW"}},{"key":"pnEHBPitBUkRpQITYt","value":{"stringValue":"mAlMVugUYnKYVeb"}},{"key":"XCVuliBnprHwspAHWeG","value":{"stringValue":"kfsGckFpFuPbvJHNTrR"}},{"key":"cWXLIlwIHqSc","value":{"stringValue":"kDjjEpARclocv"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"UxrkQFkOxbIAa","value":{"stringValue":"XdyxwxufaRjNwJERC"}},{"key":"xMPbUkkfnOOaQ","value":{"stringValue":"YuUlvb"}},{"key":"qhiKEZg","value":{"stringValue":"IzBPri"}},{"key":"gECoxhsZk","value":{"stringValue":"TziSWdhbUugReg"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"CydXjRxtkamHI","value":{"stringValue":"dSaUGKspZWboVvgAhQ"}},{"key":"AltWzwxOJqvnqGJPT","value":{"stringValue":"nKLBIuEFVMScO"}},{"key":"ZkRaQrVhRkBuLQfiH","value":{"stringValue":"XYQvgfcwnYMfuDWat"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"VCVHrbEjEVPribaJE","value":{"stringValue":"zNCZbozWdEniSWiELz"}},{"key":"sqtYWQxANdHrB","value":{"stringValue":"mXRSaX"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"Jt6XxxhbvKY=","name":"LOScBC","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010065000","attributes":[{"key":"PgGnPQGdlw","value":{"stringValue":"pQbeVSN"}},{"key":"jewTcxNrUkntz","value":{"stringValue":"jtLSRH"}},{"key":"XxmlFBhlFjFtaTz","value":{"stringValue":"GVKuVhQhyKw"}},{"key":"cRtLmhrTRHG","value":{"stringValue":"HygRjMki"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"aNqnotlbrWqP","value":{"stringValue":"pmjQFwyXG"}},{"key":"ourrHPL","value":{"stringValue":"QLHKhQiLTrFbIoQg"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"sfEeWLmoTKQGxUaJ","value":{"stringValue":"tfcWDXpm"}},{"key":"pJSrdCO","value":{"stringValue":"UqyPGVkNsBQCzSzqLnA"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"AhdwBcHKLyUCq","value":{"stringValue":"suArnggKjYoDb"}},{"key":"YpljnZRRwhzapGyrBl","value":{"stringValue":"wRAedskpBHcQF"}},{"key":"KSjmMHSVf","value":{"stringValue":"TIYEDiKxDfWtVR"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"jBcEQlEbLNFaxzJrJmH","value":{"stringValue":"VlTExeIMMvOmEqlQX"}},{"key":"uasbrAUlkRNMy","value":{"stringValue":"MkTWLj"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"PgnMdYdFSpE=","name":"qMluYLWbzmfMgZEdrK","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010083000","attributes":[{"key":"gGvosEUKGQRULXKsB","value":{"stringValue":"EcQcYWmCoCLzVom"}},{"key":"zfKQIdcPDYIetQjkun","value":{"stringValue":"QCHrzOHnqqVOujscWcl"}},{"key":"DvQbkIHHC","value":{"stringValue":"WuDdlYWZwTWMtseZMP"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"inGsHt","value":{"stringValue":"wvizUmLFDZgup"}},{"key":"JCAYnrqOXRoIASzbM","value":{"stringValue":"OPqqMouE"}},{"key":"vbEEei","value":{"stringValue":"rYxUSEp"}},{"key":"ZLtDPwQJ","value":{"stringValue":"bfLEcHiea"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"dSYfyZtDQRcTiy","value":{"stringValue":"TleARBTKBoALxOpIl"}},{"key":"xtlpsePzJtnvnAh","value":{"stringValue":"bYjXZBK"}},{"key":"LkQqVcPnHsFSK","value":{"stringValue":"vXjIAC"}},{"key":"ywYAFvlyL","value":{"stringValue":"LADrhw"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"ZZZjjjEXhiE=","name":"nYcnnruZ","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010135000","attributes":[{"key":"ooAzEFlj","value":{"stringValue":"KTptZkDxhGgKAbRwWp"}},{"key":"OtXHoSWQjU","value":{"stringValue":"UKGPVhNjSUQNquyheX"}},{"key":"XTDDRPLTkq","value":{"stringValue":"dKvXnqcYyqaZ"}},{"key":"BwagWshtnm","value":{"stringValue":"HSxEQSvWy"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"coCxywhiSItYge","value":{"stringValue":"nbFnbxdcUKHpbxPk"}},{"key":"uXjtuyLgairgVJatrMi","value":{"stringValue":"ZIrJSfmezfXdk"}},{"key":"MvsrLncUO","value":{"stringValue":"UjyGCaYOXvjrdKsyh"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"IVQYumABWBnnmStuwsU","value":{"stringValue":"iZTRuszyBtMBCkpWn"}},{"key":"vVRLLpGOLpzePnYA","value":{"stringValue":"lTZXrAyqSCnhmm"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"ilSeiIswpBPqLeGYKw","value":{"stringValue":"LgBIsIZCAlSKzdbvd"}},{"key":"DsSqHF","value":{"stringValue":"SObgumeUmFzQkXNQta"}},{"key":"cpHEhIjftgiHjreKKo","value":{"stringValue":"wlAKri"}},{"key":"iCucsxFlpXdlp","value":{"stringValue":"lDpTjMPmrpsfesUr"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"Lyw1oxb+rG4=","name":"mGIhUuZBliC","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010053000","attributes":[{"key":"pjrtsQIxPgRW","value":{"stringValue":"FtGYKkjVyCmAnKlUpow"}},{"key":"KPFdqKAGHAP","value":{"stringValue":"obeWRLnJVWgrbao"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"ElwsGOxg","value":{"stringValue":"ApTtjPHltInAPjUP"}},{"key":"nydKmjI","value":{"stringValue":"uRXMNpLZwHTNQLjxf"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"uQpSav","value":{"stringValue":"eBjUnEcUVOBXYmkpBFh"}},{"key":"gbmhxpY","value":{"stringValue":"DvqSMIAoCTODE"}},{"key":"AOFhfEKwktirIpTAjZ","value":{"stringValue":"aqhTgflRtFivJgjYttV"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"L9a1PWIsubE=","name":"FHAlmrggDpaFlifQ","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010103000","attributes":[{"key":"anCueu","value":{"stringValue":"wRJTSZCariOyYEO"}},{"key":"jpRSaTaWmbtNmlRwIvZ","value":{"stringValue":"LjsuZlejPLBoFpxEOHY"}},{"key":"dxRyrxlqYGqR","value":{"stringValue":"vhMkvo"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"uGlInzHhlCCgxHjllnl","value":{"stringValue":"TuicDMUXtDQk"}},{"key":"tcHxvds","value":{"stringValue":"cNBKzI"}},{"key":"HXqSdqNZydMdPEmLu","value":{"stringValue":"XoGKYL"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"TDUWJkr","value":{"stringValue":"CYUWEAskWthyswnP"}},{"key":"lpjuAMcUDpMcGa","value":{"stringValue":"zHzCIMUehxMW"}},{"key":"WrMZbrHyanynvB","value":{"stringValue":"SvMkhOHSPlmFRfgbwD"}},{"key":"MKXqHdw","value":{"stringValue":"AmkgyJtaxigL"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"MpS549+H9hA=","name":"EFWLufr","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010069000","attributes":[{"key":"yfkeRWHIGJx","value":{"stringValue":"DVLARpsZlIEVMYF"}},{"key":"xGZXxtUAaWXSwXLw","value":{"stringValue":"utVhbzqyt"}},{"key":"lbuBdw","value":{"stringValue":"ENsJRZrr"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"lzLgvxvPsuIi","value":{"stringValue":"wOxjDZf"}},{"key":"CgeRzdus","value":{"stringValue":"IxhDnAKZAQPgC"}},{"key":"JlpaAYCkNcvUs","value":{"stringValue":"LfGfibHmKDVFRhlm"}},{"key":"BYLyOxvBxpABNcLyerI","value":{"stringValue":"eRMXQXHGGKZ"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"exJIKzNLuQpivFR","value":{"stringValue":"bZKKhwiTQIUyjeDR"}},{"key":"DrjfjoPvAJRpryxbzUX","value":{"stringValue":"NkVJDgkL"}},{"key":"KqAoRLBdyxUXsOLNww","value":{"stringValue":"YjEXKZXupUznf"}},{"key":"oqaXDavLq","value":{"stringValue":"bPHyPlpqt"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"kXBVQgXHLaX","value":{"stringValue":"nTPSNRiN"}},{"key":"jWGnNaXrQDRMeMoulA","value":{"stringValue":"KTDGGpDAaeEnAcpXsV"}},{"key":"jIGwLJtnNYZnGlPQ","value":{"stringValue":"TpECaIhioBhqSorHH"}},{"key":"zlyZvHWuOlkf","value":{"stringValue":"GAjalthQM"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"EythrgiPR","value":{"stringValue":"QIoHylqBlTefQrJCPRi"}},{"key":"tUzOjBdF","value":{"stringValue":"YcIXKHtW"}},{"key":"RZhGkIlOcxLrRxJGY","value":{"stringValue":"lkewfXmB"}},{"key":"WulynXwr","value":{"stringValue":"pgSDHEWTTUzAcFNajgd"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"bsRFW5tyGRI=","name":"pbnInJR","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010124000","attributes":[{"key":"UELDFc","value":{"stringValue":"EeVmJCKBuV"}},{"key":"HJmIocgg","value":{"stringValue":"TGsAvHOobieEwNVh"}},{"key":"YSneHmbyIlBjrAhKRqN","value":{"stringValue":"yTvbeztAhuUUqV"}},{"key":"DWhfviPvZoj","value":{"stringValue":"sUVdSKtKv"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"yIvUhsSqenEeu","value":{"stringValue":"vWvHPUSPhPdO"}},{"key":"tuImtdA","value":{"stringValue":"xcxlPEXPhJMZc"}},{"key":"RYJGSnZnlYoOAL","value":{"stringValue":"HMrutauiDrePizLwHCv"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"HOuaRwIdcUXAkCA","value":{"stringValue":"JKREExh"}},{"key":"RINfiG","value":{"stringValue":"UDgQtUzMChsSbdIMJbx"}},{"key":"voVzWNVvcLRCSNCQLw","value":{"stringValue":"AFGiPTVGFvrxgKPxC"}},{"key":"xfDjbuUlemPkvWfRVL","value":{"stringValue":"HRRtYDWRa"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"wDPOhLPIIrLZquAWnw","value":{"stringValue":"qMUBVPOl"}},{"key":"vKXztLmhlkrqcL","value":{"stringValue":"bzxnZILgFBqlHzdLRUQ"}}]}],"status":{}}]}]}]} \ No newline at end of file diff --git a/go.mod b/go.mod index 14bc665efff..da0a3dac60f 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/drone/envsubst v1.0.3 github.com/dustin/go-humanize v1.0.0 github.com/go-kit/kit v0.11.0 + github.com/go-test/deep v1.0.7 github.com/gogo/protobuf v1.3.2 github.com/gogo/status v1.0.3 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b diff --git a/go.sum b/go.sum index b1653401456..727d1fe43d5 100644 --- a/go.sum +++ b/go.sum @@ -791,6 +791,8 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= +github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/go-zookeeper/zk v1.0.2 h1:4mx0EYENAdX/B/rbunjlt5+4RTA/a9SMHBRuSKdGxPM= github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= diff --git a/vendor/github.com/go-test/deep/.gitignore b/vendor/github.com/go-test/deep/.gitignore new file mode 100644 index 00000000000..53f12f0f0e8 --- /dev/null +++ b/vendor/github.com/go-test/deep/.gitignore @@ -0,0 +1,2 @@ +*.swp +*.out diff --git a/vendor/github.com/go-test/deep/.travis.yml b/vendor/github.com/go-test/deep/.travis.yml new file mode 100644 index 00000000000..7f85421e62c --- /dev/null +++ b/vendor/github.com/go-test/deep/.travis.yml @@ -0,0 +1,13 @@ +language: go + +go: + - "1.12" + - "1.13" + - "1.14" + +before_install: + - go get github.com/mattn/goveralls + - go get golang.org/x/tools/cover + +script: + - $HOME/gopath/bin/goveralls -service=travis-ci -package github.com/go-test/deep diff --git a/vendor/github.com/go-test/deep/CHANGES.md b/vendor/github.com/go-test/deep/CHANGES.md new file mode 100644 index 00000000000..ac00b9a97ec --- /dev/null +++ b/vendor/github.com/go-test/deep/CHANGES.md @@ -0,0 +1,42 @@ +# go-test/deep Changelog + +## v1.0.7 released 2020-07-11 + +* Fixed issue #39: Confusing diff when comparing distinct types with the same name (PR #44) + +## v1.0.6 released 2020-04-21 + +* Added `NilMapsAreEmpty` variable which causes a nil map to equal an empty map (PR #43) (@yalegko) + +## v1.0.5 released 2020-01-16 + +* Added `NilSlicesAreEmpty` variable which causes a nil slice to be equal to an empty slice (PR #27) (@Anaminus) + +## v1.0.4 released 2019-09-15 + +* Added \`deep:"-"\` structure field tag to ignore field (PR #38) (@flga) + +## v1.0.3 released 2019-08-18 + +* Fixed issue #31: panic on typed primitives that implement error interface + +## v1.0.2 released 2019-07-14 + +* Enabled Go module (@radeksimko) +* Changed supported and tested Go versions: 1.10, 1.11, and 1.12 (dropped 1.9) +* Changed Error equality: additional struct fields are compared too (PR #29) (@andrewmostello) +* Fixed typos and ineffassign issues (PR #25) (@tariq1890) +* Fixed diff order for nil comparison (PR #16) (@gmarik) +* Fixed slice equality when slices are extracted from the same array (PR #11) (@risteli) +* Fixed test spelling and messages (PR #19) (@sofuture) +* Fixed issue #15: panic on comparing struct with anonymous time.Time +* Fixed issue #18: Panic when comparing structs with time.Time value and CompareUnexportedFields is true +* Fixed issue #21: Set default MaxDepth = 0 (disabled) (PR #23) + +## v1.0.1 released 2018-01-28 + +* Fixed issue #12: Arrays are not properly compared (@samlitowitz) + +## v1.0.0 releaesd 2017-10-27 + +* First release diff --git a/vendor/github.com/go-test/deep/LICENSE b/vendor/github.com/go-test/deep/LICENSE new file mode 100644 index 00000000000..228ef16f74e --- /dev/null +++ b/vendor/github.com/go-test/deep/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright 2015-2017 Daniel Nichter + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/go-test/deep/README.md b/vendor/github.com/go-test/deep/README.md new file mode 100644 index 00000000000..3b78eac7c1f --- /dev/null +++ b/vendor/github.com/go-test/deep/README.md @@ -0,0 +1,51 @@ +# Deep Variable Equality for Humans + +[![Go Report Card](https://goreportcard.com/badge/github.com/go-test/deep)](https://goreportcard.com/report/github.com/go-test/deep) [![Build Status](https://travis-ci.org/go-test/deep.svg?branch=master)](https://travis-ci.org/go-test/deep) [![Coverage Status](https://coveralls.io/repos/github/go-test/deep/badge.svg?branch=master)](https://coveralls.io/github/go-test/deep?branch=master) [![GoDoc](https://godoc.org/github.com/go-test/deep?status.svg)](https://godoc.org/github.com/go-test/deep) + +This package provides a single function: `deep.Equal`. It's like [reflect.DeepEqual](http://golang.org/pkg/reflect/#DeepEqual) but much friendlier to humans (or any sentient being) for two reason: + +* `deep.Equal` returns a list of differences +* `deep.Equal` does not compare unexported fields (by default) + +`reflect.DeepEqual` is good (like all things Golang!), but it's a game of [Hunt the Wumpus](https://en.wikipedia.org/wiki/Hunt_the_Wumpus). For large maps, slices, and structs, finding the difference is difficult. + +`deep.Equal` doesn't play games with you, it lists the differences: + +```go +package main_test + +import ( + "testing" + "github.com/go-test/deep" +) + +type T struct { + Name string + Numbers []float64 +} + +func TestDeepEqual(t *testing.T) { + // Can you spot the difference? + t1 := T{ + Name: "Isabella", + Numbers: []float64{1.13459, 2.29343, 3.010100010}, + } + t2 := T{ + Name: "Isabella", + Numbers: []float64{1.13459, 2.29843, 3.010100010}, + } + + if diff := deep.Equal(t1, t2); diff != nil { + t.Error(diff) + } +} +``` + + +``` +$ go test +--- FAIL: TestDeepEqual (0.00s) + main_test.go:25: [Numbers.slice[1]: 2.29343 != 2.29843] +``` + +The difference is in `Numbers.slice[1]`: the two values aren't equal using Go `==`. diff --git a/vendor/github.com/go-test/deep/deep.go b/vendor/github.com/go-test/deep/deep.go new file mode 100644 index 00000000000..dbc89c04bb9 --- /dev/null +++ b/vendor/github.com/go-test/deep/deep.go @@ -0,0 +1,417 @@ +// Package deep provides function deep.Equal which is like reflect.DeepEqual but +// returns a list of differences. This is helpful when comparing complex types +// like structures and maps. +package deep + +import ( + "errors" + "fmt" + "log" + "reflect" + "strings" +) + +var ( + // FloatPrecision is the number of decimal places to round float values + // to when comparing. + FloatPrecision = 10 + + // MaxDiff specifies the maximum number of differences to return. + MaxDiff = 10 + + // MaxDepth specifies the maximum levels of a struct to recurse into, + // if greater than zero. If zero, there is no limit. + MaxDepth = 0 + + // LogErrors causes errors to be logged to STDERR when true. + LogErrors = false + + // CompareUnexportedFields causes unexported struct fields, like s in + // T{s int}, to be compared when true. + CompareUnexportedFields = false + + // NilSlicesAreEmpty causes a nil slice to be equal to an empty slice. + NilSlicesAreEmpty = false + + // NilMapsAreEmpty causes a nil map to be equal to an empty map. + NilMapsAreEmpty = false +) + +var ( + // ErrMaxRecursion is logged when MaxDepth is reached. + ErrMaxRecursion = errors.New("recursed to MaxDepth") + + // ErrTypeMismatch is logged when Equal passed two different types of values. + ErrTypeMismatch = errors.New("variables are different reflect.Type") + + // ErrNotHandled is logged when a primitive Go kind is not handled. + ErrNotHandled = errors.New("cannot compare the reflect.Kind") +) + +type cmp struct { + diff []string + buff []string + floatFormat string +} + +var errorType = reflect.TypeOf((*error)(nil)).Elem() + +// Equal compares variables a and b, recursing into their structure up to +// MaxDepth levels deep (if greater than zero), and returns a list of differences, +// or nil if there are none. Some differences may not be found if an error is +// also returned. +// +// If a type has an Equal method, like time.Equal, it is called to check for +// equality. +// +// When comparing a struct, if a field has the tag `deep:"-"` then it will be +// ignored. +func Equal(a, b interface{}) []string { + aVal := reflect.ValueOf(a) + bVal := reflect.ValueOf(b) + c := &cmp{ + diff: []string{}, + buff: []string{}, + floatFormat: fmt.Sprintf("%%.%df", FloatPrecision), + } + if a == nil && b == nil { + return nil + } else if a == nil && b != nil { + c.saveDiff("", b) + } else if a != nil && b == nil { + c.saveDiff(a, "") + } + if len(c.diff) > 0 { + return c.diff + } + + c.equals(aVal, bVal, 0) + if len(c.diff) > 0 { + return c.diff // diffs + } + return nil // no diffs +} + +func (c *cmp) equals(a, b reflect.Value, level int) { + if MaxDepth > 0 && level > MaxDepth { + logError(ErrMaxRecursion) + return + } + + // Check if one value is nil, e.g. T{x: *X} and T.x is nil + if !a.IsValid() || !b.IsValid() { + if a.IsValid() && !b.IsValid() { + c.saveDiff(a.Type(), "") + } else if !a.IsValid() && b.IsValid() { + c.saveDiff("", b.Type()) + } + return + } + + // If different types, they can't be equal + aType := a.Type() + bType := b.Type() + if aType != bType { + // Built-in types don't have a name, so don't report [3]int != [2]int as " != " + if aType.Name() == "" || aType.Name() != bType.Name() { + c.saveDiff(aType, bType) + } else { + // Type names can be the same, e.g. pkg/v1.Error and pkg/v2.Error + // are both exported as pkg, so unless we include the full pkg path + // the diff will be "pkg.Error != pkg.Error" + // https://github.com/go-test/deep/issues/39 + aFullType := aType.PkgPath() + "." + aType.Name() + bFullType := bType.PkgPath() + "." + bType.Name() + c.saveDiff(aFullType, bFullType) + } + logError(ErrTypeMismatch) + return + } + + // Primitive https://golang.org/pkg/reflect/#Kind + aKind := a.Kind() + bKind := b.Kind() + + // Do a and b have underlying elements? Yes if they're ptr or interface. + aElem := aKind == reflect.Ptr || aKind == reflect.Interface + bElem := bKind == reflect.Ptr || bKind == reflect.Interface + + // If both types implement the error interface, compare the error strings. + // This must be done before dereferencing because the interface is on a + // pointer receiver. Re https://github.com/go-test/deep/issues/31, a/b might + // be primitive kinds; see TestErrorPrimitiveKind. + if aType.Implements(errorType) && bType.Implements(errorType) { + if (!aElem || !a.IsNil()) && (!bElem || !b.IsNil()) { + aString := a.MethodByName("Error").Call(nil)[0].String() + bString := b.MethodByName("Error").Call(nil)[0].String() + if aString != bString { + c.saveDiff(aString, bString) + return + } + } + } + + // Dereference pointers and interface{} + if aElem || bElem { + if aElem { + a = a.Elem() + } + if bElem { + b = b.Elem() + } + c.equals(a, b, level+1) + return + } + + switch aKind { + + ///////////////////////////////////////////////////////////////////// + // Iterable kinds + ///////////////////////////////////////////////////////////////////// + + case reflect.Struct: + /* + The variables are structs like: + type T struct { + FirstName string + LastName string + } + Type = .T, Kind = reflect.Struct + + Iterate through the fields (FirstName, LastName), recurse into their values. + */ + + // Types with an Equal() method, like time.Time, only if struct field + // is exported (CanInterface) + if eqFunc := a.MethodByName("Equal"); eqFunc.IsValid() && eqFunc.CanInterface() { + // Handle https://github.com/go-test/deep/issues/15: + // Don't call T.Equal if the method is from an embedded struct, like: + // type Foo struct { time.Time } + // First, we'll encounter Equal(Ttime, time.Time) but if we pass b + // as the 2nd arg we'll panic: "Call using pkg.Foo as type time.Time" + // As far as I can tell, there's no way to see that the method is from + // time.Time not Foo. So we check the type of the 1st (0) arg and skip + // unless it's b type. Later, we'll encounter the time.Time anonymous/ + // embedded field and then we'll have Equal(time.Time, time.Time). + funcType := eqFunc.Type() + if funcType.NumIn() == 1 && funcType.In(0) == bType { + retVals := eqFunc.Call([]reflect.Value{b}) + if !retVals[0].Bool() { + c.saveDiff(a, b) + } + return + } + } + + for i := 0; i < a.NumField(); i++ { + if aType.Field(i).PkgPath != "" && !CompareUnexportedFields { + continue // skip unexported field, e.g. s in type T struct {s string} + } + + if aType.Field(i).Tag.Get("deep") == "-" { + continue // field wants to be ignored + } + + c.push(aType.Field(i).Name) // push field name to buff + + // Get the Value for each field, e.g. FirstName has Type = string, + // Kind = reflect.String. + af := a.Field(i) + bf := b.Field(i) + + // Recurse to compare the field values + c.equals(af, bf, level+1) + + c.pop() // pop field name from buff + + if len(c.diff) >= MaxDiff { + break + } + } + case reflect.Map: + /* + The variables are maps like: + map[string]int{ + "foo": 1, + "bar": 2, + } + Type = map[string]int, Kind = reflect.Map + + Or: + type T map[string]int{} + Type = .T, Kind = reflect.Map + + Iterate through the map keys (foo, bar), recurse into their values. + */ + + if a.IsNil() || b.IsNil() { + if NilMapsAreEmpty { + if a.IsNil() && b.Len() != 0 { + c.saveDiff("", b) + return + } else if a.Len() != 0 && b.IsNil() { + c.saveDiff(a, "") + return + } + } else { + if a.IsNil() && !b.IsNil() { + c.saveDiff("", b) + } else if !a.IsNil() && b.IsNil() { + c.saveDiff(a, "") + } + } + return + } + + if a.Pointer() == b.Pointer() { + return + } + + for _, key := range a.MapKeys() { + c.push(fmt.Sprintf("map[%v]", key)) + + aVal := a.MapIndex(key) + bVal := b.MapIndex(key) + if bVal.IsValid() { + c.equals(aVal, bVal, level+1) + } else { + c.saveDiff(aVal, "") + } + + c.pop() + + if len(c.diff) >= MaxDiff { + return + } + } + + for _, key := range b.MapKeys() { + if aVal := a.MapIndex(key); aVal.IsValid() { + continue + } + + c.push(fmt.Sprintf("map[%v]", key)) + c.saveDiff("", b.MapIndex(key)) + c.pop() + if len(c.diff) >= MaxDiff { + return + } + } + case reflect.Array: + n := a.Len() + for i := 0; i < n; i++ { + c.push(fmt.Sprintf("array[%d]", i)) + c.equals(a.Index(i), b.Index(i), level+1) + c.pop() + if len(c.diff) >= MaxDiff { + break + } + } + case reflect.Slice: + if NilSlicesAreEmpty { + if a.IsNil() && b.Len() != 0 { + c.saveDiff("", b) + return + } else if a.Len() != 0 && b.IsNil() { + c.saveDiff(a, "") + return + } + } else { + if a.IsNil() && !b.IsNil() { + c.saveDiff("", b) + return + } else if !a.IsNil() && b.IsNil() { + c.saveDiff(a, "") + return + } + } + + aLen := a.Len() + bLen := b.Len() + + if a.Pointer() == b.Pointer() && aLen == bLen { + return + } + + n := aLen + if bLen > aLen { + n = bLen + } + for i := 0; i < n; i++ { + c.push(fmt.Sprintf("slice[%d]", i)) + if i < aLen && i < bLen { + c.equals(a.Index(i), b.Index(i), level+1) + } else if i < aLen { + c.saveDiff(a.Index(i), "") + } else { + c.saveDiff("", b.Index(i)) + } + c.pop() + if len(c.diff) >= MaxDiff { + break + } + } + + ///////////////////////////////////////////////////////////////////// + // Primitive kinds + ///////////////////////////////////////////////////////////////////// + + case reflect.Float32, reflect.Float64: + // Round floats to FloatPrecision decimal places to compare with + // user-defined precision. As is commonly know, floats have "imprecision" + // such that 0.1 becomes 0.100000001490116119384765625. This cannot + // be avoided; it can only be handled. Issue 30 suggested that floats + // be compared using an epsilon: equal = |a-b| < epsilon. + // In many cases the result is the same, but I think epsilon is a little + // less clear for users to reason about. See issue 30 for details. + aval := fmt.Sprintf(c.floatFormat, a.Float()) + bval := fmt.Sprintf(c.floatFormat, b.Float()) + if aval != bval { + c.saveDiff(a.Float(), b.Float()) + } + case reflect.Bool: + if a.Bool() != b.Bool() { + c.saveDiff(a.Bool(), b.Bool()) + } + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if a.Int() != b.Int() { + c.saveDiff(a.Int(), b.Int()) + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + if a.Uint() != b.Uint() { + c.saveDiff(a.Uint(), b.Uint()) + } + case reflect.String: + if a.String() != b.String() { + c.saveDiff(a.String(), b.String()) + } + + default: + logError(ErrNotHandled) + } +} + +func (c *cmp) push(name string) { + c.buff = append(c.buff, name) +} + +func (c *cmp) pop() { + if len(c.buff) > 0 { + c.buff = c.buff[0 : len(c.buff)-1] + } +} + +func (c *cmp) saveDiff(aval, bval interface{}) { + if len(c.buff) > 0 { + varName := strings.Join(c.buff, ".") + c.diff = append(c.diff, fmt.Sprintf("%s: %v != %v", varName, aval, bval)) + } else { + c.diff = append(c.diff, fmt.Sprintf("%v != %v", aval, bval)) + } +} + +func logError(err error) { + if LogErrors { + log.Println(err) + } +} diff --git a/vendor/github.com/go-test/deep/go.mod b/vendor/github.com/go-test/deep/go.mod new file mode 100644 index 00000000000..7ac85c448ac --- /dev/null +++ b/vendor/github.com/go-test/deep/go.mod @@ -0,0 +1,3 @@ +module github.com/go-test/deep + +go 1.13 diff --git a/vendor/github.com/go-test/deep/go.sum b/vendor/github.com/go-test/deep/go.sum new file mode 100644 index 00000000000..e69de29bb2d diff --git a/vendor/modules.txt b/vendor/modules.txt index 46c0b1848ab..ea8d4d91069 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -366,6 +366,9 @@ github.com/go-redis/redis/v8/internal/rand github.com/go-redis/redis/v8/internal/util # github.com/go-stack/stack v1.8.0 github.com/go-stack/stack +# github.com/go-test/deep v1.0.7 +## explicit +github.com/go-test/deep # github.com/gocql/gocql v0.0.0-20200526081602-cd04bd7f22a7 => github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85 github.com/gocql/gocql github.com/gocql/gocql/internal/lru From e51104fc62517761c85e753eac14a2727d72b6ba Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Fri, 27 Aug 2021 13:04:54 -0600 Subject: [PATCH 05/14] Round the seed to the nearest interval --- cmd/tempo-vulture/main.go | 20 +-- cmd/tempo-vulture/main_test.go | 284 ++------------------------------- 2 files changed, 19 insertions(+), 285 deletions(-) diff --git a/cmd/tempo-vulture/main.go b/cmd/tempo-vulture/main.go index 2925a715216..767d0061850 100644 --- a/cmd/tempo-vulture/main.go +++ b/cmd/tempo-vulture/main.go @@ -91,7 +91,7 @@ func main() { } for now := range tickerWrite.C { - r := newRand(now) + r := newRand(now.Round(interval)) traceIDHigh := r.Int63() traceIDLow := r.Int63() @@ -124,9 +124,7 @@ func main() { // Read go func() { for now := range tickerRead.C { - - intervals := intervalsBetween(startTime, now, interval) - intervals = trimOutdatedIntervals(intervals, tempoRetentionDuration) + intervals := intervalsBetween(startTime, now, interval, tempoRetentionDuration) startTime = intervals[0] // pick past interval and re-generate trace @@ -154,27 +152,19 @@ func main() { log.Fatal(http.ListenAndServe(prometheusListenAddress, nil)) } -func intervalsBetween(start, stop time.Time, interval time.Duration) []time.Time { +func intervalsBetween(start, stop time.Time, interval time.Duration, retention time.Duration) []time.Time { if stop.Before(start) { return nil } intervals := []time.Time{start} - next := start.Add(interval) + next := start.Round(interval) for next.Before(stop) { intervals = append(intervals, next) next = next.Add(interval) } - return intervals -} - -func trimOutdatedIntervals(intervals []time.Time, retention time.Duration) []time.Time { - if len(intervals) == 0 { - return nil - } - oldest := intervals[len(intervals)-1].Add(-retention) for i, t := range intervals { @@ -187,7 +177,7 @@ func trimOutdatedIntervals(intervals []time.Time, retention time.Duration) []tim } } - return nil + return intervals } func newJaegerGRPCClient(endpoint string) (*jaeger_grpc.Reporter, error) { diff --git a/cmd/tempo-vulture/main_test.go b/cmd/tempo-vulture/main_test.go index ba478bda0a4..b8b1cbf97e9 100644 --- a/cmd/tempo-vulture/main_test.go +++ b/cmd/tempo-vulture/main_test.go @@ -9,9 +9,7 @@ import ( "github.com/go-test/deep" "github.com/gogo/protobuf/jsonpb" "github.com/grafana/tempo/pkg/tempopb" - v1common "github.com/grafana/tempo/pkg/tempopb/common/v1" v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1" - "github.com/jaegertracing/jaeger/thrift-gen/jaeger" thrift "github.com/jaegertracing/jaeger/thrift-gen/jaeger" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -208,40 +206,7 @@ func TestGenerateRandomLogs(t *testing.T) { } func TestIntervalsBetween(t *testing.T) { - now := time.Now() - cases := []struct { - start time.Time - stop time.Time - interval time.Duration - count int - }{ - { - start: now.Add(-1 * time.Minute), - stop: now, - interval: 11 * time.Second, - count: 6, - }, - { - start: now.Add(-1 * time.Hour), - stop: now, - interval: 33 * time.Second, - count: 110, - }, - } - - for _, tc := range cases { - result := intervalsBetween(tc.start, tc.stop, tc.interval) - require.Equal(t, tc.count, len(result)) - - if tc.count > 0 { - require.Equal(t, tc.start, result[0]) - require.True(t, result[len(result)-1].Before(tc.stop)) - } - } -} - -func TestTrimOutdatedIntervals(t *testing.T) { - now := time.Now() + nowish := time.Unix(1630010049, 0) cases := []struct { start time.Time @@ -251,36 +216,29 @@ func TestTrimOutdatedIntervals(t *testing.T) { count int }{ { - start: now.Add(-1 * time.Minute), - stop: now, + start: nowish.Add(-1 * time.Minute), + stop: nowish, interval: 11 * time.Second, - count: 3, - retention: 30 * time.Second, + retention: 1 * time.Hour, + count: 7, }, { - start: now.Add(-1 * time.Hour), - stop: now, + start: nowish.Add(-1 * time.Hour), + stop: nowish, interval: 33 * time.Second, + retention: 1 * time.Hour, count: 110, - retention: 24 * time.Hour, - }, - { - start: now.Add(-25 * time.Hour), - stop: now, - interval: 33 * time.Second, - count: 2619, - retention: 24 * time.Hour, }, } for _, tc := range cases { - intervals := intervalsBetween(tc.start, tc.stop, tc.interval) - intervals = trimOutdatedIntervals(intervals, tc.retention) - - require.NotNil(t, intervals) - require.Equal(t, tc.count, len(intervals)) + result := intervalsBetween(tc.start, tc.stop, tc.interval, tc.retention) + require.Equal(t, tc.count, len(result)) - require.True(t, intervals[len(intervals)-1].Before(tc.stop)) + if tc.count > 0 { + require.Equal(t, tc.start, result[0]) + require.True(t, result[len(result)-1].Before(tc.stop)) + } } } @@ -301,219 +259,6 @@ func TestEqualTraces(t *testing.T) { require.True(t, equalTraces(pb1, pb2)) } -func TestJagerBatchToPbTrace(t *testing.T) { - nowish, err := time.Parse(time.RFC3339, "2021-08-25T10:15:31-06:00") - require.NoError(t, err) - r1 := newRand(nowish) - - cases := []struct { - batch *jaeger.Batch - expected *tempopb.Trace - }{ - { - batch: makeThriftBatch(r1.Int63(), r1.Int63(), r1, nowish), - expected: &tempopb.Trace{ - Batches: []*v1.ResourceSpans{ - { - InstrumentationLibrarySpans: []*v1.InstrumentationLibrarySpans{ - { - Spans: []*v1.Span{ - { - Name: "tgvrGx", - TraceId: []byte("0855b1342fd8f95914c1a43aae1b6c73"), - SpanId: []byte("39d2aec5556367e6"), - Events: []*v1.Span_Event{ - { - TimeUnixNano: 1629908131, - Attributes: []*v1common.KeyValue{ - { - Key: "fQKOABsE", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "rcEoMvnzLV"}, - }, - }, - { - Key: "IAeBiaSjF", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "YRmKJvUVSWD"}, - }, - }, - }, - }, - { - TimeUnixNano: 1629908131, - Attributes: []*v1common.KeyValue{ - { - Key: "hirzqJYPlSpH", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "GNowiq"}, - }, - }, - { - Key: "obDItNCcENJowMkHL", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "xmcFdTYhG"}, - }, - }, - }, - }, - { - TimeUnixNano: 1629908131, - Attributes: []*v1common.KeyValue{ - { - Key: "GrGxVkxyURMAFXv", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "dZiIwT"}, - }, - }, - { - Key: "HXfssLZ", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "njhpnYnxIaSiU"}, - }, - }, - { - Key: "kvuXiofamXq", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "bcxZfGtAvySamMkU"}, - }, - }, - { - Key: "AlSiwOjoJRVL", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "KiEJFRaxYF"}, - }, - }, - }, - }, - }, - Attributes: []*v1common.KeyValue{ - { - Key: "yZJMlOghq", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "paPjzXSe"}, - }, - }, - { - Key: "QDbZDAMYeQ", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "rXBYKuwWuCS"}, - }, - }, - }, - }, - { - Name: "BUZFbNNaAauNncS", - TraceId: []byte("0855b1342fd8f95914c1a43aae1b6c73"), - SpanId: []byte("3d278f63b44225b7"), - Events: []*v1.Span_Event{ - { - TimeUnixNano: 1629908131, - Attributes: []*v1common.KeyValue{ - { - Key: "KwxnCyqkJ", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "CfZmobCxYfqCkWR"}, - }, - }, - { - Key: "KQWDjfuUkSsLnxecMIC", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "zSgonMaGYEhS"}, - }, - }, - { - Key: "yOuqOyqCOvsdFGfW", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "ueDVCAMFiqMNPTr"}, - }, - }, - }, - }, - { - TimeUnixNano: 1629908131, - Attributes: []*v1common.KeyValue{ - { - Key: "vbgCDBMw", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "NnCdKhtWYsLq"}, - }, - }, - { - Key: "qborQobPng", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "jKsNTBoFbpvlaKkS"}, - }, - }, - { - Key: "FDRdKcKVtOAUvtKHnA", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "wrqriKCgwG"}, - }, - }, - { - Key: "IFSlfd", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "LbsCDuTrkW"}, - }, - }, - }, - }, - { - TimeUnixNano: 1629908131, - Attributes: []*v1common.KeyValue{ - { - Key: "iPSWrw", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "VOHOFrBpolYmHozHW"}, - }, - }, - { - Key: "HoUnCBfEAQMFGW", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "pncMfPLHwyqA"}, - }, - }, - }, - }, - }, - Attributes: []*v1common.KeyValue{ - { - Key: "XfdGaWAVs", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "NEoskqRbDSmtzKvZkp"}, - }, - }, - { - Key: "VLqUydNkPt", - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: "vPxSkJq"}, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - for _, tc := range cases { - result := jaegerBatchToPbTrace(tc.batch) - if diff := deep.Equal(tc.expected, result); diff != nil { - t.Logf("expected: %+v", tc.expected) - t.Logf("result: %+v", result) - for _, d := range diff { - t.Error(d) - } - } - // require.Equal(t, tc.expected, result) - } -} - func TestNewRand(t *testing.T) { now := time.Now() @@ -559,7 +304,6 @@ func TestResponseFixture(t *testing.T) { t.Error(d) } } - } func stringPointer(s string) *string { return &s } From c1866653ef90fd9a0bc16887fc416cae55fcd0a1 Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Mon, 30 Aug 2021 08:30:30 -0700 Subject: [PATCH 06/14] Adjust timestamp usage for interval rounding --- cmd/tempo-vulture/main.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/tempo-vulture/main.go b/cmd/tempo-vulture/main.go index 767d0061850..320bb46f765 100644 --- a/cmd/tempo-vulture/main.go +++ b/cmd/tempo-vulture/main.go @@ -91,7 +91,8 @@ func main() { } for now := range tickerWrite.C { - r := newRand(now.Round(interval)) + timestamp := now.Round(interval) + r := newRand(timestamp) traceIDHigh := r.Int63() traceIDLow := r.Int63() @@ -99,7 +100,7 @@ func main() { log := logger.With( zap.String("org_id", tempoOrgID), zap.String("write_trace_id", fmt.Sprintf("%016x%016x", traceIDHigh, traceIDLow)), - zap.Int64("seed", now.Unix()), + zap.Int64("seed", timestamp.Unix()), ) log.Info("sending trace") @@ -111,7 +112,7 @@ func main() { metricErrorTotal.Inc() continue } - err = c.EmitBatch(ctx, makeThriftBatch(traceIDHigh, traceIDLow, r, now)) + err = c.EmitBatch(ctx, makeThriftBatch(traceIDHigh, traceIDLow, r, timestamp)) if err != nil { log.Error("error pushing batch to Tempo", zap.Error(err)) metricErrorTotal.Inc() From 30ddb6e20da5b16a23a4ce1ffb0aa685e1a7d138 Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Mon, 30 Aug 2021 08:30:54 -0700 Subject: [PATCH 07/14] Avoid recursion for random number generation --- cmd/tempo-vulture/main.go | 4 +- cmd/tempo-vulture/main_test.go | 81 ++++++++++++++------------- cmd/tempo-vulture/testdata/trace.json | 2 +- 3 files changed, 43 insertions(+), 44 deletions(-) diff --git a/cmd/tempo-vulture/main.go b/cmd/tempo-vulture/main.go index 320bb46f765..e7d06388ce6 100644 --- a/cmd/tempo-vulture/main.go +++ b/cmd/tempo-vulture/main.go @@ -338,10 +338,8 @@ func jaegerBatchToPbTrace(batch *jaeger.Batch) *tempopb.Trace { } func generateRandomInt(min int64, max int64, r *rand.Rand) int64 { + min++ number := min + r.Int63n(max-min) - if number == min { - return generateRandomInt(min, max, r) - } return number } diff --git a/cmd/tempo-vulture/main_test.go b/cmd/tempo-vulture/main_test.go index b8b1cbf97e9..12628d7e2c1 100644 --- a/cmd/tempo-vulture/main_test.go +++ b/cmd/tempo-vulture/main_test.go @@ -77,12 +77,12 @@ func TestGenerateRandomInt(t *testing.T) { { min: 1, max: 5, - result: 3, + result: 4, }, { min: 10, max: 50, - result: 41, + result: 33, }, { min: 1, @@ -101,7 +101,7 @@ func TestGenerateRandomString(t *testing.T) { r := rand.New(rand.NewSource(1)) strings := []string{ - "VlBzgbaiCM", + "VlBzgbaiCMRAjWwhTH", } for _, s := range strings { @@ -115,16 +115,20 @@ func TestGenerateRandomTags(t *testing.T) { expected := []*thrift.Tag{ { - Key: "CMRAjWwhTHctcuAx", - VStr: stringPointer("lBzgba"), + Key: "tcuAxhxKQ", + VStr: stringPointer("lBzgbaiCMRAjWwhTH"), }, { - Key: "oEFfRsWxPLDnJOb", - VStr: stringPointer("xKQFDaFpLSjFbc"), + Key: "sWxPLDnJObCsN", + VStr: stringPointer("DaFpLSjFbcXoEFf"), }, { - Key: "eQYhYzRyWJjP", - VStr: stringPointer("sNVlgTeMaPEZQ"), + Key: "EZQleQYhYzR", + VStr: stringPointer("lgTeMa"), + }, + { + Key: "FetHsbZR", + VStr: stringPointer("WJjPjzpfRFEgmot"), }, } result := generateRandomTags(r) @@ -141,20 +145,16 @@ func TestGenerateRandomLogs(t *testing.T) { Timestamp: now.Unix(), Fields: []*thrift.Tag{ { - Key: "whTHctcuAx", - VStr: stringPointer("BzgbaiCMRAj"), - }, - { - Key: "oEFfRsWxPLDnJOb", - VStr: stringPointer("xKQFDaFpLSjFbc"), + Key: "THctcuAxhxKQFD", + VStr: stringPointer("BzgbaiCMRAjWw"), }, { - Key: "eQYhYzRyWJjP", - VStr: stringPointer("sNVlgTeMaPEZQ"), + Key: "LDnJObCsNVlgTeMaP", + VStr: stringPointer("FpLSjFbcXoEFfRsWx"), }, { - Key: "ZRjxAwnwekrBEmf", - VStr: stringPointer("zpfRFEgmotaFetHs"), + Key: "PjzpfRFEgmotaFetHsb", + VStr: stringPointer("ZQleQYhYzRyWJ"), }, }, }, @@ -162,20 +162,12 @@ func TestGenerateRandomLogs(t *testing.T) { Timestamp: now.Unix(), Fields: []*thrift.Tag{ { - Key: "TMtTCoaN", - VStr: stringPointer("cEkXBAkjQZLC"), - }, - { - Key: "tNswYNsGRussVmaozFZ", - VStr: stringPointer("tyyiNKAReKJyiXJrsc"), - }, - { - Key: "GLOpbU", - VStr: stringPointer("sbOJiFQGZsnwTKSmVo"), + Key: "BAkjQZLCtTMtTCoa", + VStr: stringPointer("jxAwnwekrBEmfdzdcEk"), }, { - Key: "VjaRzLNTXYeUCWKs", - VStr: stringPointer("pEdKupdOMe"), + Key: "cctNswYN", + VStr: stringPointer("atyyiNKAReKJyiXJr"), }, }, }, @@ -183,20 +175,29 @@ func TestGenerateRandomLogs(t *testing.T) { Timestamp: now.Unix(), Fields: []*thrift.Tag{ { - Key: "SJfjzaLbtZsyMGe", - VStr: stringPointer("yRAOmBTv"), + Key: "OJiFQG", + VStr: stringPointer("RussVmaozFZBs"), }, { - Key: "HOvgSeycJPJHYNufN", - VStr: stringPointer("DtRzQMDQiYCOh"), + Key: "KupdOMeRVja", + VStr: stringPointer("snwTKSmVoiGLOpbUOpE"), }, { - Key: "qfgqVMkPYVkU", - VStr: stringPointer("JhhjUVRu"), + Key: "WKsXbGyRA", + VStr: stringPointer("zLNTXYeU"), + }, + }, + }, + { + Timestamp: now.Unix(), + Fields: []*thrift.Tag{ + { + Key: "yMGeuDtRzQMDQ", + VStr: stringPointer("BTvKSJfjzaLbtZ"), }, { - Key: "rKCtzkjkZIvaBj", - VStr: stringPointer("UpiFvIZRgBmy"), + Key: "HYNufNjJhhjUVRu", + VStr: stringPointer("YCOhgHOvgSeycJP"), }, }, }, @@ -294,7 +295,7 @@ func TestResponseFixture(t *testing.T) { err = jsonpb.Unmarshal(f, response) require.NoError(t, err) - seed := time.Unix(1630010049, 0) + seed := time.Unix(1630337285, 0) expected := constructTraceFromEpoch(seed) assert.True(t, equalTraces(expected, response)) diff --git a/cmd/tempo-vulture/testdata/trace.json b/cmd/tempo-vulture/testdata/trace.json index fda2de7f6d6..ff6c33bcd0b 100644 --- a/cmd/tempo-vulture/testdata/trace.json +++ b/cmd/tempo-vulture/testdata/trace.json @@ -1 +1 @@ -{"batches":[{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"B52cYuiJF10=","name":"dXSIILHG","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010084000","attributes":[{"key":"YComJVvR","value":{"stringValue":"LiSeCjsOhSK"}},{"key":"nXZITjCuHTMmrkWIUN","value":{"stringValue":"YNjrChKeMgQiizwms"}},{"key":"qXPRlfeJAtnp","value":{"stringValue":"mFztHAvsquXlRvcl"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"zBbqtcyeqUbvOTA","value":{"stringValue":"gmdNyySNZdCpzWS"}},{"key":"FdTUPJYUMYpNRukN","value":{"stringValue":"YHBZKpUm"}},{"key":"mkWcCBbLzyj","value":{"stringValue":"tVjwJDMCeMkAj"}},{"key":"IriGfu","value":{"stringValue":"hclSVzeAk"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"XbsVqF","value":{"stringValue":"sOqwCOYdbLGdjl"}},{"key":"IcqPCjgxnrUdamAeQsf","value":{"stringValue":"RhuUDuAonjZYAdHz"}},{"key":"cVNvIcMuZjqivzrYZD","value":{"stringValue":"gzBluUlTTMFAJdJl"}},{"key":"rBdBTwBQRK","value":{"stringValue":"xMGpUqIFAfwAg"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"WMlNQsDlDkw=","name":"hyQLsgSJujQGDZqijWv","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010136000","attributes":[{"key":"qQTmMPw","value":{"stringValue":"ZFqMCqFK"}},{"key":"CSdtrFLoGUSv","value":{"stringValue":"qwnxLuZDwqYbG"}},{"key":"EzrZXducoHOhu","value":{"stringValue":"GtujMzPpdftAny"}},{"key":"imJuNWdgsCQejF","value":{"stringValue":"VQaSvYgziEpssgmzfB"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"vVSUmGOsiiqEcFYX","value":{"stringValue":"UMORYTNnqYpmIOMedA"}},{"key":"mpssrwckYSI","value":{"stringValue":"RMjIJJPHIk"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"VUAanGHvPBu","value":{"stringValue":"UJtJCASPYVbVka"}},{"key":"mCyaNwXswASBsKE","value":{"stringValue":"ZAmabkGedrWaWSgRHl"}},{"key":"yVBFRIHbP","value":{"stringValue":"NXwiazN"}},{"key":"NWkRwwtluVUHNfLc","value":{"stringValue":"xylxAygkROmY"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"nBngXmJxU","value":{"stringValue":"BAzsiJFmN"}},{"key":"iCLFwEoPJXeKoX","value":{"stringValue":"TzGJGB"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"jTvFSeAFPB","value":{"stringValue":"NoTAAmaoUQyUnIct"}},{"key":"VFRXqfmHITSpAFlvWk","value":{"stringValue":"RFjPuEFrFeMGHYWt"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"CQSClXtArh4=","name":"RdaJbyrJRBRd","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010071000","attributes":[{"key":"tVkrSzdI","value":{"stringValue":"bpxfaZedtxNmvHq"}},{"key":"VRPfphfp","value":{"stringValue":"LECQpHmvPxJKm"}},{"key":"AtcZyW","value":{"stringValue":"EvWlLQgoLlYMsVCwGG"}},{"key":"devnfpndUHi","value":{"stringValue":"UPhHzDADXs"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"QftMoHmlPunVaYP","value":{"stringValue":"hFIPzCvD"}},{"key":"qAirvJAQAPMsni","value":{"stringValue":"VrDkafQjoJBZKITVz"}},{"key":"yjWzKmSUCkXbHjauh","value":{"stringValue":"oMUPgpgklMeHPC"}},{"key":"HgFvIyuRBPT","value":{"stringValue":"wNDWqPiccSUsHy"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"bYWcWFMUnrFc","value":{"stringValue":"jPpvdbirVF"}},{"key":"rfvaLBFInDMp","value":{"stringValue":"MiINJdaEOmYQPT"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"Ie8lTXlA1F8=","name":"QaIbddwXWENcQZRxT","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010050000","attributes":[{"key":"jYPNPCunkgDGsnBrXK","value":{"stringValue":"cuhqrMaUqmvdfvQLyGy"}},{"key":"rAyPwuBuSlhwXZhMkeI","value":{"stringValue":"wrnMZeoxFqgARjAwSC"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"vwYWXyFodYXopPrypP","value":{"stringValue":"XWmOGpRKWxnrLY"}},{"key":"RBzHFTxFVfgivdWiTX","value":{"stringValue":"mSyhFUjmtJeWkfMWgI"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"yAfgJLkE","value":{"stringValue":"VAMzPXiEiUFh"}},{"key":"uaLXKJazqiDhP","value":{"stringValue":"iDrhqoZSeGZRQMusze"}},{"key":"txTKAwscNpOCzile","value":{"stringValue":"GhZwJQfKqYoFMb"}},{"key":"mhCauyTaVHOennCvm","value":{"stringValue":"cVvfQmqwIRERFnFwLiC"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"iVJikT","value":{"stringValue":"femOvXGIZoQGhjgpKsJ"}},{"key":"ywsjFWwEIUa","value":{"stringValue":"sfsKHqWEYdjdLQr"}},{"key":"BucYrgmwIQZgO","value":{"stringValue":"dfDPSiZOmHIsVqw"}},{"key":"jmpCPSw","value":{"stringValue":"cClcOCZEeYTobQ"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"C8w1NbaQWV8=","name":"CRtzZQyi","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010141000","attributes":[{"key":"sZvujGDwYgKPLL","value":{"stringValue":"fQCaIgIJYLQdwedZ"}},{"key":"ulNVBRxsiPbzlR","value":{"stringValue":"WhiIODkhnAgIDi"}},{"key":"sCtIDmiOewpOPb","value":{"stringValue":"aoSFRjkVuifYSF"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"UVsBIKnOmfirCioNFYX","value":{"stringValue":"pjrelGPPP"}},{"key":"LnHHcOAztxOXtbz","value":{"stringValue":"fNMHFKJIl"}},{"key":"NrAWfynHE","value":{"stringValue":"mqymPXHDQcd"}},{"key":"SCIBQtglWFH","value":{"stringValue":"WBhFfkkjdHZdKohxJ"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"VQaWHgFUcOxWj","value":{"stringValue":"jNjpwIYdVNPGNuB"}},{"key":"MDOVJhOGKqbQHkIcD","value":{"stringValue":"pjSKazuHzEAWDDjJ"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"NffbLD","value":{"stringValue":"WqAkMs"}},{"key":"vOskTL","value":{"stringValue":"orsxShrfECmmf"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"fkDc8yzldLc=","name":"kVqQrSUgdc","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010142000","attributes":[{"key":"MiXkyZqOt","value":{"stringValue":"sEeQcF"}},{"key":"YtcCaHNmHEHwc","value":{"stringValue":"aPxIzKgZLvqCwhC"}},{"key":"btyfggwUTvxCVQBCVV","value":{"stringValue":"BSiOWAgXRMi"}},{"key":"xZFXyoAJAjiUfwLHWq","value":{"stringValue":"LfNkqh"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"uQykWtqrTmGPGn","value":{"stringValue":"dsZzdhOSzUjjPYePX"}},{"key":"GgcPQCbMkiHnt","value":{"stringValue":"HEpbugqmBtbpP"}},{"key":"vIKSRpfHfHdmwpbuXtv","value":{"stringValue":"gcGLuwItVrhFMknr"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"eAzKTARGdRuGw","value":{"stringValue":"AfBEUsINzEcPgJOLyjm"}},{"key":"vrThHoVEipqf","value":{"stringValue":"FIFThsqxl"}},{"key":"IwfRDigObD","value":{"stringValue":"HSOGfzRqIPNBB"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"Gg5JjDKfvsg=","name":"kpmJvsfwVrx","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010129000","attributes":[{"key":"LgqtYKtZes","value":{"stringValue":"EvRBbDYhscZtwFuQ"}},{"key":"wzYOfvsnJikc","value":{"stringValue":"UPEgVsbOTGQSNxvYBy"}},{"key":"tCrMGdWxJGBdmsO","value":{"stringValue":"QCrYEfxX"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"evtqvUXMwYXm","value":{"stringValue":"xiqicllPHpNxzBuG"}},{"key":"DRmggxKduzSdYiSuXuR","value":{"stringValue":"pbrxaZQWy"}},{"key":"ELDBIpUiOTPyGJIy","value":{"stringValue":"YKgbIlhlvmJIslNODF"}},{"key":"SPSiNJOGEYwPgED","value":{"stringValue":"QDYtwnNSDahQRTYnIXf"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"KzUUQyIdlwAzwOh","value":{"stringValue":"jxvwsrkGjRjTDutmlBz"}},{"key":"tWiJGsiVpP","value":{"stringValue":"sShvrDETGn"}},{"key":"lZNXHWPZGhZYrQIfp","value":{"stringValue":"wMZIDZFoVvHgDxHC"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"zMVXgGngeYIrmawTSsn","value":{"stringValue":"vHpICdsivq"}},{"key":"rkvKKIdUr","value":{"stringValue":"keIfoTuqzdVaYdVu"}},{"key":"amMnEJwJk","value":{"stringValue":"kKDaKDQDBfntGip"}},{"key":"AVFKhownIOD","value":{"stringValue":"dYiiKhJPPJRgzQZl"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"ywvengUyoge","value":{"stringValue":"RQlQqkCOcKQg"}},{"key":"PVnLUwZjvqhTqHS","value":{"stringValue":"QHAMpzWQNbhg"}},{"key":"dtARzwsmkdaYKd","value":{"stringValue":"aQNZbsRolx"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"P+UE8bwUpK4=","name":"gCXHBx","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010091000","attributes":[{"key":"NZCRKBzKhFH","value":{"stringValue":"bCWEhcTrHhFgwYvfxSp"}},{"key":"buXdrVTq","value":{"stringValue":"ViwjTHCpsOTIZyigxbu"}},{"key":"cxsnpDrbOzCmtMzzh","value":{"stringValue":"RwYHSHsYxIAqlRgawHM"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"bwFnUHTt","value":{"stringValue":"WfPLuanhsAKpqAEad"}},{"key":"SmYPos","value":{"stringValue":"ioCTdsZAFROinDWH"}},{"key":"rKuDihtNLiDVMtJ","value":{"stringValue":"dpcvPI"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"lNOPpuX","value":{"stringValue":"KtEAmBWmmk"}},{"key":"nlFlSjOXI","value":{"stringValue":"ICysfmWymehSzKU"}},{"key":"gRrfERtq","value":{"stringValue":"ISLasi"}},{"key":"mcyYqyYjmo","value":{"stringValue":"GKBNNdRxElaB"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"XnYC8ZgG6cA=","name":"lQFwjsK","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010121000","attributes":[{"key":"kAhGGBugJAVKJuP","value":{"stringValue":"NBUMYIUdtOUqRPdXu"}},{"key":"qEdHMwGuEVHrWxFir","value":{"stringValue":"JdMLzzFVzDJYjTjxAYf"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"vNVBJdsWhxCWWWxcv","value":{"stringValue":"mDDDUQgAtymfygxUMb"}},{"key":"AbtYSxmyFb","value":{"stringValue":"UOeyYffq"}},{"key":"qAdFxSaZBUM","value":{"stringValue":"uHaGHSmA"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"FASMjFPCZMXDB","value":{"stringValue":"PTktOPgcpegaKAWaR"}},{"key":"VuGGKekpXaoGLAzuPNv","value":{"stringValue":"YqbWdoxUoBDzRetZXjG"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"dqGIBGv1rXk=","name":"JVIsaXWdhAWeWbdCVe","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010101000","attributes":[{"key":"shkuxzuLXoNKubj","value":{"stringValue":"hwzwRUSBSKd"}},{"key":"fmgWAEqFHEsdeuNTtn","value":{"stringValue":"TOfOgvbtBPx"}},{"key":"BboKZLlvwr","value":{"stringValue":"oNPGmHrmhWwoku"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"GrAxmeQXxnvjeLQSZfl","value":{"stringValue":"hnpfKucKClOErWvSW"}},{"key":"pnEHBPitBUkRpQITYt","value":{"stringValue":"mAlMVugUYnKYVeb"}},{"key":"XCVuliBnprHwspAHWeG","value":{"stringValue":"kfsGckFpFuPbvJHNTrR"}},{"key":"cWXLIlwIHqSc","value":{"stringValue":"kDjjEpARclocv"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"UxrkQFkOxbIAa","value":{"stringValue":"XdyxwxufaRjNwJERC"}},{"key":"xMPbUkkfnOOaQ","value":{"stringValue":"YuUlvb"}},{"key":"qhiKEZg","value":{"stringValue":"IzBPri"}},{"key":"gECoxhsZk","value":{"stringValue":"TziSWdhbUugReg"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"CydXjRxtkamHI","value":{"stringValue":"dSaUGKspZWboVvgAhQ"}},{"key":"AltWzwxOJqvnqGJPT","value":{"stringValue":"nKLBIuEFVMScO"}},{"key":"ZkRaQrVhRkBuLQfiH","value":{"stringValue":"XYQvgfcwnYMfuDWat"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"VCVHrbEjEVPribaJE","value":{"stringValue":"zNCZbozWdEniSWiELz"}},{"key":"sqtYWQxANdHrB","value":{"stringValue":"mXRSaX"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"Jt6XxxhbvKY=","name":"LOScBC","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010065000","attributes":[{"key":"PgGnPQGdlw","value":{"stringValue":"pQbeVSN"}},{"key":"jewTcxNrUkntz","value":{"stringValue":"jtLSRH"}},{"key":"XxmlFBhlFjFtaTz","value":{"stringValue":"GVKuVhQhyKw"}},{"key":"cRtLmhrTRHG","value":{"stringValue":"HygRjMki"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"aNqnotlbrWqP","value":{"stringValue":"pmjQFwyXG"}},{"key":"ourrHPL","value":{"stringValue":"QLHKhQiLTrFbIoQg"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"sfEeWLmoTKQGxUaJ","value":{"stringValue":"tfcWDXpm"}},{"key":"pJSrdCO","value":{"stringValue":"UqyPGVkNsBQCzSzqLnA"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"AhdwBcHKLyUCq","value":{"stringValue":"suArnggKjYoDb"}},{"key":"YpljnZRRwhzapGyrBl","value":{"stringValue":"wRAedskpBHcQF"}},{"key":"KSjmMHSVf","value":{"stringValue":"TIYEDiKxDfWtVR"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"jBcEQlEbLNFaxzJrJmH","value":{"stringValue":"VlTExeIMMvOmEqlQX"}},{"key":"uasbrAUlkRNMy","value":{"stringValue":"MkTWLj"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"PgnMdYdFSpE=","name":"qMluYLWbzmfMgZEdrK","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010083000","attributes":[{"key":"gGvosEUKGQRULXKsB","value":{"stringValue":"EcQcYWmCoCLzVom"}},{"key":"zfKQIdcPDYIetQjkun","value":{"stringValue":"QCHrzOHnqqVOujscWcl"}},{"key":"DvQbkIHHC","value":{"stringValue":"WuDdlYWZwTWMtseZMP"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"inGsHt","value":{"stringValue":"wvizUmLFDZgup"}},{"key":"JCAYnrqOXRoIASzbM","value":{"stringValue":"OPqqMouE"}},{"key":"vbEEei","value":{"stringValue":"rYxUSEp"}},{"key":"ZLtDPwQJ","value":{"stringValue":"bfLEcHiea"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"dSYfyZtDQRcTiy","value":{"stringValue":"TleARBTKBoALxOpIl"}},{"key":"xtlpsePzJtnvnAh","value":{"stringValue":"bYjXZBK"}},{"key":"LkQqVcPnHsFSK","value":{"stringValue":"vXjIAC"}},{"key":"ywYAFvlyL","value":{"stringValue":"LADrhw"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"ZZZjjjEXhiE=","name":"nYcnnruZ","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010135000","attributes":[{"key":"ooAzEFlj","value":{"stringValue":"KTptZkDxhGgKAbRwWp"}},{"key":"OtXHoSWQjU","value":{"stringValue":"UKGPVhNjSUQNquyheX"}},{"key":"XTDDRPLTkq","value":{"stringValue":"dKvXnqcYyqaZ"}},{"key":"BwagWshtnm","value":{"stringValue":"HSxEQSvWy"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"coCxywhiSItYge","value":{"stringValue":"nbFnbxdcUKHpbxPk"}},{"key":"uXjtuyLgairgVJatrMi","value":{"stringValue":"ZIrJSfmezfXdk"}},{"key":"MvsrLncUO","value":{"stringValue":"UjyGCaYOXvjrdKsyh"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"IVQYumABWBnnmStuwsU","value":{"stringValue":"iZTRuszyBtMBCkpWn"}},{"key":"vVRLLpGOLpzePnYA","value":{"stringValue":"lTZXrAyqSCnhmm"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"ilSeiIswpBPqLeGYKw","value":{"stringValue":"LgBIsIZCAlSKzdbvd"}},{"key":"DsSqHF","value":{"stringValue":"SObgumeUmFzQkXNQta"}},{"key":"cpHEhIjftgiHjreKKo","value":{"stringValue":"wlAKri"}},{"key":"iCucsxFlpXdlp","value":{"stringValue":"lDpTjMPmrpsfesUr"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"Lyw1oxb+rG4=","name":"mGIhUuZBliC","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010053000","attributes":[{"key":"pjrtsQIxPgRW","value":{"stringValue":"FtGYKkjVyCmAnKlUpow"}},{"key":"KPFdqKAGHAP","value":{"stringValue":"obeWRLnJVWgrbao"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"ElwsGOxg","value":{"stringValue":"ApTtjPHltInAPjUP"}},{"key":"nydKmjI","value":{"stringValue":"uRXMNpLZwHTNQLjxf"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"uQpSav","value":{"stringValue":"eBjUnEcUVOBXYmkpBFh"}},{"key":"gbmhxpY","value":{"stringValue":"DvqSMIAoCTODE"}},{"key":"AOFhfEKwktirIpTAjZ","value":{"stringValue":"aqhTgflRtFivJgjYttV"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"L9a1PWIsubE=","name":"FHAlmrggDpaFlifQ","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010103000","attributes":[{"key":"anCueu","value":{"stringValue":"wRJTSZCariOyYEO"}},{"key":"jpRSaTaWmbtNmlRwIvZ","value":{"stringValue":"LjsuZlejPLBoFpxEOHY"}},{"key":"dxRyrxlqYGqR","value":{"stringValue":"vhMkvo"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"uGlInzHhlCCgxHjllnl","value":{"stringValue":"TuicDMUXtDQk"}},{"key":"tcHxvds","value":{"stringValue":"cNBKzI"}},{"key":"HXqSdqNZydMdPEmLu","value":{"stringValue":"XoGKYL"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"TDUWJkr","value":{"stringValue":"CYUWEAskWthyswnP"}},{"key":"lpjuAMcUDpMcGa","value":{"stringValue":"zHzCIMUehxMW"}},{"key":"WrMZbrHyanynvB","value":{"stringValue":"SvMkhOHSPlmFRfgbwD"}},{"key":"MKXqHdw","value":{"stringValue":"AmkgyJtaxigL"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"MpS549+H9hA=","name":"EFWLufr","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010069000","attributes":[{"key":"yfkeRWHIGJx","value":{"stringValue":"DVLARpsZlIEVMYF"}},{"key":"xGZXxtUAaWXSwXLw","value":{"stringValue":"utVhbzqyt"}},{"key":"lbuBdw","value":{"stringValue":"ENsJRZrr"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"lzLgvxvPsuIi","value":{"stringValue":"wOxjDZf"}},{"key":"CgeRzdus","value":{"stringValue":"IxhDnAKZAQPgC"}},{"key":"JlpaAYCkNcvUs","value":{"stringValue":"LfGfibHmKDVFRhlm"}},{"key":"BYLyOxvBxpABNcLyerI","value":{"stringValue":"eRMXQXHGGKZ"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"exJIKzNLuQpivFR","value":{"stringValue":"bZKKhwiTQIUyjeDR"}},{"key":"DrjfjoPvAJRpryxbzUX","value":{"stringValue":"NkVJDgkL"}},{"key":"KqAoRLBdyxUXsOLNww","value":{"stringValue":"YjEXKZXupUznf"}},{"key":"oqaXDavLq","value":{"stringValue":"bPHyPlpqt"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"kXBVQgXHLaX","value":{"stringValue":"nTPSNRiN"}},{"key":"jWGnNaXrQDRMeMoulA","value":{"stringValue":"KTDGGpDAaeEnAcpXsV"}},{"key":"jIGwLJtnNYZnGlPQ","value":{"stringValue":"TpECaIhioBhqSorHH"}},{"key":"zlyZvHWuOlkf","value":{"stringValue":"GAjalthQM"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"EythrgiPR","value":{"stringValue":"QIoHylqBlTefQrJCPRi"}},{"key":"tUzOjBdF","value":{"stringValue":"YcIXKHtW"}},{"key":"RZhGkIlOcxLrRxJGY","value":{"stringValue":"lkewfXmB"}},{"key":"WulynXwr","value":{"stringValue":"pgSDHEWTTUzAcFNajgd"}}]}],"status":{}},{"traceId":"WSxczJcdjLg+p/Z9kHcE7w==","spanId":"bsRFW5tyGRI=","name":"pbnInJR","startTimeUnixNano":"1630010049000","endTimeUnixNano":"1630010124000","attributes":[{"key":"UELDFc","value":{"stringValue":"EeVmJCKBuV"}},{"key":"HJmIocgg","value":{"stringValue":"TGsAvHOobieEwNVh"}},{"key":"YSneHmbyIlBjrAhKRqN","value":{"stringValue":"yTvbeztAhuUUqV"}},{"key":"DWhfviPvZoj","value":{"stringValue":"sUVdSKtKv"}}],"events":[{"timeUnixNano":"1630010049000","attributes":[{"key":"yIvUhsSqenEeu","value":{"stringValue":"vWvHPUSPhPdO"}},{"key":"tuImtdA","value":{"stringValue":"xcxlPEXPhJMZc"}},{"key":"RYJGSnZnlYoOAL","value":{"stringValue":"HMrutauiDrePizLwHCv"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"HOuaRwIdcUXAkCA","value":{"stringValue":"JKREExh"}},{"key":"RINfiG","value":{"stringValue":"UDgQtUzMChsSbdIMJbx"}},{"key":"voVzWNVvcLRCSNCQLw","value":{"stringValue":"AFGiPTVGFvrxgKPxC"}},{"key":"xfDjbuUlemPkvWfRVL","value":{"stringValue":"HRRtYDWRa"}}]},{"timeUnixNano":"1630010049000","attributes":[{"key":"wDPOhLPIIrLZquAWnw","value":{"stringValue":"qMUBVPOl"}},{"key":"vKXztLmhlkrqcL","value":{"stringValue":"bzxnZILgFBqlHzdLRUQ"}}]}],"status":{}}]}]}]} \ No newline at end of file +{"batches":[{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"AWqRa/jgO3A=","name":"siAIsMcYaLiHosIn","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337300000","attributes":[{"key":"YKjbqy","value":{"stringValue":"RFjakBBK"}},{"key":"IQmfzJqWkIOkkl","value":{"stringValue":"OzrkKh"}},{"key":"VhfUUqQZVDmgGzP","value":{"stringValue":"OiYtSgQol"}},{"key":"SEaAVox","value":{"stringValue":"FmsrnMZpzyxRdPFm"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"cDYjEDBZInESJ","value":{"stringValue":"lvLaEKXNLDy"}},{"key":"uNxbpOTJwRFNefj","value":{"stringValue":"pEDBSslN"}},{"key":"UPrbps","value":{"stringValue":"wTgvhQdlMTV"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"eQnBahW","value":{"stringValue":"hCLbrlozeRwMPHZdlRg"}},{"key":"meYNljWQPTkv","value":{"stringValue":"iyczel"}},{"key":"wJJAEeFHsUEpJpFKsJp","value":{"stringValue":"NkZDrQU"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"JADnpdgYedE","value":{"stringValue":"nhrwaPdVkfmF"}},{"key":"PuVazRsMD","value":{"stringValue":"LDurdBjJRYIvQfFrvS"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"S6sLz2qyscY=","name":"AwRsceBcCuxJozJ","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337384000","attributes":[{"key":"gxDdvxC","value":{"stringValue":"jGNScdUxcrnof"}},{"key":"qNIiRhTeuxrupvEbnjq","value":{"stringValue":"VHPIfQHzmXasSnZoqLv"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"MRxsYrgUE","value":{"stringValue":"MtlfGoqMB"}},{"key":"KOIbttfDR","value":{"stringValue":"sAndKgviLrQi"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"YjmCpPn","value":{"stringValue":"IumQzOAwiZGEca"}},{"key":"uqQHWtCJktZiKDRKiW","value":{"stringValue":"tQLlfmpjpWdVfHHL"}},{"key":"QFYtVQfcmXKaOATwMI","value":{"stringValue":"EhIdRfypN"}},{"key":"bTdbWRrxF","value":{"stringValue":"wbzrdooaoEYLRZDq"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"PGQoFopHVpnJvcELp","value":{"stringValue":"vjxYmySNPLDUT"}},{"key":"GxqpLGkFL","value":{"stringValue":"fbAvYtuijCYO"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"WwP9lQKi7EA=","name":"PQTaxaZjIsclt","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337355000","attributes":[{"key":"ipQpJRQmKxx","value":{"stringValue":"OuGwdVQylNnpEu"}},{"key":"wusbbE","value":{"stringValue":"CAAsxEFhjWlScYT"}},{"key":"kKMToTOUQBE","value":{"stringValue":"UIQSitovsImks"}},{"key":"SrGOfoYIWjeMBaxMIss","value":{"stringValue":"ONIoxpsPhpmjMwnJjGt"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"PMRJWxvFsARuhmbfxx","value":{"stringValue":"bwYxWtSQrUPyYZ"}},{"key":"vNsxmzWnpqyZCU","value":{"stringValue":"wodMkqdKnici"}},{"key":"BGKQNDMus","value":{"stringValue":"XeimgOWS"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"ipayseSZj","value":{"stringValue":"YXhaguHAVT"}},{"key":"jWGzDAvASZRBhXlTLy","value":{"stringValue":"fNPftqOtAO"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"rbyhnmrwxGxr","value":{"stringValue":"iApvWymkr"}},{"key":"zeksHAq","value":{"stringValue":"VMFoZBwgCrCuW"}},{"key":"NtijHAymWuahzie","value":{"stringValue":"pdkDLDTKRDSUzBT"}},{"key":"jdFaGKBYLlllkdantSE","value":{"stringValue":"XjeuZGg"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"cIMbVLjsEZGdrDblXKG","value":{"stringValue":"qcpTJGCxVUjsQdGKeDU"}},{"key":"zkevWZgPv","value":{"stringValue":"oiyVxJEwTLzzAp"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"CNfuRPgvHT8=","name":"eNdtMCwoiiy","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337332000","attributes":[{"key":"csXgqBntXg","value":{"stringValue":"Ejdnmpjpq"}},{"key":"tFwCQBKcRHfamq","value":{"stringValue":"KTXAmsYtNYytsULnFQz"}},{"key":"zsdtdcaVbFEi","value":{"stringValue":"AvWwiTLznA"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"wnJkdVdcCz","value":{"stringValue":"lpCmgoApUPEsmsGltsC"}},{"key":"MmHJsVKJJh","value":{"stringValue":"dcDAuZrPMUsLRO"}},{"key":"ikZAaAMCQhtHCvry","value":{"stringValue":"NiggATaqXGBLzSZD"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"ssjUfNDiKwOFiG","value":{"stringValue":"iasMgWNUXQZOnVRpuIJ"}},{"key":"FOLwTugM","value":{"stringValue":"PyhutH"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"siotvDXiEffGlNhyOGV","value":{"stringValue":"qYJpTcSoLiFiu"}},{"key":"QhGkbyxGsCa","value":{"stringValue":"YvLeefB"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"IMlsRa","value":{"stringValue":"vamFsbFAnp"}},{"key":"DXUYxfrutX","value":{"stringValue":"vdcMWlib"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"OdxUMu2WQHI=","name":"RiFeXAzlwkeWuLdYNBw","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337355000","attributes":[{"key":"NsGhmVEddhbuUrki","value":{"stringValue":"aheKAxIXipMgnNbU"}},{"key":"XYWqGC","value":{"stringValue":"JPWoWEWTwuiJBlIN"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"cLudZFgqMlTOxvWiOyc","value":{"stringValue":"SdWQvRO"}},{"key":"FOURBQQBk","value":{"stringValue":"wXSueFbCrwaGBTChLZv"}},{"key":"IEXSwDFHOIXrC","value":{"stringValue":"YdpzHXR"}},{"key":"rdHuZMUzLNpF","value":{"stringValue":"UEroZVU"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"CeqLPBfgInTVoxx","value":{"stringValue":"rPQChmPIJjtus"}},{"key":"cSKMiXCwnvvELmLvx","value":{"stringValue":"VatZzrAYdCgxYNP"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"XWIjH1Sa5zA=","name":"MUPZKLmOUCFwKG","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337299000","attributes":[{"key":"qqmelYIRTpRImPfJrvJ","value":{"stringValue":"reZITQLGH"}},{"key":"QrKwTttaD","value":{"stringValue":"DxZpJWrKwMkZQhwHgEP"}},{"key":"DUikeEKuEnjrujLMl","value":{"stringValue":"OOwBJUwVkxtksinPou"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"GlAskaQNMHKMkRsV","value":{"stringValue":"lNUJDA"}},{"key":"KLQjXCNrUOB","value":{"stringValue":"smtlulYR"}},{"key":"EZQIEoboyMBSM","value":{"stringValue":"gWdTjtHWD"}},{"key":"rwZbCPnalUzmhea","value":{"stringValue":"TPWPYnbZMa"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"sRKMapSNV","value":{"stringValue":"jkaCsfdDafRoPIgtM"}},{"key":"jRAWKHzqIkb","value":{"stringValue":"YclLeRKNRySA"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"cewA/Q1puw4=","name":"zOmGmzAkMAHjIXpPHJN","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337339000","attributes":[{"key":"rfugMxt","value":{"stringValue":"kKJDJqurMrJhixVO"}},{"key":"WGzONyKlzlHZqZFdEOh","value":{"stringValue":"UcYjFhIoYeU"}},{"key":"gUQfiHFeiTerOm","value":{"stringValue":"JgRrYZPZtgHdSkBTbJJ"}},{"key":"GRXxOgMYXHD","value":{"stringValue":"xWnDUVNMjGUbNkGA"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"WXiEPHKGIVxV","value":{"stringValue":"knEnXoqo"}},{"key":"TaOarqPWRznGxfJj","value":{"stringValue":"qAlkxrdSmIkUIjogU"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"YRHcyoL","value":{"stringValue":"BGPpyMOtQJkfrHosLKR"}},{"key":"igzjvcopmpQDW","value":{"stringValue":"QHXPNwAkoTBC"}},{"key":"GhjEEZMfDd","value":{"stringValue":"lKIvZt"}},{"key":"ecEvZLzZHDvBEnWixL","value":{"stringValue":"aADWzVk"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"CZiqmmd9Oio=","name":"auyRemrzg","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337319000","attributes":[{"key":"AVXeWYdCsBiCvSi","value":{"stringValue":"BTaGIvqoWk"}},{"key":"eHsjQgWiBVTlILE","value":{"stringValue":"cCkFDtvvXZLxcc"}},{"key":"qBpJYkx","value":{"stringValue":"hmVabyMcrgsfexJe"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"xJvLNLCSOT","value":{"stringValue":"ivtwLKghUjSzqTGvF"}},{"key":"QitArzKzezUGv","value":{"stringValue":"dCeDwwpfinGIvi"}},{"key":"fWpOevz","value":{"stringValue":"AFuuJWx"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"yGDhkmE","value":{"stringValue":"ZxbxFzJoXLXUgI"}},{"key":"XzkhfAoJCyRIBX","value":{"stringValue":"NSOSOenInbFQmNw"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"GBrowR","value":{"stringValue":"SvXxdAdzS"}},{"key":"cQMPdhdtVrkNTMva","value":{"stringValue":"wWzDqdmshyTXo"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"WzPERnsWJgPyfKm","value":{"stringValue":"mkuuCybJKPBOsUxou"}},{"key":"hxfQejkHpcEKS","value":{"stringValue":"ZLeQHms"}},{"key":"RQuVgmkpKKcMiPXz","value":{"stringValue":"LXbrcIdL"}},{"key":"QBUAtrwpdeOhu","value":{"stringValue":"BsBEwaa"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"L19TYiCkcI4=","name":"gwuTsnTe","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337335000","attributes":[{"key":"SlSVyEABNDT","value":{"stringValue":"ybHTlhE"}},{"key":"ivXUoFdZeNIFNix","value":{"stringValue":"yTnbKzXMNDrJ"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"ggNxDNzuGEObeRn","value":{"stringValue":"lJwsKMIJlfjqvT"}},{"key":"QCjTxCSt","value":{"stringValue":"KOqViDYzwzcx"}},{"key":"NbEZXmw","value":{"stringValue":"ezcdAsHmufvgcRhTdIq"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"zIlVkufWDme","value":{"stringValue":"hReIzkLVOuAtDLC"}},{"key":"ybUKznYWiLCpVgHXgY","value":{"stringValue":"QkmAfRWWjkZvGHwObC"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"JukUrpGcMcAJYKTX","value":{"stringValue":"AqhyQQj"}},{"key":"YqGAXdeNTmwxmzushR","value":{"stringValue":"OBqGqqsaFwMiy"}},{"key":"hDEMcZpAy","value":{"stringValue":"mOpfeYHTvRiLFaV"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"IOXGsLwvBuJF","value":{"stringValue":"hsvCOEiQQC"}},{"key":"RUSTkHdeCEpqYftH","value":{"stringValue":"lHqOuyL"}},{"key":"WlXoKcIZpFbQJxxk","value":{"stringValue":"WDYMfCYQBkTx"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"CbCTa10HoP4=","name":"UewHrvsL","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337321000","attributes":[{"key":"tribOSEx","value":{"stringValue":"fZLFUNKxX"}},{"key":"iDEMMVgdFpGCsOK","value":{"stringValue":"IayQmZWxwd"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"dptQyioEeAISiz","value":{"stringValue":"VgQwqU"}},{"key":"kGeXvP","value":{"stringValue":"pfpBnVMtbQ"}},{"key":"PybpuSVRYbTl","value":{"stringValue":"jMJIMDMvCrFMdwf"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"pcgWgdKS","value":{"stringValue":"FfRAeLZhYnjChWUSPFk"}},{"key":"LawQoLYPfkOoAxm","value":{"stringValue":"GlHFqiGVRaIq"}},{"key":"lRqVDtetqMdWQL","value":{"stringValue":"abBeCVplE"}},{"key":"epioGrgjVLodHA","value":{"stringValue":"fIoqngoVd"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"E0I5GA5vYY8=","name":"HxAsKmHeqBcsztxMxwB","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337376000","attributes":[{"key":"jORCZuGLthPEs","value":{"stringValue":"VOunqlhLRFbRrJxd"}},{"key":"UPhAkUWsVBNkrbKnRk","value":{"stringValue":"KoiVutOFBl"}},{"key":"qTzuGGovr","value":{"stringValue":"QRsaKOOoAAC"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"eAnLXdPXeU","value":{"stringValue":"fvNUHtZEEnRunAj"}},{"key":"ytUjKNbAMorfWCAL","value":{"stringValue":"aoxLDqFe"}},{"key":"qnaeuF","value":{"stringValue":"oPHBDwnNzXNsHtCtox"}},{"key":"UAxalBIfeLNd","value":{"stringValue":"BMeJSJzx"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"rBmeXvCyECIBIwRyuRE","value":{"stringValue":"tEmUEHOfQjE"}},{"key":"OGWlZdexFXLoZeD","value":{"stringValue":"xdXhPfsAup"}},{"key":"NzOWbltjFy","value":{"stringValue":"wXqMwzO"}},{"key":"Amvspmln","value":{"stringValue":"ykXVgcMbcLp"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"hlcdXqTYixF","value":{"stringValue":"FxdjBgCKRKiYbvEQomr"}},{"key":"qnxRqVUQdz","value":{"stringValue":"YNvyEMAksWXInNYFff"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"DVmgOrXV8uY=","name":"rtcsTDMeVherspeZVzL","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337306000","attributes":[{"key":"fpnPai","value":{"stringValue":"wJrdYisE"}},{"key":"kyPYQngTyxGb","value":{"stringValue":"PUswnqYvKvZku"}},{"key":"yHUTTpv","value":{"stringValue":"SGHoGVBixMOOTNDl"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"tmZXdv","value":{"stringValue":"nWmUZIqh"}},{"key":"QFHiIhvJbhRSgrvr","value":{"stringValue":"lSRwOoK"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"NUsBMwE","value":{"stringValue":"pVQHCYoIWDyXTkq"}},{"key":"rKhBziihoiXWDbjHIqL","value":{"stringValue":"HAbZIuuBWceSsHIfqt"}},{"key":"IzfInREsp","value":{"stringValue":"xGUKedrUKdE"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"bsGszLjsCOgqER","value":{"stringValue":"AChHjIfMJsZCnWp"}},{"key":"zgfXhVy","value":{"stringValue":"mVhrmA"}},{"key":"WheJRgXuWRzvd","value":{"stringValue":"PTimPGaUwdWOgNg"}},{"key":"ZdnaboMFvwQX","value":{"stringValue":"byUHzrbYQIZUrS"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"kQSbVkdvazgzkqvH","value":{"stringValue":"LkRhPNM"}},{"key":"gCqdRtbWIpsMg","value":{"stringValue":"RgVUxaSITSjRnq"}},{"key":"nGuDbGhDnCqH","value":{"stringValue":"kOnnHuDZjcvm"}},{"key":"EMdnHz","value":{"stringValue":"MNiCeXCscegRygGM"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"IXVIcrmAZuI=","name":"eCkYsxMCjhZMURVRz","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337346000","attributes":[{"key":"oufuKJYQsDjPzMTbb","value":{"stringValue":"pWvmkwhqVICfmNesS"}},{"key":"uRgYHlvQTZw","value":{"stringValue":"CWQhuLgsRDc"}},{"key":"FiDnRenpXpz","value":{"stringValue":"ESvjgtUp"}},{"key":"tCQgFrYbwqLHC","value":{"stringValue":"onvWVLHBbNolMyhYMW"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"hJWOMFoKinlCP","value":{"stringValue":"dnSRtnKcBAmSNgmyd"}},{"key":"eXPfBkKMtMhiAi","value":{"stringValue":"eYayxkoaCimFndG"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"udkMCjqZZnizImSrKM","value":{"stringValue":"rlVJCixnluD"}},{"key":"HoXTkhnFnx","value":{"stringValue":"SuErVbuD"}},{"key":"zioCWGZVXpuekAfRQZC","value":{"stringValue":"pTLvHiSKqH"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"eZXrtAU","value":{"stringValue":"uggKoQooYJHdEXu"}},{"key":"aoIjMszkmP","value":{"stringValue":"cJBFDRxbnI"}},{"key":"KpPUdDodOaBHM","value":{"stringValue":"VaeDYrOSS"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"qBLvIwXGGNx","value":{"stringValue":"LITkvEyd"}},{"key":"WYIwfkiYsVmlYFqLiw","value":{"stringValue":"UqGZOBvnRvJvAOGaNkM"}},{"key":"SexRVGtOzjWSbybDu","value":{"stringValue":"GqWDNvlRxsP"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"E3AdNSTlx5I=","name":"RuMameQTKYjfmh","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337304000","attributes":[{"key":"qtHREirD","value":{"stringValue":"DVCHriEiNYFWxa"}},{"key":"WTmxtEXMgc","value":{"stringValue":"lVmxUDNhOcRDUiNZ"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"FICrOgzUcAy","value":{"stringValue":"MlJIqRRsKnqrugpwlfV"}},{"key":"IBlCdbmUBrCBE","value":{"stringValue":"gbnwaTZlhdwgxuKpdOV"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"XtJdYfjXHkXYDPTqM","value":{"stringValue":"jSFfchCrNoSw"}},{"key":"pELYMMrVIpHNRd","value":{"stringValue":"DjjPCOGyFwdkTXJG"}},{"key":"YtFZLuczqFNlUg","value":{"stringValue":"AKdQycHp"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"ImN16IupBAI=","name":"TgFMdQTKRhuYH","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337361000","attributes":[{"key":"MpKaKHA","value":{"stringValue":"vvsHwEdxVxZTp"}},{"key":"VLWzcPkquhQCQGD","value":{"stringValue":"eljBipj"}},{"key":"RjElkFZQSehUmAQWCm","value":{"stringValue":"JAnfLQUpJwfWJURa"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"zZogvVXORBQjis","value":{"stringValue":"lCWgjZZZcsiZ"}},{"key":"djrgJoZHP","value":{"stringValue":"smgVVYKjafDtiitoLRV"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"XEyAoHqSM","value":{"stringValue":"XXMQMvNfbgwaWbEx"}},{"key":"xrlmkboulYC","value":{"stringValue":"rXygiHsZnZ"}},{"key":"iZkXdLy","value":{"stringValue":"MZypYNHxreZlKmstV"}},{"key":"fVEUsVJnsNTVJGHAkSP","value":{"stringValue":"ROgdDtLDUWdUQa"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"CcErif","value":{"stringValue":"doFPZtmCnQ"}},{"key":"fwZofsiGmXWzUCT","value":{"stringValue":"VwobctjQJape"}},{"key":"mrLJRRwPHgInK","value":{"stringValue":"QYuTwwkNSsBgGpEYO"}},{"key":"LilhkyHfGOQb","value":{"stringValue":"kDslGsmiSNtLYU"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"lBCmtDfua","value":{"stringValue":"oiMnQo"}},{"key":"vPkEdi","value":{"stringValue":"uvRcFu"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"Q28BPulhLTI=","name":"ExtKAJrGkQSomNJhXL","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337366000","attributes":[{"key":"DaSmOxJZcxOosOTxkd","value":{"stringValue":"PXnCaOLYJQWvJpa"}},{"key":"MRglSfFidQcYaV","value":{"stringValue":"JBoHhHkxJwXMguH"}},{"key":"IYEyTKpQuDxyBIWW","value":{"stringValue":"GmfsOCOIkRKJE"}},{"key":"FwauhDJyLbPs","value":{"stringValue":"SDaXSqeXXSnTL"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"KlPpcvGlXvlsU","value":{"stringValue":"AzgAaTXYGp"}},{"key":"ZDKghk","value":{"stringValue":"loHKVfMKMZLgYHhG"}},{"key":"RTQfAOtgW","value":{"stringValue":"RosNdtxqrLa"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"TUdpwGKEvtZAiGc","value":{"stringValue":"LDIufvORJpraDq"}},{"key":"SKPljIKU","value":{"stringValue":"AQRoob"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"EJTXBpFlZ","value":{"stringValue":"EtWyLQsVixJksQBItQH"}},{"key":"vWRYuYj","value":{"stringValue":"jzGOFIgWws"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"owFhlqFsDoSfRczn","value":{"stringValue":"XACCcn"}},{"key":"MaFwkHUBK","value":{"stringValue":"QDQddRJCaoGWiPtAvSc"}},{"key":"bQkaeRSdIFKF","value":{"stringValue":"rVlIbCbEqFDBCUR"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"F+6LoW5pgSI=","name":"cwlDVjBTv","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337307000","attributes":[{"key":"QzGGsJLTcpbU","value":{"stringValue":"UewWstKbUFwd"}},{"key":"TnpiNVvatcnAmunMv","value":{"stringValue":"wMLdCWcIbOBgDQ"}},{"key":"EGtSsQcXhLFyT","value":{"stringValue":"lDccorOzJrZmwEGIkFl"}},{"key":"ltdighzARkjmao","value":{"stringValue":"GsDLIxpxwvWLoMWCByC"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"EVlaXurhXMrhz","value":{"stringValue":"PqIEnHAju"}},{"key":"rRJGcbHWNqgGUNZ","value":{"stringValue":"TmJzWMKnWeVPqIZRXc"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"sUWEMWERINmlZTFIu","value":{"stringValue":"vDcykEHEGHdKHkeUv"}},{"key":"GqcarihhJzFDptSGpv","value":{"stringValue":"FDpKAopeKGFhfygdYNX"}},{"key":"pNocHoazU","value":{"stringValue":"xojqYySKtPiCEagQMHa"}},{"key":"WEqHymAGDnIT","value":{"stringValue":"JWKouglwEivYXIUtsM"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"JgjPKGtHQnqgIBi","value":{"stringValue":"sROLfHDtZ"}},{"key":"ApOKpNditinURYwmc","value":{"stringValue":"WomyacPmFFIhjXzvanD"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"yZkViIxHbe","value":{"stringValue":"bMpVbaPTdhZzYC"}},{"key":"IkaOeFnhyZjfLNgRqdW","value":{"stringValue":"UvkbpcdD"}},{"key":"ZnHDrNcLuk","value":{"stringValue":"hhFxLbc"}},{"key":"HFhaEw","value":{"stringValue":"Kdutwaf"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"ViwMxYTccqo=","name":"EmwEqLAkcMTnqoFAVjB","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337382000","attributes":[{"key":"WtPdaHAHHfvuydWrFEd","value":{"stringValue":"UqqyRSwGbje"}},{"key":"akwHpFFVldVT","value":{"stringValue":"qYJHBnjRblBHIAGqmTu"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"zMXRgiJUYLbTLLWgk","value":{"stringValue":"afeIWhVJYYgcZfqOW"}},{"key":"qCbwmNY","value":{"stringValue":"ZXqcAvsgmQnrePNoTSt"}},{"key":"BEVBRJrkRArXKmpUXp","value":{"stringValue":"JMFOiAjBWgDDKEBa"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"UxAXVrNwAzSKbqLxCx","value":{"stringValue":"VMaALuhPhMrJkmF"}},{"key":"ItrFpYAlpgS","value":{"stringValue":"fTyVWa"}},{"key":"ThXqtDQK","value":{"stringValue":"CCkxvDiMPQMcYpq"}},{"key":"mgNUeIh","value":{"stringValue":"BpnwrfD"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"kpByyHIIRIcZJsBzzcl","value":{"stringValue":"MlvLzjXROX"}},{"key":"EFOGeqFYxOAKuCZSoyk","value":{"stringValue":"vFEKdXPkOac"}},{"key":"nxVTngojFTQayh","value":{"stringValue":"onvxRXbtZfvTwIf"}},{"key":"RMLqcDKvYwtrIT","value":{"stringValue":"HRmlZr"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"GHD7aeMK8O0=","name":"vjhgJxqbQqFCO","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337343000","attributes":[{"key":"teiTNeOnUxQWcfvrC","value":{"stringValue":"FhJNwGplFKhYZW"}},{"key":"rAiexbjjMNLX","value":{"stringValue":"wTMVOQy"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"tpqIQUmZNpLuY","value":{"stringValue":"fmcCiDqRyzYUQF"}},{"key":"THReqcxbTkGwIR","value":{"stringValue":"YJOcUZTKwGd"}},{"key":"AIElxwrTC","value":{"stringValue":"rTAvRCAhw"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"vJciNqGlrB","value":{"stringValue":"NXJyAUw"}},{"key":"QwWntLbN","value":{"stringValue":"YTlrTAYOGiQKJBFIEk"}},{"key":"rTRObHiuXnsMkXG","value":{"stringValue":"zEEIQQKwqAtmwycv"}},{"key":"rIPGqI","value":{"stringValue":"jaokDDsbsf"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"Noc+VxdtRxs=","name":"vvBuwoPBChkPez","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337365000","attributes":[{"key":"juBtTBuc","value":{"stringValue":"pJRJadMUmskzX"}},{"key":"OxVxDLyVCepH","value":{"stringValue":"CeDqwnVO"}},{"key":"xXfTeZzVcC","value":{"stringValue":"QZIZiOrxLgfukP"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"oNmjGJVRposg","value":{"stringValue":"fhgzGEIVq"}},{"key":"neoccWhzkcWNSUaC","value":{"stringValue":"rOOPUIHPVKvqaPogzET"}},{"key":"bWvzAHwNePvb","value":{"stringValue":"JybPZtTcsnsPGo"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"vxINyVolqOMO","value":{"stringValue":"wnYciBFFEcxFufPmz"}},{"key":"LygqzGJxCbt","value":{"stringValue":"drtsvFwGaQBtrbAELu"}},{"key":"xrZhANJ","value":{"stringValue":"UJCbmgYEGcYP"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"SWRqIkYxU34=","name":"cvKyTGhRNWbiNlyc","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337383000","attributes":[{"key":"glVsBiMStIxMvCRDA","value":{"stringValue":"VuTsMcMmasppPNn"}},{"key":"gqOkvqvyNWwb","value":{"stringValue":"uvaSSvQdrM"}},{"key":"VKsfbHfwpdYgfjUqp","value":{"stringValue":"DZaUIcFZzszRc"}},{"key":"ktylpVKYdiOrdWcxr","value":{"stringValue":"QVVyGiVfrVIo"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"LTCoRM","value":{"stringValue":"oCGWyVpnw"}},{"key":"tbuXsctwcorlxpl","value":{"stringValue":"CXKYpkWBHnhfr"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"RiilTpajsnOLWWjrY","value":{"stringValue":"PTXledGGALJzbgHV"}},{"key":"LRwsbVbRqsIpqVya","value":{"stringValue":"RFFsWuy"}},{"key":"HEMhCDsFn","value":{"stringValue":"shOkrYZnnJ"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"WvgBwciSVhmOGYwOe","value":{"stringValue":"NdLPtxDri"}},{"key":"eVlGCvvTJCss","value":{"stringValue":"hJRrVa"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"GKSWjKp+DDc=","name":"cmomyRzCpKNFAs","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337298000","attributes":[{"key":"ZnEAPUeJi","value":{"stringValue":"oKiXfdRNVMyPsnVn"}},{"key":"SdJoVTVlGiiqz","value":{"stringValue":"dbNDaqcRohLnYIeh"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"SXvjRTUJaP","value":{"stringValue":"XSmiAnTUiCsig"}},{"key":"UAOEhou","value":{"stringValue":"ZVowCIU"}},{"key":"mfiGJn","value":{"stringValue":"NcEEMgpwVewvp"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"JKnbgZXzGgkZfMOocF","value":{"stringValue":"tKGYiDKUvTNkDU"}},{"key":"VORQiC","value":{"stringValue":"aNMnOnSdFGG"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"YaXuJzNpdRiMwD","value":{"stringValue":"rBwTBvRqyLwAU"}},{"key":"IftMrRDugFFr","value":{"stringValue":"yJZoGrjBz"}},{"key":"DPWZaLvEdjvjXEyNDN","value":{"stringValue":"CUuvLUpdL"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"PFJiaZUL/aE=","name":"bmPKziRHCYE","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337355000","attributes":[{"key":"KRqtTwPWqVxgXtTqMam","value":{"stringValue":"iVTYXaBfi"}},{"key":"eXxGKtJ","value":{"stringValue":"SNlPyigRZIRkgxXm"}},{"key":"QygelsNcPllnkWXR","value":{"stringValue":"cQfYYiulNcgBhCX"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"MnOITZakiHcgdRMSV","value":{"stringValue":"mnseNVAsgXtT"}},{"key":"yrNBsB","value":{"stringValue":"luRHwcvcYH"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"OxGCQXtkXf","value":{"stringValue":"BjPzZirm"}},{"key":"KkwvsAvNGvPDGe","value":{"stringValue":"lIQzgPrmxMHlnbu"}},{"key":"wtQfGXu","value":{"stringValue":"pYwOUgpWlBhzMQsvdt"}},{"key":"NByhICREVRFml","value":{"stringValue":"LBrRZBRxPkEHLocmmnK"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"XaOrPTaRNfylYoDC","value":{"stringValue":"JKgSHrouHJclNiYLuS"}},{"key":"fcQSSRJQwppmLtdfR","value":{"stringValue":"WPZVckcpwibDSX"}},{"key":"jVckBSWTgYFQQvOR","value":{"stringValue":"NzmZlGp"}},{"key":"CRmrSkQkr","value":{"stringValue":"gtTppgvFTrvpygNCc"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"VLpclgUcL1Q=","name":"hpxDduZvNoBoyR","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337384000","attributes":[{"key":"IaEeDt","value":{"stringValue":"tZOfXFAEYkJPF"}},{"key":"btJLqRlyfgMvoUN","value":{"stringValue":"pMbefhs"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"gHGQyVxMSwwsYzGY","value":{"stringValue":"aeZvmQdXe"}},{"key":"rRJLvSZAOBudTIaM","value":{"stringValue":"LPHSehzPGENHPfIkX"}},{"key":"mVxNZKtvPSYsqEHFYp","value":{"stringValue":"hBYTXGCuaxQBsdMM"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"VlKEMBJfpd","value":{"stringValue":"vhExMTvklZIxlFhJ"}},{"key":"oMTVTSvE","value":{"stringValue":"Hhokwv"}},{"key":"NEyZQzJqFWCx","value":{"stringValue":"VyOPGCcQuNRyJq"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"fsuHrs","value":{"stringValue":"zloLLGoeMwwJmxv"}},{"key":"aUoXwnHV","value":{"stringValue":"DPRIDC"}},{"key":"AiXUqHdKvGtgo","value":{"stringValue":"OryumUEithkztDWXpGG"}},{"key":"DQSAnCBoJIfKQo","value":{"stringValue":"tSOAAUr"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"Hc6gZtIab/c=","name":"fhuQeHsnv","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337373000","attributes":[{"key":"HdTENVOrwBHs","value":{"stringValue":"cUNKdNxizRqUaJ"}},{"key":"hMJwWRY","value":{"stringValue":"MvCnmaNXatOWkWALD"}},{"key":"umHbKnXNlvINnMr","value":{"stringValue":"gzzTDMeNXAdHOiY"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"eyENaxqMqYVeNb","value":{"stringValue":"ZqxDdvk"}},{"key":"xlsFmyWne","value":{"stringValue":"jxRRGauwKPkrGn"}},{"key":"xVlLJOfeifGIny","value":{"stringValue":"clGmyZhfxDjkRraDnN"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"gQklqGTnO","value":{"stringValue":"NBeHkWymMhWNsK"}},{"key":"NqIUdqtTXMzTjCF","value":{"stringValue":"NDoNpQtZMzv"}},{"key":"vTvFJTCEdCrTkkOZ","value":{"stringValue":"YxVkTitQXqTXKh"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"fRNgHNQPGsr","value":{"stringValue":"gyQxThY"}},{"key":"iIsYvzjcdnjriQJg","value":{"stringValue":"nksMjKNutEEMJS"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"jQFjwu","value":{"stringValue":"bOwtkMsEqADzXy"}},{"key":"XrsFTkKBgWWFWS","value":{"stringValue":"jKQxIDHKTdxNvuJnuO"}},{"key":"QQfWpBKqDzqQFOMDg","value":{"stringValue":"NpcoqqHtvcGdcAuanQB"}},{"key":"KeIcyDUeXybJkRu","value":{"stringValue":"LFgzuhAyDwshv"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"aOMOZO5S25I=","name":"PBKOonDx","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337322000","attributes":[{"key":"RTlSkMKMQggrNoe","value":{"stringValue":"vRdnXJwkz"}},{"key":"btubWcJIwUcKqVYtV","value":{"stringValue":"cnVcmabHbkSr"}},{"key":"YahOZNmsD","value":{"stringValue":"fFZBukcH"}},{"key":"RPAaeJrXJvBNUF","value":{"stringValue":"skOIxAHoJeaC"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"ipdVsjrgvo","value":{"stringValue":"AatKedQGkHagQwj"}},{"key":"LTkBFqnKgQeZYztsDx","value":{"stringValue":"pKsCyIpkTs"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"ZfpSOYtxNYhUVl","value":{"stringValue":"GJCmOYkAUmgq"}},{"key":"sxINTajwLoNZjQFgDz","value":{"stringValue":"COQjMbmYE"}},{"key":"caubQm","value":{"stringValue":"PUXMcpUbObnoF"}},{"key":"aoVhfoOTYfqTEN","value":{"stringValue":"FwZTuekCmzEcdBzSWQC"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"rCVHLLStoKXNKsmd","value":{"stringValue":"PWfagykGmiJbmgtpDOO"}},{"key":"tyVUhIskZgQSE","value":{"stringValue":"FhVFCjm"}},{"key":"YMBrbLSIeI","value":{"stringValue":"eyyjMVjbkxarpt"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"HtUKPYSlLCQ=","name":"vqRRfMGoxqI","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337348000","attributes":[{"key":"KaGXXwAZyMBLjYi","value":{"stringValue":"cWgCenOAqpiYq"}},{"key":"zvgzMzog","value":{"stringValue":"ZGdSEiAAPltpLgOL"}},{"key":"GpNcnxePZ","value":{"stringValue":"iUvGcVXvh"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"gnRdDnWKhlmjQvfKeQ","value":{"stringValue":"oskFdCwWPgFEbr"}},{"key":"ItdrPuHxYKpfDBL","value":{"stringValue":"rcHwuJAEOzJtIB"}},{"key":"MaOEMzbNTtXXIs","value":{"stringValue":"HHHomyiNebV"}},{"key":"dnHoCymRJJIOiqZzpTg","value":{"stringValue":"oxSOZen"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"MpxeMibfbGNZ","value":{"stringValue":"aVGUVhNAHaRJEVxgy"}},{"key":"jkarfXJhxJdl","value":{"stringValue":"kUfcUD"}},{"key":"HvJPPcseH","value":{"stringValue":"RZEbCz"}},{"key":"imaQljJlgezdxmAreco","value":{"stringValue":"ZsMuTnicHYRV"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"MJoTcuFRpa","value":{"stringValue":"imufrPbiL"}},{"key":"ysyzQWzBp","value":{"stringValue":"mJYMKYilJ"}},{"key":"GBtHQz","value":{"stringValue":"bpqUeDNAIOS"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"T9YWKv8JsCQ=","name":"vzdXGggFHqTGbcjC","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337329000","attributes":[{"key":"JpMdxmPSsurgrJlXo","value":{"stringValue":"YCfmjj"}},{"key":"edWOEDBwNfEfViImqZA","value":{"stringValue":"DRkVISOdpMWXlmsaNB"}},{"key":"pyzTyoWVa","value":{"stringValue":"WBgyjYTQAD"}},{"key":"DFAjDHJL","value":{"stringValue":"luxUErL"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"IUawxcrnpcbxAYzUyVQ","value":{"stringValue":"cmySqDaT"}},{"key":"JHJaKJWmqvqzDTTtMc","value":{"stringValue":"rhspZHJDrEmimmkCGk"}},{"key":"EqvkMe","value":{"stringValue":"izaejOtjlPXfu"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"gksgRTgzAfBB","value":{"stringValue":"xRgLgQRhF"}},{"key":"IzhzAeJgMkpnNnayTn","value":{"stringValue":"JbRUmNVcRtmMgTw"}},{"key":"SNOmGzJonxEeSJvY","value":{"stringValue":"AyWvATUBgeT"}},{"key":"SRuSEbwGhhjKZyHQS","value":{"stringValue":"KDzuDNYzluuuLkRa"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"vtkTsNBeIlNb","value":{"stringValue":"hWrdWPSzhID"}},{"key":"ieDnNygvMMGSfsGQ","value":{"stringValue":"UlhYOXTG"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"a6+jaViQM1k=","name":"qxdotAFdQgLCgItD","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337324000","attributes":[{"key":"IMXjVxJqxblDV","value":{"stringValue":"xmbmuWRgbDSoh"}},{"key":"ZJJexpD","value":{"stringValue":"bColsQLmoGLtK"}},{"key":"VYsFnHTzkxiL","value":{"stringValue":"kAriXZbs"}},{"key":"tleTnpjbfh","value":{"stringValue":"qDhzuKN"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"ZnoFgkeUUgzimkInvev","value":{"stringValue":"VCcBHdHDRIigpAjn"}},{"key":"STcHhJEyWRTGVAgw","value":{"stringValue":"qNybTZKnVgkV"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"rwhmtVEE","value":{"stringValue":"JvXljHIYEejwTBcIjFg"}},{"key":"GNRlcIQkIUpGiox","value":{"stringValue":"dzwlWVBXijS"}},{"key":"nlXTmrCZvnEk","value":{"stringValue":"GLKOREfHxGnTYJBX"}},{"key":"WqLxmfpWGLC","value":{"stringValue":"pcYXCcAnUFkbagdnWmx"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"Id6+/upuPOc=","name":"kMpwjjDzYesTlWmNH","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337337000","attributes":[{"key":"TbFHWp","value":{"stringValue":"rpAZuaVZuJfB"}},{"key":"kwVCllVouZTedHBeuhI","value":{"stringValue":"fYpaygHunLXCkv"}},{"key":"xDqGHamfZOBuX","value":{"stringValue":"SQApJJiGU"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"dwDiGplhwVQRsTM","value":{"stringValue":"tnHHrJPrIIrg"}},{"key":"NibjYVBXnNutEuxbU","value":{"stringValue":"KLqDQoGggVdnTIyS"}},{"key":"lTDEoYjLkvAMbyR","value":{"stringValue":"YymyozCsK"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"ohwZncz","value":{"stringValue":"xOhyqsIxuZ"}},{"key":"OrAuhaUQB","value":{"stringValue":"gADKcYpBnrLvfekUM"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"dsEUgtlrkKayP","value":{"stringValue":"VlVbriDfZAM"}},{"key":"rnPmtC","value":{"stringValue":"dRHSUgJOzyD"}},{"key":"CRnSTJEFUzLLBqmYPe","value":{"stringValue":"yZsdXEmrMPow"}},{"key":"xuvyjBzr","value":{"stringValue":"hDbTLzTQvhocHIfQ"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"XR0f6+E1nkE=","name":"FHfcJJLRpx","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337300000","attributes":[{"key":"bzalEE","value":{"stringValue":"mhUdvTJGaCgbQsls"}},{"key":"yuldufjRhuckjhzhh","value":{"stringValue":"pLVjLixrDnCpwGdrF"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"dMJTzNVaaWLWHD","value":{"stringValue":"IymAfO"}},{"key":"zxltnJMdSIMacj","value":{"stringValue":"FyxfeczITQPahV"}},{"key":"tHXzIvAmksBOiHEiy","value":{"stringValue":"DzypwwKtoUouBBr"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"MmjbERwhDhtr","value":{"stringValue":"EQyKEUGMCaYhxV"}},{"key":"TAwiUYR","value":{"stringValue":"FRHvdpaPbRVQbqA"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"jCIEJSbkKsMUr","value":{"stringValue":"WfmaTPIeCnlnBVMcN"}},{"key":"efNDJQRakoXWMkVuH","value":{"stringValue":"htzXOf"}},{"key":"LSWgcmz","value":{"stringValue":"krHmawFrlSQvji"}},{"key":"jSRVHRvQZEUQwWYDs","value":{"stringValue":"gjQIMhFCGbAawTldj"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"sYkgRN","value":{"stringValue":"RRxSle"}},{"key":"rfIyJywVxrw","value":{"stringValue":"dXLJzPbdxmPS"}},{"key":"QpWEMqfGavgZxCIc","value":{"stringValue":"TSjQOEtjP"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"I5vvoWQS7LY=","name":"UcjHpCyTZZEO","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337365000","attributes":[{"key":"xxiZyjMuTUxRt","value":{"stringValue":"BVokDixDkXMxvt"}},{"key":"ZbAXJWwhvhuWQpQ","value":{"stringValue":"XGtobhwYo"}},{"key":"OOuvlCZxoxKWpuXe","value":{"stringValue":"pvZiLpxAShCH"}},{"key":"tUhfFZbFtRlp","value":{"stringValue":"rRzfCAreuzGYNjA"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"wIUCVJvAjoDDcqyG","value":{"stringValue":"YDbmuTwFrf"}},{"key":"lloCbRWwPSvfsRk","value":{"stringValue":"LILGkcBH"}},{"key":"GeSUVrbQnZOQNY","value":{"stringValue":"eekSkRyTqsPgzM"}},{"key":"GwnPZnCC","value":{"stringValue":"FOnSGWCMtDgxOVAO"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"VbapmaZRmJTcwOR","value":{"stringValue":"unwbnvjulNyXoB"}},{"key":"TnRaqG","value":{"stringValue":"klESJEOaKWGdPZOxtiN"}},{"key":"eUpuzzYvgV","value":{"stringValue":"kWvpayHjmPh"}},{"key":"aLrSrJedOG","value":{"stringValue":"wLjxhVLinDyxDjHa"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"O9IcGLFx9KQ=","name":"bWJoGVLa","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337320000","attributes":[{"key":"ibklcE","value":{"stringValue":"GjIXkGiM"}},{"key":"WaQEmZuSg","value":{"stringValue":"fNxLiFakG"}},{"key":"liBpjrcexQYGq","value":{"stringValue":"qtrxxRoToWkiqRewFC"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"dnPVsZikCpaBlO","value":{"stringValue":"vBYOfBpweBFdz"}},{"key":"DeciFxYxLUPBuCOCUj","value":{"stringValue":"vYrwibeaWUVYnsb"}},{"key":"cHZgxoEQJTve","value":{"stringValue":"ydoZBOTTMo"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"VSKTAmIBv","value":{"stringValue":"OLkMPyr"}},{"key":"PzVQhjPhQztlrw","value":{"stringValue":"QmiJMiYTrdRYqLgem"}},{"key":"vmGLBTGZDBSdnq","value":{"stringValue":"WcgNTW"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"W8KNOLib+WE=","name":"WbMDPqmySqCS","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337303000","attributes":[{"key":"aSaElaONewo","value":{"stringValue":"vNYAmCzWHNFvi"}},{"key":"znftdOTWGPFG","value":{"stringValue":"JGVthfOTZaTOYeLjwA"}},{"key":"ACjTRDzzEUsPhwcwUPy","value":{"stringValue":"guGltMHjRXXiT"}},{"key":"ppLinNslkOjfwOed","value":{"stringValue":"WUhSSpYTyhzYQ"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"dSNrAyhFpa","value":{"stringValue":"SQULVqSVSklPQbBa"}},{"key":"fnmwAImoTTxDgCoOjQG","value":{"stringValue":"XTPRQSuVauabX"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"YezalmNzyctUEBNjC","value":{"stringValue":"WpaMBWpZFPqs"}},{"key":"OYvlcTpTfAKRpLVmQS","value":{"stringValue":"qhJiQav"}},{"key":"UkMbHQ","value":{"stringValue":"LYkMytnkqyHximPiyw"}},{"key":"bsIFuaVYdELoTSnA","value":{"stringValue":"KLaaGQNSpTjVWeVxqF"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"cikrabXODTg=","name":"JPDSjFbdFPjZO","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337331000","attributes":[{"key":"YWcOCmgWsiH","value":{"stringValue":"eMMSrPSOCouUShmxGa"}},{"key":"usakNZ","value":{"stringValue":"MfxGIvAr"}},{"key":"hMQPgpnH","value":{"stringValue":"wCxETdyyQZBWm"}},{"key":"okeFWh","value":{"stringValue":"aOrXDJklZzVnuwbd"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"nQPJPsdqGVFabr","value":{"stringValue":"PnNeQpLVGlZnm"}},{"key":"xHZiYu","value":{"stringValue":"hXLFTXPTcRV"}},{"key":"PexElIP","value":{"stringValue":"ziLQauRSL"}},{"key":"VXjwNbLG","value":{"stringValue":"wafJQpfAFqA"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"DZprHAhafyyfyuGRWj","value":{"stringValue":"CkhgQJGXEsAg"}},{"key":"jYVOPoilgApqCADqXq","value":{"stringValue":"thhVWl"}},{"key":"HQiPxlHQC","value":{"stringValue":"YDzpUgQeH"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"kOqLNchLfnybOEGPLX","value":{"stringValue":"ZRwQulUkTcSoZOX"}},{"key":"WNjKwwJUqcTDWPfmDm","value":{"stringValue":"gXCrTPvkBBYQCQgP"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"Kx5mK0yTpP8=","name":"DOvLaMGwFBzjaXRi","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337339000","attributes":[{"key":"ObmotpRuqwWmwRr","value":{"stringValue":"rnETFabBcNeWOe"}},{"key":"jgdQPaKacKBCcSVlQN","value":{"stringValue":"DpfTfJsxdajyZbuvley"}},{"key":"hqcmwUFby","value":{"stringValue":"ioQuohxFveBpaUnDJ"}},{"key":"jhYgTWSNMdlyAKEV","value":{"stringValue":"IgBLtSDuinbkkAgi"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"BkryibdLWCLOY","value":{"stringValue":"VGBaWVpXOSxNmN"}},{"key":"EZuMQqFccs","value":{"stringValue":"OeuezLnDkmKthApy"}},{"key":"vPkBBhcDWhy","value":{"stringValue":"yRWFJfNTUqxPzCOHUWQ"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"cTCLQrgXi","value":{"stringValue":"iWjWYLnQ"}},{"key":"aAIqDnhexZrbrycU","value":{"stringValue":"LLFztZBuggliCs"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"UUMPEpXjLREghIkZeVR","value":{"stringValue":"SqRCGssgsSHcGU"}},{"key":"HIFzawK","value":{"stringValue":"oJQyIVSI"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"NZEuaIMkjC4=","name":"vQnGxF","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337316000","attributes":[{"key":"inseWvepd","value":{"stringValue":"osUwTjIaSjrvm"}},{"key":"wfdBlCDisM","value":{"stringValue":"qdHxUveWyhhesEUZCB"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"QzihGYLzmiou","value":{"stringValue":"UFggwkTXMZKnRsebfN"}},{"key":"uUiKqVqfwl","value":{"stringValue":"bOpmmCuTQOQUXVvd"}},{"key":"qqctPfUnJdnuECpr","value":{"stringValue":"EjuOuLICxrrJC"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"wluVpXTxDydfGBxm","value":{"stringValue":"qzUorsGUXpUU"}},{"key":"RYnFKYCYCCKCQ","value":{"stringValue":"hSEMYNrUCTTwajF"}},{"key":"RWbtDzI","value":{"stringValue":"eHljxUrsn"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"UHaVZVlXRsI=","name":"jljRoPdmVHWZh","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337374000","attributes":[{"key":"PwIaAzctDbciPAkPBb","value":{"stringValue":"sdXfHGCY"}},{"key":"LskeqiQkG","value":{"stringValue":"zVDsmgZrgphPwU"}},{"key":"MRlaOPVNjT","value":{"stringValue":"bOvlHLyRMqyZjbiwYd"}},{"key":"vCINaw","value":{"stringValue":"YbTIqdYtr"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"zSoYOLyNd","value":{"stringValue":"NuqeBGSiLmjWfMjJP"}},{"key":"PTfLqxtlTIoxjl","value":{"stringValue":"UCvKirFyLSUxKo"}},{"key":"gIufAXnj","value":{"stringValue":"gtYHUEkdtzsJusPr"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"XcjgkV","value":{"stringValue":"sIvmnKioYG"}},{"key":"MfJmeFoRMUTiMMRo","value":{"stringValue":"jpapHOnwUMyIXjsplau"}},{"key":"vUSsGFBYfnFO","value":{"stringValue":"lTDuKRzllBJTkBkFzp"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"RheqSgdUd","value":{"stringValue":"niDJUgZ"}},{"key":"jTQVQBCkXozeFsFl","value":{"stringValue":"BTRXNXcJiuhD"}},{"key":"MRuEBXQmHOI","value":{"stringValue":"FGgliqZ"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"Y8DFJ8c1HMI=","name":"XIuxvmeD","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337371000","attributes":[{"key":"fnafbsD","value":{"stringValue":"oGhTUbZ"}},{"key":"dncSHCdpfBI","value":{"stringValue":"rTWHywlyI"}},{"key":"RYQUcnglzJew","value":{"stringValue":"beJIAoRxlJjEWNsWZ"}},{"key":"vIoFRlJMSep","value":{"stringValue":"FDHocmPNss"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"ZvBqBBGpAtBPij","value":{"stringValue":"LgFEDmgKY"}},{"key":"aMgXDISBTz","value":{"stringValue":"vXBnRCKqzSQeOkY"}},{"key":"usmVzs","value":{"stringValue":"BulmAwOMSnockvZN"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"ADBfwsAzf","value":{"stringValue":"lWRAGFlVWodY"}},{"key":"LMRLmJLGh","value":{"stringValue":"yemWvguWOqOuOf"}},{"key":"abVisbylppTgENlo","value":{"stringValue":"saiTqYFpFpVEuwQ"}},{"key":"ltpJLiAA","value":{"stringValue":"HVRbfdWTtPPtIIjgTi"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"uttxYSUBtHmI","value":{"stringValue":"kDbUqouyMjVDKvEQ"}},{"key":"ZmYtDkQnewdI","value":{"stringValue":"EzGvgoFirMZtWx"}},{"key":"LvWTxf","value":{"stringValue":"VtgMDPKgOeNOGy"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"EnqojOajVWKHXcX","value":{"stringValue":"gpHwCPSYZIjozBnkxH"}},{"key":"MvlgaCPpOt","value":{"stringValue":"yNTBqxhqJUKAcBvqw"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"Lba2V9BwTNo=","name":"YitmjKgomwmSzzfCcvz","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337339000","attributes":[{"key":"RrjrtBCxn","value":{"stringValue":"KglyEl"}},{"key":"AseomFRQJKdGLWzdg","value":{"stringValue":"vgDhrZPmXhuSLvebxM"}},{"key":"DltyLFasoQq","value":{"stringValue":"ZwsSHKoePyPzqfp"}},{"key":"mWzkAa","value":{"stringValue":"sZCvwmAnBrWgaDikQ"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"isCYAFCcAKmx","value":{"stringValue":"FUAtPAxN"}},{"key":"cGHuTw","value":{"stringValue":"OujyUEFSCUmdl"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"JFqRlrrXD","value":{"stringValue":"ySNWEnQJWXXQsxgcH"}},{"key":"qioCpznx","value":{"stringValue":"SQlrKYpFA"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"vAnfzHtOBeXaOqS","value":{"stringValue":"RNvPvnFHnPS"}},{"key":"rjawmnRqAYfoOLbTJo","value":{"stringValue":"VlKcYxSa"}},{"key":"ZQMZCqEQo","value":{"stringValue":"JwvTGKIcAANlmQGHd"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"aqhuvbr+LUA=","name":"KmLUDlbLBLHEYnIia","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337372000","attributes":[{"key":"oHcQJPDmZgIo","value":{"stringValue":"KvgiZPtun"}},{"key":"WMawQnQ","value":{"stringValue":"oFaXONFIaPWZfds"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"upkFUoLPbbKuJGl","value":{"stringValue":"hNTGrSt"}},{"key":"iJtOeCgiUWLVDZlCAH","value":{"stringValue":"tLzyukdRx"}},{"key":"mOiAklAfoblFBBmblH","value":{"stringValue":"UaNQCaGQUxsYY"}},{"key":"LsdzVILMoZnwlo","value":{"stringValue":"LCAeezURhB"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"BOdNIKWvPKMsrO","value":{"stringValue":"TYMWsZHtgOrgOTF"}},{"key":"XbPbqLdqgGRFaK","value":{"stringValue":"QMtcfUocE"}},{"key":"XvwcpYQbZdQqzyJ","value":{"stringValue":"seyMkCW"}},{"key":"mKHCvhulFdJoIaw","value":{"stringValue":"MVSInWLpIfPiP"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"TxrifWlsfz","value":{"stringValue":"CrqOonrf"}},{"key":"bWJcWu","value":{"stringValue":"pQGKHzXGxn"}},{"key":"FzMluQBNbImp","value":{"stringValue":"RRofwdhmk"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"PfkqmgX","value":{"stringValue":"adSVfYjQybh"}},{"key":"UnlNkRYILCNhi","value":{"stringValue":"SEbmvrRM"}},{"key":"xVvPNDKSh","value":{"stringValue":"LRtzTO"}},{"key":"mMjVbifBh","value":{"stringValue":"AlZlcIqa"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"Nl7C8oVyzvw=","name":"SNtuBXLuzULEzFea","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337342000","attributes":[{"key":"ZKaGmauNRRa","value":{"stringValue":"UvtWWkgWDgaFVHKT"}},{"key":"qzOqPmlfPWwCeIejJF","value":{"stringValue":"LtSDaJPEkVEjTyc"}},{"key":"PbFIXUgUU","value":{"stringValue":"iJuguaRwVWqgx"}},{"key":"uCOafRazuLpTuIjiuDf","value":{"stringValue":"MoIIWNKtY"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"KFLgqQVzRpmmO","value":{"stringValue":"oPnzHCGXtWlgppn"}},{"key":"gusRzLcUnfg","value":{"stringValue":"MPQamlYoeea"}},{"key":"xNIWya","value":{"stringValue":"OWpzEGImgmPvSbeX"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"koDPDtxgEAwvbel","value":{"stringValue":"HKktYTpxlLfCfWOlLEd"}},{"key":"SbvwlkkqPjCJ","value":{"stringValue":"zdEdXBv"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"fLmRaTNmhermhYv","value":{"stringValue":"ChMJrBPnyIYFWFo"}},{"key":"XxUakhiohBYXv","value":{"stringValue":"EtJPosIHZEEgZzakrid"}},{"key":"KyqmAXtyprhK","value":{"stringValue":"CEIelnLtyxyOR"}},{"key":"NTSJClVCfDFXZc","value":{"stringValue":"gTYuSeY"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"MfvjZfdQwkyhSx","value":{"stringValue":"zbzgWWUjbYsyhaz"}},{"key":"QSNfIpCZkRplBYNj","value":{"stringValue":"kuCbzjDKkYIybmgb"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"PZUmRoCdEG8=","name":"ZUOjlXiBblakolDwtRS","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337382000","attributes":[{"key":"IrssEEdaGlN","value":{"stringValue":"PKabHzrCzk"}},{"key":"QHtKZiA","value":{"stringValue":"KqRMRXD"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"bhANVGXiKvgcTw","value":{"stringValue":"fDvZxscLCr"}},{"key":"LuSXQNFGJ","value":{"stringValue":"RMTCNHiPTxnjzUR"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"ABYXMHnUcl","value":{"stringValue":"AaBEodRwcoq"}},{"key":"ztJLvr","value":{"stringValue":"ORSuEKKuibEAVbVJt"}},{"key":"UvJvzrANdmmKIy","value":{"stringValue":"kNPJPhjvcsYczbxmqu"}},{"key":"uzUpFmPsYCQLGQKrlAE","value":{"stringValue":"ukEppgOKEJVAiW"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"agMkChDIluw=","name":"PmaWiqarhHqzl","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337329000","attributes":[{"key":"EEPVLjJNSZTbMl","value":{"stringValue":"WUCVQDJY"}},{"key":"aKbLHmhWOgvkrBrjiPF","value":{"stringValue":"FGciGTKYqIWUrtOHq"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"ySjUjntVBr","value":{"stringValue":"vaIybhRB"}},{"key":"KcsMPsjAGGIq","value":{"stringValue":"UACtdHiJYDcWtKti"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"GKBTIIUGzqDUbujrvZ","value":{"stringValue":"FXrVKtqCjKhWCt"}},{"key":"EraADLLZOxjIsmgQj","value":{"stringValue":"XNJbECPFPSkeJjQ"}},{"key":"psCaOUNqljX","value":{"stringValue":"VtXSQcerctaYVHMLgsK"}},{"key":"lFEvdJ","value":{"stringValue":"RQsIYRhCMtu"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"jgdoRUncpzqi","value":{"stringValue":"qUwSnKQZxjrdHf"}},{"key":"lOmjkJmtBIKneqyUdS","value":{"stringValue":"rauNZVkALdVnS"}},{"key":"ZxASzlnJikwlBHbuIye","value":{"stringValue":"qAPGIBruBQNkCpDNoW"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"JLeATuqfQhrlCqEIIy","value":{"stringValue":"VwmaqqmAsJZECkur"}},{"key":"COcoxImLqJpDfXBnn","value":{"stringValue":"FtBUQQrKTckohjQGP"}},{"key":"fXVpIVGnsyCTsgwHw","value":{"stringValue":"OAYglQwkwWA"}},{"key":"oGFgiewpjcsoVSQL","value":{"stringValue":"pcwjGgJKGLEkH"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"PQAmBBtIZdI=","name":"TvsRHt","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337340000","attributes":[{"key":"uwpbpKHiSI","value":{"stringValue":"iuuPimYhBiHegTpkg"}},{"key":"XUyZTQMOcxQ","value":{"stringValue":"qeBHfsMbJuj"}},{"key":"WnzimAAYJZcghC","value":{"stringValue":"sqrFWExr"}},{"key":"hlNpRpzuLszIdKuTgo","value":{"stringValue":"APgyraxGK"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"CxLEkMJGsuihQI","value":{"stringValue":"gAiuzMCXuzzTrSz"}},{"key":"XxhxmzhQAgAkGFHVJUt","value":{"stringValue":"Hexmfsw"}},{"key":"rMpHfwW","value":{"stringValue":"mSyTPeN"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"lhhMoBOakq","value":{"stringValue":"DIIohkMLfVuvcH"}},{"key":"oeLOZBOPGW","value":{"stringValue":"FYnzSQRwT"}},{"key":"lBmHzFsyFSqd","value":{"stringValue":"LPtxIDetfVLChxEl"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"oKMjqQxgrZseUDvDwGr","value":{"stringValue":"uXMslttsniGppFSFHN"}},{"key":"RCSKrggEybWM","value":{"stringValue":"onWksrmLoKnwaTaIxuq"}},{"key":"kJSSaPJakvtIlQtYkC","value":{"stringValue":"FqiRbOCXkwAVsw"}},{"key":"mPUBQjhl","value":{"stringValue":"ullcfcUDtfxvozFdh"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"dSukzECaKmY=","name":"rJahDnlEAIR","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337333000","attributes":[{"key":"UdxyidhgvaL","value":{"stringValue":"sAXQwqDdCf"}},{"key":"gVWLhzIpepiyNbYH","value":{"stringValue":"UpOMZxOloDEd"}},{"key":"wGiMTiOpijrMC","value":{"stringValue":"EBCmzAgUVZclXvbTNY"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"puzvLAAJ","value":{"stringValue":"UxuhiO"}},{"key":"XIlIeDSegVeXzhHlD","value":{"stringValue":"qcXHtmHPI"}},{"key":"pKsuXE","value":{"stringValue":"GcEdIPjNFN"}},{"key":"UNRrfcJjlSGa","value":{"stringValue":"UBhgiQcwN"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"CbYfWCG","value":{"stringValue":"SCKRBfQMYxaEzw"}},{"key":"ypZvrqmYHKqw","value":{"stringValue":"pLDHLDAeogh"}},{"key":"cnEwQfluHESZJluXfMl","value":{"stringValue":"BdvsJEoMrNYPLzO"}},{"key":"pPbpzUkfciPMb","value":{"stringValue":"XoWAqPNMejIwq"}}]}],"status":{}}]}]},{"resource":{},"instrumentationLibrarySpans":[{"instrumentationLibrary":{},"spans":[{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"QWmmdGlBzho=","name":"pEyulMxxM","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337336000","attributes":[{"key":"NlLWNfbBHeiqqdYB","value":{"stringValue":"myaPlRIjqgJqiUVV"}},{"key":"KIYOArL","value":{"stringValue":"OhommiJqMUHbHQb"}},{"key":"YCXtKezls","value":{"stringValue":"tpPefGmRKEm"}},{"key":"fkGOSuCdndWkVPCz","value":{"stringValue":"fCpEzvckQHtYIDSrWgj"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"RlFXsehxRfxVmhXS","value":{"stringValue":"JPwjsdqUjtbiYVP"}},{"key":"tLZVVLwM","value":{"stringValue":"COTKbKwnsS"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"UfWlHf","value":{"stringValue":"MhRKbDIRWdeppMIv"}},{"key":"ilprteZJyYpb","value":{"stringValue":"XzekXOTMfkMhcQRgpm"}},{"key":"TOqEcsbSjjCDHwbPth","value":{"stringValue":"zznRGmxqFaQPLOvq"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"NynRtofwSXQAJRpuT","value":{"stringValue":"jeJfUPpcEYpWOTzV"}},{"key":"kJTtbVeD","value":{"stringValue":"nPVBBPbWirC"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"zVjRXdHc","value":{"stringValue":"UCYrKzV"}},{"key":"ZFmRbEtKsQLNCsp","value":{"stringValue":"YUzbzCTTfYCI"}},{"key":"LmyAVNzjGqwtpzaZYdB","value":{"stringValue":"bzgGRhLBdEteuuPZU"}}]}],"status":{}},{"traceId":"E+T0Wj7NYJAM7fMtkt6jRg==","spanId":"Z2M4gdZaS84=","name":"gMbLmM","startTimeUnixNano":"1630337285000","endTimeUnixNano":"1630337313000","attributes":[{"key":"pUnVctxZDTeCN","value":{"stringValue":"jgvBAqQVylReRPd"}},{"key":"YWlOPszdyTA","value":{"stringValue":"TfNPQWBpRh"}}],"events":[{"timeUnixNano":"1630337285000","attributes":[{"key":"sRKGlIOeJLzCclGq","value":{"stringValue":"kcgIKgMTqVQ"}},{"key":"qYHvMwmT","value":{"stringValue":"iOFsgDDYOHMvHiWYvj"}}]},{"timeUnixNano":"1630337285000","attributes":[{"key":"hRnXqUMG","value":{"stringValue":"uyloKxPqkzfWGHvFkpU"}},{"key":"rZXLaNKJetkRrdR","value":{"stringValue":"jqEWFYT"}}]}],"status":{}}]}]}]} \ No newline at end of file From c9d6ce77f7ddf5648491f682166cde6afc40047b Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Mon, 30 Aug 2021 10:30:52 -0700 Subject: [PATCH 08/14] Reuse code for converting thrift.Batch to tempopb.Trace --- cmd/tempo-vulture/main.go | 112 +++++------------- cmd/tempo-vulture/main_test.go | 17 --- .../cmd/agent/app/reporter/grpc/reporter.go | 1 + 3 files changed, 30 insertions(+), 100 deletions(-) diff --git a/cmd/tempo-vulture/main.go b/cmd/tempo-vulture/main.go index e7d06388ce6..7f0971f2388 100644 --- a/cmd/tempo-vulture/main.go +++ b/cmd/tempo-vulture/main.go @@ -3,7 +3,6 @@ package main import ( "bytes" "context" - "encoding/hex" "flag" "fmt" "log" @@ -17,16 +16,14 @@ import ( "github.com/go-test/deep" "github.com/grafana/tempo/pkg/model" "github.com/grafana/tempo/pkg/tempopb" - v1common "github.com/grafana/tempo/pkg/tempopb/common/v1" - v1resource "github.com/grafana/tempo/pkg/tempopb/resource/v1" - v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1" "github.com/grafana/tempo/pkg/util" jaeger_grpc "github.com/jaegertracing/jaeger/cmd/agent/app/reporter/grpc" - "github.com/jaegertracing/jaeger/thrift-gen/jaeger" thrift "github.com/jaegertracing/jaeger/thrift-gen/jaeger" zaplogfmt "github.com/jsternberg/zap-logfmt" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/weaveworks/common/user" + jaegerTrans "go.opentelemetry.io/collector/translator/trace/jaeger" "go.uber.org/zap" "go.uber.org/zap/zapcore" "google.golang.org/grpc" @@ -261,82 +258,6 @@ func makeThriftBatch(TraceIDHigh int64, TraceIDLow int64, r *rand.Rand, now time return &thrift.Batch{Spans: spans} } -func jaegerBatchToPbTrace(batch *jaeger.Batch) *tempopb.Trace { - trace := &tempopb.Trace{ - Batches: []*v1.ResourceSpans{}, - } - libs := []*v1.InstrumentationLibrarySpans{ - { - InstrumentationLibrary: &v1common.InstrumentationLibrary{}, - }, - } - - for _, s := range batch.Spans { - traceIDHex := fmt.Sprintf("%016x%016x", s.TraceIdHigh, s.TraceIdLow) - traceID, err := util.HexStringToTraceID(traceIDHex) - if err != nil { - logger.Error(err.Error()) - } - - spanIDHex := fmt.Sprintf("%016x", s.SpanId) - spanID, err := hex.DecodeString(spanIDHex) - if err != nil { - logger.Error(err.Error()) - } - - startTime := time.Unix(s.StartTime, 0) - stopTime := startTime.Add(time.Duration(s.Duration) * time.Second) - - span := &v1.Span{ - TraceId: traceID, - SpanId: spanID, - Name: s.OperationName, - // TODO use time.UnixMilli() when upgraded to Go 1.17 - StartTimeUnixNano: uint64(startTime.UnixNano() / int64(time.Millisecond)), - EndTimeUnixNano: uint64(stopTime.UnixNano() / int64(time.Millisecond)), - Status: &v1.Status{}, - } - - for _, tag := range s.Tags { - span.Attributes = append(span.Attributes, &v1common.KeyValue{ - Key: tag.Key, - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: *tag.VStr}, - }, - }) - } - - for _, event := range s.Logs { - attrs := []*v1common.KeyValue{} - - for _, tag := range event.Fields { - attrs = append(attrs, &v1common.KeyValue{ - Key: tag.Key, - Value: &v1common.AnyValue{ - Value: &v1common.AnyValue_StringValue{StringValue: *tag.VStr}, - }, - }) - } - - t := time.Unix(event.Timestamp, 0) - - span.Events = append(span.Events, &v1.Span_Event{ - // TODO use time.UnixMilli() when upgraded to Go 1.17 - TimeUnixNano: uint64(t.UnixNano() / int64(time.Millisecond)), - Attributes: attrs, - }) - } - libs[0].Spans = append(libs[0].Spans, span) - } - - trace.Batches = append(trace.Batches, &v1.ResourceSpans{ - Resource: &v1resource.Resource{}, - InstrumentationLibrarySpans: libs, - }) - - return trace -} - func generateRandomInt(min int64, max int64, r *rand.Rand) int64 { min++ number := min + r.Int63n(max-min) @@ -449,8 +370,33 @@ func constructTraceFromEpoch(epoch time.Time) *tempopb.Trace { for i := int64(0); i < generateRandomInt(1, 100, r); i++ { batch := makeThriftBatch(traceIDHigh, traceIDLow, r, epoch) - result := jaegerBatchToPbTrace(batch) - trace.Batches = append(trace.Batches, result.Batches...) + internalTrace := jaegerTrans.ThriftBatchToInternalTraces(batch) + conv, err := internalTrace.ToOtlpProtoBytes() + if err != nil { + logger.Error(err.Error()) + } + + t := tempopb.Trace{} + err = t.Unmarshal(conv) + if err != nil { + logger.Error(err.Error()) + } + + // Due to the several transforms above, some manual mangling is required to + // get the parentSpanID to match. In the case of an empty []byte in place + // for the ParentSpanId, we set to nil here to ensure that the final result + // matches the json.Unmarshal value when tempo is queried. + for ib, b := range t.Batches { + for il, l := range b.InstrumentationLibrarySpans { + for is, s := range l.Spans { + if len(s.GetParentSpanId()) == 0 { + t.Batches[ib].InstrumentationLibrarySpans[il].Spans[is].ParentSpanId = nil + } + } + } + } + + trace.Batches = append(trace.Batches, t.Batches...) } return trace diff --git a/cmd/tempo-vulture/main_test.go b/cmd/tempo-vulture/main_test.go index 12628d7e2c1..78510aa431a 100644 --- a/cmd/tempo-vulture/main_test.go +++ b/cmd/tempo-vulture/main_test.go @@ -243,23 +243,6 @@ func TestIntervalsBetween(t *testing.T) { } } -func TestEqualTraces(t *testing.T) { - now := time.Now() - - require.Equal(t, newRand(now).Int(), newRand(now).Int()) - - r1 := newRand(now) - batch1 := makeThriftBatch(r1.Int63(), r1.Int63(), r1, now) - pb1 := jaegerBatchToPbTrace(batch1) - - r2 := newRand(now) - batch2 := makeThriftBatch(r2.Int63(), r2.Int63(), r2, now) - pb2 := jaegerBatchToPbTrace(batch2) - - require.Equal(t, pb1, pb2) - require.True(t, equalTraces(pb1, pb2)) -} - func TestNewRand(t *testing.T) { now := time.Now() diff --git a/vendor/github.com/jaegertracing/jaeger/cmd/agent/app/reporter/grpc/reporter.go b/vendor/github.com/jaegertracing/jaeger/cmd/agent/app/reporter/grpc/reporter.go index 217c5ae42f2..39032efffea 100644 --- a/vendor/github.com/jaegertracing/jaeger/cmd/agent/app/reporter/grpc/reporter.go +++ b/vendor/github.com/jaegertracing/jaeger/cmd/agent/app/reporter/grpc/reporter.go @@ -49,6 +49,7 @@ func NewReporter(conn *grpc.ClientConn, agentTags map[string]string, logger *zap } // EmitBatch implements EmitBatch() of Reporter + func (r *Reporter) EmitBatch(ctx context.Context, b *thrift.Batch) error { return r.send(ctx, jConverter.ToDomain(b.Spans, nil), jConverter.ToDomainProcess(b.Process)) } From d219dbdfb591d61266ae47324321390c340c73bc Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Mon, 30 Aug 2021 13:17:41 -0700 Subject: [PATCH 09/14] Choose interval times only between start and stop --- cmd/tempo-vulture/main.go | 4 +++- cmd/tempo-vulture/main_test.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/tempo-vulture/main.go b/cmd/tempo-vulture/main.go index 7f0971f2388..753eb54d99b 100644 --- a/cmd/tempo-vulture/main.go +++ b/cmd/tempo-vulture/main.go @@ -159,7 +159,9 @@ func intervalsBetween(start, stop time.Time, interval time.Duration, retention t next := start.Round(interval) for next.Before(stop) { - intervals = append(intervals, next) + if next.After(start) { + intervals = append(intervals, next) + } next = next.Add(interval) } diff --git a/cmd/tempo-vulture/main_test.go b/cmd/tempo-vulture/main_test.go index 78510aa431a..3fc3476b267 100644 --- a/cmd/tempo-vulture/main_test.go +++ b/cmd/tempo-vulture/main_test.go @@ -221,7 +221,7 @@ func TestIntervalsBetween(t *testing.T) { stop: nowish, interval: 11 * time.Second, retention: 1 * time.Hour, - count: 7, + count: 6, }, { start: nowish.Add(-1 * time.Hour), From e5095a94535f98a067db80154178db16e421f7bb Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Mon, 30 Aug 2021 13:35:55 -0700 Subject: [PATCH 10/14] Adjust method usage for simplicity --- cmd/tempo-vulture/main.go | 15 +++++++++------ .../cmd/agent/app/reporter/grpc/reporter.go | 1 - 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cmd/tempo-vulture/main.go b/cmd/tempo-vulture/main.go index 753eb54d99b..3cd33629938 100644 --- a/cmd/tempo-vulture/main.go +++ b/cmd/tempo-vulture/main.go @@ -122,12 +122,8 @@ func main() { // Read go func() { for now := range tickerRead.C { - intervals := intervalsBetween(startTime, now, interval, tempoRetentionDuration) - startTime = intervals[0] - - // pick past interval and re-generate trace - pick := generateRandomInt(0, int64(len(intervals)), newRand(now)) - seed := intervals[pick] + var seed time.Time + startTime, seed = selectPastTimestamp(startTime, now, interval, tempoRetentionDuration) r := newRand(seed) hexID := fmt.Sprintf("%016x%016x", r.Int63(), r.Int63()) @@ -150,6 +146,13 @@ func main() { log.Fatal(http.ListenAndServe(prometheusListenAddress, nil)) } +func selectPastTimestamp(start, stop time.Time, interval time.Duration, retention time.Duration) (newStart, ts time.Time) { + intervals := intervalsBetween(start, stop, interval, retention) + // pick past interval and re-generate trace + pick := generateRandomInt(0, int64(len(intervals)), newRand(intervals[0])) + return intervals[0], intervals[pick] +} + func intervalsBetween(start, stop time.Time, interval time.Duration, retention time.Duration) []time.Time { if stop.Before(start) { return nil diff --git a/vendor/github.com/jaegertracing/jaeger/cmd/agent/app/reporter/grpc/reporter.go b/vendor/github.com/jaegertracing/jaeger/cmd/agent/app/reporter/grpc/reporter.go index 39032efffea..217c5ae42f2 100644 --- a/vendor/github.com/jaegertracing/jaeger/cmd/agent/app/reporter/grpc/reporter.go +++ b/vendor/github.com/jaegertracing/jaeger/cmd/agent/app/reporter/grpc/reporter.go @@ -49,7 +49,6 @@ func NewReporter(conn *grpc.ClientConn, agentTags map[string]string, logger *zap } // EmitBatch implements EmitBatch() of Reporter - func (r *Reporter) EmitBatch(ctx context.Context, b *thrift.Batch) error { return r.send(ctx, jConverter.ToDomain(b.Spans, nil), jConverter.ToDomainProcess(b.Process)) } From 15de0e5f644c11cf6f112d417e938ce3e3e63047 Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Wed, 1 Sep 2021 06:25:53 -0700 Subject: [PATCH 11/14] Return early to avoid incrementing multiple metrics on failure --- cmd/tempo-vulture/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/tempo-vulture/main.go b/cmd/tempo-vulture/main.go index 3cd33629938..6c2709e981f 100644 --- a/cmd/tempo-vulture/main.go +++ b/cmd/tempo-vulture/main.go @@ -296,12 +296,14 @@ func queryTempoAndAnalyze(baseURL string, seed time.Time, traceID string) (trace if len(trace.Batches) == 0 { logger.Error("trace contains 0 batches") tm.notFound++ + return tm, nil } // iterate through if hasMissingSpans(trace) { logger.Error("trace has missing spans") tm.missingSpans++ + return tm, nil } // Get the expected @@ -317,6 +319,7 @@ func queryTempoAndAnalyze(baseURL string, seed time.Time, traceID string) (trace ) } } + return tm, nil } return tm, nil From 95668986e59aae5cb0a2cd63ff752d5e61e481cd Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Wed, 1 Sep 2021 07:14:03 -0700 Subject: [PATCH 12/14] Avoid reading before we expect writing to have taken place --- cmd/tempo-vulture/main.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/tempo-vulture/main.go b/cmd/tempo-vulture/main.go index 6c2709e981f..c9113ede41b 100644 --- a/cmd/tempo-vulture/main.go +++ b/cmd/tempo-vulture/main.go @@ -75,7 +75,8 @@ func main() { logger.Info("Tempo Vulture starting") - startTime := time.Now() + actualStartTime := time.Now() + startTime := actualStartTime tickerWrite := time.NewTicker(tempoWriteBackoffDuration) tickerRead := time.NewTicker(tempoReadBackoffDuration) interval := tempoWriteBackoffDuration @@ -125,6 +126,12 @@ func main() { var seed time.Time startTime, seed = selectPastTimestamp(startTime, now, interval, tempoRetentionDuration) + // Don't attempt to read on the first itteration if we can't reasonably + // expect the write loop to have fired yet. + if seed.Before(actualStartTime.Add(tempoWriteBackoffDuration)) { + continue + } + r := newRand(seed) hexID := fmt.Sprintf("%016x%016x", r.Int63(), r.Int63()) From 0e781c12a4e3b5ce0c862705074bcf9ac8d75fc5 Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Wed, 1 Sep 2021 08:48:30 -0700 Subject: [PATCH 13/14] Update interval selection for brevity --- cmd/tempo-vulture/main.go | 42 ++++++++++------------------------ cmd/tempo-vulture/main_test.go | 37 ------------------------------ 2 files changed, 12 insertions(+), 67 deletions(-) diff --git a/cmd/tempo-vulture/main.go b/cmd/tempo-vulture/main.go index c9113ede41b..bbf6b39b551 100644 --- a/cmd/tempo-vulture/main.go +++ b/cmd/tempo-vulture/main.go @@ -132,6 +132,11 @@ func main() { continue } + // Don't attempt to read future traces. + if seed.After(now) { + continue + } + r := newRand(seed) hexID := fmt.Sprintf("%016x%016x", r.Int63(), r.Int63()) @@ -154,40 +159,17 @@ func main() { } func selectPastTimestamp(start, stop time.Time, interval time.Duration, retention time.Duration) (newStart, ts time.Time) { - intervals := intervalsBetween(start, stop, interval, retention) - // pick past interval and re-generate trace - pick := generateRandomInt(0, int64(len(intervals)), newRand(intervals[0])) - return intervals[0], intervals[pick] -} + oldest := stop.Add(-retention) -func intervalsBetween(start, stop time.Time, interval time.Duration, retention time.Duration) []time.Time { - if stop.Before(start) { - return nil + if oldest.After(start) { + newStart = oldest + } else { + newStart = start } - intervals := []time.Time{start} - next := start.Round(interval) - - for next.Before(stop) { - if next.After(start) { - intervals = append(intervals, next) - } - next = next.Add(interval) - } - - oldest := intervals[len(intervals)-1].Add(-retention) - - for i, t := range intervals { - if t.Before(oldest) { - continue - } - - if t.After(oldest) { - return intervals[i:] - } - } + ts = time.Unix(generateRandomInt(newStart.Unix(), stop.Unix(), newRand(start)), 0) - return intervals + return newStart.Round(interval), ts.Round(interval) } func newJaegerGRPCClient(endpoint string) (*jaeger_grpc.Reporter, error) { diff --git a/cmd/tempo-vulture/main_test.go b/cmd/tempo-vulture/main_test.go index 3fc3476b267..c4115439669 100644 --- a/cmd/tempo-vulture/main_test.go +++ b/cmd/tempo-vulture/main_test.go @@ -206,43 +206,6 @@ func TestGenerateRandomLogs(t *testing.T) { require.Equal(t, expected, result) } -func TestIntervalsBetween(t *testing.T) { - nowish := time.Unix(1630010049, 0) - - cases := []struct { - start time.Time - stop time.Time - interval time.Duration - retention time.Duration - count int - }{ - { - start: nowish.Add(-1 * time.Minute), - stop: nowish, - interval: 11 * time.Second, - retention: 1 * time.Hour, - count: 6, - }, - { - start: nowish.Add(-1 * time.Hour), - stop: nowish, - interval: 33 * time.Second, - retention: 1 * time.Hour, - count: 110, - }, - } - - for _, tc := range cases { - result := intervalsBetween(tc.start, tc.stop, tc.interval, tc.retention) - require.Equal(t, tc.count, len(result)) - - if tc.count > 0 { - require.Equal(t, tc.start, result[0]) - require.True(t, result[len(result)-1].Before(tc.stop)) - } - } -} - func TestNewRand(t *testing.T) { now := time.Now() From a119dc2c35a084ee7216cb159b4eda9d083b5fd5 Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Thu, 2 Sep 2021 09:09:43 -0700 Subject: [PATCH 14/14] Tidy up for PR feedback --- cmd/tempo-vulture/main.go | 8 ++++---- cmd/tempo-vulture/main_test.go | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd/tempo-vulture/main.go b/cmd/tempo-vulture/main.go index bbf6b39b551..3f1f3207fe4 100644 --- a/cmd/tempo-vulture/main.go +++ b/cmd/tempo-vulture/main.go @@ -383,11 +383,11 @@ func constructTraceFromEpoch(epoch time.Time) *tempopb.Trace { // get the parentSpanID to match. In the case of an empty []byte in place // for the ParentSpanId, we set to nil here to ensure that the final result // matches the json.Unmarshal value when tempo is queried. - for ib, b := range t.Batches { - for il, l := range b.InstrumentationLibrarySpans { - for is, s := range l.Spans { + for _, b := range t.Batches { + for _, l := range b.InstrumentationLibrarySpans { + for _, s := range l.Spans { if len(s.GetParentSpanId()) == 0 { - t.Batches[ib].InstrumentationLibrarySpans[il].Spans[is].ParentSpanId = nil + s.ParentSpanId = nil } } } diff --git a/cmd/tempo-vulture/main_test.go b/cmd/tempo-vulture/main_test.go index c4115439669..de777ddd6f3 100644 --- a/cmd/tempo-vulture/main_test.go +++ b/cmd/tempo-vulture/main_test.go @@ -253,4 +253,11 @@ func TestResponseFixture(t *testing.T) { } } +func TestEqualTraces(t *testing.T) { + seed := time.Now() + a := constructTraceFromEpoch(seed) + b := constructTraceFromEpoch(seed) + require.True(t, equalTraces(a, b)) +} + func stringPointer(s string) *string { return &s }