From c751c2dfbe200960218867e924b72545c0bb536d Mon Sep 17 00:00:00 2001 From: Song Gao Date: Mon, 9 Dec 2024 11:17:43 +0800 Subject: [PATCH 1/5] revise Signed-off-by: Song Gao --- internal/server/rule_init.go | 10 ++--- internal/server/rule_manager.go | 4 +- internal/server/server.go | 4 +- internal/topo/node/cache/sync_cache.go | 14 +++++++ .../prom_metrcis.go => metrics/metrics.go | 6 ++- metrics/sync_cache.go | 39 +++++++++++++++++++ 6 files changed, 66 insertions(+), 11 deletions(-) rename internal/server/promMetrics/prom_metrcis.go => metrics/metrics.go (96%) create mode 100644 metrics/sync_cache.go diff --git a/internal/server/rule_init.go b/internal/server/rule_init.go index ed21769154..660dfb284d 100644 --- a/internal/server/rule_init.go +++ b/internal/server/rule_init.go @@ -26,8 +26,8 @@ import ( "github.com/lf-edge/ekuiper/v2/internal/conf" "github.com/lf-edge/ekuiper/v2/internal/pkg/def" "github.com/lf-edge/ekuiper/v2/internal/pkg/schedule" - "github.com/lf-edge/ekuiper/v2/internal/server/promMetrics" "github.com/lf-edge/ekuiper/v2/internal/topo/rule" + "github.com/lf-edge/ekuiper/v2/metrics" "github.com/lf-edge/ekuiper/v2/pkg/ast" "github.com/lf-edge/ekuiper/v2/pkg/timex" ) @@ -150,10 +150,10 @@ func handleAllRuleStatusMetrics(rs []ruleWrapper) { stopCount++ v = RuleStopped } - promMetrics.SetRuleStatus(id, int(v)) + metrics.SetRuleStatus(id, int(v)) } - promMetrics.SetRuleStatusCountGauge(true, runningCount) - promMetrics.SetRuleStatusCountGauge(false, stopCount) + metrics.SetRuleStatusCountGauge(true, runningCount) + metrics.SetRuleStatusCountGauge(false, stopCount) } } @@ -272,7 +272,7 @@ func StartCPUProfiling(ctx context.Context, cpuProfile Profiler) error { continue } for labelValue, t := range ruleUsage.Stats { - promMetrics.SetRuleCPUUsageGauge(labelValue, t) + metrics.SetRuleCPUUsageGauge(labelValue, t) } } } diff --git a/internal/server/rule_manager.go b/internal/server/rule_manager.go index aeeec7b44d..42fbab1ae6 100644 --- a/internal/server/rule_manager.go +++ b/internal/server/rule_manager.go @@ -24,10 +24,10 @@ import ( "github.com/lf-edge/ekuiper/v2/internal/conf" "github.com/lf-edge/ekuiper/v2/internal/pkg/def" "github.com/lf-edge/ekuiper/v2/internal/pkg/store" - "github.com/lf-edge/ekuiper/v2/internal/server/promMetrics" "github.com/lf-edge/ekuiper/v2/internal/topo/planner" "github.com/lf-edge/ekuiper/v2/internal/topo/rule" "github.com/lf-edge/ekuiper/v2/internal/xsql" + "github.com/lf-edge/ekuiper/v2/metrics" "github.com/lf-edge/ekuiper/v2/pkg/errorx" "github.com/lf-edge/ekuiper/v2/pkg/infra" "github.com/lf-edge/ekuiper/v2/pkg/replace" @@ -475,6 +475,6 @@ func getRuleState(name string) (rule.RunState, error) { func deleteRuleMetrics(name string) { if conf.Config != nil && conf.Config.Basic.Prometheus { - promMetrics.RemoveRuleStatus(name) + metrics.RemoveRuleStatus(name) } } diff --git a/internal/server/server.go b/internal/server/server.go index e1190d48e2..afec4a30eb 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -44,8 +44,8 @@ import ( "github.com/lf-edge/ekuiper/v2/internal/plugin/portable/runtime" "github.com/lf-edge/ekuiper/v2/internal/processor" "github.com/lf-edge/ekuiper/v2/internal/server/bump" - "github.com/lf-edge/ekuiper/v2/internal/server/promMetrics" "github.com/lf-edge/ekuiper/v2/internal/topo/rule" + "github.com/lf-edge/ekuiper/v2/metrics" "github.com/lf-edge/ekuiper/v2/pkg/cast" "github.com/lf-edge/ekuiper/v2/pkg/connection" "github.com/lf-edge/ekuiper/v2/pkg/modules" @@ -141,7 +141,7 @@ func StartUp(Version string) { conf.SetupEnv() conf.InitConf() if conf.Config.Basic.Prometheus { - promMetrics.RegisterMetrics() + metrics.RegisterMetrics() } // Print inited modules diff --git a/internal/topo/node/cache/sync_cache.go b/internal/topo/node/cache/sync_cache.go index c969d1bada..52b66bf785 100644 --- a/internal/topo/node/cache/sync_cache.go +++ b/internal/topo/node/cache/sync_cache.go @@ -24,6 +24,7 @@ import ( "github.com/lf-edge/ekuiper/v2/internal/conf" "github.com/lf-edge/ekuiper/v2/internal/pkg/store" + "github.com/lf-edge/ekuiper/v2/metrics" "github.com/lf-edge/ekuiper/v2/pkg/kv" ) @@ -94,7 +95,15 @@ func (p *page) reset() { p.L = 0 } +const ( + syncCacheAdd = "add" + syncCachePop = "pop" + syncCacheFlush = "flush" +) + type SyncCache struct { + RuleID string + OpID string // cache config cacheConf *conf.SinkConf maxDiskPage int @@ -119,6 +128,8 @@ func NewSyncCache(ctx api.StreamContext, cacheConf *conf.SinkConf) (*SyncCache, ctx.GetLogger().Warnf("disk page is less than 2, so set it to 2") } c := &SyncCache{ + RuleID: ctx.GetRuleId(), + OpID: ctx.GetOpId(), cacheConf: cacheConf, // add one more slot so that there will be at least one slot between head and tail to find out the head/tail id maxDiskPage: diskPage, @@ -142,6 +153,7 @@ func (c *SyncCache) AddCache(ctx api.StreamContext, item any) error { } else { ctx.GetLogger().Debugf("added cache to disk buffer page %v", c.writeBufferPage) } + metrics.SyncCacheCounter.WithLabelValues(syncCacheAdd, c.RuleID, c.OpID).Inc() c.CacheLength++ ctx.GetLogger().Debugf("added cache %d", c.CacheLength) return nil @@ -229,6 +241,7 @@ func (c *SyncCache) PopCache(ctx api.StreamContext) (any, bool) { ctx.GetLogger().Debugf("deleted cache: %d", c.CacheLength) } ctx.GetLogger().Debugf("deleted cache. CacheLength: %d, diskSize: %d, readPage: %v", c.CacheLength, c.diskSize, c.readBufferPage) + metrics.SyncCacheCounter.WithLabelValues(syncCachePop, c.RuleID, c.OpID).Inc() return result, true } @@ -357,5 +370,6 @@ func (c *SyncCache) Flush(ctx api.StreamContext) { ctx.GetLogger().Warnf("fail to store disk cache size %v", err) } _ = c.store.Set("storeSig", 1) + metrics.SyncCacheCounter.WithLabelValues(syncCacheFlush, c.RuleID, c.OpID).Inc() } } diff --git a/internal/server/promMetrics/prom_metrcis.go b/metrics/metrics.go similarity index 96% rename from internal/server/promMetrics/prom_metrcis.go rename to metrics/metrics.go index 96d471d7ad..b4d7aae1e5 100644 --- a/internal/server/promMetrics/prom_metrcis.go +++ b/metrics/metrics.go @@ -12,13 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package promMetrics +package metrics import "github.com/prometheus/client_golang/prometheus" const ( + LblType = "type" LblStatusType = "status" LblRuleIDType = "ruleID" + LblOpIDType = "opID" LBlRuleRunning = "running" LblRuleStop = "stop" @@ -54,7 +56,7 @@ func InitServerMetrics() { } func RegisterMetrics() { - InitServerMetrics() + RegisterSyncCache() prometheus.MustRegister(RuleStatusCountGauge) prometheus.MustRegister(RuleStatusGauge) prometheus.MustRegister(RuleCPUUsageGauge) diff --git a/metrics/sync_cache.go b/metrics/sync_cache.go new file mode 100644 index 0000000000..8ff690218f --- /dev/null +++ b/metrics/sync_cache.go @@ -0,0 +1,39 @@ +// Copyright 2024 EMQ Technologies Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +var ( + SyncCacheHist = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "kuiper", + Subsystem: "sync_cache", + Name: "hist", + Buckets: prometheus.ExponentialBuckets(10, 2, 20), // 10us ~ 5s + Help: "hist of sync cache", + }, []string{LblType, LblRuleIDType, LblOpIDType}) + + SyncCacheCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "kuiper", + Subsystem: "sync_cache", + Name: "counter", + Help: "counter of sync cache", + }, []string{LblType, LblRuleIDType, LblOpIDType}) +) + +func RegisterSyncCache() { + prometheus.MustRegister(SyncCacheCounter) + prometheus.MustRegister(SyncCacheHist) +} From a3730db36066184d5182db0bf33a8937fe8d51a5 Mon Sep 17 00:00:00 2001 From: Song Gao Date: Mon, 9 Dec 2024 12:04:00 +0800 Subject: [PATCH 2/5] fix Signed-off-by: Song Gao --- internal/server/server.go | 5 ----- metrics/metrics.go | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/internal/server/server.go b/internal/server/server.go index afec4a30eb..b8d223dcf2 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -45,7 +45,6 @@ import ( "github.com/lf-edge/ekuiper/v2/internal/processor" "github.com/lf-edge/ekuiper/v2/internal/server/bump" "github.com/lf-edge/ekuiper/v2/internal/topo/rule" - "github.com/lf-edge/ekuiper/v2/metrics" "github.com/lf-edge/ekuiper/v2/pkg/cast" "github.com/lf-edge/ekuiper/v2/pkg/connection" "github.com/lf-edge/ekuiper/v2/pkg/modules" @@ -140,10 +139,6 @@ func StartUp(Version string) { createPaths() conf.SetupEnv() conf.InitConf() - if conf.Config.Basic.Prometheus { - metrics.RegisterMetrics() - } - // Print inited modules for n := range modules.Sources { conf.Log.Infof("register source %s", n) diff --git a/metrics/metrics.go b/metrics/metrics.go index b4d7aae1e5..75265792be 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -55,7 +55,7 @@ func InitServerMetrics() { }, []string{LblRuleIDType}) } -func RegisterMetrics() { +func init() { RegisterSyncCache() prometheus.MustRegister(RuleStatusCountGauge) prometheus.MustRegister(RuleStatusGauge) From 0d7289c67b6f5ed3733f6a176dfd182d59033b2c Mon Sep 17 00:00:00 2001 From: Song Gao Date: Mon, 9 Dec 2024 12:04:33 +0800 Subject: [PATCH 3/5] fix Signed-off-by: Song Gao --- metrics/metrics.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/metrics/metrics.go b/metrics/metrics.go index 75265792be..c3523a57df 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -27,12 +27,6 @@ const ( ) var ( - RuleStatusCountGauge *prometheus.GaugeVec - RuleStatusGauge *prometheus.GaugeVec - RuleCPUUsageGauge *prometheus.GaugeVec -) - -func InitServerMetrics() { RuleStatusCountGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: "kuiper", Subsystem: "rule", @@ -53,7 +47,7 @@ func InitServerMetrics() { Name: "cpu_ms", Help: "gauge of rule CPU usage", }, []string{LblRuleIDType}) -} +) func init() { RegisterSyncCache() From 925dc0166c99caf9c385056604fe0f729c076928 Mon Sep 17 00:00:00 2001 From: Song Gao Date: Mon, 9 Dec 2024 16:02:51 +0800 Subject: [PATCH 4/5] fix Signed-off-by: Song Gao --- internal/topo/node/cache/sync_cache.go | 21 ++++++++++++++++++--- internal/topo/node/cache_op.go | 1 + metrics/metrics.go | 4 ++-- metrics/sync_cache.go | 2 +- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/internal/topo/node/cache/sync_cache.go b/internal/topo/node/cache/sync_cache.go index 52b66bf785..5c7f55d30d 100644 --- a/internal/topo/node/cache/sync_cache.go +++ b/internal/topo/node/cache/sync_cache.go @@ -99,6 +99,8 @@ const ( syncCacheAdd = "add" syncCachePop = "pop" syncCacheFlush = "flush" + syncCacheDrop = "drop" + syncCacheLoad = "load" ) type SyncCache struct { @@ -128,8 +130,6 @@ func NewSyncCache(ctx api.StreamContext, cacheConf *conf.SinkConf) (*SyncCache, ctx.GetLogger().Warnf("disk page is less than 2, so set it to 2") } c := &SyncCache{ - RuleID: ctx.GetRuleId(), - OpID: ctx.GetOpId(), cacheConf: cacheConf, // add one more slot so that there will be at least one slot between head and tail to find out the head/tail id maxDiskPage: diskPage, @@ -140,6 +140,11 @@ func NewSyncCache(ctx api.StreamContext, cacheConf *conf.SinkConf) (*SyncCache, return c, err } +func (c *SyncCache) SetupMeta(ctx api.StreamContext) { + c.RuleID = ctx.GetRuleId() + c.OpID = ctx.GetOpId() +} + // AddCache not thread safe! func (c *SyncCache) AddCache(ctx api.StreamContext, item any) error { isBufferNotFull := c.writeBufferPage.append(item) @@ -160,6 +165,11 @@ func (c *SyncCache) AddCache(ctx api.StreamContext, item any) error { } func (c *SyncCache) appendWriteCache(ctx api.StreamContext) error { + metrics.SyncCacheCounter.WithLabelValues(syncCacheFlush, c.RuleID, c.OpID).Inc() + start := time.Now() + defer func() { + metrics.SyncCacheHist.WithLabelValues(syncCacheFlush, c.RuleID, c.OpID).Observe(float64(time.Since(start).Microseconds())) + }() if c.diskSize == c.maxDiskPage { // disk full, replace read buffer page err := c.deleteDiskPage(ctx, false) @@ -247,6 +257,7 @@ func (c *SyncCache) PopCache(ctx api.StreamContext) (any, bool) { // loaded means whether load the page to memory or just drop func (c *SyncCache) deleteDiskPage(ctx api.StreamContext, loaded bool) error { + metrics.SyncCacheCounter.WithLabelValues(syncCacheDrop, c.RuleID, c.OpID).Inc() _ = c.store.Delete(strconv.Itoa(c.diskPageHead)) ctx.GetLogger().Warnf("drop a read page of %d items in memory", c.readBufferPage.L) c.diskPageHead++ @@ -271,6 +282,11 @@ func (c *SyncCache) deleteDiskPage(ctx api.StreamContext, loaded bool) error { } func (c *SyncCache) loadFromDisk(ctx api.StreamContext) error { + metrics.SyncCacheCounter.WithLabelValues(syncCacheLoad, c.RuleID, c.OpID).Inc() + start := time.Now() + defer func() { + metrics.SyncCacheHist.WithLabelValues(syncCacheLoad, c.RuleID, c.OpID).Observe(float64(time.Since(start).Microseconds())) + }() // load page from the disk ctx.GetLogger().Debugf("loading from disk %d. CacheLength: %d, diskSize: %d", c.diskPageTail, c.CacheLength, c.diskSize) // caution, must create a new page instance @@ -370,6 +386,5 @@ func (c *SyncCache) Flush(ctx api.StreamContext) { ctx.GetLogger().Warnf("fail to store disk cache size %v", err) } _ = c.store.Set("storeSig", 1) - metrics.SyncCacheCounter.WithLabelValues(syncCacheFlush, c.RuleID, c.OpID).Inc() } } diff --git a/internal/topo/node/cache_op.go b/internal/topo/node/cache_op.go index 7411f4a992..5850579453 100644 --- a/internal/topo/node/cache_op.go +++ b/internal/topo/node/cache_op.go @@ -69,6 +69,7 @@ func (s *CacheOp) Exec(ctx api.StreamContext, errCh chan<- error) { if len(s.outputs) > 1 { infra.DrainError(ctx, fmt.Errorf("cache op should have only 1 output but got %+v", s.outputs), errCh) } + s.cache.SetupMeta(ctx) s.prepareExec(ctx, errCh, "op") go func() { err := infra.SafeRun(func() error { diff --git a/metrics/metrics.go b/metrics/metrics.go index c3523a57df..90e70b586c 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -19,8 +19,8 @@ import "github.com/prometheus/client_golang/prometheus" const ( LblType = "type" LblStatusType = "status" - LblRuleIDType = "ruleID" - LblOpIDType = "opID" + LblRuleIDType = "rule" + LblOpIDType = "op" LBlRuleRunning = "running" LblRuleStop = "stop" diff --git a/metrics/sync_cache.go b/metrics/sync_cache.go index 8ff690218f..28065b612a 100644 --- a/metrics/sync_cache.go +++ b/metrics/sync_cache.go @@ -20,7 +20,7 @@ var ( SyncCacheHist = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "kuiper", Subsystem: "sync_cache", - Name: "hist", + Name: "duration", Buckets: prometheus.ExponentialBuckets(10, 2, 20), // 10us ~ 5s Help: "hist of sync cache", }, []string{LblType, LblRuleIDType, LblOpIDType}) From 844beeedcfac5259303c9b9baff177df3920aeb8 Mon Sep 17 00:00:00 2001 From: Song Gao Date: Mon, 9 Dec 2024 16:07:42 +0800 Subject: [PATCH 5/5] update metrics Signed-off-by: Song Gao --- metrics/metrics.json | 814 ++++++++++++++++++------------------------- 1 file changed, 338 insertions(+), 476 deletions(-) diff --git a/metrics/metrics.json b/metrics/metrics.json index 55138a0473..cfcffe4322 100644 --- a/metrics/metrics.json +++ b/metrics/metrics.json @@ -25,12 +25,12 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 1, + "id": 18, "links": [], "liveNow": false, "panels": [ { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -107,7 +107,7 @@ "options": { "legend": { "calcs": [ - "lastNotNull" + "last" ], "displayMode": "table", "placement": "right", @@ -128,7 +128,7 @@ "exemplar": false, "expr": "(time() - process_start_time_seconds{instance=\"$instance\"})", "instant": false, - "legendFormat": "{{instance}}", + "legendFormat": "up", "range": true, "refId": "A" } @@ -214,25 +214,37 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", "expr": "go_memstats_heap_sys_bytes{instance=\"$instance\"}", - "legendFormat": "{{instance}}-heap-sys", + "legendFormat": "heap-sys", "range": true, "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", "expr": "go_memstats_alloc_bytes{instance=\"$instance\"}", "hide": false, - "legendFormat": "{{instance}}-heap-alloc", + "legendFormat": "heap-alloc", "range": true, "refId": "B" }, { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", "expr": "go_memstats_heap_inuse_bytes{instance=\"$instance\"}", "hide": false, - "legendFormat": "{{instance}}-heap-inuse", + "legendFormat": "heap-inuse", "range": true, "refId": "C" } @@ -305,7 +317,7 @@ "options": { "legend": { "calcs": [ - "lastNotNull" + "last" ], "displayMode": "table", "placement": "right", @@ -318,9 +330,13 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "code", "expr": "irate(process_cpu_seconds_total{instance=\"$instance\"}[30s])", - "legendFormat": "{{instance}}", + "legendFormat": "cpu_usage", "range": true, "refId": "A" } @@ -405,9 +421,13 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", - "expr": "go_goroutines", - "legendFormat": "{{instance}}", + "expr": "go_goroutines{instance=\"$instance\"}", + "legendFormat": "goroutines", "range": true, "refId": "A" } @@ -465,35 +485,9 @@ "value": 80 } ] - }, - "unit": "cps" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "172.31.24.64:20499-total-records-in" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] } - ] + }, + "overrides": [] }, "gridPos": { "h": 8, @@ -501,11 +495,11 @@ "x": 0, "y": 17 }, - "id": 29, + "id": 31, "options": { "legend": { "calcs": [ - "lastNotNull" + "last" ], "displayMode": "table", "placement": "right", @@ -518,14 +512,18 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", - "expr": "sum by(instance) (rate(kuiper_source_records_in_total{instance=\"$instance\"}[1m]))", - "legendFormat": "{{instance}}", + "expr": "kuiper_rule_status{instance=\"$instance\"}", + "legendFormat": "{{rule}}", "range": true, "refId": "A" } ], - "title": "Server Total Records in QPS", + "title": "Rule Status", "type": "timeseries" }, { @@ -585,10 +583,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 18 + "x": 12, + "y": 17 }, - "id": 31, + "id": 32, "options": { "legend": { "calcs": [ @@ -605,18 +603,36 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", - "expr": "kuiper_rule_status", - "legendFormat": "{{instance}}-{{ruleID}}", + "expr": "kuiper_rule_count", + "legendFormat": "{{status}}", "range": true, "refId": "A" } ], - "title": "Rule Status", + "title": "Rule Count", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 4, + "panels": [], + "title": "Operator Panel", + "type": "row" + }, { "datasource": "${DS_TEST-CLUSTER}", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -626,7 +642,7 @@ "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", - "axisPlacement": "auto", + "axisPlacement": "left", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, @@ -665,17 +681,18 @@ "value": 80 } ] - } + }, + "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 18 + "x": 0, + "y": 26 }, - "id": 32, + "id": 17, "options": { "legend": { "calcs": [ @@ -692,18 +709,23 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", - "expr": "kuiper_rule_count", - "legendFormat": "{{instance}}-{{status}}", + "expr": "rate(kuiper_op_records_in_total{rule=\"$rule\", instance=\"$instance\"}[1m])", + "legendFormat": "{{op}}", "range": true, "refId": "A" } ], - "title": "Rule Count", + "title": "Records in QPS", "type": "timeseries" }, { "datasource": "${DS_TEST-CLUSTER}", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -713,7 +735,7 @@ "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", - "axisPlacement": "auto", + "axisPlacement": "left", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, @@ -761,13 +783,13 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 26 }, - "id": 30, + "id": 18, "options": { "legend": { "calcs": [ - "lastNotNull" + "last" ], "displayMode": "table", "placement": "right", @@ -780,33 +802,22 @@ }, "targets": [ { - "datasource": "${DS_TEST-CLUSTER}", + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", - "expr": "sum by(instance) (rate(kuiper_sink_records_out_total{instance=\"$instance\"}[1m]))", - "legendFormat": "{{instance}}", + "expr": "rate(kuiper_op_records_out_total{rule=\"$rule\", instance=\"$instance\"}[1m])", + "legendFormat": "{{op}}", "range": true, "refId": "A" } ], - "title": "Server Total Records Out QPS", + "title": "Records Out QPS", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 4, - "panels": [], - "title": "Operator Panel", - "type": "row" - }, { "datasource": "${DS_TEST-CLUSTER}", - "description": "", "fieldConfig": { "defaults": { "color": { @@ -856,7 +867,7 @@ } ] }, - "unit": "cps" + "unit": "µs" }, "overrides": [] }, @@ -864,13 +875,17 @@ "h": 8, "w": 12, "x": 0, - "y": 18 + "y": 34 }, - "id": 17, + "id": 19, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "last", + "min", + "max" + ], + "displayMode": "table", "placement": "right", "showLegend": true }, @@ -881,19 +896,27 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", - "expr": "rate(kuiper_op_records_in_total{rule=\"$rule\", instance=\"$instance\"}[1m])", - "legendFormat": "{{instance}}-{{op}}-{{exported_instance}}", + "exemplar": false, + "expr": "histogram_quantile(0.9, sum by(le, op, exported_instance, instance) (rate(kuiper_op_process_latency_us_hist_bucket{rule=\"$rule\", instance=\"$instance\"}[1m])))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{op}}", "range": true, - "refId": "A" + "refId": "C" } ], - "title": "Records in QPS", + "title": "Process Latency P90", "type": "timeseries" }, { "datasource": "${DS_TEST-CLUSTER}", - "description": "", "fieldConfig": { "defaults": { "color": { @@ -903,7 +926,7 @@ "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", - "axisPlacement": "left", + "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, @@ -951,13 +974,15 @@ "h": 8, "w": 12, "x": 12, - "y": 18 + "y": 34 }, - "id": 18, + "id": 20, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "last" + ], + "displayMode": "table", "placement": "right", "showLegend": true }, @@ -968,18 +993,36 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", - "expr": "rate(kuiper_op_records_out_total{rule=\"$rule\", instance=\"$instance\"}[1m])", - "legendFormat": "{{instance}}-{{op}}-{{exported_instance}}", + "expr": "rate(kuiper_op_exceptions_total{rule=\"$rule\", instance=\"$instance\"}[1m])", + "legendFormat": "{{op}}", "range": true, "refId": "A" } ], - "title": "Records Out QPS", + "title": "Exceptions QPS", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 3, + "panels": [], + "title": "Sink Panel", + "type": "row" + }, { "datasource": "${DS_TEST-CLUSTER}", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -1029,7 +1072,7 @@ } ] }, - "unit": "µs" + "unit": "cps" }, "overrides": [] }, @@ -1037,13 +1080,15 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 43 }, - "id": 19, + "id": 13, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "last" + ], + "displayMode": "table", "placement": "right", "showLegend": true }, @@ -1054,23 +1099,23 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.9, sum by(le, op, exported_instance, instance) (rate(kuiper_op_process_latency_us_hist_bucket{rule=\"$rule\", instance=\"$instance\"}[1m])))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{instance}}-{{op}}-{{exported_instance}}", + "expr": "rate(kuiper_sink_records_in_total{rule=\"$rule\", instance=\"$instance\"}[1m])", + "legendFormat": "{{op}}", "range": true, - "refId": "C" + "refId": "A" } ], - "title": "Process Latency P90", + "title": "Records in QPS", "type": "timeseries" }, { "datasource": "${DS_TEST-CLUSTER}", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -1120,7 +1165,7 @@ } ] }, - "unit": "µs" + "unit": "cps" }, "overrides": [] }, @@ -1128,13 +1173,15 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 43 }, - "id": 23, + "id": 14, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "last" + ], + "displayMode": "table", "placement": "right", "showLegend": true }, @@ -1145,19 +1192,18 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(le, op, exported_instance, instance) (rate(kuiper_op_process_latency_us_hist_bucket{rule=\"$rule\", instance=\"$instance\"}[1m])))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{instance}}-{{op}}-{{exported_instance}}", + "expr": "rate(kuiper_sink_records_out_total{rule=\"$rule\", instance=\"$instance\"}[1m])", + "legendFormat": "{{op}}", "range": true, - "refId": "C" + "refId": "A" } ], - "title": "Process Latency P99", + "title": "Records Out QPS", "type": "timeseries" }, { @@ -1171,7 +1217,7 @@ "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", - "axisPlacement": "auto", + "axisPlacement": "left", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, @@ -1211,7 +1257,7 @@ } ] }, - "unit": "cps" + "unit": "µs" }, "overrides": [] }, @@ -1219,13 +1265,17 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 51 }, - "id": 20, + "id": 15, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "last", + "min", + "max" + ], + "displayMode": "table", "placement": "right", "showLegend": true }, @@ -1236,32 +1286,27 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", - "expr": "rate(kuiper_op_exceptions_total{rule=\"$rule\", instance=\"$instance\"}[1m])", - "legendFormat": "{{instance}}-{{op}}-{{exported_instance}}", + "exemplar": false, + "expr": "histogram_quantile(0.9, sum by(le, op, exported_instance, instance) (rate(kuiper_sink_process_latency_us_hist_bucket{rule=\"$rule\", instance=\"$instance\"}[1m])))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{op}}", "range": true, - "refId": "A" + "refId": "C" } ], - "title": "Exceptions QPS", + "title": "Process Latency P90", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 42 - }, - "id": 3, - "panels": [], - "title": "Sink Panel", - "type": "row" - }, { "datasource": "${DS_TEST-CLUSTER}", - "description": "", "fieldConfig": { "defaults": { "color": { @@ -1271,7 +1316,7 @@ "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", - "axisPlacement": "left", + "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, @@ -1318,14 +1363,16 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 43 + "x": 12, + "y": 51 }, - "id": 13, + "id": 16, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "last" + ], + "displayMode": "table", "placement": "right", "showLegend": true }, @@ -1336,16 +1383,33 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", - "expr": "rate(kuiper_sink_records_in_total{rule=\"$rule\", instance=\"$instance\"}[1m])", - "legendFormat": "{{instance}}-{{op}}-{{exported_instance}}", + "expr": "rate(kuiper_sink_exceptions_total{rule=\"$rule\", instance=\"$instance\"}[1m])", + "legendFormat": "{{op}}", "range": true, "refId": "A" } ], - "title": "Records in QPS", + "title": "Exceptions QPS", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 2, + "panels": [], + "title": "Source Panel", + "type": "row" + }, { "datasource": "${DS_TEST-CLUSTER}", "description": "", @@ -1405,14 +1469,16 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 43 + "x": 0, + "y": 60 }, - "id": 14, + "id": 1, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "last" + ], + "displayMode": "table", "placement": "right", "showLegend": true }, @@ -1423,18 +1489,23 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", - "expr": "rate(kuiper_sink_records_out_total{rule=\"$rule\", instance=\"$instance\"}[1m])", - "legendFormat": "{{instance}}-{{op}}-{{exported_instance}}", + "expr": "rate(kuiper_source_records_in_total{rule=\"$rule\", instance=\"$instance\"}[1m])", + "legendFormat": "{{op}}", "range": true, "refId": "A" } ], - "title": "Records Out QPS", + "title": "Records in QPS", "type": "timeseries" }, { "datasource": "${DS_TEST-CLUSTER}", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -1484,21 +1555,23 @@ } ] }, - "unit": "µs" + "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 51 + "x": 12, + "y": 60 }, - "id": 15, + "id": 5, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "last" + ], + "displayMode": "table", "placement": "right", "showLegend": true }, @@ -1509,19 +1582,18 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.9, sum by(le, op, exported_instance, instance) (rate(kuiper_sink_process_latency_us_hist_bucket{rule=\"$rule\", instance=\"$instance\"}[1m])))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{instance}}-{{op}}-{{exported_instance}}", + "expr": "rate(kuiper_source_records_out_total{rule=\"$rule\", instance=\"$instance\"}[1m])", + "legendFormat": "{{op}}", "range": true, - "refId": "C" + "refId": "A" } ], - "title": "Process Latency P90", + "title": "Records Out QPS", "type": "timeseries" }, { @@ -1582,14 +1654,18 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 51 + "x": 0, + "y": 68 }, - "id": 22, + "id": 6, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "last", + "min", + "max" + ], + "displayMode": "table", "placement": "right", "showLegend": true }, @@ -1600,19 +1676,23 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(le, op, exported_instance, instance) (rate(kuiper_sink_process_latency_us_hist_bucket{rule=\"$rule\", instance=\"$instance\"}[1m])))", + "expr": "histogram_quantile(0.9, sum by(le, op, instance) (rate(kuiper_source_process_latency_us_hist_bucket{rule=\"$rule\", instance=\"$instance\"}[1m])))", "format": "time_series", "hide": false, "instant": false, "interval": "", - "legendFormat": "{{instance}}-{{op}}-{{exported_instance}}", + "legendFormat": "{{op}}", "range": true, "refId": "C" } ], - "title": "Process Latency P99", + "title": "Process Latency P90", "type": "timeseries" }, { @@ -1673,14 +1753,16 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 59 + "x": 12, + "y": 68 }, - "id": 16, + "id": 8, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "last" + ], + "displayMode": "table", "placement": "right", "showLegend": true }, @@ -1691,9 +1773,13 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", - "expr": "rate(kuiper_sink_exceptions_total{rule=\"$rule\", instance=\"$instance\"}[1m])", - "legendFormat": "{{instance}}-{{op}}-{{exported_instance}}", + "expr": "rate(kuiper_source_exceptions_total{rule=\"$rule\", instance=\"$instance\"}[1m])", + "legendFormat": "{{op}}", "range": true, "refId": "A" } @@ -1707,11 +1793,11 @@ "h": 1, "w": 24, "x": 0, - "y": 67 + "y": 76 }, - "id": 2, + "id": 33, "panels": [], - "title": "Source Panel", + "title": "Sink Cache", "type": "row" }, { @@ -1774,13 +1860,15 @@ "h": 8, "w": 12, "x": 0, - "y": 68 + "y": 77 }, - "id": 1, + "id": 34, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "last" + ], + "displayMode": "table", "placement": "right", "showLegend": true }, @@ -1791,101 +1879,19 @@ }, "targets": [ { - "editorMode": "builder", - "expr": "rate(kuiper_source_records_in_total{rule=\"$rule\", instance=\"$instance\"}[1m])", - "legendFormat": "{{instance}}-{{op}}-{{exported_instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Records in QPS", - "type": "timeseries" - }, - { - "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "left", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" }, - "unit": "cps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 68 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { "editorMode": "builder", - "expr": "rate(kuiper_source_records_out_total{rule=\"$rule\", instance=\"$instance\"}[1m])", - "legendFormat": "{{instance}}-{{op}}-{{exported_instance}}", + "expr": "rate(kuiper_sync_cache_counter{instance=\"$instance\", rule=\"$rule\"}[1m])", + "hide": false, + "legendFormat": "{{type}}", "range": true, "refId": "A" } ], - "title": "Records Out QPS", + "title": "Sync Cache QPS", "type": "timeseries" }, { @@ -1941,110 +1947,48 @@ }, "unit": "µs" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 76 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.9, sum by(le, op, exported_instance, instance) (rate(kuiper_source_process_latency_us_hist_bucket{rule=\"$rule\", instance=\"$instance\"}[1m])))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{instance}}-{{op}}-{{exported_instance}}", - "range": true, - "refId": "C" - } - ], - "title": "Process Latency P90", - "type": "timeseries" - }, - { - "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "left", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "flush" + ], + "prefix": "All except:", + "readOnly": true + } }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, + "properties": [ { - "color": "red", - "value": 80 + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } } ] - }, - "unit": "µs" - }, - "overrides": [] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 76 + "y": 77 }, - "id": 21, + "id": 35, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "last", + "min", + "max" + ], + "displayMode": "table", "placement": "right", "showLegend": true }, @@ -2055,109 +1999,27 @@ }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "ad687d3e-23e5-4418-bb75-0dfb04fb4721" + }, "editorMode": "builder", "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(le, op, exported_instance, instance) (rate(kuiper_source_process_latency_us_hist_bucket{rule=\"$rule\", instance=\"$instance\"}[1m])))", + "expr": "histogram_quantile(0.9, sum by(le, instance, type, rule) (rate(kuiper_sync_cache_duration_bucket{rule=\"$rule\", instance=\"$instance\"}[1m])))", "format": "time_series", "hide": false, "instant": false, "interval": "", - "legendFormat": "{{instance}}-{{op}}-{{exported_instance}}", + "legendFormat": "{{type}}", "range": true, "refId": "C" } ], - "title": "Process Latency P99", - "type": "timeseries" - }, - { - "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 84 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "editorMode": "builder", - "expr": "rate(kuiper_source_exceptions_total{rule=\"$rule\", instance=\"$instance\"}[1m])", - "legendFormat": "{{instance}}-{{op}}-{{exported_instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Exceptions QPS", + "title": "Sync Cache Process latency P90", "type": "timeseries" } ], - "refresh": "", + "refresh": "5s", "schemaVersion": 38, "style": "dark", "tags": [], @@ -2209,6 +2071,6 @@ "timezone": "", "title": "kuiper panel", "uid": "da14abfc-1b97-4833-a86f-fad5d3147e80", - "version": 15, + "version": 1, "weekStart": "" }