From d6a60773e0e30095bf7438bdaf10558dfbaac92b Mon Sep 17 00:00:00 2001 From: "Eduardo J. Ortega U" <5791035+ejortegau@users.noreply.github.com> Date: Tue, 18 Jul 2023 14:33:22 +0200 Subject: [PATCH 1/3] Per workload TxThrottler metrics With these changes, TxThrottler metrics for requests received and requests throttled now include a label to identify workload associated with the request. Signed-off-by: Eduardo J. Ortega U <5791035+ejortegau@users.noreply.github.com> --- go/vt/vttablet/tabletserver/query_executor.go | 4 ++-- go/vt/vttablet/tabletserver/tabletserver.go | 2 +- .../tabletserver/txthrottler/tx_throttler.go | 16 +++++++-------- .../txthrottler/tx_throttler_test.go | 20 +++++++++---------- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/go/vt/vttablet/tabletserver/query_executor.go b/go/vt/vttablet/tabletserver/query_executor.go index 0ac3dc78155..0723a158b99 100644 --- a/go/vt/vttablet/tabletserver/query_executor.go +++ b/go/vt/vttablet/tabletserver/query_executor.go @@ -221,7 +221,7 @@ func (qre *QueryExecutor) execAutocommit(f func(conn *StatefulConnection) (*sqlt } qre.options.TransactionIsolation = querypb.ExecuteOptions_AUTOCOMMIT - if qre.tsv.txThrottler.Throttle(qre.tsv.getPriorityFromOptions(qre.options)) { + if qre.tsv.txThrottler.Throttle(qre.tsv.getPriorityFromOptions(qre.options), qre.options.GetWorkloadName()) { return nil, errTxThrottled } @@ -236,7 +236,7 @@ func (qre *QueryExecutor) execAutocommit(f func(conn *StatefulConnection) (*sqlt } func (qre *QueryExecutor) execAsTransaction(f func(conn *StatefulConnection) (*sqltypes.Result, error)) (*sqltypes.Result, error) { - if qre.tsv.txThrottler.Throttle(qre.tsv.getPriorityFromOptions(qre.options)) { + if qre.tsv.txThrottler.Throttle(qre.tsv.getPriorityFromOptions(qre.options), qre.options.GetWorkloadName()) { return nil, errTxThrottled } conn, beginSQL, _, err := qre.tsv.te.txPool.Begin(qre.ctx, qre.options, false, 0, nil, qre.setting) diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 9c01ec1ba1e..70089f85a2e 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -494,7 +494,7 @@ func (tsv *TabletServer) begin(ctx context.Context, target *querypb.Target, save target, options, false, /* allowOnShutdown */ func(ctx context.Context, logStats *tabletenv.LogStats) error { startTime := time.Now() - if tsv.txThrottler.Throttle(tsv.getPriorityFromOptions(options)) { + if tsv.txThrottler.Throttle(tsv.getPriorityFromOptions(options), options.GetWorkloadName()) { return errTxThrottled } var connSetting *pools.Setting diff --git a/go/vt/vttablet/tabletserver/txthrottler/tx_throttler.go b/go/vt/vttablet/tabletserver/txthrottler/tx_throttler.go index 30e2ec19c56..ad1126e887c 100644 --- a/go/vt/vttablet/tabletserver/txthrottler/tx_throttler.go +++ b/go/vt/vttablet/tabletserver/txthrottler/tx_throttler.go @@ -68,7 +68,7 @@ type TxThrottler interface { InitDBConfig(target *querypb.Target) Open() (err error) Close() - Throttle(priority int) (result bool) + Throttle(priority int, workload string) (result bool) } func init() { @@ -143,8 +143,8 @@ type txThrottler struct { // stats throttlerRunning *stats.Gauge - requestsTotal *stats.Counter - requestsThrottled *stats.Counter + requestsTotal *stats.CountersWithSingleLabel + requestsThrottled *stats.CountersWithSingleLabel } // txThrottlerConfig holds the parameters that need to be @@ -205,8 +205,8 @@ func NewTxThrottler(env tabletenv.Env, topoServer *topo.Server) TxThrottler { config: throttlerConfig, topoServer: topoServer, throttlerRunning: env.Exporter().NewGauge("TransactionThrottlerRunning", "transaction throttler running state"), - requestsTotal: env.Exporter().NewCounter("TransactionThrottlerRequests", "transaction throttler requests"), - requestsThrottled: env.Exporter().NewCounter("TransactionThrottlerThrottled", "transaction throttler requests throttled"), + requestsTotal: env.Exporter().NewCountersWithSingleLabel("TransactionThrottlerRequests", "transaction throttler requests", "workload"), + requestsThrottled: env.Exporter().NewCountersWithSingleLabel("TransactionThrottlerThrottled", "transaction throttler requests throttled", "workload"), } } @@ -249,7 +249,7 @@ func (t *txThrottler) Close() { // It returns true if the transaction should not proceed (the caller // should back off). Throttle requires that Open() was previously called // successfully. -func (t *txThrottler) Throttle(priority int) (result bool) { +func (t *txThrottler) Throttle(priority int, workload string) (result bool) { if !t.config.enabled { return false } @@ -261,9 +261,9 @@ func (t *txThrottler) Throttle(priority int) (result bool) { // are less likely to be throttled. result = t.state.throttle() && rand.Intn(sqlparser.MaxPriorityValue) < priority - t.requestsTotal.Add(1) + t.requestsTotal.Add(workload, 1) if result { - t.requestsThrottled.Add(1) + t.requestsThrottled.Add(workload, 1) } return result diff --git a/go/vt/vttablet/tabletserver/txthrottler/tx_throttler_test.go b/go/vt/vttablet/tabletserver/txthrottler/tx_throttler_test.go index ffb88bf21f6..bad64e7ff21 100644 --- a/go/vt/vttablet/tabletserver/txthrottler/tx_throttler_test.go +++ b/go/vt/vttablet/tabletserver/txthrottler/tx_throttler_test.go @@ -49,7 +49,7 @@ func TestDisabledThrottler(t *testing.T) { Shard: "shard", }) assert.Nil(t, throttler.Open()) - assert.False(t, throttler.Throttle(0)) + assert.False(t, throttler.Throttle(0, "some_workload")) throttlerImpl, _ := throttler.(*txThrottler) assert.Zero(t, throttlerImpl.throttlerRunning.Get()) throttler.Close() @@ -125,9 +125,9 @@ func TestEnabledThrottler(t *testing.T) { assert.Nil(t, throttlerImpl.Open()) assert.Equal(t, int64(1), throttlerImpl.throttlerRunning.Get()) - assert.False(t, throttlerImpl.Throttle(100)) - assert.Equal(t, int64(1), throttlerImpl.requestsTotal.Get()) - assert.Zero(t, throttlerImpl.requestsThrottled.Get()) + assert.False(t, throttlerImpl.Throttle(100, "some_workload")) + assert.Equal(t, int64(1), throttlerImpl.requestsTotal.Counts()["some_workload"]) + assert.Zero(t, throttlerImpl.requestsThrottled.Counts()["some_workload"]) throttlerImpl.state.StatsUpdate(tabletStats) // This calls replication lag thing rdonlyTabletStats := &discovery.TabletHealth{ @@ -138,14 +138,14 @@ func TestEnabledThrottler(t *testing.T) { // This call should not be forwarded to the go/vt/throttlerImpl.Throttler object. throttlerImpl.state.StatsUpdate(rdonlyTabletStats) // The second throttle call should reject. - assert.True(t, throttlerImpl.Throttle(100)) - assert.Equal(t, int64(2), throttlerImpl.requestsTotal.Get()) - assert.Equal(t, int64(1), throttlerImpl.requestsThrottled.Get()) + assert.True(t, throttlerImpl.Throttle(100, "some_workload")) + assert.Equal(t, int64(2), throttlerImpl.requestsTotal.Counts()["some_workload"]) + assert.Equal(t, int64(1), throttlerImpl.requestsThrottled.Counts()["some_workload"]) // This call should not throttle due to priority. Check that's the case and counters agree. - assert.False(t, throttlerImpl.Throttle(0)) - assert.Equal(t, int64(3), throttlerImpl.requestsTotal.Get()) - assert.Equal(t, int64(1), throttlerImpl.requestsThrottled.Get()) + assert.False(t, throttlerImpl.Throttle(0, "some_workload")) + assert.Equal(t, int64(3), throttlerImpl.requestsTotal.Counts()["some_workload"]) + assert.Equal(t, int64(1), throttlerImpl.requestsThrottled.Counts()["some_workload"]) throttlerImpl.Close() assert.Zero(t, throttlerImpl.throttlerRunning.Get()) } From 5ee8f876c4131d4adb7a5084f4c1171f2b5c772f Mon Sep 17 00:00:00 2001 From: "Eduardo J. Ortega U" <5791035+ejortegau@users.noreply.github.com> Date: Tue, 18 Jul 2023 14:50:06 +0200 Subject: [PATCH 2/3] Fix unit tests Signed-off-by: Eduardo J. Ortega U <5791035+ejortegau@users.noreply.github.com> --- go/vt/vttablet/tabletserver/query_executor_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletserver/query_executor_test.go b/go/vt/vttablet/tabletserver/query_executor_test.go index 3ab653bf50c..8a341791169 100644 --- a/go/vt/vttablet/tabletserver/query_executor_test.go +++ b/go/vt/vttablet/tabletserver/query_executor_test.go @@ -1806,6 +1806,6 @@ func (m mockTxThrottler) Open() (err error) { func (m mockTxThrottler) Close() { } -func (m mockTxThrottler) Throttle(priority int) (result bool) { +func (m mockTxThrottler) Throttle(priority int, workload string) (result bool) { return m.throttle } From 87588643ef6cd6682d818329a262128845d6484f Mon Sep 17 00:00:00 2001 From: "Eduardo J. Ortega U" <5791035+ejortegau@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:06:32 +0200 Subject: [PATCH 3/3] Empty commit to re-trigger CI Signed-off-by: Eduardo J. Ortega U <5791035+ejortegau@users.noreply.github.com>