From 24d24bb4199cb680b4111a044e0998dfb041b1ad Mon Sep 17 00:00:00 2001 From: Manan Gupta <35839558+GuptaManan100@users.noreply.github.com> Date: Thu, 15 Jun 2023 10:01:05 +0530 Subject: [PATCH] Add metric for showing the errant GTIDs in VTOrc (#13281) * feat: add metric for errant gtid Signed-off-by: Manan Gupta * feat: update the errant gtid map always, so that once errant gtids are fixed, they are cleared Signed-off-by: Manan Gupta * feat: fix the test to match the new expectations Signed-off-by: Manan Gupta * feat: add mutex to protect the access to the map storing errant gtids Signed-off-by: Manan Gupta * feat: update the errant gtid map in a goroutine Signed-off-by: Manan Gupta --------- Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/api/api_test.go | 20 ++++++++++++++++++++ go/vt/vtorc/inst/instance_dao.go | 18 ++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/go/test/endtoend/vtorc/api/api_test.go b/go/test/endtoend/vtorc/api/api_test.go index 3af5725d3e9..16f1df2b856 100644 --- a/go/test/endtoend/vtorc/api/api_test.go +++ b/go/test/endtoend/vtorc/api/api_test.go @@ -17,7 +17,9 @@ limitations under the License. package api import ( + "encoding/json" "fmt" + "reflect" "testing" "time" @@ -197,5 +199,23 @@ func TestAPIEndpoints(t *testing.T) { require.NoError(t, err) assert.Equal(t, 400, status, resp) assert.Equal(t, "Filtering by shard without keyspace isn't supported\n", resp) + + // Also verify that the metric for errant GTIDs is reporting the correct information. + _, resp, err = utils.MakeAPICall(t, vtorc, "/debug/vars") + require.NoError(t, err) + resultMap := make(map[string]any) + err = json.Unmarshal([]byte(resp), &resultMap) + require.NoError(t, err) + errantGTIDMap := reflect.ValueOf(resultMap["ErrantGtidMap"]) + errantGtidTablets := errantGTIDMap.MapKeys() + require.Len(t, errantGtidTablets, 3) + + errantGTIDinReplica := "" + for _, tabletKey := range errantGtidTablets { + if tabletKey.String() == replica.Alias { + errantGTIDinReplica = errantGTIDMap.MapIndex(tabletKey).Interface().(string) + } + } + require.NotEmpty(t, errantGTIDinReplica) }) } diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index 26c1eb1bfa6..7d122ea61a0 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -31,6 +31,7 @@ import ( "github.com/rcrowley/go-metrics" "github.com/sjmudd/stopwatch" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/external/golib/sqlutils" vitessmysql "vitess.io/vitess/go/mysql" @@ -56,6 +57,12 @@ const ( var instanceReadChan = make(chan bool, backendDBConcurrency) var instanceWriteChan = make(chan bool, backendDBConcurrency) +var ( + // Mutex to protect the access of the following variable + errantGtidMapMu = sync.Mutex{} + errantGtidMap = make(map[string]string) +) + // Constant strings for Group Replication information // See https://dev.mysql.com/doc/refman/8.0/en/replication-group-members-table.html for additional information. const ( @@ -88,6 +95,11 @@ func init() { _ = metrics.Register("instance.write", writeInstanceCounter) _ = writeBufferLatency.AddMany([]string{"wait", "write"}) writeBufferLatency.Start("wait") + stats.NewStringMapFuncWithMultiLabels("ErrantGtidMap", "Metric to track the errant GTIDs detected by VTOrc", []string{"TabletAlias"}, "ErrantGtid", func() map[string]string { + errantGtidMapMu.Lock() + defer errantGtidMapMu.Unlock() + return errantGtidMap + }) go initializeInstanceDao() } @@ -429,6 +441,12 @@ Cleanup: instance.GtidErrant, err = vitessmysql.Subtract(redactedExecutedGtidSet.String(), redactedPrimaryExecutedGtidSet.String()) } } + // update the errant gtid map + go func() { + errantGtidMapMu.Lock() + defer errantGtidMapMu.Unlock() + errantGtidMap[topoproto.TabletAliasString(tablet.Alias)] = instance.GtidErrant + }() } latency.Stop("instance")