From c6db17cff4072ce7edc0ca0a4f416ce5aded0acd Mon Sep 17 00:00:00 2001 From: Omar Date: Mon, 25 Oct 2021 15:23:28 +0000 Subject: [PATCH 1/7] Added substrate_number_leaves metrics --- .gitignore | 1 + dot/metrics/collector.go | 6 ++++-- dot/metrics/metrics.go | 6 +++--- dot/network/service.go | 4 ++-- dot/node.go | 2 +- dot/state/service.go | 2 ++ dot/state/service_test.go | 4 ++-- lib/grandpa/grandpa_test.go | 2 +- 8 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index b349c464f9..aef1849652 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ *.idea *.vscode *.sublime +*.devcontainer # Output of go coverage tool when used with LiteIDE *.out diff --git a/dot/metrics/collector.go b/dot/metrics/collector.go index 82f17e2336..9e2bbc291e 100644 --- a/dot/metrics/collector.go +++ b/dot/metrics/collector.go @@ -65,7 +65,8 @@ func (c *Collector) AddGauge(g GaugeMetrics) { } func (c *Collector) startCollectGauges() { - t := time.NewTicker(Refresh) + //TODO: Should we better add individual RefreshInterval for each `GaugeMetrics`or `label inside the gauges map`? + t := time.NewTicker(RefreshInterval) defer func() { t.Stop() c.wg.Done() @@ -89,6 +90,7 @@ func (c *Collector) startCollectGauges() { } func (c *Collector) startCollectProccessMetrics() { + //TODO: Should we better add individual RefreshInterval for each `GaugeMetrics`or `label inside the gauges map`? cpuStats := make([]*ethmetrics.CPUStats, 2) memStats := make([]*runtime.MemStats, 2) for i := 0; i < len(memStats); i++ { @@ -110,7 +112,7 @@ func (c *Collector) startCollectProccessMetrics() { memUsed = ethmetrics.GetOrRegisterGauge("system/memory/used", ethmetrics.DefaultRegistry) ) - t := time.NewTicker(Refresh) + t := time.NewTicker(RefreshInterval) defer func() { t.Stop() c.wg.Done() diff --git a/dot/metrics/metrics.go b/dot/metrics/metrics.go index 94c2067317..ab44a95086 100644 --- a/dot/metrics/metrics.go +++ b/dot/metrics/metrics.go @@ -29,9 +29,9 @@ import ( var logger = log.New("pkg", "metrics") const ( - // Refresh is the refresh time for publishing metrics. - Refresh = time.Second * 10 - refreshFreq = int64(Refresh / time.Second) + // RefreshInterval is the refresh time for publishing metrics. + RefreshInterval = time.Second * 10 + refreshFreq = int64(RefreshInterval / time.Second) ) // PublishMetrics function will export the /metrics endpoint to prometheus process diff --git a/dot/network/service.go b/dot/network/service.go index 5e9fba5d41..b852c093d4 100644 --- a/dot/network/service.go +++ b/dot/network/service.go @@ -301,7 +301,7 @@ func (s *Service) collectNetworkMetrics() { syncedBlocks.Update(num.Int64()) } - time.Sleep(gssmrmetrics.Refresh) + time.Sleep(gssmrmetrics.RefreshInterval) } } @@ -713,7 +713,7 @@ func (s *Service) NodeRoles() byte { return s.cfg.Roles } -// CollectGauge will be used to collect coutable metrics from network service +// CollectGauge will be used to collect countable metrics from network service func (s *Service) CollectGauge() map[string]int64 { var isSynced int64 if !s.syncer.IsSynced() { diff --git a/dot/node.go b/dot/node.go index 6b30f4784c..9ac82d5459 100644 --- a/dot/node.go +++ b/dot/node.go @@ -211,7 +211,7 @@ func LoadGlobalNodeName(basepath string) (nodename string, err error) { // NewNode creates a new dot node from a dot node configuration func NewNode(cfg *Config, ks *keystore.GlobalKeystore, stopFunc func()) (*Node, error) { // set garbage collection percent to 10% - // can be overwritten by setting the GOGC env veriable, which defaults to 100 + // can be overwritten by setting the GOGC env variable, which defaults to 100 prev := debug.SetGCPercent(10) if prev != 100 { debug.SetGCPercent(prev) diff --git a/dot/state/service.go b/dot/state/service.go index 1d937d6ea0..54b74434b2 100644 --- a/dot/state/service.go +++ b/dot/state/service.go @@ -34,6 +34,7 @@ import ( const readyPoolTransactionsMetrics = "gossamer/ready/pool/transaction/metrics" const readyPriorityQueueTransactions = "gossamer/ready/queue/transaction/metrics" +const substrateNumberLeaves = "gossamer/substrate_number_leaves/metrics" var logger = log.New("pkg", "state") @@ -358,5 +359,6 @@ func (s *Service) CollectGauge() map[string]int64 { return map[string]int64{ readyPoolTransactionsMetrics: int64(s.Transaction.pool.Len()), readyPriorityQueueTransactions: int64(s.Transaction.queue.Len()), + substrateNumberLeaves: int64(len(s.Block.Leaves())), } } diff --git a/dot/state/service_test.go b/dot/state/service_test.go index dcd3a46d98..102e7d2bfa 100644 --- a/dot/state/service_test.go +++ b/dot/state/service_test.go @@ -434,7 +434,7 @@ func TestStateServiceMetrics(t *testing.T) { hashes[i] = h } - time.Sleep(time.Second + metrics.Refresh) + time.Sleep(time.Second + metrics.RefreshInterval) gpool := ethmetrics.GetOrRegisterGauge(readyPoolTransactionsMetrics, nil) gqueue := ethmetrics.GetOrRegisterGauge(readyPriorityQueueTransactions, nil) @@ -444,7 +444,7 @@ func TestStateServiceMetrics(t *testing.T) { serv.Transaction.pool.Remove(hashes[0]) serv.Transaction.queue.Pop() - time.Sleep(time.Second + metrics.Refresh) + time.Sleep(time.Second + metrics.RefreshInterval) require.Equal(t, int64(1), gpool.Value()) require.Equal(t, int64(1), gqueue.Value()) } diff --git a/lib/grandpa/grandpa_test.go b/lib/grandpa/grandpa_test.go index 6502fed404..8e843cd80a 100644 --- a/lib/grandpa/grandpa_test.go +++ b/lib/grandpa/grandpa_test.go @@ -1274,7 +1274,7 @@ func TestFinalRoundGaugeMetric(t *testing.T) { go coll.Start() - time.Sleep(metrics.Refresh + time.Second) + time.Sleep(metrics.RefreshInterval + time.Second) gauge := ethmetrics.GetOrRegisterGauge(finalityGrandpaRoundMetrics, nil) require.Equal(t, gauge.Value(), int64(180)) } From 4b8ed182a534c1e5579990beb48aec886546099a Mon Sep 17 00:00:00 2001 From: Omar Date: Wed, 27 Oct 2021 13:19:58 +0000 Subject: [PATCH 2/7] Linter fix with: gofmt -w . --- dot/metrics/metrics.go | 4 ++-- scripts/ci.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dot/metrics/metrics.go b/dot/metrics/metrics.go index ab44a95086..ae22c59e05 100644 --- a/dot/metrics/metrics.go +++ b/dot/metrics/metrics.go @@ -30,8 +30,8 @@ var logger = log.New("pkg", "metrics") const ( // RefreshInterval is the refresh time for publishing metrics. - RefreshInterval = time.Second * 10 - refreshFreq = int64(RefreshInterval / time.Second) + RefreshInterval = time.Second * 10 + refreshFreq = int64(RefreshInterval / time.Second) ) // PublishMetrics function will export the /metrics endpoint to prometheus process diff --git a/scripts/ci.go b/scripts/ci.go index 457dcd5bba..06c22ad644 100755 --- a/scripts/ci.go +++ b/scripts/ci.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the gossamer library. If not, see . +//go:build none // +build none package main From fbd3702446981c77b2bdf63f11bd41d484a26cf0 Mon Sep 17 00:00:00 2001 From: Omar Date: Wed, 27 Oct 2021 13:25:32 +0000 Subject: [PATCH 3/7] Grouped constants --- dot/state/service.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dot/state/service.go b/dot/state/service.go index 54b74434b2..95ecd487d6 100644 --- a/dot/state/service.go +++ b/dot/state/service.go @@ -32,9 +32,11 @@ import ( log "github.com/ChainSafe/log15" ) -const readyPoolTransactionsMetrics = "gossamer/ready/pool/transaction/metrics" -const readyPriorityQueueTransactions = "gossamer/ready/queue/transaction/metrics" -const substrateNumberLeaves = "gossamer/substrate_number_leaves/metrics" +const ( + readyPoolTransactionsMetrics = "gossamer/ready/pool/transaction/metrics" + readyPriorityQueueTransactions = "gossamer/ready/queue/transaction/metrics" + substrateNumberLeaves = "gossamer/substrate_number_leaves/metrics" +) var logger = log.New("pkg", "state") From 9bed42b61049cbfa629543bf16a8c0dcb84abb16 Mon Sep 17 00:00:00 2001 From: Omar Date: Thu, 28 Oct 2021 15:36:06 +0000 Subject: [PATCH 4/7] =?UTF-8?q?Fixed=20memmory=20issue=20=E2=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dot/state/service_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/dot/state/service_test.go b/dot/state/service_test.go index 102e7d2bfa..e6f4f378af 100644 --- a/dot/state/service_test.go +++ b/dot/state/service_test.go @@ -39,6 +39,23 @@ import ( "github.com/stretchr/testify/require" ) +// var testGenesisHeader = &types.Header{ +// Number: big.NewInt(0), +// StateRoot: trie.EmptyHash, +// Digest: types.NewDigest(), +// } + +// func newTestBlockState(t *testing.T, header *types.Header) *BlockState { +// db := NewInMemoryDB(t) +// if header == nil { +// header = testGenesisHeader +// } + +// bs, err := NewBlockStateFromGenesis(db, header) +// require.NoError(t, err) +// return bs +// } + // helper method to create and start test state service func newTestService(t *testing.T) (state *Service) { testDir := utils.NewTestDir(t) @@ -410,6 +427,7 @@ func TestStateServiceMetrics(t *testing.T) { ethmetrics.Enabled = true serv := NewService(config) serv.Transaction = NewTransactionState() + serv.Block = newTestBlockState(t, testGenesisHeader) m := metrics.NewCollector(context.Background()) m.AddGauge(serv) From 06f354e20752c0be3821307e864c336df87a41fb Mon Sep 17 00:00:00 2001 From: Omar Date: Thu, 28 Oct 2021 15:56:12 +0000 Subject: [PATCH 5/7] Micro optimization --- dot/network/service.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dot/network/service.go b/dot/network/service.go index b852c093d4..cf233916ff 100644 --- a/dot/network/service.go +++ b/dot/network/service.go @@ -715,11 +715,9 @@ func (s *Service) NodeRoles() byte { // CollectGauge will be used to collect countable metrics from network service func (s *Service) CollectGauge() map[string]int64 { - var isSynced int64 + var isSynced int64 = 0 if !s.syncer.IsSynced() { isSynced = 1 - } else { - isSynced = 0 } return map[string]int64{ From 1906c8c542731dfda0ee7b3c6aa955535103c0f0 Mon Sep 17 00:00:00 2001 From: Omar Date: Thu, 28 Oct 2021 16:12:12 +0000 Subject: [PATCH 6/7] Fixed for lint --- dot/network/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dot/network/service.go b/dot/network/service.go index f5f4eebc3c..c192486859 100644 --- a/dot/network/service.go +++ b/dot/network/service.go @@ -715,7 +715,7 @@ func (s *Service) NodeRoles() byte { // CollectGauge will be used to collect countable metrics from network service func (s *Service) CollectGauge() map[string]int64 { - var isSynced int64 = 0 + var isSynced int64 if !s.syncer.IsSynced() { isSynced = 1 } From bd513a74940b87bd961418f099e7ee6d773abc56 Mon Sep 17 00:00:00 2001 From: Omar Date: Thu, 28 Oct 2021 16:16:42 +0000 Subject: [PATCH 7/7] Removed unnecessary comments --- dot/state/service_test.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/dot/state/service_test.go b/dot/state/service_test.go index 80e06deadf..193cbb4a53 100644 --- a/dot/state/service_test.go +++ b/dot/state/service_test.go @@ -39,23 +39,6 @@ import ( "github.com/stretchr/testify/require" ) -// var testGenesisHeader = &types.Header{ -// Number: big.NewInt(0), -// StateRoot: trie.EmptyHash, -// Digest: types.NewDigest(), -// } - -// func newTestBlockState(t *testing.T, header *types.Header) *BlockState { -// db := NewInMemoryDB(t) -// if header == nil { -// header = testGenesisHeader -// } - -// bs, err := NewBlockStateFromGenesis(db, header) -// require.NoError(t, err) -// return bs -// } - // helper method to create and start test state service func newTestService(t *testing.T) (state *Service) { testDir := utils.NewTestDir(t)