Skip to content

Commit

Permalink
e2e: add manifest fields to enable Prometheus on nodes (backport #313) (
Browse files Browse the repository at this point in the history 
#573)

* e2e: add manifest fields to enable Prometheus on nodes (#313)

* Add Prometheus and PrometheusListenAddr flags

* Add Prometheus flag at root level, for all nodes

* Add comment

* Add PrometheusProxyPort; remove PrometheusListenAddr

* Update startup log message

* Update startup log message

* Don't print the whole line in docker template

* Remove flag from ManifestNode

* Do not generate PrometheusProxyPort if config is disabled

(cherry picked from commit a95a1df)

# Conflicts:
#	test/e2e/pkg/testnet.go

* Solve merge conflicts

---------

Co-authored-by: Hernán Vanzetto <15466498+hvanz@users.noreply.github.com>
Co-authored-by: hvanz <hernan.vanzetto@gmail.com>
  • Loading branch information
@mergify @hvanz
3 people committed Mar 23, 2023
1 parent f2591e2 commit cc7601f
Show file tree
Hide file tree
Showing 5 changed files with 70 additions and 34 deletions.
6 changes: 6 additions & 0 deletions test/e2e/pkg/infra/docker/docker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,9 @@ services:
ports:
- 26656
- {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657
{{- if .PrometheusProxyPort }}
- {{ .PrometheusProxyPort }}:26660
{{- end }}
- 6060
volumes:
- ./{{ .Name }}:/cometbft
Expand All @@ -107,6 +110,9 @@ services:
ports:
- 26656
- {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657
{{- if .PrometheusProxyPort }}
- {{ .PrometheusProxyPort }}:26660
{{- end }}
- 6060
volumes:
- ./{{ .Name }}:/cometbft
Expand Down
4 changes: 4 additions & 0 deletions test/e2e/pkg/manifest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,10 @@ type Manifest struct {
LoadTxSizeBytes int `toml:"load_tx_size_bytes"`
LoadTxBatchSize int `toml:"load_tx_batch_size"`
LoadTxConnections int `toml:"load_tx_connections"`

// Enable or disable Prometheus metrics on all nodes.
// Defaults to false (disabled).
Prometheus bool `toml:"prometheus"`
}

// ManifestNode represents a node in a testnet manifest.
Expand Down
83 changes: 50 additions & 33 deletions test/e2e/pkg/testnet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ import (
)

const (
randomSeed int64 = 2308084734268
proxyPortFirst uint32 = 5701
randomSeed int64 = 2308084734268
proxyPortFirst uint32 = 5701
prometheusProxyPortFirst uint32 = 6701

defaultBatchSize = 2
defaultConnections = 1
Expand Down Expand Up @@ -72,35 +73,36 @@ type Testnet struct {
LoadTxConnections int
ABCIProtocol string
UpgradeVersion string
Prometheus bool
}

// Node represents a CometBFT node in a testnet.
type Node struct {
Name string
Version string
Testnet *Testnet
Mode Mode
PrivvalKey crypto.PrivKey
NodeKey crypto.PrivKey
IP net.IP
ProxyPort uint32
StartAt int64
FastSync string
StateSync bool
Mempool string
Database string
ABCIProtocol Protocol
PrivvalProtocol Protocol
PersistInterval uint64
SnapshotInterval uint64
RetainBlocks uint64
Seeds []*Node
PersistentPeers []*Node
Perturbations []Perturbation
Misbehaviors map[int64]string

// SendNoLoad determines if the e2e test should send load to this node.
SendNoLoad bool
Name string
Version string
Testnet *Testnet
Mode Mode
PrivvalKey crypto.PrivKey
NodeKey crypto.PrivKey
IP net.IP
ProxyPort uint32
StartAt int64
FastSync string
StateSync bool
Mempool string
Database string
ABCIProtocol Protocol
PrivvalProtocol Protocol
PersistInterval uint64
SnapshotInterval uint64
RetainBlocks uint64
Seeds []*Node
PersistentPeers []*Node
Perturbations []Perturbation
Misbehaviors map[int64]string
SendNoLoad bool
Prometheus bool
PrometheusProxyPort uint32
}

// LoadTestnet loads a testnet from a manifest file, using the filename to
Expand All @@ -112,6 +114,7 @@ func LoadTestnet(manifest Manifest, fname string, ifd InfrastructureData) (*Test
dir := strings.TrimSuffix(fname, filepath.Ext(fname))
keyGen := newKeyGenerator(randomSeed)
proxyPortGen := newPortGenerator(proxyPortFirst)
prometheusProxyPortGen := newPortGenerator(prometheusProxyPortFirst)
_, ipNet, err := net.ParseCIDR(ifd.Network)
if err != nil {
return nil, fmt.Errorf("invalid IP network address %q: %w", ifd.Network, err)
Expand All @@ -132,6 +135,7 @@ func LoadTestnet(manifest Manifest, fname string, ifd InfrastructureData) (*Test
LoadTxConnections: manifest.LoadTxConnections,
ABCIProtocol: manifest.ABCIProtocol,
UpgradeVersion: manifest.UpgradeVersion,
Prometheus: manifest.Prometheus,
}
if len(manifest.KeyType) != 0 {
testnet.KeyType = manifest.KeyType
Expand Down Expand Up @@ -195,6 +199,7 @@ func LoadTestnet(manifest Manifest, fname string, ifd InfrastructureData) (*Test
Perturbations: []Perturbation{},
Misbehaviors: make(map[int64]string),
SendNoLoad: nodeManifest.SendNoLoad,
Prometheus: testnet.Prometheus,
}
if node.StartAt == testnet.InitialHeight {
node.StartAt = 0 // normalize to 0 for initial nodes, since code expects this
Expand All @@ -214,6 +219,9 @@ func LoadTestnet(manifest Manifest, fname string, ifd InfrastructureData) (*Test
if nodeManifest.PersistInterval != nil {
node.PersistInterval = *nodeManifest.PersistInterval
}
if node.Prometheus {
node.PrometheusProxyPort = prometheusProxyPortGen.Next()
}
for _, p := range nodeManifest.Perturb {
node.Perturbations = append(node.Perturbations, Perturbation(p))
}
Expand Down Expand Up @@ -327,13 +335,22 @@ func (n Node) Validate(testnet Testnet) error {
if !testnet.IP.Contains(n.IP) {
return fmt.Errorf("node IP %v is not in testnet network %v", n.IP, testnet.IP)
}
if n.ProxyPort > 0 {
if n.ProxyPort <= 1024 {
return fmt.Errorf("local port %v must be >1024", n.ProxyPort)
if n.ProxyPort == n.PrometheusProxyPort {
return fmt.Errorf("node local port %v used also for Prometheus local port", n.ProxyPort)
}
if n.ProxyPort > 0 && n.ProxyPort <= 1024 {
return fmt.Errorf("local port %v must be >1024", n.ProxyPort)
}
if n.PrometheusProxyPort > 0 && n.PrometheusProxyPort <= 1024 {
return fmt.Errorf("local port %v must be >1024", n.PrometheusProxyPort)
}
for _, peer := range testnet.Nodes {
if peer.Name != n.Name && peer.ProxyPort == n.ProxyPort {
return fmt.Errorf("peer %q also has local port %v", peer.Name, n.ProxyPort)
}
for _, peer := range testnet.Nodes {
if peer.Name != n.Name && peer.ProxyPort == n.ProxyPort {
return fmt.Errorf("peer %q also has local port %v", peer.Name, n.ProxyPort)
if n.PrometheusProxyPort > 0 {
if peer.Name != n.Name && peer.PrometheusProxyPort == n.PrometheusProxyPort {
return fmt.Errorf("peer %q also has local port %v", peer.Name, n.PrometheusProxyPort)
}
}
}
Expand Down
5 changes: 5 additions & 0 deletions test/e2e/runner/setup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,11 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) {
}
cfg.P2P.PersistentPeers += peer.AddressP2P(true)
}

if node.Prometheus {
cfg.Instrumentation.Prometheus = true
}

return cfg, nil
}

Expand Down
6 changes: 5 additions & 1 deletion test/e2e/runner/start.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,11 @@ func Start(testnet *e2e.Testnet) error {
if _, err := waitForNode(node, 0, 15*time.Second); err != nil {
return err
}
logger.Info("start", "msg", log.NewLazySprintf("Node %v up on http://127.0.0.1:%v", node.Name, node.ProxyPort))
if node.PrometheusProxyPort > 0 {
logger.Info("start", "msg", log.NewLazySprintf("Node %v up on http://127.0.0.1:%v; with Prometheus on http://127.0.0.1:%v/metrics", node.Name, node.ProxyPort, node.PrometheusProxyPort))
} else {
logger.Info("start", "msg", log.NewLazySprintf("Node %v up on http://127.0.0.1:%v", node.Name, node.ProxyPort))
}
}

networkHeight := testnet.InitialHeight
Expand Down

0 comments on commit cc7601f

Please sign in to comment.