Antrea Prometheus integration (antrea-io#236)

Integrate with Prometheus monitoring solution. Integration of the Prometheus client into Antrea controller and agent allows the exposure of various metrics to Prometheus server. In addition to Antrea's own set of metrics, Prometheus client will also expose metrics which are defined by various components which are part of the Antrea ecosystem, e.g golang, Prometheus itself etc.
ksamoray · Apr 21, 2020 · ea126e6 · ea126e6
1 parent d9fdf59
commit ea126e6
Show file tree

Hide file tree

Showing 12 changed files with 294 additions and 14 deletions.
diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -332,6 +332,15 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
     #apiPort: 10350
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+    #enablePrometheusMetrics: false
+
+    # Enable golang metrics exposure via Prometheus.
+    #enablePrometheusGoMetrics: false
+
+    # Enable process metrics exposure via Prometheus.
+    #enablePrometheusProcessMetrics: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -354,12 +363,21 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
     #apiPort: 10349
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
+
+    # Enable golang metrics exposure via Prometheus.
+    #enablePrometheusGoMetrics: false
+
+    # Enable process metrics exposure via Prometheus.
+    #enablePrometheusProcessMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-t4m46b8f6h
+  name: antrea-config-kmc5g2c4d4
   namespace: kube-system
 ---
 apiVersion: v1
@@ -459,7 +477,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-t4m46b8f6h
+          name: antrea-config-kmc5g2c4d4
         name: antrea-config
       - hostPath:
           path: /var/log/antrea
@@ -658,7 +676,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-t4m46b8f6h
+          name: antrea-config-kmc5g2c4d4
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -332,6 +332,15 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
     #apiPort: 10350
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+    #enablePrometheusMetrics: false
+
+    # Enable golang metrics exposure via Prometheus.
+    #enablePrometheusGoMetrics: false
+
+    # Enable process metrics exposure via Prometheus.
+    #enablePrometheusProcessMetrics: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -354,12 +363,21 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
     #apiPort: 10349
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
+
+    # Enable golang metrics exposure via Prometheus.
+    #enablePrometheusGoMetrics: false
+
+    # Enable process metrics exposure via Prometheus.
+    #enablePrometheusProcessMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-5754dg84hf
+  name: antrea-config-ccg2965597
   namespace: kube-system
 ---
 apiVersion: v1
@@ -459,7 +477,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-5754dg84hf
+          name: antrea-config-ccg2965597
         name: antrea-config
       - hostPath:
           path: /var/log/antrea
@@ -658,7 +676,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-5754dg84hf
+          name: antrea-config-ccg2965597
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -332,6 +332,15 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
     #apiPort: 10350
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+    #enablePrometheusMetrics: false
+
+    # Enable golang metrics exposure via Prometheus.
+    #enablePrometheusGoMetrics: false
+
+    # Enable process metrics exposure via Prometheus.
+    #enablePrometheusProcessMetrics: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -354,12 +363,21 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
     #apiPort: 10349
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
+
+    # Enable golang metrics exposure via Prometheus.
+    #enablePrometheusGoMetrics: false
+
+    # Enable process metrics exposure via Prometheus.
+    #enablePrometheusProcessMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-c7579447k2
+  name: antrea-config-4btg5ctkgk
   namespace: kube-system
 ---
 apiVersion: v1
@@ -468,7 +486,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-c7579447k2
+          name: antrea-config-4btg5ctkgk
         name: antrea-config
       - hostPath:
           path: /var/log/antrea
@@ -699,7 +717,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-c7579447k2
+          name: antrea-config-4btg5ctkgk
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -332,6 +332,15 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
     #apiPort: 10350
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+    #enablePrometheusMetrics: false
+
+    # Enable golang metrics exposure via Prometheus.
+    #enablePrometheusGoMetrics: false
+
+    # Enable process metrics exposure via Prometheus.
+    #enablePrometheusProcessMetrics: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -354,12 +363,21 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
     #apiPort: 10349
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
+
+    # Enable golang metrics exposure via Prometheus.
+    #enablePrometheusGoMetrics: false
+
+    # Enable process metrics exposure via Prometheus.
+    #enablePrometheusProcessMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-428d4tg64g
+  name: antrea-config-cgb74bc9c7
   namespace: kube-system
 ---
 apiVersion: v1
@@ -459,7 +477,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-428d4tg64g
+          name: antrea-config-cgb74bc9c7
         name: antrea-config
       - hostPath:
           path: /var/log/antrea
@@ -658,7 +676,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-428d4tg64g
+          name: antrea-config-cgb74bc9c7
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -46,3 +46,12 @@
 # Note that if it's set to another value, the `containerPort` of the `api` port of the
 # `antrea-agent` container must be set to the same value.
 #apiPort: 10350
+
+# Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+#enablePrometheusMetrics: false
+
+# Enable golang metrics exposure via Prometheus.
+#enablePrometheusGoMetrics: false
+
+# Enable process metrics exposure via Prometheus.
+#enablePrometheusProcessMetrics: false
diff --git a/build/yamls/base/conf/antrea-controller.conf b/build/yamls/base/conf/antrea-controller.conf
@@ -2,3 +2,12 @@
 # Note that if it's set to another value, the `containerPort` of the `api` port of the
 # `antrea-controller` container must be set to the same value.
 #apiPort: 10349
+
+# Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+#enablePrometheusMetrics: false
+
+# Enable golang metrics exposure via Prometheus.
+#enablePrometheusGoMetrics: false
+
+# Enable process metrics exposure via Prometheus.
+#enablePrometheusProcessMetrics: false
diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -30,6 +30,7 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/agent/controller/networkpolicy"
 	"github.com/vmware-tanzu/antrea/pkg/agent/controller/noderoute"
 	"github.com/vmware-tanzu/antrea/pkg/agent/interfacestore"
+	"github.com/vmware-tanzu/antrea/pkg/agent/metrics"
 	"github.com/vmware-tanzu/antrea/pkg/agent/openflow"
 	"github.com/vmware-tanzu/antrea/pkg/agent/querier"
 	"github.com/vmware-tanzu/antrea/pkg/agent/route"
@@ -161,6 +162,13 @@ func run(o *Options) error {
 		ovsBridgeClient,
 		networkPolicyController)
 
+	if o.config.EnablePrometheusMetrics {
+		go metrics.InitializePrometheusMetrics(
+			o.config.EnablePrometheusGoMetrics,
+			o.config.EnablePrometheusProcessMetrics,
+			o.config.OVSBridge, ifaceStore, ofClient)
+	}
+
 	agentMonitor := monitor.NewAgentMonitor(crdClient, agentQuerier)
 
 	go agentMonitor.Run(stopCh)

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -75,4 +75,13 @@ type AgentConfig struct {
 	// APIPort is the port for the antrea-agent APIServer to serve on.
 	// Defaults to 10350.
 	APIPort int `yaml:"apiPort,omitempty"`
+	// Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+	// Defaults to false.
+	EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"`
+	// Enable golang metrics exposure via Prometheus
+	// Defaults to false.
+	EnablePrometheusGoMetrics bool `yaml:"enablePrometheusGoMetrics,omitempty"`
+	// Enable process metrics exposure via Prometheus
+	// Defaults to false.
+	EnablePrometheusProcessMetrics bool `yaml:"enablePrometheusProcessMetrics,omitempty"`
 }
diff --git a/cmd/antrea-controller/config.go b/cmd/antrea-controller/config.go
@@ -25,4 +25,13 @@ type ControllerConfig struct {
 	// APIPort is the port for the antrea-controller APIServer to serve on.
 	// Defaults to 10349.
 	APIPort int `yaml:"apiPort,omitempty"`
+	// Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+	// Defaults to false.
+	EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"`
+	// Enable golang metrics exposure via Prometheus
+	// Defaults to false.
+	EnablePrometheusGoMetrics bool `yaml:"enablePrometheusGoMetrics,omitempty"`
+	// Enable process metrics exposure via Prometheus
+	// Defaults to false.
+	EnablePrometheusProcessMetrics bool `yaml:"enablePrometheusProcessMetrics,omitempty"`
 }
diff --git a/cmd/antrea-controller/controller.go b/cmd/antrea-controller/controller.go
@@ -17,8 +17,12 @@ package main
 import (
 	"fmt"
 	"net"
+	"net/http"
+	"os"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	genericopenapi "k8s.io/apiserver/pkg/endpoints/openapi"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
@@ -78,7 +82,8 @@ func run(o *Options) error {
 		addressGroupStore,
 		appliedToGroupStore,
 		networkPolicyStore,
-		controllerQuerier)
+		controllerQuerier,
+		o.config.EnablePrometheusMetrics)
 	if err != nil {
 		return fmt.Errorf("error creating API server config: %v", err)
 	}
@@ -100,23 +105,63 @@ func run(o *Options) error {
 
 	go apiServer.GenericAPIServer.PrepareRun().Run(stopCh)
 
+	if o.config.EnablePrometheusMetrics {
+		go initializePrometheusMetrics(
+			o.config.EnablePrometheusGoMetrics,
+			o.config.EnablePrometheusProcessMetrics)
+	}
+
 	<-stopCh
 	klog.Info("Stopping Antrea controller")
 	return nil
 }
 
+// Initialize Prometheus metrics collection.
+func initializePrometheusMetrics(
+	enablePrometheusGoMetrics bool,
+	enablePrometheusProcessMetrics bool) {
+	hostname, err := os.Hostname()
+	if err != nil {
+		klog.Errorf("Failed to retrieve agent node name, %v", err)
+	}
+
+	klog.Info("Initializing prometheus")
+	gaugeHost := prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "antrea_controller_host",
+		Help: "Antrea controller hostname (as a label), typically used in grouping/aggregating stats; " +
+			"The value of the gauge is always set to 1.",
+		ConstLabels: prometheus.Labels{"host": hostname},
+	})
+	gaugeHost.Set(1)
+	prometheus.MustRegister(gaugeHost)
+	http.Handle("/metrics", promhttp.Handler())
+
+	if !enablePrometheusGoMetrics {
+		klog.Info("Golang metrics are disabled")
+		prometheus.Unregister(prometheus.NewGoCollector())
+	}
+	if !enablePrometheusProcessMetrics {
+		klog.Info("Process metrics are disabled")
+		prometheus.Unregister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
+	}
+}
+
 func createAPIServerConfig(kubeconfig string,
 	bindPort int,
 	addressGroupStore storage.Interface,
 	appliedToGroupStore storage.Interface,
 	networkPolicyStore storage.Interface,
-	controllerQuerier querier.ControllerQuerier) (*apiserver.Config, error) {
+	controllerQuerier querier.ControllerQuerier,
+	enablePrometheusMetrics bool) (*apiserver.Config, error) {
 	// TODO:
 	// 1. Support user-provided certificate.
 	secureServing := genericoptions.NewSecureServingOptions().WithLoopback()
 	authentication := genericoptions.NewDelegatingAuthenticationOptions()
 	authorization := genericoptions.NewDelegatingAuthorizationOptions()
 
+	if enablePrometheusMetrics {
+		authorization.WithAlwaysAllowPaths("/metrics")
+	}
 	// Set the PairName but leave certificate directory blank to generate in-memory by default
 	secureServing.ServerCert.CertDirectory = ""
 	secureServing.ServerCert.PairName = "antrea-apiserver"

diff --git a/go.mod b/go.mod
@@ -25,6 +25,7 @@ require (
 	github.com/imdario/mergo v0.3.7 // indirect
 	github.com/j-keck/arping v1.0.0
 	github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd
+	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
 	github.com/satori/go.uuid v1.2.0
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.3