Metric collector and exporter

docs/content/en/docs/configuration/command-line.md

@@ -22,13 +22,16 @@ The following command-line options are supported:
 | [`--annotation-prefix`](#annotation-prefix)             | prefix without `/`         | `ingress.kubernetes.io` | v0.8  |
 | [`--default-backend-service`](#default-backend-service) | namespace/servicename      | haproxy's 404 page      |       |
 | [`--default-ssl-certificate`](#default-ssl-certificate) | namespace/secretname       | fake, auto generated    |       |
+| [`--healthz-port`](#stats)                              | port number                | `10254`                 |       |
 | [`--ingress-class`](#ingress-class)                     | name                       | `haproxy`               |       |
 | [`--kubeconfig`](#kubeconfig)                           | /path/to/kubeconfig        | in cluster config       |       |
 | [`--max-old-config-files`](#max-old-config-files)       | num of files               | `0`                     |       |
+| [`--profiling`](#stats)                                 | [true\|false]              | `true`                  |       |
 | [`--publish-service`](#publish-service)                 | namespace/servicename      |                         |       |
 | [`--rate-limit-update`](#rate-limit-update)             | uploads per second (float) | `0.5`                   |       |
 | [`--reload-strategy`](#reload-strategy)                 | [native\|reusesocket]      | `reusesocket`           |       |
 | [`--sort-backends`](#sort-backends)                     | [true\|false]              | `false`                 |       |
+| [`--stats-collect-processing-period`](#stats)           | time                       | `500ms`                 | v0.10 |
 | [`--tcp-services-configmap`](#tcp-services-configmap)   | namespace/configmapname    | no tcp svc              |       |
 | [`--verify-hostname`](#verify-hostname)                 | [true\|false]              | `true`                  |       |
 | [`--wait-before-shutdown`](#wait-before-shutdown)       | seconds as integer         | `0`                     | v0.8  |
@@ -179,6 +182,24 @@ in the same order.
 
 ---
 
+## Stats
+
+Configures an endpoint with statistics, debugging and health checks. The following URIs are provided:
+
+* `/healthz`: a healthz URI for the haproxy-ingress
+* `/metrics`: Prometheus compatible metrics exporter
+* `/debug/pprof`: profiling tools
+* `/build`: build information - controller name, version, git commit hash and repository
+* `/stop`: stops haproxy-ingress controller
+
+Options:
+
+* `--healthz-port`: Defines the port number haproxy-ingress should listen to. Defaults to `10254`.
+* `--profiling`: Configures if the profiling URI should be enabled. Defaults to `true`.
+* `--stats-collect-processing-period`: Defines the interval between two consecutive readings of haproxy's `Idle_pct`, used to generate `haproxy_processing_seconds_total` metric. haproxy updates Idle_pct every `500ms`, which makes that the best configuration value, and it's also the default if not configured. Values higher than `500ms` will produce a less accurate collect. Change to 0 (zero) to disable this metric.
+
+---
+
 ## --tcp-services-configmap
 
 Configure `--tcp-services-configmap` argument with `namespace/configmapname` resource with TCP

pkg/common/ingress/controller/controller.go

@@ -78,12 +78,13 @@ type Configuration struct {
 	AcmeTokenConfigmapName  string
 	AcmeTrackTLSAnn         bool
 
-	TCPConfigMapName      string
-	DefaultSSLCertificate string
-	VerifyHostname        bool
-	DefaultHealthzURL     string
-	PublishService        string
-	Backend               ingress.Controller
+	TCPConfigMapName       string
+	DefaultSSLCertificate  string
+	VerifyHostname         bool
+	DefaultHealthzURL      string
+	StatsCollectProcPeriod time.Duration
+	PublishService         string
+	Backend                ingress.Controller
 
 	UpdateStatus           bool
 	UseNodeInternalIP      bool
@@ -241,11 +242,11 @@ func (ic *GenericController) Start() {
 }
 
 // CreateDefaultSSLCertificate ...
-func (ic *GenericController) CreateDefaultSSLCertificate() (path, hash string) {
+func (ic *GenericController) CreateDefaultSSLCertificate() (path, hash string, notAfter time.Time) {
 	defCert, defKey := ssl.GetFakeSSLCert()
 	c, err := ssl.AddOrUpdateCertAndKey("default-fake-certificate", defCert, defKey, []byte{})
 	if err != nil {
 		glog.Fatalf("Error generating self signed certificate: %v", err)
 	}
-	return c.PemFileName, c.PemSHA
+	return c.PemFileName, c.PemSHA, c.Certificate.NotAfter
 }

pkg/common/ingress/controller/launch.go

@@ -107,6 +107,11 @@ func NewIngressController(backend ingress.Controller) *GenericController {
 
 		healthzPort = flags.Int("healthz-port", 10254, "port for healthz endpoint.")
 
+		statsCollectProcPeriod = flags.Duration("stats-collect-processing-period", 500*time.Millisecond,
+			`Defines the interval between two consecutive readings of haproxy's Idle_pct. haproxy
+		updates Idle_pct every 500ms, which makes that the best configuration value.
+		Change to 0 (zero) to disable this metric.`)
+
 		profiling = flags.Bool("profiling", true, `Enable profiling via web interface host:port/debug/pprof/`)
 
 		defSSLCertificate = flags.String("default-ssl-certificate", "", `Name of the secret
@@ -288,6 +293,7 @@ func NewIngressController(backend ingress.Controller) *GenericController {
 		DefaultSSLCertificate:   *defSSLCertificate,
 		VerifyHostname:          *verifyHostname,
 		DefaultHealthzURL:       *defHealthzURL,
+		StatsCollectProcPeriod:  *statsCollectProcPeriod,
 		PublishService:          *publishSvc,
 		Backend:                 backend,
 		ForceNamespaceIsolation: *forceIsolation,

pkg/controller/cache.go

@@ -136,7 +136,7 @@ func (c *cache) buildSecretName(defaultNamespace, secretName string) (string, er
 	)
 }
 
-func (c *cache) GetTLSSecretPath(defaultNamespace, secretName string) (file convtypes.File, err error) {
+func (c *cache) GetTLSSecretPath(defaultNamespace, secretName string) (file convtypes.CrtFile, err error) {
 	fullname, err := c.buildSecretName(defaultNamespace, secretName)
 	if err != nil {
 		return file, err
@@ -148,9 +148,10 @@ func (c *cache) GetTLSSecretPath(defaultNamespace, secretName string) (file conv
 	if sslCert.PemFileName == "" {
 		return file, fmt.Errorf("secret '%s' does not have keys 'tls.crt' and 'tls.key'", fullname)
 	}
-	file = convtypes.File{
+	file = convtypes.CrtFile{
 		Filename: sslCert.PemFileName,
 		SHA1Hash: sslCert.PemSHA,
+		NotAfter: sslCert.Certificate.NotAfter,
 	}
 	return file, nil
 }

pkg/controller/controller.go

@@ -47,6 +47,7 @@ type HAProxyController struct {
 	instance          haproxy.Instance
 	logger            *logger
 	cache             *cache
+	metrics           *metrics
 	stopCh            chan struct{}
 	acmeQueue         utils.Queue
 	leaderelector     types.LeaderElector
@@ -93,6 +94,7 @@ func (hc *HAProxyController) configController() {
 	hc.stopCh = hc.controller.GetStopCh()
 	hc.logger = &logger{depth: 1}
 	hc.cache = newCache(hc.cfg.Client, hc.storeLister, hc.controller)
+	hc.metrics = createMetrics()
 	var acmeSigner acme.Signer
 	if hc.cfg.AcmeServer {
 		electorID := fmt.Sprintf("%s-%s", hc.cfg.AcmeElectionID, hc.cfg.IngressClass)
@@ -111,6 +113,7 @@ func (hc *HAProxyController) configController() {
 		AcmeSigner:        acmeSigner,
 		AcmeQueue:         hc.acmeQueue,
 		LeaderElector:     hc.leaderelector,
+		Metrics:           hc.metrics,
 		ReloadStrategy:    *hc.reloadStrategy,
 		MaxOldConfigFiles: *hc.maxOldConfigFiles,
 		ValidateConfig:    *hc.validateConfig,
@@ -130,6 +133,11 @@ func (hc *HAProxyController) configController() {
 }
 
 func (hc *HAProxyController) startServices() {
+	if hc.cfg.StatsCollectProcPeriod.Milliseconds() > 0 {
+		go wait.Until(func() {
+			hc.instance.CalcIdleMetric()
+		}, hc.cfg.StatsCollectProcPeriod, hc.stopCh)
+	}
 	if hc.leaderelector != nil {
 		go hc.leaderelector.Run()
 	}
@@ -145,7 +153,7 @@ func (hc *HAProxyController) startServices() {
 	}
 }
 
-func (hc *HAProxyController) createDefaultSSLFile(cache convtypes.Cache) (tlsFile convtypes.File) {
+func (hc *HAProxyController) createDefaultSSLFile(cache convtypes.Cache) (tlsFile convtypes.CrtFile) {
 	if hc.cfg.DefaultSSLCertificate != "" {
 		tlsFile, err := cache.GetTLSSecretPath("", hc.cfg.DefaultSSLCertificate)
 		if err == nil {
@@ -155,10 +163,11 @@ func (hc *HAProxyController) createDefaultSSLFile(cache convtypes.Cache) (tlsFil
 	} else {
 		glog.Info("using auto generated fake certificate")
 	}
-	path, hash := hc.controller.CreateDefaultSSLCertificate()
-	tlsFile = convtypes.File{
+	path, hash, notAfter := hc.controller.CreateDefaultSSLCertificate()
+	tlsFile = convtypes.CrtFile{
 		Filename: path,
 		SHA1Hash: hash,
+		NotAfter: notAfter,
 	}
 	return tlsFile
 }
@@ -251,7 +260,7 @@ func (hc *HAProxyController) SyncIngress(item interface{}) error {
 	//
 	hc.updateCount++
 	hc.logger.Info("Starting HAProxy update id=%d", hc.updateCount)
-	timer := utils.NewTimer()
+	timer := utils.NewTimer(hc.metrics.ControllerProcTime)
 	var ingress []*extensions.Ingress
 	for _, iing := range hc.storeLister.Ingress.List() {
 		ing := iing.(*extensions.Ingress)
@@ -277,7 +286,7 @@ func (hc *HAProxyController) SyncIngress(item interface{}) error {
 		globalConfig,
 	)
 	ingConverter.Sync(ingress)
-	timer.Tick("ingress")
+	timer.Tick("parse_ingress")
 
 	//
 	// configmap converters
@@ -291,7 +300,7 @@ func (hc *HAProxyController) SyncIngress(item interface{}) error {
 				hc.cache,
 			)
 			tcpSvcConverter.Sync(tcpConfigmap.Data)
-			timer.Tick("tcpServices")
+			timer.Tick("parse_tcp_svc")
 		} else {
 			hc.logger.Error("error reading TCP services: %v", err)
 		}

pkg/controller/metrics.go

@@ -0,0 +1,150 @@
+/*
+Copyright 2019 The HAProxy Ingress Controller Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type metrics struct {
+	responseTime       *prometheus.HistogramVec
+	ctlProcTimeSum     *prometheus.CounterVec
+	ctlProcTimeCount   *prometheus.CounterVec
+	procSecondsCounter *prometheus.CounterVec
+	updatesCounter     *prometheus.CounterVec
+	updateSuccessGauge *prometheus.GaugeVec
+	certExpireGauge    *prometheus.GaugeVec
+	lastTrack          time.Time
+}
+
+func createMetrics() *metrics {
+	namespace := "haproxyingress"
+	metrics := &metrics{
+		responseTime: prometheus.NewHistogramVec(
+			prometheus.HistogramOpts{
+				Namespace: namespace,
+				Name:      "haproxy_response_time_seconds",
+				Help:      "Response time to commands sent via admin socket",
+				Buckets:   []float64{.0005, .001, .002, .005, .01},
+			},
+			[]string{"command"},
+		),
+		ctlProcTimeSum: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "controller_processing_time_seconds_sum",
+				Help:      "Cumulative time in seconds spent on haproxy-ingress tasks",
+			},
+			[]string{"task"},
+		),
+		ctlProcTimeCount: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "controller_processing_time_seconds_count",
+				Help:      "Cumulative number of haproxy-ingress tasks executed",
+			},
+			[]string{"task"},
+		),
+		procSecondsCounter: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "haproxy_processing_seconds_total",
+				Help:      "Cumulative time in seconds a single thread spent processing requests, based on Idle_pct.",
+			},
+			[]string{},
+		),
+		updatesCounter: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Name:      "updates_total",
+				Help:      "Cumulative number of Ingress controller updates. Status can be noop, dynamic, full.",
+			},
+			[]string{"status"},
+		),
+		updateSuccessGauge: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: namespace,
+				Name:      "update_success",
+				Help:      "Whether the last haproxy update was successful.",
+			},
+			[]string{},
+		),
+		certExpireGauge: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: namespace,
+				Name:      "cert_expire_date_epoch",
+				Help:      "The SSL certificate expiration date in unix epoch time.",
+			},
+			[]string{"hostname"},
+		),
+	}
+	prometheus.MustRegister(metrics.responseTime)
+	prometheus.MustRegister(metrics.ctlProcTimeSum)
+	prometheus.MustRegister(metrics.ctlProcTimeCount)
+	prometheus.MustRegister(metrics.procSecondsCounter)
+	prometheus.MustRegister(metrics.updatesCounter)
+	prometheus.MustRegister(metrics.updateSuccessGauge)
+	prometheus.MustRegister(metrics.certExpireGauge)
+	return metrics
+}
+
+func (m *metrics) HAProxyShowInfoResponseTime(duration time.Duration) {
+	m.responseTime.WithLabelValues("show_info").Observe(duration.Seconds())
+}
+
+func (m *metrics) HAProxySetServerResponseTime(duration time.Duration) {
+	m.responseTime.WithLabelValues("set_server").Observe(duration.Seconds())
+}
+
+func (m *metrics) ControllerProcTime(task string, duration time.Duration) {
+	m.ctlProcTimeSum.WithLabelValues(task).Add(duration.Seconds())
+	m.ctlProcTimeCount.WithLabelValues(task).Inc()
+}
+
+func (m *metrics) AddIdleFactor(idle int) {
+	now := time.Now()
+	if m.lastTrack.IsZero() {
+		m.lastTrack = now
+		return
+	}
+	totalTime := now.Sub(m.lastTrack).Seconds()
+	m.lastTrack = now
+	m.procSecondsCounter.WithLabelValues().Add(float64(100-idle) * totalTime / 100)
+}
+
+func (m *metrics) IncUpdateNoop() {
+	m.updatesCounter.WithLabelValues("noop").Inc()
+}
+
+func (m *metrics) IncUpdateDynamic() {
+	m.updatesCounter.WithLabelValues("dynamic").Inc()
+}
+
+func (m *metrics) IncUpdateFull() {
+	m.updatesCounter.WithLabelValues("full").Inc()
+}
+
+func (m *metrics) UpdateSuccessful(success bool) {
+	value := map[bool]float64{false: 0, true: 1}
+	m.updateSuccessGauge.WithLabelValues().Set(value[success])
+}
+
+func (m *metrics) SetCertExpireDate(hostname string, notAfter time.Time) {
+	m.certExpireGauge.WithLabelValues(hostname).Set(float64(notAfter.Unix()))
+}

pkg/converters/configmap/tcpservices.go

@@ -82,7 +82,7 @@ func (c *tcpSvcConverter) Sync(tcpservices map[string]string) {
 			c.logger.Warn("skipping TCP service on public port %d: %v", svc.port, err)
 			continue
 		}
-		var crtfile convtypes.File
+		var crtfile convtypes.CrtFile
 		if svc.secret != "" {
 			crtfile, err = c.cache.GetTLSSecretPath("", svc.secret)
 			if err != nil {

pkg/converters/helper_test/cachemock.go

@@ -20,6 +20,7 @@ import (
 	"crypto/sha1"
 	"fmt"
 	"strings"
+	"time"
 
 	api "k8s.io/api/core/v1"
 
@@ -101,15 +102,16 @@ func (c *CacheMock) GetPod(podName string) (*api.Pod, error) {
 }
 
 // GetTLSSecretPath ...
-func (c *CacheMock) GetTLSSecretPath(defaultNamespace, secretName string) (convtypes.File, error) {
+func (c *CacheMock) GetTLSSecretPath(defaultNamespace, secretName string) (convtypes.CrtFile, error) {
 	fullname := c.buildSecretName(defaultNamespace, secretName)
 	if path, found := c.SecretTLSPath[fullname]; found {
-		return convtypes.File{
+		return convtypes.CrtFile{
 			Filename: path,
 			SHA1Hash: fmt.Sprintf("%x", sha1.Sum([]byte(path))),
+			NotAfter: time.Now().AddDate(0, 0, 30),
 		}, nil
 	}
-	return convtypes.File{}, fmt.Errorf("secret not found: '%s'", fullname)
+	return convtypes.CrtFile{}, fmt.Errorf("secret not found: '%s'", fullname)
 }
 
 // GetCASecretPath ...