Implement metrics server override

artifacts/flagger/crd.yaml

@@ -68,6 +68,9 @@ spec:
             provider:
               description: Traffic managent provider
               type: string
+            metricsServer:
+              description: Prometheus URL
+              type: string
             progressDeadlineSeconds:
               description: Deployment progress deadline
               type: number

charts/flagger/templates/crd.yaml

@@ -69,6 +69,9 @@ spec:
             provider:
               description: Traffic managent provider
               type: string
+            metricsServer:
+              description: Prometheus URL
+              type: string
             progressDeadlineSeconds:
               description: Deployment progress deadline
               type: number

charts/flagger/templates/prometheus.yaml

@@ -133,38 +133,22 @@ data:
       scheme: https
       tls_config:
         ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        insecure_skip_verify: true
       bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
       relabel_configs:
       - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
         action: keep
         regex: kubernetes;https
 
-    # Scrape config for nodes
-    - job_name: 'kubernetes-nodes'
-      scheme: https
-      tls_config:
-        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-      kubernetes_sd_configs:
-      - role: node
-      relabel_configs:
-      - action: labelmap
-        regex: __meta_kubernetes_node_label_(.+)
-      - target_label: __address__
-        replacement: kubernetes.default.svc:443
-      - source_labels: [__meta_kubernetes_node_name]
-        regex: (.+)
-        target_label: __metrics_path__
-        replacement: /api/v1/nodes/${1}/proxy/metrics
-
     # scrape config for cAdvisor
     - job_name: 'kubernetes-cadvisor'
       scheme: https
       tls_config:
         ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        insecure_skip_verify: true
       bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
       kubernetes_sd_configs:
-      - role: node
+        - role: node
       relabel_configs:
       - action: labelmap
         regex: __meta_kubernetes_node_label_(.+)
@@ -174,6 +158,14 @@ data:
         regex: (.+)
         target_label: __metrics_path__
         replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
+      # exclude high cardinality metrics
+      metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: (container|machine)_(cpu|memory|network|fs)_(.+)
+        action: keep
+      - source_labels: [__name__]
+        regex: container_memory_failures_total
+        action: drop
 
     # scrape config for pods
     - job_name: kubernetes-pods

kustomize/base/flagger/crd.yaml

@@ -68,6 +68,9 @@ spec:
             provider:
               description: Traffic managent provider
               type: string
+            metricsServer:
+              description: Prometheus URL
+              type: string
             progressDeadlineSeconds:
               description: Deployment progress deadline
               type: number

kustomize/base/prometheus/prometheus.yml

@@ -2,12 +2,11 @@ global:
   scrape_interval: 5s
 scrape_configs:
 
-# Scrape config for AppMesh Envoy sidecar
+# scrape config for AppMesh Envoy sidecar
 - job_name: 'appmesh-envoy'
   metrics_path: /stats/prometheus
   kubernetes_sd_configs:
   - role: pod
-
   relabel_configs:
   - source_labels: [__meta_kubernetes_pod_container_name]
     action: keep
@@ -25,8 +24,7 @@ scrape_configs:
   - source_labels: [__meta_kubernetes_pod_name]
     action: replace
     target_label: kubernetes_pod_name
-
-  # Exclude high cardinality metrics
+  # exclude high cardinality metrics
   metric_relabel_configs:
   - source_labels: [ cluster_name ]
     regex: '(outbound|inbound|prometheus_stats).*'
@@ -56,7 +54,7 @@ scrape_configs:
     regex: 'envoy_cluster_(lb|retry|bind|internal|max|original).*'
     action: drop
 
-# Scrape config for API servers
+# scrape config for API servers
 - job_name: 'kubernetes-apiservers'
   kubernetes_sd_configs:
   - role: endpoints
@@ -66,35 +64,19 @@ scrape_configs:
   scheme: https
   tls_config:
     ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    insecure_skip_verify: true
   bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
   relabel_configs:
   - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
     action: keep
     regex: kubernetes;https
 
-# Scrape config for nodes
-- job_name: 'kubernetes-nodes'
-  scheme: https
-  tls_config:
-    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-  kubernetes_sd_configs:
-  - role: node
-  relabel_configs:
-  - action: labelmap
-    regex: __meta_kubernetes_node_label_(.+)
-  - target_label: __address__
-    replacement: kubernetes.default.svc:443
-  - source_labels: [__meta_kubernetes_node_name]
-    regex: (.+)
-    target_label: __metrics_path__
-    replacement: /api/v1/nodes/${1}/proxy/metrics
-
 # scrape config for cAdvisor
 - job_name: 'kubernetes-cadvisor'
   scheme: https
   tls_config:
     ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    insecure_skip_verify: true
   bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
   kubernetes_sd_configs:
   - role: node
@@ -107,6 +89,14 @@ scrape_configs:
     regex: (.+)
     target_label: __metrics_path__
     replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
+  # exclude high cardinality metrics
+  metric_relabel_configs:
+  - source_labels: [__name__]
+    regex: (container|machine)_(cpu|memory|network|fs)_(.+)
+    action: keep
+  - source_labels: [__name__]
+    regex: container_memory_failures_total
+    action: drop
 
 # scrape config for pods
 - job_name: kubernetes-pods

pkg/apis/flagger/v1alpha3/types.go

@@ -51,6 +51,10 @@ type CanarySpec struct {
 	// +optional
 	Provider string `json:"provider,omitempty"`
 
+	// if specified overwrites the -metrics-server flag for this particular canary
+	// +optional
+	MetricsServer string `json:"metricsServer,omitempty"`
+
 	// reference to target resource
 	TargetRef hpav1.CrossVersionObjectReference `json:"targetRef"`
 

pkg/controller/scheduler.go

@@ -2,6 +2,7 @@ package controller
 
 import (
 	"fmt"
+	"github.com/weaveworks/flagger/pkg/metrics"
 	"strings"
 	"time"
 
@@ -749,7 +750,21 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
 	}
 
 	// create observer based on the mesh provider
-	observer := c.observerFactory.Observer(metricsProvider)
+	observerFactory := c.observerFactory
+	observer := observerFactory.Observer(metricsProvider)
+
+	// override the global metrics server if one is specified in the canary spec
+	metricsServer := c.observerFactory.Client.GetMetricsServer()
+	if r.Spec.MetricsServer != "" {
+		metricsServer = r.Spec.MetricsServer
+		var err error
+		observerFactory, err = metrics.NewFactory(metricsServer, metricsProvider, 5*time.Second)
+		if err != nil {
+			c.recordEventErrorf(r, "Error building Prometheus client for %s %v", r.Spec.MetricsServer, err)
+			return false
+		}
+		observer = observerFactory.Observer(metricsProvider)
+	}
 
 	// run metrics checks
 	for _, metric := range r.Spec.CanaryAnalysis.Metrics {
@@ -764,7 +779,7 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
 					c.recordEventWarningf(r, "Halt advancement no values found for metric %s probably %s.%s is not receiving traffic",
 						metric.Name, r.Spec.TargetRef.Name, r.Namespace)
 				} else {
-					c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.observerFactory.Client.GetMetricsServer(), err)
+					c.recordEventErrorf(r, "Metrics server %s query failed: %v", metricsServer, err)
 				}
 				return false
 			}
@@ -784,7 +799,7 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
 					c.recordEventWarningf(r, "Halt advancement no values found for metric %s probably %s.%s is not receiving traffic",
 						metric.Name, r.Spec.TargetRef.Name, r.Namespace)
 				} else {
-					c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.observerFactory.Client.GetMetricsServer(), err)
+					c.recordEventErrorf(r, "Metrics server %s query failed: %v", metricsServer, err)
 				}
 				return false
 			}
@@ -800,13 +815,13 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
 
 		// custom checks
 		if metric.Query != "" {
-			val, err := c.observerFactory.Client.RunQuery(metric.Query)
+			val, err := observerFactory.Client.RunQuery(metric.Query)
 			if err != nil {
 				if strings.Contains(err.Error(), "no values found") {
 					c.recordEventWarningf(r, "Halt advancement no values found for metric %s probably %s.%s is not receiving traffic",
 						metric.Name, r.Spec.TargetRef.Name, r.Namespace)
 				} else {
-					c.recordEventErrorf(r, "Metrics server %s query failed for %s: %v", c.observerFactory.Client.GetMetricsServer(), metric.Name, err)
+					c.recordEventErrorf(r, "Metrics server %s query failed for %s: %v", metricsServer, metric.Name, err)
 				}
 				return false
 			}