Update prometheus guide to a working example and add Envoy metrics po…

…rt (projectcontour#6269)

Updates docs page guide instructions and sample manifests

Also removes prometheus annotations from Contour/Envoy in the example YAMLs and provisioner generated resources and adds the metrics port to Envoy DaemonSet/Deployment so that Prometheus PodMonitors can reach the metrics server. The prometheus-operator project uses PodMonitors rather than annotations to configure Prometheus to discover scrape endpoints.

Signed-off-by: Sunjay Bhatia <sunjayb@vmware.com>
Signed-off-by: Saman Mahdanian <saman@mahdanian.xyz>

changelogs/unreleased/6269-sunjayBhatia-deprecation.md

@@ -0,0 +1,4 @@
+## Contour sample YAML manifests no longer use `prometheus.io/` annotations
+
+The annotations for notifying a Prometheus instance on how to scrape metrics from Contour and Envoy pods have been removed from the deployment YAMLs and the Gateway provisioner.
+The suggested mechanism for doing so now is to use [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) and the [`PodMonitor`](https://prometheus-operator.dev/docs/operator/design/#podmonitor) resource.

changelogs/unreleased/6269-sunjayBhatia-minor.md

@@ -0,0 +1,4 @@
+## Update examples for monitoring Contour and Envoy
+
+Updates the [documentation](https://projectcontour.io/docs/main/guides/prometheus/) and examples for deploying a monitoring stack (Prometheus and Grafana) to scrape metrics from Contour and Envoy.
+Adds a metrics port to the Envoy DaemonSet/Deployment in the example YAMLs to expose port `8002` so that `PodMonitor` resources can be used to find metrics endpoints.

examples/contour/03-contour.yaml

@@ -19,9 +19,6 @@ spec:
       app: contour
   template:
     metadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8000"
       labels:
         app: contour
     spec:

examples/contour/03-envoy.yaml

@@ -16,10 +16,6 @@ spec:
       app: envoy
   template:
     metadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8002"
-        prometheus.io/path: "/stats/prometheus"
       labels:
         app: envoy
     spec:
@@ -73,6 +69,10 @@ spec:
           hostPort: 443
           name: https
           protocol: TCP
+        - containerPort: 8002
+          hostPort: 8002
+          name: metrics
+          protocol: TCP
         readinessProbe:
           httpGet:
             path: /ready

examples/deployment/03-envoy-deployment.yaml

@@ -19,10 +19,6 @@ spec:
       app: envoy
   template:
     metadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8002"
-        prometheus.io/path: "/stats/prometheus"
       labels:
         app: envoy
     spec:
@@ -85,6 +81,10 @@ spec:
               hostPort: 443
               name: https
               protocol: TCP
+            - containerPort: 8002
+              hostPort: 8002
+              name: metrics
+              protocol: TCP
           readinessProbe:
             httpGet:
               path: /ready

examples/grafana/02-grafana-configmap.yaml → examples/grafana/dashboards.yaml

@@ -1,12 +1,10 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  labels:
-    app: grafana
-  name: grafana-dashs
-  namespace: projectcontour-monitoring
+  name: grafana-dashboard-contour-httpproxies
+  namespace: monitoring
 data:
-  contour-proxy.json: |
+  contour-httpproxies.json: |
     {
         "__inputs": [{
             "name": "DS_PROMETHEUS",
@@ -809,7 +807,14 @@ data:
         "uid": "KYcCfvKik",
         "version": 1
     }
-  envoy.json: |
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-dashboard-contour-envoy
+  namespace: monitoring
+data:
+  contour-envoy.json: |
     {
       "annotations": {
         "list": [
@@ -2912,7 +2917,14 @@ data:
       "uid": "khVnG8iiz",
       "version": 7
     }
-  apiserver.json: |
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-dashboard-contour-api
+  namespace: monitoring
+data:
+  contour-api.json: |
     {
       "annotations": {
         "list": [
@@ -3246,174 +3258,3 @@ data:
       "uid": "u4C2S-QZz",
       "version": 8
     }
-
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  labels:
-    app: grafana
-  name: grafana-config
-  namespace: projectcontour-monitoring
-data:
-  grafana.ini: |
-    ; instance_name = ${HOSTNAME}
-    [paths]
-    data = /var/lib/grafana/data
-    logs = /var/log/grafana
-    plugins = /var/lib/grafana/plugins
-
-    [server]
-    ;protocol = http
-    ;http_addr =
-    ;http_port = 3000
-    ;domain = localhost
-    ;enforce_domain = false
-    ;root_url = %(protocol)s://%(domain)s:%(http_port)s/
-    ;router_logging = false
-    ;static_root_path = public
-    ;enable_gzip = false
-    ;cert_file =
-    ;cert_key =
-
-    [database]
-    ;type = sqlite3
-    ;host = 127.0.0.1:3306
-    ;name = grafana
-    ;user = root
-    ;password =
-    ;ssl_mode = disable
-    ;path = grafana.db
-
-    [session]
-    ;provider = file
-    ;provider_config = sessions
-    ;cookie_name = grafana_sess
-    ;cookie_secure = false
-    ;session_life_time = 86400
-
-    [analytics]
-    ;reporting_enabled = true
-    check_for_updates = true
-    ;google_analytics_ua_id =
-
-    [security]
-    ;admin_user = admin
-    ;admin_password =
-    ;secret_key =
-    ;login_remember_days = 7
-    ;cookie_username = grafana_user
-    ;cookie_remember_name = grafana_remember
-    ;disable_gravatar = false
-    ;data_source_proxy_whitelist =
-
-    [snapshots]
-    ;external_enabled = true
-    ;external_snapshot_url = https://snapshots-origin.raintank.io
-    ;external_snapshot_name = Publish to snapshot.raintank.io
-
-    [users]
-    ;allow_sign_up = true
-    ;allow_org_create = true
-    ;auto_assign_org = true
-    ;auto_assign_org_role = Viewer
-    ;login_hint = email or username
-    ;default_theme = dark
-
-    [auth.anonymous]
-    ;enabled = false
-    ;org_name = Main Org.
-    ;org_role = Viewer
-
-    [auth.proxy]
-    ;enabled = false
-    ;header_name = X-WEBAUTH-USER
-    ;header_property = username
-    ;auto_sign_up = true
-
-    [auth.basic]
-    ;enabled = true
-
-    [auth.ldap]
-    ;enabled = false
-    ;config_file = /etc/grafana/ldap.toml
-
-    [smtp]
-    ;enabled = false
-    ;host = localhost:25
-    ;user =
-    ;password =
-    ;cert_file =
-    ;key_file =
-    ;skip_verify = false
-    ;from_address = admin@grafana.localhost
-
-    [emails]
-    ;welcome_email_on_sign_up = false
-
-    [log]
-    mode = console
-    level = info
-
-    [log.console]
-    ;level =
-    ;format = console
-
-    [event_publisher]
-    ;enabled = false
-    ;rabbitmq_url = amqp://localhost/
-    ;exchange = grafana_events
-
-    [dashboards.json]
-    enabled = true
-    path = /var/lib/grafana/dashboards
-
-    [metrics]
-    ;enabled           = true
-    ;interval_seconds  = 10
-
-    ; [metrics.graphite]
-    ; address = localhost:2003
-    ; prefix = prod.grafana.%(instance_name)s.
-
-    [grafana_net]
-    url = https://grafana.net
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  labels:
-    app: grafana
-  name: grafana-dash-provider
-  namespace: projectcontour-monitoring
-data:
-  providers.yaml: |
-    apiVersion: 1
-    providers:
-    - name: 'default'
-      orgId: 1
-      folder: ''
-      type: file
-      disableDeletion: false
-      editable: false
-      options:
-        path: /var/lib/grafana/dashboards
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  labels:
-    app: grafana
-  name: grafana-datasources-provider
-  namespace: projectcontour-monitoring
-data:
-  providers.yaml: |
-    apiVersion: 1
-    datasources:
-    - name: 'prometheus'
-      type: prometheus
-      access: proxy
-      orgId: 1
-      url: http://prometheus:9090
-      isDefault: true
-      editable: false