update prometheus exporter to 1.0.0 (#4769)

* update prometheus exporter to 1.0.0

cmd/nginx-ingress/main.go

@@ -28,6 +28,7 @@ import (
 	"github.com/nginxinc/nginx-plus-go-client/client"
 	nginxCollector "github.com/nginxinc/nginx-prometheus-exporter/collector"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/promlog"
 	api_v1 "k8s.io/api/core/v1"
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	util_version "k8s.io/apimachinery/pkg/util/version"
@@ -694,16 +695,17 @@ func createPlusAndLatencyCollectors(
 
 			serverZoneVariableLabels := []string{"resource_type", "resource_name", "resource_namespace"}
 			streamServerZoneVariableLabels := []string{"resource_type", "resource_name", "resource_namespace"}
+			cacheZoneLabels := []string{"resource_type", "resource_name", "resource_namespace"}
+			workerPIDVariableLabels := []string{"resource_type", "resource_name", "resource_namespace"}
 			variableLabelNames := nginxCollector.NewVariableLabelNames(upstreamServerVariableLabels, serverZoneVariableLabels, upstreamServerPeerVariableLabelNames,
-				streamUpstreamServerVariableLabels, streamServerZoneVariableLabels, streamUpstreamServerPeerVariableLabelNames)
-			plusCollector = nginxCollector.NewNginxPlusCollector(plusClient, "nginx_ingress_nginxplus", variableLabelNames, constLabels)
+				streamUpstreamServerVariableLabels, streamServerZoneVariableLabels, streamUpstreamServerPeerVariableLabelNames, cacheZoneLabels, workerPIDVariableLabels)
+			promlogConfig := &promlog.Config{}
+			logger := promlog.New(promlogConfig)
+			plusCollector = nginxCollector.NewNginxPlusCollector(plusClient, "nginx_ingress_nginxplus", variableLabelNames, constLabels, logger)
 			go metrics.RunPrometheusListenerForNginxPlus(*prometheusMetricsListenPort, plusCollector, registry, prometheusSecret)
 		} else {
 			httpClient := getSocketClient("/var/lib/nginx/nginx-status.sock")
-			client, err := metrics.NewNginxMetricsClient(httpClient)
-			if err != nil {
-				glog.Errorf("Error creating the Nginx client for Prometheus metrics: %v", err)
-			}
+			client := metrics.NewNginxMetricsClient(httpClient)
 			go metrics.RunPrometheusListenerForNginx(*prometheusMetricsListenPort, client, registry, constLabels, prometheusSecret)
 		}
 		if *enableLatencyMetrics {

go.mod

@@ -13,9 +13,10 @@ require (
 	github.com/google/go-cmp v0.6.0
 	github.com/kr/pretty v0.3.1
 	github.com/nginxinc/nginx-plus-go-client v1.2.0
-	github.com/nginxinc/nginx-prometheus-exporter v0.11.0
+	github.com/nginxinc/nginx-prometheus-exporter v1.0.0
 	github.com/nginxinc/nginx-service-mesh v1.7.0
 	github.com/prometheus/client_golang v1.17.0
+	github.com/prometheus/common v0.45.0
 	github.com/spiffe/go-spiffe/v2 v2.1.6
 	github.com/stretchr/testify v1.8.4
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
@@ -55,7 +56,9 @@ require (
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
 	github.com/go-jose/go-jose/v3 v3.0.1 // indirect
+	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-ldap/ldap/v3 v3.4.5 // indirect
+	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/go-logr/logr v1.3.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zapr v1.2.4 // indirect
@@ -86,7 +89,6 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
-	github.com/prometheus/common v0.45.0 // indirect
 	github.com/prometheus/procfs v0.11.1 // indirect
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/spf13/cobra v1.7.0 // indirect

go.sum

@@ -84,8 +84,12 @@ github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk=
 github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA=
 github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
+github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
+github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
 github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8=
 github.com/go-ldap/ldap/v3 v3.4.5/go.mod h1:bMGIq3AGbytbaMwf8wdv5Phdxz0FWHTIYMSzyrYgnQs=
+github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -186,8 +190,8 @@ github.com/nginxinc/glog v1.1.2 h1:zyoZXhCoYvNMJq4qMsKislUCGyJ4eU2gNl3Nt7cjESg=
 github.com/nginxinc/glog v1.1.2/go.mod h1:Q2FpGp/qFhJEVnuC88BVfbLDPmio9aHYUj4al6w0138=
 github.com/nginxinc/nginx-plus-go-client v1.2.0 h1:NVfRsHbMJ7lOhkqMG52uvODiDBhQZNp20c0tV2lU3wg=
 github.com/nginxinc/nginx-plus-go-client v1.2.0/go.mod h1:n8OFLzrJulJ2fur28Cwa1Qp5DZNS2VicLV+Adt30LQ4=
-github.com/nginxinc/nginx-prometheus-exporter v0.11.0 h1:21xjnqNgxtni2jDgAQ90bl15uDnrTreO9sIlu1YsX/U=
-github.com/nginxinc/nginx-prometheus-exporter v0.11.0/go.mod h1:GdyHnWAb8q8OW1Pssrrqbcqra0SH0Vn6UXICMmyWkw8=
+github.com/nginxinc/nginx-prometheus-exporter v1.0.0 h1:rw5q6j6FQe9EWzJy5HzRgRBJ2tSVyC9By6k9ZFQ7lD8=
+github.com/nginxinc/nginx-prometheus-exporter v1.0.0/go.mod h1:SPohlKx0SiOuZYi04js53GWWb0HhD281AT8q4ApVMIE=
 github.com/nginxinc/nginx-service-mesh v1.7.0 h1:oxKr+Jdbxkos10VTy5xF2UHCcmfIhqWNlsOK/zPnZDM=
 github.com/nginxinc/nginx-service-mesh v1.7.0/go.mod h1:8tREM3kSEUGyk8JT8hdCf/9ol2kEo7hLR8b+m5Yd8Fs=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=

internal/configs/configurator_test.go

@@ -388,6 +388,8 @@ type mockLabelUpdater struct {
 	streamUpstreamServerPeerLabels map[string][]string
 	streamUpstreamServerLabels     map[string][]string
 	streamServerZoneLabels         map[string][]string
+	cacheZoneLabels                map[string][]string
+	workerPIDVariableLabels        map[string][]string
 }
 
 func newFakeLabelUpdater() *mockLabelUpdater {
@@ -398,6 +400,8 @@ func newFakeLabelUpdater() *mockLabelUpdater {
 		streamUpstreamServerPeerLabels: make(map[string][]string),
 		streamUpstreamServerLabels:     make(map[string][]string),
 		streamServerZoneLabels:         make(map[string][]string),
+		cacheZoneLabels:                make(map[string][]string),
+		workerPIDVariableLabels:        make(map[string][]string),
 	}
 }
 
@@ -485,6 +489,34 @@ func (u *mockLabelUpdater) DeleteStreamServerZoneLabels(zoneNames []string) {
 	}
 }
 
+// UpdateCacheZoneLabels updates the Cache Zone Labels
+func (u *mockLabelUpdater) UpdateCacheZoneLabels(cacheZoneLabelValues map[string][]string) {
+	for k, v := range cacheZoneLabelValues {
+		u.cacheZoneLabels[k] = v
+	}
+}
+
+// DeleteCacheZoneLabels deletes the Cache Zone Labels
+func (u *mockLabelUpdater) DeleteCacheZoneLabels(cacheZoneNames []string) {
+	for _, k := range cacheZoneNames {
+		delete(u.cacheZoneLabels, k)
+	}
+}
+
+// UpdateWorkerLabels updates the Worker Labels
+func (u *mockLabelUpdater) UpdateWorkerLabels(workerValues map[string][]string) {
+	for k, v := range workerValues {
+		u.workerPIDVariableLabels[k] = v
+	}
+}
+
+// DeleteWorkerLabels deletes the Worker Labels
+func (u *mockLabelUpdater) DeleteWorkerLabels(workerNames []string) {
+	for _, k := range workerNames {
+		delete(u.workerPIDVariableLabels, k)
+	}
+}
+
 type mockLatencyCollector struct {
 	upstreamServerLabels        map[string][]string
 	upstreamServerPeerLabels    map[string][]string
@@ -613,6 +645,8 @@ func TestUpdateIngressMetricsLabels(t *testing.T) {
 		streamUpstreamServerPeerLabels: make(map[string][]string),
 		streamUpstreamServerLabels:     make(map[string][]string),
 		streamServerZoneLabels:         make(map[string][]string),
+		cacheZoneLabels:                make(map[string][]string),
+		workerPIDVariableLabels:        make(map[string][]string),
 	}
 	expectedLatencyCollector := &mockLatencyCollector{
 		upstreamServerLabels:     upstreamServerLabels,
@@ -661,7 +695,9 @@ func TestUpdateIngressMetricsLabels(t *testing.T) {
 		upstreamServerPeerLabels:       upstreamServerPeerLabels,
 		streamUpstreamServerPeerLabels: make(map[string][]string),
 		streamUpstreamServerLabels:     make(map[string][]string),
-		streamServerZoneLabels:         map[string][]string{},
+		streamServerZoneLabels:         make(map[string][]string),
+		cacheZoneLabels:                make(map[string][]string),
+		workerPIDVariableLabels:        make(map[string][]string),
 	}
 	expectedLatencyCollector = &mockLatencyCollector{
 		upstreamServerLabels:        upstreamServerLabels,
@@ -688,6 +724,8 @@ func TestUpdateIngressMetricsLabels(t *testing.T) {
 		streamUpstreamServerPeerLabels: map[string][]string{},
 		streamUpstreamServerLabels:     map[string][]string{},
 		streamServerZoneLabels:         map[string][]string{},
+		cacheZoneLabels:                map[string][]string{},
+		workerPIDVariableLabels:        map[string][]string{},
 	}
 	expectedLatencyCollector = &mockLatencyCollector{
 		upstreamServerLabels:        upstreamServerLabels,
@@ -780,6 +818,8 @@ func TestUpdateVirtualServerMetricsLabels(t *testing.T) {
 		streamUpstreamServerPeerLabels: map[string][]string{},
 		streamUpstreamServerLabels:     map[string][]string{},
 		streamServerZoneLabels:         map[string][]string{},
+		cacheZoneLabels:                map[string][]string{},
+		workerPIDVariableLabels:        map[string][]string{},
 	}
 
 	expectedLatencyCollector := &mockLatencyCollector{
@@ -829,6 +869,8 @@ func TestUpdateVirtualServerMetricsLabels(t *testing.T) {
 		streamUpstreamServerPeerLabels: map[string][]string{},
 		streamUpstreamServerLabels:     map[string][]string{},
 		streamServerZoneLabels:         map[string][]string{},
+		cacheZoneLabels:                map[string][]string{},
+		workerPIDVariableLabels:        map[string][]string{},
 	}
 
 	expectedLatencyCollector = &mockLatencyCollector{
@@ -853,6 +895,8 @@ func TestUpdateVirtualServerMetricsLabels(t *testing.T) {
 		streamUpstreamServerPeerLabels: map[string][]string{},
 		streamUpstreamServerLabels:     map[string][]string{},
 		streamServerZoneLabels:         map[string][]string{},
+		cacheZoneLabels:                map[string][]string{},
+		workerPIDVariableLabels:        map[string][]string{},
 	}
 
 	expectedLatencyCollector = &mockLatencyCollector{
@@ -948,6 +992,8 @@ func TestUpdateTransportServerMetricsLabels(t *testing.T) {
 		upstreamServerPeerLabels:       make(map[string][]string),
 		upstreamServerLabels:           make(map[string][]string),
 		serverZoneLabels:               make(map[string][]string),
+		cacheZoneLabels:                make(map[string][]string),
+		workerPIDVariableLabels:        make(map[string][]string),
 	}
 
 	cnf.updateTransportServerMetricsLabels(tsEx, streamUpstreams)
@@ -989,6 +1035,8 @@ func TestUpdateTransportServerMetricsLabels(t *testing.T) {
 		upstreamServerPeerLabels:       map[string][]string{},
 		upstreamServerLabels:           map[string][]string{},
 		serverZoneLabels:               map[string][]string{},
+		cacheZoneLabels:                map[string][]string{},
+		workerPIDVariableLabels:        map[string][]string{},
 	}
 
 	cnf.updateTransportServerMetricsLabels(tsEx, updatedStreamUpstreams)
@@ -1003,6 +1051,8 @@ func TestUpdateTransportServerMetricsLabels(t *testing.T) {
 		streamUpstreamServerPeerLabels: map[string][]string{},
 		streamUpstreamServerLabels:     map[string][]string{},
 		streamServerZoneLabels:         map[string][]string{},
+		cacheZoneLabels:                map[string][]string{},
+		workerPIDVariableLabels:        map[string][]string{},
 	}
 
 	cnf.deleteTransportServerMetricsLabels("default/test-transportserver")
@@ -1063,6 +1113,8 @@ func TestUpdateTransportServerMetricsLabels(t *testing.T) {
 		upstreamServerPeerLabels:       make(map[string][]string),
 		upstreamServerLabels:           make(map[string][]string),
 		serverZoneLabels:               make(map[string][]string),
+		cacheZoneLabels:                make(map[string][]string),
+		workerPIDVariableLabels:        make(map[string][]string),
 	}
 
 	cnf.updateTransportServerMetricsLabels(tsExTLS, streamUpstreams)
@@ -1077,6 +1129,8 @@ func TestUpdateTransportServerMetricsLabels(t *testing.T) {
 		streamUpstreamServerPeerLabels: map[string][]string{},
 		streamUpstreamServerLabels:     map[string][]string{},
 		streamServerZoneLabels:         map[string][]string{},
+		cacheZoneLabels:                map[string][]string{},
+		workerPIDVariableLabels:        map[string][]string{},
 	}
 
 	cnf.deleteTransportServerMetricsLabels("default/test-transportserver-tls")

internal/metrics/listener.go

@@ -15,17 +15,20 @@ import (
 	nginxCollector "github.com/nginxinc/nginx-prometheus-exporter/collector"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/prometheus/common/promlog"
 	v1 "k8s.io/api/core/v1"
 )
 
 // NewNginxMetricsClient creates an NginxClient to fetch stats from NGINX over an unix socket
-func NewNginxMetricsClient(httpClient *http.Client) (*prometheusClient.NginxClient, error) {
+func NewNginxMetricsClient(httpClient *http.Client) *prometheusClient.NginxClient {
 	return prometheusClient.NewNginxClient(httpClient, "http://config-status/stub_status")
 }
 
 // RunPrometheusListenerForNginx runs an http server to expose Prometheus metrics for NGINX
 func RunPrometheusListenerForNginx(port int, client *prometheusClient.NginxClient, registry *prometheus.Registry, constLabels map[string]string, prometheusSecret *v1.Secret) {
-	registry.MustRegister(nginxCollector.NewNginxCollector(client, "nginx_ingress_nginx", constLabels))
+	promlogConfig := &promlog.Config{}
+	logger := promlog.New(promlogConfig)
+	registry.MustRegister(nginxCollector.NewNginxCollector(client, "nginx_ingress_nginx", constLabels, logger))
 	runServer(strconv.Itoa(port), registry, prometheusSecret)
 }
 

tests/Makefile

@@ -3,15 +3,15 @@ ROOT_DIR                = $(shell git rev-parse --show-toplevel)
 CONTEXT                 =
 PULL_POLICY             = IfNotPresent
 DEPLOYMENT_TYPE         = deployment
-IC_TYPE                 = nginx-ingress
 SERVICE                 = nodeport
 NODE_IP                 =
 TEST_PREFIX             = test-runner
 KUBE_CONFIG_FOLDER      = ${HOME}/.kube
 KIND_KUBE_CONFIG_FOLDER = $(KUBE_CONFIG_FOLDER)/kind
-SHOW_IC_LOGS            = no
 DOCKERFILEPATH          := ${ROOT_DIR}/tests/Dockerfile
 IP_FAMILY               = dual
+IC_TYPE                 ?= nginx-ingress ## The Ingress Controller type to use, "nginx-ingress" or "nginx-plus-ingress".  Defaults to "nginx-ingress"
+SHOW_IC_LOGS            ?= no ## Should the tests show the Ingress Controller logs on failure, "yes" or "no". Defaults to "no"
 TEST_TAG                ?= latest ## The Tag to use for the test image. e.g. commitsha
 PREFIX                  ?= nginx/nginx-ingress ## The name of the image. For example, nginx/nginx-ingress
 TAG                     ?= edge ## The tag of the image. For example, edge
@@ -52,7 +52,10 @@ update-test-kind-config:
 
 .PHONY: run-tests-in-kind
 run-tests-in-kind: update-test-kind-config ## Run tests in Kind
-	docker run --network=kind --rm -v $(KIND_KUBE_CONFIG_FOLDER):/root/.kube $(TEST_PREFIX):$(TEST_TAG) \
+	docker run --network=kind --rm \
+		-v $(KIND_KUBE_CONFIG_FOLDER):/root/.kube \
+		-v $(ROOT_DIR)/tests:/workspace/tests \
+		$(TEST_PREFIX):$(TEST_TAG) \
  		--context=kind-$(strip $(K8S_CLUSTER_NAME)) \
  		--image=$(BUILD_IMAGE) --image-pull-policy=$(PULL_POLICY) \
  		--deployment-type=$(DEPLOYMENT_TYPE) \

tests/suite/test_prometheus_metrics.py

@@ -298,3 +298,45 @@ def test_total_metrics(
         wait_before_test()
 
         assert_ts_total_metric(ingress_controller_endpoint, ts_type, 0)
+
+
+@pytest.mark.ingresses
+@pytest.mark.smoke
+@pytest.mark.skip_for_nginx_oss
+class TestPrometheusExporterPlus:
+    @pytest.mark.parametrize(
+        "ingress_controller, expected_metrics",
+        [
+            pytest.param(
+                {"extra_args": ["-enable-prometheus-metrics", "-enable-oidc"]},
+                [
+                    'nginx_ingress_nginxplus_cache_bypass_bytes{class="nginx",zone="jwk"}',
+                    'nginx_ingress_nginxplus_cache_expired_bytes{class="nginx",zone="jwk"}',
+                    'nginx_ingress_nginxplus_cache_hit_responses{class="nginx",zone="jwk"}',
+                    'nginx_ingress_nginxplus_cache_miss_responses{class="nginx",zone="jwk"}',
+                    'nginx_ingress_nginxplus_cache_size{class="nginx",zone="jwk"}',
+                    'nginxplus_worker_connection_accepted{class="nginx"',
+                    'nginxplus_worker_connection_active{class="nginx"',
+                    'nginxplus_worker_http_requests_total{class="nginx"',
+                ],
+            )
+        ],
+        indirect=["ingress_controller"],
+    )
+    def test_plus_metrics(
+        self,
+        ingress_controller_endpoint,
+        ingress_controller,
+        expected_metrics,
+        ingress_setup,
+    ):
+        ensure_connection(ingress_setup.req_url, 200, {"host": ingress_setup.ingress_host})
+        resp = requests.get(ingress_setup.req_url, headers={"host": ingress_setup.ingress_host}, verify=False)
+        assert resp.status_code == 200
+        req_url = f"http://{ingress_controller_endpoint.public_ip}:{ingress_controller_endpoint.metrics_port}/metrics"
+        ensure_connection(req_url, 200)
+        resp = requests.get(req_url)
+        assert resp.status_code == 200, f"Expected 200 code for /metrics but got {resp.status_code}"
+        resp_content = resp.content.decode("utf-8")
+        for item in expected_metrics:
+            assert item in resp_content