feat(prometheus) add nginx timer metrics (#8387)

* feat(prometheus) add nginx timer metrics

This adds gauges to track ngx.timer.running_count() and ngx.timer.pending_count() as requested in #7670.

* style(prometheus) rename metrics from current timers to just timers

Per suggestion, to avoid confusion.

* fix(prometheus) fix timer tests failing

The tests were accidentally matching in plain mode so '%d+' was not
understood.

* perf(prometheus) localize ngx timer functions

kong/plugins/prometheus/exporter.lua

@@ -4,6 +4,8 @@ local find = string.find
 local lower = string.lower
 local concat = table.concat
 local select = select
+local ngx_timer_pending_count = ngx.timer.pending_count
+local ngx_timer_running_count = ngx.timer.running_count
 local balancer = require("kong.runloop.balancer")
 local get_all_upstreams = balancer.get_all_upstreams
 if not balancer.get_all_upstreams then -- API changed since after Kong 2.5
@@ -48,6 +50,9 @@ local function init()
       "Number of Stream connections",
       {"state"})
   end
+  metrics.timers = prometheus:gauge("nginx_timers",
+                                    "Number of nginx timers",
+                                    {"state"})
   metrics.db_reachable = prometheus:gauge("datastore_reachable",
                                           "Datastore reachable from Kong, " ..
                                           "0 is unreachable")
@@ -102,7 +107,6 @@ local function init()
                                           "HTTP status codes for customer per service/route in Kong",
                                           {"service", "route", "code", "consumer"})
 
-
   -- Hybrid mode status
   if role == "control_plane" then
     metrics.data_plane_last_seen = prometheus:gauge("data_plane_last_seen",
@@ -314,6 +318,9 @@ local function metric_data()
   metrics.connections:set(ngx.var.connections_writing or 0, { "writing" })
   metrics.connections:set(ngx.var.connections_waiting or 0, { "waiting" })
 
+  metrics.timers:set(ngx_timer_running_count(), {"running"})
+  metrics.timers:set(ngx_timer_pending_count(), {"pending"})
+
   -- db reachable?
   local ok, err = kong.db.connector:connect()
   if ok then

spec/03-plugins/26-prometheus/04-status_api_spec.lua

@@ -286,6 +286,16 @@ describe("Plugin: prometheus (access via status API)", function()
     assert.matches('kong_datastore_reachable 1', body, nil, true)
   end)
 
+  it("exposes nginx timer metrics", function()
+    local res = assert(status_client:send {
+      method  = "GET",
+      path    = "/metrics",
+    })
+    local body = assert.res_status(200, res)
+    assert.matches('kong_nginx_timers{state="running"} %d+', body)
+    assert.matches('kong_nginx_timers{state="pending"} %d+', body)
+  end)
+
   it("exposes upstream's target health metrics - healthchecks-off", function()
     local body
     helpers.wait_until(function()