From 36cb74b22f74a0be42f70aa00958de5b5ce9a463 Mon Sep 17 00:00:00 2001 From: Alejandro Pedraza Date: Tue, 25 Apr 2023 15:43:23 -0500 Subject: [PATCH] Fix `server_port_subscribers` metric in Destination Fixes #10764 `GetProfile' streams create a `server_port_subscribers` gauge that tracks the number of listeners interested in a given Server. Because of an oversight, the gauge was only being registered until the second listener was added. For just one listener the gauge was absent. But whenever the `GetProfile` stream ended, the gauge was deleted which resulted in this error if it wasn't registered to begin with: ``` level=warning msg="unable to delete server_port_subscribers metric with labels map[name:voting namespace:emojivoto port:4191]" addr=":8086" component=server ``` One can check that the gauge wasn't being created by installing viz and emojivoto, and checking the following returns empty: ```bash $ linkerd diagnostics controller-metrics|ag server.port.subscribers ``` After this fix, one can see the metric getting populated: ```bash $ linkerd diagnostics controller-metrics | grep server.port.subscribers # HELP server_port_subscribers Number of subscribers to Server changes associated with a pod's port. # TYPE server_port_subscribers gauge server_port_subscribers{name="emoji",namespace="emojivoto",port="4191"} 1 server_port_subscribers{name="linkerd",namespace="linkerd",port="4191"} 1 server_port_subscribers{name="linkerd",namespace="linkerd",port="9990"} 1 server_port_subscribers{name="linkerd",namespace="linkerd",port="9995"} 1 server_port_subscribers{name="linkerd",namespace="linkerd",port="9996"} 1 server_port_subscribers{name="linkerd",namespace="linkerd",port="9997"} 1 server_port_subscribers{name="metrics",namespace="linkerd-viz",port="4191"} 1 server_port_subscribers{name="metrics",namespace="linkerd-viz",port="9995"} 1 server_port_subscribers{name="tap",namespace="linkerd-viz",port="4191"} 1 server_port_subscribers{name="tap",namespace="linkerd-viz",port="9995"} 1 server_port_subscribers{name="tap",namespace="linkerd-viz",port="9998"} 1 server_port_subscribers{name="vote",namespace="emojivoto",port="4191"} 1 server_port_subscribers{name="voting",namespace="emojivoto",port="4191"} 1 server_port_subscribers{name="web",namespace="emojivoto",port="4191"} 1 server_port_subscribers{name="web",namespace="linkerd-viz",port="4191"} 1 server_port_subscribers{name="web",namespace="linkerd-viz",port="9994"} 1 ``` And when scaling down the voting deployment, one can see how the metric with `name="voting"` is removed. --- controller/api/destination/watcher/server_watcher.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/controller/api/destination/watcher/server_watcher.go b/controller/api/destination/watcher/server_watcher.go index 3aef1e72a9375..1783687a10f96 100644 --- a/controller/api/destination/watcher/server_watcher.go +++ b/controller/api/destination/watcher/server_watcher.go @@ -88,11 +88,10 @@ func (sw *ServerWatcher) Subscribe(pod *corev1.Pod, port Port, listener ServerUp } ppp, ok := sw.subscriptions[pp] if !ok { - sw.subscriptions[pp] = podPortPublisher{ + ppp = podPortPublisher{ pod: pod, - listeners: []ServerUpdateListener{listener}, + listeners: []ServerUpdateListener{}, } - return } ppp.listeners = append(ppp.listeners, listener) sw.subscriptions[pp] = ppp