metrics process empty at dashboard

[Kofl]

Hi,

we are running cadvisor fine, but processes are not listed as metrics in the dashboard, although we did not exlude process from the metrics. Any advice welcome.

VERSION=v0.37.0
sudo docker run
--volume=/:/rootfs:ro
--volume=/var/run:/var/run:rw
--volume=/sys:/sys:ro
--volume=/var/lib/docker/:/var/lib/docker:ro
--volume=/cgroup:/cgroup:ro
--publish=8080:8080
--detach=true
--name=cadvisor-docker
gcr.io/cadvisor/cadvisor:$VERSION --docker_only=true --disable_metrics=tcp,advtcp,udp,sched,hugetlb

CentOS Linux release 7.7.1908 (Core)
Docker version 19.03.6, build 369ce74a3

[root@docker cadvisor]# docker logs 6c810ec3be53
W0829 15:30:06.261919 1 nvidia.go:61] NVIDIA GPU metrics will not be available: no NVIDIA devices found

[root@docker cadvisor]# docker inspect 6c810ec3be53
[
{
"Id": "6c810ec3be538307d17ba07d2abb24060f2ae8dfc1f7eae4014618aef910bff9",
"Created": "2020-08-29T15:30:05.7636614Z",
"Path": "/usr/bin/cadvisor",
"Args": [
"-logtostderr",
"--docker_only=true",
"--disable_metrics=tcp,advtcp,udp,sched,hugetlb"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 14761,
"ExitCode": 0,
"Error": "",
"StartedAt": "2020-08-29T15:30:06.1727475Z",
"FinishedAt": "0001-01-01T00:00:00Z",
"Health": {
"Status": "healthy",
"FailingStreak": 0,
"Log": [
{
"Start": "2020-08-29T17:30:36.1735321+02:00",
"End": "2020-08-29T17:30:36.4315535+02:00",
"ExitCode": 0,
"Output": ""
}
]
}
},
"Image": "sha256:6e37caacdfed121615ab71a0ba76f7da1117f1b08b70aa23b1ecb8ed71c60e36",
"ResolvConfPath": "/var/lib/docker/containers/6c810ec3be538307d17ba07d2abb24060f2ae8dfc1f7eae4014618aef910bff9/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/6c810ec3be538307d17ba07d2abb24060f2ae8dfc1f7eae4014618aef910bff9/hostname",
"HostsPath": "/var/lib/docker/containers/6c810ec3be538307d17ba07d2abb24060f2ae8dfc1f7eae4014618aef910bff9/hosts",
"LogPath": "/var/lib/docker/containers/6c810ec3be538307d17ba07d2abb24060f2ae8dfc1f7eae4014618aef910bff9/6c810ec3be538307d17ba07d2abb24060f2ae8dfc1f7eae4014618aef910bff9-json.log",
"Name": "/cadvisor-docker",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"/:/rootfs:ro",
"/var/run:/var/run:rw",
"/sys:/sys:ro",
"/var/lib/docker/:/var/lib/docker:ro",
"/cgroup:/cgroup:ro"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "default",
"PortBindings": {
"8080/tcp": [
{
"HostIp": "",
"HostPort": "8080"
}
]
},
"RestartPolicy": {
"Name": "no",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"Capabilities": null,
"Dns": [],
"DnsOptions": [],
"DnsSearch": [],
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "private",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": true,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": [
"label=disable"
],
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 0,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": [],
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [
{
"PathOnHost": "/dev/kmsg",
"PathInContainer": "/dev/kmsg",
"CgroupPermissions": "rwm"
}
],
"DeviceCgroupRules": null,
"DeviceRequests": null,
"KernelMemory": 0,
"KernelMemoryTCP": 0,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": null,
"OomKillDisable": false,
"PidsLimit": null,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": null,
"ReadonlyPaths": null
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/3ed3486e068d357fb676b18fc3f5cb3303ae916ec8b68059ce347db4bf18357d-init/diff:/var/lib/docker/overlay2/b01962751fc21def15096029fce50c8235d79473ccc53f6d17ba0b5d8c1480a0/diff:/var/lib/docker/overlay2/68d7da07b2cf55a60121496dbdd6c66eafb2cc9304fd3bdd324fa43a19e9d0a7/diff:/var/lib/docker/overlay2/fdc0ac3d9e8806c132b3b123f484aaeae1ecdea9cf0410ee6872395ac9e34782/diff:/var/lib/docker/overlay2/193d7b14e7ea0eb0bd6d6cd4a5f7abfb00f9c789d714ada6b130338e2aae0989/diff",
"MergedDir": "/var/lib/docker/overlay2/3ed3486e068d357fb676b18fc3f5cb3303ae916ec8b68059ce347db4bf18357d/merged",
"UpperDir": "/var/lib/docker/overlay2/3ed3486e068d357fb676b18fc3f5cb3303ae916ec8b68059ce347db4bf18357d/diff",
"WorkDir": "/var/lib/docker/overlay2/3ed3486e068d357fb676b18fc3f5cb3303ae916ec8b68059ce347db4bf18357d/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "bind",
"Source": "/var/lib/docker",
"Destination": "/var/lib/docker",
"Mode": "ro",
"RW": false,
"Propagation": "rslave"
},
{
"Type": "bind",
"Source": "/cgroup",
"Destination": "/cgroup",
"Mode": "ro",
"RW": false,
"Propagation": "rprivate"
},
{
"Type": "bind",
"Source": "/",
"Destination": "/rootfs",
"Mode": "ro",
"RW": false,
"Propagation": "rslave"
},
{
"Type": "bind",
"Source": "/var/run",
"Destination": "/var/run",
"Mode": "rw",
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "bind",
"Source": "/sys",
"Destination": "/sys",
"Mode": "ro",
"RW": false,
"Propagation": "rprivate"
}
],
"Config": {
"Hostname": "6c810ec3be53",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"8080/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"CADVISOR_HEALTHCHECK_URL=http://localhost:8080/healthz"
],
"Cmd": [
"--docker_only=true",
"--disable_metrics=tcp,advtcp,udp,sched,hugetlb"
],
"Healthcheck": {
"Test": [
"CMD-SHELL",
"wget --quiet --tries=1 --spider $CADVISOR_HEALTHCHECK_URL || exit 1"
],
"Interval": 30000000000,
"Timeout": 3000000000
},
"Image": "gcr.io/cadvisor/cadvisor:v0.37.0",
"Volumes": null,
"WorkingDir": "",
"Entrypoint": [
"/usr/bin/cadvisor",
"-logtostderr"
],
"OnBuild": null,
"Labels": {}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "e8eb7035bb46d9e13c8cb539513098762fd80985f877efa6d9d7e7d5635a6a31",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {
"8080/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "8080"
}
]
},
"SandboxKey": "/var/run/docker/netns/e8eb7035bb46",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "47d382fe5a66064e9d65009363219d3e1a0bba5f48ea01312e9adcc5d574fb82",
"Gateway": "172.17.0.1",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"MacAddress": "02:42:ac:11:00:03",
"Networks": {
"bridge": {
"IPAMConfig": null,
"Links": null,
"Aliases": null,
"NetworkID": "3492871bc481ff70e6bed8a2c77a8ad19c84a819c5a667482e71874b7b87322c",
"EndpointID": "47d382fe5a66064e9d65009363219d3e1a0bba5f48ea01312e9adcc5d574fb82",
"Gateway": "172.17.0.1",
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "02:42:ac:11:00:03",
"DriverOpts": null
}
}
}
}
]

[dashpole]

Which dashboard are you referring to? Are you using the cAdvisor UI, or something else?

[Kofl]

Sorry, I am referring to the cAdvisor UI. No processes are listed in the overview and also when I dig into the containers.

Tried v0.37.0 and v0.36.0

[dashpole]

I did a bit of testing myself, and I think this is a UI bug. I ran the docker container locally as you did above, and the Processes section for http://localhost:8080/containers/ was empty.

I was able to query the processes endpoint successfully: curl localhost:8080/api/v2.0/ps.

cadvisor/cmd/internal/pages/assets/js/containers.js

Line 604 in e3f3ee1

function drawProcesses(isRoot, rootDir, processInfo) {

is the code for rendering the processes.

I see this error in my console:

Uncaught Error: Type mismatch. Value /init.scope does not match type number in column index 11
    at jp (gcharts.js:173)
    at lga (gcharts.js:189)
    at fp.W.z0 (gcharts.js:191)
    at fp.W.cB (gcharts.js:191)
    at drawTable (containers.js:42)
    at drawProcesses (containers.js:653)
    at containers.js:1099
    at Object.<anonymous> (containers.js:193)
    at c (jquery-3.5.1.min.js:2)
    at Object.fireWith [as resolveWith] (jquery-3.5.1.min.js:2)

[dashpole]

I think this was broken by #2291. We we append the container name after PSR if it is "root", but then put push the PSR value after the container name, hence the mismatch in types. I'll send out a fix