Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add CPU metrics for pending jobs #53

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Add CPU metrics for pending jobs #53

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5702eac
Add new metric for pending CPUs per account
atombaby Apr 20, 2021
60e61a5
Add metric for pending CPUs by user
atombaby Apr 20, 2021
4d5fe11
Include updated collectors in README
atombaby Apr 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,9 @@ See the related [test data](https://github.com/vpenso/prometheus-slurm-exporter/
The following information about jobs are also extracted via [squeue](https://slurm.schedmd.com/squeue.html):

* **Running/Pending/Suspended** jobs per SLURM Account.
* **Running/Pending** CPUs per SLURM Account.
* **Running/Pending/Suspended** jobs per SLURM User.
* **Running/Pending** CPUs per SLURM User.

### Scheduler Information

Expand Down
10 changes: 9 additions & 1 deletion accounts.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ func AccountsData() []byte {

type JobMetrics struct {
pending float64
pending_cpus float64
running float64
running_cpus float64
suspended float64
Expand All @@ -56,7 +57,7 @@ func ParseAccountsMetrics(input []byte) map[string]*JobMetrics {
account := strings.Split(line,"|")[1]
_,key := accounts[account]
if !key {
accounts[account] = &JobMetrics{0,0,0,0}
accounts[account] = &JobMetrics{0,0,0,0,0}
}
state := strings.Split(line,"|")[2]
state = strings.ToLower(state)
Expand All @@ -67,6 +68,7 @@ func ParseAccountsMetrics(input []byte) map[string]*JobMetrics {
switch {
case pending.MatchString(state) == true:
accounts[account].pending++
accounts[account].pending_cpus += cpus
case running.MatchString(state) == true:
accounts[account].running++
accounts[account].running_cpus += cpus
Expand All @@ -80,6 +82,7 @@ func ParseAccountsMetrics(input []byte) map[string]*JobMetrics {

type AccountsCollector struct {
pending *prometheus.Desc
pending_cpus *prometheus.Desc
running *prometheus.Desc
running_cpus *prometheus.Desc
suspended *prometheus.Desc
Expand All @@ -89,6 +92,7 @@ func NewAccountsCollector() *AccountsCollector {
labels := []string{"account"}
return &AccountsCollector{
pending: prometheus.NewDesc("slurm_account_jobs_pending", "Pending jobs for account", labels, nil),
pending_cpus: prometheus.NewDesc("slurm_account_cpus_pending", "Pending jobs for account", labels, nil),
running: prometheus.NewDesc("slurm_account_jobs_running", "Running jobs for account", labels, nil),
running_cpus: prometheus.NewDesc("slurm_account_cpus_running", "Running cpus for account", labels, nil),
suspended: prometheus.NewDesc("slurm_account_jobs_suspended", "Suspended jobs for account", labels, nil),
Expand All @@ -97,6 +101,7 @@ func NewAccountsCollector() *AccountsCollector {

func (ac *AccountsCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- ac.pending
ch <- ac.pending_cpus
ch <- ac.running
ch <- ac.running_cpus
ch <- ac.suspended
Expand All @@ -108,6 +113,9 @@ func (ac *AccountsCollector) Collect(ch chan<- prometheus.Metric) {
if am[a].pending > 0 {
ch <- prometheus.MustNewConstMetric(ac.pending, prometheus.GaugeValue, am[a].pending, a)
}
if am[a].pending_cpus > 0 {
ch <- prometheus.MustNewConstMetric(ac.pending_cpus, prometheus.GaugeValue, am[a].pending_cpus, a)
}
if am[a].running > 0 {
ch <- prometheus.MustNewConstMetric(ac.running, prometheus.GaugeValue, am[a].running, a)
}
Expand Down
10 changes: 9 additions & 1 deletion users.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ func UsersData() []byte {

type UserJobMetrics struct {
pending float64
pending_cpus float64
running float64
running_cpus float64
suspended float64
Expand All @@ -56,7 +57,7 @@ func ParseUsersMetrics(input []byte) map[string]*UserJobMetrics {
user := strings.Split(line,"|")[1]
_,key := users[user]
if !key {
users[user] = &UserJobMetrics{0,0,0,0}
users[user] = &UserJobMetrics{0,0,0,0,0}
}
state := strings.Split(line,"|")[2]
state = strings.ToLower(state)
Expand All @@ -67,6 +68,7 @@ func ParseUsersMetrics(input []byte) map[string]*UserJobMetrics {
switch {
case pending.MatchString(state) == true:
users[user].pending++
users[user].pending_cpus += cpus
case running.MatchString(state) == true:
users[user].running++
users[user].running_cpus += cpus
Expand All @@ -80,6 +82,7 @@ func ParseUsersMetrics(input []byte) map[string]*UserJobMetrics {

type UsersCollector struct {
pending *prometheus.Desc
pending_cpus *prometheus.Desc
running *prometheus.Desc
running_cpus *prometheus.Desc
suspended *prometheus.Desc
Expand All @@ -89,6 +92,7 @@ func NewUsersCollector() *UsersCollector {
labels := []string{"user"}
return &UsersCollector {
pending: prometheus.NewDesc("slurm_user_jobs_pending", "Pending jobs for user", labels, nil),
pending_cpus: prometheus.NewDesc("slurm_user_cpus_pending", "Pending jobs for user", labels, nil),
running: prometheus.NewDesc("slurm_user_jobs_running", "Running jobs for user", labels, nil),
running_cpus: prometheus.NewDesc("slurm_user_cpus_running", "Running cpus for user", labels, nil),
suspended: prometheus.NewDesc("slurm_user_jobs_suspended", "Suspended jobs for user", labels, nil),
Expand All @@ -97,6 +101,7 @@ func NewUsersCollector() *UsersCollector {

func (uc *UsersCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- uc.pending
ch <- uc.pending_cpus
ch <- uc.running
ch <- uc.running_cpus
ch <- uc.suspended
Expand All @@ -108,6 +113,9 @@ func (uc *UsersCollector) Collect(ch chan<- prometheus.Metric) {
if um[u].pending > 0 {
ch <- prometheus.MustNewConstMetric(uc.pending, prometheus.GaugeValue, um[u].pending, u)
}
if um[u].pending_cpus > 0 {
ch <- prometheus.MustNewConstMetric(uc.pending_cpus, prometheus.GaugeValue, um[u].pending_cpus, u)
}
if um[u].running > 0 {
ch <- prometheus.MustNewConstMetric(uc.running, prometheus.GaugeValue, um[u].running, u)
}
Expand Down