From 08ae941a871a982f163aa40a3cd5e23bc4a03d8e Mon Sep 17 00:00:00 2001 From: programokey <872127164@qq.com> Date: Fri, 6 Jul 2018 02:11:10 -0700 Subject: [PATCH] add some system monitor terms --- tools/prometheus/system/metrics.go | 121 ++++++++++++++++++++--------- 1 file changed, 83 insertions(+), 38 deletions(-) diff --git a/tools/prometheus/system/metrics.go b/tools/prometheus/system/metrics.go index 32c479e81..1b9f6c826 100644 --- a/tools/prometheus/system/metrics.go +++ b/tools/prometheus/system/metrics.go @@ -1,41 +1,83 @@ package system import ( + "errors" + "fmt" "github.com/go-kit/kit/metrics" "github.com/go-kit/kit/metrics/prometheus" stdprometheus "github.com/prometheus/client_golang/prometheus" - "time" - "fmt" - "os/exec" + "github.com/shirou/gopsutil/disk" + "github.com/shirou/gopsutil/process" "io/ioutil" - "strings" + "os/exec" "strconv" - "errors" - "github.com/shirou/gopsutil/process" - "github.com/shirou/gopsutil/disk" + "strings" + "time" ) -type Metrics struct{ - CPUUtilization metrics.Gauge + +type Metrics struct { + CPUUtilization metrics.Gauge MemoUtilization metrics.Gauge - OpenedFilesNum metrics.Gauge - DirSize metrics.Gauge + OpenedFilesNum metrics.Gauge + DirSize metrics.Gauge + metrics []metrics.Gauge + dirPaths []string + processes []process.Process +} + +func (metrics *Metrics) AddDirectory(path string) { + metrics.dirPaths = append(metrics.dirPaths, path) + name := fmt.Sprintf("Direcotry_Size_%s", strings.Replace(path, "/", "_", -1)) + help := fmt.Sprintf("total Size of files in %s", path) + metrics.metrics = append(metrics.metrics, prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: name, + Help: help, + }, []string{})) } +func (metrics *Metrics) AddProcess(command string) { + pid, err := getPid(command) + if err != nil { + return + } + process := process.Process{Pid: int32(pid)} + metrics.processes = append(metrics.processes, process) + + metrics.metrics = append(metrics.metrics, prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: fmt.Sprintf("CPU_Percent_%d", pid), + Help: fmt.Sprintf("CPU Utilization Percantage of processes with pid %d", pid), + }, []string{})) + + metrics.metrics = append(metrics.metrics, prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: fmt.Sprintf("Memo_Percent_%d", pid), + Help: fmt.Sprintf("Memory Utilization Percantage of processes with pid %d", pid), + }, []string{})) + + metrics.metrics = append(metrics.metrics, prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: fmt.Sprintf("Opened_Files_Number_%d", pid), + Help: fmt.Sprintf("Number of Opened Files of processes with pid %d", pid), + }, []string{})) + +} // PrometheusMetrics returns Metrics build using Prometheus client library. func PrometheusMetrics() *Metrics { return &Metrics{ - CPUUtilization:prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + CPUUtilization: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Subsystem: "system", Name: "CPU_Percent", Help: "CPU Utilization Percantage", }, []string{}), - MemoUtilization:prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + MemoUtilization: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Subsystem: "system", Name: "Memo_Percent", Help: "Memo Utilization Percantage", }, []string{}), - OpenedFilesNum:prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + OpenedFilesNum: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Subsystem: "system", Name: "Opened_Files_Number", Help: "Number of Opened Files, socket and other IO is included", @@ -45,47 +87,50 @@ func PrometheusMetrics() *Metrics { Name: "Directory_Size", Help: "total size of files in given directory (in bytes)", }, []string{}), + metrics: make([]metrics.Gauge, 0), + dirPaths: make([]string, 0), + processes: make([]process.Process, 0), } } -func Monitor(command string, dir_path string, metrics *Metrics)(error){ +func Monitor(command string, dir_path string, metrics *Metrics) error { pid, err := getPid(command) - if err != nil{ + if err != nil { return err } - go func(){ - for{ - time.Sleep(1*time.Second) + go func() { + for { + time.Sleep(1 * time.Second) metrics.RecordMetrics(int32(pid), dir_path) } }() return nil } -func (metrics Metrics)RecordMetrics(pid int32, dir_path string) { - proc := process.Process{Pid:pid} +func (metrics Metrics) RecordMetrics(pid int32, dir_path string) { + proc := process.Process{Pid: pid} - if cpu_util, err := proc.CPUPercent();err != nil{ + if cpu_util, err := proc.CPUPercent(); err != nil { metrics.CPUUtilization.Set(float64(-1)) - }else{ + } else { metrics.CPUUtilization.Set(cpu_util) } - if memo_util, err := proc.MemoryPercent();err != nil{ + if memo_util, err := proc.MemoryPercent(); err != nil { metrics.MemoUtilization.Set(float64(-1)) - }else { + } else { metrics.MemoUtilization.Set(float64(memo_util)) } - if files, err := proc.OpenFiles();err != nil{ + if files, err := proc.OpenFiles(); err != nil { metrics.OpenedFilesNum.Set(float64(-1)) - }else { + } else { metrics.OpenedFilesNum.Set(float64(len(files))) } - if usage, err := disk.Usage(dir_path); err != nil{ + if usage, err := disk.Usage(dir_path); err != nil { metrics.DirSize.Set(float64(-1)) - }else{ + } else { metrics.DirSize.Set(float64(usage.Used)) } } @@ -93,13 +138,13 @@ func (metrics Metrics)RecordMetrics(pid int32, dir_path string) { //get the pid of process that start by the given command //the first pid return by "ps -aux|grep ", // the process whose command contains "grep" is omitted -func getPid(command string)(pid int, err error){ +func getPid(command string) (pid int, err error) { command_str := fmt.Sprintf("ps -aux|grep '%s'", command) cmd := exec.Command("/bin/bash", "-c", command_str) stdout, err := cmd.StdoutPipe() - if err != nil{ + if err != nil { fmt.Printf("Error:can not obtain stdout pipe for command:%s\n", err) return 0, err } @@ -114,14 +159,14 @@ func getPid(command string)(pid int, err error){ fmt.Println("ReadAll Stdout:", err.Error()) return 0, err } - for _, item := range (strings.Split(string(bytes), "\n")){ - if !strings.Contains(item, "grep"){ - for j, s := range (strings.Split(item, " ")){ - if j > 0 && s != ""{ + for _, item := range strings.Split(string(bytes), "\n") { + if !strings.Contains(item, "grep") { + for j, s := range strings.Split(item, " ") { + if j > 0 && s != "" { pid, err = strconv.Atoi(s) - if err == nil{ + if err == nil { return pid, nil - }else { + } else { return 0, err } } @@ -129,4 +174,4 @@ func getPid(command string)(pid int, err error){ } } return 0, errors.New("cannot find the process") -} \ No newline at end of file +}