From 4d96b0f8c149b9e173077fac48cc2dca0e351c4a Mon Sep 17 00:00:00 2001 From: programokey <872127164@qq.com> Date: Fri, 6 Jul 2018 01:15:13 -0700 Subject: [PATCH 1/4] add some system monitor terms --- tools/prometheus/system/metrics.go | 1 + 1 file changed, 1 insertion(+) create mode 100644 tools/prometheus/system/metrics.go diff --git a/tools/prometheus/system/metrics.go b/tools/prometheus/system/metrics.go new file mode 100644 index 000000000..9b140a34d --- /dev/null +++ b/tools/prometheus/system/metrics.go @@ -0,0 +1 @@ +package system From 95d824b55bfabe9edfed00755ad0d3c136672534 Mon Sep 17 00:00:00 2001 From: programokey <872127164@qq.com> Date: Fri, 6 Jul 2018 01:19:20 -0700 Subject: [PATCH 2/4] add some system monitor terms --- tools/prometheus/system/metrics.go | 131 +++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/tools/prometheus/system/metrics.go b/tools/prometheus/system/metrics.go index 9b140a34d..32c479e81 100644 --- a/tools/prometheus/system/metrics.go +++ b/tools/prometheus/system/metrics.go @@ -1 +1,132 @@ package system + +import ( + "github.com/go-kit/kit/metrics" + "github.com/go-kit/kit/metrics/prometheus" + stdprometheus "github.com/prometheus/client_golang/prometheus" + "time" + "fmt" + "os/exec" + "io/ioutil" + "strings" + "strconv" + "errors" + "github.com/shirou/gopsutil/process" + "github.com/shirou/gopsutil/disk" +) +type Metrics struct{ + CPUUtilization metrics.Gauge + MemoUtilization metrics.Gauge + OpenedFilesNum metrics.Gauge + DirSize metrics.Gauge +} + + +// PrometheusMetrics returns Metrics build using Prometheus client library. +func PrometheusMetrics() *Metrics { + return &Metrics{ + CPUUtilization:prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: "CPU_Percent", + Help: "CPU Utilization Percantage", + }, []string{}), + MemoUtilization:prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: "Memo_Percent", + Help: "Memo Utilization Percantage", + }, []string{}), + OpenedFilesNum:prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: "Opened_Files_Number", + Help: "Number of Opened Files, socket and other IO is included", + }, []string{}), + DirSize: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: "Directory_Size", + Help: "total size of files in given directory (in bytes)", + }, []string{}), + } +} + +func Monitor(command string, dir_path string, metrics *Metrics)(error){ + pid, err := getPid(command) + if err != nil{ + return err + } + go func(){ + for{ + time.Sleep(1*time.Second) + metrics.RecordMetrics(int32(pid), dir_path) + } + }() + return nil +} + +func (metrics Metrics)RecordMetrics(pid int32, dir_path string) { + proc := process.Process{Pid:pid} + + if cpu_util, err := proc.CPUPercent();err != nil{ + metrics.CPUUtilization.Set(float64(-1)) + }else{ + metrics.CPUUtilization.Set(cpu_util) + } + + if memo_util, err := proc.MemoryPercent();err != nil{ + metrics.MemoUtilization.Set(float64(-1)) + }else { + metrics.MemoUtilization.Set(float64(memo_util)) + } + + if files, err := proc.OpenFiles();err != nil{ + metrics.OpenedFilesNum.Set(float64(-1)) + }else { + metrics.OpenedFilesNum.Set(float64(len(files))) + } + + if usage, err := disk.Usage(dir_path); err != nil{ + metrics.DirSize.Set(float64(-1)) + }else{ + metrics.DirSize.Set(float64(usage.Used)) + } +} + +//get the pid of process that start by the given command +//the first pid return by "ps -aux|grep ", +// the process whose command contains "grep" is omitted +func getPid(command string)(pid int, err error){ + command_str := fmt.Sprintf("ps -aux|grep '%s'", command) + cmd := exec.Command("/bin/bash", "-c", command_str) + + stdout, err := cmd.StdoutPipe() + + if err != nil{ + fmt.Printf("Error:can not obtain stdout pipe for command:%s\n", err) + return 0, err + } + + if err := cmd.Start(); err != nil { + fmt.Println("Error:Invalid command,", err) + return 0, err + } + + bytes, err := ioutil.ReadAll(stdout) + if err != nil { + fmt.Println("ReadAll Stdout:", err.Error()) + return 0, err + } + for _, item := range (strings.Split(string(bytes), "\n")){ + if !strings.Contains(item, "grep"){ + for j, s := range (strings.Split(item, " ")){ + if j > 0 && s != ""{ + pid, err = strconv.Atoi(s) + if err == nil{ + return pid, nil + }else { + return 0, err + } + } + } + } + } + return 0, errors.New("cannot find the process") +} \ No newline at end of file From 2af5d3670efe664d1541578c686b7de8e7402839 Mon Sep 17 00:00:00 2001 From: programokey <872127164@qq.com> Date: Fri, 6 Jul 2018 01:28:53 -0700 Subject: [PATCH 3/4] add some system monitor terms --- tools/prometheus/provider.go | 5 +++-- tools/prometheus/server.go | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tools/prometheus/provider.go b/tools/prometheus/provider.go index 429af8c27..68d580d8f 100644 --- a/tools/prometheus/provider.go +++ b/tools/prometheus/provider.go @@ -3,11 +3,12 @@ package prometheus import ( "github.com/irisnet/irishub/tools/prometheus/p2p" cs "github.com/irisnet/irishub/tools/prometheus/consensus" + sys "github.com/programokey/irishub/tools/prometheus/system" mempl "github.com/irisnet/irishub/tools/prometheus/mempool" ) // DefaultMetricsProvider returns consensus, p2p and mempool Metrics build // using Prometheus client library. -func DefaultMetricsProvider() (*cs.Metrics, *p2p.Metrics, *mempl.Metrics) { - return cs.PrometheusMetrics(), p2p.PrometheusMetrics(), mempl.PrometheusMetrics() +func DefaultMetricsProvider() (*cs.Metrics, *p2p.Metrics, *mempl.Metrics, *sys.Metrics) { + return cs.PrometheusMetrics(), p2p.PrometheusMetrics(), mempl.PrometheusMetrics(), sys.PrometheusMetrics() } \ No newline at end of file diff --git a/tools/prometheus/server.go b/tools/prometheus/server.go index 0c912abe8..618259546 100644 --- a/tools/prometheus/server.go +++ b/tools/prometheus/server.go @@ -9,6 +9,7 @@ import ( "github.com/cosmos/cosmos-sdk/client/context" "github.com/cosmos/cosmos-sdk/wire" // XXX fix "github.com/irisnet/irishub/tools/prometheus/consensus" + sys "github.com/programokey/irishub/tools/prometheus/system" ) @@ -18,12 +19,16 @@ func MonitorCommand(storeName string, cdc *wire.Codec) *cobra.Command { Short: "irishub monitor", RunE: func(cmd *cobra.Command, args []string) error { //TODO - csMetrics,_,_ := DefaultMetricsProvider() + csMetrics,_,_ , sysMertrics:= DefaultMetricsProvider() ctx := context.NewCoreContextFromViper() //监控共识参数 consensus.Monitor(ctx,*csMetrics,cdc,storeName) + //monitor system info, first parameter is the command of the process to be monitor + // and the second parameter is the directory that you want to get total size of its' files + sys.Monitor("irishub", "/", sysMertrics) + srv := &http.Server{ Addr: ":26660", Handler: promhttp.Handler(), From 08ae941a871a982f163aa40a3cd5e23bc4a03d8e Mon Sep 17 00:00:00 2001 From: programokey <872127164@qq.com> Date: Fri, 6 Jul 2018 02:11:10 -0700 Subject: [PATCH 4/4] add some system monitor terms --- tools/prometheus/system/metrics.go | 121 ++++++++++++++++++++--------- 1 file changed, 83 insertions(+), 38 deletions(-) diff --git a/tools/prometheus/system/metrics.go b/tools/prometheus/system/metrics.go index 32c479e81..1b9f6c826 100644 --- a/tools/prometheus/system/metrics.go +++ b/tools/prometheus/system/metrics.go @@ -1,41 +1,83 @@ package system import ( + "errors" + "fmt" "github.com/go-kit/kit/metrics" "github.com/go-kit/kit/metrics/prometheus" stdprometheus "github.com/prometheus/client_golang/prometheus" - "time" - "fmt" - "os/exec" + "github.com/shirou/gopsutil/disk" + "github.com/shirou/gopsutil/process" "io/ioutil" - "strings" + "os/exec" "strconv" - "errors" - "github.com/shirou/gopsutil/process" - "github.com/shirou/gopsutil/disk" + "strings" + "time" ) -type Metrics struct{ - CPUUtilization metrics.Gauge + +type Metrics struct { + CPUUtilization metrics.Gauge MemoUtilization metrics.Gauge - OpenedFilesNum metrics.Gauge - DirSize metrics.Gauge + OpenedFilesNum metrics.Gauge + DirSize metrics.Gauge + metrics []metrics.Gauge + dirPaths []string + processes []process.Process +} + +func (metrics *Metrics) AddDirectory(path string) { + metrics.dirPaths = append(metrics.dirPaths, path) + name := fmt.Sprintf("Direcotry_Size_%s", strings.Replace(path, "/", "_", -1)) + help := fmt.Sprintf("total Size of files in %s", path) + metrics.metrics = append(metrics.metrics, prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: name, + Help: help, + }, []string{})) } +func (metrics *Metrics) AddProcess(command string) { + pid, err := getPid(command) + if err != nil { + return + } + process := process.Process{Pid: int32(pid)} + metrics.processes = append(metrics.processes, process) + + metrics.metrics = append(metrics.metrics, prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: fmt.Sprintf("CPU_Percent_%d", pid), + Help: fmt.Sprintf("CPU Utilization Percantage of processes with pid %d", pid), + }, []string{})) + + metrics.metrics = append(metrics.metrics, prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: fmt.Sprintf("Memo_Percent_%d", pid), + Help: fmt.Sprintf("Memory Utilization Percantage of processes with pid %d", pid), + }, []string{})) + + metrics.metrics = append(metrics.metrics, prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: fmt.Sprintf("Opened_Files_Number_%d", pid), + Help: fmt.Sprintf("Number of Opened Files of processes with pid %d", pid), + }, []string{})) + +} // PrometheusMetrics returns Metrics build using Prometheus client library. func PrometheusMetrics() *Metrics { return &Metrics{ - CPUUtilization:prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + CPUUtilization: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Subsystem: "system", Name: "CPU_Percent", Help: "CPU Utilization Percantage", }, []string{}), - MemoUtilization:prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + MemoUtilization: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Subsystem: "system", Name: "Memo_Percent", Help: "Memo Utilization Percantage", }, []string{}), - OpenedFilesNum:prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + OpenedFilesNum: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Subsystem: "system", Name: "Opened_Files_Number", Help: "Number of Opened Files, socket and other IO is included", @@ -45,47 +87,50 @@ func PrometheusMetrics() *Metrics { Name: "Directory_Size", Help: "total size of files in given directory (in bytes)", }, []string{}), + metrics: make([]metrics.Gauge, 0), + dirPaths: make([]string, 0), + processes: make([]process.Process, 0), } } -func Monitor(command string, dir_path string, metrics *Metrics)(error){ +func Monitor(command string, dir_path string, metrics *Metrics) error { pid, err := getPid(command) - if err != nil{ + if err != nil { return err } - go func(){ - for{ - time.Sleep(1*time.Second) + go func() { + for { + time.Sleep(1 * time.Second) metrics.RecordMetrics(int32(pid), dir_path) } }() return nil } -func (metrics Metrics)RecordMetrics(pid int32, dir_path string) { - proc := process.Process{Pid:pid} +func (metrics Metrics) RecordMetrics(pid int32, dir_path string) { + proc := process.Process{Pid: pid} - if cpu_util, err := proc.CPUPercent();err != nil{ + if cpu_util, err := proc.CPUPercent(); err != nil { metrics.CPUUtilization.Set(float64(-1)) - }else{ + } else { metrics.CPUUtilization.Set(cpu_util) } - if memo_util, err := proc.MemoryPercent();err != nil{ + if memo_util, err := proc.MemoryPercent(); err != nil { metrics.MemoUtilization.Set(float64(-1)) - }else { + } else { metrics.MemoUtilization.Set(float64(memo_util)) } - if files, err := proc.OpenFiles();err != nil{ + if files, err := proc.OpenFiles(); err != nil { metrics.OpenedFilesNum.Set(float64(-1)) - }else { + } else { metrics.OpenedFilesNum.Set(float64(len(files))) } - if usage, err := disk.Usage(dir_path); err != nil{ + if usage, err := disk.Usage(dir_path); err != nil { metrics.DirSize.Set(float64(-1)) - }else{ + } else { metrics.DirSize.Set(float64(usage.Used)) } } @@ -93,13 +138,13 @@ func (metrics Metrics)RecordMetrics(pid int32, dir_path string) { //get the pid of process that start by the given command //the first pid return by "ps -aux|grep ", // the process whose command contains "grep" is omitted -func getPid(command string)(pid int, err error){ +func getPid(command string) (pid int, err error) { command_str := fmt.Sprintf("ps -aux|grep '%s'", command) cmd := exec.Command("/bin/bash", "-c", command_str) stdout, err := cmd.StdoutPipe() - if err != nil{ + if err != nil { fmt.Printf("Error:can not obtain stdout pipe for command:%s\n", err) return 0, err } @@ -114,14 +159,14 @@ func getPid(command string)(pid int, err error){ fmt.Println("ReadAll Stdout:", err.Error()) return 0, err } - for _, item := range (strings.Split(string(bytes), "\n")){ - if !strings.Contains(item, "grep"){ - for j, s := range (strings.Split(item, " ")){ - if j > 0 && s != ""{ + for _, item := range strings.Split(string(bytes), "\n") { + if !strings.Contains(item, "grep") { + for j, s := range strings.Split(item, " ") { + if j > 0 && s != "" { pid, err = strconv.Atoi(s) - if err == nil{ + if err == nil { return pid, nil - }else { + } else { return 0, err } } @@ -129,4 +174,4 @@ func getPid(command string)(pid int, err error){ } } return 0, errors.New("cannot find the process") -} \ No newline at end of file +}