diff --git a/tools/prometheus/provider.go b/tools/prometheus/provider.go index 429af8c27..68d580d8f 100644 --- a/tools/prometheus/provider.go +++ b/tools/prometheus/provider.go @@ -3,11 +3,12 @@ package prometheus import ( "github.com/irisnet/irishub/tools/prometheus/p2p" cs "github.com/irisnet/irishub/tools/prometheus/consensus" + sys "github.com/programokey/irishub/tools/prometheus/system" mempl "github.com/irisnet/irishub/tools/prometheus/mempool" ) // DefaultMetricsProvider returns consensus, p2p and mempool Metrics build // using Prometheus client library. -func DefaultMetricsProvider() (*cs.Metrics, *p2p.Metrics, *mempl.Metrics) { - return cs.PrometheusMetrics(), p2p.PrometheusMetrics(), mempl.PrometheusMetrics() +func DefaultMetricsProvider() (*cs.Metrics, *p2p.Metrics, *mempl.Metrics, *sys.Metrics) { + return cs.PrometheusMetrics(), p2p.PrometheusMetrics(), mempl.PrometheusMetrics(), sys.PrometheusMetrics() } \ No newline at end of file diff --git a/tools/prometheus/server.go b/tools/prometheus/server.go index 0c912abe8..618259546 100644 --- a/tools/prometheus/server.go +++ b/tools/prometheus/server.go @@ -9,6 +9,7 @@ import ( "github.com/cosmos/cosmos-sdk/client/context" "github.com/cosmos/cosmos-sdk/wire" // XXX fix "github.com/irisnet/irishub/tools/prometheus/consensus" + sys "github.com/programokey/irishub/tools/prometheus/system" ) @@ -18,12 +19,16 @@ func MonitorCommand(storeName string, cdc *wire.Codec) *cobra.Command { Short: "irishub monitor", RunE: func(cmd *cobra.Command, args []string) error { //TODO - csMetrics,_,_ := DefaultMetricsProvider() + csMetrics,_,_ , sysMertrics:= DefaultMetricsProvider() ctx := context.NewCoreContextFromViper() //监控共识参数 consensus.Monitor(ctx,*csMetrics,cdc,storeName) + //monitor system info, first parameter is the command of the process to be monitor + // and the second parameter is the directory that you want to get total size of its' files + sys.Monitor("irishub", "/", sysMertrics) + srv := &http.Server{ Addr: ":26660", Handler: promhttp.Handler(), diff --git a/tools/prometheus/system/metrics.go b/tools/prometheus/system/metrics.go new file mode 100644 index 000000000..1b9f6c826 --- /dev/null +++ b/tools/prometheus/system/metrics.go @@ -0,0 +1,177 @@ +package system + +import ( + "errors" + "fmt" + "github.com/go-kit/kit/metrics" + "github.com/go-kit/kit/metrics/prometheus" + stdprometheus "github.com/prometheus/client_golang/prometheus" + "github.com/shirou/gopsutil/disk" + "github.com/shirou/gopsutil/process" + "io/ioutil" + "os/exec" + "strconv" + "strings" + "time" +) + +type Metrics struct { + CPUUtilization metrics.Gauge + MemoUtilization metrics.Gauge + OpenedFilesNum metrics.Gauge + DirSize metrics.Gauge + metrics []metrics.Gauge + dirPaths []string + processes []process.Process +} + +func (metrics *Metrics) AddDirectory(path string) { + metrics.dirPaths = append(metrics.dirPaths, path) + name := fmt.Sprintf("Direcotry_Size_%s", strings.Replace(path, "/", "_", -1)) + help := fmt.Sprintf("total Size of files in %s", path) + metrics.metrics = append(metrics.metrics, prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: name, + Help: help, + }, []string{})) +} + +func (metrics *Metrics) AddProcess(command string) { + pid, err := getPid(command) + if err != nil { + return + } + process := process.Process{Pid: int32(pid)} + metrics.processes = append(metrics.processes, process) + + metrics.metrics = append(metrics.metrics, prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: fmt.Sprintf("CPU_Percent_%d", pid), + Help: fmt.Sprintf("CPU Utilization Percantage of processes with pid %d", pid), + }, []string{})) + + metrics.metrics = append(metrics.metrics, prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: fmt.Sprintf("Memo_Percent_%d", pid), + Help: fmt.Sprintf("Memory Utilization Percantage of processes with pid %d", pid), + }, []string{})) + + metrics.metrics = append(metrics.metrics, prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: fmt.Sprintf("Opened_Files_Number_%d", pid), + Help: fmt.Sprintf("Number of Opened Files of processes with pid %d", pid), + }, []string{})) + +} + +// PrometheusMetrics returns Metrics build using Prometheus client library. +func PrometheusMetrics() *Metrics { + return &Metrics{ + CPUUtilization: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: "CPU_Percent", + Help: "CPU Utilization Percantage", + }, []string{}), + MemoUtilization: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: "Memo_Percent", + Help: "Memo Utilization Percantage", + }, []string{}), + OpenedFilesNum: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: "Opened_Files_Number", + Help: "Number of Opened Files, socket and other IO is included", + }, []string{}), + DirSize: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Subsystem: "system", + Name: "Directory_Size", + Help: "total size of files in given directory (in bytes)", + }, []string{}), + metrics: make([]metrics.Gauge, 0), + dirPaths: make([]string, 0), + processes: make([]process.Process, 0), + } +} + +func Monitor(command string, dir_path string, metrics *Metrics) error { + pid, err := getPid(command) + if err != nil { + return err + } + go func() { + for { + time.Sleep(1 * time.Second) + metrics.RecordMetrics(int32(pid), dir_path) + } + }() + return nil +} + +func (metrics Metrics) RecordMetrics(pid int32, dir_path string) { + proc := process.Process{Pid: pid} + + if cpu_util, err := proc.CPUPercent(); err != nil { + metrics.CPUUtilization.Set(float64(-1)) + } else { + metrics.CPUUtilization.Set(cpu_util) + } + + if memo_util, err := proc.MemoryPercent(); err != nil { + metrics.MemoUtilization.Set(float64(-1)) + } else { + metrics.MemoUtilization.Set(float64(memo_util)) + } + + if files, err := proc.OpenFiles(); err != nil { + metrics.OpenedFilesNum.Set(float64(-1)) + } else { + metrics.OpenedFilesNum.Set(float64(len(files))) + } + + if usage, err := disk.Usage(dir_path); err != nil { + metrics.DirSize.Set(float64(-1)) + } else { + metrics.DirSize.Set(float64(usage.Used)) + } +} + +//get the pid of process that start by the given command +//the first pid return by "ps -aux|grep ", +// the process whose command contains "grep" is omitted +func getPid(command string) (pid int, err error) { + command_str := fmt.Sprintf("ps -aux|grep '%s'", command) + cmd := exec.Command("/bin/bash", "-c", command_str) + + stdout, err := cmd.StdoutPipe() + + if err != nil { + fmt.Printf("Error:can not obtain stdout pipe for command:%s\n", err) + return 0, err + } + + if err := cmd.Start(); err != nil { + fmt.Println("Error:Invalid command,", err) + return 0, err + } + + bytes, err := ioutil.ReadAll(stdout) + if err != nil { + fmt.Println("ReadAll Stdout:", err.Error()) + return 0, err + } + for _, item := range strings.Split(string(bytes), "\n") { + if !strings.Contains(item, "grep") { + for j, s := range strings.Split(item, " ") { + if j > 0 && s != "" { + pid, err = strconv.Atoi(s) + if err == nil { + return pid, nil + } else { + return 0, err + } + } + } + } + } + return 0, errors.New("cannot find the process") +}