diff --git a/topbeat/beat/config.go b/topbeat/beat/config.go index 8a8c393d093..994ea33c485 100644 --- a/topbeat/beat/config.go +++ b/topbeat/beat/config.go @@ -4,9 +4,10 @@ type TopConfig struct { Period *int64 Procs *[]string Stats struct { - System *bool - Proc *bool - Filesystem *bool + System *bool `yaml:"system"` + Proc *bool `yaml:"process"` + Filesystem *bool `yaml:"filesystem"` + CpuPerCore *bool `yaml:"cpu_per_core"` } } diff --git a/topbeat/beat/topbeat.go b/topbeat/beat/topbeat.go index f3068a2278e..088a75315e4 100644 --- a/topbeat/beat/topbeat.go +++ b/topbeat/beat/topbeat.go @@ -27,9 +27,10 @@ type Topbeat struct { TbConfig ConfigSettings events publisher.Client - sysStats bool - procStats bool - fsStats bool + sysStats bool + procStats bool + fsStats bool + cpuPerCore bool done chan struct{} } @@ -72,6 +73,11 @@ func (tb *Topbeat) Config(b *beat.Beat) error { } else { tb.fsStats = true } + if tb.TbConfig.Input.Stats.CpuPerCore != nil { + tb.cpuPerCore = *tb.TbConfig.Input.Stats.CpuPerCore + } else { + tb.cpuPerCore = false + } if !tb.sysStats && !tb.procStats && !tb.fsStats { return errors.New("Invalid statistics configuration") @@ -83,6 +89,7 @@ func (tb *Topbeat) Config(b *beat.Beat) error { logp.Debug("topbeat", "System statistics %t\n", tb.sysStats) logp.Debug("topbeat", "Process statistics %t\n", tb.procStats) logp.Debug("topbeat", "File system statistics %t\n", tb.fsStats) + logp.Debug("topbeat", "Cpu usage per core %t\n", tb.cpuPerCore) return nil } @@ -263,8 +270,14 @@ func (t *Topbeat) exportSystemStats() error { "swap": swap_stat, } - for coreNumber, core := range cpu_core_stat { - event["cpu"+strconv.Itoa(coreNumber)] = core + if t.cpuPerCore { + + cpus := common.MapStr{} + + for coreNumber, stat := range cpu_core_stat { + cpus["cpu"+strconv.Itoa(coreNumber)] = stat + } + event["cpus"] = cpus } t.events.PublishEvent(event) diff --git a/topbeat/docs/configuration.asciidoc b/topbeat/docs/configuration.asciidoc index 72a4429e47d..6f73e86ffc4 100644 --- a/topbeat/docs/configuration.asciidoc +++ b/topbeat/docs/configuration.asciidoc @@ -36,9 +36,19 @@ input: # Statistics to collect (all enabled by default) stats: + # per system statistics, by default is true system: true - proc: true + + # per process statistics, by default is true + process: true + + # file system information, by default is true filesystem: true + + # cpu usage per core, by default is false + cpu_per_core: false + + ------------------------------------------------------------------------------ ==== Options @@ -58,11 +68,32 @@ default, all the running processes are monitored. The statistics to collect. You can specify the following settings: -* `system: true` to capture system-wide statistics, including statistics about +* `system` to export system-wide statistics, including statistics about system load, CPU usage, memory usage, and swap usage. -* `proc: true` to capture per-process statistics, such as the process name, +* `process` to export per-process statistics, such as the process name, parent pid, state, pid, CPU usage, and memory usage. -* `filesystem: true` to capture file system statistics, including details about the +* `filesystem` to export file system statistics, including details about the mounted disks, such as the total and used disk space, and details about each disk, such as the device name and the mounting place. +* `cpu_per_core` to export the cpu usage per core. By default is false and it makes sense only if `system: true`. +The details are grouped under `cpus`. For each core of the server, few details are exported like: + +[source,json] +---------------------------------------------------------------------------------- + "cpus": { + "cpu0": { + "user": 373170, + "user_p": 0, + "nice": 0, + "system": 463408, + "system_p": 0, + "idle": 2904305, + "iowait": 0, + "irq": 0, + "softirq": 0, + "steal": 0 + }, + ... + } +---------------------------------------------------------------------------------- diff --git a/topbeat/docs/fields.asciidoc b/topbeat/docs/fields.asciidoc index 32dd550266a..6a614d3aaa8 100644 --- a/topbeat/docs/fields.asciidoc +++ b/topbeat/docs/fields.asciidoc @@ -169,6 +169,78 @@ type: int The amount of CPU time spent in involuntary wait by the virtual CPU while the hypervisor was servicing another processor. Available only on Unix. +=== cpus Fields + +This group contains cpu usage per core statistics. + + +=== cpuX Fields + +This group contains cpu usage statistics of the core X, where 0 + The amount of CPU time spent in user space on core X. + + - name: user_p + path: cpus.cpuX.user_p + type: float + description: > + The percentage of CPU time spent in user space on core X. + + - name: nice + path: cpus.cpuX.nice + type: int + description: > + The amount of CPU time spent on low-priority processes on core X. + + + - name: system + path: cpus.cpuX.system + type: int + description: > + The amount of CPU time spent in kernel space on core X. + + - name: system_p + path: cpus.cpuX.system_p + type: float + description: > + The percentage of CPU time spent in kernel space on core X. + + - name: idle + path: cpus.cpuX.idle + type: int + description: > + The amount of CPU time spent idle on core X. + + - name: iowait + path: cpus.cpuX.iowait + type: int + description: > + The amount of CPU time spent in wait (on disk) on core X. + + - name: softirq + path: cpus.cpuX.softirq + type: int + description: + The amount of CPU time spent servicing and handling software interrupts on core X. + + + - name: steal + path: cpus.cpuX.steal + type: int + description: > + The amount of CPU time spent in involuntary wait by the virtual CPU while the hypervisor + was servicing another processor on core X. + Available only on Unix. + + - name: mem type: group diff --git a/topbeat/etc/topbeat.yml b/topbeat/etc/topbeat.yml index 4ac7dffd88e..89112e38ae0 100644 --- a/topbeat/etc/topbeat.yml +++ b/topbeat/etc/topbeat.yml @@ -11,9 +11,19 @@ input: # Statistics to collect (all enabled by default) stats: + # per system statistics, by default is true system: true + + # per process statistics, by default is true proc: true + + # file system information, by default is true filesystem: true + + # cpu usage per core, by default is false + cpu_per_core: false + + ############################################################################### ############################# Libbeat Config ################################## # Base config file used by all other beats for using libbeat features diff --git a/topbeat/output.json b/topbeat/output.json new file mode 100644 index 00000000000..9371dfa7679 --- /dev/null +++ b/topbeat/output.json @@ -0,0 +1,209 @@ +type: system + +{ + "_index": "topbeat-2015.12.10", + "_type": "system", + "_id": "AVGL4gp3-qPEyihyAYjW", + "_score": null, + "_source": { + "@timestamp": "2015-12-10T12:33:45.584Z", + "beat": { + "hostname": "mar", + "name": "mar" + }, + "count": 1, + "cpu": { + "user": 1124477, + "user_p": 0, + "nice": 0, + "system": 937191, + "system_p": 0, + "idle": 12901569, + "iowait": 0, + "irq": 0, + "softirq": 0, + "steal": 0 + }, + "cpus": { + "cpu0": { + "user": 373170, + "user_p": 0, + "nice": 0, + "system": 463408, + "system_p": 0, + "idle": 2904305, + "iowait": 0, + "irq": 0, + "softirq": 0, + "steal": 0 + }, + "cpu1": { + "user": 183293, + "user_p": 0, + "nice": 0, + "system": 87867, + "system_p": 0, + "idle": 3469625, + "iowait": 0, + "irq": 0, + "softirq": 0, + "steal": 0 + }, + "cpu2": { + "user": 376911, + "user_p": 0, + "nice": 0, + "system": 296693, + "system_p": 0, + "idle": 3067182, + "iowait": 0, + "irq": 0, + "softirq": 0, + "steal": 0 + }, + "cpu3": { + "user": 191103, + "user_p": 0, + "nice": 0, + "system": 89223, + "system_p": 0, + "idle": 3460457, + "iowait": 0, + "irq": 0, + "softirq": 0, + "steal": 0 + } + }, + "load": { + "load1": 1.66357421875, + "load5": 1.81396484375, + "load15": 1.78466796875 + }, + "mem": { + "total": 17179869184, + "used": 14505914368, + "free": 2673954816, + "used_p": 0.84, + "actual_used": 10005078016, + "actual_free": 7174791168, + "actual_used_p": 0.58 + }, + "swap": { + "total": 0, + "used": 0, + "free": 0, + "used_p": 0, + "actual_used": 0, + "actual_free": 0, + "actual_used_p": 0 + }, + "type": "system" + }, + "fields": { + "@timestamp": [ + 1449750825584 + ] + }, + "highlight": { + "type": [ + "@kibana-highlighted-field@system@/kibana-highlighted-field@" + ] + }, + "sort": [ + 1449750825584 + ] +} + +type: process + + +{ + "_index": "topbeat-2015.12.10", + "_type": "process", + "_id": "AVGL44HJ-qPEyihyAY4q", + "_score": null, + "_source": { + "@timestamp": "2015-12-10T12:35:20.684Z", + "beat": { + "hostname": "mar", + "name": "mar" + }, + "count": 1, + "proc": { + "cpu": { + "user": 3397, + "user_p": 0, + "system": 2972, + "total": 6369, + "start_time": "17:48" + }, + "mem": { + "size": 2532204544, + "rss": 3616768, + "rss_p": 0, + "share": 0 + }, + "name": "cfprefsd", + "pid": 228, + "ppid": 1, + "state": "running" + }, + "type": "process" + }, + "fields": { + "@timestamp": [ + 1449750920684 + ] + }, + "highlight": { + "type": [ + "@kibana-highlighted-field@process@/kibana-highlighted-field@" + ] + }, + "sort": [ + 1449750920684 + ] +} + +type: filesystem + +{ + "_index": "topbeat-2015.12.10", + "_type": "filesystem", + "_id": "AVGL5B4F-qPEyihyAZDq", + "_score": null, + "_source": { + "@timestamp": "2015-12-10T12:36:00.678Z", + "beat": { + "hostname": "mar", + "name": "mar" + }, + "count": 1, + "fs": { + "device_name": "/dev/disk1", + "total": 249779191808, + "used": 207223169024, + "used_p": 0.83, + "free": 42556022784, + "avail": 42293878784, + "files": 60981246, + "free_files": 10325654, + "mount_point": "/" + }, + "type": "filesystem" + }, + "fields": { + "@timestamp": [ + 1449750960678 + ] + }, + "highlight": { + "type": [ + "@kibana-highlighted-field@filesystem@/kibana-highlighted-field@" + ] + }, + "sort": [ + 1449750960678 + ] +} + diff --git a/topbeat/tests/system/config/topbeat.yml.j2 b/topbeat/tests/system/config/topbeat.yml.j2 index 347e2bf5fa7..435e729c4ff 100644 --- a/topbeat/tests/system/config/topbeat.yml.j2 +++ b/topbeat/tests/system/config/topbeat.yml.j2 @@ -15,8 +15,9 @@ input: # Statistics to collect (all enabled by default) stats: system: {{ "false" if system_stats == false else "true" }} - proc: {{ "false" if proc_stats == false else "true" }} + process: {{ "false" if process_stats == false else "true" }} filesystem: {{ "false" if filesystem_stats == false else "true" }} + cpu_per_core: {{ "false" if cpu_per_core == false else "true" }} ############################# Output ########################################## diff --git a/topbeat/tests/system/test_cpu_per_core.py b/topbeat/tests/system/test_cpu_per_core.py new file mode 100644 index 00000000000..8569de14615 --- /dev/null +++ b/topbeat/tests/system/test_cpu_per_core.py @@ -0,0 +1,45 @@ +from topbeat import TestCase + +import os + +""" +Contains tests for ide statistics. +""" + + +class Test(TestCase): + def test_cpu_per_core(self): + """ + Checks that cpu usage per core statistics are exported + when the config option is enabled. + """ + # the test applies only for Unix systems + if os.name == "nt": + return + + self.render_config_template( + system_stats=True, + process_stats=False, + filesystem_stats=False, + cpu_per_core=True + ) + topbeat = self.start_topbeat() + self.wait_until(lambda: self.output_has(lines=1)) + topbeat.kill_and_wait() + + output = self.read_output()[0] + + for key in [ + "cpus.cpu0.user_p", + "cpus.cpu0.system_p", + "cpus.cpu0.user", + "cpus.cpu0.system", + "cpus.cpu0.nice", + "cpus.cpu0.idle", + "cpus.cpu0.iowait", + "cpus.cpu0.irq", + "cpus.cpu0.softirq", + "cpus.cpu0.steal", + + ]: + assert type(output[key]) in [int, float] diff --git a/topbeat/tests/system/test_filesystem.py b/topbeat/tests/system/test_filesystem.py index fc2cde3d22c..0dd9d210bc1 100644 --- a/topbeat/tests/system/test_filesystem.py +++ b/topbeat/tests/system/test_filesystem.py @@ -14,7 +14,7 @@ def test_filesystems(self): """ self.render_config_template( system_stats=False, - proc_stats=False, + process_stats=False, filesystem_stats=True ) topbeat = self.start_topbeat() diff --git a/topbeat/tests/system/test_procs.py b/topbeat/tests/system/test_procs.py index d58ed5fb425..c6603247d46 100644 --- a/topbeat/tests/system/test_procs.py +++ b/topbeat/tests/system/test_procs.py @@ -15,7 +15,7 @@ def test_procs(self): """ self.render_config_template( system_stats=False, - proc_stats=True, + process_stats=True, filesystem_stats=False, proc_patterns=["topbeat.test"] # monitor itself ) diff --git a/topbeat/tests/system/test_system.py b/topbeat/tests/system/test_system.py index 4fb38e6ce77..ef4c2dbdf6f 100644 --- a/topbeat/tests/system/test_system.py +++ b/topbeat/tests/system/test_system.py @@ -15,7 +15,7 @@ def test_system_wide(self): """ self.render_config_template( system_stats=True, - proc_stats=False, + process_stats=False, filesystem_stats=False ) topbeat = self.start_topbeat()