First refactor of the system module - system/cpu and system/core (#25771

) (#26359)

* initial commit

* finux linux refactor

* fix up freebsd

* port main metricset, start openbsd

* start work on openbsd vagrantfile

* refactors of API, add darwin support

* fix darwin implementation

* refactor API, move tests, remove old code, take a crack at AIX

* fix aix init func

* fix tests

* regenerate core data.json

* small fixes, fix host field

* update tests

* run correct mage commands

* try to fix system tests

* more fixes for windows python tests

* refactor CPU struct, use reflection

* refactor reflection code, add validation

* move metrics to its own internal folder

* move directories, again

* move directories, again

* use optional Uint type

* cleanup opt files

* move around naming of opt types

* fix up if block

* change opt names

* move around opt methods, cpu stat reader refactor

* fix IsZero usage

* add changelog

(cherry picked from commit 2871d29)

CHANGELOG.next.asciidoc

@@ -373,6 +373,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Use working set bytes to calculate the pod memory limit pct when memory usage is not reported (ie. Windows pods). {pull}25428[25428]
 - Fix copy-paste error in libbeat docs. {pull}25448[25448]
 - Fix azure billing dashboard. {pull}25554[25554]
+- Major refactor of system/cpu and system/core metrics. {pull}25771[25771]
 
 *Packetbeat*
 

Vagrantfile

@@ -301,6 +301,9 @@ Vagrant.configure("2") do |config|
 
     c.vm.hostname = "beats-tester"
     c.vm.provision "shell", inline: $unixProvision, privileged: false
+    c.vm.provision "shell", inline: $freebsdShellUpdate, privileged: true
+    c.vm.provision "shell", inline: gvmProvision(arch="amd64", os="freebsd"), privileged: false
+    c.vm.provision "shell", inline: "sudo mount -t linprocfs /dev/null /proc", privileged: false
   end
 
   # OpenBSD 5.9-stable

libbeat/metric/system/cpu/cpu.go

@@ -26,165 +26,6 @@ import (
 	sigar "github.com/elastic/gosigar"
 )
 
-// CPU Monitor
-
-// Monitor is used to monitor the overall CPU usage of the system.
-type Monitor struct {
-	lastSample *sigar.Cpu
-}
-
-// Sample collects a new sample of the CPU usage metrics.
-func (m *Monitor) Sample() (*Metrics, error) {
-	cpuSample := &sigar.Cpu{}
-	if err := cpuSample.Get(); err != nil {
-		return nil, err
-	}
-
-	oldLastSample := m.lastSample
-	m.lastSample = cpuSample
-	return &Metrics{oldLastSample, cpuSample}, nil
-}
-
-// Percentages stores all CPU values in percentages collected by a Beat.
-type Percentages struct {
-	User    float64
-	System  float64
-	Idle    float64
-	IOWait  float64
-	IRQ     float64
-	Nice    float64
-	SoftIRQ float64
-	Steal   float64
-	Total   float64
-}
-
-// Ticks stores all CPU values in number of tick collected by a Beat.
-type Ticks struct {
-	User    uint64
-	System  uint64
-	Idle    uint64
-	IOWait  uint64
-	IRQ     uint64
-	Nice    uint64
-	SoftIRQ uint64
-	Steal   uint64
-}
-
-// Metrics stores the current and the last sample collected by a Beat.
-type Metrics struct {
-	previousSample *sigar.Cpu
-	currentSample  *sigar.Cpu
-}
-
-// NormalizedPercentages returns CPU percentage usage information that is
-// normalized by the number of CPU cores. The values will range from
-// 0 to 100%.
-func (m *Metrics) NormalizedPercentages() Percentages {
-	return cpuPercentages(m.previousSample, m.currentSample, 1)
-}
-
-// Percentages returns CPU percentage usage information. The values range from
-// 0 to 100% * NumCPU.
-func (m *Metrics) Percentages() Percentages {
-	return cpuPercentages(m.previousSample, m.currentSample, runtime.NumCPU())
-}
-
-// cpuPercentages calculates the amount of CPU time used between the two given
-// samples. The CPU percentages are divided by given numCPU value and rounded
-// using Round.
-func cpuPercentages(s0, s1 *sigar.Cpu, numCPU int) Percentages {
-	if s0 == nil || s1 == nil {
-		return Percentages{}
-	}
-
-	// timeDelta is the total amount of CPU time available across all CPU cores.
-	timeDelta := s1.Total() - s0.Total()
-	if timeDelta <= 0 {
-		return Percentages{}
-	}
-
-	calculatePct := func(v0, v1 uint64) float64 {
-		cpuDelta := int64(v1 - v0)
-		pct := float64(cpuDelta) / float64(timeDelta)
-		return common.Round(pct*float64(numCPU), common.DefaultDecimalPlacesCount)
-	}
-
-	calculateTotalPct := func() float64 {
-		// IOWait time is excluded from the total as per #7627.
-		idle := calculatePct(s0.Idle, s1.Idle) + calculatePct(s0.Wait, s1.Wait)
-		return common.Round(float64(numCPU)-idle, common.DefaultDecimalPlacesCount)
-	}
-
-	return Percentages{
-		User:    calculatePct(s0.User, s1.User),
-		System:  calculatePct(s0.Sys, s1.Sys),
-		Idle:    calculatePct(s0.Idle, s1.Idle),
-		IOWait:  calculatePct(s0.Wait, s1.Wait),
-		IRQ:     calculatePct(s0.Irq, s1.Irq),
-		Nice:    calculatePct(s0.Nice, s1.Nice),
-		SoftIRQ: calculatePct(s0.SoftIrq, s1.SoftIrq),
-		Steal:   calculatePct(s0.Stolen, s1.Stolen),
-		Total:   calculateTotalPct(),
-	}
-}
-
-// Ticks returns the number of CPU ticks from the last collected sample.
-func (m *Metrics) Ticks() Ticks {
-	return Ticks{
-		User:    m.currentSample.User,
-		System:  m.currentSample.Sys,
-		Idle:    m.currentSample.Idle,
-		IOWait:  m.currentSample.Wait,
-		IRQ:     m.currentSample.Irq,
-		Nice:    m.currentSample.Nice,
-		SoftIRQ: m.currentSample.SoftIrq,
-		Steal:   m.currentSample.Stolen,
-	}
-}
-
-// CPU Core Monitor
-
-// CoreMetrics is used to monitor the usage of individual CPU cores.
-type CoreMetrics Metrics
-
-// Percentages returns CPU percentage usage information for the core. The values
-// range from [0, 100%].
-func (m *CoreMetrics) Percentages() Percentages { return (*Metrics)(m).NormalizedPercentages() }
-
-// Ticks returns the raw number of "ticks". The value is a counter (though it
-// may roll overfunc (m *CoreMetrics) Ticks() Ticks { return (*Metrics)(m).Ticks() }
-func (m *CoreMetrics) Ticks() Ticks { return (*Metrics)(m).Ticks() }
-
-// CoresMonitor is used to monitor the usage information of all the CPU
-// cores in the system.
-type CoresMonitor struct {
-	lastSample []sigar.Cpu
-}
-
-// Sample collects a new sample of the metrics from all CPU cores.
-func (m *CoresMonitor) Sample() ([]CoreMetrics, error) {
-	var cores sigar.CpuList
-	if err := cores.Get(); err != nil {
-		return nil, err
-	}
-
-	lastSample := m.lastSample
-	m.lastSample = cores.List
-
-	cpuMetrics := make([]CoreMetrics, len(cores.List))
-	for i := 0; i < len(cores.List); i++ {
-		if len(lastSample) > i {
-			cpuMetrics[i] = CoreMetrics{&lastSample[i], &cores.List[i]}
-		} else {
-			cpuMetrics[i] = CoreMetrics{nil, &cores.List[i]}
-		}
-	}
-
-	return cpuMetrics, nil
-}
-
-// CPU Load
-
 // Load returns CPU load information for the previous 1, 5, and 15 minute
 // periods.
 func Load() (*LoadMetrics, error) {

metricbeat/docs/fields.asciidoc

@@ -43145,6 +43145,18 @@ type: long
 
 --
 
+*`system.core.total.pct`*::
++
+--
+Total active time spent by the core
+
+
+type: scaled_float
+
+format: percent
+
+--
+
 *`system.core.user.pct`*::
 +
 --

metricbeat/internal/metrics/cpu/doc.go

@@ -0,0 +1,18 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package cpu