Skip to content

Commit

Permalink
Correct issues with sysload (#343)
Browse files Browse the repository at this point in the history 
* include system processes into FreeBSD number of processes

* add an option to only collect the ALL CPU

* typo

* only split lines one time

* add FreeBSD memory, ARC & swap info

* force flush of data to tcollector.py

* Be sure to always return the same type

because FreeBSD top prints 100% instead of 100.0%
  • Loading branch information
@johann8384
Ben RUBSON authored and johann8384 committed Sep 20, 2016
1 parent a0b4e23 commit 790aabd
Show file tree
Hide file tree
Showing 2 changed files with 158 additions and 24 deletions.
179 changes: 156 additions & 23 deletions collectors/0/sysload.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
In addition, for FreeBSD, it reports :
- load average (1m, 5m, 15m)
- number of processes (total, running, sleeping)
- number of processes (total, starting, running, sleeping, stopped, zombie, waiting, lock)
'''

import errno
Expand All @@ -51,6 +51,22 @@

DEFAULT_COLLECTION_INTERVAL=15

def convert_to_bytes(string):
"""Take a string in the form 1234K, and convert to bytes"""
factors = {
"K": 1024,
"M": 1024 * 1024,
"G": 1024 * 1024 * 1024,
"T": 1024 * 1024 * 1024 * 1024,
"P": 1024 * 1024 * 1024 * 1024 * 1024,
}
for f, fm in factors.items():
if string.endswith(f):
number = float(string[:-1])
number = number * fm
return long(number)
return long(string)

signal_received = None
def handlesignal(signum, stack):
global signal_received
Expand All @@ -60,9 +76,11 @@ def main():
"""top main loop"""

collection_interval=DEFAULT_COLLECTION_INTERVAL
collect_every_cpu=True
if(sysload_conf):
config = sysload_conf.get_config()
collection_interval=config['collection_interval']
collect_every_cpu=config['collect_every_cpu']

global signal_received

Expand All @@ -71,15 +89,27 @@ def main():

try:
if platform.system() == "FreeBSD":
p_top = subprocess.Popen(
["top", "-u", "-t", "-I", "-P", "-n", "-s"+str(collection_interval), "-d"+str((365*24*3600)/collection_interval)],
stdout=subprocess.PIPE,
)
if collect_every_cpu:
p_top = subprocess.Popen(
["top", "-S", "-P", "-n", "-s"+str(collection_interval), "-dinfinity", "0"],
stdout=subprocess.PIPE,
)
else:
p_top = subprocess.Popen(
["top", "-S", "-n", "-s"+str(collection_interval), "-dinfinity", "0"],
stdout=subprocess.PIPE,
)
else:
p_top = subprocess.Popen(
["mpstat", "-P", "ALL", str(collection_interval)],
stdout=subprocess.PIPE,
)
if collect_every_cpu:
p_top = subprocess.Popen(
["mpstat", "-P", "ALL", str(collection_interval)],
stdout=subprocess.PIPE,
)
else:
p_top = subprocess.Popen(
["mpstat", str(collection_interval)],
stdout=subprocess.PIPE,
)
except OSError, e:
if e.errno == errno.ENOENT:
# it makes no sense to run this collector here
Expand All @@ -100,44 +130,147 @@ def main():
# end of the program, die
break

fields = re.sub(r"%( [uni][a-z]+,?)?| AM | PM ", "", line).split()
# CPU: --> CPU all: : FreeBSD, to match the all CPU
# %( [uni][a-z]+,?)? : FreeBSD, so that top output matches mpstat output
# AM : Linux, mpstat output depending on locale
# PM : Linux, mpstat output depending on locale
# .* load : FreeBSD, to correctly match load averages
# , : FreeBSD, to correctly match processes: Mem: ARC: and Swap:
fields = re.sub("CPU:", "CPU all:", re.sub(r"%( [uni][a-z]+,?)?| AM | PM |.* load |,", "", line)).split()
if len(fields) <= 0:
continue

if (((fields[0] == "CPU") or (re.match("[0-9][0-9]:[0-9][0-9]:[0-9][0-9]",fields[0]))) and (re.match("[0-9]+:?",fields[1]))):
if(fields[1] == "0"):
if (((fields[0] == "CPU") or (re.match("[0-9][0-9]:[0-9][0-9]:[0-9][0-9]",fields[0]))) and ((collect_every_cpu and re.match("[0-9]+:?",fields[1])) or ((not collect_every_cpu) and re.match("all:?",fields[1])))):
if((fields[1] == "all") or (fields[1] == "0")):
timestamp = int(time.time())
cpuid=fields[1].replace(":","")
cpuuser=fields[2]
cpunice=fields[3]
cpusystem=fields[4]
cpuinterrupt=fields[6]
cpuidle=fields[-1]
print ("cpu.usr %s %s cpu=%s" % (timestamp, cpuuser, cpuid))
print ("cpu.nice %s %s cpu=%s" % (timestamp, cpunice, cpuid))
print ("cpu.sys %s %s cpu=%s" % (timestamp, cpusystem, cpuid))
print ("cpu.irq %s %s cpu=%s" % (timestamp, cpuinterrupt, cpuid))
print ("cpu.idle %s %s cpu=%s" % (timestamp, cpuidle, cpuid))
print ("cpu.usr %s %s cpu=%s" % (timestamp, float(cpuuser), cpuid))
print ("cpu.nice %s %s cpu=%s" % (timestamp, float(cpunice), cpuid))
print ("cpu.sys %s %s cpu=%s" % (timestamp, float(cpusystem), cpuid))
print ("cpu.irq %s %s cpu=%s" % (timestamp, float(cpuinterrupt), cpuid))
print ("cpu.idle %s %s cpu=%s" % (timestamp, float(cpuidle), cpuid))

elif (re.match("(.* load averages: *)",line)):
elif(fields[0] == "averages:"):
timestamp = int(time.time())
fields = re.sub(r".* load averages: *|,", "", line).split()
print ("load.1m %s %s" % (timestamp, fields[0]))
print ("load.5m %s %s" % (timestamp, fields[1]))
print ("load.15m %s %s" % (timestamp, fields[2]))
print ("load.1m %s %s" % (timestamp, fields[1]))
print ("load.5m %s %s" % (timestamp, fields[2]))
print ("load.15m %s %s" % (timestamp, fields[3]))

elif (re.match("[0-9]+ processes:",line)):
fields = re.sub(r",", "", line).split()
starting=0
running=0
sleeping=0
stopped=0
zombie=0
waiting=0
lock=0
for i in range(len(fields)):
if(fields[i] == "starting"):
starting=fields[i-1]
if(fields[i] == "running"):
running=fields[i-1]
if(fields[i] == "sleeping"):
sleeping=fields[i-1]
if(fields[i] == "stopped"):
stopped=fields[i-1]
if(fields[i] == "zombie"):
zombie=fields[i-1]
if(fields[i] == "waiting"):
waiting=fields[i-1]
if(fields[i] == "lock"):
lock=fields[i-1]
print ("ps.all %s %s" % (timestamp, fields[0]))
print ("ps.start %s %s" % (timestamp, starting))
print ("ps.run %s %s" % (timestamp, running))
print ("ps.sleep %s %s" % (timestamp, sleeping))
print ("ps.stop %s %s" % (timestamp, stopped))
print ("ps.zomb %s %s" % (timestamp, zombie))
print ("ps.wait %s %s" % (timestamp, waiting))
print ("ps.lock %s %s" % (timestamp, lock))

elif(fields[0] == "Mem:"):
active=0
inact=0
wired=0
cache=0
buf=0
free=0
for i in range(len(fields)):
if(fields[i] == "Active"):
active=convert_to_bytes(fields[i-1])
if(fields[i] == "Inact"):
inact=convert_to_bytes(fields[i-1])
if(fields[i] == "Wired"):
wired=convert_to_bytes(fields[i-1])
if(fields[i] == "Cache"):
cache=convert_to_bytes(fields[i-1])
if(fields[i] == "Buf"):
buf=convert_to_bytes(fields[i-1])
if(fields[i] == "Free"):
free=convert_to_bytes(fields[i-1])
print ("mem.active %s %s" % (timestamp, active))
print ("mem.inact %s %s" % (timestamp, inact))
print ("mem.wired %s %s" % (timestamp, wired))
print ("mem.cache %s %s" % (timestamp, cache))
print ("mem.buf %s %s" % (timestamp, buf))
print ("mem.free %s %s" % (timestamp, free))

elif(fields[0] == "ARC:"):
total=0
mru=0
mfu=0
anon=0
header=0
other=0
for i in range(len(fields)):
if(fields[i] == "Total"):
total=convert_to_bytes(fields[i-1])
if(fields[i] == "MRU"):
mru=convert_to_bytes(fields[i-1])
if(fields[i] == "MFU"):
mfu=convert_to_bytes(fields[i-1])
if(fields[i] == "Anon"):
anon=convert_to_bytes(fields[i-1])
if(fields[i] == "Header"):
header=convert_to_bytes(fields[i-1])
if(fields[i] == "Other"):
other=convert_to_bytes(fields[i-1])
print ("arc.total %s %s" % (timestamp, total))
print ("arc.mru %s %s" % (timestamp, mru))
print ("arc.mfu %s %s" % (timestamp, mfu))
print ("arc.anon %s %s" % (timestamp, anon))
print ("arc.header %s %s" % (timestamp, header))
print ("arc.other %s %s" % (timestamp, other))

elif(fields[0] == "Swap:"):
total=0
free=0
inuse=0
inps=0
outps=0
for i in range(len(fields)):
if(fields[i] == "Total"):
total=convert_to_bytes(fields[i-1])
if(fields[i] == "Free"):
free=convert_to_bytes(fields[i-1])
if(fields[i] == "Inuse"):
inuse=convert_to_bytes(fields[i-1])
if(fields[i] == "In"):
inps=convert_to_bytes(fields[i-1])/collection_interval
if(fields[i] == "Out"):
outps=convert_to_bytes(fields[i-1])/collection_interval
print ("swap.total %s %s" % (timestamp, total))
print ("swap.free %s %s" % (timestamp, free))
print ("swap.inuse %s %s" % (timestamp, inuse))
print ("swap.inps %s %s" % (timestamp, inps))
print ("swap.outps %s %s" % (timestamp, outps))

sys.stdout.flush()

if signal_received is None:
signal_received = signal.SIGTERM
Expand Down
3 changes: 2 additions & 1 deletion collectors/etc/sysload_conf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
def get_config():

config = {
'collection_interval': 15 # Seconds, how often to collect metric data
'collection_interval': 15, # Seconds, how often to collect metric data
'collect_every_cpu': True # True will collect statistics for every CPU, False for the "ALL" CPU
}

return config

0 comments on commit 790aabd

Please sign in to comment.