measure memory usage in benchmarks (on linux). improve tests by using…

… exit-when-finished feature of client_test

tools/benchmark_checking.py

@@ -3,9 +3,15 @@
 
 import argparse
 import os
+import platform
 import shutil
 import subprocess
 import sys
+import time
+
+from linux_vmstat import capture_sample
+from linux_vmstat import plot_output
+from linux_vmstat import print_output_to_file
 
 
 def main():
@@ -26,7 +32,7 @@ def main():
             print('ERROR: connection_tester failed: %d' % ret)
             sys.exit(1)
 
-    if not os.path.exists('checking_benchmark'):
+    if not os.path.exists("checking_benchmark.torrent"):
         ret = os.system('../examples/connection_tester gen-data -t checking_benchmark.torrent -p .')
         if ret != 0:
             print('ERROR: connection_tester failed: %d' % ret)
@@ -60,7 +66,20 @@ def run_test(name, client_arg):
     client_out = open('%s/client.out' % output_dir, 'w+')
     print('client_cmd: "{cmd}"'.format(cmd=client_cmd))
     c = subprocess.Popen(client_cmd.split(' '), stdout=client_out, stderr=client_out, stdin=subprocess.PIPE)
-    c.wait()
+    start_time = time.time()
+
+    if platform.system() == "Linux":
+        out = {}
+        while c.returncode is None:
+            capture_sample(c.pid, start_time, out)
+            time.sleep(0.1)
+            c.poll()
+
+        stats_filename = f"{output_dir}/memory_stats.log"
+        keys = print_output_to_file(out, stats_filename)
+        plot_output(stats_filename, keys)
+    else:
+        c.wait()
 
     client_out.close()
 
@@ -101,4 +120,4 @@ def parse_args():
 
 
 if __name__ == '__main__':
-    main()
+    main()

tools/linux_vmstat.py

@@ -0,0 +1,123 @@
+# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
+
+import os
+from time import time
+from typing import Dict
+from typing import List
+from typing import Set
+
+
+def capture_sample(pid: int, start_time: int, output: Dict[str, List[int]]) -> None:
+    try:
+        with open(f"/proc/{pid}/smaps_rollup") as f:
+            sample = f.read()
+            timestamp = int((time() - start_time) * 1000)
+    except Exception:
+        return
+
+    if "time" not in output:
+        output["time"] = [timestamp]
+    else:
+        output["time"].append(timestamp)
+
+    for line in sample.split("\n"):
+        if "[rollup]" in line:
+            continue
+        if line.strip() == "":
+            continue
+
+        key, value = line.split(":")
+        val = int(value.split()[0].strip())
+        key = key.strip()
+
+        if key not in output:
+            output[key] = [val]
+        else:
+            output[key].append(val)
+
+
+# example output:
+# 8affffff000-7fffba926000 ---p 00000000 00:00 0  [rollup]
+#    Rss:               76932 kB
+#    Pss:               17508 kB
+#    Pss_Anon:          11376 kB
+#    Pss_File:           6101 kB
+#    Pss_Shmem:            30 kB
+#    Shared_Clean:      65380 kB
+#    Shared_Dirty:         88 kB
+#    Private_Clean:        80 kB
+#    Private_Dirty:     11384 kB
+#    Referenced:        76932 kB
+#    Anonymous:         11376 kB
+#    LazyFree:              0 kB
+#    AnonHugePages:         0 kB
+#    ShmemPmdMapped:        0 kB
+#    FilePmdMapped:         0 kB
+#    Shared_Hugetlb:        0 kB
+#    Private_Hugetlb:       0 kB
+#    Swap:                  0 kB
+#    SwapPss:               0 kB
+#    Locked:                0 kB
+
+
+def print_output_to_file(out: Dict[str, List[int]], filename: str) -> List[str]:
+
+    if out == {}:
+        return []
+
+    with open(filename, "w+") as stats_output:
+        non_zero_keys: Set[str] = set()
+        non_zero_keys.add("time")
+        keys = out.keys()
+        for key in keys:
+            stats_output.write(f"{key} ")
+        stats_output.write("\n")
+        idx = 0
+        while len(out["time"]) > idx:
+            for key in keys:
+                stats_output.write(f"{out[key][idx]} ")
+                if out[key][idx] != 0:
+                    non_zero_keys.add(key)
+            stats_output.write("\n")
+            idx += 1
+    return [k if k in non_zero_keys else "" for k in keys]
+
+
+def plot_output(filename: str, keys: List[str]) -> None:
+    if "time" not in keys:
+        return
+
+    output_dir, in_file = os.path.split(filename)
+    gnuplot_file = f"{output_dir}/plot_{in_file}.gnuplot"
+    with open(gnuplot_file, "w+") as f:
+        f.write(
+            f"""set term png size 1200,700
+set output "{in_file}.png"
+set format y '%.0f MB'
+set title "libtorrent memory usage"
+set ylabel "Memory Size"
+set xlabel "time (s)"
+set xrange [0:*]
+set yrange [2:*]
+set logscale y 2
+set grid
+plot """
+        )
+
+        plot_string = ""
+        tidx = keys.index("time") + 1
+        idx = 0
+        for p in keys:
+            idx += 1
+            if p == "time" or p == "":
+                continue
+            # escape underscores, since gnuplot interprets those as markup
+            p = p.replace("_", "\\\\_")
+            plot_string += (
+                f'"{in_file}" using (${tidx}/1000):(${idx}/1024) '
+                + f'title "{p}" with steps, \\\n'
+            )
+        plot_string = plot_string[0:-4]
+        f.write(plot_string)
+
+    os.system(f"(cd {output_dir}; gnuplot {os.path.split(gnuplot_file)[1]})")

tools/run_benchmark.py

@@ -7,6 +7,11 @@
 import shutil
 import subprocess
 import sys
+import platform
+
+from linux_vmstat import capture_sample
+from linux_vmstat import plot_output
+from linux_vmstat import print_output_to_file
 
 
 def main():
@@ -29,15 +34,15 @@ def main():
     rm_file_or_dir('cpu_benchmark')
 
     if not os.path.exists('cpu_benchmark.torrent'):
-        ret = os.system('../examples/connection_tester gen-torrent -s 10000 -n 15 -t cpu_benchmark.torrent')
+        ret = os.system('../examples/connection_tester gen-torrent -s 20000 -n 15 -t cpu_benchmark.torrent')
         if ret != 0:
             print('ERROR: connection_tester failed: %d' % ret)
             sys.exit(1)
 
     rm_file_or_dir('t')
 
-    run_test('download', 'upload', '', args.download_peers)
-    run_test('upload', 'download', '-G', args.download_peers)
+    run_test('download', 'upload', '-1', args.download_peers)
+    run_test('upload', 'download', '-G -e 240', args.download_peers)
 
 
 def run_test(name, test_cmd, client_arg, num_peers):
@@ -57,8 +62,8 @@ def run_test(name, test_cmd, client_arg, num_peers):
     start = time.time()
     client_cmd = f'../examples/client_test -k --listen_interfaces=127.0.0.1:{port} cpu_benchmark.torrent ' + \
         f'--disable_hash_checks=1 --enable_dht=0 --enable_lsd=0 --enable_upnp=0 --enable_natpmp=0 ' + \
-        f'-e 120 {client_arg} -O --allow_multiple_connections_per_ip=1 --connections_limit={num_peers*2} -T {num_peers*2} ' + \
-        f'-f {output_dir}/events.log --alert_mask=8747'
+        f'{client_arg} -O --allow_multiple_connections_per_ip=1 --connections_limit={num_peers*2} -T {num_peers*2} ' + \
+        f'-f {output_dir}/events.log --alert_mask=error,status,connect,performance_warning,storage,peer'
 
     test_cmd = f'../examples/connection_tester {test_cmd} -c {num_peers} -d 127.0.0.1 -p {port} -t cpu_benchmark.torrent'
 
@@ -70,15 +75,22 @@ def run_test(name, test_cmd, client_arg, num_peers):
     print(f'test_cmd: "{test_cmd}"')
     t = subprocess.Popen(test_cmd.split(' '), stdout=test_out, stderr=test_out)
 
-    t.wait()
-
-    end = time.time()
+    if platform.system() == "Linux":
+        out = {}
+        while c.returncode is None:
+            capture_sample(c.pid, start, out)
+            time.sleep(0.1)
+            c.poll()
+        end = time.time()
+
+        stats_filename = f"{output_dir}/memory_stats.log"
+        keys = print_output_to_file(out, stats_filename)
+        plot_output(stats_filename, keys)
+    else:
+        c.wait()
+        end = time.time()
 
-    try:
-        c.communicate('q')
-    except Exception:
-        pass
-    c.wait()
+    t.wait()
 
     client_out.close()
     test_out.close()