Onionperf updates

README.md

@@ -50,7 +50,7 @@ See the [resource/](resource) directory for example config files.
 
 ## More documentation
 
-See [doc/Tools-Setup.md](doc/Tools-Setup.md) for setup instructions for
+See [tools/README](tools/README) for setup instructions for
 the TGenTools toolkit that can be used to parse and plot `tgen` log output.
 
 See [doc/TGen-Overview.md](doc/TGen-Overview.md) for an overview of how to use

tools/LICENSE

@@ -0,0 +1,31 @@
+DISTRIBUTION STATEMENT
+
+Approved for public release: distribution unlimited.
+
+Redistributions of source and binary forms, with or without
+modification, are permitted if redistributions retain the above
+distribution statement and the following disclaimer.
+
+DISCLAIMER
+
+THE SOFTWARE IS SUPPLIED “AS IS” WITHOUT WARRANTY OF ANY KIND.
+
+AS THE OWNER OF THE SOFTWARE, THE UNITED STATES, THE UNITED STATES
+DEPARTMENT OF DEFENSE, AND THEIR EMPLOYEES: (1) DISCLAIM ANY
+WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY
+IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE, TITLE OR NON-INFRINGEMENT, (2) DO NOT ASSUME ANY LEGAL
+LIABILITY OR RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR
+USEFULNESS OF THE SOFTWARE, (3) DO NOT REPRESENT THAT USE OF THE
+SOFTWARE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS, (4) DO NOT WARRANT
+THAT THE SOFTWARE WILL FUNCTION UNINTERRUPTED, THAT IT IS ERROR-FREE
+OR THAT ANY ERRORS WILL BE CORRECTED.
+
+PORTIONS OF THE SOFTWARE RESULTED FROM WORK DEVELOPED BY OR FOR THE
+U.S. GOVERNMENT SUBJECT TO THE FOLLOWING LICENSE: THE GOVERNMENT IS
+GRANTED FOR ITSELF AND OTHERS ACTING ON ITS BEHALF A PAID-UP,
+NONEXCLUSIVE, IRREVOCABLE WORLDWIDE LICENSE IN THIS COMPUTER SOFTWARE
+TO REPRODUCE, PREPARE DERIVATIVE WORKS, TO PERFORM OR DISPLAY ANY
+PORTION OF THAT WORK, AND TO PERMIT OTHERS TO DO SO FOR GOVERNMENT
+PURPOSES.
+

tools/setup.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-from distutils.core import setup
+from setuptools import setup
 
 setup(name='TGenTools',
       version="1.0.0",

tools/tgentools/analysis.py

@@ -234,9 +234,9 @@ def __init__(self, line):
         time_usec_max = 0.0
         if self.time_info != None:
             for key in self.time_info:
-                val = int(self.time_info[key])
-                time_usec_max = max(time_usec_max, val)
-
+                if 'usecs' in key:
+                    val = int(self.time_info[key])
+                    time_usec_max = max(time_usec_max, val)
         self.unix_ts_start = self.unix_ts_end - (time_usec_max / 1000000.0)  # usecs to secs
 
 class StreamSuccessEvent(StreamCompleteEvent):
@@ -262,31 +262,46 @@ def __init__(self, tid):
         self.last_event = None
         self.payload_recv_progress = {decile:None for decile in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]}
         self.payload_send_progress = {decile:None for decile in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]}
+        self.payload_recv_bytes = {partial:None for partial in [10240, 20480, 51200, 102400, 204800, 512000, 1048576, 2097152, 5242880]}
+        self.payload_send_bytes = {partial:None for partial in [10240, 20480, 51200, 102400, 204800, 512000, 1048576, 2097152, 5242880]}
 
     def __set_progress_helper(self, status_event, bytes_key, progress_dict):
-        progress = status_event.byte_info[bytes_key].strip('%')
+        divide = 1
+        if '%' in status_event.byte_info[bytes_key]:
+            progress = status_event.byte_info[bytes_key].strip('%')
+            divide = 100
+        else:
+            progress = status_event.byte_info[bytes_key]
         if progress != '?':
-            frac = float(progress)
-            # set only the highest decile that we meet or exceed
-            for decile in sorted(progress_dict.keys(), reverse=True):
-                if frac >= decile:
-                    if progress_dict[decile] is None:
-                        progress_dict[decile] = status_event.unix_ts_end
+            progress_instance = float(progress)/divide
+            for item in sorted(progress_dict.keys()):
+                if progress_instance >= item and progress_dict[item] is None:
+                    progress_dict[item] = status_event.unix_ts_end
                     return
 
     def add_event(self, status_event):
         self.__set_progress_helper(status_event, 'payload-progress-recv', self.payload_recv_progress)
         self.__set_progress_helper(status_event, 'payload-progress-send', self.payload_send_progress)
+        self.__set_progress_helper(status_event, 'payload-bytes-recv', self.payload_recv_bytes)
+        self.__set_progress_helper(status_event, 'payload-bytes-send', self.payload_send_bytes)
         self.last_event = status_event
 
     def get_data(self):
         e = self.last_event
         if e is None or not e.is_complete:
             return None
         d = e.__dict__
-        d['elapsed_seconds'] = {}
-        d['elapsed_seconds']['payload_progress_recv'] = {decile: self.payload_recv_progress[decile] - e.unix_ts_start for decile in self.payload_recv_progress if self.payload_recv_progress[decile] is not None}
-        d['elapsed_seconds']['payload_progress_send'] = {decile: self.payload_send_progress[decile] - e.unix_ts_start for decile in self.payload_send_progress if self.payload_send_progress[decile] is not None}
+        if not e.is_error:
+            d['elapsed_seconds'] = {
+                'payload_progress_recv': {
+                  decile: round((self.payload_recv_progress[decile] - e.unix_ts_start), 6) for decile in self.payload_recv_progress if self.payload_recv_progress[decile] is not None},
+                'payload_progress_send': {
+                  decile: round((self.payload_send_progress[decile] - e.unix_ts_start), 6) for decile in self.payload_send_progress if self.payload_send_progress[decile] is not None},
+                'payload_bytes_recv' : {
+                  partial: round((self.payload_recv_bytes[partial] - e.unix_ts_start), 6) for partial in self.payload_recv_bytes if self.payload_recv_bytes[partial] is not None},
+                'payload_bytes_send' : {
+                  partial: round((self.payload_send_bytes[partial] - e.unix_ts_start), 6) for partial in self.payload_send_bytes if self.payload_send_bytes[partial] is not None}
+                 }
         return d
 
 class Parser(object):

tools/tgentools/util.py

@@ -4,7 +4,7 @@
   See LICENSE for licensing information
 '''
 
-import sys, os, socket, logging, random, re, shutil, datetime, urllib
+import sys, os, socket, logging, random, re, shutil, datetime, urllib, gzip
 from subprocess import Popen, PIPE, STDOUT
 from threading import Lock
 from abc import ABCMeta, abstractmethod
@@ -167,6 +167,9 @@ def open(self):
                 cmd = "xz --decompress --stdout {0}".format(self.filename)
                 xzproc = Popen(cmd.split(), stdout=PIPE)
                 self.source = xzproc.stdout
+            elif self.filename.endswith(".gz"):
+                self.compress = True
+                self.source = gzip.open(self.filename, 'rt')
             else:
                 self.source = open(self.filename, 'r')