From 317d19f54c6b0ab0ce10a645ab3b06e1a02963c3 Mon Sep 17 00:00:00 2001 From: Ana Custura Date: Mon, 29 Jun 2020 17:14:40 +0100 Subject: [PATCH 1/8] Update to setuptools --- tools/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/setup.py b/tools/setup.py index 3ec3584..38425f1 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from distutils.core import setup +from setuptools import setup setup(name='TGenTools', version="1.0.0", From e77bd7f5ce2e9970a81dc3ae7dd0f32fefe5e669 Mon Sep 17 00:00:00 2001 From: Ana Custura Date: Mon, 29 Jun 2020 17:18:26 +0100 Subject: [PATCH 2/8] Copy LICENSE and tools-setup.md as project README --- tools/LICENSE | 31 ++++++++++++++++++++++++ tools/README | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 tools/LICENSE create mode 100644 tools/README diff --git a/tools/LICENSE b/tools/LICENSE new file mode 100644 index 0000000..405dcbb --- /dev/null +++ b/tools/LICENSE @@ -0,0 +1,31 @@ +DISTRIBUTION STATEMENT + +Approved for public release: distribution unlimited. + +Redistributions of source and binary forms, with or without +modification, are permitted if redistributions retain the above +distribution statement and the following disclaimer. + +DISCLAIMER + +THE SOFTWARE IS SUPPLIED “AS IS” WITHOUT WARRANTY OF ANY KIND. + +AS THE OWNER OF THE SOFTWARE, THE UNITED STATES, THE UNITED STATES +DEPARTMENT OF DEFENSE, AND THEIR EMPLOYEES: (1) DISCLAIM ANY +WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY +IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE, TITLE OR NON-INFRINGEMENT, (2) DO NOT ASSUME ANY LEGAL +LIABILITY OR RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR +USEFULNESS OF THE SOFTWARE, (3) DO NOT REPRESENT THAT USE OF THE +SOFTWARE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS, (4) DO NOT WARRANT +THAT THE SOFTWARE WILL FUNCTION UNINTERRUPTED, THAT IT IS ERROR-FREE +OR THAT ANY ERRORS WILL BE CORRECTED. + +PORTIONS OF THE SOFTWARE RESULTED FROM WORK DEVELOPED BY OR FOR THE +U.S. GOVERNMENT SUBJECT TO THE FOLLOWING LICENSE: THE GOVERNMENT IS +GRANTED FOR ITSELF AND OTHERS ACTING ON ITS BEHALF A PAID-UP, +NONEXCLUSIVE, IRREVOCABLE WORLDWIDE LICENSE IN THIS COMPUTER SOFTWARE +TO REPRODUCE, PREPARE DERIVATIVE WORKS, TO PERFORM OR DISPLAY ANY +PORTION OF THAT WORK, AND TO PERMIT OTHERS TO DO SO FOR GOVERNMENT +PURPOSES. + diff --git a/tools/README b/tools/README new file mode 100644 index 0000000..baf9231 --- /dev/null +++ b/tools/README @@ -0,0 +1,67 @@ +# TGenTools + +TGenTools is a toolkit to parse and plot `tgen` log files. TGenTools is +not required to run `tgen`, but will be helpful to understand its output. + +## Install system dependencies + +Dependencies in Fedora/RedHat: + + sudo yum install python python-devel python-pip python-virtualenv libxml2 libxml2-devel libxslt libxslt-devel libpng libpng-devel freetype freetype-devel blas blas-devel lapack lapack-devel + +Dependencies in Ubuntu/Debian: + + sudo apt-get install python python-dev python-pip python-virtualenv libxml2 libxml2-dev libxslt1.1 libxslt1-dev libpng12-0 libpng12-dev libfreetype6 libfreetype6-dev libblas-dev liblapack-dev + +## Install TGenTools Python modules + +We show how to install python modules using `pip` (although you can also +use your OS package manager). We recommend using virtual environments to +keep all of the dependencies self-contained and to avoid conflicts with +your other python projects. + + virtualenv toolsenv + source toolsenv/bin/activate + pip install -r path/to/tgen/tools/requirements.txt + pip install -I path/to/tgen/tools + +## Run TGenTools + +TGenTools has several modes of operation and a help menu for each. For a +description of each mode, use the following (make sure you have activated +the tgen virtual environment with `source toolsenv/bin/activate` first): + +``` +tgentools -h +``` + + + **parse**: Analyze TGen output + + **plot**: Visualize TGen analysis results + + **edit**: Edit TGen configuration files in place + +## Example parsing and plotting TGen output + +Assuming you have already run `tgen` and saved the output to a log file +called `tgen.client.log`, you can then parse the log file like this: + + tgentools parse tgen.client.log + +This produces the `tgen.analysis.json.xz` file, the format of which is +outlined in [doc/Tools-JSON-Format.md](Tools-JSON-Format.md). +The analysis file can be plotted: + + tgentools plot --data tgen.analysis.json.xz "tgen-test" + +This will save new PDFs containing several graphs in the current directory. +Depending on the data that was analyzed, the graphs may include: + +- Time to download first byte of transfer, across all transfers +- Time to download last byte of transfer, across all transfers +- Median time to download last byte of transfer, per client +- Mean time to download last byte of transfer, per client +- Max time to download last byte of transfer, per client +- Number of transfer successes, per client +- Number of transfer errors, per client +- Bytes transferred before error, across all transfers with error +- Median bytes transferred before error, per client +- Mean bytes transferred before error, per client From bc1e071d9fbe6b0a11906cebe2e07424a0e82359 Mon Sep 17 00:00:00 2001 From: Ana Custura Date: Mon, 29 Jun 2020 17:37:11 +0100 Subject: [PATCH 3/8] Adds support gz compressed tgen logs --- tools/tgentools/util.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/tgentools/util.py b/tools/tgentools/util.py index 1f005a4..f599a26 100644 --- a/tools/tgentools/util.py +++ b/tools/tgentools/util.py @@ -4,7 +4,7 @@ See LICENSE for licensing information ''' -import sys, os, socket, logging, random, re, shutil, datetime, urllib +import sys, os, socket, logging, random, re, shutil, datetime, urllib, gzip from subprocess import Popen, PIPE, STDOUT from threading import Lock from abc import ABCMeta, abstractmethod @@ -167,6 +167,9 @@ def open(self): cmd = "xz --decompress --stdout {0}".format(self.filename) xzproc = Popen(cmd.split(), stdout=PIPE) self.source = xzproc.stdout + elif self.filename.endswith(".gz"): + self.compress = True + self.source = gzip.open(self.filename, 'rt') else: self.source = open(self.filename, 'r') From 9bdea4670c4baf54defa7cfd87439fac1be7d778 Mon Sep 17 00:00:00 2001 From: Ana Custura Date: Mon, 29 Jun 2020 19:33:02 +0100 Subject: [PATCH 4/8] Fixes bug where the start of transfer is incorrectly recorded --- tools/tgentools/analysis.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/tgentools/analysis.py b/tools/tgentools/analysis.py index 0f99fac..1a82d12 100644 --- a/tools/tgentools/analysis.py +++ b/tools/tgentools/analysis.py @@ -234,9 +234,9 @@ def __init__(self, line): time_usec_max = 0.0 if self.time_info != None: for key in self.time_info: - val = int(self.time_info[key]) - time_usec_max = max(time_usec_max, val) - + if 'usecs' in key: + val = int(self.time_info[key]) + time_usec_max = max(time_usec_max, val) self.unix_ts_start = self.unix_ts_end - (time_usec_max / 1000000.0) # usecs to secs class StreamSuccessEvent(StreamCompleteEvent): From d5fec131505d797566ab0f3fb5e4f27f192a1a89 Mon Sep 17 00:00:00 2001 From: Ana Custura Date: Mon, 29 Jun 2020 19:35:00 +0100 Subject: [PATCH 5/8] Renames variables, fixes conversion between percentage and decile --- tools/tgentools/analysis.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tools/tgentools/analysis.py b/tools/tgentools/analysis.py index 1a82d12..e27276e 100644 --- a/tools/tgentools/analysis.py +++ b/tools/tgentools/analysis.py @@ -264,14 +264,17 @@ def __init__(self, tid): self.payload_send_progress = {decile:None for decile in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]} def __set_progress_helper(self, status_event, bytes_key, progress_dict): - progress = status_event.byte_info[bytes_key].strip('%') + divide = 1 + if '%' in status_event.byte_info[bytes_key]: + progress = status_event.byte_info[bytes_key].strip('%') + divide = 100 + else: + progress = status_event.byte_info[bytes_key] if progress != '?': - frac = float(progress) - # set only the highest decile that we meet or exceed - for decile in sorted(progress_dict.keys(), reverse=True): - if frac >= decile: - if progress_dict[decile] is None: - progress_dict[decile] = status_event.unix_ts_end + progress_instance = float(progress)/divide + for item in sorted(progress_dict.keys(), reverse=True): + if progress_instance >= item and progress_dict[item] is None: + progress_dict[item] = status_event.unix_ts_end return def add_event(self, status_event): From bbf2b32015699e3b996ad5be9364b7e0c5bfc53e Mon Sep 17 00:00:00 2001 From: Ana Custura Date: Mon, 29 Jun 2020 19:36:51 +0100 Subject: [PATCH 6/8] Adds support for time to complete certain numbers of bytes --- tools/tgentools/analysis.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tools/tgentools/analysis.py b/tools/tgentools/analysis.py index e27276e..b32e63b 100644 --- a/tools/tgentools/analysis.py +++ b/tools/tgentools/analysis.py @@ -262,6 +262,8 @@ def __init__(self, tid): self.last_event = None self.payload_recv_progress = {decile:None for decile in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]} self.payload_send_progress = {decile:None for decile in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]} + self.payload_recv_bytes = {partial:None for partial in [10240, 20480, 51200, 102400, 204800, 512000, 1048576, 2097152, 5242880]} + self.payload_send_bytes = {partial:None for partial in [10240, 20480, 51200, 102400, 204800, 512000, 1048576, 2097152, 5242880]} def __set_progress_helper(self, status_event, bytes_key, progress_dict): divide = 1 @@ -280,6 +282,8 @@ def __set_progress_helper(self, status_event, bytes_key, progress_dict): def add_event(self, status_event): self.__set_progress_helper(status_event, 'payload-progress-recv', self.payload_recv_progress) self.__set_progress_helper(status_event, 'payload-progress-send', self.payload_send_progress) + self.__set_progress_helper(status_event, 'payload-bytes-recv', self.payload_recv_bytes) + self.__set_progress_helper(status_event, 'payload-bytes-send', self.payload_send_bytes) self.last_event = status_event def get_data(self): @@ -287,9 +291,17 @@ def get_data(self): if e is None or not e.is_complete: return None d = e.__dict__ - d['elapsed_seconds'] = {} - d['elapsed_seconds']['payload_progress_recv'] = {decile: self.payload_recv_progress[decile] - e.unix_ts_start for decile in self.payload_recv_progress if self.payload_recv_progress[decile] is not None} - d['elapsed_seconds']['payload_progress_send'] = {decile: self.payload_send_progress[decile] - e.unix_ts_start for decile in self.payload_send_progress if self.payload_send_progress[decile] is not None} + if not e.is_error: + d['elapsed_seconds'] = { + 'payload_progress_recv': { + decile: round((self.payload_recv_progress[decile] - e.unix_ts_start), 6) for decile in self.payload_recv_progress if self.payload_recv_progress[decile] is not None}, + 'payload_progress_send': { + decile: round((self.payload_send_progress[decile] - e.unix_ts_start), 6) for decile in self.payload_send_progress if self.payload_send_progress[decile] is not None}, + 'payload_bytes_recv' : { + partial: round((self.payload_recv_bytes[partial] - e.unix_ts_start), 6) for partial in self.payload_recv_bytes if self.payload_recv_bytes[partial] is not None}, + 'payload_bytes_send' : { + partial: round((self.payload_send_bytes[partial] - e.unix_ts_start), 6) for partial in self.payload_send_bytes if self.payload_send_bytes[partial] is not None} + } return d class Parser(object): From 1694596d5150f3b2ac07a17c70b3c7e00d585eba Mon Sep 17 00:00:00 2001 From: Ana Custura Date: Wed, 8 Jul 2020 14:59:30 +0100 Subject: [PATCH 7/8] Record time in ascending order --- tools/tgentools/analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tgentools/analysis.py b/tools/tgentools/analysis.py index b32e63b..6b0cfad 100644 --- a/tools/tgentools/analysis.py +++ b/tools/tgentools/analysis.py @@ -274,7 +274,7 @@ def __set_progress_helper(self, status_event, bytes_key, progress_dict): progress = status_event.byte_info[bytes_key] if progress != '?': progress_instance = float(progress)/divide - for item in sorted(progress_dict.keys(), reverse=True): + for item in sorted(progress_dict.keys()): if progress_instance >= item and progress_dict[item] is None: progress_dict[item] = status_event.unix_ts_end return From 725a9810ae49faad0393a14ffa1ab653c44dcf78 Mon Sep 17 00:00:00 2001 From: Ana Custura Date: Wed, 8 Jul 2020 17:16:11 +0100 Subject: [PATCH 8/8] Move TGenTools documentation to the tools/ directory --- README.md | 2 +- doc/Tools-Setup.md | 67 ---------------------------------------------- 2 files changed, 1 insertion(+), 68 deletions(-) delete mode 100644 doc/Tools-Setup.md diff --git a/README.md b/README.md index 3f7fcf2..d04799a 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ See the [resource/](resource) directory for example config files. ## More documentation -See [doc/Tools-Setup.md](doc/Tools-Setup.md) for setup instructions for +See [tools/README](tools/README) for setup instructions for the TGenTools toolkit that can be used to parse and plot `tgen` log output. See [doc/TGen-Overview.md](doc/TGen-Overview.md) for an overview of how to use diff --git a/doc/Tools-Setup.md b/doc/Tools-Setup.md deleted file mode 100644 index baf9231..0000000 --- a/doc/Tools-Setup.md +++ /dev/null @@ -1,67 +0,0 @@ -# TGenTools - -TGenTools is a toolkit to parse and plot `tgen` log files. TGenTools is -not required to run `tgen`, but will be helpful to understand its output. - -## Install system dependencies - -Dependencies in Fedora/RedHat: - - sudo yum install python python-devel python-pip python-virtualenv libxml2 libxml2-devel libxslt libxslt-devel libpng libpng-devel freetype freetype-devel blas blas-devel lapack lapack-devel - -Dependencies in Ubuntu/Debian: - - sudo apt-get install python python-dev python-pip python-virtualenv libxml2 libxml2-dev libxslt1.1 libxslt1-dev libpng12-0 libpng12-dev libfreetype6 libfreetype6-dev libblas-dev liblapack-dev - -## Install TGenTools Python modules - -We show how to install python modules using `pip` (although you can also -use your OS package manager). We recommend using virtual environments to -keep all of the dependencies self-contained and to avoid conflicts with -your other python projects. - - virtualenv toolsenv - source toolsenv/bin/activate - pip install -r path/to/tgen/tools/requirements.txt - pip install -I path/to/tgen/tools - -## Run TGenTools - -TGenTools has several modes of operation and a help menu for each. For a -description of each mode, use the following (make sure you have activated -the tgen virtual environment with `source toolsenv/bin/activate` first): - -``` -tgentools -h -``` - - + **parse**: Analyze TGen output - + **plot**: Visualize TGen analysis results - + **edit**: Edit TGen configuration files in place - -## Example parsing and plotting TGen output - -Assuming you have already run `tgen` and saved the output to a log file -called `tgen.client.log`, you can then parse the log file like this: - - tgentools parse tgen.client.log - -This produces the `tgen.analysis.json.xz` file, the format of which is -outlined in [doc/Tools-JSON-Format.md](Tools-JSON-Format.md). -The analysis file can be plotted: - - tgentools plot --data tgen.analysis.json.xz "tgen-test" - -This will save new PDFs containing several graphs in the current directory. -Depending on the data that was analyzed, the graphs may include: - -- Time to download first byte of transfer, across all transfers -- Time to download last byte of transfer, across all transfers -- Median time to download last byte of transfer, per client -- Mean time to download last byte of transfer, per client -- Max time to download last byte of transfer, per client -- Number of transfer successes, per client -- Number of transfer errors, per client -- Bytes transferred before error, across all transfers with error -- Median bytes transferred before error, per client -- Mean bytes transferred before error, per client