From 04c958f31b21d9a8323164d0245f2d327b4d1e28 Mon Sep 17 00:00:00 2001 From: florian-vuillemot Date: Sat, 5 Dec 2020 18:22:27 +0100 Subject: [PATCH 1/8] Update shebangs --- tools/setup.py | 2 +- tools/tgentools/tgentools | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/setup.py b/tools/setup.py index 38425f1..cacbc41 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from setuptools import setup diff --git a/tools/tgentools/tgentools b/tools/tgentools/tgentools index b012a81..0e4c0de 100755 --- a/tools/tgentools/tgentools +++ b/tools/tgentools/tgentools @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 ''' tgentools From 006c5c5651635d07d4f1c957919b95b9e6d42501 Mon Sep 17 00:00:00 2001 From: florian-vuillemot Date: Sat, 5 Dec 2020 18:22:59 +0100 Subject: [PATCH 2/8] Update doc to be Python3 compatible --- tools/README | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tools/README b/tools/README index baf9231..1ed5a1f 100644 --- a/tools/README +++ b/tools/README @@ -7,23 +7,30 @@ not required to run `tgen`, but will be helpful to understand its output. Dependencies in Fedora/RedHat: - sudo yum install python python-devel python-pip python-virtualenv libxml2 libxml2-devel libxslt libxslt-devel libpng libpng-devel freetype freetype-devel blas blas-devel lapack lapack-devel + sudo yum install python3 python3-pip libxml2 libxml2-devel libxslt libxslt-devel libpng libpng-devel freetype freetype-devel blas blas-devel lapack lapack-devel Dependencies in Ubuntu/Debian: - sudo apt-get install python python-dev python-pip python-virtualenv libxml2 libxml2-dev libxslt1.1 libxslt1-dev libpng12-0 libpng12-dev libfreetype6 libfreetype6-dev libblas-dev liblapack-dev + sudo apt-get install python3 python3-pip libxml2 libxml2-dev libxslt1.1 libxslt1-dev libpng-dev libfreetype6 libfreetype6-dev libblas-dev liblapack-dev ## Install TGenTools Python modules -We show how to install python modules using `pip` (although you can also +We show how to install python modules using `pip3` (although you can also use your OS package manager). We recommend using virtual environments to keep all of the dependencies self-contained and to avoid conflicts with your other python projects. virtualenv toolsenv source toolsenv/bin/activate - pip install -r path/to/tgen/tools/requirements.txt - pip install -I path/to/tgen/tools + pip3 install -r path/to/tgen/tools/requirements.txt + pip3 install -I path/to/tgen/tools + +## Install TGenTools + +Build then install locally + python3 setup.py build + python3 setup.py install + ## Run TGenTools From 0a00629a21fe4647d4d868ee7f65511edc5669e8 Mon Sep 17 00:00:00 2001 From: florian-vuillemot Date: Sat, 5 Dec 2020 19:05:15 +0100 Subject: [PATCH 3/8] Convert script test from python2 to python3 --- test/expected-results/plot-results.sh | 2 +- tools/scripts/generate_mmodel_graphml.py | 4 ++-- tools/scripts/generate_tgen_config.py | 2 +- tools/scripts/parse-tgen.py | 20 ++++++++++---------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/test/expected-results/plot-results.sh b/test/expected-results/plot-results.sh index e291456..6240e21 100644 --- a/test/expected-results/plot-results.sh +++ b/test/expected-results/plot-results.sh @@ -2,7 +2,7 @@ # note that the plot script requires matplotlib, numpy, and scipy for seed in 123 321 do - python ../../tools/scripts/plot-dist.py ${seed} + python3 ../../tools/scripts/plot-dist.py ${seed} done echo "See the PDF files in the build-test directory" diff --git a/tools/scripts/generate_mmodel_graphml.py b/tools/scripts/generate_mmodel_graphml.py index 6b39d47..fe99ad3 100644 --- a/tools/scripts/generate_mmodel_graphml.py +++ b/tools/scripts/generate_mmodel_graphml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import networkx @@ -143,7 +143,7 @@ def print_as_c_string(filename): with open(filename, 'r') as inf: for line in inf: escaped = line.replace('"', '\\"') - print '"{}"'.format(escaped.rstrip()) + print('"{}"'.format(escaped.rstrip())) if __name__ == "__main__": main() diff --git a/tools/scripts/generate_tgen_config.py b/tools/scripts/generate_tgen_config.py index 423ad12..7de8282 100644 --- a/tools/scripts/generate_tgen_config.py +++ b/tools/scripts/generate_tgen_config.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import sys import networkx as nx diff --git a/tools/scripts/parse-tgen.py b/tools/scripts/parse-tgen.py index fa00ebb..24658c7 100644 --- a/tools/scripts/parse-tgen.py +++ b/tools/scripts/parse-tgen.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import sys, os, argparse, re, json from multiprocessing import Pool, cpu_count @@ -15,19 +15,19 @@ sizes reach tens of gigabytes. Use the help menu to understand usage: -$ python parse-tgen.py -h +$ python3 parse-tgen.py -h The standard way to run the script is to give the path to a directory tree under which one or several tgen log files exist: -$ python parse-tgen.py shadow.data/hosts/ -$ python parse-tgen.py ./ +$ python3 parse-tgen.py shadow.data/hosts/ +$ python3 parse-tgen.py ./ This path will be searched for log files whose names match those created by shadow; additional patterns can be added with the '-e' option. A single tgen log file can also be passed on STDIN with the special '-' path: -$ cat tgen.log | python parse-tgen.py - -$ xzcat tgen.log.xz | python parse-tgen.py - +$ cat tgen.log | python3 parse-tgen.py - +$ xzcat tgen.log.xz | python3 parse-tgen.py - The default mode is to filter and parse the log files using a single process; this will be done with multiple worker processes when passing @@ -87,7 +87,7 @@ def run(args): while not mr.ready(): mr.wait(1) r = mr.get() except KeyboardInterrupt: - print >> sys.stderr, "interrupted, terminating process pool" + print("interrupted, terminating process pool", file=sys.stderr) p.terminate() p.join() sys.exit() @@ -106,10 +106,10 @@ def run(args): success_count += item[2] error_count += item[3] - print >> sys.stderr, "done processing input: {0} total successes, {1} total errors, {2} files with names, {3} files without names".format(success_count, error_count, name_count, noname_count) - print >> sys.stderr, "dumping stats in {0}".format(args.prefix) + print("done processing input: {0} total successes, {1} total errors, {2} files with names, {3} files without names".format(success_count, error_count, name_count, noname_count), file=sys.stderr) + print("dumping stats in {0}".format(args.prefix), file=sys.stderr) dump(d, args.prefix, TGENJSON) - print >> sys.stderr, "all done!" + print("all done!", file=sys.stderr) def process_tgen_log(filename): signal(SIGINT, SIG_IGN) # ignore interrupts From a2ef7d63511d4069815f1d68ae5df91de10cdb4d Mon Sep 17 00:00:00 2001 From: florian-vuillemot Date: Sat, 5 Dec 2020 19:11:03 +0100 Subject: [PATCH 4/8] Replace README python by python3 --- README.md | 2 +- doc/TGen-Markov-Models.md | 2 +- doc/TGen-Overview.md | 2 +- doc/Tools-JSON-Format.md | 4 ++-- tools/README | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d04799a..9dd1b1d 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ TGen instances. The characteristics of the traffic (e.g., size, timing, number of parallel flows, etc.) can be configured by the user. TGen can generate complex traffic patterns. Users write relatively simple -python scripts to generate `graphml` files that are then used as TGen +python3 scripts to generate `graphml` files that are then used as TGen configuration files that instruct TGen how to generate traffic. TGen also supports the use of Markov models in order to generate TCP flows and packet streams according to common probability distributions. diff --git a/doc/TGen-Markov-Models.md b/doc/TGen-Markov-Models.md index 48eeab9..a3db278 100644 --- a/doc/TGen-Markov-Models.md +++ b/doc/TGen-Markov-Models.md @@ -35,7 +35,7 @@ pass TGen's Markov model validation. As with the config file, TGen uses the `graphml` file format to represent Markov models. As we explain the structure supported by TGen, we provide examples of generating the corresponding `graphml` elements and atrributes -using `python` and the `networkx` python module (installing the TGenTools +using `python3` and the `networkx` python3 module (installing the TGenTools toolkit will install the networkx module). Models are constructed as directed graphs: diff --git a/doc/TGen-Overview.md b/doc/TGen-Overview.md index 1009e65..8c171cf 100644 --- a/doc/TGen-Overview.md +++ b/doc/TGen-Overview.md @@ -40,7 +40,7 @@ Be warned that edge weights must be used carefully, especially when combined wit # Examples -An easy way to generate TGen behavior graphs is to use python and the networkx python module. +An easy way to generate TGen behavior graphs is to use python3 and the networkx python3 module. The scripts are simple, but capable of generating complex behavior profiles. Example scripts for generating TGen configuration files can be found in the diff --git a/doc/Tools-JSON-Format.md b/doc/Tools-JSON-Format.md index f6a4fd0..b83584d 100644 --- a/doc/Tools-JSON-Format.md +++ b/doc/Tools-JSON-Format.md @@ -1,10 +1,10 @@ # JSON DB Structure for Analysis Results -The `tgentools` python toolkit can be run in `parse` mode to parse a set of +The `tgentools` python3 toolkit can be run in `parse` mode to parse a set of tgen log files. It writes useful statistics to a json file. This document describes the structure of the json database file that gets exported -when running the `tgentools` python toolkit in `parse` mode. +when running the `tgentools` python3 toolkit in `parse` mode. The structure is given here with variable keys marked as such. diff --git a/tools/README b/tools/README index 1ed5a1f..df448dc 100644 --- a/tools/README +++ b/tools/README @@ -13,9 +13,9 @@ Dependencies in Ubuntu/Debian: sudo apt-get install python3 python3-pip libxml2 libxml2-dev libxslt1.1 libxslt1-dev libpng-dev libfreetype6 libfreetype6-dev libblas-dev liblapack-dev -## Install TGenTools Python modules +## Install TGenTools Python3 modules -We show how to install python modules using `pip3` (although you can also +We show how to install python3 modules using `pip3` (although you can also use your OS package manager). We recommend using virtual environments to keep all of the dependencies self-contained and to avoid conflicts with your other python projects. From d0e0fb79ed6192767e1a7f1ae6381a904ee58598 Mon Sep 17 00:00:00 2001 From: florian-vuillemot Date: Sat, 5 Dec 2020 20:37:09 +0100 Subject: [PATCH 5/8] Fix tools README dependencies --- tools/README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/README b/tools/README index df448dc..f515468 100644 --- a/tools/README +++ b/tools/README @@ -11,7 +11,7 @@ Dependencies in Fedora/RedHat: Dependencies in Ubuntu/Debian: - sudo apt-get install python3 python3-pip libxml2 libxml2-dev libxslt1.1 libxslt1-dev libpng-dev libfreetype6 libfreetype6-dev libblas-dev liblapack-dev + sudo apt-get install python3 python3-dev python3-pip python3-virtualenv libxml2 libxml2-dev libxslt1.1 libxslt1-dev libpng16-16 libpng-dev libfreetype6 libfreetype6-dev libblas-dev liblapack-dev ## Install TGenTools Python3 modules From 7b822ee4982ffb885598fd4a992d0997604f1272 Mon Sep 17 00:00:00 2001 From: florian-vuillemot Date: Sat, 5 Dec 2020 20:56:02 +0100 Subject: [PATCH 6/8] Update missed dependencies and change virtualenv by venv --- tools/README | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/README b/tools/README index f515468..275f4de 100644 --- a/tools/README +++ b/tools/README @@ -11,7 +11,7 @@ Dependencies in Fedora/RedHat: Dependencies in Ubuntu/Debian: - sudo apt-get install python3 python3-dev python3-pip python3-virtualenv libxml2 libxml2-dev libxslt1.1 libxslt1-dev libpng16-16 libpng-dev libfreetype6 libfreetype6-dev libblas-dev liblapack-dev + sudo apt-get install python3 python3-dev python3-pip python3-venv libxml2 libxml2-dev libxslt1.1 libxslt1-dev libpng16-16 libpng-dev libfreetype6 libfreetype6-dev libblas-dev liblapack-dev ## Install TGenTools Python3 modules @@ -20,7 +20,7 @@ use your OS package manager). We recommend using virtual environments to keep all of the dependencies self-contained and to avoid conflicts with your other python projects. - virtualenv toolsenv + python3 -m venv toolsenv source toolsenv/bin/activate pip3 install -r path/to/tgen/tools/requirements.txt pip3 install -I path/to/tgen/tools From 55693eec0ebe644e91ea2a734280c84274b0bda1 Mon Sep 17 00:00:00 2001 From: florian-vuillemot Date: Sat, 5 Dec 2020 20:56:57 +0100 Subject: [PATCH 7/8] Update github workflow to python3 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd183b8..cffcdb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,10 +39,10 @@ jobs: run: bash test/run_tgen_integration_tests.sh - name: Install tgentools dependencies - run: sudo apt install -y python python-dev python-pip python-virtualenv libxml2 libxml2-dev libxslt1.1 libxslt1-dev libpng16-16 libpng-dev libfreetype6 libfreetype6-dev libblas-dev liblapack-dev + run: sudo apt install -y python3 python3-dev python3-pip python3-venv libxml2 libxml2-dev libxslt1.1 libxslt1-dev libpng16-16 libpng-dev libfreetype6 libfreetype6-dev libblas-dev liblapack-dev - name: Build tgentools - run: virtualenv build/toolsenv && source build/toolsenv/bin/activate && pip install -r tools/requirements.txt && pip install -I tools/ + run: python3 -m venv build/toolsenv && source build/toolsenv/bin/activate && pip3 install -r tools/requirements.txt && pip3 install -I tools/ - name: Test tgentools run: bash test/run_tgentools_integration_tests.sh From 4f095269775aeeecab11f8b32e48918c0dfb9b8f Mon Sep 17 00:00:00 2001 From: florian-vuillemot Date: Sun, 31 Jan 2021 10:39:24 +0100 Subject: [PATCH 8/8] Update install doc --- tools/README | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/README b/tools/README index 275f4de..50d3cef 100644 --- a/tools/README +++ b/tools/README @@ -23,15 +23,19 @@ your other python projects. python3 -m venv toolsenv source toolsenv/bin/activate pip3 install -r path/to/tgen/tools/requirements.txt - pip3 install -I path/to/tgen/tools ## Install TGenTools -Build then install locally +Install into the virtual environment: + + source toolsenv/bin/activate + pip3 install -I path/to/tgen/tools + +Or build/install locally (outside of the virtual environment): + python3 setup.py build python3 setup.py install - ## Run TGenTools TGenTools has several modes of operation and a help menu for each. For a