diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd183b8..cffcdb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,10 +39,10 @@ jobs: run: bash test/run_tgen_integration_tests.sh - name: Install tgentools dependencies - run: sudo apt install -y python python-dev python-pip python-virtualenv libxml2 libxml2-dev libxslt1.1 libxslt1-dev libpng16-16 libpng-dev libfreetype6 libfreetype6-dev libblas-dev liblapack-dev + run: sudo apt install -y python3 python3-dev python3-pip python3-venv libxml2 libxml2-dev libxslt1.1 libxslt1-dev libpng16-16 libpng-dev libfreetype6 libfreetype6-dev libblas-dev liblapack-dev - name: Build tgentools - run: virtualenv build/toolsenv && source build/toolsenv/bin/activate && pip install -r tools/requirements.txt && pip install -I tools/ + run: python3 -m venv build/toolsenv && source build/toolsenv/bin/activate && pip3 install -r tools/requirements.txt && pip3 install -I tools/ - name: Test tgentools run: bash test/run_tgentools_integration_tests.sh diff --git a/README.md b/README.md index d04799a..9dd1b1d 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ TGen instances. The characteristics of the traffic (e.g., size, timing, number of parallel flows, etc.) can be configured by the user. TGen can generate complex traffic patterns. Users write relatively simple -python scripts to generate `graphml` files that are then used as TGen +python3 scripts to generate `graphml` files that are then used as TGen configuration files that instruct TGen how to generate traffic. TGen also supports the use of Markov models in order to generate TCP flows and packet streams according to common probability distributions. diff --git a/doc/TGen-Markov-Models.md b/doc/TGen-Markov-Models.md index 48eeab9..a3db278 100644 --- a/doc/TGen-Markov-Models.md +++ b/doc/TGen-Markov-Models.md @@ -35,7 +35,7 @@ pass TGen's Markov model validation. As with the config file, TGen uses the `graphml` file format to represent Markov models. As we explain the structure supported by TGen, we provide examples of generating the corresponding `graphml` elements and atrributes -using `python` and the `networkx` python module (installing the TGenTools +using `python3` and the `networkx` python3 module (installing the TGenTools toolkit will install the networkx module). Models are constructed as directed graphs: diff --git a/doc/TGen-Overview.md b/doc/TGen-Overview.md index 1009e65..8c171cf 100644 --- a/doc/TGen-Overview.md +++ b/doc/TGen-Overview.md @@ -40,7 +40,7 @@ Be warned that edge weights must be used carefully, especially when combined wit # Examples -An easy way to generate TGen behavior graphs is to use python and the networkx python module. +An easy way to generate TGen behavior graphs is to use python3 and the networkx python3 module. The scripts are simple, but capable of generating complex behavior profiles. Example scripts for generating TGen configuration files can be found in the diff --git a/doc/Tools-JSON-Format.md b/doc/Tools-JSON-Format.md index f6a4fd0..b83584d 100644 --- a/doc/Tools-JSON-Format.md +++ b/doc/Tools-JSON-Format.md @@ -1,10 +1,10 @@ # JSON DB Structure for Analysis Results -The `tgentools` python toolkit can be run in `parse` mode to parse a set of +The `tgentools` python3 toolkit can be run in `parse` mode to parse a set of tgen log files. It writes useful statistics to a json file. This document describes the structure of the json database file that gets exported -when running the `tgentools` python toolkit in `parse` mode. +when running the `tgentools` python3 toolkit in `parse` mode. The structure is given here with variable keys marked as such. diff --git a/test/expected-results/plot-results.sh b/test/expected-results/plot-results.sh index e291456..6240e21 100644 --- a/test/expected-results/plot-results.sh +++ b/test/expected-results/plot-results.sh @@ -2,7 +2,7 @@ # note that the plot script requires matplotlib, numpy, and scipy for seed in 123 321 do - python ../../tools/scripts/plot-dist.py ${seed} + python3 ../../tools/scripts/plot-dist.py ${seed} done echo "See the PDF files in the build-test directory" diff --git a/tools/README b/tools/README index baf9231..50d3cef 100644 --- a/tools/README +++ b/tools/README @@ -7,23 +7,34 @@ not required to run `tgen`, but will be helpful to understand its output. Dependencies in Fedora/RedHat: - sudo yum install python python-devel python-pip python-virtualenv libxml2 libxml2-devel libxslt libxslt-devel libpng libpng-devel freetype freetype-devel blas blas-devel lapack lapack-devel + sudo yum install python3 python3-pip libxml2 libxml2-devel libxslt libxslt-devel libpng libpng-devel freetype freetype-devel blas blas-devel lapack lapack-devel Dependencies in Ubuntu/Debian: - sudo apt-get install python python-dev python-pip python-virtualenv libxml2 libxml2-dev libxslt1.1 libxslt1-dev libpng12-0 libpng12-dev libfreetype6 libfreetype6-dev libblas-dev liblapack-dev + sudo apt-get install python3 python3-dev python3-pip python3-venv libxml2 libxml2-dev libxslt1.1 libxslt1-dev libpng16-16 libpng-dev libfreetype6 libfreetype6-dev libblas-dev liblapack-dev -## Install TGenTools Python modules +## Install TGenTools Python3 modules -We show how to install python modules using `pip` (although you can also +We show how to install python3 modules using `pip3` (although you can also use your OS package manager). We recommend using virtual environments to keep all of the dependencies self-contained and to avoid conflicts with your other python projects. - virtualenv toolsenv + python3 -m venv toolsenv source toolsenv/bin/activate - pip install -r path/to/tgen/tools/requirements.txt - pip install -I path/to/tgen/tools + pip3 install -r path/to/tgen/tools/requirements.txt + +## Install TGenTools + +Install into the virtual environment: + + source toolsenv/bin/activate + pip3 install -I path/to/tgen/tools + +Or build/install locally (outside of the virtual environment): + + python3 setup.py build + python3 setup.py install ## Run TGenTools diff --git a/tools/scripts/generate_mmodel_graphml.py b/tools/scripts/generate_mmodel_graphml.py index 6b39d47..fe99ad3 100644 --- a/tools/scripts/generate_mmodel_graphml.py +++ b/tools/scripts/generate_mmodel_graphml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import networkx @@ -143,7 +143,7 @@ def print_as_c_string(filename): with open(filename, 'r') as inf: for line in inf: escaped = line.replace('"', '\\"') - print '"{}"'.format(escaped.rstrip()) + print('"{}"'.format(escaped.rstrip())) if __name__ == "__main__": main() diff --git a/tools/scripts/generate_tgen_config.py b/tools/scripts/generate_tgen_config.py index 423ad12..7de8282 100644 --- a/tools/scripts/generate_tgen_config.py +++ b/tools/scripts/generate_tgen_config.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import sys import networkx as nx diff --git a/tools/scripts/parse-tgen.py b/tools/scripts/parse-tgen.py index fa00ebb..24658c7 100644 --- a/tools/scripts/parse-tgen.py +++ b/tools/scripts/parse-tgen.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import sys, os, argparse, re, json from multiprocessing import Pool, cpu_count @@ -15,19 +15,19 @@ sizes reach tens of gigabytes. Use the help menu to understand usage: -$ python parse-tgen.py -h +$ python3 parse-tgen.py -h The standard way to run the script is to give the path to a directory tree under which one or several tgen log files exist: -$ python parse-tgen.py shadow.data/hosts/ -$ python parse-tgen.py ./ +$ python3 parse-tgen.py shadow.data/hosts/ +$ python3 parse-tgen.py ./ This path will be searched for log files whose names match those created by shadow; additional patterns can be added with the '-e' option. A single tgen log file can also be passed on STDIN with the special '-' path: -$ cat tgen.log | python parse-tgen.py - -$ xzcat tgen.log.xz | python parse-tgen.py - +$ cat tgen.log | python3 parse-tgen.py - +$ xzcat tgen.log.xz | python3 parse-tgen.py - The default mode is to filter and parse the log files using a single process; this will be done with multiple worker processes when passing @@ -87,7 +87,7 @@ def run(args): while not mr.ready(): mr.wait(1) r = mr.get() except KeyboardInterrupt: - print >> sys.stderr, "interrupted, terminating process pool" + print("interrupted, terminating process pool", file=sys.stderr) p.terminate() p.join() sys.exit() @@ -106,10 +106,10 @@ def run(args): success_count += item[2] error_count += item[3] - print >> sys.stderr, "done processing input: {0} total successes, {1} total errors, {2} files with names, {3} files without names".format(success_count, error_count, name_count, noname_count) - print >> sys.stderr, "dumping stats in {0}".format(args.prefix) + print("done processing input: {0} total successes, {1} total errors, {2} files with names, {3} files without names".format(success_count, error_count, name_count, noname_count), file=sys.stderr) + print("dumping stats in {0}".format(args.prefix), file=sys.stderr) dump(d, args.prefix, TGENJSON) - print >> sys.stderr, "all done!" + print("all done!", file=sys.stderr) def process_tgen_log(filename): signal(SIGINT, SIG_IGN) # ignore interrupts diff --git a/tools/setup.py b/tools/setup.py index 38425f1..cacbc41 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from setuptools import setup diff --git a/tools/tgentools/tgentools b/tools/tgentools/tgentools index b012a81..0e4c0de 100755 --- a/tools/tgentools/tgentools +++ b/tools/tgentools/tgentools @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 ''' tgentools