From 467c24fa4e5c5e22758abb5ca63f6ce263f137da Mon Sep 17 00:00:00 2001 From: Pagin Matteo Date: Fri, 27 Jan 2023 10:15:12 +0100 Subject: [PATCH 1/4] Iniitial work on fixing get_command_from_result --- sem/cli.py | 2 ++ sem/runner.py | 4 ++-- sem/utils.py | 12 +++++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/sem/cli.py b/sem/cli.py index e289083..c082c0c 100644 --- a/sem/cli.py +++ b/sem/cli.py @@ -235,9 +235,11 @@ def command(results_dir, result_id): click.echo("Simulation command:") click.echo(sem.utils.get_command_from_result(campaign.db.get_script(), + campaign.dir, result)) click.echo("Debug command:") click.echo(sem.utils.get_command_from_result(campaign.db.get_script(), + campaign.dir, result, debug=True)) diff --git a/sem/runner.py b/sem/runner.py index ac9df28..3cca5ae 100644 --- a/sem/runner.py +++ b/sem/runner.py @@ -352,8 +352,8 @@ def run_simulations(self, parameter_list, data_folder, callbacks: [CallbackBase] with open(stdout_file_path, 'r') as stdout_file, open( stderr_file_path, 'r') as stderr_file: - complete_command = sem.utils.get_command_from_result(self.script, current_result) - complete_command_debug = sem.utils.get_command_from_result(self.script, current_result, debug=True) + complete_command = sem.utils.get_command_from_result(self.script, self.path, current_result) + complete_command_debug = sem.utils.get_command_from_result(self.script, self.path, current_result, debug=True) error_message = ('\nSimulation exited with an error.\n' 'Params: %s\n' 'Stderr: %s\n' diff --git a/sem/utils.py b/sem/utils.py index 5b5c50c..988cd07 100644 --- a/sem/utils.py +++ b/sem/utils.py @@ -1,4 +1,5 @@ import io +import os import math import copy import warnings @@ -101,21 +102,26 @@ def list_param_combinations(param_ranges): return [param_ranges_copy] -def get_command_from_result(script, result, debug=False): +def get_command_from_result(script, path, result, debug=False): """ Return the command that is needed to obtain a certain result. Args: params (dict): Dictionary containing parameter: value pairs. + path (str): The path to the ns-3 folder, used to discern which + build system (waf/CMake) is used debug (bool): Whether the command should include the debugging template. """ + command = "python3 " + command += "./ns3 run " if os.path.exists(os.path.join(path, "ns3")) else " ./waf --run " + if not debug: - command = "python3 waf --run \"" + script + " " + " ".join( + command += script + " " + " ".join( ['--%s=%s' % (param, value) for param, value in result['params'].items()]) + "\"" else: - command = "python3 waf --run " + script + " --command-template=\"" +\ + command += script + " --command-template=\"" +\ "gdb --args %s " + " ".join(['--%s=%s' % (param, value) for param, value in result['params'].items()]) + "\"" From 3e3b72eb88d8b162d516133eae82fad54cfb7682 Mon Sep 17 00:00:00 2001 From: pagmatt Date: Fri, 21 Apr 2023 15:55:19 -0400 Subject: [PATCH 2/4] Test runner also with waf-based versions of ns-3 --- tests/conftest.py | 24 ++++++++++++++++++++++-- tests/test_runner.py | 27 +++++++++++++++++++-------- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e48f381..fd233fb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,7 +42,7 @@ def ns_3_compiled(tmpdir): # Relocate build by running the same command in the new directory if subprocess.call([build_program, 'configure', '--disable-gtk', - '--build-profile=optimized', '--enable-modules=core', + '--build-profile=optimized', '--enable-modules=core', '--enable-examples', '--out=build/optimized'], cwd=ns_3_tempdir, stdout=subprocess.DEVNULL, @@ -70,7 +70,7 @@ def ns_3_compiled_debug(tmpdir): # Relocate build by running the same command in the new directory if subprocess.call([build_program, 'configure', '--disable-gtk', - '--build-profile=debug', '--enable-modules=core', + '--build-profile=debug', '--enable-modules=core', '--enable-examples', '--out=build'], cwd=ns_3_tempdir, stdout=subprocess.DEVNULL, @@ -85,6 +85,26 @@ def ns_3_compiled_debug(tmpdir): return ns_3_tempdir +@pytest.fixture(scope='function') +def ns_3_compiled_examples(): + # Configure and build WAF-based ns-3 + build_program = get_build_program(ns_3_examples) + + if subprocess.call([build_program, 'configure', '--disable-gtk', + '--build-profile=optimized', '--enable-modules=core', + '--out=build/optimized'], + cwd=ns_3_examples, + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT) != 0: + raise Exception("Examples configuration failed.") + + if subprocess.call([build_program, 'build'], + cwd=ns_3_examples, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) > 0: + raise Exception("Examples build failed.") + + return ns_3_examples @pytest.fixture(scope='function') def config(tmpdir, ns_3_compiled): diff --git a/tests/test_runner.py b/tests/test_runner.py index 9c9e2db..18d3672 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -6,11 +6,21 @@ # Runner creation # ################### +""" +First param: Runner type +Second param: Whether to use optimized build +Third param: Whether to use the CMake-version of ns-3 +""" +@pytest.fixture(scope='function', params=[['ParallelRunner', True, True], + ['ParallelRunner', True, False]]) +def runner(ns_3_compiled, ns_3_compiled_debug, ns_3_compiled_examples, config, request): + ns_3_folder = ns_3_compiled + if request.param[1] is False: + ns_3_folder = ns_3_compiled_debug + if request.param[2] is True: + assert(request.param[1] is True) + ns_3_folder = ns_3_compiled_examples -@pytest.fixture(scope='function', params=[['ParallelRunner', True], - ['ParallelRunner', False]]) -def runner(ns_3_compiled, ns_3_compiled_debug, config, request): - ns_3_folder = ns_3_compiled if request.param[1] is True else ns_3_compiled_debug if request.param[0] == 'SimulationRunner': return SimulationRunner(ns_3_folder, config['script'], optimized=request.param[1]) @@ -18,7 +28,6 @@ def runner(ns_3_compiled, ns_3_compiled_debug, config, request): return ParallelRunner(ns_3_folder, config['script'], optimized=request.param[1]) - def test_get_available_parameters(runner, config): # Try getting the available parameters of the script assert runner.get_available_parameters() == config['params'] @@ -26,13 +35,15 @@ def test_get_available_parameters(runner, config): @pytest.mark.parametrize('runner', [ - ['SimulationRunner', True], - ['ParallelRunner', True], + ['SimulationRunner', True, True], + ['ParallelRunner', True, True], + ['ParallelRunner', True, False], ], indirect=True) def test_run_simulations(runner, config, parameter_combination): - # Make sure that simulations run without any issue + # Make sure that simulations run without any issue, + # with CMake optimized and debug builds, and Waf optimized builds data_dir = os.path.join(config['campaign_dir'], 'data') list(runner.run_simulations([parameter_combination], data_dir)) From 8de1e5b2f25bffab9138fa31d70ba8e4c44fc50e Mon Sep 17 00:00:00 2001 From: pagmatt Date: Tue, 18 Apr 2023 15:02:40 -0400 Subject: [PATCH 3/4] improve ns3 command --- sem/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sem/utils.py b/sem/utils.py index 988cd07..d8116a4 100644 --- a/sem/utils.py +++ b/sem/utils.py @@ -113,8 +113,7 @@ def get_command_from_result(script, path, result, debug=False): debug (bool): Whether the command should include the debugging template. """ - command = "python3 " - command += "./ns3 run " if os.path.exists(os.path.join(path, "ns3")) else " ./waf --run " + command = "./ns3 run " if os.path.exists(os.path.join(path, "ns3")) else "python3 ./waf --run " if not debug: command += script + " " + " ".join( From f381ee5bc80963671ef9822304f196d0a8592b71 Mon Sep 17 00:00:00 2001 From: pagmatt Date: Fri, 21 Apr 2023 15:54:49 -0400 Subject: [PATCH 4/4] Add initial get_command_from_result test --- sem/__init__.py | 4 ++-- sem/utils.py | 4 ++-- tests/test_utils.py | 45 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/sem/__init__.py b/sem/__init__.py index 011ff39..fbf6e08 100644 --- a/sem/__init__.py +++ b/sem/__init__.py @@ -4,11 +4,11 @@ from .lptrunner import LptRunner from .gridrunner import BUILD_GRID_PARAMS, SIMULATION_GRID_PARAMS from .database import DatabaseManager -from .utils import list_param_combinations, automatic_parser, stdout_automatic_parser, only_load_some_files, CallbackBase +from .utils import list_param_combinations, automatic_parser, stdout_automatic_parser, only_load_some_files, get_command_from_result, CallbackBase from .cli import cli __all__ = ('CampaignManager', 'SimulationRunner', 'ParallelRunner', 'LptRunner', 'DatabaseManager', 'list_param_combinations', 'automatic_parser', - 'only_load_some_files', 'CallbackBase') + 'only_load_some_files', 'get_command_from_result', 'CallbackBase') name = 'sem' diff --git a/sem/utils.py b/sem/utils.py index d8116a4..ffbd40e 100644 --- a/sem/utils.py +++ b/sem/utils.py @@ -116,11 +116,11 @@ def get_command_from_result(script, path, result, debug=False): command = "./ns3 run " if os.path.exists(os.path.join(path, "ns3")) else "python3 ./waf --run " if not debug: - command += script + " " + " ".join( + command += "\"" + script + " " + " ".join( ['--%s=%s' % (param, value) for param, value in result['params'].items()]) + "\"" else: - command += script + " --command-template=\"" +\ + command += "\"" + script + " --command-template=\"" +\ "gdb --args %s " + " ".join(['--%s=%s' % (param, value) for param, value in result['params'].items()]) + "\"" diff --git a/tests/test_utils.py b/tests/test_utils.py index 29d7689..9deab9f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,9 +1,24 @@ -from sem import list_param_combinations, automatic_parser, stdout_automatic_parser, CallbackBase, CampaignManager +from sem import list_param_combinations, automatic_parser, stdout_automatic_parser, get_command_from_result, CallbackBase, CampaignManager import json import numpy as np +import pytest from operator import getitem +@pytest.fixture(scope='function', params=[['compiled', False]]) +def ns_3_compiled_folder_and_command(ns_3_compiled, ns_3_compiled_examples, request): + if request.param[0] == 'compiled': + if request.param[1] is False: + return [ns_3_compiled, False, './ns3 run \"hash-example --dict=/usr/share/dict/american-english --time=False --RngRun=0\"'] + # elif request.param[1] is True: + # return [ns_3_compiled, True, './ns3 run \"hash-example --dict=/usr/share/dict/american-english --time=False --RngRun=0\"'] + elif request.param[0] == 'compiled_examples': + if request.param[1] is False: + return [ns_3_compiled_examples, False, 'python3 ./waf --run \"hash-example --dict=/usr/share/dict/american-english --time=False --RngRun=0\"'] + # elif request.param[1] is True: + # return [ns_3_compiled_examples, True, 'python3 ./waf --run \"hash-example --dict=/usr/share/dict/american-english --time=False --RngRun=0\"'] + + def test_list_param_combinations(): # Two possible combinations d1 = {'a': [1, 2]} @@ -139,3 +154,31 @@ def test_callback(ns_3_compiled, config, parameter_combination): campaign.run_missing_simulations( param_list=[parameter_combination], callbacks=[cb]) assert expected_output == cb.output +@pytest.mark.parametrize('ns_3_compiled_folder_and_command', + [ + ['compiled', False], + ['compiled_examples', False] + ], + indirect=True) + + +def test_get_cmnd_from_result(ns_3_compiled_folder_and_command, config, parameter_combination): + + # Create an ns-3 campaign to run simulations and obtain a result + ns_3_folder = ns_3_compiled_folder_and_command[0] + hardcoded_command = ns_3_compiled_folder_and_command[2] + + cmpgn = CampaignManager.new(ns_3_folder, + config['script'], + config['campaign_dir'], + overwrite=True, + skip_configuration=True) + + cmpgn.run_simulations([parameter_combination], show_progress=False) + + # Retrieve the results, and compare the output of get_command_from_result + # with the expected command + result = cmpgn.db.get_complete_results()[0] + cmnd = get_command_from_result( + config['script'], ns_3_folder, result) + assert (hardcoded_command == cmnd)