From ea2ae61bdfb30e82f5542267ae4f37dff0955782 Mon Sep 17 00:00:00 2001 From: Althea Denlinger Date: Thu, 3 Feb 2022 15:44:28 -0700 Subject: [PATCH 1/6] Move run module into run/serial.py --- compass/run/__init__.py | 0 compass/{run.py => run/serial.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 compass/run/__init__.py rename compass/{run.py => run/serial.py} (100%) diff --git a/compass/run/__init__.py b/compass/run/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/compass/run.py b/compass/run/serial.py similarity index 100% rename from compass/run.py rename to compass/run/serial.py From b736a2956ac6d414cda14a7da180156a5f6117ba Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 10 Feb 2022 15:25:27 -0700 Subject: [PATCH 2/6] Point `compass run` to `main()` in `serial.py` --- compass/__main__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/compass/__main__.py b/compass/__main__.py index 24b9f89f4b..12f994f4f4 100644 --- a/compass/__main__.py +++ b/compass/__main__.py @@ -4,8 +4,9 @@ import argparse import os -from compass import list, setup, clean, suite, run, cache +from compass import list, setup, clean, suite, cache from compass.version import __version__ +import compass.run.serial as run_serial def main(): @@ -49,7 +50,7 @@ def main(): 'setup': setup.main, 'clean': clean.main, 'suite': suite.main, - 'run': run.main} + 'run': run_serial.main} if allow_cache: commands['cache'] = cache.main From 457d65a0929335cca6f2853b2122bb17110b7438 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 10 Feb 2022 15:27:49 -0700 Subject: [PATCH 3/6] Change test cases to run like suites This reduces redundancy and gives better output (such as timing) when running at test case on its own. --- compass/run/serial.py | 139 +++++++++++++++++------------------------- compass/setup.py | 5 +- 2 files changed, 61 insertions(+), 83 deletions(-) diff --git a/compass/run/serial.py b/compass/run/serial.py index 0b7d417501..b422850334 100644 --- a/compass/run/serial.py +++ b/compass/run/serial.py @@ -12,9 +12,10 @@ from compass.config import CompassConfigParser -def run_suite(suite_name, quiet=False): +def run_suite(suite_name, quiet=False, is_test_case=False, steps_to_run=None, + steps_not_to_run=None): """ - Run the given test suite + Run the given test suite (or test case) Parameters ---------- @@ -25,6 +26,17 @@ def run_suite(suite_name, quiet=False): Whether step names are not included in the output as the test suite progresses + is_test_case : bool + Whether this is a test case instead of a full test suite + + steps_to_run : list of str, optional + A list of the steps to run if this is a test case, not a full suite. + The default behavior is to run the default steps unless they are in + ``steps_not_to_run`` + + steps_not_to_run : list of str, optional + A list of steps not to run if this is a test case, not a full suite. + Typically, these are steps to remove from the defaults """ # ANSI fail text: https://stackoverflow.com/a/287944/7728169 start_fail = '\033[91m' @@ -59,10 +71,11 @@ def run_suite(suite_name, quiet=False): os.environ['PYTHONUNBUFFERED'] = '1' - try: - os.makedirs('case_outputs') - except OSError: - pass + if not is_test_case: + try: + os.makedirs('case_outputs') + except OSError: + pass failures = 0 cwd = os.getcwd() @@ -75,9 +88,14 @@ def run_suite(suite_name, quiet=False): logger.info('{}'.format(test_name)) test_name = test_case.path.replace('/', '_') - log_filename = '{}/case_outputs/{}.log'.format(cwd, test_name) - with LoggingContext(test_name, log_filename=log_filename) as \ - test_logger: + if is_test_case: + log_filename = None + test_logger = logger + else: + log_filename = '{}/case_outputs/{}.log'.format(cwd, test_name) + test_logger = None + with LoggingContext(test_name, logger=test_logger, + log_filename=log_filename) as test_logger: if quiet: # just log the step names and any failure messages to the # log file @@ -87,7 +105,7 @@ def run_suite(suite_name, quiet=False): test_case.stdout_logger = logger test_case.logger = test_logger test_case.log_filename = log_filename - test_case.new_step_log_file = False + test_case.new_step_log_file = is_test_case os.chdir(test_case.work_dir) @@ -99,8 +117,8 @@ def run_suite(suite_name, quiet=False): mpas_tools.io.default_format = config.get('io', 'format') mpas_tools.io.default_engine = config.get('io', 'engine') - test_case.steps_to_run = config.get( - 'test_case', 'steps_to_run').replace(',', ' ').split() + test_case.steps_to_run = _update_steps_to_run( + steps_to_run, steps_not_to_run, config, test_case.steps) test_start = time.time() log_method_call(method=test_case.run, logger=test_logger) @@ -196,75 +214,6 @@ def run_suite(suite_name, quiet=False): sys.exit(1) -def run_test_case(steps_to_run=None, steps_not_to_run=None): - """ - Used by the framework to run a test case when ``compass run`` gets called - in the test case's work directory - - Parameters - ---------- - steps_to_run : list of str, optional - A list of the steps to run. The default behavior is to run the default - steps unless they are in ``steps_not_to_run`` - - steps_not_to_run : list of str, optional - A list of steps not to run. Typically, these are steps to remove from - the defaults - """ - with open('test_case.pickle', 'rb') as handle: - test_case = pickle.load(handle) - - config = CompassConfigParser() - config.add_from_file(test_case.config_filename) - - check_parallel_system(config) - - test_case.config = config - set_cores_per_node(test_case.config) - - mpas_tools.io.default_format = config.get('io', 'format') - mpas_tools.io.default_engine = config.get('io', 'engine') - - if steps_to_run is None: - steps_to_run = config.get('test_case', - 'steps_to_run').replace(',', ' ').split() - - for step in steps_to_run: - if step not in test_case.steps: - raise ValueError( - 'A step "{}" was requested but is not one of the steps in ' - 'this test case:\n{}'.format(step, list(test_case.steps))) - - if steps_not_to_run is not None: - for step in steps_not_to_run: - if step not in test_case.steps: - raise ValueError( - 'A step "{}" was flagged not to run but is not one of the ' - 'steps in this test case:' - '\n{}'.format(step, list(test_case.steps))) - - steps_to_run = [step for step in steps_to_run if step not in - steps_not_to_run] - - test_case.steps_to_run = steps_to_run - - # start logging to stdout/stderr - test_name = test_case.path.replace('/', '_') - test_case.new_step_log_file = True - with LoggingContext(name=test_name) as logger: - test_case.logger = logger - test_case.stdout_logger = logger - log_method_call(method=test_case.run, logger=logger) - logger.info('') - logger.info('Running steps: {}'.format(', '.join(steps_to_run))) - test_case.run() - - logger.info('') - log_method_call(method=test_case.validate, logger=logger) - logger.info('') - test_case.validate() - - def run_step(): """ Used by the framework to run a step when ``compass run`` gets called in the @@ -322,7 +271,8 @@ def main(): if args.suite is not None: run_suite(args.suite, quiet=args.quiet) elif os.path.exists('test_case.pickle'): - run_test_case(args.steps, args.no_steps) + run_suite(suite_name='test_case', quiet=args.quiet, is_test_case=True, + steps_to_run=args.steps, steps_not_to_run=args.no_steps) elif os.path.exists('step.pickle'): run_step() else: @@ -336,3 +286,28 @@ def main(): else: raise ValueError('More than one suite was found. Please specify ' 'which to run: compass run ') + + +def _update_steps_to_run(steps_to_run, steps_not_to_run, config, steps): + if steps_to_run is None: + steps_to_run = config.get('test_case', + 'steps_to_run').replace(',', ' ').split() + + for step in steps_to_run: + if step not in steps: + raise ValueError( + 'A step "{}" was requested but is not one of the steps in ' + 'this test case:\n{}'.format(step, list(steps))) + + if steps_not_to_run is not None: + for step in steps_not_to_run: + if step not in steps: + raise ValueError( + 'A step "{}" was flagged not to run but is not one of the ' + 'steps in this test case:' + '\n{}'.format(step, list(steps))) + + steps_to_run = [step for step in steps_to_run if step not in + steps_not_to_run] + + return steps_to_run diff --git a/compass/setup.py b/compass/setup.py index b5a72ec51b..60b4aad1c9 100644 --- a/compass/setup.py +++ b/compass/setup.py @@ -307,7 +307,10 @@ def setup_case(path, test_case, config_file, machine, work_dir, baseline_dir, # pickle the test case and step for use at runtime pickle_filename = os.path.join(test_case.work_dir, 'test_case.pickle') with open(pickle_filename, 'wb') as handle: - pickle.dump(test_case, handle, protocol=pickle.HIGHEST_PROTOCOL) + test_suite = {'name': 'test_case', + 'test_cases': {test_case.path: test_case}, + 'work_dir': test_case.work_dir} + pickle.dump(test_suite, handle, protocol=pickle.HIGHEST_PROTOCOL) if 'LOAD_COMPASS_ENV' in os.environ: script_filename = os.environ['LOAD_COMPASS_ENV'] From 54469fef8bb97e22390cb7634ccd95f4b4d502ca Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 1 Apr 2022 15:54:33 +0200 Subject: [PATCH 4/6] Update docs --- docs/developers_guide/api.rst | 3 +- docs/developers_guide/framework.rst | 54 +++++++++++++++-------------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/docs/developers_guide/api.rst b/docs/developers_guide/api.rst index 135007aa2f..3238e9c677 100644 --- a/docs/developers_guide/api.rst +++ b/docs/developers_guide/api.rst @@ -80,13 +80,12 @@ suite run ~~~ -.. currentmodule:: compass.run +.. currentmodule:: compass.run.serial .. autosummary:: :toctree: generated/ run_suite - run_test_case run_step diff --git a/docs/developers_guide/framework.rst b/docs/developers_guide/framework.rst index 5d15a6279a..a1f96dcbc1 100644 --- a/docs/developers_guide/framework.rst +++ b/docs/developers_guide/framework.rst @@ -72,32 +72,34 @@ with before at least some steps in the suite will fail. .. _dev_run: -run module -~~~~~~~~~~ - -The :py:func:`compass.run.run_suite()`, :py:func:`compass.run.run_test_case()`, -and :py:func:`compass.run.run_step()` functions are used to run a test suite, -test case or step, respectively, from the base, test case or step work -directory, respectively, using ``compass run``. Each of these functions reads -a local pickle file to retrieve information about the test suite, test case -and/or step that was stored during setup. - -:py:func:`compass.run.run_suite()` runs each test case in the test suite in -the order that they are given in the text file defining the suite -(``compass//suites/.txt``). It displays a ``PASS`` or -``FAIL`` message for the test execution, as well as similar messages for -validation involving output within the test case or suite and validation -against a baseline (depending on the implementation of the ``validate()`` -method in the test case and whether a baseline was provided during setup). -Output from test cases and their steps are stored in log files in -the ``case_output`` subdirectory of the base work directory. - -:py:func:`compass.run.run_test_case()` and :py:func:`compass.run.run_step()` -run a single test case. In the latter case, only the selected step from the -test case is run, skipping any others. If running the full test case, output -from individual steps are stored in log files ``.log`` in the test case's -work directory. The results of validation (if any) are displayed in the final -stage of running the test case. +run.serial module +~~~~~~~~~~~~~~~~~ + +The function :py:func:`compass.run.serial.run_suite()` is used to run a +test suite or test case and :py:func:`compass.run.serial.run_step()` is used to +run a step using ``compass run``. Suites run from the base work directory +with a pickle file starting with the suite name, or ``custom.pickle`` if a +suite name was not given. Test cases or steps run from their respective +subdirectories with a ``testcase.pickle`` or ``step.pickle`` file in them. +Both of these functions reads the local pickle file to retrieve information +about the test suite, test case and/or step that was stored during setup. + +If :py:func:`compass.run.serial.run_suite()` is used for a test suite, it will +run each test case in the test suite in the order that they are given in the +text file defining the suite (``compass//suites/.txt``). +Output from test cases and their steps are stored in log files in the +``case_output`` subdirectory of the base work directory. If the function is +used for a single test case, it will run the steps of that test case, writing +output for each step to a log file starting with the step's name. In either +case (suite or individual test), it displays a ``PASS`` or ``FAIL`` message for +the test execution, as well as similar messages for validation involving output +within the test case or suite and validation against a baseline (depending on +the implementation of the ``validate()`` method in the test case and whether a +baseline was provided during setup). + +:py:func:`compass.run.run_step()` runs only the selected step from a given +test case, skipping any others, displaying the output in the terminal window +rather than a log file. .. _dev_cache: From 56a443a42c84fadde2c26b0d65aea3114cf47634 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 11 Apr 2022 21:31:18 +0200 Subject: [PATCH 5/6] Rename function for running test cases/suites It is now `run_tests` rather than `run_suite`. --- compass/run/serial.py | 10 +++++----- docs/developers_guide/api.rst | 2 +- docs/developers_guide/framework.rst | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/compass/run/serial.py b/compass/run/serial.py index b422850334..44ca2404d8 100644 --- a/compass/run/serial.py +++ b/compass/run/serial.py @@ -12,10 +12,10 @@ from compass.config import CompassConfigParser -def run_suite(suite_name, quiet=False, is_test_case=False, steps_to_run=None, +def run_tests(suite_name, quiet=False, is_test_case=False, steps_to_run=None, steps_not_to_run=None): """ - Run the given test suite (or test case) + Run the given test suite or test case Parameters ---------- @@ -269,9 +269,9 @@ def main(): "their own.") args = parser.parse_args(sys.argv[2:]) if args.suite is not None: - run_suite(args.suite, quiet=args.quiet) + run_tests(args.suite, quiet=args.quiet) elif os.path.exists('test_case.pickle'): - run_suite(suite_name='test_case', quiet=args.quiet, is_test_case=True, + run_tests(suite_name='test_case', quiet=args.quiet, is_test_case=True, steps_to_run=args.steps, steps_not_to_run=args.no_steps) elif os.path.exists('step.pickle'): run_step() @@ -279,7 +279,7 @@ def main(): pickles = glob.glob('*.pickle') if len(pickles) == 1: suite = os.path.splitext(os.path.basename(pickles[0]))[0] - run_suite(suite, quiet=args.quiet) + run_tests(suite, quiet=args.quiet) elif len(pickles) == 0: raise OSError('No pickle files were found. Are you sure this is ' 'a compass suite, test-case or step work directory?') diff --git a/docs/developers_guide/api.rst b/docs/developers_guide/api.rst index 3238e9c677..58e048e0d4 100644 --- a/docs/developers_guide/api.rst +++ b/docs/developers_guide/api.rst @@ -85,7 +85,7 @@ run .. autosummary:: :toctree: generated/ - run_suite + run_tests run_step diff --git a/docs/developers_guide/framework.rst b/docs/developers_guide/framework.rst index a1f96dcbc1..d66beddf61 100644 --- a/docs/developers_guide/framework.rst +++ b/docs/developers_guide/framework.rst @@ -75,7 +75,7 @@ with before at least some steps in the suite will fail. run.serial module ~~~~~~~~~~~~~~~~~ -The function :py:func:`compass.run.serial.run_suite()` is used to run a +The function :py:func:`compass.run.serial.run_tests()` is used to run a test suite or test case and :py:func:`compass.run.serial.run_step()` is used to run a step using ``compass run``. Suites run from the base work directory with a pickle file starting with the suite name, or ``custom.pickle`` if a @@ -84,7 +84,7 @@ subdirectories with a ``testcase.pickle`` or ``step.pickle`` file in them. Both of these functions reads the local pickle file to retrieve information about the test suite, test case and/or step that was stored during setup. -If :py:func:`compass.run.serial.run_suite()` is used for a test suite, it will +If :py:func:`compass.run.serial.run_tests()` is used for a test suite, it will run each test case in the test suite in the order that they are given in the text file defining the suite (``compass//suites/.txt``). Output from test cases and their steps are stored in log files in the From bc4f9efdaa00ccf05a8323fee4a2563965ad1a65 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 13 Apr 2022 11:58:08 +0200 Subject: [PATCH 6/6] List steps that will run for each test case --- compass/run/serial.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compass/run/serial.py b/compass/run/serial.py index 44ca2404d8..be9b418c80 100644 --- a/compass/run/serial.py +++ b/compass/run/serial.py @@ -123,6 +123,8 @@ def run_tests(suite_name, quiet=False, is_test_case=False, steps_to_run=None, test_start = time.time() log_method_call(method=test_case.run, logger=test_logger) test_logger.info('') + test_list = ', '.join(test_case.steps_to_run) + test_logger.info(f'Running steps: {test_list}') try: test_case.run() run_status = success_str