Skip to content

Commit

Permalink
Merge pull request #350 from xylar/run_test_cases_like_suites
Browse files Browse the repository at this point in the history 
Change test cases to run like suites
  • Loading branch information
@xylar
xylar authored Apr 15, 2022
2 parents 43ed76e + bc4f9ef commit 019a16f
Show file tree
Hide file tree
Showing 6 changed files with 98 additions and 116 deletions.
5 changes: 3 additions & 2 deletions compass/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@
import argparse
import os

from compass import list, setup, clean, suite, run, cache
from compass import list, setup, clean, suite, cache
from compass.version import __version__
import compass.run.serial as run_serial


def main():
Expand Down Expand Up @@ -49,7 +50,7 @@ def main():
'setup': setup.main,
'clean': clean.main,
'suite': suite.main,
'run': run.main}
'run': run_serial.main}
if allow_cache:
commands['cache'] = cache.main

Expand Down
Empty file added compass/run/__init__.py
View file
Empty file.
145 changes: 61 additions & 84 deletions compass/run.py → compass/run/serial.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,10 @@
from compass.config import CompassConfigParser


def run_suite(suite_name, quiet=False):
def run_tests(suite_name, quiet=False, is_test_case=False, steps_to_run=None,
steps_not_to_run=None):
"""
Run the given test suite
Run the given test suite or test case
Parameters
----------
Expand All @@ -25,6 +26,17 @@ def run_suite(suite_name, quiet=False):
Whether step names are not included in the output as the test suite
progresses
is_test_case : bool
Whether this is a test case instead of a full test suite
steps_to_run : list of str, optional
A list of the steps to run if this is a test case, not a full suite.
The default behavior is to run the default steps unless they are in
``steps_not_to_run``
steps_not_to_run : list of str, optional
A list of steps not to run if this is a test case, not a full suite.
Typically, these are steps to remove from the defaults
"""
# ANSI fail text: https://stackoverflow.com/a/287944/7728169
start_fail = '\033[91m'
Expand Down Expand Up @@ -59,10 +71,11 @@ def run_suite(suite_name, quiet=False):

os.environ['PYTHONUNBUFFERED'] = '1'

try:
os.makedirs('case_outputs')
except OSError:
pass
if not is_test_case:
try:
os.makedirs('case_outputs')
except OSError:
pass

failures = 0
cwd = os.getcwd()
Expand All @@ -75,9 +88,14 @@ def run_suite(suite_name, quiet=False):
logger.info('{}'.format(test_name))

test_name = test_case.path.replace('/', '_')
log_filename = '{}/case_outputs/{}.log'.format(cwd, test_name)
with LoggingContext(test_name, log_filename=log_filename) as \
test_logger:
if is_test_case:
log_filename = None
test_logger = logger
else:
log_filename = '{}/case_outputs/{}.log'.format(cwd, test_name)
test_logger = None
with LoggingContext(test_name, logger=test_logger,
log_filename=log_filename) as test_logger:
if quiet:
# just log the step names and any failure messages to the
# log file
Expand All @@ -87,7 +105,7 @@ def run_suite(suite_name, quiet=False):
test_case.stdout_logger = logger
test_case.logger = test_logger
test_case.log_filename = log_filename
test_case.new_step_log_file = False
test_case.new_step_log_file = is_test_case

os.chdir(test_case.work_dir)

Expand All @@ -99,12 +117,14 @@ def run_suite(suite_name, quiet=False):
mpas_tools.io.default_format = config.get('io', 'format')
mpas_tools.io.default_engine = config.get('io', 'engine')

test_case.steps_to_run = config.get(
'test_case', 'steps_to_run').replace(',', ' ').split()
test_case.steps_to_run = _update_steps_to_run(
steps_to_run, steps_not_to_run, config, test_case.steps)

test_start = time.time()
log_method_call(method=test_case.run, logger=test_logger)
test_logger.info('')
test_list = ', '.join(test_case.steps_to_run)
test_logger.info(f'Running steps: {test_list}')
try:
test_case.run()
run_status = success_str
Expand Down Expand Up @@ -196,75 +216,6 @@ def run_suite(suite_name, quiet=False):
sys.exit(1)


def run_test_case(steps_to_run=None, steps_not_to_run=None):
"""
Used by the framework to run a test case when ``compass run`` gets called
in the test case's work directory
Parameters
----------
steps_to_run : list of str, optional
A list of the steps to run. The default behavior is to run the default
steps unless they are in ``steps_not_to_run``
steps_not_to_run : list of str, optional
A list of steps not to run. Typically, these are steps to remove from
the defaults
"""
with open('test_case.pickle', 'rb') as handle:
test_case = pickle.load(handle)

config = CompassConfigParser()
config.add_from_file(test_case.config_filename)

check_parallel_system(config)

test_case.config = config
set_cores_per_node(test_case.config)

mpas_tools.io.default_format = config.get('io', 'format')
mpas_tools.io.default_engine = config.get('io', 'engine')

if steps_to_run is None:
steps_to_run = config.get('test_case',
'steps_to_run').replace(',', ' ').split()

for step in steps_to_run:
if step not in test_case.steps:
raise ValueError(
'A step "{}" was requested but is not one of the steps in '
'this test case:\n{}'.format(step, list(test_case.steps)))

if steps_not_to_run is not None:
for step in steps_not_to_run:
if step not in test_case.steps:
raise ValueError(
'A step "{}" was flagged not to run but is not one of the '
'steps in this test case:'
'\n{}'.format(step, list(test_case.steps)))

steps_to_run = [step for step in steps_to_run if step not in
steps_not_to_run]

test_case.steps_to_run = steps_to_run

# start logging to stdout/stderr
test_name = test_case.path.replace('/', '_')
test_case.new_step_log_file = True
with LoggingContext(name=test_name) as logger:
test_case.logger = logger
test_case.stdout_logger = logger
log_method_call(method=test_case.run, logger=logger)
logger.info('')
logger.info('Running steps: {}'.format(', '.join(steps_to_run)))
test_case.run()

logger.info('')
log_method_call(method=test_case.validate, logger=logger)
logger.info('')
test_case.validate()


def run_step():
"""
Used by the framework to run a step when ``compass run`` gets called in the
Expand Down Expand Up @@ -320,19 +271,45 @@ def main():
"their own.")
args = parser.parse_args(sys.argv[2:])
if args.suite is not None:
run_suite(args.suite, quiet=args.quiet)
run_tests(args.suite, quiet=args.quiet)
elif os.path.exists('test_case.pickle'):
run_test_case(args.steps, args.no_steps)
run_tests(suite_name='test_case', quiet=args.quiet, is_test_case=True,
steps_to_run=args.steps, steps_not_to_run=args.no_steps)
elif os.path.exists('step.pickle'):
run_step()
else:
pickles = glob.glob('*.pickle')
if len(pickles) == 1:
suite = os.path.splitext(os.path.basename(pickles[0]))[0]
run_suite(suite, quiet=args.quiet)
run_tests(suite, quiet=args.quiet)
elif len(pickles) == 0:
raise OSError('No pickle files were found. Are you sure this is '
'a compass suite, test-case or step work directory?')
else:
raise ValueError('More than one suite was found. Please specify '
'which to run: compass run <suite>')


def _update_steps_to_run(steps_to_run, steps_not_to_run, config, steps):
if steps_to_run is None:
steps_to_run = config.get('test_case',
'steps_to_run').replace(',', ' ').split()

for step in steps_to_run:
if step not in steps:
raise ValueError(
'A step "{}" was requested but is not one of the steps in '
'this test case:\n{}'.format(step, list(steps)))

if steps_not_to_run is not None:
for step in steps_not_to_run:
if step not in steps:
raise ValueError(
'A step "{}" was flagged not to run but is not one of the '
'steps in this test case:'
'\n{}'.format(step, list(steps)))

steps_to_run = [step for step in steps_to_run if step not in
steps_not_to_run]

return steps_to_run
5 changes: 4 additions & 1 deletion compass/setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,10 @@ def setup_case(path, test_case, config_file, machine, work_dir, baseline_dir,
# pickle the test case and step for use at runtime
pickle_filename = os.path.join(test_case.work_dir, 'test_case.pickle')
with open(pickle_filename, 'wb') as handle:
pickle.dump(test_case, handle, protocol=pickle.HIGHEST_PROTOCOL)
test_suite = {'name': 'test_case',
'test_cases': {test_case.path: test_case},
'work_dir': test_case.work_dir}
pickle.dump(test_suite, handle, protocol=pickle.HIGHEST_PROTOCOL)

if 'LOAD_COMPASS_ENV' in os.environ:
script_filename = os.environ['LOAD_COMPASS_ENV']
Expand Down
5 changes: 2 additions & 3 deletions docs/developers_guide/api.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,13 +80,12 @@ suite
run
~~~

.. currentmodule:: compass.run
.. currentmodule:: compass.run.serial

.. autosummary::
:toctree: generated/

run_suite
run_test_case
run_tests
run_step


Expand Down
54 changes: 28 additions & 26 deletions docs/developers_guide/framework.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,32 +72,34 @@ with before at least some steps in the suite will fail.

.. _dev_run:

run module
~~~~~~~~~~

The :py:func:`compass.run.run_suite()`, :py:func:`compass.run.run_test_case()`,
and :py:func:`compass.run.run_step()` functions are used to run a test suite,
test case or step, respectively, from the base, test case or step work
directory, respectively, using ``compass run``. Each of these functions reads
a local pickle file to retrieve information about the test suite, test case
and/or step that was stored during setup.

:py:func:`compass.run.run_suite()` runs each test case in the test suite in
the order that they are given in the text file defining the suite
(``compass/<mpas_core>/suites/<suite_name>.txt``). It displays a ``PASS`` or
``FAIL`` message for the test execution, as well as similar messages for
validation involving output within the test case or suite and validation
against a baseline (depending on the implementation of the ``validate()``
method in the test case and whether a baseline was provided during setup).
Output from test cases and their steps are stored in log files in
the ``case_output`` subdirectory of the base work directory.

:py:func:`compass.run.run_test_case()` and :py:func:`compass.run.run_step()`
run a single test case. In the latter case, only the selected step from the
test case is run, skipping any others. If running the full test case, output
from individual steps are stored in log files ``<step>.log`` in the test case's
work directory. The results of validation (if any) are displayed in the final
stage of running the test case.
run.serial module
~~~~~~~~~~~~~~~~~

The function :py:func:`compass.run.serial.run_tests()` is used to run a
test suite or test case and :py:func:`compass.run.serial.run_step()` is used to
run a step using ``compass run``. Suites run from the base work directory
with a pickle file starting with the suite name, or ``custom.pickle`` if a
suite name was not given. Test cases or steps run from their respective
subdirectories with a ``testcase.pickle`` or ``step.pickle`` file in them.
Both of these functions reads the local pickle file to retrieve information
about the test suite, test case and/or step that was stored during setup.

If :py:func:`compass.run.serial.run_tests()` is used for a test suite, it will
run each test case in the test suite in the order that they are given in the
text file defining the suite (``compass/<mpas_core>/suites/<suite_name>.txt``).
Output from test cases and their steps are stored in log files in the
``case_output`` subdirectory of the base work directory. If the function is
used for a single test case, it will run the steps of that test case, writing
output for each step to a log file starting with the step's name. In either
case (suite or individual test), it displays a ``PASS`` or ``FAIL`` message for
the test execution, as well as similar messages for validation involving output
within the test case or suite and validation against a baseline (depending on
the implementation of the ``validate()`` method in the test case and whether a
baseline was provided during setup).

:py:func:`compass.run.run_step()` runs only the selected step from a given
test case, skipping any others, displaying the output in the terminal window
rather than a log file.

.. _dev_cache:

Expand Down

0 comments on commit 019a16f

Please sign in to comment.