Merge pull request #28 from nicjhan/diagnostic-tests

ice_ocean_SIS2/Baltic/input.nml

@@ -15,6 +15,11 @@
                              'SIS_override' /
 
  &diag_manager_nml
+         max_output_fields=1000
+         max_input_fields=1000
+         max_num_axis_sets=1000
+         max_files=1000
+         max_axes=2000
  /
 
  &coupler_nml

tools/tests/README.md

@@ -0,0 +1,45 @@
+
+# Running Python tests
+
+This directory contains model tests written in Python. They rely on the pytest testing framework.
+
+## Install pytest
+
+If you don't have pytest installed you can install it with:
+```
+$ sudo easy_install pytest
+```
+or, if you don't have sudo rights,
+```
+$ easy_install --user pytest
+```
+
+If you don't have easy_install, you can install that with:
+```
+$ wget https://bootstrap.pypa.io/ez_setup.py
+$ sudo python ez_setup.py
+```
+or
+```
+$ wget https://bootstrap.pypa.io/ez_setup.py
+$ python ez_setup.py --user
+```
+If you use the --user options above then binaries and packages will be installed to $HOME/.local/bin and $HOME/.local/lib . You may need to include these directories in your $PATH and $PYTHONPATH if they are not already.
+
+## Run the tests
+
+Then you can run the tests with:
+```
+$ py.test
+```
+Or a subset of the tests with:
+```
+$ py.test <test_file.py>
+```
+Also see:
+```
+$ py.test --help
+```
+Pay particular attention to the 'custom options' section.
+
+If you don't have py.test installed on your machine, then you can do all of the above by replacing py.test with: python runtest.py

tools/tests/conftest.py

@@ -0,0 +1,79 @@
+
+import os
+
+import pytest
+from dump_all_diagnostics import dump_diags
+from experiment import experiment_dict, exp_id_from_path
+
+def pytest_addoption(parser):
+    parser.addoption('--exps', default=None,
+                     help="""comma-separated no spaces list of experiments to
+                             pass to test functions. Also you must use the '='
+                             sign otherwise py.test gets confused, e.g:
+                             $ py.test --exps=ice_ocean_SIS2/Baltic/,ocean_only/benchmark""")
+    parser.addoption('--full', action='store_true', default=False,
+                     help="""Run on all experiments/test cases. By default
+                             tests are run on a 'fast' subset of experiments.
+                             Note that this overrides the --exps option.""")
+
+def pytest_generate_tests(metafunc):
+    """
+    Parameterize tests. Presently handles those that have 'exp' as an argument.
+    """
+    if 'exp' in metafunc.fixturenames:
+        if metafunc.config.option.full:
+            # Run tests on all experiments.
+            exps = experiment_dict.values()
+        elif metafunc.config.option.exps is not None:
+            # Only run on the given experiments.
+            exps = []
+            for p in metafunc.config.option.exps.split(','):
+                assert(os.path.exists(p))
+                id = exp_id_from_path(os.path.abspath(p))
+                exps.append(experiment_dict[id])
+        else:
+            # Default is to run on a fast subset of the experiments.
+            exps = [experiment_dict['ice_ocean_SIS2/Baltic']]
+
+        metafunc.parametrize('exp', exps, indirect=True)
+
+@pytest.fixture
+def exp(request):
+    """
+    Called before each test, use this to dump all the experiment data.
+    """
+    exp = request.param
+
+    # Run the experiment to get latest code changes. This will do nothing if
+    # the experiment has already been run.
+    exp.run()
+    # Dump all available diagnostics, if they haven't been already.
+    if not exp.has_dumped_diags:
+        # Before dumping we delete old ones if they exist.
+        diags = exp.get_available_diags()
+        for d in diags:
+            try:
+                os.remove(d.output)
+            except OSError:
+                pass
+
+        dump_diags(exp, diags)
+        exp.has_dumped_diags = True
+    return exp
+
+def restore_after_test():
+    """
+    Restore experiment state after running a test.
+
+    - The diag_table files needs to be switched back (?)
+    """
+    pass
+
+@pytest.fixture(scope='module')
+def prepare_to_test():
+    """
+    Called once for a test module.
+
+    - Make a backup of the diag_table, to be restored later. (?)
+    """
+    pass

tools/tests/dump_all_diagnostics.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import sys, os
+import argparse
+from experiment import Experiment
+from experiment import exp_id_from_path
+
+"""
+This script is used to run an experiment/test case and dump all available
+diagnostics. This can be useful for testing or to get a feel for the range of
+model outputs.
+"""
+
+def dump_diags(exp, diags):
+    """
+    Run the model dumping the given diagnostics into individual files.
+    """
+
+    # Create a new diag_table that puts diagnostics into individual files. This
+    # is a trick to get the highest frequency output for each diagnostic.
+    #
+    # By default if only a single file is used, when '0' is set as the
+    # frequency, the diag manager will choose the minimum frequency and dump
+    # all diagnostics with that. This will result in the slower diagnostics
+    # being filled up with missing values which is not desirable.
+
+    with open(os.path.join(exp.path, 'diag_table'), 'w') as f:
+        print('All {} diags'.format(exp.name), file=f)
+        print('1 1 1 0 0 0', file=f)
+        for d in diags:
+            print('"{}_{}", 0, "seconds", 1, "seconds",' \
+                  '"time"'.format(d.model, d.name), file=f)
+        for d in diags:
+            m = d.model
+            n = d.name
+            print('"{}", "{}", "{}", "{}_{}", "all",' \
+                  '.false., "none", 2'.format(m, n, n, m, n), file=f)
+    return exp.force_run()
+
+def main():
+
+    description = "Run an experiment and dump all it's available diagnostics."
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument('exp_path',
+                        help='path to experiment whose diagnostics will be dumped.')
+
+    args = parser.parse_args()
+    if not os.path.exists(args.exp_path):
+      print('Experiment {} is not a valid path!'.format(args.exp_path),
+             file=sys.stderr)
+      parser.print_help()
+      return 1
+
+    exp_id = exp_id_from_path(args.exp_path)
+    exp = Experiment(exp_id)
+    diags = exp.get_available_diags()
+    return dump_diags(exp, diags)
+
+if __name__ == '__main__':
+    sys.exit(main())