NOAA-GFDL#98 Be able to build models before testing begins.

tools/tests/experiment.py

@@ -8,6 +8,7 @@
 import string
 import subprocess as sp
 import run_config as rc
+from model import Model
 
 # Only support Python version >= 2.7
 assert(not(sys.version_info[0] == 2) or sys.version_info[1] >= 7)
@@ -66,28 +67,34 @@ def exp_id_from_path(path):
 
 class Experiment:
 
-    def __init__(self, id, platform='gnu'):
+    def __init__(self, id, compiler='gnu', build='repro', memory_type='dynamic'):
         """
         Python representation of an experiment/test case.
 
         The id is a string of the form <model>/<exp>/<variation>.
         """
 
-        self.platform = platform
+        self.compiler = compiler
+        self.build = build
+        self.memory_type = memory_type
         id = id.split('/')
-        self.model = id[0]
+        self.model_name = id[0]
         self.name = id[1]
         if len(id) == 3:
             self.variation = id[2]
         else:
             self.variation = None
 
-        self.path = os.path.join(_mom_examples_path, self.model, self.name)
+        self.model = Model(self.model_name, _mom_examples_path)
+
+        self.path = os.path.join(_mom_examples_path, self.model_name,
+                                 self.name)
         if self.variation is not None:
             self.path = os.path.join(self.path, self.variation)
 
         # Path to executable, may not exist yet.
-        rel_path = 'build/{}/{}/repro/MOM6'.format(self.platform, self.model)
+        rel_path = 'build/{}/{}/{}/MOM6'.format(self.compiler,
+                                                self.model_name, build)
         self.exec_path = os.path.join(_mom_examples_path, rel_path)
 
         # Lists of available and unfinished diagnostics.
@@ -101,8 +108,9 @@ def __init__(self, id, platform='gnu'):
         # It helps with testing and human readability if this is sorted.
         self.available_diags.sort(key=lambda d: d.full_name)
 
-        # Whether this experiment has been run/built. Want to try to avoid
+        # Whether this experiment has been built, run. Want to try to avoid
         # repeating this if possible.
+        self.has_built = False
         self.has_run = False
         # Another thing to avoid repeating.
         self.has_dumped_diags = False
@@ -128,24 +136,35 @@ def _parse_available_diags(self):
                 diags.extend([Diagnostic(m, d, self.path) for m, d in matches])
         return diags
 
-    def force_build(self):
+    def force_build_model(self):
         """
         Do a clean build of the configuration.
         """
-        raise NotImplementedError
 
-    def build(self):
+        ret = self.model.build_shared(self.compiler, self.build)
+        if ret == 0:
+            ret = self.model.build_model(self.compiler, self.build,
+                                         self.memory_type)
+        if ret == 0:
+            self.has_built = True
+
+        return ret
+
+    def build_model(self):
         """
         Build the configuration for this experiment.
         """
-        raise NotImplementedError
+        if not self.has_built:
+            return self.force_build_model()
 
     def run(self):
         """
         Run the experiment if it hasn't already.
         """
+
+        self.build_model()
         if not self.has_run:
-            self.force_run()
+            return self.force_run()
 
     def force_run(self):
         """

tools/tests/model.py

@@ -0,0 +1,116 @@
+
+from __future__ import print_function
+
+import sys
+import os
+import subprocess as sp
+import shlex
+
+_build_fms_script = """
+../../../../mkmf/bin/list_paths ../../../../src/FMS &&
+../../../../mkmf/bin/mkmf -t ../../../../mkmf/templates/{site}-{compiler}.mk -p libfms.a -c "-Duse_libMPI -Duse_netCDF -DSPMD" path_names &&
+source ../../../../mkmf/env/{site}-{compiler}.env && make NETCDF=3 {build}=1 libfms.a -j
+"""
+
+_build_ocean_script = """
+pwd &&
+../../../../../mkmf/bin/list_paths ./ ../../../../../src/MOM6/{{config_src/{memory_type},config_src/solo_driver,src/{{*,*/*}}}}/
+../../../../../mkmf/bin/mkmf -t ../../../../../mkmf/templates/{site}-{compiler}.mk -o '-I../../../shared/{build}' -p 'MOM6 -L../../../shared/{build} -lfms' -c "-Duse_libMPI -Duse_netCDF -DSPMD" path_names &&
+source ../../../../../mkmf/env/{site}-{compiler}.env && make NETCDF=3 {build}=1 MOM6 -j
+"""
+
+_build_ocean_ice_script = """
+pwd &&
+../../../../../mkmf/bin/list_paths ./ ../../../../../src/MOM6/config_src/{{{memory_type},coupled_driver}} ../../../../../src/MOM6/src/{{*,*/*}}/ ../../../../../src/{{atmos_null,coupler,land_null,SIS2,ice_ocean_extras,icebergs,FMS/coupler,FMS/include}} &&
+../../../../../mkmf/bin/mkmf -t ../../../../../mkmf/templates/{site}-{compiler}.mk -o '-I../../../shared/{build}' -p 'MOM6 -L../../../shared/{build} -lfms' -c '-Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO' path_names &&
+source ../../../../../mkmf/env/{site}-{compiler}.env && make NETCDF=3 {build}=1 MOM6 -j
+"""
+
+def mkdir_p(path):
+    """Create a new directory; ignore if it already exists."""
+    import errno
+    try:
+        os.makedirs(path)
+    except OSError as exc:
+        if exc.errno != errno.EEXIST:
+            raise
+
+def get_shared_build_dir(mom_dir, compiler, build):
+    return os.path.join(mom_dir, 'build', compiler, 'shared', build)
+
+
+def get_model_build_dir(mom_dir, compiler, model_name, build, memory_type):
+    return os.path.join(mom_dir, 'build', compiler, model_name, build,
+                         memory_type)
+
+class Model:
+
+    def __init__(self, name, mom_dir):
+        self.name = name
+        self.mom_dir = mom_dir
+        self.site = 'raijin'
+
+    def build(self, compiler, build, memory_type):
+        self.build_shared(compiler, build)
+        self.build_model(compiler, build, memory_type)
+
+    def build_shared(self, compiler, build):
+        saved_path = os.getcwd()
+        ret = 0
+
+        # Build FMS
+        shared_dir = get_shared_build_dir(self.mom_dir, compiler, build)
+        mkdir_p(shared_dir)
+        os.chdir(shared_dir)
+        command = _build_fms_script.format(site=self.site, build=build,
+                                           compiler=compiler)
+        try:
+            output = sp.check_output(command, stderr=sp.STDOUT, shell=True)
+        except sp.CalledProcessError as e:
+            ret = e.returncode
+            print(e.output, file=sys.stderr)
+        finally:
+            os.chdir(saved_path)
+
+        with open(os.path.join(shared_dir, 'build.out'), 'w') as f:
+            f.write(output)
+
+        return ret
+
+
+    def build_model(self, compiler='gnu', build='REPRO', memory_type='dynamic'):
+        """
+        Build this model.
+        """
+        saved_path = os.getcwd()
+        ret = 0
+
+        # Build either ocean_only or ice and ocean.
+
+        model_dir = get_model_build_dir(self.mom_dir, compiler, self.name,
+                                        build, memory_type)
+        mkdir_p(model_dir)
+        os.chdir(model_dir)
+        if self.name == 'ocean_only':
+            command = _build_ocean_script.format(site=self.site, build=build,
+                                                 compiler=compiler,
+                                                 memory_type=memory_type)
+        elif self.name == 'ice_ocean_SIS2':
+            command = _build_ocean_ice_script.format(site=self.site, build=build,
+                                                     compiler=compiler,
+                                                     memory_type=memory_type)
+        else:
+            print('Unsupported model type', file=sys.stderr)
+            assert False
+        try:
+            output = sp.check_output(command, stderr=sp.STDOUT, shell=True)
+        except sp.CalledProcessError as e:
+            ret = e.returncode
+            print(e.output, file=sys.stderr)
+        finally:
+            os.chdir(saved_path)
+
+        with open(os.path.join(model_dir, 'build.out'), 'w') as f:
+            f.write(output)
+
+        return ret