Merge pull request #230 from climbfuji/update_master_from_gmtb_develo…

…p_20191016

Update master from gmtb/develop 2019/10/16

scripts/ccpp_prebuild.py

@@ -3,18 +3,20 @@
 # Standard modules
 import argparse
 import collections
+import importlib
 import itertools
 import logging
 import os
 import re
 import sys
 
-# Local modules
+# CCPP framework imports
 from common import encode_container, decode_container, decode_container_as_dict, execute
-from common import CCPP_INTERNAL_VARIABLES, CCPP_STATIC_API_MODULE
+from common import CCPP_INTERNAL_VARIABLES, CCPP_STATIC_API_MODULE, CCPP_INTERNAL_VARIABLE_DEFINITON_FILE
 from common import split_var_name_and_array_reference
 from metadata_parser import merge_dictionaries, parse_scheme_tables, parse_variable_tables
-from mkcap import Cap, CapsMakefile, CapsCMakefile, SchemesMakefile, SchemesCMakefile
+from mkcap import Cap, CapsMakefile, CapsCMakefile, CapsSourcefile, \
+                  SchemesMakefile, SchemesCMakefile, SchemesSourcefile
 from mkdoc import metadata_to_html, metadata_to_latex
 from mkstatic import API, Suite, Group
 
@@ -28,19 +30,11 @@
 parser.add_argument('--debug',      action='store_true', help='enable debugging output', default=False)
 parser.add_argument('--static',     action='store_true', help='enable a static build for a given suite definition file', default=False)
 parser.add_argument('--suites',     action='store', help='suite definition files to use (comma-separated, for static build only, without path)', default='')
+parser.add_argument('--builddir',   action='store', help='relative path to CCPP build directory', required=False, default='.')
 
 # BASEDIR is the current directory where this script is executed
 BASEDIR = os.getcwd()
 
-# SCRIPTDIR is the directory where the ccpp_prebuild.py and its Python modules are located
-SCRIPTDIR = os.path.abspath(os.path.split(__file__)[0])
-
-# SRCDIR is the directory where the CCPP framework source code (C, Fortran) is located
-SRCDIR = os.path.abspath(os.path.join(SCRIPTDIR, '..', 'src'))
-
-# Definition of variables (metadata tables) that are provided by CCPP
-CCPP_INTERNAL_VARIABLE_DEFINITON_FILE = os.path.join(SRCDIR, 'ccpp_types.F90')
-
 ###############################################################################
 # Functions and subroutines                                                   #
 ###############################################################################
@@ -57,9 +51,10 @@ def parse_arguments():
         parser.print_help()
         sys.exit(-1)
     sdfs = [ 'suite_{0}.xml'.format(x) for x in args.suites.split(',')]
-    return (success, configfile, clean, debug, static, sdfs)
+    builddir = args.builddir
+    return (success, configfile, clean, debug, static, sdfs, builddir)
 
-def import_config(configfile):
+def import_config(configfile, builddir):
     """Import the configuration from a given configuration file"""
     success = True
     config = {}
@@ -71,35 +66,37 @@ def import_config(configfile):
 
     # Import the host-model specific CCPP prebuild config;
     # split into path and module name for import
-    configpath = os.path.abspath(os.path.split(configfile)[0])
-    configmodule = os.path.split(configfile)[1].rstrip('.py')
+    configpath = os.path.abspath(os.path.dirname(configfile))
+    configmodule = os.path.splitext(os.path.basename(configfile))[0]
     sys.path.append(configpath)
-    ccpp_prebuild_config = __import__(configmodule)
+    ccpp_prebuild_config = importlib.import_module(configmodule)
 
     # Definitions in host-model dependent CCPP prebuild config script
     config['variable_definition_files'] = ccpp_prebuild_config.VARIABLE_DEFINITION_FILES
     config['scheme_files']              = ccpp_prebuild_config.SCHEME_FILES
     config['scheme_files_dependencies'] = ccpp_prebuild_config.SCHEME_FILES_DEPENDENCIES
-    config['schemes_makefile']          = ccpp_prebuild_config.SCHEMES_MAKEFILE
-    config['schemes_cmakefile']         = ccpp_prebuild_config.SCHEMES_CMAKEFILE
+    config['schemes_makefile']          = ccpp_prebuild_config.SCHEMES_MAKEFILE.format(build_dir=builddir)
+    config['schemes_cmakefile']         = ccpp_prebuild_config.SCHEMES_CMAKEFILE.format(build_dir=builddir)
+    config['schemes_sourcefile']        = ccpp_prebuild_config.SCHEMES_SOURCEFILE.format(build_dir=builddir)
     config['target_files']              = ccpp_prebuild_config.TARGET_FILES
-    config['caps_makefile']             = ccpp_prebuild_config.CAPS_MAKEFILE
-    config['caps_cmakefile']            = ccpp_prebuild_config.CAPS_CMAKEFILE
-    config['caps_dir']                  = ccpp_prebuild_config.CAPS_DIR
+    config['caps_makefile']             = ccpp_prebuild_config.CAPS_MAKEFILE.format(build_dir=builddir)
+    config['caps_cmakefile']            = ccpp_prebuild_config.CAPS_CMAKEFILE.format(build_dir=builddir)
+    config['caps_sourcefile']           = ccpp_prebuild_config.CAPS_SOURCEFILE.format(build_dir=builddir)
+    config['caps_dir']                  = ccpp_prebuild_config.CAPS_DIR.format(build_dir=builddir)
     config['suites_dir']                = ccpp_prebuild_config.SUITES_DIR
     config['optional_arguments']        = ccpp_prebuild_config.OPTIONAL_ARGUMENTS
     config['module_include_file']       = ccpp_prebuild_config.MODULE_INCLUDE_FILE
     config['fields_include_file']       = ccpp_prebuild_config.FIELDS_INCLUDE_FILE
     config['host_model']                = ccpp_prebuild_config.HOST_MODEL_IDENTIFIER
-    config['html_vartable_file']        = ccpp_prebuild_config.HTML_VARTABLE_FILE
-    config['latex_vartable_file']       = ccpp_prebuild_config.LATEX_VARTABLE_FILE
-    # For static build: location of static API file
-    config['static_api_dir'] = ccpp_prebuild_config.STATIC_API_DIR
-
+    config['html_vartable_file']        = ccpp_prebuild_config.HTML_VARTABLE_FILE.format(build_dir=builddir)
+    config['latex_vartable_file']       = ccpp_prebuild_config.LATEX_VARTABLE_FILE.format(build_dir=builddir)
+    # For static build: location of static API file, and shell script to source
+    config['static_api_dir']            = ccpp_prebuild_config.STATIC_API_DIR.format(build_dir=builddir)
+    config['static_api_srcfile']        = ccpp_prebuild_config.STATIC_API_SRCFILE.format(build_dir=builddir)
     # Template code in host-model dependent CCPP prebuild config script
     config['ccpp_data_structure']            = ccpp_prebuild_config.CCPP_DATA_STRUCTURE
 
-    # Add model-intependent, CCPP-internal variable definition files
+    # Add model-independent, CCPP-internal variable definition files
     config['variable_definition_files'].append(CCPP_INTERNAL_VARIABLE_DEFINITON_FILE)
 
     # To handle new metadata: import DDT references (if exist)
@@ -134,8 +131,10 @@ def clean_files(config, static):
     files_to_remove = [
         config['schemes_makefile'],
         config['schemes_cmakefile'],
+        config['schemes_sourcefile'],
         config['caps_makefile'],
         config['caps_cmakefile'],
+        config['caps_sourcefile'],
         config['html_vartable_file'],
         config['latex_vartable_file'],
         ]
@@ -613,65 +612,76 @@ def generate_suite_and_group_caps(suites, metadata_request, metadata_define, arg
 def generate_static_api(suites, static_api_dir):
     """Generate API for static build for a given suite"""
     success = True
-    # Change to caps directory
+    # Change to caps directory, create if necessary
+    if not os.path.isdir(static_api_dir):
+        os.makedirs(static_api_dir)
     os.chdir(static_api_dir)
-    api = API(suites=suites)
+    api = API(suites=suites, directory=static_api_dir)
     logging.info('Generating static API {0} in {1} ...'.format(api.filename, static_api_dir))
     api.write()
     os.chdir(BASEDIR)
     return (success, api)
 
-def generate_schemes_makefile(schemes, schemes_makefile, schemes_cmakefile):
+def generate_schemes_makefile(schemes, schemes_makefile, schemes_cmakefile, schemes_sourcefile):
     """Generate makefile/cmakefile snippets for all schemes."""
     logging.info('Generating schemes makefile/cmakefile snippet ...')
     success = True
     makefile = SchemesMakefile()
     makefile.filename = schemes_makefile
     cmakefile = SchemesCMakefile()
     cmakefile.filename = schemes_cmakefile
+    sourcefile = SchemesSourcefile()
+    sourcefile.filename = schemes_sourcefile
     # Adjust relative file path to schemes from caps makefile
     schemes_with_path = []
+    schemes_with_abspath = []
     schemes_makefile_dir = os.path.split(os.path.abspath(schemes_makefile))[0]
     for scheme in schemes:
         (scheme_filepath, scheme_filename) = os.path.split(os.path.abspath(scheme))
         relative_path = './{0}'.format(os.path.relpath(scheme_filepath, schemes_makefile_dir))
         schemes_with_path.append(os.path.join(relative_path, scheme_filename))
-    makefile.write(schemes_with_path)
-    cmakefile.write(schemes_with_path)
-    logging.info('Added {0} schemes to {1} and {2}'.format(
-           len(schemes_with_path), makefile.filename, cmakefile.filename))
+        schemes_with_abspath.append(os.path.abspath(scheme))
+    makefile.write(schemes_with_abspath)
+    cmakefile.write(schemes_with_abspath)
+    sourcefile.write(schemes_with_abspath)
+    logging.info('Added {0} schemes to {1}, {2}, {3}'.format(
+           len(schemes_with_path), makefile.filename, cmakefile.filename, sourcefile.filename))
     return success
 
-def generate_caps_makefile(caps, caps_makefile, caps_cmakefile, caps_dir):
+def generate_caps_makefile(caps, caps_makefile, caps_cmakefile, caps_sourcefile, caps_dir):
     """Generate makefile/cmakefile snippets for all caps."""
     logging.info('Generating caps makefile/cmakefile snippet ...')
     success = True
     makefile = CapsMakefile()
     makefile.filename = caps_makefile
     cmakefile = CapsCMakefile()
     cmakefile.filename = caps_cmakefile
+    sourcefile = CapsSourcefile()
+    sourcefile.filename = caps_sourcefile
     # Adjust relative file path to schemes from caps makefile
     caps_makefile_dir = os.path.split(os.path.abspath(caps_makefile))[0]
     relative_path = './{0}'.format(os.path.relpath(caps_dir, caps_makefile_dir))
     caps_with_path = [ os.path.join(relative_path, cap) for cap in caps]
-    makefile.write(caps_with_path)
-    cmakefile.write(caps_with_path)
+    caps_with_abspath = [ os.path.abspath(os.path.join(caps_dir, cap)) for cap in caps]
+    makefile.write(caps_with_abspath)
+    cmakefile.write(caps_with_abspath)
+    sourcefile.write(caps_with_abspath)
     logging.info('Added {0} auto-generated caps to {1} and {2}'.format(
                           len(caps_with_path), makefile.filename, cmakefile.filename))
     return success
 
 def main():
     """Main routine that handles the CCPP prebuild for different host models."""
     # Parse command line arguments
-    (success, configfile, clean, debug, static, sdfs) = parse_arguments()
+    (success, configfile, clean, debug, static, sdfs, builddir) = parse_arguments()
     if not success:
         raise Exception('Call to parse_arguments failed.')
 
     success = setup_logging(debug)
     if not success:
         raise Exception('Call to setup_logging failed.')
 
-    (success, config) = import_config(configfile)
+    (success, config) = import_config(configfile, builddir)
     if not success:
         raise Exception('Call to import_config failed.')
 
@@ -765,7 +775,8 @@ def main():
 
     # Add filenames of schemes to makefile - add dependencies for schemes
     success = generate_schemes_makefile(config['scheme_files_dependencies'] + config['scheme_files'].keys(),
-                                                    config['schemes_makefile'], config['schemes_cmakefile'])
+                                        config['schemes_makefile'], config['schemes_cmakefile'],
+                                        config['schemes_sourcefile'])
     if not success:
         raise Exception('Call to generate_schemes_makefile failed.')
 
@@ -780,6 +791,9 @@ def main():
         if not success: 
             raise Exception('Call to generate_static_api failed.')
 
+        success = api.write_sourcefile(config['static_api_srcfile'])
+        if not success: 
+            raise Exception("Writing API sourcefile {sourcefile} failed".format(sourcefile=config['static_api_srcfile']))
     else:
         # Generate scheme caps for each individual scheme
         (success, scheme_caps) = generate_scheme_caps(metadata_define, metadata_request, arguments_request,
@@ -792,7 +806,8 @@ def main():
         all_caps = suite_and_group_caps
     else:
         all_caps = scheme_caps
-    success = generate_caps_makefile(all_caps, config['caps_makefile'], config['caps_cmakefile'], config['caps_dir'])
+    success = generate_caps_makefile(all_caps, config['caps_makefile'], config['caps_cmakefile'],
+                                     config['caps_sourcefile'], config['caps_dir'])
     if not success:
         raise Exception('Call to generate_caps_makefile failed.')
 

scripts/common.py

@@ -2,6 +2,7 @@
 
 import keyword
 import logging
+import os
 import re
 import subprocess
 import sys
@@ -14,6 +15,16 @@
 
 CCPP_TYPE = 'ccpp_t'
 
+# SCRIPTDIR is the directory where ccpp_prebuild.py and its Python modules are located
+SCRIPTDIR = os.path.abspath(os.path.dirname(__file__))
+
+# SRCDIR is the directory where the CCPP framework source code (C, Fortran) is located
+SRCDIR = os.path.abspath(os.path.join(SCRIPTDIR, os.pardir, 'src'))
+
+# Definition of variables (metadata tables) that are provided by CCPP
+CCPP_INTERNAL_VARIABLE_DEFINITON_FILE = os.path.join(SRCDIR, 'ccpp_types.F90')
+
+# List of internal variables provided by the CCPP
 CCPP_INTERNAL_VARIABLES = {
     CCPP_ERROR_FLAG_VARIABLE : 'cdata%errflg',
     CCPP_ERROR_MSG_VARIABLE  : 'cdata%errmsg',

scripts/metadata2html.py

@@ -2,30 +2,89 @@
 
 import argparse
 import logging
+import importlib
 import os
 import sys
 
 # CCPP framework imports
+from common import CCPP_INTERNAL_VARIABLE_DEFINITON_FILE
 from parse_tools import init_log, set_log_level
 from metadata_table import MetadataHeader
 
+###############################################################################
+# Set up the command line argument parser and other global variables          #
+###############################################################################
+
 parser = argparse.ArgumentParser()
-parser.add_argument('--metafile', '-m', action='store',
-                    help='name of metadata file to convert',
-                    required=True)
+method = parser.add_mutually_exclusive_group(required=True)
+method.add_argument('--config', '-c', action='store',
+                    help='path to CCPP prebuild configuration file')
+method.add_argument('--metafile', '-m', action='store',
+                    help='name of metadata file to convert (requires -o)')
 parser.add_argument('--outputdir', '-o', action='store',
                     help='directory where to write the html files',
-                    required=True)
+                    required='--metafile' in sys.argv or '-m' in sys.argv)
 
-attributes = [ 'local_name', 'standard_name', 'long_name', 'units',
+# List and order of variable attributes to output to HTML
+ATTRIBUTES = [ 'local_name', 'standard_name', 'long_name', 'units',
                'type', 'dimensions', 'kind', 'intent', 'optional' ]
 
+###############################################################################
+# Functions and subroutines                                                   #
+###############################################################################
+
 def parse_arguments():
     """Parse command line arguments."""
     args = parser.parse_args()
+    config = args.config
     filename = args.metafile
     outdir = args.outputdir
-    return (filename, outdir)
+    return (config, filename, outdir)
+
+def import_config(configfile, logger):
+    """Import the configuration from a given configuration file"""
+
+    if not os.path.isfile(configfile):
+        raise Exception("Configuration file {0} not found".format(configfile))
+
+    # Import the host-model specific CCPP prebuild config;
+    # split into path and module name for import
+    configpath = os.path.abspath(os.path.dirname(configfile))
+    configmodule = os.path.splitext(os.path.basename(configfile))[0]
+    sys.path.append(configpath)
+    ccpp_prebuild_config = importlib.import_module(configmodule)
+
+    config = {}
+    # Definitions in host-model dependent CCPP prebuild config script
+    config['variable_definition_files'] = ccpp_prebuild_config.VARIABLE_DEFINITION_FILES
+    config['scheme_files']              = ccpp_prebuild_config.SCHEME_FILES
+    # Add model-independent, CCPP-internal variable definition files
+    config['variable_definition_files'].append(CCPP_INTERNAL_VARIABLE_DEFINITON_FILE)
+    # Output directory for converted metadata tables
+    config['metadata_html_output_dir'] = ccpp_prebuild_config.METADATA_HTML_OUTPUT_DIR
+
+    return config
+
+def get_metadata_files_from_config(config, logger):
+    """Create a list of metadata filenames for a CCPP prebuild configuration"""
+    filenames = []
+    for sourcefile in config['variable_definition_files'] + config['scheme_files'].keys():
+        metafile = os.path.splitext(sourcefile)[0]+'.meta'
+        if os.path.isfile(metafile):
+            filenames.append(metafile)
+        else:
+            # DH* Warn for now, raise exception later when
+            # old metadata format is no longer supported
+            logger.warn("Metadata file {} for source file {} not found, assuming old metadata format".format(
+                                                                                       metafile, sourcefile))
+    return filenames
+
+def get_output_directory_from_config(config, logger):
+    """Return the html output directory for a CCPP prebuild configuration"""
+    outdir = config['metadata_html_output_dir']
+    if not os.path.isdir(outdir):
+        raise Exception("Output directory {} for converted metadata tables does not exist".format(outdir))
+    return outdir
 
 def convert_to_html(filename_in, outdir, logger):
     """Convert a metadata file into html (one html file for each table)"""
@@ -34,7 +93,7 @@ def convert_to_html(filename_in, outdir, logger):
     logger.info("Converting file {} to HTML".format(filename_in))
     metadata_headers = MetadataHeader.parse_metadata_file(filename_in)
     for metadata_header in metadata_headers:
-        filename_out = metadata_header.to_html(outdir, attributes)
+        filename_out = metadata_header.to_html(outdir, ATTRIBUTES)
         if filename_out:
             logger.info("  ... wrote {}".format(filename_out))
 
@@ -43,8 +102,15 @@ def main():
     logger = init_log('metadata2html')
     set_log_level(logger, logging.INFO)
     # Convert metadata file
-    (filename, outdir) = parse_arguments()
-    convert_to_html(filename, outdir, logger)
+    (configfile, filename, outdir) = parse_arguments()
+    if configfile:
+        config = import_config(configfile, logger)
+        filenames = get_metadata_files_from_config(config, logger)
+        outdir = get_output_directory_from_config(config, logger)
+        for filename in filenames:
+            convert_to_html(filename, outdir, logger)
+    else:
+        convert_to_html(filename, outdir, logger)
 
 if __name__ == '__main__':
     main()