Merge pull request #292 from climbfuji/python3_decoding_parse_all_suites

Fix/report Python3 decoding errors, parse all suites if --suites=... argument not present

scripts/ccpp_prebuild.py

@@ -15,6 +15,7 @@
 from common import encode_container, decode_container, decode_container_as_dict, execute
 from common import CCPP_INTERNAL_VARIABLES, CCPP_STATIC_API_MODULE, CCPP_INTERNAL_VARIABLE_DEFINITON_FILE
 from common import STANDARD_VARIABLE_TYPES, STANDARD_INTEGER_TYPE, CCPP_TYPE
+from common import SUITE_DEFINITION_FILENAME_PATTERN
 from common import split_var_name_and_array_reference
 from metadata_parser import merge_dictionaries, parse_scheme_tables, parse_variable_tables
 from mkcap import CapsMakefile, CapsCMakefile, CapsSourcefile, \
@@ -48,10 +49,10 @@ def parse_arguments():
     configfile = args.config
     clean = args.clean
     debug = args.debug
-    if not args.suites:
-        parser.print_help()
-        sys.exit(-1)
-    sdfs = [ 'suite_{0}.xml'.format(x) for x in args.suites.split(',')]
+    if args.suites:
+        sdfs = [ 'suite_{0}.xml'.format(x) for x in args.suites.split(',')]
+    else:
+        sdfs = None
     builddir = args.builddir
     return (success, configfile, clean, debug, sdfs, builddir)
 
@@ -160,6 +161,19 @@ def clean_files(config):
     execute(cmd)
     return success
 
+def get_all_suites(suites_dir):
+    success = False
+    logging.info("No suites were given, compiling a list of all suites")
+    sdfs = []
+    for f in os.listdir(suites_dir):
+        match = SUITE_DEFINITION_FILENAME_PATTERN.match(f)
+        if match:
+            logging.info('Adding suite definition file {}'.format(f))
+            sdfs.append(f)
+    if sdfs:
+        success = True
+    return (success, sdfs)
+
 def parse_suites(suites_dir, sdfs):
     """Parse suite definition files for prebuild"""
     logging.info('Parsing suite definition files ...')
@@ -695,6 +709,12 @@ def main():
         logging.info('CCPP prebuild clean completed successfully, exiting.')
         sys.exit(0)
 
+    # If no suite definition files were given, get all of them
+    if not sdfs:
+        (success, sdfs) = get_all_suites(config['suites_dir'])
+        if not success:
+            raise Exception('Call to get_all_sdfs failed.')
+
     # Parse suite definition files for prebuild
     (success, suites) = parse_suites(config['suites_dir'], sdfs)
     if not success:

scripts/common.py

@@ -41,6 +41,9 @@
 CCPP_STATIC_API_MODULE = 'ccpp_static_api'
 CCPP_STATIC_SUBROUTINE_NAME = 'ccpp_physics_{stage}'
 
+# Filename pattern for suite definition files
+SUITE_DEFINITION_FILENAME_PATTERN = re.compile('^suite_(.*)\.xml$')
+
 def execute(cmd, abort = True):
     """Runs a local command in a shell. Waits for completion and
     returns status, stdout and stderr. If abort = True, abort in
@@ -56,18 +59,18 @@ def execute(cmd, abort = True):
     status = p.returncode
     if debug:
         message = 'Execution of "{0}" returned with exit code {1}\n'.format(cmd, status)
-        message += '    stdout: "{0}"\n'.format(stdout.rstrip('\n'))
-        message += '    stderr: "{0}"'.format(stderr.rstrip('\n'))
+        message += '    stdout: "{0}"\n'.format(stdout.decode('ascii').rstrip('\n'))
+        message += '    stderr: "{0}"'.format(stderr.decode('ascii').rstrip('\n'))
         logging.debug(message)
     if not status == 0:
         message = 'Execution of command {0} failed, exit code {1}\n'.format(cmd, status)
-        message += '    stdout: "{0}"\n'.format(stdout.rstrip('\n'))
-        message += '    stderr: "{0}"'.format(stderr.rstrip('\n'))
+        message += '    stdout: "{0}"\n'.format(stdout.decode('ascii').rstrip('\n'))
+        message += '    stderr: "{0}"'.format(stderr.decode('ascii').rstrip('\n'))
         if abort:
             raise Exception(message)
         else:
             logging.error(message)
-    return (status, stdout.rstrip('\n'), stderr.rstrip('\n'))
+    return (status, stdout.decode('ascii').rstrip('\n'), stderr.decode('ascii').rstrip('\n'))
 
 def split_var_name_and_array_reference(var_name):
     """Split an expression like foo(:,a,1:ddt%ngas)

scripts/metadata_parser.py

@@ -157,7 +157,10 @@ def parse_variable_tables(filename):
 
     # Read all lines of the file at once
     with (open(filename, 'r')) as file:
-        file_lines = file.readlines()
+        try:
+            file_lines = file.readlines()
+        except UnicodeDecodeError:
+            raise Exception("Decoding error while trying to read file {}, check that the file only contains ASCII characters".format(filename))
 
     lines = []
     buffer = ''
@@ -464,7 +467,10 @@ def parse_scheme_tables(filename):
 
     # Read all lines of the file at once
     with (open(filename, 'r')) as file:
-        file_lines = file.readlines()
+        try:
+            file_lines = file.readlines()
+        except UnicodeDecodeError:
+            raise Exception("Decoding error while trying to read file {}, check that the file only contains ASCII characters".format(filename))
 
     lines = []
     original_line_numbers = []