From 07991bb7ba26679117a91c67ab5e804d7717e81d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 May 2020 16:08:33 -0600 Subject: [PATCH] Fix Python 3 error in scripts/common.py, raise proper exceptions when decoding non-ascii files in scripts/metadata_parser.py, parse all suite definition files if suites argument is not present when calling ccpp_prebuild.py --- scripts/ccpp_prebuild.py | 28 ++++++++++++++++++++++++---- scripts/common.py | 13 ++++++++----- scripts/metadata_parser.py | 10 ++++++++-- 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/scripts/ccpp_prebuild.py b/scripts/ccpp_prebuild.py index abe81c6a..1e3dc561 100755 --- a/scripts/ccpp_prebuild.py +++ b/scripts/ccpp_prebuild.py @@ -15,6 +15,7 @@ from common import encode_container, decode_container, decode_container_as_dict, execute from common import CCPP_INTERNAL_VARIABLES, CCPP_STATIC_API_MODULE, CCPP_INTERNAL_VARIABLE_DEFINITON_FILE from common import STANDARD_VARIABLE_TYPES, STANDARD_INTEGER_TYPE, CCPP_TYPE +from common import SUITE_DEFINITION_FILENAME_PATTERN from common import split_var_name_and_array_reference from metadata_parser import merge_dictionaries, parse_scheme_tables, parse_variable_tables from mkcap import CapsMakefile, CapsCMakefile, CapsSourcefile, \ @@ -48,10 +49,10 @@ def parse_arguments(): configfile = args.config clean = args.clean debug = args.debug - if not args.suites: - parser.print_help() - sys.exit(-1) - sdfs = [ 'suite_{0}.xml'.format(x) for x in args.suites.split(',')] + if args.suites: + sdfs = [ 'suite_{0}.xml'.format(x) for x in args.suites.split(',')] + else: + sdfs = None builddir = args.builddir return (success, configfile, clean, debug, sdfs, builddir) @@ -160,6 +161,19 @@ def clean_files(config): execute(cmd) return success +def get_all_suites(suites_dir): + success = False + logging.info("No suites were given, compiling a list of all suites") + sdfs = [] + for f in os.listdir(suites_dir): + match = SUITE_DEFINITION_FILENAME_PATTERN.match(f) + if match: + logging.info('Adding suite definition file {}'.format(f)) + sdfs.append(f) + if sdfs: + success = True + return (success, sdfs) + def parse_suites(suites_dir, sdfs): """Parse suite definition files for prebuild""" logging.info('Parsing suite definition files ...') @@ -695,6 +709,12 @@ def main(): logging.info('CCPP prebuild clean completed successfully, exiting.') sys.exit(0) + # If no suite definition files were given, get all of them + if not sdfs: + (success, sdfs) = get_all_suites(config['suites_dir']) + if not success: + raise Exception('Call to get_all_sdfs failed.') + # Parse suite definition files for prebuild (success, suites) = parse_suites(config['suites_dir'], sdfs) if not success: diff --git a/scripts/common.py b/scripts/common.py index 520e6c8c..801ecfcc 100755 --- a/scripts/common.py +++ b/scripts/common.py @@ -41,6 +41,9 @@ CCPP_STATIC_API_MODULE = 'ccpp_static_api' CCPP_STATIC_SUBROUTINE_NAME = 'ccpp_physics_{stage}' +# Filename pattern for suite definition files +SUITE_DEFINITION_FILENAME_PATTERN = re.compile('^suite_(.*)\.xml$') + def execute(cmd, abort = True): """Runs a local command in a shell. Waits for completion and returns status, stdout and stderr. If abort = True, abort in @@ -56,18 +59,18 @@ def execute(cmd, abort = True): status = p.returncode if debug: message = 'Execution of "{0}" returned with exit code {1}\n'.format(cmd, status) - message += ' stdout: "{0}"\n'.format(stdout.rstrip('\n')) - message += ' stderr: "{0}"'.format(stderr.rstrip('\n')) + message += ' stdout: "{0}"\n'.format(stdout.decode('ascii').rstrip('\n')) + message += ' stderr: "{0}"'.format(stderr.decode('ascii').rstrip('\n')) logging.debug(message) if not status == 0: message = 'Execution of command {0} failed, exit code {1}\n'.format(cmd, status) - message += ' stdout: "{0}"\n'.format(stdout.rstrip('\n')) - message += ' stderr: "{0}"'.format(stderr.rstrip('\n')) + message += ' stdout: "{0}"\n'.format(stdout.decode('ascii').rstrip('\n')) + message += ' stderr: "{0}"'.format(stderr.decode('ascii').rstrip('\n')) if abort: raise Exception(message) else: logging.error(message) - return (status, stdout.rstrip('\n'), stderr.rstrip('\n')) + return (status, stdout.decode('ascii').rstrip('\n'), stderr.decode('ascii').rstrip('\n')) def split_var_name_and_array_reference(var_name): """Split an expression like foo(:,a,1:ddt%ngas) diff --git a/scripts/metadata_parser.py b/scripts/metadata_parser.py index 5be32299..4f8077e2 100755 --- a/scripts/metadata_parser.py +++ b/scripts/metadata_parser.py @@ -157,7 +157,10 @@ def parse_variable_tables(filename): # Read all lines of the file at once with (open(filename, 'r')) as file: - file_lines = file.readlines() + try: + file_lines = file.readlines() + except UnicodeDecodeError: + raise Exception("Decoding error while trying to read file {}, check that the file only contains ASCII characters".format(filename)) lines = [] buffer = '' @@ -464,7 +467,10 @@ def parse_scheme_tables(filename): # Read all lines of the file at once with (open(filename, 'r')) as file: - file_lines = file.readlines() + try: + file_lines = file.readlines() + except UnicodeDecodeError: + raise Exception("Decoding error while trying to read file {}, check that the file only contains ASCII characters".format(filename)) lines = [] original_line_numbers = []