From 8e7caae6b0383a3d274b3edd079145dd6adfe6b1 Mon Sep 17 00:00:00 2001 From: Justin Bich Date: Sun, 11 Feb 2024 16:29:11 +0100 Subject: [PATCH] Improve import manipulation of Fortran files --- utils/srcmanip/check_import_usage | 101 +++++++++++++++++++++++++++++ utils/srcmanip/sort_modules | 102 +++++++++++++++++++++++++----- 2 files changed, 187 insertions(+), 16 deletions(-) create mode 100755 utils/srcmanip/check_import_usage diff --git a/utils/srcmanip/check_import_usage b/utils/srcmanip/check_import_usage new file mode 100755 index 0000000000..0d93a04e75 --- /dev/null +++ b/utils/srcmanip/check_import_usage @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +"""Simple script to ckeck usage of imports in Fortran files""" + +import argparse +import fnmatch +import os +import re + + +_DESCRIPTION = "Checks the imports in DFTB+ Fortran file(s) for usage" + +_PAT_USE_MODULE = re.compile( + r"""^(?P[ ]*)use + (?P(?:\s*,\s*intrinsic)?) + (?P[ ]*::[ ]*|[ ]*) + (?P\w+) + (?P.*?(?:&[ ]*\n(?:[ ]*&)?.*?)*)\n + """, + re.VERBOSE | re.MULTILINE, +) + +WORD_PATTERN = re.compile(r"""\w+""") + + +def main(): + """Main script driver.""" + args = parse_arguments() + filenames = [] + if args.folders: + filenames = get_files(args.folders) + if args.files: + filenames += args.files + for fname in filenames: + with open(fname, "r", encoding="utf-8") as file: + file_txt = file.read() + matches = list(_PAT_USE_MODULE.finditer(file_txt)) + matches.reverse() + rest_list = [] + for match in matches: + if match["rest"]: + rest_list.append(match["rest"]) + file_txt = file_txt[: match.start()] + file_txt[match.end() :] + word_set = set( + match.group().lower() for match in WORD_PATTERN.finditer(file_txt) + ) + import_set = get_import_set(rest_list) + unused_imports = import_set - word_set + if unused_imports: + print( + f"{fname} contains the following unsued imports: ", unused_imports + ) + else: + print(f"{fname} OK") + + +def parse_arguments(): + """Parses the command line arguments""" + parser = argparse.ArgumentParser(description=_DESCRIPTION) + msg = "File to process" + parser.add_argument("--files", nargs="+", metavar="FILE", help=msg) + msg = "Folder to process" + parser.add_argument("--folders", nargs="+", metavar="FOLDER", help=msg) + args = parser.parse_args() + if not (args.folders or args.files): + parser.error("No Files/Folders specified, add '--files' or '--folders'") + return args + + +def get_files(folders): + """Find all '*F90' or '*f90' files in folders""" + file_list = [] + for folder in folders: + for root, _, files in os.walk(folder): + for file in files: + if fnmatch.fnmatch(file, "*.[fF]90"): + file_list.append(os.path.join(root, file)) + return file_list + + +def get_import_set(rest_list): + """Creates set of imported methods and functions""" + import_set = set() + for rest in rest_list: + _, imports = re.split(r"only\s*\:", rest, 1) + imports_list = imports.split(",") + for imp in imports_list: + imp = re.sub(r"\&\s*&", "", imp).strip() + if "operator" in imp.lower(): + continue + if "assignment" in imp.lower(): + continue + if "=>" in imp: + import_set.add(imp.split("=>")[0].lower()) + else: + import_set.add(imp.lower()) + return import_set + + +if __name__ == "__main__": + main() diff --git a/utils/srcmanip/sort_modules b/utils/srcmanip/sort_modules index c9ef7c1863..03705bf03c 100755 --- a/utils/srcmanip/sort_modules +++ b/utils/srcmanip/sort_modules @@ -3,29 +3,41 @@ """Simple script to sort module imports in Fortran files""" import argparse +import fnmatch +import os import re import sys + from typing import Dict, List, Tuple -_DESCRIPTION = "Sorts the 'use' statements in DFTB+ Fortran file" +_DESCRIPTION = "Sorts the 'use' statements in DFTB+ Fortran file(s)" _PAT_USE_MODULE = re.compile( r"""^(?P[ ]*)use (?P(?:\s*,\s*intrinsic)?) - (?P[ ]*:: |[ ]+) + (?P[ ]*::[ ]*|[ ]*) (?P\w+) (?P.*?(?:&[ ]*\n(?:[ ]*&)?.*?)*)\n - """, re.VERBOSE | re.MULTILINE) + """, + re.VERBOSE | re.MULTILINE, +) def main(): """Main script driver.""" args = _parse_arguments() - for fname in args.filenames: - txt = open(fname, "r").read() - blocks, output = _process_file_content(txt, fname) - open(fname, "w").write("\n".join(output) + "\n") + filenames = [] + if args.folders: + filenames = _get_files(args.folders) + if args.files: + filenames += args.files + for fname in filenames: + with open(fname, "r", encoding="utf-8") as file: + txt = file.read() + blocks, output = _process_file_content(txt, fname) + with open(fname, "w", encoding="utf-8") as file: + file.write("\n".join(output) + "\n") if blocks > 1: print(f"{fname}: multiple blocks found!") @@ -35,17 +47,34 @@ def _parse_arguments() -> argparse.Namespace: parser = argparse.ArgumentParser(description=_DESCRIPTION) msg = "File to process" - parser.add_argument("filenames", nargs="+", metavar="FILE", help=msg) + parser.add_argument("--files", nargs="+", metavar="FILE", help=msg) + msg = "Folder to process" + parser.add_argument("--folders", nargs="+", metavar="FOLDER", help=msg) args = parser.parse_args() + if not (args.folders or args.files): + parser.error("No Files/Folders specified, add '--files' or '--folders'") return args +def _get_files(folders: List[str]) -> List[str]: + """Find all '*F90' or '*f90' files in folders""" + + file_list = [] + for folder in folders: + for root, _, files in os.walk(folder): + for file in files: + if fnmatch.fnmatch(file, "*.[fF]90"): + file_list.append(os.path.join(root, file)) + return file_list + + def _process_file_content(txt: str, fname: str) -> Tuple[int, List[str]]: """Processes the content of a file.""" output = [] - matches = [(match.group('name').lower(), match) - for match in _PAT_USE_MODULE.finditer(txt)] + matches = [ + (match.group("name").lower(), match) for match in _PAT_USE_MODULE.finditer(txt) + ] lastpos = 0 buffer = {} blocks = 0 @@ -63,7 +92,7 @@ def _process_file_content(txt: str, fname: str) -> Tuple[int, List[str]]: if buffer: output += _get_sorted_modules(buffer) blocks += 1 - output.append(txt[lastpos : ].rstrip()) + output.append(txt[lastpos:].rstrip()) return blocks, output @@ -74,9 +103,9 @@ def _get_sorted_modules(modules: Dict[str, re.Match]) -> List[str]: third_party_modules = [] dftbplus_modules = [] for name in modules: - if name.startswith('dftbplus_'): + if name.startswith("dftbp_"): dftbplus_modules.append(name) - elif name.startswith('iso_') or name == 'mpi': + elif name.startswith("iso_") or name == "mpi": intrinsic_modules.append(name) else: third_party_modules.append(name) @@ -87,12 +116,53 @@ def _get_sorted_modules(modules: Dict[str, re.Match]) -> List[str]: output = [] for name in intrinsic_modules + third_party_modules + dftbplus_modules: fields = modules[name] - output.append(f"{fields['indent']}use{fields['attrib']}" \ - f"{fields['separator']}{fields['name'].lower()}" \ - f"{fields['rest']}") + if fields["rest"]: + output.append(_sort_rest(modules[name])) + else: + output.append( + f"{fields['indent']}use{fields['attrib']}" + f"{fields['separator']}{fields['name'].lower()}" + ) return output +def _sort_rest(fields: re.Match) -> str: + """Sorts imported functions and methods in 'fields['rest']'""" + + _, imports = re.split(r"only\s*\:", fields["rest"], 1) + imports_list = imports.split(",") + imports_dict = {} + for imp in imports_list: + value_key = re.sub(r"\&\s*&", "", imp).strip() + try: + value, key = value_key.split("=>") + imports_dict[key.strip()] = value.strip() + except ValueError: + imports_dict[value_key.strip()] = None + sorted_imports = sorted(imports_dict.items()) + + output = ( + f"{fields['indent']}use{fields['attrib']}" + f"{fields['separator']}{fields['name'].lower()}" + ", only :" + ) + current_lenght = len(output) + + for key, item in sorted_imports: + if item is not None: + imp = f" {item} => {key}," + else: + imp = f" {key}," + if (current_lenght + len(imp)) <= 100: + current_lenght += len(imp) + output += imp + else: + output += "&\n" + imp = fields["indent"] + " " * 4 + f"&{imp}" + current_lenght = len(imp) + output += imp + return output[:-1] + + def _fatal_error(msg: str): """Prints an error message and stops"""