From 3fdf0c48ad209897d97f81623d7651b098d538db Mon Sep 17 00:00:00 2001 From: Justin Bich Date: Sat, 3 Feb 2024 16:24:37 +0100 Subject: [PATCH] Update header manipulation --- utils/srcmanip/change_header_year | 103 ++++++++++++++++ utils/srcmanip/set_new_source_headers | 141 +++++++++++++++++++++ utils/srcmanip/set_source_headers | 171 -------------------------- 3 files changed, 244 insertions(+), 171 deletions(-) create mode 100755 utils/srcmanip/change_header_year create mode 100755 utils/srcmanip/set_new_source_headers delete mode 100755 utils/srcmanip/set_source_headers diff --git a/utils/srcmanip/change_header_year b/utils/srcmanip/change_header_year new file mode 100755 index 0000000000..056d9f811d --- /dev/null +++ b/utils/srcmanip/change_header_year @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +import os +import re +import fnmatch +import argparse + + +DESCRIPTION = "Replaces the year in files with existing copyright header." + +YEAR = "2024" + +PATTERN = re.compile( + r"""Copyright\s+\(C\)\s+(\d{4}\s*-\s*)*(?P\d{4})\s+DFTB\+\s+developers\s+group""" +) + +FOLDERS = ["app/", "cmake/", "src/", "sys/", "test/", "tools/", "utils/"] + +FILES = ["LICENSE"] + +CHANGE_MANUALLY = ["src/dftbp/common/release.F90", "doc/dptools/api/conf.py"] + +IGNORED_FOLDERS = ["__pycache__"] + +IGNORED_FILES = [ + "*~", + "*.pyc", + "*/make.deps", + "*/make.extdeps", + "*.mgf", + "*.egg", + "*.skf", +] + + +def main(): + """Main function""" + args = parse_arguments() + rootdir = os.path.abspath(args.rootdir) + process_files(rootdir) + print_files_manually(rootdir) + + +def parse_arguments(): + """Parses the arguments""" + parser = argparse.ArgumentParser(description=DESCRIPTION) + msg = "Root directory of the project (default: current directory)" + parser.add_argument("--rootdir", default=os.getcwd(), help=msg) + args = parser.parse_args() + return args + + +def process_files(rootdir): + """Replaces the year in header""" + file_list = find_files(rootdir) + number_files = len(file_list) + indent = len(str(number_files)) + 1 + for num, file in enumerate(file_list): + changed_date = False + with open(file, "r", encoding="utf-8") as fp: + newcontent = "" + for line in fp.readlines(): + hit = PATTERN.search(line) + if hit: + changed_date = True + start, stop = hit.span("year") + newcontent += line[:start] + YEAR + line[stop:] + else: + newcontent += line + with open(file, "w", encoding="utf-8") as fp: + fp.write(newcontent) + if changed_date: + print(f"{num + 1:>{indent}} of {number_files}: ", file, " CHANGED") + else: + print(f"{num + 1:>{indent}} of {number_files}: ", file, " UNCHANGED") + + +def find_files(rootdir): + """Walks through 'FOLDERS' and 'FILES' in rootdir and adds them to + 'file_list'""" + file_list = [] + for file in FILES: + file_list.append(os.path.join(rootdir, file)) + for folder in FOLDERS: + for root, dirs, files in os.walk(os.path.join(rootdir, folder)): + dirs[:] = [d for d in dirs if d not in IGNORED_FOLDERS] + for file in files: + for ignored in IGNORED_FILES: + if fnmatch.fnmatch(file, ignored): + break + else: + file_list.append(os.path.join(root, file)) + return file_list + + +def print_files_manually(rootdir): + """Prints the files that needs to be changed manually""" + print("Please change the year in the following files manually:") + for file in CHANGE_MANUALLY: + print(os.path.join(rootdir, file)) + + +if __name__ == "__main__": + main() diff --git a/utils/srcmanip/set_new_source_headers b/utils/srcmanip/set_new_source_headers new file mode 100755 index 0000000000..3e871087b0 --- /dev/null +++ b/utils/srcmanip/set_new_source_headers @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +import sys +import os +import re +import fnmatch +import argparse + +if sys.hexversion < 0x03050000: + sys.stderr.write("Error: this script requires Python 3.5 or newer") + sys.exit(1) + + +DESCRIPTION = "Creates the copyright header in all relevant source files." + +COPYRIGHT = """DFTB+: general package for performing fast atomistic simulations +Copyright (C) 2006 - 2024 DFTB+ developers group + +See the LICENSE file for terms of usage and distribution.""" + + +FOLDERS = ["app/", "cmake/", "src/", "sys/", "test/", "tools/", "utils/"] + +FILE_TYPES = [ + ("*.[fF]90", "fortran"), + ("*.[hc]", "c"), + ("*.fpp", "fortran"), + ("*.inc", "fortran"), + ("*.fypp", "fypp"), + ("make.*", "makefile"), + ("makefile", "makefile"), + ("*.sh", "shell"), + ("*.py", "python"), +] + + +def main(): + """Main function""" + args = parse_arguments() + process_files(os.path.abspath(args.rootdir)) + + +def parse_arguments(): + """Parses the arguments""" + parser = argparse.ArgumentParser(description=DESCRIPTION) + msg = "Root directory of the project (default: current directory)" + parser.add_argument("--rootdir", default=os.getcwd(), help=msg) + args = parser.parse_args() + return args + + +def find_files(rootdir): + """Walks through 'FOLDERS' and 'FILE_TYPES' in rootdir and adds them to + 'file_list'""" + file_list = [] + for folder in FOLDERS: + for root, _, files in os.walk(os.path.join(rootdir, folder)): + for file in files: + for fpattern, ftype in FILE_TYPES: + if fnmatch.fnmatch(file, fpattern): + file_list.append((os.path.join(root, file), ftype)) + return file_list + + +def process_files(rootdir): + """Iterates over files and inserts headers accordingly""" + file_list = find_files(rootdir) + for file, ftype in file_list: + headerpattern, newheader = HEADER_BY_TYPE[ftype] + set_header(file, headerpattern, newheader) + + +def set_header(fname, headerpattern, header): + """Adds headers to files that do not contain headers""" + with open(fname, "r", encoding="utf-8") as fp: + txt = fp.readlines() + first_line = txt[0] + hit = headerpattern.search("".join(txt)) + if not hit: + if SHEBANG.match(first_line) is None: + newtxt = header + "\n\n" + first_line + "".join(txt[1:]) + else: + newtxt = first_line + header + "\n\n" + "".join(txt[1:]) + with open(fname, "w", encoding="utf-8") as fp: + fp.write(newtxt) + print("Header added in: ", fname) + + +def pretty_header(txt, commentbegin, commentend, marker, linelength): + """Creates 'pretty header'""" + commentlen = len(commentbegin) + len(commentend) + headerseparator = commentbegin + marker * (linelength - commentlen) + commentend + headerlines = [headerseparator] + for line in txt.split("\n"): + line = line.strip() + padding = linelength - commentlen - 2 - len(line) + headerlines.append(commentbegin + " " + line + " " * padding + commentend) + headerlines.append(headerseparator) + return "\n".join(headerlines) + + +RAWTEXT_HEADER_PATTERN = re.compile( + r"[ \t]*\|---+\|[ \t]*\n(?:[ \t]*\|.*\n)*?[ \t]*\|---+\|[ \t]*$", re.MULTILINE +) +RAWTEXT_HEADER = pretty_header(COPYRIGHT, "!", "!", "-", 80) + +FORTRAN_HEADER_PATTERN = re.compile( + r"[ \t]*!---+![ \t]*\n(?:[ \t]*!.*\n)*?[ \t]*!---+![ \t]*$", re.MULTILINE +) +FORTRAN_HEADER = pretty_header(COPYRIGHT, "!", "!", "-", 100) + +SCRIPT_HEADER_PATTERN = re.compile( + r"[ \t]*#---+#[ \t]*\n(?:[ \t]*#.*\n)*?[ \t]*#---+#[ \t]*$", re.MULTILINE +) +SCRIPT_HEADER = pretty_header(COPYRIGHT, "#", "#", "-", 80) + +FYPP_HEADER_PATTERN = re.compile( + r"[ \t]*#!---+![ \t]*\n(?:[ \t]*#!.*\n)*?[ \t]*#!---+![ \t]*$", re.MULTILINE +) +FYPP_HEADER = pretty_header(COPYRIGHT, "#!", "!", "-", 100) + +C_HEADER_PATTERN = re.compile( + r"[ \t]*/\*---+\*/[ \t]*\n(?:[ \t]*/\*.*\n)*?[ \t]*/\*---+\*/[ \t]*$", re.MULTILINE +) +C_HEADER = pretty_header(COPYRIGHT, "/*", "*/", "-", 100) + +SHEBANG = re.compile(r"^#!(.*)") + + +HEADER_BY_TYPE = { + "fortran": (FORTRAN_HEADER_PATTERN, FORTRAN_HEADER), + "makefile": (SCRIPT_HEADER_PATTERN, SCRIPT_HEADER), + "python": (SCRIPT_HEADER_PATTERN, SCRIPT_HEADER), + "rawtext": (RAWTEXT_HEADER_PATTERN, RAWTEXT_HEADER), + "shell": (SCRIPT_HEADER_PATTERN, SCRIPT_HEADER), + "fypp": (FYPP_HEADER_PATTERN, FYPP_HEADER), + "c": (C_HEADER_PATTERN, C_HEADER), +} + + +if __name__ == "__main__": + main() diff --git a/utils/srcmanip/set_source_headers b/utils/srcmanip/set_source_headers deleted file mode 100755 index 1ab3c10fd2..0000000000 --- a/utils/srcmanip/set_source_headers +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python3 -import sys -import os -import re -import glob -import fnmatch -import argparse - -if sys.hexversion < 0x03050000: - sys.stderr.write('Error: this script requires Python 3.5 or newer') - sys.exit(1) - - -DESCRIPTION = 'Replaces the copyright header in all relevant source files.' - -COPYRIGHT = '''DFTB+: general package for performing fast atomistic simulations -Copyright (C) 2006 - 2023 DFTB+ developers group - -See the LICENSE file for terms of usage and distribution.''' - -# File patterns to ignore -IGNORED_FILES = [ - '*~', '*.pyc', '*/make.deps', '*/make.extdeps' -] - -# Order matters: File are processed according the first matching glob -# and ignored in any other matching globs afterwards. If type is None, -# the file is skipped, but for performance resons you should use the -# IGNORED_FILES list to indicate ignored files, whenever possible. -FILE_TYPES = [ - ('api/mm/*.[fF]90', 'fortran'), - ('api/mm/*.[hc]', 'c'), - ('makefile', 'makefile'), - ('app/**/*.[fF]90', 'fortran'), - ('app/**/*.fpp', 'fortran'), - ('app/**/*.inc', 'fortran'), - ('app/**/*.h', 'c'), - ('app/**/*.fypp', 'fypp'), - ('app/**/make.*', 'makefile'), - ('src/**/*.fpp', 'fortran'), - ('src/**/*.[fF]90', 'fortran'), - ('src/**/*.inc', 'fortran'), - ('src/**/*.h', 'c'), - ('src/**/*.fypp', 'fypp'), - ('src/**/make.*', 'makefile'), - ('test/app/dftb+/*/*.sh', 'shell'), - ('test/app/dftb+/*/*/*.sh', 'shell'), - ('test/app/dftb+/bin/*', 'shell'), - ('test/app/modes+/bin/*', 'shell'), - ('test/app/phonons/bin/*', 'shell'), - ('tools/misc/*', 'python'), - ('tools/dptools/bin/*', 'python'), - ('tools/dptools/src/**/*.py', 'python'), - ('tools/pythonapi/src/**/*.py', 'python'), - ('test/tools/dptools/**/*.py', 'python'), - ('test/api/mm/testers/*.[fF]90', 'fortran'), - ('test/api/mm/testers/*.[hc]', 'c'), - ('test/api/pyapi/**/*.py', 'python'), - ('test/unittests/**/*.[fF]90', 'fortran'), - ('utils/*', 'python'), - ('utils/build/*.sh', 'shell'), - ('utils/build/*', 'python'), - ('utils/build/cr_makedep/*', 'python'), - ('utils/build/fpp/*', 'shell'), -] - - -def main(): - args = parse_arguments() - process_files(os.path.abspath(args.rootdir)) - - -def parse_arguments(): - parser = argparse.ArgumentParser(description=DESCRIPTION) - msg = 'Root directory of the project (default: current directory)' - parser.add_argument('--rootdir', default=os.getcwd(), help=msg) - args = parser.parse_args() - return args - - -def process_files(rootdir): - processed = set() - for fpattern, ftype in FILE_TYPES: - headerpattern, newheader = HEADER_BY_TYPE[ftype] - paths = glob.glob(os.path.join(rootdir, fpattern), recursive=True) - for path in paths: - if os.path.isfile(path) and path not in processed: - processed.add(path) - if headerpattern is None: - continue - for ignored in IGNORED_FILES: - if fnmatch.fnmatch(path, ignored): - break - else: - print(path) - replace_header(str(path), headerpattern, newheader) - - -def replace_header(fname, headerpattern, header): - with open(fname, 'r') as fp: - txt = fp.readlines() - first_line = txt[0] - - newtxt, nsub = headerpattern.subn(header, "".join(txt)) - if nsub: - print("{:d} substitutions in {}".format(nsub, fname)) - else: - print("HEADER ADDED in {}".format(fname)) - if not nsub: - if SHEBANG.match(first_line) is None: - newtxt = header + '\n\n' + first_line + "".join(txt[1:]) - else: - newtxt = first_line + header + '\n\n' + "".join(txt[1:]) - with open(fname, 'w') as fp: - fp.write(newtxt) - - -def pretty_header(txt, commentbegin, commentend, marker, linelength): - commentlen = len(commentbegin) + len(commentend) - headerseparator = (commentbegin + marker * (linelength - commentlen) - + commentend) - headerlines = [headerseparator] - for line in txt.split('\n'): - line = line.strip() - padding = linelength - commentlen - 2 - len(line) - headerlines.append(commentbegin + ' ' + line + ' ' * padding - + commentend) - headerlines.append(headerseparator) - return '\n'.join(headerlines) - - -RAWTEXT_HEADER_PATTERN = re.compile( - r'[ \t]*\|---+\|[ \t]*\n(?:[ \t]*\|.*\n)*?[ \t]*\|---+\|[ \t]*$', - re.MULTILINE) -RAWTEXT_HEADER = pretty_header(COPYRIGHT, '!', '!', '-', 80) - -FORTRAN_HEADER_PATTERN = re.compile( - r'[ \t]*!---+![ \t]*\n(?:[ \t]*!.*\n)*?[ \t]*!---+![ \t]*$', re.MULTILINE) -FORTRAN_HEADER = pretty_header(COPYRIGHT, '!', '!', '-', 100) - -SCRIPT_HEADER_PATTERN = re.compile( - r'[ \t]*#---+#[ \t]*\n(?:[ \t]*#.*\n)*?[ \t]*#---+#[ \t]*$', re.MULTILINE) -SCRIPT_HEADER = pretty_header(COPYRIGHT, '#', '#', '-', 80) - -FYPP_HEADER_PATTERN = re.compile( - r'[ \t]*#!---+![ \t]*\n(?:[ \t]*#!.*\n)*?[ \t]*#!---+![ \t]*$', - re.MULTILINE) -FYPP_HEADER = pretty_header(COPYRIGHT, '#!', '!', '-', 100) - -C_HEADER_PATTERN = re.compile( - r'[ \t]*/\*---+\*/[ \t]*\n(?:[ \t]*/\*.*\n)*?[ \t]*/\*---+\*/[ \t]*$', - re.MULTILINE) -C_HEADER = pretty_header(COPYRIGHT, '/*', '*/', '-', 100) - -SHEBANG = re.compile(r'^#!(.*)') - - -HEADER_BY_TYPE = { - None: (None, None), - 'fortran': (FORTRAN_HEADER_PATTERN, FORTRAN_HEADER), - 'makefile': (SCRIPT_HEADER_PATTERN, SCRIPT_HEADER), - 'python': (SCRIPT_HEADER_PATTERN, SCRIPT_HEADER), - 'rawtext': (RAWTEXT_HEADER_PATTERN, RAWTEXT_HEADER), - 'shell': (SCRIPT_HEADER_PATTERN, SCRIPT_HEADER), - 'fypp': (FYPP_HEADER_PATTERN, FYPP_HEADER), - 'c': (C_HEADER_PATTERN, C_HEADER), -} - - -if __name__ == '__main__': - main()