From cdf55558c5c81132e07ebd56ccd579cbac6f967f Mon Sep 17 00:00:00 2001 From: hlky Date: Sun, 20 Oct 2024 10:35:15 +0100 Subject: [PATCH 1/4] Separator in regex --- utils/modular_model_converter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/modular_model_converter.py b/utils/modular_model_converter.py index c107a4831862..4cdaf7b0f438 100644 --- a/utils/modular_model_converter.py +++ b/utils/modular_model_converter.py @@ -1142,8 +1142,9 @@ def convert_modular_file(modular_file, old_model_name=None, new_model_name=None, for file, node in cst_transformers.files.items(): if node != {}: # Get relative path starting from src/transformers/ + sep = os.sep * 2 if os.sep == "\\" else os.sep relative_path = re.search( - rf"(src{os.sep}transformers{os.sep}.*|examples{os.sep}.*)", os.path.abspath(modular_file) + rf"(src{sep}transformers{sep}.*|examples{sep}.*)", os.path.abspath(modular_file) ).group(1) header = AUTO_GENERATED_MESSAGE.format( From 6eb5c38b967bbf2118688682c3c116d77eac5fc5 Mon Sep 17 00:00:00 2001 From: hlky Date: Sun, 20 Oct 2024 10:36:05 +0100 Subject: [PATCH 2/4] Standardize separator for relative path in auto generated message --- utils/modular_model_converter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/modular_model_converter.py b/utils/modular_model_converter.py index 4cdaf7b0f438..136ad54c763e 100644 --- a/utils/modular_model_converter.py +++ b/utils/modular_model_converter.py @@ -1146,6 +1146,8 @@ def convert_modular_file(modular_file, old_model_name=None, new_model_name=None, relative_path = re.search( rf"(src{sep}transformers{sep}.*|examples{sep}.*)", os.path.abspath(modular_file) ).group(1) + if os.sep == "\\": + relative_path = relative_path.replace(os.sep, "/") header = AUTO_GENERATED_MESSAGE.format( relative_path=relative_path, short_name=os.path.basename(relative_path) From dde4b9b6f37f19461e435e073450e740e65b9d07 Mon Sep 17 00:00:00 2001 From: hlky Date: Sun, 20 Oct 2024 10:36:22 +0100 Subject: [PATCH 3/4] open() encoding --- utils/modular_model_converter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/modular_model_converter.py b/utils/modular_model_converter.py index 136ad54c763e..558a8465e079 100644 --- a/utils/modular_model_converter.py +++ b/utils/modular_model_converter.py @@ -56,7 +56,7 @@ def get_module_source_from_name(module_name: str) -> str: if spec is None or spec.origin is None: return f"Module {module_name} not found" - with open(spec.origin, "r") as file: + with open(spec.origin, "r", encoding="utf-8") as file: source_code = file.read() return source_code @@ -1132,7 +1132,7 @@ def convert_modular_file(modular_file, old_model_name=None, new_model_name=None, if pattern is not None: model_name = pattern.groups()[0] # Parse the Python file - with open(modular_file, "r") as file: + with open(modular_file, "r", encoding="utf-8") as file: code = file.read() module = cst.parse_module(code) wrapper = MetadataWrapper(module) @@ -1167,7 +1167,7 @@ def save_modeling_file(modular_file, converted_file): [line for line in converted_file[file_type][0].strip().split("\n") if not line.strip().startswith("#")] ) if len(converted_file[file_type][0].strip()) > 0 and non_comment_lines > 0: - with open(modular_file.replace("modular_", f"{file_type}_"), "w") as f: + with open(modular_file.replace("modular_", f"{file_type}_"), "w", encoding="utf-8") as f: f.write(converted_file[file_type][0]) else: non_comment_lines = len( @@ -1175,7 +1175,7 @@ def save_modeling_file(modular_file, converted_file): ) if len(converted_file[file_type][1].strip()) > 0 and non_comment_lines > 0: logger.warning("The modeling code contains errors, it's written without formatting") - with open(modular_file.replace("modular_", f"{file_type}_"), "w") as f: + with open(modular_file.replace("modular_", f"{file_type}_"), "w", encoding="utf-8") as f: f.write(converted_file[file_type][1]) From c47af794326913b01d15e9d7a454edaf56a5ca57 Mon Sep 17 00:00:00 2001 From: hlky Date: Thu, 24 Oct 2024 17:37:52 +0100 Subject: [PATCH 4/4] Replace `\` on `os.path.abspath` --- utils/modular_model_converter.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/utils/modular_model_converter.py b/utils/modular_model_converter.py index 558a8465e079..bda143c25772 100644 --- a/utils/modular_model_converter.py +++ b/utils/modular_model_converter.py @@ -1142,12 +1142,9 @@ def convert_modular_file(modular_file, old_model_name=None, new_model_name=None, for file, node in cst_transformers.files.items(): if node != {}: # Get relative path starting from src/transformers/ - sep = os.sep * 2 if os.sep == "\\" else os.sep relative_path = re.search( - rf"(src{sep}transformers{sep}.*|examples{sep}.*)", os.path.abspath(modular_file) + r"(src/transformers/.*|examples/.*)", os.path.abspath(modular_file).replace("\\", "/") ).group(1) - if os.sep == "\\": - relative_path = relative_path.replace(os.sep, "/") header = AUTO_GENERATED_MESSAGE.format( relative_path=relative_path, short_name=os.path.basename(relative_path)