From 6c9a795c263d3b68b5d1598dfa484ce3e111e7da Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Thu, 18 Jan 2024 22:55:37 +0800 Subject: [PATCH 01/26] fix: support cross compiling for wasm with make generator --- pylib/gyp/generator/make.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pylib/gyp/generator/make.py b/pylib/gyp/generator/make.py index 1b997494..e6e95a37 100644 --- a/pylib/gyp/generator/make.py +++ b/pylib/gyp/generator/make.py @@ -859,7 +859,11 @@ def Write( self.output = self.ComputeMacBundleOutput(spec) self.output_binary = self.ComputeMacBundleBinaryOutput(spec) else: - self.output = self.output_binary = self.ComputeOutput(spec) + if self.flavor == "win": + # prevent from generating copy targets on Windows + self.output = self.output_binary = self.ComputeOutput(spec).replace('\\', '/') + else: + self.output = self.output_binary = self.ComputeOutput(spec) self.is_standalone_static_library = bool( spec.get("standalone_static_library", 0) @@ -2441,13 +2445,17 @@ def CalculateMakefilePath(build_file, base_name): flock_command = "flock" copy_archive_arguments = "-af" makedep_arguments = "-MMD" + + # some linkers don't support --start-group/--end-group (e.g. wasm-ld) + link_commands = LINK_COMMANDS_LINUX.replace(' -Wl,--start-group', '').replace(' -Wl,--end-group', '') if gyp.common.CrossCompileRequested() else LINK_COMMANDS_LINUX + header_params = { "default_target": default_target, "builddir": builddir_name, "default_configuration": default_configuration, "flock": flock_command, "flock_index": 1, - "link_commands": LINK_COMMANDS_LINUX, + "link_commands": link_commands, "extra_commands": "", "srcdir": srcdir, "copy_archive_args": copy_archive_arguments, @@ -2463,7 +2471,8 @@ def CalculateMakefilePath(build_file, base_name): "LINK.host": GetEnvironFallback(("LINK_host", "LINK"), "$(CXX.host)"), "PLI.host": GetEnvironFallback(("PLI_host", "PLI"), "pli"), } - if flavor == "mac": + # If cross-compiling, reserve linux link commands and do not use gyp-mac-tool + if flavor == "mac" and not gyp.common.CrossCompileRequested(): flock_command = "./gyp-mac-tool flock" header_params.update( { From 326246b749c5697d1a829c36c06e042cf2417450 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Fri, 19 Jan 2024 12:39:23 +0800 Subject: [PATCH 02/26] fix: lint --- pylib/gyp/generator/make.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pylib/gyp/generator/make.py b/pylib/gyp/generator/make.py index e6e95a37..3036a089 100644 --- a/pylib/gyp/generator/make.py +++ b/pylib/gyp/generator/make.py @@ -861,7 +861,9 @@ def Write( else: if self.flavor == "win": # prevent from generating copy targets on Windows - self.output = self.output_binary = self.ComputeOutput(spec).replace('\\', '/') + self.output = self.output_binary = self.ComputeOutput(spec).replace( + '\\', '/' + ) else: self.output = self.output_binary = self.ComputeOutput(spec) @@ -2447,7 +2449,9 @@ def CalculateMakefilePath(build_file, base_name): makedep_arguments = "-MMD" # some linkers don't support --start-group/--end-group (e.g. wasm-ld) - link_commands = LINK_COMMANDS_LINUX.replace(' -Wl,--start-group', '').replace(' -Wl,--end-group', '') if gyp.common.CrossCompileRequested() else LINK_COMMANDS_LINUX + link_commands = LINK_COMMANDS_LINUX.replace(' -Wl,--start-group', '').replace( + ' -Wl,--end-group', '' + ) if gyp.common.CrossCompileRequested() else LINK_COMMANDS_LINUX header_params = { "default_target": default_target, From 9bc78054b675b320bb980de14cabd0a2240d2ac9 Mon Sep 17 00:00:00 2001 From: Toyo Li Date: Fri, 19 Jan 2024 21:04:38 +0800 Subject: [PATCH 03/26] refactor for readability Co-authored-by: Christian Clauss --- pylib/gyp/generator/make.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pylib/gyp/generator/make.py b/pylib/gyp/generator/make.py index 3036a089..ba9be4ba 100644 --- a/pylib/gyp/generator/make.py +++ b/pylib/gyp/generator/make.py @@ -2449,9 +2449,11 @@ def CalculateMakefilePath(build_file, base_name): makedep_arguments = "-MMD" # some linkers don't support --start-group/--end-group (e.g. wasm-ld) - link_commands = LINK_COMMANDS_LINUX.replace(' -Wl,--start-group', '').replace( - ' -Wl,--end-group', '' - ) if gyp.common.CrossCompileRequested() else LINK_COMMANDS_LINUX + link_commands = LINK_COMMANDS_LINUX + if gyp.common.CrossCompileRequested(): + link_commands = link_commands.replace(' -Wl,--start-group', '').replace( + ' -Wl,--end-group', '' + ) header_params = { "default_target": default_target, From 882c1f9bdb3774dc686ba84ac047ab0abd64f70a Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Wed, 24 Jan 2024 00:09:08 +0800 Subject: [PATCH 04/26] replace separator in make generator on Windows --- pylib/gyp/generator/make.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pylib/gyp/generator/make.py b/pylib/gyp/generator/make.py index ba9be4ba..f0589363 100644 --- a/pylib/gyp/generator/make.py +++ b/pylib/gyp/generator/make.py @@ -25,6 +25,7 @@ import os import re import subprocess +import sys import gyp import gyp.common import gyp.xcode_emulation @@ -724,6 +725,10 @@ def QuoteIfNecessary(string): string = '"' + string.replace('"', '\\"') + '"' return string +def ReplaceSep(string): + if sys.platform == 'win32': + string = string.replace('\\', '/') + return string def StringToMakefileVariable(string): """Convert a string to a value that is acceptable as a make variable name.""" @@ -2069,7 +2074,7 @@ def WriteList(self, value_list, variable=None, prefix="", quoter=QuoteIfNecessar """ values = "" if value_list: - value_list = [quoter(prefix + value) for value in value_list] + value_list = [ReplaceSep(quoter(prefix + value)) for value in value_list] values = " \\\n\t" + " \\\n\t".join(value_list) self.fp.write(f"{variable} :={values}\n\n") From 82d5d76bdcae91760aab25a582ad726974c99936 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Wed, 24 Jan 2024 09:02:09 +0800 Subject: [PATCH 05/26] snake_case --- pylib/gyp/generator/make.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pylib/gyp/generator/make.py b/pylib/gyp/generator/make.py index f0589363..f25800f5 100644 --- a/pylib/gyp/generator/make.py +++ b/pylib/gyp/generator/make.py @@ -725,7 +725,7 @@ def QuoteIfNecessary(string): string = '"' + string.replace('"', '\\"') + '"' return string -def ReplaceSep(string): +def replace_sep(string): if sys.platform == 'win32': string = string.replace('\\', '/') return string @@ -2074,7 +2074,7 @@ def WriteList(self, value_list, variable=None, prefix="", quoter=QuoteIfNecessar """ values = "" if value_list: - value_list = [ReplaceSep(quoter(prefix + value)) for value in value_list] + value_list = [replace_sep(quoter(prefix + value)) for value in value_list] values = " \\\n\t" + " \\\n\t".join(value_list) self.fp.write(f"{variable} :={values}\n\n") From 6a757fc3e38f50d0e294a6e369c42a53a4a382bb Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Sun, 28 Jan 2024 00:55:29 +0800 Subject: [PATCH 06/26] found more place to replace sep --- pylib/gyp/generator/make.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/pylib/gyp/generator/make.py b/pylib/gyp/generator/make.py index f25800f5..5635299e 100644 --- a/pylib/gyp/generator/make.py +++ b/pylib/gyp/generator/make.py @@ -727,7 +727,7 @@ def QuoteIfNecessary(string): def replace_sep(string): if sys.platform == 'win32': - string = string.replace('\\', '/') + string = string.replace('\\\\', '/').replace('\\', '/') return string def StringToMakefileVariable(string): @@ -864,13 +864,7 @@ def Write( self.output = self.ComputeMacBundleOutput(spec) self.output_binary = self.ComputeMacBundleBinaryOutput(spec) else: - if self.flavor == "win": - # prevent from generating copy targets on Windows - self.output = self.output_binary = self.ComputeOutput(spec).replace( - '\\', '/' - ) - else: - self.output = self.output_binary = self.ComputeOutput(spec) + self.output = self.output_binary = replace_sep(self.ComputeOutput(spec)) self.is_standalone_static_library = bool( spec.get("standalone_static_library", 0) @@ -996,7 +990,7 @@ def WriteSubMake(self, output_filename, makefile_path, targets, build_dir): # sub-project dir (see test/subdirectory/gyptest-subdir-all.py). self.WriteLn( "export builddir_name ?= %s" - % os.path.join(os.path.dirname(output_filename), build_dir) + % replace_sep(os.path.join(os.path.dirname(output_filename), build_dir)) ) self.WriteLn(".PHONY: all") self.WriteLn("all:") @@ -2380,10 +2374,10 @@ def WriteAutoRegenerationRule(params, root_makefile, makefile_name, build_files) "\t$(call do_cmd,regen_makefile)\n\n" % { "makefile_name": makefile_name, - "deps": " ".join(SourceifyAndQuoteSpaces(bf) for bf in build_files), - "cmd": gyp.common.EncodePOSIXShellList( + "deps": replace_sep(" ".join(SourceifyAndQuoteSpaces(bf) for bf in build_files)), + "cmd": replace_sep(gyp.common.EncodePOSIXShellList( [gyp_binary, "-fmake"] + gyp.RegenerateFlags(options) + build_files_args - ), + )), } ) From 7fd6b7eb74e5305ff77afd2ba85cc652de82dd1d Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Sun, 28 Jan 2024 00:58:45 +0800 Subject: [PATCH 07/26] lint --- pylib/gyp/generator/make.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pylib/gyp/generator/make.py b/pylib/gyp/generator/make.py index 5635299e..ab1a120e 100644 --- a/pylib/gyp/generator/make.py +++ b/pylib/gyp/generator/make.py @@ -2374,7 +2374,9 @@ def WriteAutoRegenerationRule(params, root_makefile, makefile_name, build_files) "\t$(call do_cmd,regen_makefile)\n\n" % { "makefile_name": makefile_name, - "deps": replace_sep(" ".join(SourceifyAndQuoteSpaces(bf) for bf in build_files)), + "deps": replace_sep( + " ".join(SourceifyAndQuoteSpaces(bf) for bf in build_files) + ), "cmd": replace_sep(gyp.common.EncodePOSIXShellList( [gyp_binary, "-fmake"] + gyp.RegenerateFlags(options) + build_files_args )), From be69f9f616333300857e7d4d403ac289f1a5cc87 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Sun, 28 Jan 2024 18:51:53 +0800 Subject: [PATCH 08/26] replace sep in compiler path --- pylib/gyp/generator/make.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/pylib/gyp/generator/make.py b/pylib/gyp/generator/make.py index ab1a120e..d406c4a5 100644 --- a/pylib/gyp/generator/make.py +++ b/pylib/gyp/generator/make.py @@ -2442,7 +2442,7 @@ def CalculateMakefilePath(build_file, base_name): makefile_path = os.path.join( options.toplevel_dir, options.generator_output, makefile_name ) - srcdir = gyp.common.RelativePath(srcdir, options.generator_output) + srcdir = replace_sep(gyp.common.RelativePath(srcdir, options.generator_output)) srcdir_prefix = "$(srcdir)/" flock_command = "flock" @@ -2456,6 +2456,17 @@ def CalculateMakefilePath(build_file, base_name): ' -Wl,--end-group', '' ) + CC_target = replace_sep(GetEnvironFallback(("CC_target", "CC"), "$(CC)")) + AR_target = replace_sep(GetEnvironFallback(("AR_target", "AR"), "$(AR)")) + CXX_target = replace_sep(GetEnvironFallback(("CXX_target", "CXX"), "$(CXX)")) + LINK_target = replace_sep(GetEnvironFallback(("LINK_target", "LINK"), "$(LINK)")) + PLI_target = replace_sep(GetEnvironFallback(("PLI_target", "PLI"), "pli")) + CC_host = replace_sep(GetEnvironFallback(("CC_host", "CC"), "gcc")) + AR_host = replace_sep(GetEnvironFallback(("AR_host", "AR"), "ar")) + CXX_host = replace_sep(GetEnvironFallback(("CXX_host", "CXX"), "g++")) + LINK_host = replace_sep(GetEnvironFallback(("LINK_host", "LINK"), "$(CXX.host)")) + PLI_host = replace_sep(GetEnvironFallback(("PLI_host", "PLI"), "pli")) + header_params = { "default_target": default_target, "builddir": builddir_name, @@ -2467,16 +2478,16 @@ def CalculateMakefilePath(build_file, base_name): "srcdir": srcdir, "copy_archive_args": copy_archive_arguments, "makedep_args": makedep_arguments, - "CC.target": GetEnvironFallback(("CC_target", "CC"), "$(CC)"), - "AR.target": GetEnvironFallback(("AR_target", "AR"), "$(AR)"), - "CXX.target": GetEnvironFallback(("CXX_target", "CXX"), "$(CXX)"), - "LINK.target": GetEnvironFallback(("LINK_target", "LINK"), "$(LINK)"), - "PLI.target": GetEnvironFallback(("PLI_target", "PLI"), "pli"), - "CC.host": GetEnvironFallback(("CC_host", "CC"), "gcc"), - "AR.host": GetEnvironFallback(("AR_host", "AR"), "ar"), - "CXX.host": GetEnvironFallback(("CXX_host", "CXX"), "g++"), - "LINK.host": GetEnvironFallback(("LINK_host", "LINK"), "$(CXX.host)"), - "PLI.host": GetEnvironFallback(("PLI_host", "PLI"), "pli"), + "CC.target": CC_target, + "AR.target": AR_target, + "CXX.target": CXX_target, + "LINK.target": LINK_target, + "PLI.target": PLI_target, + "CC.host": CC_host, + "AR.host": AR_host, + "CXX.host": CXX_host, + "LINK.host": LINK_host, + "PLI.host": PLI_host, } # If cross-compiling, reserve linux link commands and do not use gyp-mac-tool if flavor == "mac" and not gyp.common.CrossCompileRequested(): From 7a62b19447d3011dc36ad3a1c74dffed733fa67f Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Sun, 28 Jan 2024 21:51:39 +0800 Subject: [PATCH 09/26] fix sed unterminated `s' command error on Windows --- pylib/gyp/generator/make.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pylib/gyp/generator/make.py b/pylib/gyp/generator/make.py index d406c4a5..beb045ed 100644 --- a/pylib/gyp/generator/make.py +++ b/pylib/gyp/generator/make.py @@ -448,8 +448,12 @@ def CalculateGeneratorInputInfo(params): # Add extra rules as in (2). # We remove slashes and replace spaces with new lines; # remove blank lines; -# delete the first line and append a colon to the remaining lines. -sed -e 's|\\||' -e 'y| |\n|' $(depfile).raw |\ +# delete the first line and append a colon to the remaining lines.""" + + (r""" +sed -e 's|\\\\||' -e 'y| |\n|' $(depfile).raw |\\""" if sys.platform == 'win32' + else r""" +sed -e 's|\\||' -e 'y| |\n|' $(depfile).raw |\\""") + + r""" grep -v '^$$' |\ sed -e 1d -e 's|$$|:|' \ >> $(depfile) From 64290b6abce447735faf6ea9958325cf5483cde1 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Sun, 28 Jan 2024 21:59:19 +0800 Subject: [PATCH 10/26] path includes `\` so replace the ended `\` only --- pylib/gyp/generator/make.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib/gyp/generator/make.py b/pylib/gyp/generator/make.py index beb045ed..fa00e162 100644 --- a/pylib/gyp/generator/make.py +++ b/pylib/gyp/generator/make.py @@ -450,7 +450,7 @@ def CalculateGeneratorInputInfo(params): # remove blank lines; # delete the first line and append a colon to the remaining lines.""" + (r""" -sed -e 's|\\\\||' -e 'y| |\n|' $(depfile).raw |\\""" if sys.platform == 'win32' +sed -e 's/\\\\$$//' -e 'y| |\n|' $(depfile).raw |\\""" if sys.platform == 'win32' else r""" sed -e 's|\\||' -e 'y| |\n|' $(depfile).raw |\\""") + r""" From 77b8e9cd4397d37e34855a835d5314b6caf9505e Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Sun, 28 Jan 2024 23:54:44 +0800 Subject: [PATCH 11/26] replace `\` with `/` in depfile on win --- pylib/gyp/generator/make.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pylib/gyp/generator/make.py b/pylib/gyp/generator/make.py index fa00e162..68bc2efd 100644 --- a/pylib/gyp/generator/make.py +++ b/pylib/gyp/generator/make.py @@ -443,14 +443,18 @@ def CalculateGeneratorInputInfo(params): define fixup_dep # The depfile may not exist if the input file didn't have any #includes. touch $(depfile).raw -# Fixup path as in (1). -sed -e "s|^$(notdir $@)|$@|" $(depfile).raw >> $(depfile) +# Fixup path as in (1).""" + + (r""" +sed -e "s|^$(notdir $@)|$@|" -re 's/\\\\([^$$])/\/\1/g' $(depfile).raw >> $(depfile)""" if sys.platform == 'win32' + else r""" +sed -e "s|^$(notdir $@)|$@|" $(depfile).raw >> $(depfile)""") + + r""" # Add extra rules as in (2). # We remove slashes and replace spaces with new lines; # remove blank lines; # delete the first line and append a colon to the remaining lines.""" + (r""" -sed -e 's/\\\\$$//' -e 'y| |\n|' $(depfile).raw |\\""" if sys.platform == 'win32' +sed -e 's/\\\\$$//' -e 's/\\\\/\//g' -e 'y| |\n|' $(depfile).raw |\\""" if sys.platform == 'win32' else r""" sed -e 's|\\||' -e 'y| |\n|' $(depfile).raw |\\""") + r""" From 9f110ec52f19268f2239702c1a7ac6a8570cb6a6 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Mon, 29 Jan 2024 00:04:24 +0800 Subject: [PATCH 12/26] lint --- pylib/gyp/generator/make.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pylib/gyp/generator/make.py b/pylib/gyp/generator/make.py index 68bc2efd..c7f54459 100644 --- a/pylib/gyp/generator/make.py +++ b/pylib/gyp/generator/make.py @@ -445,8 +445,8 @@ def CalculateGeneratorInputInfo(params): touch $(depfile).raw # Fixup path as in (1).""" + (r""" -sed -e "s|^$(notdir $@)|$@|" -re 's/\\\\([^$$])/\/\1/g' $(depfile).raw >> $(depfile)""" if sys.platform == 'win32' - else r""" +sed -e "s|^$(notdir $@)|$@|" -re 's/\\\\([^$$])/\/\1/g' $(depfile).raw >> $(depfile)""" + if sys.platform == 'win32' else r""" sed -e "s|^$(notdir $@)|$@|" $(depfile).raw >> $(depfile)""") + r""" # Add extra rules as in (2). @@ -454,8 +454,8 @@ def CalculateGeneratorInputInfo(params): # remove blank lines; # delete the first line and append a colon to the remaining lines.""" + (r""" -sed -e 's/\\\\$$//' -e 's/\\\\/\//g' -e 'y| |\n|' $(depfile).raw |\\""" if sys.platform == 'win32' - else r""" +sed -e 's/\\\\$$//' -e 's/\\\\/\//g' -e 'y| |\n|' $(depfile).raw |\\""" + if sys.platform == 'win32' else r""" sed -e 's|\\||' -e 'y| |\n|' $(depfile).raw |\\""") + r""" grep -v '^$$' |\ From 1d82674d9a4acd4722e00fafa27fe3d877192c36 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Tue, 30 Jan 2024 18:12:33 +0800 Subject: [PATCH 13/26] fix: trailing `\` in raw string --- pylib/gyp/generator/make.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pylib/gyp/generator/make.py b/pylib/gyp/generator/make.py index c7f54459..142a59c7 100644 --- a/pylib/gyp/generator/make.py +++ b/pylib/gyp/generator/make.py @@ -455,8 +455,8 @@ def CalculateGeneratorInputInfo(params): # delete the first line and append a colon to the remaining lines.""" + (r""" sed -e 's/\\\\$$//' -e 's/\\\\/\//g' -e 'y| |\n|' $(depfile).raw |\\""" - if sys.platform == 'win32' else r""" -sed -e 's|\\||' -e 'y| |\n|' $(depfile).raw |\\""") + + if sys.platform == 'win32' else """ +sed -e 's|\\\\||' -e 'y| |\\n|' $(depfile).raw |\\""") + r""" grep -v '^$$' |\ sed -e 1d -e 's|$$|:|' \ From 97d765cf1579949517db83f83f10e4ae2fb81e02 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Tue, 30 Jan 2024 18:41:22 +0800 Subject: [PATCH 14/26] revert: flavor can be set via `-f make-linux` so no need to change the mac params --- pylib/gyp/generator/make.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pylib/gyp/generator/make.py b/pylib/gyp/generator/make.py index 142a59c7..aaad35d1 100644 --- a/pylib/gyp/generator/make.py +++ b/pylib/gyp/generator/make.py @@ -2497,8 +2497,7 @@ def CalculateMakefilePath(build_file, base_name): "LINK.host": LINK_host, "PLI.host": PLI_host, } - # If cross-compiling, reserve linux link commands and do not use gyp-mac-tool - if flavor == "mac" and not gyp.common.CrossCompileRequested(): + if flavor == "mac": flock_command = "./gyp-mac-tool flock" header_params.update( { From ef484854230ef3e52cd1e8d59a9f7360acab6a07 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Tue, 30 Jan 2024 20:31:41 +0800 Subject: [PATCH 15/26] fix: also do not use raw string in windows branch due to trailing `\` break editor highlight --- pylib/gyp/generator/make.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pylib/gyp/generator/make.py b/pylib/gyp/generator/make.py index aaad35d1..b70dde65 100644 --- a/pylib/gyp/generator/make.py +++ b/pylib/gyp/generator/make.py @@ -453,8 +453,8 @@ def CalculateGeneratorInputInfo(params): # We remove slashes and replace spaces with new lines; # remove blank lines; # delete the first line and append a colon to the remaining lines.""" + - (r""" -sed -e 's/\\\\$$//' -e 's/\\\\/\//g' -e 'y| |\n|' $(depfile).raw |\\""" + (""" +sed -e 's/\\\\\\\\$$//' -e 's/\\\\\\\\/\\//g' -e 'y| |\\n|' $(depfile).raw |\\""" if sys.platform == 'win32' else """ sed -e 's|\\\\||' -e 'y| |\\n|' $(depfile).raw |\\""") + r""" From f4410afd4d698c781ffcb283e6439c080386b935 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Wed, 20 Mar 2024 17:05:17 +0800 Subject: [PATCH 16/26] fix: respect user specified AR_target environment variable --- pylib/gyp/generator/make.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib/gyp/generator/make.py b/pylib/gyp/generator/make.py index b70dde65..28d64221 100644 --- a/pylib/gyp/generator/make.py +++ b/pylib/gyp/generator/make.py @@ -379,7 +379,7 @@ def CalculateGeneratorInputInfo(params): CXXFLAGS.target ?= $(CPPFLAGS) $(CXXFLAGS) LINK.target ?= %(LINK.target)s LDFLAGS.target ?= $(LDFLAGS) -AR.target ?= $(AR) +AR.target ?= %(AR.target)s PLI.target ?= %(PLI.target)s # C++ apps need to be linked with g++. From 840574c0c9d79918d588347afa9be3d111aef4ee Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Mon, 1 Apr 2024 18:15:29 +0800 Subject: [PATCH 17/26] feat: detect wasm flavor --- pylib/gyp/common.py | 44 +++++++++++++++++++++++++++++++++++++ pylib/gyp/generator/make.py | 4 ++-- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/pylib/gyp/common.py b/pylib/gyp/common.py index b73a0c55..64277f75 100644 --- a/pylib/gyp/common.py +++ b/pylib/gyp/common.py @@ -422,6 +422,37 @@ def EnsureDirExists(path): except OSError: pass +def GetCrossCompilerPredefines(): + CC = os.environ.get("CC_target") or os.environ.get("CC") + CFLAGS = os.environ.get("CFLAGS") + CXX = os.environ.get("CXX_target") or os.environ.get("CXX") + CXXFLAGS = os.environ.get("CXXFLAGS") + cmd = [] + + if CC: + cmd += CC.split(" ") + if CFLAGS: + cmd += CFLAGS.split(" ") + elif CXX: + cmd += CXX.split(" ") + if CXXFLAGS: + cmd += CXXFLAGS.split(" ") + else: + return None + + out = subprocess.Popen( + [*cmd, "-dM", "-E", "-x", "c", "/dev/null"], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + lines = out.communicate()[0].decode("utf-8").split("\n") + defines = {} + for line in lines: + if not line: + continue + define_directive, key, *value = line.split(" ") + assert define_directive == "#define" + defines[key] = " ".join(value) + return defines def GetFlavor(params): """Returns |params.flavor| if it's set, the system's default flavor else.""" @@ -433,6 +464,19 @@ def GetFlavor(params): if "flavor" in params: return params["flavor"] + + try: + defines = GetCrossCompilerPredefines() + if "__EMSCRIPTEN__" in defines: + return "emscripten" + if "__wasm__" in defines: + if "__wasi__" in defines: + return "wasi" + else: + return "wasm" + except Exception: + pass + if sys.platform in flavors: return flavors[sys.platform] if sys.platform.startswith("sunos"): diff --git a/pylib/gyp/generator/make.py b/pylib/gyp/generator/make.py index 28d64221..392d9009 100644 --- a/pylib/gyp/generator/make.py +++ b/pylib/gyp/generator/make.py @@ -2457,9 +2457,9 @@ def CalculateMakefilePath(build_file, base_name): copy_archive_arguments = "-af" makedep_arguments = "-MMD" - # some linkers don't support --start-group/--end-group (e.g. wasm-ld) + # wasm-ld doesn't support --start-group/--end-group link_commands = LINK_COMMANDS_LINUX - if gyp.common.CrossCompileRequested(): + if flavor in ["wasi", "wasm"]: link_commands = link_commands.replace(' -Wl,--start-group', '').replace( ' -Wl,--end-group', '' ) From 3a727601607f935f8349f0620387ad321118ba7c Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Mon, 1 Apr 2024 18:23:46 +0800 Subject: [PATCH 18/26] lint: Too many return statements --- pylib/gyp/common.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/pylib/gyp/common.py b/pylib/gyp/common.py index 64277f75..0e5abe03 100644 --- a/pylib/gyp/common.py +++ b/pylib/gyp/common.py @@ -454,7 +454,7 @@ def GetCrossCompilerPredefines(): defines[key] = " ".join(value) return defines -def GetFlavor(params): +def GetFlavorByPlatform(): """Returns |params.flavor| if it's set, the system's default flavor else.""" flavors = { "cygwin": "win", @@ -462,21 +462,6 @@ def GetFlavor(params): "darwin": "mac", } - if "flavor" in params: - return params["flavor"] - - try: - defines = GetCrossCompilerPredefines() - if "__EMSCRIPTEN__" in defines: - return "emscripten" - if "__wasm__" in defines: - if "__wasi__" in defines: - return "wasi" - else: - return "wasm" - except Exception: - pass - if sys.platform in flavors: return flavors[sys.platform] if sys.platform.startswith("sunos"): @@ -496,6 +481,24 @@ def GetFlavor(params): return "linux" +def GetFlavor(params): + if "flavor" in params: + return params["flavor"] + + try: + defines = GetCrossCompilerPredefines() + if "__EMSCRIPTEN__" in defines: + return "emscripten" + if "__wasm__" in defines: + if "__wasi__" in defines: + return "wasi" + else: + return "wasm" + except Exception: + pass + + return GetFlavorByPlatform() + def CopyTool(flavor, out_path, generator_flags={}): """Finds (flock|mac|win)_tool.gyp in the gyp directory and copies it From 626f0a6c2dbad075060fe9a390e950de32873e0b Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Mon, 1 Apr 2024 21:13:58 +0800 Subject: [PATCH 19/26] fix get compiler predefines on windows --- pylib/gyp/common.py | 42 +++++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/pylib/gyp/common.py b/pylib/gyp/common.py index 0e5abe03..125c8a68 100644 --- a/pylib/gyp/common.py +++ b/pylib/gyp/common.py @@ -440,19 +440,35 @@ def GetCrossCompilerPredefines(): else: return None - out = subprocess.Popen( - [*cmd, "-dM", "-E", "-x", "c", "/dev/null"], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - lines = out.communicate()[0].decode("utf-8").split("\n") - defines = {} - for line in lines: - if not line: - continue - define_directive, key, *value = line.split(" ") - assert define_directive == "#define" - defines[key] = " ".join(value) - return defines + if sys.platform == "win32": + fd, input = tempfile.mkstemp(suffix=".c") + try: + os.close(fd) + except Exception: + os.unlink(input) + raise + else: + input = "/dev/null" + + try: + out = subprocess.Popen( + [*cmd, "-dM", "-E", "-x", "c", input], + shell=sys.platform == "win32", + stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + lines = out.communicate()[0].decode("utf-8").replace( + "\r\n", "\n").split("\n") + defines = {} + for line in lines: + if not line: + continue + define_directive, key, *value = line.split(" ") + assert define_directive == "#define" + defines[key] = " ".join(value) + return defines + finally: + if sys.platform == "win32": + os.unlink(input) def GetFlavorByPlatform(): """Returns |params.flavor| if it's set, the system's default flavor else.""" From 9e9cb995da63d3a77c1ef5302cb75b0a52987426 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Mon, 1 Apr 2024 21:33:14 +0800 Subject: [PATCH 20/26] GetCrossCompilerPredefines always return dict --- pylib/gyp/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pylib/gyp/common.py b/pylib/gyp/common.py index 125c8a68..dae8873c 100644 --- a/pylib/gyp/common.py +++ b/pylib/gyp/common.py @@ -428,6 +428,7 @@ def GetCrossCompilerPredefines(): CXX = os.environ.get("CXX_target") or os.environ.get("CXX") CXXFLAGS = os.environ.get("CXXFLAGS") cmd = [] + defines = {} if CC: cmd += CC.split(" ") @@ -438,7 +439,7 @@ def GetCrossCompilerPredefines(): if CXXFLAGS: cmd += CXXFLAGS.split(" ") else: - return None + return defines if sys.platform == "win32": fd, input = tempfile.mkstemp(suffix=".c") @@ -458,7 +459,6 @@ def GetCrossCompilerPredefines(): ) lines = out.communicate()[0].decode("utf-8").replace( "\r\n", "\n").split("\n") - defines = {} for line in lines: if not line: continue From 3b49a7bbb6b849ce4bae9ad9976be7fbed547869 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Mon, 1 Apr 2024 21:53:02 +0800 Subject: [PATCH 21/26] do not broad exceptions --- pylib/gyp/common.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/pylib/gyp/common.py b/pylib/gyp/common.py index dae8873c..a57ba656 100644 --- a/pylib/gyp/common.py +++ b/pylib/gyp/common.py @@ -422,7 +422,7 @@ def EnsureDirExists(path): except OSError: pass -def GetCrossCompilerPredefines(): +def GetCrossCompilerPredefines(): # -> dict CC = os.environ.get("CC_target") or os.environ.get("CC") CFLAGS = os.environ.get("CFLAGS") CXX = os.environ.get("CXX_target") or os.environ.get("CXX") @@ -443,11 +443,7 @@ def GetCrossCompilerPredefines(): if sys.platform == "win32": fd, input = tempfile.mkstemp(suffix=".c") - try: - os.close(fd) - except Exception: - os.unlink(input) - raise + os.close(fd) else: input = "/dev/null" @@ -501,17 +497,11 @@ def GetFlavor(params): if "flavor" in params: return params["flavor"] - try: - defines = GetCrossCompilerPredefines() - if "__EMSCRIPTEN__" in defines: - return "emscripten" - if "__wasm__" in defines: - if "__wasi__" in defines: - return "wasi" - else: - return "wasm" - except Exception: - pass + defines = GetCrossCompilerPredefines() + if "__EMSCRIPTEN__" in defines: + return "emscripten" + if "__wasm__" in defines: + return "wasi" if "__wasi__" in defines else "wasm" return GetFlavorByPlatform() From 83274d90e55f0e54cf20fbd8996342513ee462e9 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Mon, 1 Apr 2024 23:40:16 +0800 Subject: [PATCH 22/26] test: GetCrossCompilerPredefines --- pylib/gyp/common_test.py | 61 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/pylib/gyp/common_test.py b/pylib/gyp/common_test.py index 05344085..f7383e43 100755 --- a/pylib/gyp/common_test.py +++ b/pylib/gyp/common_test.py @@ -9,7 +9,10 @@ import gyp.common import unittest import sys - +import os +import subprocess +from unittest import mock +from unittest.mock import patch, MagicMock class TestTopologicallySorted(unittest.TestCase): def test_Valid(self): @@ -73,6 +76,62 @@ def test_platform_default(self): def test_param(self): self.assertFlavor("foobar", "linux2", {"flavor": "foobar"}) + class MockCommunicate: + def __init__(self, stdout): + self.stdout = stdout + + def decode(self, encoding): + return self.stdout + + @mock.patch("os.close") + @mock.patch("os.unlink") + @mock.patch("tempfile.mkstemp") + def test_GetCrossCompilerPredefines(self, mock_mkstemp, mock_unlink, mock_close): + mock_close.return_value = None + mock_unlink.return_value = None + mock_mkstemp.return_value = (0, "temp.c") + + def mock_run(env, defines_stdout): + with mock.patch("subprocess.Popen") as mock_popen: + mock_process = MagicMock() + mock_process.communicate.return_value = ( + TestGetFlavor.MockCommunicate(defines_stdout), None) + mock_process.stdout = MagicMock() + mock_popen.return_value = mock_process + expected_input = "temp.c" if sys.platform == "win32" else "/dev/null" + with mock.patch.dict(os.environ, env): + defines = gyp.common.GetCrossCompilerPredefines() + flavor = gyp.common.GetFlavor({}) + if env.get("CC_target"): + mock_popen.assert_called_with( + [env["CC_target"], "-dM", "-E", "-x", "c", expected_input], + shell=sys.platform == "win32", + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + return [defines, flavor] + + [defines1, _] = mock_run({}, "") + self.assertDictEqual({}, defines1) + + [defines2, flavor2] = mock_run( + { "CC_target": "/opt/wasi-sdk/bin/clang" }, + "#define __wasm__ 1\n#define __wasi__ 1\n" + ) + self.assertDictEqual({ "__wasm__": "1", "__wasi__": "1" }, defines2) + self.assertEqual("wasi", flavor2) + + [defines3, flavor3] = mock_run( + { "CC_target": "/opt/wasi-sdk/bin/clang" }, + "#define __wasm__ 1\n" + ) + self.assertDictEqual({ "__wasm__": "1" }, defines3) + self.assertEqual("wasm", flavor3) + + [defines4, flavor4] = mock_run( + { "CC_target": "/emsdk/upstream/emscripten/emcc" }, + "#define __EMSCRIPTEN__ 1\n" + ) + self.assertDictEqual({ "__EMSCRIPTEN__": "1" }, defines4) + self.assertEqual("emscripten", flavor4) if __name__ == "__main__": unittest.main() From 04024d4db2ac0d4d29f75604a9bf3b730710c169 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Mon, 1 Apr 2024 23:42:19 +0800 Subject: [PATCH 23/26] fix lint --- pylib/gyp/common_test.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pylib/gyp/common_test.py b/pylib/gyp/common_test.py index f7383e43..43b22547 100755 --- a/pylib/gyp/common_test.py +++ b/pylib/gyp/common_test.py @@ -11,7 +11,6 @@ import sys import os import subprocess -from unittest import mock from unittest.mock import patch, MagicMock class TestTopologicallySorted(unittest.TestCase): @@ -83,23 +82,23 @@ def __init__(self, stdout): def decode(self, encoding): return self.stdout - @mock.patch("os.close") - @mock.patch("os.unlink") - @mock.patch("tempfile.mkstemp") + @patch("os.close") + @patch("os.unlink") + @patch("tempfile.mkstemp") def test_GetCrossCompilerPredefines(self, mock_mkstemp, mock_unlink, mock_close): mock_close.return_value = None mock_unlink.return_value = None mock_mkstemp.return_value = (0, "temp.c") def mock_run(env, defines_stdout): - with mock.patch("subprocess.Popen") as mock_popen: + with patch("subprocess.Popen") as mock_popen: mock_process = MagicMock() mock_process.communicate.return_value = ( TestGetFlavor.MockCommunicate(defines_stdout), None) mock_process.stdout = MagicMock() mock_popen.return_value = mock_process expected_input = "temp.c" if sys.platform == "win32" else "/dev/null" - with mock.patch.dict(os.environ, env): + with patch.dict(os.environ, env): defines = gyp.common.GetCrossCompilerPredefines() flavor = gyp.common.GetFlavor({}) if env.get("CC_target"): From a6cfb16c0cbf481069b4231bf63e16587cf4eeeb Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Mon, 1 Apr 2024 23:50:01 +0800 Subject: [PATCH 24/26] refactor: do not put so many lines in try block --- pylib/gyp/common.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/pylib/gyp/common.py b/pylib/gyp/common.py index a57ba656..01dfd301 100644 --- a/pylib/gyp/common.py +++ b/pylib/gyp/common.py @@ -443,28 +443,32 @@ def GetCrossCompilerPredefines(): # -> dict if sys.platform == "win32": fd, input = tempfile.mkstemp(suffix=".c") - os.close(fd) + try: + os.close(fd) + out = subprocess.Popen( + [*cmd, "-dM", "-E", "-x", "c", input], + shell=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + finally: + os.unlink(input) else: input = "/dev/null" - - try: out = subprocess.Popen( [*cmd, "-dM", "-E", "-x", "c", input], - shell=sys.platform == "win32", + shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) - lines = out.communicate()[0].decode("utf-8").replace( - "\r\n", "\n").split("\n") - for line in lines: - if not line: - continue - define_directive, key, *value = line.split(" ") - assert define_directive == "#define" - defines[key] = " ".join(value) - return defines - finally: - if sys.platform == "win32": - os.unlink(input) + + lines = out.communicate()[0].decode("utf-8").replace( + "\r\n", "\n").split("\n") + for line in lines: + if not line: + continue + define_directive, key, *value = line.split(" ") + assert define_directive == "#define" + defines[key] = " ".join(value) + return defines def GetFlavorByPlatform(): """Returns |params.flavor| if it's set, the system's default flavor else.""" From 4bf22c4b70a385a506d566430db59cf1c72a4602 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Tue, 2 Apr 2024 00:01:51 +0800 Subject: [PATCH 25/26] fix: finally block should wait until subprocess terminate --- pylib/gyp/common.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pylib/gyp/common.py b/pylib/gyp/common.py index 01dfd301..30326f89 100644 --- a/pylib/gyp/common.py +++ b/pylib/gyp/common.py @@ -450,6 +450,7 @@ def GetCrossCompilerPredefines(): # -> dict shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) + stdout = out.communicate()[0] finally: os.unlink(input) else: @@ -459,9 +460,9 @@ def GetCrossCompilerPredefines(): # -> dict shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) + stdout = out.communicate()[0] - lines = out.communicate()[0].decode("utf-8").replace( - "\r\n", "\n").split("\n") + lines = stdout.decode("utf-8").replace("\r\n", "\n").split("\n") for line in lines: if not line: continue From 3a962e9702594fee81c1ec36588217c17fa24d72 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Tue, 2 Apr 2024 00:43:47 +0800 Subject: [PATCH 26/26] suggestion change --- pylib/gyp/common.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/pylib/gyp/common.py b/pylib/gyp/common.py index 30326f89..db64dc52 100644 --- a/pylib/gyp/common.py +++ b/pylib/gyp/common.py @@ -423,23 +423,17 @@ def EnsureDirExists(path): pass def GetCrossCompilerPredefines(): # -> dict - CC = os.environ.get("CC_target") or os.environ.get("CC") - CFLAGS = os.environ.get("CFLAGS") - CXX = os.environ.get("CXX_target") or os.environ.get("CXX") - CXXFLAGS = os.environ.get("CXXFLAGS") cmd = [] - defines = {} - - if CC: + if CC := os.environ.get("CC_target") or os.environ.get("CC"): cmd += CC.split(" ") - if CFLAGS: + if CFLAGS := os.environ.get("CFLAGS"): cmd += CFLAGS.split(" ") - elif CXX: + elif CXX := os.environ.get("CXX_target") or os.environ.get("CXX"): cmd += CXX.split(" ") - if CXXFLAGS: + if CXXFLAGS := os.environ.get("CXXFLAGS"): cmd += CXXFLAGS.split(" ") else: - return defines + return {} if sys.platform == "win32": fd, input = tempfile.mkstemp(suffix=".c") @@ -462,6 +456,7 @@ def GetCrossCompilerPredefines(): # -> dict ) stdout = out.communicate()[0] + defines = {} lines = stdout.decode("utf-8").replace("\r\n", "\n").split("\n") for line in lines: if not line: