From 90ccf2187332ff900d46a58a27cb0353577d37cb Mon Sep 17 00:00:00 2001 From: "Joel E. Denny" Date: Thu, 11 Jul 2024 11:41:34 -0400 Subject: [PATCH] [LinkerWrapper] Extend with usual pass options (#96704) The goal of this patch is to enable utilizing LLVM plugin passes and remarks for GPU offload code at link time. Specifically, this patch extends clang-linker-wrapper's `--offload-opt` (and consequently `-mllvm`) to accept the various LLVM pass options that tools like opt usually accept. Those options include `--passes`, `--load-pass-plugin`, and various remarks options. Unlike many other LLVM options that are inherited from linked code by clang-linker-wrapper (e.g., `-pass-remarks` is already implemented in `llvm/lib/IR/DiagnosticHandler.cpp`), these options are implemented separately as needed by each tool (e.g., opt, llc). Fortunately, this patch is able to handle most of the implementation by passing the option values to `lto::Config`. For testing plugin support, this patch uses the simple `Bye` plugin from LLVM core, but that requires several small Clang test suite config extensions. --- clang/test/CMakeLists.txt | 3 + clang/test/Driver/linker-wrapper-llvm-help.c | 10 +++ clang/test/Driver/linker-wrapper-passes.c | 72 ++++++++++++++++++ clang/test/lit.cfg.py | 12 +++ clang/test/lit.site.cfg.py.in | 4 + .../tools/clang-linker-wrapper/CMakeLists.txt | 2 + .../ClangLinkerWrapper.cpp | 74 +++++++++++++++++++ .../clang-linker-wrapper/LinkerWrapperOpts.td | 8 +- 8 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 clang/test/Driver/linker-wrapper-llvm-help.c create mode 100644 clang/test/Driver/linker-wrapper-passes.c diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt index 5fceb1d7103341..8303269a9ad074 100644 --- a/clang/test/CMakeLists.txt +++ b/clang/test/CMakeLists.txt @@ -11,6 +11,9 @@ llvm_canonicalize_cmake_booleans( CLANG_SPAWN_CC1 CLANG_ENABLE_CIR ENABLE_BACKTRACES + LLVM_BUILD_EXAMPLES + LLVM_BYE_LINK_INTO_TOOLS + LLVM_ENABLE_PLUGINS LLVM_ENABLE_ZLIB LLVM_ENABLE_ZSTD LLVM_ENABLE_PER_TARGET_RUNTIME_DIR diff --git a/clang/test/Driver/linker-wrapper-llvm-help.c b/clang/test/Driver/linker-wrapper-llvm-help.c new file mode 100644 index 00000000000000..ffd1cf78bcd9ad --- /dev/null +++ b/clang/test/Driver/linker-wrapper-llvm-help.c @@ -0,0 +1,10 @@ +// Check that these simple command lines for listing LLVM options are supported, +// as claimed by 'clang-linker-wrapper --help'. + +// RUN: clang-linker-wrapper -mllvm --help 2>&1 | FileCheck %s +// RUN: clang-linker-wrapper --offload-opt=--help 2>&1 | FileCheck %s + +// Look for a few options supported only after -mllvm and --offload-opt. +// CHECK: OPTIONS: +// CHECK-DAG: --passes= +// CHECK-DAG: --load-pass-plugin= diff --git a/clang/test/Driver/linker-wrapper-passes.c b/clang/test/Driver/linker-wrapper-passes.c new file mode 100644 index 00000000000000..aadcf472e9b636 --- /dev/null +++ b/clang/test/Driver/linker-wrapper-passes.c @@ -0,0 +1,72 @@ +// Check various clang-linker-wrapper pass options after -offload-opt. + +// REQUIRES: llvm-plugins, llvm-examples +// REQUIRES: x86-registered-target +// REQUIRES: amdgpu-registered-target + +// Setup. +// RUN: mkdir -p %t +// RUN: %clang -cc1 -emit-llvm-bc -o %t/host-x86_64-unknown-linux-gnu.bc \ +// RUN: -disable-O0-optnone -triple=x86_64-unknown-linux-gnu %s +// RUN: %clang -cc1 -emit-llvm-bc -o %t/openmp-amdgcn-amd-amdhsa.bc \ +// RUN: -disable-O0-optnone -triple=amdgcn-amd-amdhsa %s +// RUN: opt %t/openmp-amdgcn-amd-amdhsa.bc -o %t/openmp-amdgcn-amd-amdhsa.bc \ +// RUN: -passes=forceattrs -force-remove-attribute=f:noinline +// RUN: clang-offload-packager -o %t/openmp-x86_64-unknown-linux-gnu.out \ +// RUN: --image=file=%t/openmp-amdgcn-amd-amdhsa.bc,triple=amdgcn-amd-amdhsa +// RUN: %clang -cc1 -S -o %t/host-x86_64-unknown-linux-gnu.s \ +// RUN: -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa \ +// RUN: -fembed-offload-object=%t/openmp-x86_64-unknown-linux-gnu.out \ +// RUN: %t/host-x86_64-unknown-linux-gnu.bc +// RUN: %clang -cc1as -o %t/host-x86_64-unknown-linux-gnu.o \ +// RUN: -triple x86_64-unknown-linux-gnu -filetype obj -target-cpu x86-64 \ +// RUN: %t/host-x86_64-unknown-linux-gnu.s + +// Check plugin, -passes, and no remarks. +// RUN: clang-linker-wrapper -o a.out --embed-bitcode \ +// RUN: --linker-path=/usr/bin/true %t/host-x86_64-unknown-linux-gnu.o \ +// RUN: %offload-opt-loadbye --offload-opt=-wave-goodbye \ +// RUN: --offload-opt=-passes="function(goodbye),module(inline)" 2>&1 | \ +// RUN: FileCheck -match-full-lines -check-prefixes=OUT %s + +// Check plugin, -p, and remarks. +// RUN: clang-linker-wrapper -o a.out --embed-bitcode \ +// RUN: --linker-path=/usr/bin/true %t/host-x86_64-unknown-linux-gnu.o \ +// RUN: %offload-opt-loadbye --offload-opt=-wave-goodbye \ +// RUN: --offload-opt=-p="function(goodbye),module(inline)" \ +// RUN: --offload-opt=-pass-remarks=inline \ +// RUN: --offload-opt=-pass-remarks-output=%t/remarks.yml \ +// RUN: --offload-opt=-pass-remarks-filter=inline \ +// RUN: --offload-opt=-pass-remarks-format=yaml 2>&1 | \ +// RUN: FileCheck -match-full-lines -check-prefixes=OUT,REM %s +// RUN: FileCheck -input-file=%t/remarks.yml -match-full-lines \ +// RUN: -check-prefixes=YML %s + +// Check handling of bad plugin. +// RUN: not clang-linker-wrapper \ +// RUN: --offload-opt=-load-pass-plugin=%t/nonexistent.so 2>&1 | \ +// RUN: FileCheck -match-full-lines -check-prefixes=BAD-PLUGIN %s + +// OUT-NOT: {{.}} +// OUT: Bye: f +// OUT-NEXT: Bye: test +// REM-NEXT: remark: {{.*}} 'f' inlined into 'test' {{.*}} +// OUT-NOT: {{.}} + +// YML-NOT: {{.}} +// YML: --- !Passed +// YML-NEXT: Pass: inline +// YML-NEXT: Name: Inlined +// YML-NEXT: Function: test +// YML-NEXT: Args: +// YML: - Callee: f +// YML: - Caller: test +// YML: ... +// YML-NOT: {{.}} + +// BAD-PLUGIN-NOT: {{.}} +// BAD-PLUGIN: {{.*}}Could not load library {{.*}}nonexistent.so{{.*}} +// BAD-PLUGIN-NOT: {{.}} + +void f() {} +void test() { f(); } diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py index e5630a07424c7c..2e0fbc2c9e1dd4 100644 --- a/clang/test/lit.cfg.py +++ b/clang/test/lit.cfg.py @@ -109,6 +109,15 @@ if config.clang_examples: config.available_features.add("examples") +if config.llvm_examples: + config.available_features.add("llvm-examples") + +if config.llvm_linked_bye_extension: + config.substitutions.append(("%offload-opt-loadbye", "")) +else: + loadbye = f"-load-pass-plugin={config.llvm_shlib_dir}/Bye{config.llvm_shlib_ext}" + config.substitutions.append(("%offload-opt-loadbye", f"--offload-opt={loadbye}")) + def have_host_jit_feature_support(feature_name): clang_repl_exe = lit.util.which("clang-repl", config.clang_tools_dir) @@ -213,6 +222,9 @@ def have_host_clang_repl_cuda(): if config.has_plugins and config.llvm_plugin_ext: config.available_features.add("plugins") +if config.llvm_has_plugins and config.llvm_plugin_ext: + config.available_features.add("llvm-plugins") + if config.clang_default_pie_on_linux: config.available_features.add("default-pie-on-linux") diff --git a/clang/test/lit.site.cfg.py.in b/clang/test/lit.site.cfg.py.in index 1cbd876ac5bb93..2cc70e52f1aa15 100644 --- a/clang/test/lit.site.cfg.py.in +++ b/clang/test/lit.site.cfg.py.in @@ -7,6 +7,7 @@ config.llvm_obj_root = path(r"@LLVM_BINARY_DIR@") config.llvm_tools_dir = lit_config.substitute(path(r"@LLVM_TOOLS_DIR@")) config.llvm_libs_dir = lit_config.substitute(path(r"@LLVM_LIBS_DIR@")) config.llvm_shlib_dir = lit_config.substitute(path(r"@SHLIBDIR@")) +config.llvm_shlib_ext = "@SHLIBEXT@" config.llvm_plugin_ext = "@LLVM_PLUGIN_EXT@" config.lit_tools_dir = path(r"@LLVM_LIT_TOOLS_DIR@") config.errc_messages = "@LLVM_LIT_ERRC_MESSAGES@" @@ -39,7 +40,10 @@ config.python_executable = "@Python3_EXECUTABLE@" config.use_z3_solver = lit_config.params.get('USE_Z3_SOLVER', "@USE_Z3_SOLVER@") config.has_plugins = @CLANG_PLUGIN_SUPPORT@ config.clang_vendor_uti = "@CLANG_VENDOR_UTI@" +config.llvm_examples = @LLVM_BUILD_EXAMPLES@ +config.llvm_linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@ config.llvm_external_lit = path(r"@LLVM_EXTERNAL_LIT@") +config.llvm_has_plugins = @LLVM_ENABLE_PLUGINS@ config.standalone_build = @CLANG_BUILT_STANDALONE@ config.ppc_linux_default_ieeelongdouble = @PPC_LINUX_DEFAULT_IEEELONGDOUBLE@ config.have_llvm_driver = @LLVM_TOOL_LLVM_DRIVER_BUILD@ diff --git a/clang/tools/clang-linker-wrapper/CMakeLists.txt b/clang/tools/clang-linker-wrapper/CMakeLists.txt index 5556869affaa62..bf37d8031025ed 100644 --- a/clang/tools/clang-linker-wrapper/CMakeLists.txt +++ b/clang/tools/clang-linker-wrapper/CMakeLists.txt @@ -41,3 +41,5 @@ target_link_libraries(clang-linker-wrapper PRIVATE ${CLANG_LINKER_WRAPPER_LIB_DEPS} ) + +export_executable_symbols_for_plugins(clang-linker-wrapper) diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp index 9027076119cf94..cb4cc5debae879 100644 --- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp +++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp @@ -37,6 +37,8 @@ #include "llvm/Option/ArgList.h" #include "llvm/Option/OptTable.h" #include "llvm/Option/Option.h" +#include "llvm/Passes/PassPlugin.h" +#include "llvm/Remarks/HotnessThresholdParser.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Errc.h" #include "llvm/Support/FileOutputBuffer.h" @@ -62,6 +64,54 @@ using namespace llvm; using namespace llvm::opt; using namespace llvm::object; +// Various tools (e.g., llc and opt) duplicate this series of declarations for +// options related to passes and remarks. + +static cl::opt RemarksWithHotness( + "pass-remarks-with-hotness", + cl::desc("With PGO, include profile count in optimization remarks"), + cl::Hidden); + +static cl::opt, false, remarks::HotnessThresholdParser> + RemarksHotnessThreshold( + "pass-remarks-hotness-threshold", + cl::desc("Minimum profile count required for " + "an optimization remark to be output. " + "Use 'auto' to apply the threshold from profile summary."), + cl::value_desc("N or 'auto'"), cl::init(0), cl::Hidden); + +static cl::opt + RemarksFilename("pass-remarks-output", + cl::desc("Output filename for pass remarks"), + cl::value_desc("filename")); + +static cl::opt + RemarksPasses("pass-remarks-filter", + cl::desc("Only record optimization remarks from passes whose " + "names match the given regular expression"), + cl::value_desc("regex")); + +static cl::opt RemarksFormat( + "pass-remarks-format", + cl::desc("The format used for serializing remarks (default: YAML)"), + cl::value_desc("format"), cl::init("yaml")); + +static cl::list + PassPlugins("load-pass-plugin", + cl::desc("Load passes from plugin library")); + +static cl::opt PassPipeline( + "passes", + cl::desc( + "A textual description of the pass pipeline. To have analysis passes " + "available before a certain pass, add 'require'. " + "'-passes' overrides the pass pipeline (but not all effects) from " + "specifying '--opt-level=O?' (O2 is the default) to " + "clang-linker-wrapper. Be sure to include the corresponding " + "'default' in '-passes'.")); +static cl::alias PassPipeline2("p", cl::aliasopt(PassPipeline), + cl::desc("Alias for -passes")); + /// Path of the current binary. static const char *LinkerExecutable; @@ -628,6 +678,12 @@ std::unique_ptr createLTO( Conf.CPU = Arch.str(); Conf.Options = codegen::InitTargetOptionsFromCodeGenFlags(Triple); + Conf.RemarksFilename = RemarksFilename; + Conf.RemarksPasses = RemarksPasses; + Conf.RemarksWithHotness = RemarksWithHotness; + Conf.RemarksHotnessThreshold = RemarksHotnessThreshold; + Conf.RemarksFormat = RemarksFormat; + StringRef OptLevel = Args.getLastArgValue(OPT_opt_level, "O2"); Conf.MAttrs = Features; std::optional CGOptLevelOrNone = @@ -637,6 +693,17 @@ std::unique_ptr createLTO( Conf.OptLevel = OptLevel[1] - '0'; Conf.DefaultTriple = Triple.getTriple(); + // TODO: Should we complain about combining --opt-level and -passes, as opt + // does? That might be too limiting in clang-linker-wrapper, so for now we + // just warn in the help entry for -passes that the default corresponding + // to --opt-level=O? should be included there. The problem is that + // --opt-level produces effects in clang-linker-wrapper beyond what -passes + // appears to be able to achieve, so rejecting the combination of --opt-level + // and -passes would apparently make it impossible to combine those effects + // with a custom pass pipeline. + Conf.OptPipeline = PassPipeline; + Conf.PassPlugins = PassPlugins; + LTOError = false; Conf.DiagHandler = diagnosticHandler; @@ -1660,6 +1727,13 @@ int main(int Argc, char **Argv) { NewArgv.push_back(Arg->getValue()); for (const opt::Arg *Arg : Args.filtered(OPT_offload_opt_eq_minus)) NewArgv.push_back(Args.MakeArgString(StringRef("-") + Arg->getValue())); + SmallVector PluginList; + PassPlugins.setCallback([&](const std::string &PluginPath) { + auto Plugin = PassPlugin::Load(PluginPath); + if (!Plugin) + report_fatal_error(Plugin.takeError(), /*gen_crash_diag=*/false); + PluginList.emplace_back(Plugin.get()); + }); cl::ParseCommandLineOptions(NewArgv.size(), &NewArgv[0]); Verbose = Args.hasArg(OPT_verbose); diff --git a/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td b/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td index eb31b98a3f5452..9c27e588fc4f57 100644 --- a/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td +++ b/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td @@ -94,9 +94,13 @@ def linker_arg_EQ : Joined<["--"], "linker-arg=">, // Arguments for the LLVM backend. def mllvm : Separate<["-"], "mllvm">, Flags<[WrapperOnlyOption]>, - MetaVarName<"">, HelpText<"Arguments passed to the LLVM invocation">; + MetaVarName<"">, + HelpText<"Arguments passed to LLVM, including Clang invocations, for which " + "the '-mllvm' prefix is preserved. Use '-mllvm --help' for a list " + "of options.">; def offload_opt_eq_minus : Joined<["--", "-"], "offload-opt=-">, Flags<[HelpHidden, WrapperOnlyOption]>, - HelpText<"Options passed to LLVM">; + HelpText<"Options passed to LLVM, not including the Clang invocation. Use " + "'--offload-opt=--help' for a list of options.">; // Standard linker flags also used by the linker wrapper. def sysroot_EQ : Joined<["--"], "sysroot=">, HelpText<"Set the system root">;