Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[PGO] Add ability to mark cold functions as optsize/minsize/optnone #69030

Merged
merged 7 commits into from
Feb 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions clang/lib/CodeGen/BackendUtil.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -748,7 +748,8 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
CodeGenOpts.InstrProfileOutput.empty() ? getDefaultProfileGenName()
: CodeGenOpts.InstrProfileOutput,
"", "", CodeGenOpts.MemoryProfileUsePath, nullptr, PGOOptions::IRInstr,
PGOOptions::NoCSAction, CodeGenOpts.DebugInfoForProfiling,
PGOOptions::NoCSAction, PGOOptions::ColdFuncOpt::Default,
CodeGenOpts.DebugInfoForProfiling,
/*PseudoProbeForProfiling=*/false, CodeGenOpts.AtomicProfileUpdate);
else if (CodeGenOpts.hasProfileIRUse()) {
// -fprofile-use.
Expand All @@ -757,28 +758,32 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
PGOOpt = PGOOptions(
CodeGenOpts.ProfileInstrumentUsePath, "",
CodeGenOpts.ProfileRemappingFile, CodeGenOpts.MemoryProfileUsePath, VFS,
PGOOptions::IRUse, CSAction, CodeGenOpts.DebugInfoForProfiling);
PGOOptions::IRUse, CSAction, PGOOptions::ColdFuncOpt::Default,
CodeGenOpts.DebugInfoForProfiling);
} else if (!CodeGenOpts.SampleProfileFile.empty())
// -fprofile-sample-use
PGOOpt = PGOOptions(
CodeGenOpts.SampleProfileFile, "", CodeGenOpts.ProfileRemappingFile,
CodeGenOpts.MemoryProfileUsePath, VFS, PGOOptions::SampleUse,
PGOOptions::NoCSAction, CodeGenOpts.DebugInfoForProfiling,
CodeGenOpts.PseudoProbeForProfiling);
PGOOptions::NoCSAction, PGOOptions::ColdFuncOpt::Default,
CodeGenOpts.DebugInfoForProfiling, CodeGenOpts.PseudoProbeForProfiling);
else if (!CodeGenOpts.MemoryProfileUsePath.empty())
// -fmemory-profile-use (without any of the above options)
PGOOpt = PGOOptions("", "", "", CodeGenOpts.MemoryProfileUsePath, VFS,
PGOOptions::NoAction, PGOOptions::NoCSAction,
PGOOptions::ColdFuncOpt::Default,
CodeGenOpts.DebugInfoForProfiling);
else if (CodeGenOpts.PseudoProbeForProfiling)
// -fpseudo-probe-for-profiling
PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr,
PGOOptions::NoAction, PGOOptions::NoCSAction,
PGOOptions::ColdFuncOpt::Default,
CodeGenOpts.DebugInfoForProfiling, true);
else if (CodeGenOpts.DebugInfoForProfiling)
// -fdebug-info-for-profiling
PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr,
PGOOptions::NoAction, PGOOptions::NoCSAction, true);
PGOOptions::NoAction, PGOOptions::NoCSAction,
PGOOptions::ColdFuncOpt::Default, true);

// Check to see if we want to generate a CS profile.
if (CodeGenOpts.hasProfileCSIRInstr()) {
Expand All @@ -801,7 +806,8 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
? getDefaultProfileGenName()
: CodeGenOpts.InstrProfileOutput,
"", /*MemoryProfile=*/"", nullptr, PGOOptions::NoAction,
PGOOptions::CSIRInstr, CodeGenOpts.DebugInfoForProfiling);
PGOOptions::CSIRInstr, PGOOptions::ColdFuncOpt::Default,
CodeGenOpts.DebugInfoForProfiling);
}
if (TM)
TM->setPGOOption(PGOOpt);
Expand Down
3 changes: 3 additions & 0 deletions llvm/include/llvm/Support/PGOOptions.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,12 @@ class FileSystem;
struct PGOOptions {
enum PGOAction { NoAction, IRInstr, IRUse, SampleUse };
enum CSPGOAction { NoCSAction, CSIRInstr, CSIRUse };
enum class ColdFuncOpt { Default, OptSize, MinSize, OptNone };
PGOOptions(std::string ProfileFile, std::string CSProfileGenFile,
std::string ProfileRemappingFile, std::string MemoryProfile,
IntrusiveRefCntPtr<vfs::FileSystem> FS,
PGOAction Action = NoAction, CSPGOAction CSAction = NoCSAction,
ColdFuncOpt ColdType = ColdFuncOpt::Default,
bool DebugInfoForProfiling = false,
bool PseudoProbeForProfiling = false,
bool AtomicCounterUpdate = false);
Expand All @@ -44,6 +46,7 @@ struct PGOOptions {
std::string MemoryProfile;
PGOAction Action;
CSPGOAction CSAction;
ColdFuncOpt ColdOptType;
bool DebugInfoForProfiling;
bool PseudoProbeForProfiling;
bool AtomicCounterUpdate;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
//===- PGOForceFunctionAttrs.h - --------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_PGOFORCEFUNCTIONATTRS_H
#define LLVM_TRANSFORMS_INSTRUMENTATION_PGOFORCEFUNCTIONATTRS_H

#include "llvm/IR/PassManager.h"
#include "llvm/Support/PGOOptions.h"

namespace llvm {

struct PGOForceFunctionAttrsPass
: public PassInfoMixin<PGOForceFunctionAttrsPass> {
PGOForceFunctionAttrsPass(PGOOptions::ColdFuncOpt ColdType)
: ColdType(ColdType) {}
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);

private:
PGOOptions::ColdFuncOpt ColdType;
};

} // namespace llvm

#endif // LLVM_TRANSFORMS_INSTRUMENTATION_PGOFORCEFUNCTIONATTRS_H
12 changes: 8 additions & 4 deletions llvm/lib/LTO/LTOBackend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -243,19 +243,23 @@ static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM,
if (!Conf.SampleProfile.empty())
PGOOpt = PGOOptions(Conf.SampleProfile, "", Conf.ProfileRemapping,
/*MemoryProfile=*/"", FS, PGOOptions::SampleUse,
PGOOptions::NoCSAction, true);
PGOOptions::NoCSAction,
PGOOptions::ColdFuncOpt::Default, true);
else if (Conf.RunCSIRInstr) {
PGOOpt = PGOOptions("", Conf.CSIRProfile, Conf.ProfileRemapping,
/*MemoryProfile=*/"", FS, PGOOptions::IRUse,
PGOOptions::CSIRInstr, Conf.AddFSDiscriminator);
PGOOptions::CSIRInstr, PGOOptions::ColdFuncOpt::Default,
Conf.AddFSDiscriminator);
} else if (!Conf.CSIRProfile.empty()) {
PGOOpt = PGOOptions(Conf.CSIRProfile, "", Conf.ProfileRemapping,
/*MemoryProfile=*/"", FS, PGOOptions::IRUse,
PGOOptions::CSIRUse, Conf.AddFSDiscriminator);
PGOOptions::CSIRUse, PGOOptions::ColdFuncOpt::Default,
Conf.AddFSDiscriminator);
NoPGOWarnMismatch = !Conf.PGOWarnMismatch;
} else if (Conf.AddFSDiscriminator) {
PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr,
PGOOptions::NoAction, PGOOptions::NoCSAction, true);
PGOOptions::NoAction, PGOOptions::NoCSAction,
PGOOptions::ColdFuncOpt::Default, true);
}
TM->setPGOOption(PGOOpt);

Expand Down
1 change: 1 addition & 0 deletions llvm/lib/Passes/PassBuilder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@
#include "llvm/Transforms/Instrumentation/KCFI.h"
#include "llvm/Transforms/Instrumentation/MemProfiler.h"
#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
#include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h"
#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
#include "llvm/Transforms/Instrumentation/PoisonChecking.h"
#include "llvm/Transforms/Instrumentation/SanitizerBinaryMetadata.h"
Expand Down
10 changes: 10 additions & 0 deletions llvm/lib/Passes/PassBuilderPipelines.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@
#include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
#include "llvm/Transforms/Instrumentation/MemProfiler.h"
#include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h"
#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
#include "llvm/Transforms/Scalar/ADCE.h"
#include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
Expand Down Expand Up @@ -212,6 +213,12 @@ static cl::opt<bool>
cl::desc("Enable DFA jump threading"),
cl::init(false), cl::Hidden);

// TODO: turn on and remove flag
static cl::opt<bool> EnablePGOForceFunctionAttrs(
"enable-pgo-force-function-attrs",
cl::desc("Enable pass to set function attributes based on PGO profiles"),
cl::init(false));

static cl::opt<bool>
EnableHotColdSplit("hot-cold-split",
cl::desc("Enable hot-cold splitting pass"));
Expand Down Expand Up @@ -1137,6 +1144,9 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
if (EnableSyntheticCounts && !PGOOpt)
MPM.addPass(SyntheticCountsPropagation());

if (EnablePGOForceFunctionAttrs)
MPM.addPass(PGOForceFunctionAttrsPass(PGOOpt->ColdOptType));

MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/true));

if (EnableModuleInliner)
Expand Down
1 change: 1 addition & 0 deletions llvm/lib/Passes/PassRegistry.def
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ MODULE_PASS("lower-emutls", LowerEmuTLSPass())
MODULE_PASS("lower-global-dtors", LowerGlobalDtorsPass())
MODULE_PASS("lower-ifunc", LowerIFuncPass())
MODULE_PASS("lowertypetests", LowerTypeTestsPass())
MODULE_PASS("pgo-force-function-attrs", PGOForceFunctionAttrsPass(PGOOpt ? PGOOpt->ColdOptType : PGOOptions::ColdFuncOpt::Default))
MODULE_PASS("memprof-context-disambiguation", MemProfContextDisambiguation())
MODULE_PASS("memprof-module", ModuleMemProfilerPass())
MODULE_PASS("mergefunc", MergeFunctionsPass())
Expand Down
7 changes: 4 additions & 3 deletions llvm/lib/Support/PGOOptions.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,12 @@ PGOOptions::PGOOptions(std::string ProfileFile, std::string CSProfileGenFile,
std::string ProfileRemappingFile,
std::string MemoryProfile,
IntrusiveRefCntPtr<vfs::FileSystem> FS, PGOAction Action,
CSPGOAction CSAction, bool DebugInfoForProfiling,
bool PseudoProbeForProfiling, bool AtomicCounterUpdate)
CSPGOAction CSAction, ColdFuncOpt ColdType,
bool DebugInfoForProfiling, bool PseudoProbeForProfiling,
bool AtomicCounterUpdate)
: ProfileFile(ProfileFile), CSProfileGenFile(CSProfileGenFile),
ProfileRemappingFile(ProfileRemappingFile), MemoryProfile(MemoryProfile),
Action(Action), CSAction(CSAction),
Action(Action), CSAction(CSAction), ColdOptType(ColdType),
DebugInfoForProfiling(DebugInfoForProfiling ||
(Action == SampleUse && !PseudoProbeForProfiling)),
PseudoProbeForProfiling(PseudoProbeForProfiling),
Expand Down
1 change: 1 addition & 0 deletions llvm/lib/Transforms/Instrumentation/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ add_llvm_component_library(LLVMInstrumentation
InstrOrderFile.cpp
InstrProfiling.cpp
KCFI.cpp
PGOForceFunctionAttrs.cpp
PGOInstrumentation.cpp
PGOMemOPSizeOpt.cpp
PoisonChecking.cpp
Expand Down
61 changes: 61 additions & 0 deletions llvm/lib/Transforms/Instrumentation/PGOForceFunctionAttrs.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/IR/PassManager.h"
#include "llvm/Support/ErrorHandling.h"

using namespace llvm;

static bool shouldRunOnFunction(Function &F, ProfileSummaryInfo &PSI,
FunctionAnalysisManager &FAM) {
if (F.isDeclaration())
return false;
// Respect existing attributes.
if (F.hasOptNone() || F.hasOptSize() || F.hasMinSize())
return false;
if (F.hasFnAttribute(Attribute::Cold))
return true;
if (!PSI.hasProfileSummary())
return false;
BlockFrequencyInfo &BFI = FAM.getResult<BlockFrequencyAnalysis>(F);
return PSI.isFunctionColdInCallGraph(&F, BFI);
}

PreservedAnalyses PGOForceFunctionAttrsPass::run(Module &M,
ModuleAnalysisManager &AM) {
if (ColdType == PGOOptions::ColdFuncOpt::Default)
return PreservedAnalyses::all();
ProfileSummaryInfo &PSI = AM.getResult<ProfileSummaryAnalysis>(M);
FunctionAnalysisManager &FAM =
AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
bool MadeChange = false;
for (Function &F : M) {
if (!shouldRunOnFunction(F, PSI, FAM))
continue;
MadeChange = true;
switch (ColdType) {
case PGOOptions::ColdFuncOpt::Default:
llvm_unreachable("bailed out for default above");
break;
case PGOOptions::ColdFuncOpt::OptSize:
F.addFnAttr(Attribute::OptimizeForSize);
break;
case PGOOptions::ColdFuncOpt::MinSize:
F.addFnAttr(Attribute::MinSize);
break;
case PGOOptions::ColdFuncOpt::OptNone:
F.addFnAttr(Attribute::OptimizeNone);
F.addFnAttr(Attribute::NoInline);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What if functions originally has AlwaysInline attribute?

More generally, what if function originally has a conflicting attribute from what's being set here?

break;
}
}
return MadeChange ? PreservedAnalyses::none() : PreservedAnalyses::all();
}
102 changes: 102 additions & 0 deletions llvm/test/Instrumentation/PGOForceFunctionAttrs/basic.ll
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
; RUN: opt < %s -passes=pgo-force-function-attrs -pgo-kind=pgo-instr-use-pipeline -S -pgo-cold-func-opt=default | FileCheck %s --check-prefixes=NONE,CHECK
; RUN: opt < %s -passes=pgo-force-function-attrs -pgo-kind=pgo-instr-use-pipeline -S -pgo-cold-func-opt=optsize | FileCheck %s --check-prefixes=OPTSIZE,CHECK
; RUN: opt < %s -passes=pgo-force-function-attrs -pgo-kind=pgo-instr-use-pipeline -S -pgo-cold-func-opt=minsize | FileCheck %s --check-prefixes=MINSIZE,CHECK
; RUN: opt < %s -passes=pgo-force-function-attrs -pgo-kind=pgo-instr-use-pipeline -S -pgo-cold-func-opt=optnone | FileCheck %s --check-prefixes=OPTNONE,CHECK

; Should be no changes without profile data
; RUN: opt < %s -passes=pgo-force-function-attrs -S -pgo-cold-func-opt=minsize | FileCheck %s --check-prefixes=NONE,CHECK

; NONE-NOT: Function Attrs:
; OPTSIZE: Function Attrs: optsize{{$}}
; MINSIZE: Function Attrs: minsize{{$}}
; OPTNONE: Function Attrs: noinline optnone{{$}}
; CHECK: define void @cold()

; CHECK: Function Attrs: optsize{{$}}
; CHECK-NEXT: define void @cold_optsize()

; CHECK: Function Attrs: minsize{{$}}
; CHECK-NEXT: define void @cold_minsize()

; CHECK: Function Attrs: noinline optnone{{$}}
; CHECK-NEXT: define void @cold_optnone()

; NONE: Function Attrs: cold{{$}}
; OPTSIZE: Function Attrs: cold optsize{{$}}
; MINSIZE: Function Attrs: cold minsize{{$}}
; OPTNONE: Function Attrs: cold noinline optnone{{$}}
; CHECK-NEXT: define void @cold_attr()

; CHECK-NOT: Function Attrs: {{.*}}optsize
; CHECK-NOT: Function Attrs: {{.*}}minsize
; CHECK-NOT: Function Attrs: {{.*}}optnone

@s = global i32 0

define void @cold() !prof !27 {
store i32 1, ptr @s, align 4
ret void
}

define void @cold_optsize() optsize !prof !27 {
store i32 1, ptr @s, align 4
ret void
}

define void @cold_minsize() minsize !prof !27 {
store i32 1, ptr @s, align 4
ret void
}

define void @cold_optnone() noinline optnone !prof !27 {
store i32 1, ptr @s, align 4
ret void
}

define void @cold_attr() cold {
store i32 1, ptr @s, align 4
ret void
}

define void @hot() !prof !28 {
%l = load i32, ptr @s, align 4
%add = add nsw i32 %l, 4
store i32 %add, ptr @s, align 4
ret void
}

attributes #0 = { optsize }
attributes #1 = { minsize }
attributes #2 = { noinline optnone }

!llvm.module.flags = !{!0}

!0 = !{i32 1, !"ProfileSummary", !1}
!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
!2 = !{!"ProfileFormat", !"InstrProf"}
!3 = !{!"TotalCount", i64 9040}
!4 = !{!"MaxCount", i64 9000}
!5 = !{!"MaxInternalCount", i64 0}
!6 = !{!"MaxFunctionCount", i64 9000}
!7 = !{!"NumCounts", i64 5}
!8 = !{!"NumFunctions", i64 5}
!9 = !{!"DetailedSummary", !10}
!10 = !{!11, !12, !13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26}
!11 = !{i32 10000, i64 9000, i32 1}
!12 = !{i32 100000, i64 9000, i32 1}
!13 = !{i32 200000, i64 9000, i32 1}
!14 = !{i32 300000, i64 9000, i32 1}
!15 = !{i32 400000, i64 9000, i32 1}
!16 = !{i32 500000, i64 9000, i32 1}
!17 = !{i32 600000, i64 9000, i32 1}
!18 = !{i32 700000, i64 9000, i32 1}
!19 = !{i32 800000, i64 9000, i32 1}
!20 = !{i32 900000, i64 9000, i32 1}
!21 = !{i32 950000, i64 9000, i32 1}
!22 = !{i32 990000, i64 9000, i32 1}
!23 = !{i32 999000, i64 10, i32 5}
!24 = !{i32 999900, i64 10, i32 5}
!25 = !{i32 999990, i64 10, i32 5}
!26 = !{i32 999999, i64 10, i32 5}
!27 = !{!"function_entry_count", i64 10}
!28 = !{!"function_entry_count", i64 9000}
Loading
Loading