Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

PR for llvm/llvm-project#54059 #100

Open
wants to merge 1 commit into
base: release/14.x
Choose a base branch
from
Open

PR for llvm/llvm-project#54059 #100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions llvm/include/llvm/Analysis/LoopAccessAnalysis.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -406,8 +406,8 @@ class RuntimePointerChecking {
/// according to the assumptions that we've made during the analysis.
/// The method might also version the pointer stride according to \p Strides,
/// and add new predicates to \p PSE.
void insert(Loop *Lp, Value *Ptr, bool WritePtr, unsigned DepSetId,
unsigned ASId, const ValueToValueMap &Strides,
void insert(Loop *Lp, Value *Ptr, Type *AccessTy, bool WritePtr,
unsigned DepSetId, unsigned ASId, const ValueToValueMap &Strides,
PredicatedScalarEvolution &PSE);

/// No run-time memory checking is necessary.
Expand Down
115 changes: 63 additions & 52 deletions llvm/lib/Analysis/LoopAccessAnalysis.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,8 +189,9 @@ RuntimeCheckingPtrGroup::RuntimeCheckingPtrGroup(
///
/// There is no conflict when the intervals are disjoint:
/// NoConflict = (P2.Start >= P1.End) || (P1.Start >= P2.End)
void RuntimePointerChecking::insert(Loop *Lp, Value *Ptr, bool WritePtr,
unsigned DepSetId, unsigned ASId,
void RuntimePointerChecking::insert(Loop *Lp, Value *Ptr, Type *AccessTy,
bool WritePtr, unsigned DepSetId,
unsigned ASId,
const ValueToValueMap &Strides,
PredicatedScalarEvolution &PSE) {
// Get the stride replaced scev.
Expand Down Expand Up @@ -227,8 +228,7 @@ void RuntimePointerChecking::insert(Loop *Lp, Value *Ptr, bool WritePtr,
// Add the size of the pointed element to ScEnd.
auto &DL = Lp->getHeader()->getModule()->getDataLayout();
Type *IdxTy = DL.getIndexType(Ptr->getType());
const SCEV *EltSizeSCEV =
SE->getStoreSizeOfExpr(IdxTy, Ptr->getType()->getPointerElementType());
const SCEV *EltSizeSCEV = SE->getStoreSizeOfExpr(IdxTy, AccessTy);
ScEnd = SE->getAddExpr(ScEnd, EltSizeSCEV);

Pointers.emplace_back(Ptr, ScStart, ScEnd, WritePtr, DepSetId, ASId, Sc);
Expand Down Expand Up @@ -522,19 +522,19 @@ class AccessAnalysis {
: TheLoop(TheLoop), AST(*AA), LI(LI), DepCands(DA), PSE(PSE) {}

/// Register a load and whether it is only read from.
void addLoad(MemoryLocation &Loc, bool IsReadOnly) {
void addLoad(MemoryLocation &Loc, Type *AccessTy, bool IsReadOnly) {
Value *Ptr = const_cast<Value*>(Loc.Ptr);
AST.add(Ptr, LocationSize::beforeOrAfterPointer(), Loc.AATags);
Accesses.insert(MemAccessInfo(Ptr, false));
Accesses[MemAccessInfo(Ptr, false)].insert(AccessTy);
if (IsReadOnly)
ReadOnlyPtr.insert(Ptr);
}

/// Register a store.
void addStore(MemoryLocation &Loc) {
void addStore(MemoryLocation &Loc, Type *AccessTy) {
Value *Ptr = const_cast<Value*>(Loc.Ptr);
AST.add(Ptr, LocationSize::beforeOrAfterPointer(), Loc.AATags);
Accesses.insert(MemAccessInfo(Ptr, true));
Accesses[MemAccessInfo(Ptr, true)].insert(AccessTy);
}

/// Check if we can emit a run-time no-alias check for \p Access.
Expand All @@ -545,12 +545,11 @@ class AccessAnalysis {
/// we will attempt to use additional run-time checks in order to get
/// the bounds of the pointer.
bool createCheckForAccess(RuntimePointerChecking &RtCheck,
MemAccessInfo Access,
MemAccessInfo Access, Type *AccessTy,
const ValueToValueMap &Strides,
DenseMap<Value *, unsigned> &DepSetId,
Loop *TheLoop, unsigned &RunningDepId,
unsigned ASId, bool ShouldCheckStride,
bool Assume);
unsigned ASId, bool ShouldCheckStride, bool Assume);

/// Check whether we can check the pointers at runtime for
/// non-intersection.
Expand Down Expand Up @@ -583,14 +582,15 @@ class AccessAnalysis {
MemAccessInfoList &getDependenciesToCheck() { return CheckDeps; }

private:
typedef SetVector<MemAccessInfo> PtrAccessSet;
typedef MapVector<MemAccessInfo, SmallSetVector<Type *, 1>> PtrAccessMap;

/// Go over all memory access and check whether runtime pointer checks
/// are needed and build sets of dependency check candidates.
void processMemAccesses();

/// Set of all accesses.
PtrAccessSet Accesses;
/// Map of all accesses. Values are the types used to access memory pointed to
/// by the pointer.
PtrAccessMap Accesses;

/// The loop being checked.
const Loop *TheLoop;
Expand Down Expand Up @@ -652,12 +652,12 @@ static bool hasComputableBounds(PredicatedScalarEvolution &PSE,

/// Check whether a pointer address cannot wrap.
static bool isNoWrap(PredicatedScalarEvolution &PSE,
const ValueToValueMap &Strides, Value *Ptr, Loop *L) {
const ValueToValueMap &Strides, Value *Ptr, Type *AccessTy,
Loop *L) {
const SCEV *PtrScev = PSE.getSCEV(Ptr);
if (PSE.getSE()->isLoopInvariant(PtrScev, L))
return true;

Type *AccessTy = Ptr->getType()->getPointerElementType();
int64_t Stride = getPtrStride(PSE, AccessTy, Ptr, L, Strides);
if (Stride == 1 || PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW))
return true;
Expand Down Expand Up @@ -689,7 +689,7 @@ static void visitPointers(Value *StartPtr, const Loop &InnermostLoop,
}

bool AccessAnalysis::createCheckForAccess(RuntimePointerChecking &RtCheck,
MemAccessInfo Access,
MemAccessInfo Access, Type *AccessTy,
const ValueToValueMap &StridesMap,
DenseMap<Value *, unsigned> &DepSetId,
Loop *TheLoop, unsigned &RunningDepId,
Expand All @@ -702,7 +702,7 @@ bool AccessAnalysis::createCheckForAccess(RuntimePointerChecking &RtCheck,

// When we run after a failing dependency check we have to make sure
// we don't have wrapping pointers.
if (ShouldCheckWrap && !isNoWrap(PSE, StridesMap, Ptr, TheLoop)) {
if (ShouldCheckWrap && !isNoWrap(PSE, StridesMap, Ptr, AccessTy, TheLoop)) {
auto *Expr = PSE.getSCEV(Ptr);
if (!Assume || !isa<SCEVAddRecExpr>(Expr))
return false;
Expand All @@ -723,11 +723,11 @@ bool AccessAnalysis::createCheckForAccess(RuntimePointerChecking &RtCheck,
DepId = RunningDepId++;

bool IsWrite = Access.getInt();
RtCheck.insert(TheLoop, Ptr, IsWrite, DepId, ASId, StridesMap, PSE);
RtCheck.insert(TheLoop, Ptr, AccessTy, IsWrite, DepId, ASId, StridesMap, PSE);
LLVM_DEBUG(dbgs() << "LAA: Found a runtime check ptr:" << *Ptr << '\n');

return true;
}
}

bool AccessAnalysis::canCheckPtrAtRT(RuntimePointerChecking &RtCheck,
ScalarEvolution *SE, Loop *TheLoop,
Expand Down Expand Up @@ -788,12 +788,15 @@ bool AccessAnalysis::canCheckPtrAtRT(RuntimePointerChecking &RtCheck,
}

for (auto &Access : AccessInfos) {
if (!createCheckForAccess(RtCheck, Access, StridesMap, DepSetId, TheLoop,
RunningDepId, ASId, ShouldCheckWrap, false)) {
LLVM_DEBUG(dbgs() << "LAA: Can't find bounds for ptr:"
<< *Access.getPointer() << '\n');
Retries.push_back(Access);
CanDoAliasSetRT = false;
for (auto &AccessTy : Accesses[Access]) {
if (!createCheckForAccess(RtCheck, Access, AccessTy, StridesMap,
DepSetId, TheLoop, RunningDepId, ASId,
ShouldCheckWrap, false)) {
LLVM_DEBUG(dbgs() << "LAA: Can't find bounds for ptr:"
<< *Access.getPointer() << '\n');
Retries.push_back(Access);
CanDoAliasSetRT = false;
}
}
}

Expand All @@ -815,13 +818,16 @@ bool AccessAnalysis::canCheckPtrAtRT(RuntimePointerChecking &RtCheck,
// We know that we need these checks, so we can now be more aggressive
// and add further checks if required (overflow checks).
CanDoAliasSetRT = true;
for (auto Access : Retries)
if (!createCheckForAccess(RtCheck, Access, StridesMap, DepSetId,
TheLoop, RunningDepId, ASId,
ShouldCheckWrap, /*Assume=*/true)) {
CanDoAliasSetRT = false;
break;
for (auto Access : Retries) {
for (auto &AccessTy : Accesses[Access]) {
if (!createCheckForAccess(RtCheck, Access, AccessTy, StridesMap,
DepSetId, TheLoop, RunningDepId, ASId,
ShouldCheckWrap, /*Assume=*/true)) {
CanDoAliasSetRT = false;
break;
}
}
}
}

CanDoRT &= CanDoAliasSetRT;
Expand Down Expand Up @@ -886,9 +892,12 @@ void AccessAnalysis::processMemAccesses() {
LLVM_DEBUG(dbgs() << "LAA: Accesses(" << Accesses.size() << "):\n");
LLVM_DEBUG({
for (auto A : Accesses)
dbgs() << "\t" << *A.getPointer() << " (" <<
(A.getInt() ? "write" : (ReadOnlyPtr.count(A.getPointer()) ?
"read-only" : "read")) << ")\n";
dbgs() << "\t" << *A.first.getPointer() << " ("
<< (A.first.getInt()
? "write"
: (ReadOnlyPtr.count(A.first.getPointer()) ? "read-only"
: "read"))
<< ")\n";
});

// The AliasSetTracker has nicely partitioned our pointers by metadata
Expand All @@ -907,24 +916,24 @@ void AccessAnalysis::processMemAccesses() {
UnderlyingObjToAccessMap ObjToLastAccess;

// Set of access to check after all writes have been processed.
PtrAccessSet DeferredAccesses;
PtrAccessMap DeferredAccesses;

// Iterate over each alias set twice, once to process read/write pointers,
// and then to process read-only pointers.
for (int SetIteration = 0; SetIteration < 2; ++SetIteration) {
bool UseDeferred = SetIteration > 0;
PtrAccessSet &S = UseDeferred ? DeferredAccesses : Accesses;
PtrAccessMap &S = UseDeferred ? DeferredAccesses : Accesses;

for (const auto &AV : AS) {
Value *Ptr = AV.getValue();

// For a single memory access in AliasSetTracker, Accesses may contain
// both read and write, and they both need to be handled for CheckDeps.
for (const auto &AC : S) {
if (AC.getPointer() != Ptr)
if (AC.first.getPointer() != Ptr)
continue;

bool IsWrite = AC.getInt();
bool IsWrite = AC.first.getInt();

// If we're using the deferred access set, then it contains only
// reads.
Expand All @@ -946,7 +955,9 @@ void AccessAnalysis::processMemAccesses() {
// consecutive as "read-only" pointers (so that we check
// "a[b[i]] +="). Hence, we need the second check for "!IsWrite".
if (!UseDeferred && IsReadOnlyPtr) {
DeferredAccesses.insert(Access);
// We only use the pointer keys, the types vector values don't
// matter.
DeferredAccesses.insert({Access, {}});
continue;
}

Expand Down Expand Up @@ -1518,8 +1529,8 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
Value *BPtr = B.getPointer();
bool AIsWrite = A.getInt();
bool BIsWrite = B.getInt();
Type *ATy = APtr->getType()->getPointerElementType();
Type *BTy = BPtr->getType()->getPointerElementType();
Type *ATy = getLoadStoreType(InstMap[AIdx]);
Type *BTy = getLoadStoreType(InstMap[BIdx]);

// Two reads are independent.
if (!AIsWrite && !BIsWrite)
Expand Down Expand Up @@ -1842,8 +1853,6 @@ bool LoopAccessInfo::canAnalyzeLoop() {
void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI,
const TargetLibraryInfo *TLI,
DominatorTree *DT) {
typedef SmallPtrSet<Value*, 16> ValueSet;

// Holds the Load and Store instructions.
SmallVector<LoadInst *, 16> Loads;
SmallVector<StoreInst *, 16> Stores;
Expand Down Expand Up @@ -1975,11 +1984,11 @@ void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI,
// for read and once for write, it will only appear once (on the write
// list). This is okay, since we are going to check for conflicts between
// writes and between reads and writes, but not between reads and reads.
ValueSet Seen;
SmallSet<std::pair<Value *, Type *>, 16> Seen;

// Record uniform store addresses to identify if we have multiple stores
// to the same address.
ValueSet UniformStores;
SmallPtrSet<Value *, 16> UniformStores;

for (StoreInst *ST : Stores) {
Value *Ptr = ST->getPointerOperand();
Expand All @@ -1990,7 +1999,8 @@ void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI,

// If we did *not* see this pointer before, insert it to the read-write
// list. At this phase it is only a 'write' list.
if (Seen.insert(Ptr).second) {
Type *AccessTy = getLoadStoreType(ST);
if (Seen.insert({Ptr, AccessTy}).second) {
++NumReadWrites;

MemoryLocation Loc = MemoryLocation::get(ST);
Expand All @@ -2001,9 +2011,9 @@ void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI,
Loc.AATags.TBAA = nullptr;

visitPointers(const_cast<Value *>(Loc.Ptr), *TheLoop,
[&Accesses, Loc](Value *Ptr) {
[&Accesses, AccessTy, Loc](Value *Ptr) {
MemoryLocation NewLoc = Loc.getWithNewPtr(Ptr);
Accesses.addStore(NewLoc);
Accesses.addStore(NewLoc, AccessTy);
});
}
}
Expand All @@ -2027,7 +2037,8 @@ void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI,
// read a few words, modify, and write a few words, and some of the
// words may be written to the same address.
bool IsReadOnlyPtr = false;
if (Seen.insert(Ptr).second ||
Type *AccessTy = getLoadStoreType(LD);
if (Seen.insert({Ptr, AccessTy}).second ||
!getPtrStride(*PSE, LD->getType(), Ptr, TheLoop, SymbolicStrides)) {
++NumReads;
IsReadOnlyPtr = true;
Expand All @@ -2049,9 +2060,9 @@ void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI,
Loc.AATags.TBAA = nullptr;

visitPointers(const_cast<Value *>(Loc.Ptr), *TheLoop,
[&Accesses, Loc, IsReadOnlyPtr](Value *Ptr) {
[&Accesses, AccessTy, Loc, IsReadOnlyPtr](Value *Ptr) {
MemoryLocation NewLoc = Loc.getWithNewPtr(Ptr);
Accesses.addLoad(NewLoc, IsReadOnlyPtr);
Accesses.addLoad(NewLoc, AccessTy, IsReadOnlyPtr);
});
}

Expand Down
Loading