From 85ff15acd30f6b1b6e5ead370ec9d99e6a67b0ab Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Sun, 5 Apr 2020 18:53:24 -0700 Subject: [PATCH 01/20] Add indexTrieRoot to the SILModule to share across Analyses. ...and avoid reallocation. This is immediately necessary for LICM, in addition to its current uses. I suspect this could be used by many passes that work with addresses. RLE/DSE should absolutely migrate to it. --- .../{SILOptimizer/Utils => Basic}/IndexTrie.h | 1 + include/swift/SIL/SILModule.h | 7 +++++++ .../SILOptimizer/Analysis/AccessSummaryAnalysis.h | 15 +++------------ lib/SIL/IR/SILModule.cpp | 3 ++- .../Analysis/AccessSummaryAnalysis.cpp | 11 +++++------ .../Mandatory/DiagnoseStaticExclusivity.cpp | 2 +- lib/SILOptimizer/Transforms/CopyPropagation.cpp | 2 +- .../Transforms/DeadObjectElimination.cpp | 2 +- 8 files changed, 21 insertions(+), 22 deletions(-) rename include/swift/{SILOptimizer/Utils => Basic}/IndexTrie.h (98%) diff --git a/include/swift/SILOptimizer/Utils/IndexTrie.h b/include/swift/Basic/IndexTrie.h similarity index 98% rename from include/swift/SILOptimizer/Utils/IndexTrie.h rename to include/swift/Basic/IndexTrie.h index b7986cb4d2e22..9428ff2e87c8f 100644 --- a/include/swift/SILOptimizer/Utils/IndexTrie.h +++ b/include/swift/Basic/IndexTrie.h @@ -13,6 +13,7 @@ #ifndef SWIFT_SILOPTIMIZER_UTILS_INDEXTREE_H #define SWIFT_SILOPTIMIZER_UTILS_INDEXTREE_H +#include "swift/Basic/LLVM.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include diff --git a/include/swift/SIL/SILModule.h b/include/swift/SIL/SILModule.h index 33f7b07ff875f..162606ab5a747 100644 --- a/include/swift/SIL/SILModule.h +++ b/include/swift/SIL/SILModule.h @@ -21,6 +21,7 @@ #include "swift/AST/Builtins.h" #include "swift/AST/SILLayout.h" #include "swift/AST/SILOptions.h" +#include "swift/Basic/IndexTrie.h" #include "swift/Basic/LangOptions.h" #include "swift/Basic/ProfileCounter.h" #include "swift/Basic/Range.h" @@ -256,6 +257,10 @@ class SILModule { /// The indexed profile data to be used for PGO, or nullptr. std::unique_ptr PGOReader; + /// A trie of integer indices that gives pointer identity to a path of + /// projections, shared between all functions in the module. + std::unique_ptr indexTrieRoot; + /// The options passed into this SILModule. const SILOptions &Options; @@ -655,6 +660,8 @@ class SILModule { PGOReader = std::move(IPR); } + IndexTrieNode *getIndexTrieRoot() { return indexTrieRoot.get(); } + /// Can value operations (copies and destroys) on the given lowered type /// be performed in this module? bool isTypeABIAccessible(SILType type, diff --git a/include/swift/SILOptimizer/Analysis/AccessSummaryAnalysis.h b/include/swift/SILOptimizer/Analysis/AccessSummaryAnalysis.h index e71b6c58fea36..d8a448e7ee85a 100644 --- a/include/swift/SILOptimizer/Analysis/AccessSummaryAnalysis.h +++ b/include/swift/SILOptimizer/Analysis/AccessSummaryAnalysis.h @@ -19,10 +19,10 @@ #ifndef SWIFT_SILOPTIMIZER_ANALYSIS_ACCESS_SUMMARY_ANALYSIS_H_ #define SWIFT_SILOPTIMIZER_ANALYSIS_ACCESS_SUMMARY_ANALYSIS_H_ +#include "swift/Basic/IndexTrie.h" #include "swift/SIL/SILFunction.h" #include "swift/SIL/SILInstruction.h" #include "swift/SILOptimizer/Analysis/BottomUpIPAnalysis.h" -#include "swift/SILOptimizer/Utils/IndexTrie.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" @@ -173,22 +173,13 @@ class AccessSummaryAnalysis : public BottomUpIPAnalysis { llvm::SpecificBumpPtrAllocator Allocator; - /// A trie of integer indices that gives pointer identity to a path of - /// projections. This is shared between all functions in the module. - std::unique_ptr SubPathTrie; - public: - AccessSummaryAnalysis() : BottomUpIPAnalysis(SILAnalysisKind::AccessSummary) { - SubPathTrie.reset(new IndexTrieNode()); - } + AccessSummaryAnalysis() + : BottomUpIPAnalysis(SILAnalysisKind::AccessSummary) {} /// Returns a summary of the accesses performed by the given function. const FunctionSummary &getOrCreateSummary(SILFunction *Fn); - IndexTrieNode *getSubPathTrieRoot() { - return SubPathTrie.get(); - } - /// Returns an IndexTrieNode that represents the single subpath accessed from /// BAI or the root if no such node exists. const IndexTrieNode *findSubPathAccessed(BeginAccessInst *BAI); diff --git a/lib/SIL/IR/SILModule.cpp b/lib/SIL/IR/SILModule.cpp index 3c15b8f2ec1d3..cc30fc7f5d07b 100644 --- a/lib/SIL/IR/SILModule.cpp +++ b/lib/SIL/IR/SILModule.cpp @@ -94,7 +94,8 @@ class SILModule::SerializationCallback final SILModule::SILModule(llvm::PointerUnion context, Lowering::TypeConverter &TC, const SILOptions &Options) - : Stage(SILStage::Raw), Options(Options), serialized(false), + : Stage(SILStage::Raw), indexTrieRoot(new IndexTrieNode()), + Options(Options), serialized(false), regDeserializationNotificationHandlerForNonTransparentFuncOME(false), regDeserializationNotificationHandlerForAllFuncOME(false), SerializeSILAction(), Types(TC) { diff --git a/lib/SILOptimizer/Analysis/AccessSummaryAnalysis.cpp b/lib/SILOptimizer/Analysis/AccessSummaryAnalysis.cpp index d3906e4af74d2..0d0a7de0fda58 100644 --- a/lib/SILOptimizer/Analysis/AccessSummaryAnalysis.cpp +++ b/lib/SILOptimizer/Analysis/AccessSummaryAnalysis.cpp @@ -49,9 +49,9 @@ void AccessSummaryAnalysis::processFunction(FunctionInfo *info, /// started by a begin_access and any flows of the arguments to other /// functions. void AccessSummaryAnalysis::processArgument(FunctionInfo *info, - SILFunctionArgument *argument, - ArgumentSummary &summary, - FunctionOrder &order) { + SILFunctionArgument *argument, + ArgumentSummary &summary, + FunctionOrder &order) { unsigned argumentIndex = argument->getIndex(); // Use a worklist to track argument uses to be processed. @@ -77,7 +77,7 @@ void AccessSummaryAnalysis::processArgument(FunctionInfo *info, // call. if (!callee || callee->empty()) { summary.mergeWith(SILAccessKind::Modify, apply.getLoc(), - getSubPathTrieRoot()); + apply.getModule().getIndexTrieRoot()); continue; } unsigned operandNumber = operand->getOperandNumber(); @@ -468,7 +468,6 @@ AccessSummaryAnalysis::getOrCreateSummary(SILFunction *fn) { void AccessSummaryAnalysis::AccessSummaryAnalysis::invalidate() { FunctionInfos.clear(); Allocator.DestroyAll(); - SubPathTrie.reset(new IndexTrieNode()); } void AccessSummaryAnalysis::invalidate(SILFunction *F, InvalidationKind K) { @@ -526,7 +525,7 @@ getSingleAddressProjectionUser(SingleValueInstruction *I) { const IndexTrieNode * AccessSummaryAnalysis::findSubPathAccessed(BeginAccessInst *BAI) { - IndexTrieNode *SubPath = getSubPathTrieRoot(); + IndexTrieNode *SubPath = BAI->getModule().getIndexTrieRoot(); // For each single-user projection of BAI, construct or get a node // from the trie representing the index of the field or tuple element diff --git a/lib/SILOptimizer/Mandatory/DiagnoseStaticExclusivity.cpp b/lib/SILOptimizer/Mandatory/DiagnoseStaticExclusivity.cpp index f27984e9ec524..eace1f0eac6bb 100644 --- a/lib/SILOptimizer/Mandatory/DiagnoseStaticExclusivity.cpp +++ b/lib/SILOptimizer/Mandatory/DiagnoseStaticExclusivity.cpp @@ -754,7 +754,7 @@ static void checkCaptureAccess(ApplySite Apply, AccessState &State) { // The unknown argument access is considered a modify of the root subpath. auto argAccess = RecordedAccess(SILAccessKind::Modify, Apply.getLoc(), - State.ASA->getSubPathTrieRoot()); + Apply.getModule().getIndexTrieRoot()); // Construct a conflicting RecordedAccess if one doesn't already exist. const AccessInfo &info = AccessIt->getSecond(); diff --git a/lib/SILOptimizer/Transforms/CopyPropagation.cpp b/lib/SILOptimizer/Transforms/CopyPropagation.cpp index 82bb2269f0d4f..fdb66d4055c72 100644 --- a/lib/SILOptimizer/Transforms/CopyPropagation.cpp +++ b/lib/SILOptimizer/Transforms/CopyPropagation.cpp @@ -123,12 +123,12 @@ /// ===----------------------------------------------------------------------=== #define DEBUG_TYPE "copy-propagation" +#include "swift/Basic/IndexTrie.h" #include "swift/SIL/InstructionUtils.h" #include "swift/SIL/Projection.h" #include "swift/SILOptimizer/PassManager/Passes.h" #include "swift/SILOptimizer/PassManager/Transforms.h" #include "swift/SILOptimizer/Utils/CFGOptUtils.h" -#include "swift/SILOptimizer/Utils/IndexTrie.h" #include "swift/SILOptimizer/Utils/InstOptUtils.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Statistic.h" diff --git a/lib/SILOptimizer/Transforms/DeadObjectElimination.cpp b/lib/SILOptimizer/Transforms/DeadObjectElimination.cpp index 384d7291e07d5..58a47795051b7 100644 --- a/lib/SILOptimizer/Transforms/DeadObjectElimination.cpp +++ b/lib/SILOptimizer/Transforms/DeadObjectElimination.cpp @@ -24,6 +24,7 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "dead-object-elim" +#include "swift/Basic/IndexTrie.h" #include "swift/AST/ResilienceExpansion.h" #include "swift/SIL/BasicBlockUtils.h" #include "swift/SIL/DebugUtils.h" @@ -38,7 +39,6 @@ #include "swift/SILOptimizer/Analysis/ArraySemantic.h" #include "swift/SILOptimizer/PassManager/Passes.h" #include "swift/SILOptimizer/PassManager/Transforms.h" -#include "swift/SILOptimizer/Utils/IndexTrie.h" #include "swift/SILOptimizer/Utils/InstOptUtils.h" #include "swift/SILOptimizer/Utils/SILSSAUpdater.h" #include "swift/SILOptimizer/Utils/ValueLifetime.h" From 7774f06f10eabe16ef6b157f92e8fab5e9e5f029 Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Thu, 27 Aug 2020 17:35:10 -0700 Subject: [PATCH 02/20] Change IndexTrie to allow signed indices --- include/swift/Basic/IndexTrie.h | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/include/swift/Basic/IndexTrie.h b/include/swift/Basic/IndexTrie.h index 9428ff2e87c8f..e9a5133a67162 100644 --- a/include/swift/Basic/IndexTrie.h +++ b/include/swift/Basic/IndexTrie.h @@ -22,15 +22,18 @@ namespace swift { // Trie node representing a sequence of unsigned integer indices. class IndexTrieNode { - static const unsigned RootIdx = ~0U; - unsigned Index; +public: + static const int RootIndex = std::numeric_limits::min(); + +private: + int Index; llvm::SmallVector Children; IndexTrieNode *Parent; public: - IndexTrieNode(): Index(RootIdx), Parent(nullptr) {} + IndexTrieNode() : Index(RootIndex), Parent(nullptr) {} - explicit IndexTrieNode(unsigned V, IndexTrieNode *P): Index(V), Parent(P) {} + explicit IndexTrieNode(int V, IndexTrieNode *P) : Index(V), Parent(P) {} IndexTrieNode(IndexTrieNode &) =delete; IndexTrieNode &operator=(const IndexTrieNode&) =delete; @@ -40,19 +43,18 @@ class IndexTrieNode { delete N; } - bool isRoot() const { return Index == RootIdx; } + bool isRoot() const { return Index == RootIndex; } bool isLeaf() const { return Children.empty(); } - unsigned getIndex() const { return Index; } + int getIndex() const { return Index; } - IndexTrieNode *getChild(unsigned Idx) { - assert(Idx != RootIdx); + IndexTrieNode *getChild(int Idx) { + assert(Idx != RootIndex); - auto I = std::lower_bound(Children.begin(), Children.end(), Idx, - [](IndexTrieNode *a, unsigned i) { - return a->Index < i; - }); + auto I = + std::lower_bound(Children.begin(), Children.end(), Idx, + [](IndexTrieNode *a, int i) { return a->Index < i; }); if (I != Children.end() && (*I)->Index == Idx) return *I; auto *N = new IndexTrieNode(Idx, this); From 2767b51d615d5fcf464557ef404c491f4259e262 Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Sun, 5 Apr 2020 18:51:43 -0700 Subject: [PATCH 03/20] Change Projection to support signed indices. Change ProjectionIndex for ref_tail_addr to std::numeric_limits::max(); This is necessary to disambiguate the tail elements from ref_element_addr field zero. --- include/swift/SIL/Projection.h | 39 +++++++------- lib/SIL/Utils/Projection.cpp | 53 ++++++++++--------- lib/SILOptimizer/LoopTransforms/ArrayOpt.h | 6 +-- .../LoopTransforms/ArrayPropertyOpt.cpp | 2 +- .../LoopTransforms/COWArrayOpt.cpp | 2 +- 5 files changed, 51 insertions(+), 51 deletions(-) diff --git a/include/swift/SIL/Projection.h b/include/swift/SIL/Projection.h index c78c11ed7e2a6..ece22804b153c 100644 --- a/include/swift/SIL/Projection.h +++ b/include/swift/SIL/Projection.h @@ -67,9 +67,9 @@ inline bool isStrictSubSeqRelation(SubSeqRelation_t Seq) { /// Extract an integer index from a SILValue. /// -/// Return true if IndexVal is a constant index representable as unsigned +/// Return true if IndexVal is a constant index representable as an /// int. We do not support symbolic projections yet. -bool getIntegerIndex(SILValue IndexVal, unsigned &IndexConst); +bool getIntegerIndex(SILValue IndexVal, int &IndexConst); /// The kind of projection that we are representing. /// @@ -136,11 +136,18 @@ static inline bool isCastProjectionKind(ProjectionKind Kind) { /// that immediately contains it. /// /// This lightweight utility maps a SIL address projection to an index. +/// +/// project_box does not have a projection index. At the SIL level, the box +/// storage is considered part of the same object as the. The box projection is +/// does not affect access path so that box projections can occur on distinct +/// phi paths in the address def-use chain. struct ProjectionIndex { + static constexpr int TailIndex = std::numeric_limits::max(); + SILValue Aggregate; - unsigned Index; + int Index = std::numeric_limits::min(); - explicit ProjectionIndex(SILValue V) : Index(~0U) { + explicit ProjectionIndex(SILValue V) { switch (V->getKind()) { default: break; @@ -163,16 +170,9 @@ struct ProjectionIndex { break; } case ValueKind::RefTailAddrInst: { - RefTailAddrInst *REA = cast(V); - Index = 0; - Aggregate = REA->getOperand(); - break; - } - case ValueKind::ProjectBoxInst: { - ProjectBoxInst *PBI = cast(V); - // A box has only a single payload. - Index = 0; - Aggregate = PBI->getOperand(); + RefTailAddrInst *RTA = cast(V); + Index = TailIndex; + Aggregate = RTA->getOperand(); break; } case ValueKind::TupleElementAddrInst: { @@ -233,8 +233,7 @@ class Projection { : Projection(dyn_cast(I)) {} explicit Projection(SingleValueInstruction *I); - Projection(ProjectionKind Kind, unsigned NewIndex) - : Value(Kind, NewIndex) {} + Projection(ProjectionKind Kind, int NewIndex) : Value(Kind, NewIndex) {} Projection(ProjectionKind Kind, TypeBase *Ptr) : Value(Kind, Ptr) {} @@ -252,10 +251,8 @@ class Projection { /// Convenience method for getting the underlying index. Assumes that this /// projection is valid. Otherwise it asserts. - unsigned getIndex() const { - return Value.getIndex(); - } - + int getIndex() const { return Value.getIndex(); } + unsigned getHash() const { return (unsigned)Value.getStorage(); } /// Determine if I is a value projection instruction whose corresponding @@ -359,7 +356,7 @@ class Projection { return nullptr; case ValueKind::IndexAddrInst: { auto *i = cast(v); - unsigned scalar; + int scalar; if (getIntegerIndex(i->getIndex(), scalar)) return i; return nullptr; diff --git a/lib/SIL/Utils/Projection.cpp b/lib/SIL/Utils/Projection.cpp index e94adb426d6ac..8c6b783c842cb 100644 --- a/lib/SIL/Utils/Projection.cpp +++ b/lib/SIL/Utils/Projection.cpp @@ -12,10 +12,11 @@ #define DEBUG_TYPE "sil-projection" #include "swift/SIL/Projection.h" +#include "swift/Basic/IndexTrie.h" #include "swift/Basic/NullablePtr.h" -#include "swift/SIL/SILBuilder.h" -#include "swift/SIL/InstructionUtils.h" #include "swift/SIL/DebugUtils.h" +#include "swift/SIL/InstructionUtils.h" +#include "swift/SIL/SILBuilder.h" #include "swift/SIL/SILUndef.h" #include "llvm/ADT/None.h" #include "llvm/Support/Debug.h" @@ -42,22 +43,27 @@ static_assert(std::is_standard_layout::value, /// Return true if IndexVal is a constant index representable as unsigned /// int. We do not support symbolic projections yet, only 32-bit unsigned /// integers. -bool swift::getIntegerIndex(SILValue IndexVal, unsigned &IndexConst) { - if (auto *IndexLiteral = dyn_cast(IndexVal)) { - APInt ConstInt = IndexLiteral->getValue(); - // IntegerLiterals are signed. - if (ConstInt.isIntN(32) && ConstInt.isNonNegative()) { - IndexConst = (unsigned)ConstInt.getSExtValue(); - return true; - } - } - return false; +bool swift::getIntegerIndex(SILValue IndexVal, int &IndexConst) { + auto *IndexLiteral = dyn_cast(IndexVal); + if (!IndexLiteral) + return false; + + APInt ConstInt = IndexLiteral->getValue(); + // Reserve 1 bit for encoding. See AccessPath::Index. + if (!ConstInt.isSignedIntN(31)) + return false; + + IndexConst = ConstInt.getSExtValue(); + assert(((IndexConst << 1) >> 1) == IndexConst); + return true; } //===----------------------------------------------------------------------===// // Projection //===----------------------------------------------------------------------===// +constexpr int ProjectionIndex::TailIndex; + Projection::Projection(SingleValueInstruction *I) : Value() { if (!I) return; @@ -72,21 +78,21 @@ Projection::Projection(SingleValueInstruction *I) : Value() { auto *SEAI = cast(I); Value = ValueTy(ProjectionKind::Struct, SEAI->getFieldIndex()); assert(getKind() == ProjectionKind::Struct); - assert(getIndex() == SEAI->getFieldIndex()); + assert(getIndex() == int(SEAI->getFieldIndex())); break; } case SILInstructionKind::StructExtractInst: { auto *SEI = cast(I); Value = ValueTy(ProjectionKind::Struct, SEI->getFieldIndex()); assert(getKind() == ProjectionKind::Struct); - assert(getIndex() == SEI->getFieldIndex()); + assert(getIndex() == int(SEI->getFieldIndex())); break; } case SILInstructionKind::RefElementAddrInst: { auto *REAI = cast(I); Value = ValueTy(ProjectionKind::Class, REAI->getFieldIndex()); assert(getKind() == ProjectionKind::Class); - assert(getIndex() == REAI->getFieldIndex()); + assert(getIndex() == int(REAI->getFieldIndex())); break; } case SILInstructionKind::RefTailAddrInst: { @@ -108,28 +114,28 @@ Projection::Projection(SingleValueInstruction *I) : Value() { auto *TEI = cast(I); Value = ValueTy(ProjectionKind::Tuple, TEI->getFieldIndex()); assert(getKind() == ProjectionKind::Tuple); - assert(getIndex() == TEI->getFieldIndex()); + assert(getIndex() == int(TEI->getFieldIndex())); break; } case SILInstructionKind::TupleElementAddrInst: { auto *TEAI = cast(I); Value = ValueTy(ProjectionKind::Tuple, TEAI->getFieldIndex()); assert(getKind() == ProjectionKind::Tuple); - assert(getIndex() == TEAI->getFieldIndex()); + assert(getIndex() == int(TEAI->getFieldIndex())); break; } case SILInstructionKind::UncheckedEnumDataInst: { auto *UEDI = cast(I); Value = ValueTy(ProjectionKind::Enum, UEDI->getElementNo()); assert(getKind() == ProjectionKind::Enum); - assert(getIndex() == UEDI->getElementNo()); + assert(getIndex() == int(UEDI->getElementNo())); break; } case SILInstructionKind::UncheckedTakeEnumDataAddrInst: { auto *UTEDAI = cast(I); Value = ValueTy(ProjectionKind::Enum, UTEDAI->getElementNo()); assert(getKind() == ProjectionKind::Enum); - assert(getIndex() == UTEDAI->getElementNo()); + assert(getIndex() == int(UTEDAI->getElementNo())); break; } case SILInstructionKind::IndexAddrInst: { @@ -138,9 +144,10 @@ Projection::Projection(SingleValueInstruction *I) : Value() { // updated and a MaxLargeIndex will need to be used here. Currently we // represent large Indexes using a 64 bit integer, so we don't need to mess // with anything. - unsigned NewIndex = 0; + int NewIndex = 0; auto *IAI = cast(I); - if (getIntegerIndex(IAI->getIndex(), NewIndex)) { + // TODO: handle negative indices + if (getIntegerIndex(IAI->getIndex(), NewIndex) && NewIndex >= 0) { Value = ValueTy(ProjectionKind::Index, NewIndex); assert(getKind() == ProjectionKind::Index); assert(getIndex() == NewIndex); @@ -321,10 +328,6 @@ void Projection::getFirstLevelProjections( if (auto *C = Ty.getClassOrBoundGenericClass()) { unsigned Count = 0; - for (auto *superDecl = C->getSuperclassDecl(); superDecl != nullptr; - superDecl = superDecl->getSuperclassDecl()) { - Count += superDecl->getStoredProperties().size(); - } for (auto *VDecl : C->getStoredProperties()) { (void) VDecl; Projection P(ProjectionKind::Class, Count++); diff --git a/lib/SILOptimizer/LoopTransforms/ArrayOpt.h b/lib/SILOptimizer/LoopTransforms/ArrayOpt.h index 00bf1e0d9b492..2139d2a5aabec 100644 --- a/lib/SILOptimizer/LoopTransforms/ArrayOpt.h +++ b/lib/SILOptimizer/LoopTransforms/ArrayOpt.h @@ -63,7 +63,7 @@ class StructUseCollector { /// Do not form a path with an IndexAddrInst because we have no way to /// distinguish between indexing and subelement access. The same index could /// either refer to the next element (indexed) or a subelement. - static SILValue getAccessPath(SILValue V, SmallVectorImpl& Path) { + static SILValue getAccessPath(SILValue V, SmallVectorImpl &Path) { V = stripCasts(V); if (auto *IA = dyn_cast(V)) { // Don't include index_addr projections in the access path. We could if @@ -89,7 +89,7 @@ class StructUseCollector { VisitedSet Visited; /// Collect all uses of the value at the given address. - void collectUses(ValueBase *V, ArrayRef AccessPath) { + void collectUses(ValueBase *V, ArrayRef AccessPath) { // Save our old indent and increment. // Collect all users of the address and loads. collectAddressUses(V, AccessPath, nullptr); @@ -142,7 +142,7 @@ class StructUseCollector { /// StructVal is invalid, then the value is the address of the Struct. If /// StructVal is valid, the value is the address of an element within the /// Struct. - void collectAddressUses(ValueBase *V, ArrayRef AccessPathSuffix, + void collectAddressUses(ValueBase *V, ArrayRef AccessPathSuffix, Operand *StructVal) { for (auto *UI : V->getUses()) { // Keep the operand, not the instruction in the visited set. The same diff --git a/lib/SILOptimizer/LoopTransforms/ArrayPropertyOpt.cpp b/lib/SILOptimizer/LoopTransforms/ArrayPropertyOpt.cpp index 76a30bfaca3d2..9e9334ff59061 100644 --- a/lib/SILOptimizer/LoopTransforms/ArrayPropertyOpt.cpp +++ b/lib/SILOptimizer/LoopTransforms/ArrayPropertyOpt.cpp @@ -402,7 +402,7 @@ class ArrayPropertiesAnalysis { if (!Call.canHoist(Preheader->getTerminator(), DomTree)) return false; - SmallVector AccessPath; + SmallVector AccessPath; SILValue ArrayContainer = StructUseCollector::getAccessPath(Arr, AccessPath); diff --git a/lib/SILOptimizer/LoopTransforms/COWArrayOpt.cpp b/lib/SILOptimizer/LoopTransforms/COWArrayOpt.cpp index 1141d8dd7ea6d..ed54541cfb107 100644 --- a/lib/SILOptimizer/LoopTransforms/COWArrayOpt.cpp +++ b/lib/SILOptimizer/LoopTransforms/COWArrayOpt.cpp @@ -843,7 +843,7 @@ bool COWArrayOpt::hoistMakeMutable(ArraySemanticsCall MakeMutable, return false; } - SmallVector AccessPath; + SmallVector AccessPath; SILValue ArrayContainer = StructUseCollector::getAccessPath(CurrentArrayAddr, AccessPath); bool arrayContainerIsUnique = checkUniqueArrayContainer(ArrayContainer); From 7554a6aa3134b8b6078aac4ad21b58871a55d757 Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Wed, 26 Aug 2020 22:23:27 -0700 Subject: [PATCH 04/20] Fix a pointer addition bug in SwiftReflectionTest. --- stdlib/private/SwiftReflectionTest/SwiftReflectionTest.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/private/SwiftReflectionTest/SwiftReflectionTest.swift b/stdlib/private/SwiftReflectionTest/SwiftReflectionTest.swift index b077bd4923a2e..cb2350b845f2c 100644 --- a/stdlib/private/SwiftReflectionTest/SwiftReflectionTest.swift +++ b/stdlib/private/SwiftReflectionTest/SwiftReflectionTest.swift @@ -216,7 +216,7 @@ internal struct ReflectionInfo : Sequence { } internal func sendBytes(from address: UnsafePointer, count: Int) { - var source = address + var source = UnsafeRawPointer(address) var bytesLeft = count debugLog("BEGIN \(#function)"); defer { debugLog("END \(#function)") } while bytesLeft > 0 { From 92a181671ea7a622286281a81a97a0b2f032c76d Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Thu, 27 Aug 2020 17:38:35 -0700 Subject: [PATCH 05/20] Fix ValueTracking isUniquelyIdentified to use AccessedStorage. To clarify and unify logic, improve precision, and behave consistently with other code that does the same thing. --- .../SILOptimizer/Analysis/ValueTracking.h | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/include/swift/SILOptimizer/Analysis/ValueTracking.h b/include/swift/SILOptimizer/Analysis/ValueTracking.h index 2ab2715102cbb..32569f2f25bbf 100644 --- a/include/swift/SILOptimizer/Analysis/ValueTracking.h +++ b/include/swift/SILOptimizer/Analysis/ValueTracking.h @@ -54,9 +54,21 @@ bool pointsToLocalObject(SILValue V); /// - an address projection based on an exclusive argument with no levels of /// indirection (e.g. ref_element_addr, project_box, etc.). inline bool isUniquelyIdentified(SILValue V) { - return pointsToLocalObject(V) - || (V->getType().isAddress() - && isExclusiveArgument(getAccessedAddress(V))); + SILValue objectRef = V; + if (V->getType().isAddress()) { + auto storage = findAccessedStorage(V); + if (!storage) + return false; + + if (storage.isUniquelyIdentifiedAfterEnforcement()) + return true; + + if (!storage.isObjectAccess()) + return false; + + objectRef = storage.getObject(); + } + return pointsToLocalObject(objectRef); } enum class IsZeroKind { From cc0aa2f8b8740221420ac35e86d7e67cbcd8529b Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Thu, 27 Aug 2020 17:48:16 -0700 Subject: [PATCH 06/20] Add an AccessPath abstraction and formalize memory access Things that have come up recently but are somewhat blocked on this: - Moving AccessMarkerElimination down in the pipeline - SemanticARCOpts correctness and improvements - AliasAnalysis improvements - LICM performance regressions - RLE/DSE improvements Begin to formalize the model for valid memory access in SIL. Ignoring ownership, every access is a def-use chain in three parts: object root -> formal access base -> memory operation address AccessPath abstracts over this path and standardizes the identity of a memory access throughout the optimizer. This abstraction is the basis for a new AccessPathVerification. With that verification, we now have all the properties we need for the type of analysis requires for exclusivity enforcement, but now generalized for any memory analysis. This is suitable for an extremely lightweight analysis with no side data structures. We currently have a massive amount of ad-hoc memory analysis throughout SIL, which is incredibly unmaintainable, bug-prone, and not performance-robust. We can begin taking advantage of this verifably complete model to solve that problem. The properties this gives us are: Access analysis must be complete over memory operations: every memory operation needs a recognizable valid access. An access can be unidentified only to the extent that it is rooted in some non-address type and we can prove that it is at least *not* part of an access to a nominal class or global property. Pointer provenance is also required for future IRGen-level bitfield optimizations. Access analysis must be complete over address users: for an identified object root all memory accesses including subobjects must be discoverable. Access analysis must be symmetric: use-def and def-use analysis must be consistent. AccessPath is merely a wrapper around the existing accessed-storage utilities and IndexTrieNode. Existing passes already very succesfully use this approach, but in an ad-hoc way. With a general utility we can: - update passes to use this approach to identify memory access, reducing the space and time complexity of those algorithms. - implement an inexpensive on-the-fly, debug mode address lifetime analysis - implement a lightweight debug mode alias analysis - ultimately improve the power, efficiency, and maintainability of full alias analysis - make our type-based alias analysis sensistive to the access path --- include/swift/SIL/MemAccessUtils.h | 784 +++++++--- include/swift/SIL/PatternMatch.h | 10 + .../SILOptimizer/Analysis/ValueTracking.h | 9 +- lib/IRGen/IRGenSIL.cpp | 2 +- lib/SIL/Utils/MemAccessUtils.cpp | 1280 ++++++++++++++--- .../LoadBorrowInvalidationChecker.cpp | 40 +- lib/SILOptimizer/Analysis/AliasAnalysis.cpp | 2 +- lib/SILOptimizer/Analysis/MemoryBehavior.cpp | 13 +- lib/SILOptimizer/LoopTransforms/LICM.cpp | 2 +- .../Mandatory/DiagnoseStaticExclusivity.cpp | 10 +- 10 files changed, 1742 insertions(+), 410 deletions(-) diff --git a/include/swift/SIL/MemAccessUtils.h b/include/swift/SIL/MemAccessUtils.h index dd198db2b6814..a65effe525448 100644 --- a/include/swift/SIL/MemAccessUtils.h +++ b/include/swift/SIL/MemAccessUtils.h @@ -2,7 +2,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -22,20 +22,40 @@ /// To verify access markers, SIL checks that all memory operations either have /// an address that originates in begin_access, or originates from a pattern /// that is recognized as a non-formal-access. This implies that every SIL -/// memory operation has a recognizable address source. +/// memory operation has a recognizable address source. Given the address of a +/// memory operation, there are three levels of APIs that inspect the origin of +/// that address: /// -/// If the memory operation is part of a formal access, then getAddressAccess() -/// returns the begin_access marker. +/// 1. getAccessAddress(): Find the originating address as close as possible to +/// the address of the formal access *without* looking past any storage +/// casts. This is useful when the type of the returned access address must be +/// consistent with the memory operation's type (the same type or a parent +/// type). For a formal access, this typically returns the begin_access, but it +/// is not guaranteed to because some accesses contain storage casts. For +/// non-formal access, it returns a best-effort address corresponding to the +/// base of an access. /// -/// AccessedStorage identifies the storage location of a memory access. +/// 2. getAccessBegin(): If the memory operation is part of a formal access, +/// then this is guaranteed to return the begin_access marker. Otherwise, it +/// returns the best-effort address or pointer corresponding to the base of an +/// access. Useful to find the scope of a formal access. /// -/// identifyFormalAccess() returns the formally accessed storage of a -/// begin_access instruction. This must return a valid AccessedStorage value -/// unless the access has "Unsafe" enforcement. The formal access location may -/// be nested within an outer begin_access. For the purpose of exclusivity, -/// nested accesses are considered distinct formal accesses so they return -/// distinct AccessedStorage values even though they may access the same -/// memory. +/// 3. getAccessBase(): Find the ultimate base of any address corresponding to +/// the accessed object, regardless of whether the address is nested within +/// access scopes, and regardless of any storage casts. This returns either an +/// address type, pointer type, or box type, but never a reference type. +/// Each object's property or its tail storage is separately accessed. +/// +/// For better identification an access base, use findAccessedStorage(). It +/// returns an AccessedStorage value that identifies the storage location of a +/// memory access. It provides APIs for inspecting type of accessed storage and +/// allows for disambiguation between different types of storage and different +/// properties within a class. +/// +/// findAccessedStorage() follows the same logic as getAccessBase(), but if the +/// base is not recognized as a valid access, it returns invalid +/// AccessedStorage. It also performs further analysis to determine the root +/// reference of an object access. /// /// findAccessedStorage() returns the outermost AccessedStorage for any memory /// address. It can be called on the address of a memory operation, the address @@ -50,19 +70,28 @@ /// formal access. /// /// The AccessEnforcementWMO pass is an example of an optimistic optimization -/// that relies on the above requirements for correctness. If -/// findAccessedStorage() simply bailed out on an unrecognized memory address by -/// returning an invalid AccessedStorage, then the optimization could make -/// incorrect assumptions about the absence of access to globals or class -/// properties. +/// that relies on this requirement for correctness. If findAccessedStorage() +/// simply bailed out on an unrecognized memory address by returning an invalid +/// AccessedStorage, then the optimization could make incorrect assumptions +/// about the absence of access to globals or class properties. +/// +/// identifyFormalAccess() is similar to findAccessedStorage(), but returns the +/// formally accessed storage of a begin_access instruction. This must return a +/// valid AccessedStorage value unless the access has "Unsafe" enforcement. The +/// formal access location may be nested within an outer begin_access. For the +/// purpose of exclusivity, nested accesses are considered distinct formal +/// accesses so they return distinct AccessedStorage values even though they may +/// access the same memory. /// //===----------------------------------------------------------------------===// #ifndef SWIFT_SIL_MEMACCESSUTILS_H #define SWIFT_SIL_MEMACCESSUTILS_H +#include "swift/Basic/IndexTrie.h" #include "swift/SIL/ApplySite.h" #include "swift/SIL/InstructionUtils.h" +#include "swift/SIL/Projection.h" #include "swift/SIL/SILArgument.h" #include "swift/SIL/SILBasicBlock.h" #include "swift/SIL/SILGlobalVariable.h" @@ -75,7 +104,7 @@ namespace swift { -/// Get the base address of a formal access by stripping access markers. +/// Get the source address of a formal access by stripping access markers. /// /// Postcondition: If \p v is an address, then the returned value is also an /// address (pointer-to-address is not stripped). @@ -86,65 +115,49 @@ inline SILValue stripAccessMarkers(SILValue v) { return v; } -/// An address projection that may be inside of a formal access, such as -/// (begin_borrow, struct_element_addr, tuple_element_addr). -struct AccessProjection { - SingleValueInstruction *projectionInst = nullptr; - - /// If \p v is not a recognized access projection the result is invalid. - AccessProjection(SILValue v) { - switch (v->getKind()) { - default: - break; - - case ValueKind::StructElementAddrInst: - case ValueKind::TupleElementAddrInst: - case ValueKind::UncheckedTakeEnumDataAddrInst: - case ValueKind::TailAddrInst: - case ValueKind::IndexAddrInst: - projectionInst = cast(v); - }; - } - - operator bool() const { return projectionInst != nullptr; } - - SILValue baseAddress() const { return projectionInst->getOperand(0); } -}; - -/// Return the base address after stripping access projections. If \p v is an -/// access projection, return the enclosing begin_access. Otherwise, return a -/// "best effort" base address. +/// Return the source address after stripping as many access projections as +/// possible without losing the address type. /// -/// Precondition: \p v must be an address. +/// For formal accesses, this typically returns the begin_access, but may fail +/// for accesses that call into an addressor, which performs pointer +/// conversion. /// -/// To get the base address of the formal access behind the access marker, -/// either call stripAccessMarkers() on the returned value, or call -/// getAccessedAddress() on \p v. +/// If there is no access marker, then this returns the "best-effort" address +/// corresponding to the accessed variable. This never looks through +/// pointer_to_address or other conversions that may change the address type +/// other than via type-safe (TBAA-compatible) projection. +SILValue getAccessAddress(SILValue address); + +/// Return the source address or pointer after stripping all access projections +/// and storage casts. /// -/// To identify the underlying storage object of the access, call -/// findAccessedStorage() either on \p v or on the returned address. -SILValue getAddressAccess(SILValue v); - -/// Convenience for stripAccessMarkers(getAddressAccess(v)). -SILValue getAccessedAddress(SILValue v); +/// If this is a formal access, then it is guaranteed to return the immediately +/// enclosing begin_access and may "see through" storage casts to do so. +/// +/// If there is no access marker, then it returns a "best effort" address +/// corresponding to the accessed variable. In this case, the returned value +/// could be a non-address pointer type. +SILValue getAccessBegin(SILValue address); -/// Return true if \p accessedAddress points to a let-variable. +/// Return the source address or pointer after stripping access projections, +/// access markers, and storage casts. /// -/// Precondition: \p accessedAddress must be an address-type value representing -/// the base of a formal access (not a projection within the access). +/// The returned base address is guaranteed to match the unique AccessedStorage +/// value for the same \p address. That is, if two calls to getAccessBase() +/// return the same base address, then they must also have the same storage. +SILValue getAccessBase(SILValue address); + +/// Return true if \p address points to a let-variable. /// /// let-variables are only written during let-variable initialization, which is -/// assumed to store directly to the same, unaliased accessedAddress. +/// assumed to store directly to the same, unaliased access base. /// /// The address of a let-variable must be the base of a formal access, not an /// access projection. A 'let' member of a struct is *not* a let-variable, /// because it's memory may be written when formally modifying the outer /// struct. A let-variable is either an entire local variable, global variable, /// or class property (these are all formal access base addresses). -/// -/// The caller should derive the accessed address using -/// stripAccessMarkers(getAccessedAddress(ptr)). -bool isLetAddress(SILValue accessedAddress); +bool isLetAddress(SILValue address); /// Return true if two accesses to the same storage may conflict given the kind /// of each access. @@ -231,8 +244,8 @@ class AccessedStorage { static const char *getKindName(Kind k); - // Give object tail storage a fake property index for convenience. - static constexpr unsigned TailIndex = ~0U; + // Give object tail storage a fake large property index for convenience. + static constexpr unsigned TailIndex = std::numeric_limits::max(); /// Directly create an AccessedStorage for class or tail property access. static AccessedStorage forClass(SILValue object, unsigned propertyIndex) { @@ -363,8 +376,10 @@ class AccessedStorage { return global; } + bool isReference() const { return getKind() == Class || getKind() == Tail; } + SILValue getObject() const { - assert(getKind() == Class || getKind() == Tail); + assert(isReference()); return value; } unsigned getPropertyIndex() const { @@ -372,6 +387,27 @@ class AccessedStorage { return getElementIndex(); } + /// Return the address or reference root that the storage was based + /// on. Returns an invalid SILValue for globals or invalid storage. + SILValue getRoot() const { + switch (getKind()) { + case AccessedStorage::Box: + case AccessedStorage::Stack: + case AccessedStorage::Argument: + case AccessedStorage::Yield: + case AccessedStorage::Unidentified: + return getValue(); // Can be invalid for Unidentified storage. + case AccessedStorage::Global: + return SILValue(); + case AccessedStorage::Class: + case AccessedStorage::Tail: + return getObject(); + case AccessedStorage::Nested: + assert(false && "AccessPath cannot identify nested access"); + return SILValue(); + } + } + /// Return true if the given storage objects have identical storage locations. /// /// This compares only the AccessedStorage base class bits, ignoring the @@ -427,18 +463,17 @@ class AccessedStorage { /// If this is a uniquely identified formal access, then it cannot /// alias with any other uniquely identified access to different storage. - /// - /// This determines whether access markers may conflict, so it cannot assume - /// that exclusivity is enforced. bool isUniquelyIdentified() const { switch (getKind()) { case Box: case Stack: case Global: return true; + case Argument: + return + getArgument()->getArgumentConvention().isExclusiveIndirectParameter(); case Class: case Tail: - case Argument: case Yield: case Nested: case Unidentified: @@ -447,16 +482,10 @@ class AccessedStorage { llvm_unreachable("unhandled kind"); } - /// Return true if this a uniquely identified formal access location assuming - /// exclusivity enforcement. Do not use this to optimize access markers. - bool isUniquelyIdentifiedAfterEnforcement() const { - if (isUniquelyIdentified()) - return true; - - return getKind() == Argument - && getArgument() - ->getArgumentConvention() - .isExclusiveIndirectParameter(); + /// Return true if this storage is guaranteed not to overlap with \p other's + /// storage. + bool isDistinctFrom(const AccessedStorage &other) const { + return isDistinctFrom<&AccessedStorage::isUniquelyIdentified>(other); } /// Return true if this identifies the base of a formal access location. @@ -470,11 +499,27 @@ class AccessedStorage { return getKind() == Class; } - // Return true if this storage is guaranteed not to overlap with \p other's - // storage. + /// Returns the ValueDecl for the underlying storage, if it can be + /// determined. Otherwise returns null. + /// + /// WARNING: This is not a constant-time operation. It is for diagnostics and + /// checking via the ValueDecl if we are processing a `let` variable. + const ValueDecl *getDecl() const; + + void print(raw_ostream &os) const; + void dump() const; + +private: + // Disable direct comparison because we allow subclassing with bitfields. + // Currently, we use DenseMapInfo to unique storage, which defines key + // equalilty only in terms of the base AccessedStorage class bits. + bool operator==(const AccessedStorage &) const = delete; + bool operator!=(const AccessedStorage &) const = delete; + + template bool isDistinctFrom(const AccessedStorage &other) const { - if (isUniquelyIdentified()) { - if (other.isUniquelyIdentified() && !hasIdenticalBase(other)) + if ((this->*IsUniqueFn)()) { + if ((other.*IsUniqueFn)() && !hasIdenticalBase(other)) return true; if (other.isObjectAccess()) @@ -484,8 +529,8 @@ class AccessedStorage { // Box/Stack storage. return false; } - if (other.isUniquelyIdentified()) - return other.isDistinctFrom(*this); + if ((other.*IsUniqueFn)()) + return other.isDistinctFrom(*this); // Neither storage is uniquely identified. if (isObjectAccess()) { @@ -508,7 +553,7 @@ class AccessedStorage { return false; } if (other.isObjectAccess()) - return other.isDistinctFrom(*this); + return other.isDistinctFrom(*this); // Neither storage is from a class or tail. // @@ -516,23 +561,6 @@ class AccessedStorage { // nested/argument access. return false; } - - /// Returns the ValueDecl for the underlying storage, if it can be - /// determined. Otherwise returns null. - /// - /// WARNING: This is not a constant-time operation. It is for diagnostics and - /// checking via the ValueDecl if we are processing a `let` variable. - const ValueDecl *getDecl() const; - - void print(raw_ostream &os) const; - void dump() const; - -private: - // Disable direct comparison because we allow subclassing with bitfields. - // Currently, we use DenseMapInfo to unique storage, which defines key - // equalilty only in terms of the base AccessedStorage class bits. - bool operator==(const AccessedStorage &) const = delete; - bool operator!=(const AccessedStorage &) const = delete; }; } // end namespace swift @@ -634,6 +662,314 @@ inline AccessedStorage identifyCapturedStorage(SILValue capturedAddress) { } // end namespace swift +//===----------------------------------------------------------------------===// +// AccessPath +//===----------------------------------------------------------------------===// + +namespace swift { + +/// Identify an addressable location based the AccessedStorage and projection +/// path. +/// +/// Each unique path from a base address implies a unique memory location within +/// that object. A path prefix identifies memory that contains all paths with +/// the same prefix. The AccessPath returned by AccessPath::compute(address) +/// identifies the object seen by any memory operation that *directly* operates +/// on 'address'. The computed path is a prefix of the paths of any contained +/// subobjects. +/// +/// Path indices, encoded by AccessPath::Index, may be either subobject +/// projections or offset indices. We print subobject indices as '#n' and offset +/// indices as '@n'. +/// +/// Example Def->Use: (Path indices) +/// struct_element_addr #1: (#1) +/// ref_tail_addr -> struct_element_addr #2: (#2) +/// ref_tail_addr -> index_addr #1 -> struct_element_addr #2: (@1, #2) +/// pointer_to_address -> struct_element_addr #2: (#2) +/// pointer_to_address -> index_addr #1 -> struct_element_addr #2: (@1, #2) +/// +/// The index of ref_element_addr is part of the storage identity and does +/// not contribute to the access path indices. +/// +/// A well-formed path has at most one offset component at the begining of the +/// path (chained index_addrs are merged into one offset). In other words, +/// taking an offset from a subobject projection is not well-formed access +/// path. However, it is possible (however undesirable) for programmers to +/// convert a subobject address into a pointer (for example, via implicit +/// conversion), then advance that pointer. Since we can't absolutely prevent +/// this, we instead consider it an invalid AccessPath. This is the only case in +/// which AccessPath::storage can differ from findAccessedStorage(). +/// +/// Storing an AccessPath ammortizes to constant space. To cache identification +/// of address locations, AccessPath should be used rather than the +/// ProjectionPath which requires quadratic space in the number of address +/// values and quadratic time when comparing addresses. +/// +/// Type-cast operations such as address_to_pointer may appear on the access +/// path. It is illegal to use these operations to cast to a non-layout +/// compatible type. TODO: add enforcement for this rule. +class AccessPath { +public: + /// Create the AccessPath for any memory operation on the given address. + static AccessPath compute(SILValue address); + + // Encode a dynamic index_addr as an UnknownOffset. + static constexpr int UnknownOffset = std::numeric_limits::min() >> 1; + + struct PathNode; + + // An access path index. + // + // Note: + // - IndexTrieNode::RootIndex = INT_MIN = 0x80000000 + // - AccessedStorage::TailIndex = INT_MAX = 0x7FFFFFFF + // - AccessPath::UnknownOffset = (INT_MIN>>1) = 0xC0000000 + // - An offset index is never zero + class Index { + public: + friend struct PathNode; + + // Use the sign bit to identify offset indices. Subobject projections are + // always positive. + constexpr static unsigned IndexFlag = unsigned(1) << 31; + static int encodeOffset(int indexValue) { + assert(indexValue != 0 && "an offset index cannot be zero"); + // Must be able to sign-extended the 31-bit value. + assert(((indexValue << 1) >> 1) == indexValue); + return indexValue | IndexFlag; + } + + // Encode a positive field index, property index, or TailIndex. + static Index forSubObjectProjection(unsigned projIdx) { + assert(Index(projIdx).isSubObjectProjection()); + return Index(projIdx); + } + + static Index forOffset(unsigned projIdx) { + return Index(encodeOffset(projIdx)); + } + + private: + int indexEncoding; + Index(int indexEncoding) : indexEncoding(indexEncoding) {} + + public: + bool isSubObjectProjection() const { return indexEncoding >= 0; } + + int getSubObjectIndex() const { + assert(isSubObjectProjection()); + return indexEncoding; + } + + // Sign-extend the 31-bit value. + int getOffset() const { + assert(!isSubObjectProjection()); + return ((indexEncoding << 1) >> 1); + } + + bool isUnknownOffset() const { + return indexEncoding == AccessPath::UnknownOffset; + } + + int getEncoding() const { return indexEncoding; } + + void print(raw_ostream &os) const; + + void dump() const; + }; + + // A component of the AccessPath. + // + // Transient wrapper around the underlying IndexTrieNode that encodes either a + // subobject projection or an offset index. + struct PathNode { + IndexTrieNode *node = nullptr; + + constexpr PathNode() = default; + + PathNode(IndexTrieNode *node) : node(node) {} + + bool isValid() const { return node != nullptr; } + + bool isRoot() const { return node->isRoot(); } + + bool isLeaf() const { return node->isLeaf(); } + + Index getIndex() const { return Index(node->getIndex()); } + + PathNode getParent() const { return node->getParent(); } + + // Return the PathNode from \p subNode's path one level deeper than \p + // prefixNode. + // + // Precondition: this != subNode + PathNode findPrefix(PathNode subNode) const; + + bool operator==(PathNode other) const { return node == other.node; } + bool operator!=(PathNode other) const { return node != other.node; } + }; + +private: + AccessedStorage storage; + PathNode pathNode; + // store the single offset index independent from the PathNode to simplify + // checking for path overlap. + int offset = 0; + +public: + // AccessPaths are built by AccessPath::compute(address). + // + // AccessedStorage is only used to identify the storage location; AccessPath + // ignores its subclass bits. + AccessPath(AccessedStorage storage, PathNode pathNode, int offset) + : storage(storage), pathNode(pathNode), offset(offset) { + assert(storage.getKind() != AccessedStorage::Nested); + assert(pathNode.isValid() || !storage && "Access path requires a pathNode"); + } + + AccessPath() = default; + + bool operator==(AccessPath other) const { + return + storage.hasIdenticalBase(other.storage) && pathNode == other.pathNode; + } + bool operator!=(AccessPath other) const { return !(*this == other); } + + bool isValid() const { return pathNode.isValid(); } + + AccessedStorage getStorage() const { return storage; } + + PathNode getPathNode() const { return pathNode; } + + int getOffset() const { return offset; } + + /// Return true if this path contains \p subPath. + bool contains(AccessPath subPath) const; + + /// Return true if this path may overlap with \p otherPath. + bool mayOverlap(AccessPath otherPath) const; + + /// Return the address root that the access path was based on. Returns + /// an invalid SILValue for globals or invalid storage. + SILValue getRoot() const { return storage.getRoot(); } + + /// Get all uses of all address values that have a common AccessPath. Return + /// true if all uses were found before reaching the limit. + /// + /// This should find all uses for which calling AccessPath::compute() would + /// yield an identical AccessPath. + /// + /// This fails on global variables which have no root. To collect all uses, + /// including global variable uses, use AccessPathWithBase::collectUses. + bool + collectUses(SmallVectorImpl &uses, bool collectOverlappingUses, + unsigned useLimit = std::numeric_limits::max()) const; + + void printPath(raw_ostream &os) const; + void print(raw_ostream &os) const; + void dump() const; +}; + +// Encapsulate the result of computing an AccessPath. AccessPath does not store +// the base address of the formal access because it does not always uniquely +// indentify the access, but AccessPath users may use the base address to to +// recover the def-use chain. +// +// AccessPathWithBase::collectUses is guaranteed to be complete for all storage +// types, while AccessPath::collectUses cannot handle globals. +struct AccessPathWithBase { + AccessPath accessPath; + // The address-type value that is the base of the formal access. For + // class storage, it is the ref_element_addr. For global storage it is the + // global_addr or initializer apply. For other storage, it is the same as + // accessPath.getRoot(). + // + // base may be invalid for global_addr -> address_to_pointer -> phi patterns. + // FIXME: add a structural requirement to SIL so base is always valid in OSSA. + SILValue base; + + /// \p address identifies the object seen by any memory operation that + /// directly operates on the address. For indexable addresses, this implies an + /// operation at index zero. + static AccessPathWithBase compute(SILValue address); + + AccessPathWithBase(AccessPath accessPath, SILValue base) + : accessPath(accessPath), base(base) {} + + bool operator==(AccessPathWithBase other) const { + return accessPath == other.accessPath && base == other.base; + } + bool operator!=(AccessPathWithBase other) const { return !(*this == other); } + + /// Get all uses of all address values that have a common AccessPath. Return + /// true if all uses were found before reaching the limit. + /// + /// This should find all uses for which calling AccessPath::compute() would + /// yield an identical AccessPath and, for global variables, have the same + /// access base (e.g. from the same global_addr instruction). + bool collectUses(SmallVectorImpl &uses, + bool collectOverlappingUses, + unsigned useLimit = std::numeric_limits::max()) const; + + void print(raw_ostream &os) const; + void dump() const; +}; + +inline AccessPath AccessPath::compute(SILValue address) { + return AccessPathWithBase::compute(address).accessPath; +} + +} // end namespace swift + +namespace llvm { +/// Allow AccessPath to be used in DenseMap. +template <> struct DenseMapInfo { + static inline swift::AccessPath getEmptyKey() { + return swift::AccessPath( + DenseMapInfo::getEmptyKey(), + swift::AccessPath::PathNode( + DenseMapInfo::getEmptyKey()), 0); + } + static inline swift::AccessPath getTombstoneKey() { + return swift::AccessPath( + DenseMapInfo::getTombstoneKey(), + swift::AccessPath::PathNode( + DenseMapInfo::getTombstoneKey()), 0); + } + static inline unsigned getHashValue(const swift::AccessPath &val) { + return llvm::hash_combine( + DenseMapInfo::getHashValue(val.getStorage()), + val.getPathNode().node); + } + static bool isEqual(const swift::AccessPath &lhs, + const swift::AccessPath &rhs) { + return lhs == rhs; + } +}; +template <> struct DenseMapInfo { + static inline swift::AccessPathWithBase getEmptyKey() { + return swift::AccessPathWithBase( + DenseMapInfo::getEmptyKey(), + DenseMapInfo::getEmptyKey()); + } + static inline swift::AccessPathWithBase getTombstoneKey() { + return swift::AccessPathWithBase( + DenseMapInfo::getTombstoneKey(), + DenseMapInfo::getTombstoneKey()); + } + static inline unsigned getHashValue(const swift::AccessPathWithBase &val) { + return llvm::hash_combine( + DenseMapInfo::getHashValue(val.accessPath), + DenseMapInfo::getHashValue(val.base)); + } + static bool isEqual(const swift::AccessPathWithBase &lhs, + const swift::AccessPathWithBase &rhs) { + return lhs == rhs; + } +}; +} // end namespace llvm + //===----------------------------------------------------------------------===// // MARK: Helper API for specific formal access patterns //===----------------------------------------------------------------------===// @@ -709,7 +1045,110 @@ SILBasicBlock::iterator removeBeginAccess(BeginAccessInst *beginAccess); namespace swift { -/// Abstract CRTP class for a visitor passed to \c visitAccessUseDefChain. +/// If \p svi is an access projection, return an address-type operand for the +/// incoming address. +/// +/// An access projection is on the inside of a formal access. It includes +/// struct_element_addr and tuple_element_addr, but not ref_element_addr. +/// +/// The returned address may point to any compatible type, which may alias with +/// the projected address. Arbitrary address casts are not allowed. +inline Operand *getAccessProjectionOperand(SingleValueInstruction *svi) { + switch (svi->getKind()) { + default: + return nullptr; + + case SILInstructionKind::StructElementAddrInst: + case SILInstructionKind::TupleElementAddrInst: + case SILInstructionKind::IndexAddrInst: + case SILInstructionKind::TailAddrInst: + // open_existential_addr and unchecked_take_enum_data_addr are problematic + // because they both modify memory and are access projections. Ideally, they + // would not be casts, but will likely be eliminated with opaque values. + case SILInstructionKind::OpenExistentialAddrInst: + case SILInstructionKind::UncheckedTakeEnumDataAddrInst: + return &svi->getAllOperands()[0]; + + // Special-case this indirect enum pattern: + // unchecked_take_enum_data_addr -> load -> project_box + // (the individual load and project_box are not access projections) + // + // FIXME: Make sure this case goes away with OSSA and opaque values. If not, + // then create a special instruction for this pattern. That way we have an + // invariant that all access projections are single-value address-to-address + // conversions. Then reuse this helper for both use-def an def-use traversals. + // + // Check getAccessProjectionOperand() before isAccessedStorageCast() because + // it will consider any project_box to be a storage cast. + case SILInstructionKind::ProjectBoxInst: + if (auto *load = dyn_cast(svi->getOperand(0))) + return &load->getOperandRef(); + + return nullptr; + }; +} + +/// An address, pointer, or box cast that occurs outside of the formal +/// access. These convert the base of accessed storage without affecting the +/// AccessPath. Useful for both use-def and def-use traversal. The source +/// address must be at operand(0). +/// +/// Some of these casts, such as address_to_pointer, may also occur inside of a +/// formal access. TODO: Add stricter structural guarantee such that these never +/// occur within an access. It's important to be able to get the accessed +/// address without looking though type casts or pointer_to_address [strict], +/// which we can't do if those operations are behind access projections. +inline bool isAccessedStorageCast(SingleValueInstruction *svi) { + switch (svi->getKind()) { + default: + return false; + + // Simply pass-thru the incoming address. + case SILInstructionKind::MarkUninitializedInst: + case SILInstructionKind::UncheckedAddrCastInst: + case SILInstructionKind::MarkDependenceInst: + // Look through a project_box to identify the underlying alloc_box as the + // accesed object. It must be possible to reach either the alloc_box or the + // containing enum in this loop, only looking through simple value + // propagation such as copy_value and begin_borrow. + case SILInstructionKind::ProjectBoxInst: + case SILInstructionKind::ProjectBlockStorageInst: + case SILInstructionKind::CopyValueInst: + case SILInstructionKind::BeginBorrowInst: + // Casting to RawPointer does not affect the AccessPath. When converting + // between address types, they must be layout compatible (with truncation). + case SILInstructionKind::AddressToPointerInst: + // Access to a Builtin.RawPointer. It may be important to continue looking + // through this because some RawPointers originate from identified + // locations. See the special case for global addressors, which return + // RawPointer, above. + // + // If the inductive search does not find a valid addressor, it will + // eventually reach the default case that returns in invalid location. This + // is correct for RawPointer because, although accessing a RawPointer is + // legal SIL, there is no way to guarantee that it doesn't access class or + // global storage, so returning a valid unidentified storage object would be + // incorrect. It is the caller's responsibility to know that formal access + // to such a location can be safely ignored. + // + // For example: + // + // - KeyPath Builtins access RawPointer. However, the caller can check + // that the access `isFromBuilin` and ignore the storage. + // + // - lldb generates RawPointer access for debugger variables, but SILGen + // marks debug VarDecl access as 'Unsafe' and SIL passes don't need the + // AccessedStorage for 'Unsafe' access. + case SILInstructionKind::PointerToAddressInst: + return true; + } +} + +/// Abstract CRTP class for a visiting instructions that are part of the use-def +/// chain from an accessed address up to the storage base. +/// +/// Given the address of a memory operation begin visiting at +/// getAccessedAddress(address). template class AccessUseDefChainVisitor { protected: @@ -751,32 +1190,40 @@ class AccessUseDefChainVisitor { // Result visitBase(SILValue base, AccessedStorage::Kind kind); // Result visitNonAccess(SILValue base); // Result visitPhi(SILPhiArgument *phi); - // Result visitCast(SingleValueInstruction *cast, Operand *parentAddr); - // Result visitPathComponent(SingleValueInstruction *projectedAddr, - // Operand *parentAddr); + // Result visitStorageCast(SingleValueInstruction *cast, Operand *sourceOper); + // Result visitAccessProjection(SingleValueInstruction *cast, + // Operand *sourceOper); Result visit(SILValue sourceAddr); }; template Result AccessUseDefChainVisitor::visit(SILValue sourceAddr) { + if (auto *svi = dyn_cast(sourceAddr)) { + if (auto *projOper = getAccessProjectionOperand(svi)) + return asImpl().visitAccessProjection(svi, projOper); + + if (isAccessedStorageCast(svi)) + return asImpl().visitStorageCast(svi, &svi->getAllOperands()[0]); + } switch (sourceAddr->getKind()) { default: - if (isAddressForLocalInitOnly(sourceAddr)) - return asImpl().visitUnidentified(sourceAddr); - return asImpl().visitNonAccess(sourceAddr); + break; // MARK: Handle immediately-identifiable instructions. // An AllocBox is a fully identified memory location. case ValueKind::AllocBoxInst: return asImpl().visitBoxAccess(cast(sourceAddr)); + // An AllocStack is a fully identified memory location, which may occur // after inlining code already subjected to stack promotion. case ValueKind::AllocStackInst: return asImpl().visitStackAccess(cast(sourceAddr)); + case ValueKind::GlobalAddrInst: return asImpl().visitGlobalAccess(sourceAddr); + case ValueKind::ApplyInst: { FullApplySite apply(cast(sourceAddr)); if (auto *funcRef = apply.getReferencedFunctionOrNull()) { @@ -792,25 +1239,37 @@ Result AccessUseDefChainVisitor::visit(SILValue sourceAddr) { } case ValueKind::RefElementAddrInst: return asImpl().visitClassAccess(cast(sourceAddr)); + + // ref_tail_addr project an address from a reference. + // This is a valid address producer for nested @inout argument + // access, but it is never used for formal access of identified objects. + case ValueKind::RefTailAddrInst: + return asImpl().visitTailAccess(cast(sourceAddr)); + // A yield is effectively a nested access, enforced independently in // the caller and callee. case ValueKind::BeginApplyResult: return asImpl().visitYieldAccess(cast(sourceAddr)); + // A function argument is effectively a nested access, enforced // independently in the caller and callee. case ValueKind::SILFunctionArgument: - return asImpl().visitArgumentAccess(cast(sourceAddr)); + return asImpl().visitArgumentAccess( + cast(sourceAddr)); // View the outer begin_access as a separate location because nested // accesses do not conflict with each other. case ValueKind::BeginAccessInst: return asImpl().visitNestedAccess(cast(sourceAddr)); + // Static index_addr is handled by getAccessProjectionOperand. Dynamic + // index_addr is currently unidentified because we can't form an AccessPath + // including them. case ValueKind::SILUndef: return asImpl().visitUnidentified(sourceAddr); - // MARK: The sourceAddr producer cannot immediately be classified, - // follow the use-def chain. + // MARK: The sourceAddr producer cannot immediately be classified, + // follow the use-def chain. case ValueKind::StructExtractInst: // Handle nested access to a KeyPath projection. The projection itself @@ -834,98 +1293,11 @@ Result AccessUseDefChainVisitor::visit(SILValue sourceAddr) { checkSwitchEnumBlockArg(cast(sourceAddr)); return asImpl().visitUnidentified(sourceAddr); } - // Load a box from an indirect payload of an opaque enum. - // We must have peeked past the project_box earlier in this loop. - // (the indirectness makes it a box, the load is for address-only). - // - // %payload_adr = unchecked_take_enum_data_addr %enum : $*Enum, #Enum.case - // %box = load [take] %payload_adr : $*{ var Enum } - // - // FIXME: this case should go away with opaque values. - // - // Otherwise return invalid AccessedStorage. - case ValueKind::LoadInst: - if (sourceAddr->getType().is()) { - Operand *addrOper = &cast(sourceAddr)->getOperandRef(); - assert(isa(addrOper->get())); - return asImpl().visitCast(cast(sourceAddr), - addrOper); - } - return asImpl().visitNonAccess(sourceAddr); - - // ref_tail_addr project an address from a reference. - // This is a valid address producer for nested @inout argument - // access, but it is never used for formal access of identified objects. - case ValueKind::RefTailAddrInst: - return asImpl().visitTailAccess(cast(sourceAddr)); - - // Inductive single-operand cases: - // Look through address casts to find the source address. - case ValueKind::MarkUninitializedInst: - case ValueKind::OpenExistentialAddrInst: - case ValueKind::UncheckedAddrCastInst: - // Inductive cases that apply to any type. - case ValueKind::CopyValueInst: - case ValueKind::MarkDependenceInst: - // Look through a project_box to identify the underlying alloc_box as the - // accesed object. It must be possible to reach either the alloc_box or the - // containing enum in this loop, only looking through simple value - // propagation such as copy_value. - case ValueKind::ProjectBoxInst: - // Handle project_block_storage just like project_box. - case ValueKind::ProjectBlockStorageInst: - // Look through begin_borrow in case a local box is borrowed. - case ValueKind::BeginBorrowInst: - // Casting to RawPointer does not affect the AccessPath. When converting - // between address types, they must be layout compatible (with truncation). - case ValueKind::AddressToPointerInst: - // A tail_addr is a projection that does not affect the access path because it - // must always originate from a ref_tail_addr. Any projection within the - // object's tail storage effectively has the same access path. - case ValueKind::TailAddrInst: - return asImpl().visitCast( - cast(sourceAddr), - &cast(sourceAddr)->getAllOperands()[0]); + } // end switch + if (isAddressForLocalInitOnly(sourceAddr)) + return asImpl().visitUnidentified(sourceAddr); - // Access to a Builtin.RawPointer. It may be important to continue looking - // through this because some RawPointers originate from identified - // locations. See the special case for global addressors, which return - // RawPointer, above. - // - // If the inductive search does not find a valid addressor, it will - // eventually reach the default case that returns in invalid location. This - // is correct for RawPointer because, although accessing a RawPointer is - // legal SIL, there is no way to guarantee that it doesn't access class or - // global storage, so returning a valid unidentified storage object would be - // incorrect. It is the caller's responsibility to know that formal access - // to such a location can be safely ignored. - // - // For example: - // - // - KeyPath Builtins access RawPointer. However, the caller can check - // that the access `isFromBuilin` and ignore the storage. - // - // - lldb generates RawPointer access for debugger variables, but SILGen - // marks debug VarDecl access as 'Unsafe' and SIL passes don't need the - // AccessedStorage for 'Unsafe' access. - // - // This is always considered a path component because an IndexAddr may - // project from it. - case ValueKind::PointerToAddressInst: - return asImpl().visitPathComponent( - cast(sourceAddr), - &cast(sourceAddr)->getAllOperands()[0]); - - // Address-to-address subobject projections. Projection::isAddressProjection - // returns true for these. - case ValueKind::StructElementAddrInst: - case ValueKind::TupleElementAddrInst: - case ValueKind::UncheckedTakeEnumDataAddrInst: - case ValueKind::IndexAddrInst: - return asImpl().visitPathComponent( - cast(sourceAddr), - &cast(sourceAddr)->getAllOperands()[0]); - } + return asImpl().visitNonAccess(sourceAddr); } } // end namespace swift diff --git a/include/swift/SIL/PatternMatch.h b/include/swift/SIL/PatternMatch.h index 9a7ab469ac947..07b7449e1d9eb 100644 --- a/include/swift/SIL/PatternMatch.h +++ b/include/swift/SIL/PatternMatch.h @@ -668,6 +668,16 @@ using BuiltinApplyTy = typename Apply_match::Ty; // Define matchers for most of builtin instructions. #include "swift/AST/Builtins.def" +#undef BUILTIN_UNARY_OP_MATCH_WITH_ARG_MATCHER +#undef BUILTIN_BINARY_OP_MATCH_WITH_ARG_MATCHER +#undef BUILTIN_VARARGS_OP_MATCH_WITH_ARG_MATCHER +#undef BUILTIN_CAST_OPERATION +#undef BUILTIN_CAST_OR_BITCAST_OPERATION +#undef BUILTIN_BINARY_OPERATION_ALL +#undef BUILTIN_BINARY_PREDICATE +#undef BUILTIN_MISC_OPERATION +#undef BUILTIN + //=== // Convenience compound builtin instructions matchers that succeed // if any of the sub-matchers succeed. diff --git a/include/swift/SILOptimizer/Analysis/ValueTracking.h b/include/swift/SILOptimizer/Analysis/ValueTracking.h index 32569f2f25bbf..be2ca665ae6b4 100644 --- a/include/swift/SILOptimizer/Analysis/ValueTracking.h +++ b/include/swift/SILOptimizer/Analysis/ValueTracking.h @@ -39,8 +39,8 @@ bool pointsToLocalObject(SILValue V); /// Returns true if \p V is a uniquely identified address or reference. Two /// uniquely identified pointers with distinct roots cannot alias. However, a /// uniquely identified pointer may alias with unidentified pointers. For -/// example, the uniquely identified pointer may escape to a call that returns an -/// alias of that pointer. +/// example, the uniquely identified pointer may escape to a call that returns +/// an alias of that pointer. /// /// It may be any of: /// @@ -53,6 +53,9 @@ bool pointsToLocalObject(SILValue V); /// /// - an address projection based on an exclusive argument with no levels of /// indirection (e.g. ref_element_addr, project_box, etc.). +/// +/// TODO: Fold this into the AccessedStorage API. pointsToLocalObject should be +/// performed by AccessedStorage::isUniquelyIdentified. inline bool isUniquelyIdentified(SILValue V) { SILValue objectRef = V; if (V->getType().isAddress()) { @@ -60,7 +63,7 @@ inline bool isUniquelyIdentified(SILValue V) { if (!storage) return false; - if (storage.isUniquelyIdentifiedAfterEnforcement()) + if (storage.isUniquelyIdentified()) return true; if (!storage.isObjectAccess()) diff --git a/lib/IRGen/IRGenSIL.cpp b/lib/IRGen/IRGenSIL.cpp index 9661d638d83e9..690708643afe8 100644 --- a/lib/IRGen/IRGenSIL.cpp +++ b/lib/IRGen/IRGenSIL.cpp @@ -4125,7 +4125,7 @@ void IRGenSILFunction::visitRefTailAddrInst(RefTailAddrInst *i) { } static bool isInvariantAddress(SILValue v) { - SILValue accessedAddress = getAccessedAddress(v); + SILValue accessedAddress = getAccessAddress(v); if (auto *ptrRoot = dyn_cast(accessedAddress)) { return ptrRoot->isInvariant(); } diff --git a/lib/SIL/Utils/MemAccessUtils.cpp b/lib/SIL/Utils/MemAccessUtils.cpp index 6377fd2643c08..c34ae23d3fc93 100644 --- a/lib/SIL/Utils/MemAccessUtils.cpp +++ b/lib/SIL/Utils/MemAccessUtils.cpp @@ -13,11 +13,9 @@ #define DEBUG_TYPE "sil-access-utils" #include "swift/SIL/MemAccessUtils.h" -#include "swift/SIL/ApplySite.h" -#include "swift/SIL/Projection.h" -#include "swift/SIL/SILGlobalVariable.h" #include "swift/SIL/SILModule.h" #include "swift/SIL/SILUndef.h" +#include "llvm/Support/Debug.h" using namespace swift; @@ -25,48 +23,417 @@ using namespace swift; // MARK: General Helpers //===----------------------------------------------------------------------===// -// TODO: When the optimizer stops stripping begin_access markers, then we should -// be able to assert that the result is a BeginAccessInst and the default case -// is unreachable. -SILValue swift::getAddressAccess(SILValue v) { - while (true) { - assert(v->getType().isAddress()); - auto projection = AccessProjection(v); - if (!projection) - return v; +namespace { + +enum StorageCastTy { StopAtStorageCast, IgnoreStorageCast }; - v = projection.baseAddress(); +// Handle a single phi-web within an access use-def chain. +// +// Recursively calls the useDefVisitor on any operations that aren't recognized +// as storage casts or projections. If the useDefVisitor finds a consistent +// result for all operands, then it's result will remain valid. If the +// useDefVisitor has an invalid result after processing the phi web, then it's +// original result is restored, then the phi reported to the useDefVisitor as a +// NonAccess. +// +// Phi-web's are only allowed to contain casts and projections that do not +// affect the access path. If AccessPhiVisitor reaches an unhandled projection, +// it remembers that as the commonDefinition. If after processing the entire +// web, the commonDefinition is unique, then it calls the original useDefVisitor +// to update its result. Note that visitAccessProjection and setDefinition are +// only used by visitors that process access projections; once the accessed +// address is reached, they are no longer relevant. +template +class AccessPhiVisitor + : public AccessUseDefChainVisitor> { + + UseDefVisitor &useDefVisitor; + StorageCastTy storageCastTy; + + Optional commonDefinition; + SmallVector pointerWorklist; + SmallPtrSet nestedPhis; + +public: + AccessPhiVisitor(UseDefVisitor &useDefVisitor, StorageCastTy storageCastTy) + : useDefVisitor(useDefVisitor), storageCastTy(storageCastTy) {} + + // Main entry point. + void findPhiAccess(SILPhiArgument *phiArg) && { + auto savedResult = useDefVisitor.saveResult(); + visitPhi(phiArg); + while (!pointerWorklist.empty()) { + this->visit(pointerWorklist.pop_back_val()); + } + // If a common path component was found, recursively look for the result. + if (commonDefinition) { + if (commonDefinition.getValue()) { + useDefVisitor.reenterUseDef(commonDefinition.getValue()); + } else { + // Divergent paths were found; invalidate any previously discovered + // storage. + useDefVisitor.invalidateResult(); + } + } + // If the result is now invalid, reset it and process the current phi as an + // unrecgonized access instead. + if (!useDefVisitor.isResultValid()) { + useDefVisitor.restoreResult(savedResult); + visitNonAccess(phiArg); + } + } + + // Visitor helper. + void setDefinition(SILValue def) { + if (!commonDefinition) { + commonDefinition = def; + return; + } + if (commonDefinition.getValue() != def) + commonDefinition = SILValue(); + } + + void checkVisitorResult(SILValue result) { + assert(!result && "must override any visitor that returns a result"); + } + + // MARK: Visitor implementation. + + // Recursively call the original storageVisitor for each base. We can't simply + // look for a common definition on all phi inputs, because the base may be + // cloned on each path. For example, two global_addr instructions may refer to + // the same global storage. Those global_addr instructions may each be + // converted to a RawPointer before being passed into the non-address phi. + void visitBase(SILValue base, AccessedStorage::Kind kind) { + checkVisitorResult(useDefVisitor.visitBase(base, kind)); + } + + void visitNonAccess(SILValue value) { + checkVisitorResult(useDefVisitor.visitNonAccess(value)); + } + + void visitNestedAccess(BeginAccessInst *access) { + checkVisitorResult(useDefVisitor.visitNestedAccess(access)); + } + + void visitPhi(SILPhiArgument *phiArg) { + if (nestedPhis.insert(phiArg).second) + phiArg->getIncomingPhiValues(pointerWorklist); + } + + void visitStorageCast(SingleValueInstruction *projectedAddr, + Operand *sourceOper) { + // Allow conversions to/from pointers and addresses on disjoint phi paths + // only if the underlying useDefVisitor allows it. + if (storageCastTy == IgnoreStorageCast) + pointerWorklist.push_back(sourceOper->get()); + else + visitNonAccess(projectedAddr); + } + + void visitAccessProjection(SingleValueInstruction *projectedAddr, + Operand *sourceOper) { + // An offset index on a phi path is always conservatively considered an + // unknown offset. + if (isa(projectedAddr) || isa(projectedAddr)) { + useDefVisitor.addUnknownOffset(); + pointerWorklist.push_back(sourceOper->get()); + return; + } + // No other access projections are expected to occur on disjoint phi + // paths. Stop searching at this projection. + setDefinition(projectedAddr); + } +}; + +enum NestedAccessTy { StopAtAccessBegin, IgnoreAccessBegin }; + +// Find the origin of an access while skipping projections and casts and +// handling phis. +template +class FindAccessVisitorImpl : public AccessUseDefChainVisitor { + using SuperTy = AccessUseDefChainVisitor; + +protected: + NestedAccessTy nestedAccessTy; + StorageCastTy storageCastTy; + + SmallPtrSet visitedPhis; + bool hasUnknownOffset = false; + +public: + FindAccessVisitorImpl(NestedAccessTy nestedAccessTy, + StorageCastTy storageCastTy) + : nestedAccessTy(nestedAccessTy), storageCastTy(storageCastTy) {} + + // MARK: AccessPhiVisitor::UseDefVisitor implementation. + // + // Subclasses must implement: + // isResultValid() + // invalidateResult() + // saveResult() + // restoreResult(Result) + // addUnknownOffset() + + void reenterUseDef(SILValue sourceAddr) { + SILValue nextAddr = this->visit(sourceAddr); + while (nextAddr) { + checkNextAddressType(nextAddr, sourceAddr); + nextAddr = this->visit(nextAddr); + } + } + + // MARK: visitor implementation. + + // Override AccessUseDefChainVisitor to ignore access markers and find the + // outer access base. + SILValue visitNestedAccess(BeginAccessInst *access) { + if (nestedAccessTy == IgnoreAccessBegin) + return access->getSource(); + + return SuperTy::visitNestedAccess(access); + } + + SILValue visitPhi(SILPhiArgument *phiArg) { + // Cycles involving phis are only handled within AccessPhiVisitor. + // Path components are not allowed in phi cycles. + if (visitedPhis.insert(phiArg).second) { + AccessPhiVisitor(this->asImpl(), storageCastTy) + .findPhiAccess(phiArg); + // Each phi operand was now reentrantly processed. Stop visiting. + return SILValue(); + } + // Cannot treat unresolved phis as "unidentified" because they may alias + // with global or class access. + return this->asImpl().visitNonAccess(phiArg); + } + + SILValue visitStorageCast(SingleValueInstruction *projectedAddr, + Operand *sourceAddr) { + assert(storageCastTy == IgnoreStorageCast); + return sourceAddr->get(); + } + + SILValue visitAccessProjection(SingleValueInstruction *projectedAddr, + Operand *sourceAddr) { + if (auto *indexAddr = dyn_cast(projectedAddr)) { + if (!Projection(indexAddr).isValid()) + this->asImpl().addUnknownOffset(); + } else if (isa(projectedAddr)) { + this->asImpl().addUnknownOffset(); + } + return sourceAddr->get(); + } + +protected: + // Helper for reenterUseDef + void checkNextAddressType(SILValue nextAddr, SILValue sourceAddr) { +#ifdef NDEBUG + return; +#endif + SILType type = nextAddr->getType(); + // FIXME: This relatively expensive pointer getAnyPointerElementType check + // is only needed because keypath generation incorrectly produces + // pointer_to_address directly from stdlib Pointer types without a + // struct_extract (as is correctly done in emitAddressorAccessor), and + // the PointerToAddressInst operand type is never verified. + if (type.getASTType()->getAnyPointerElementType()) + return; + + if (type.isAddress() || isa(type.getASTType()) + || isa(type.getASTType())) { + return; + } + llvm::errs() << "Visiting "; + sourceAddr->dump(); + llvm::errs() << " not an address "; + nextAddr->dump(); + nextAddr->getFunction()->dump(); + assert(false); + } +}; + +// Implement getAccessAddress, getAccessBegin, and getAccessBase. +class FindAccessBaseVisitor + : public FindAccessVisitorImpl { + using SuperTy = FindAccessVisitorImpl; + +protected: + Optional base; + +public: + FindAccessBaseVisitor(NestedAccessTy nestedAccessTy, + StorageCastTy storageCastTy) + : FindAccessVisitorImpl(nestedAccessTy, storageCastTy) {} + + // Returns the accessed address or an invalid SILValue. + SILValue findBase(SILValue sourceAddr) && { + reenterUseDef(sourceAddr); + return base.getValueOr(SILValue()); + } + + void setResult(SILValue foundBase) { + if (!base) + base = foundBase; + else if (base.getValue() != foundBase) + base = SILValue(); + } + + // MARK: AccessPhiVisitor::UseDefVisitor implementation. + + bool isResultValid() const { return base && bool(base.getValue()); } + + void invalidateResult() { base = SILValue(); } + + Optional saveResult() const { return base; } + + void restoreResult(Optional result) { base = result; } + + void addUnknownOffset() { return; } + + // MARK: visitor implementation. + + SILValue visitBase(SILValue base, AccessedStorage::Kind kind) { + setResult(base); + return SILValue(); } -} -SILValue swift::getAccessedAddress(SILValue v) { - while (true) { - SILValue v2 = stripAccessMarkers(getAddressAccess(v)); - if (v2 == v) - return v; - v = v2; + SILValue visitNonAccess(SILValue value) { + setResult(value); + return SILValue(); } + + // Override visitStorageCast to avoid seeing through arbitrary address casts. + SILValue visitStorageCast(SingleValueInstruction *projectedAddr, + Operand *sourceAddr) { + if (storageCastTy == StopAtStorageCast) + return visitNonAccess(projectedAddr); + + return SuperTy::visitStorageCast(projectedAddr, sourceAddr); + } +}; + +} // end anonymous namespace + +SILValue swift::getAccessAddress(SILValue address) { + assert(address->getType().isAddress()); + SILValue accessAddress = + FindAccessBaseVisitor(StopAtAccessBegin, StopAtStorageCast) + .findBase(address); + assert(accessAddress->getType().isAddress()); + return accessAddress; +} + +// TODO: When the optimizer stops stripping begin_access markers and SILGen +// protects all memory operations with at least an "unsafe" access scope, then +// we should be able to assert that this returns a BeginAccessInst. +SILValue swift::getAccessBegin(SILValue address) { + assert(address->getType().isAddress()); + return FindAccessBaseVisitor(StopAtAccessBegin, IgnoreStorageCast) + .findBase(address); } -bool swift::isLetAddress(SILValue accessedAddress) { - assert(accessedAddress == getAccessedAddress(accessedAddress) - && "caller must find the address root"); +// This is allowed to be called on a non-address pointer type. +SILValue swift::getAccessBase(SILValue address) { + return FindAccessBaseVisitor(IgnoreAccessBegin, IgnoreStorageCast) + .findBase(address); +} + +bool swift::isLetAddress(SILValue address) { + SILValue base = getAccessBase(address); + if (!base) + return false; + // Is this an address of a "let" class member? - if (auto *rea = dyn_cast(accessedAddress)) + if (auto *rea = dyn_cast(base)) return rea->getField()->isLet(); // Is this an address of a global "let"? - if (auto *gai = dyn_cast(accessedAddress)) { + if (auto *gai = dyn_cast(base)) { auto *globalDecl = gai->getReferencedGlobal()->getDecl(); return globalDecl && globalDecl->isLet(); } return false; } +//===----------------------------------------------------------------------===// +// MARK: FindReferenceRoot +//===----------------------------------------------------------------------===// + +namespace { + +// Essentially RC identity where the starting point is already a reference. +// +// FIXME: We cannot currently see through type casts for true RC identity, +// because property indices are not unique within a class hierarchy. Either fix +// RefElementAddr::getFieldNo so it returns a unique index, or figure out a +// different way to encode the property's VarDecl. (Note that the lack of a +// unique property index could be the source of bugs elsewhere). +class FindReferenceRoot { + SmallPtrSet visitedPhis; + +public: + SILValue findRoot(SILValue ref) && { + SILValue root = recursiveFindRoot(ref); + assert(root && "all phi inputs must be reachable"); + return root; + } + +protected: + // Return an invalid value for a phi with no resolved inputs. + // + // FIXME: We should be able to see through RC identity like this: + // nextRoot = stripRCIdentityCasts(nextRoot); + SILValue recursiveFindRoot(SILValue ref) { + while (true) { + SILValue nextRoot = ref; + nextRoot = stripOwnershipInsts(nextRoot); + if (nextRoot == ref) + break; + ref = nextRoot; + } + + auto *phi = dyn_cast(ref); + if (!phi || !phi->isPhiArgument()) { + return ref; + } + // Handle phis... + if (!visitedPhis.insert(phi).second) { + return SILValue(); + } + SILValue commonInput; + phi->visitIncomingPhiOperands([&](Operand *operand) { + SILValue input = recursiveFindRoot(operand->get()); + // Ignore "back/cross edges" to previously visited phis. + if (!input) + return true; + + if (!commonInput) { + commonInput = input; + return true; + } + if (commonInput == input) + return true; + + commonInput = phi; + return false; + }); + return commonInput; + } +}; + +} // end anonymous namespace + +static SILValue findReferenceRoot(SILValue ref) { + return FindReferenceRoot().findRoot(ref); +} + //===----------------------------------------------------------------------===// // MARK: AccessedStorage //===----------------------------------------------------------------------===// +constexpr unsigned AccessedStorage::TailIndex; + AccessedStorage::AccessedStorage(SILValue base, Kind kind) { assert(base && "invalid storage base"); initKind(kind); @@ -113,17 +480,17 @@ AccessedStorage::AccessedStorage(SILValue base, Kind kind) { case Class: { // Do a best-effort to find the identity of the object being projected // from. It is OK to be unsound here (i.e. miss when two ref_element_addrs - // actually refer the same address) because these addresses will be - // dynamically checked, and static analysis will be sufficiently - // conservative given that classes are not "uniquely identified". + // actually refer the same address) because, when the effort fails, static + // analysis will be sufficiently conservative given that classes are not + // "uniquely identified", and these addresses will be dynamically checked. auto *REA = cast(base); - value = stripBorrow(REA->getOperand()); + value = findReferenceRoot(REA->getOperand()); setElementIndex(REA->getFieldIndex()); break; } case Tail: { auto *RTA = cast(base); - value = stripBorrow(RTA->getOperand()); + value = findReferenceRoot(RTA->getOperand()); break; } } @@ -228,200 +595,748 @@ void AccessedStorage::print(raw_ostream &os) const { break; case Tail: os << getObject(); - os << " Tail\n"; } } -void AccessedStorage::dump() const { print(llvm::dbgs()); } +LLVM_ATTRIBUTE_USED void AccessedStorage::dump() const { print(llvm::dbgs()); } namespace { -// Find common AccessedStorage that leads to all arguments of a given -// pointer phi use. Return an invalid SILValue on failure. -// -// Also guarantees that all phi inputs follow the same access path. If any phi -// inputs have different access path components, then the phi is considered an -// invalid access. This is ok because path components always have an address -// type, and we are phasing out all address-type phis. Pointer-phis will -// continue to be allowed but they cannot affect the access path. -template -class FindPhiStorageVisitor - : public AccessUseDefChainVisitor> { - StorageVisitor &storageVisitor; - Optional commonDefinition; - SmallVector pointerWorklist; - SmallPtrSet nestedPhis; + +// Implementation of AccessUseDefChainVisitor that looks for a single common +// AccessedStorage object for all projection paths. +class FindAccessedStorageVisitor + : public FindAccessVisitorImpl { + +public: + struct Result { + Optional storage; + SILValue base; + }; + +private: + Result result; + + void setResult(AccessedStorage foundStorage, SILValue foundBase) { + if (!result.storage) { + result.storage = foundStorage; + assert(!result.base); + result.base = foundBase; + } else { + // `storage` may still be invalid. If both `storage` and `foundStorage` + // are invalid, this check passes, but we return an invalid storage + // below. + if (!result.storage->hasIdenticalBase(foundStorage)) + result.storage = AccessedStorage(); + if (result.base != foundBase) + result.base = SILValue(); + } + } + +public: + FindAccessedStorageVisitor(NestedAccessTy nestedAccessTy) + : FindAccessVisitorImpl(nestedAccessTy, IgnoreStorageCast) {} + + // Main entry point + void findStorage(SILValue sourceAddr) { this->reenterUseDef(sourceAddr); } + + AccessedStorage getStorage() const { + return result.storage.getValueOr(AccessedStorage()); + } + // getBase may return an invalid value for valid Global storage because there + // may be multiple global_addr bases for identical storage. + SILValue getBase() const { return result.base; } + + // MARK: AccessPhiVisitor::UseDefVisitor implementation. + + // A valid result requires valid storage, but not a valid base. + bool isResultValid() const { + return result.storage && bool(result.storage.getValue()); + } + + void invalidateResult() { setResult(AccessedStorage(), SILValue()); } + + Result saveResult() const { return result; } + + void restoreResult(Result savedResult) { result = savedResult; } + + void addUnknownOffset() { return; } + + // MARK: visitor implementation. + + SILValue visitBase(SILValue base, AccessedStorage::Kind kind) { + setResult(AccessedStorage(base, kind), base); + return SILValue(); + } + + SILValue visitNonAccess(SILValue value) { + invalidateResult(); + return SILValue(); + } +}; + +} // end anonymous namespace + +AccessedStorage swift::findAccessedStorage(SILValue sourceAddr) { + FindAccessedStorageVisitor visitor(IgnoreAccessBegin); + visitor.findStorage(sourceAddr); + return visitor.getStorage(); +} + +AccessedStorage swift::identifyAccessedStorageImpl(SILValue sourceAddr) { + FindAccessedStorageVisitor visitor(StopAtAccessBegin); + visitor.findStorage(sourceAddr); + return visitor.getStorage(); +} + +//===----------------------------------------------------------------------===// +// AccessPath +//===----------------------------------------------------------------------===// + +bool AccessPath::contains(AccessPath subPath) const { + assert(isValid() && subPath.isValid()); + + if (!storage.hasIdenticalBase(subPath.storage)) + return false; + + // Does the offset index match? + if (offset != subPath.offset || offset == UnknownOffset) + return false; + + return pathNode.node->isPrefixOf(subPath.pathNode.node); +} + +bool AccessPath::mayOverlap(AccessPath otherPath) const { + assert(isValid() && otherPath.isValid()); + + if (storage.isDistinctFrom(otherPath.storage)) + return false; + + // If subpaths are disjoint, they do not overlap regardless of offset. + if (!pathNode.node->isPrefixOf(otherPath.pathNode.node) + && !otherPath.pathNode.node->isPrefixOf(pathNode.node)) { + return true; + } + return offset == otherPath.offset || offset == UnknownOffset + || otherPath.offset == UnknownOffset; +} + +namespace { + +// Implementation of AccessUseDefChainVisitor that builds an AccessPath. +class AccessPathVisitor : public FindAccessVisitorImpl { + using SuperTy = FindAccessVisitorImpl; + + SILModule *module; + + // This nested visitor holds the AccessedStorage and base results. + FindAccessedStorageVisitor storageVisitor; + + // Save just enough information for to checkpoint before processing phis. Phis + // can add path components and add an unknown offset. + struct Result { + FindAccessedStorageVisitor::Result storageResult; + int savedOffset; + unsigned pathLength; + + Result(FindAccessedStorageVisitor::Result storageResult, int offset, + unsigned pathLength) + : storageResult(storageResult), savedOffset(offset), + pathLength(pathLength) {} + }; + + // Only access projections affect this path. Since they are are not allowed + // beyond phis, this path is not part of AccessPathVisitor::Result. + llvm::SmallVector reversePath; + // Holds a non-zero value if an index_addr has been processed without yet + // creating a path index for it. + int pendingOffset = 0; public: - FindPhiStorageVisitor(StorageVisitor &storageVisitor) - : storageVisitor(storageVisitor) {} + AccessPathVisitor(SILModule *module) + : FindAccessVisitorImpl(IgnoreAccessBegin, IgnoreStorageCast), + module(module), storageVisitor(IgnoreAccessBegin) {} // Main entry point. - void findPhiStorage(SILPhiArgument *phiArg) { - // Visiting a phi will call storageVisitor to set the storage result - // whenever it finds a base. - visitPhi(phiArg); - while (!pointerWorklist.empty()) { - this->visit(pointerWorklist.pop_back_val()); - } - // If a common path component was found, recursively look for the storage. - if (commonDefinition) { - if (commonDefinition.getValue()) { - auto storage = storageVisitor.findStorage(commonDefinition.getValue()); - (void)storage; // The same storageVisitor called us. It has already - // recorded the storage that it found. - } else { - // If divergent paths were found, invalidate any previously discovered - // storage. - storageVisitor.setStorage(AccessedStorage()); - } + AccessPathWithBase findAccessPath(SILValue sourceAddr) && { + this->reenterUseDef(sourceAddr); + if (auto storage = storageVisitor.getStorage()) { + return AccessPathWithBase( + AccessPath(storage, computeForwardPath(), pendingOffset), + storageVisitor.getBase()); } + return AccessPathWithBase(AccessPath(), SILValue()); } - // Visitor helper. - void setDefinition(SILValue def) { - if (!commonDefinition) { - commonDefinition = def; +protected: + void addPathOffset(int offset) { + if (pendingOffset == AccessPath::UnknownOffset) + return; + + if (offset == AccessPath::UnknownOffset) { + pendingOffset = offset; return; } - if (commonDefinition.getValue() != def) - commonDefinition = SILValue(); + // Accumulate static offsets + pendingOffset = pendingOffset + offset; } - // MARK: Visitor implementation. - - void checkResult(SILValue result) { - assert(!result && "must override any visitor that returns a result"); + // Return the trie node corresponding to the current state of reversePath. + AccessPath::PathNode computeForwardPath() { + IndexTrieNode *forwardPath = module->getIndexTrieRoot(); + for (AccessPath::Index nextIndex : llvm::reverse(reversePath)) { + forwardPath = forwardPath->getChild(nextIndex.getEncoding()); + } + return AccessPath::PathNode(forwardPath); } - // Recursively call the original storageVisitor for each base. We can't simply - // look for a common definition on all phi inputs, because the base may be - // cloned on each path. For example, two global_addr instructions may refer to - // the same global storage. Those global_addr instructions may each be - // converted to a RawPointer before being passed into the non-address phi. - void visitBase(SILValue base, AccessedStorage::Kind kind) { - checkResult(storageVisitor.visitBase(base, kind)); +public: + // MARK: AccessPhiVisitor::UseDefVisitor implementation. + + bool isResultValid() const { return storageVisitor.isResultValid(); } + + void invalidateResult() { + storageVisitor.invalidateResult(); + // Don't clear reversePath. We my call restoreResult later. + pendingOffset = 0; } - void visitNonAccess(SILValue value) { - checkResult(storageVisitor.visitNonAccess(value)); + Result saveResult() const { + return Result(storageVisitor.saveResult(), pendingOffset, + reversePath.size()); } - void visitNestedAccess(BeginAccessInst *access) { - checkResult(storageVisitor.visitNestedAccess(access)); + void restoreResult(Result result) { + storageVisitor.restoreResult(result.storageResult); + pendingOffset = result.savedOffset; + assert(result.pathLength <= reversePath.size() + && "a phi should only add to the path"); + reversePath.erase(reversePath.begin() + result.pathLength, + reversePath.end()); } - void visitPhi(SILPhiArgument *phiArg) { - if (nestedPhis.insert(phiArg).second) - phiArg->getIncomingPhiValues(pointerWorklist); + void addUnknownOffset() { pendingOffset = AccessPath::UnknownOffset; } + + // MARK: visitor implementation. Return the address source as the next use-def + // value to process. An invalid SILValue stops def-use traversal. + + SILValue visitBase(SILValue base, AccessedStorage::Kind kind) { + return storageVisitor.visitBase(base, kind); } - void visitCast(SingleValueInstruction *projectedAddr, Operand *parentAddr) { - // Allow conversions to/from pointers and addresses on disjoint phi paths. - this->pointerWorklist.push_back(parentAddr->get()); + SILValue visitNonAccess(SILValue value) { + invalidateResult(); + return SILValue(); } - void visitPathComponent(SingleValueInstruction *projectedAddr, - Operand *parentAddr) { - // Path components are not expected to occur on disjoint phi paths. Stop - // searching at this projection. - setDefinition(projectedAddr); + // Override FindAccessVisitorImpl to record path components. + SILValue visitAccessProjection(SingleValueInstruction *projectedAddr, + Operand *sourceAddr) { + auto projIdx = ProjectionIndex(projectedAddr); + if (auto *indexAddr = dyn_cast(projectedAddr)) { + addPathOffset(projIdx.isValid() ? projIdx.Index + : AccessPath::UnknownOffset); + } else if (isa(projectedAddr)) { + addPathOffset(AccessPath::UnknownOffset); + } else if (projIdx.isValid()) { + if (pendingOffset) { + LLVM_DEBUG(llvm::dbgs() << "Subobject projection with offset index: " + << *projectedAddr); + // Return an invalid result even though findAccessedStorage() may be + // able to find valid storage, because an offset from a subobject is an + // invalid access path. + return visitNonAccess(projectedAddr); + } + reversePath.push_back( + AccessPath::Index::forSubObjectProjection(projIdx.Index)); + } else { + // Ignore everything in getAccessProjectionOperand that is an access + // projection with no affect on the access path. + assert(isa(projectedAddr) + || isa(projectedAddr) + || isa(projectedAddr)); + } + return sourceAddr->get(); } }; -} // namespace + +} // end anonymous namespace + +AccessPathWithBase AccessPathWithBase::compute(SILValue address) { + return AccessPathVisitor(address->getModule()).findAccessPath(address); +} namespace { -// Implementation of AccessUseDefChainVisitor that looks for a single common -// AccessedStorage object for all projection paths. -template -class FindAccessedStorageVisitorBase - : public AccessUseDefChainVisitor { -protected: - Optional storage; - SmallPtrSet visitedPhis; + +// Perform def-use DFS traversal along a given AccessPath. DFS terminates at +// each discovered use. +// +// If \p collectOverlappingUses is false, then the collected uses all have the +// same AccessPath. Uses that exactly match the AccessPath may either be exact +// uses, or may be subobject projections within that access path, including +// struct_element_addr and tuple_element_addr. The transitive uses of those +// subobject projections are not included. +// +// If \p collectOverlappingUses is true, then the collected uses also include +// uses that access an object that contains the given AccessPath. As before, +// overlapping uses do not include transitive uses of subobject projections +// contained by the current path; the def-use traversal stops at those +// projections regardless of collectOverlappingUses. However, overlapping uses +// may be at an unknown offset relative to the current path, so they don't +// necessarily contain the current path. +// +// Example: path = "(#2)" +// %base = ... // access base +// load %base // containing use +// %elt1 = struct_element_addr %base, #1 // non-use (ignored) +// load %elt1 // non-use (unseen) +// %elt2 = struct_element_addr %base, #2 // chained use (ignored) +// load %elt2 // exact use +// %sub = struct_element_addr %elt2, #i // projection use +// load %sub // interior use (ignored) +// +// A use may be a BranchInst if the corresponding phi does not have common +// AccessedStorage. +// +// For class storage, the def-use traversal starts at the reference +// root. Eventually, traversal reach the base address of the formal access: +// +// %ref = ... // reference root +// %base = ref_element_addr %refRoot // formal access address +// load %base // use +class CollectAccessPathUses { + // Origin of the def-use traversal + AccessedStorage storage; + + // Result: Exact uses, projection uses, and containing uses. + SmallVectorImpl &uses; + + bool collectOverlappingUses; + unsigned useLimit; + + // Access path indices from storage to exact uses + SmallVector pathIndices; // in use-def order + + // A point in the def-use traversal. isRef() is true only for object access + // prior to reaching the base address. + struct DFSEntry { + // Next potential use to visit and flag indicating whether traversal has + // reachaed the access base yet. + llvm::PointerIntPair useAndIsRef; + unsigned pathCursor; // position within pathIndices + int offset; // index_addr offsets seen prior to this use + + DFSEntry(Operand *use, bool isRef, unsigned pathCursor, int offset) + : useAndIsRef(use, isRef), pathCursor(pathCursor), offset(offset) {} + + Operand *getUse() const { return useAndIsRef.getPointer(); } + // Is this pointer a reference? + bool isRef() const { return useAndIsRef.getInt(); } + }; + SmallVector dfsStack; + + SmallPtrSet visitedPhis; public: - // Main entry point. May be called reentrantly by the phi visitor. - AccessedStorage findStorage(SILValue sourceAddr) { - SILValue nextAddr = this->visit(sourceAddr); - while (nextAddr) { - assert(nextAddr->getType().isAddress() - || isa(nextAddr->getType().getASTType()) - || isa(nextAddr->getType().getASTType())); - nextAddr = this->visit(nextAddr); + CollectAccessPathUses(AccessPath accessPath, SmallVectorImpl &uses, + bool collectOverlappingUses, unsigned useLimit) + : storage(accessPath.getStorage()), uses(uses), + collectOverlappingUses(collectOverlappingUses), useLimit(useLimit) { + assert(accessPath.isValid()); + for (AccessPath::PathNode currentNode = accessPath.getPathNode(); + !currentNode.isRoot(); currentNode = currentNode.getParent()) { + assert(currentNode.getIndex().isSubObjectProjection() && + "a valid AccessPath does not contain any intermediate offsets"); + pathIndices.push_back(currentNode.getIndex()); + } + if (int offset = accessPath.getOffset()) + pathIndices.push_back(AccessPath::Index::forOffset(offset)); + + // The search will start from the object root, not the formal access base, + // so add the class index to the front. + auto storage = accessPath.getStorage(); + if (storage.getKind() == AccessedStorage::Class) { + pathIndices.push_back(AccessPath::Index::forSubObjectProjection( + storage.getPropertyIndex())); + } + if (storage.getKind() == AccessedStorage::Tail) { + pathIndices.push_back(AccessPath::Index::forSubObjectProjection( + ProjectionIndex::TailIndex)); } - return storage.getValueOr(AccessedStorage()); } - void setStorage(AccessedStorage foundStorage) { - if (!storage) { - storage = foundStorage; - } else { - // `storage` may still be invalid. If both `storage` and `foundStorage` - // are invalid, this check passes, but we return an invalid storage - // below. - if (!storage->hasIdenticalBase(foundStorage)) - storage = AccessedStorage(); + // Return true if all uses were collected. This is always true as long as the + // access has a single root, or globalBase is provided, and there is no + // useLimit. + // + // For Global storage \p globalBase must be provided as the head of the + // def-use search. + bool collectUses(SILValue globalBase = SILValue()) && { + SILValue root = storage.getRoot(); + if (!root) { + assert(storage.getKind() == AccessedStorage::Global); + if (!globalBase) + return false; + + root = globalBase; } + // If the expected path has an unknown offset, then none of the uses are + // exact. + if (!collectOverlappingUses && !pathIndices.empty() + && pathIndices.back().isUnknownOffset()) { + return true; + } + pushUsers(root, + DFSEntry(nullptr, storage.isReference(), pathIndices.size(), 0)); + while (!dfsStack.empty()) { + if (!visitUser(dfsStack.pop_back_val())) + return false; + } + return true; } - // MARK: visitor implementation. +protected: + void pushUsers(SILValue def, const DFSEntry &dfs) { + for (auto *use : def->getUses()) + pushUser(DFSEntry(use, dfs.isRef(), dfs.pathCursor, dfs.offset)); + } - SILValue visitBase(SILValue base, AccessedStorage::Kind kind) { - setStorage(AccessedStorage(base, kind)); - return SILValue(); + void pushUser(DFSEntry dfs) { + Operand *use = dfs.getUse(); + if (auto *bi = dyn_cast(use->getUser())) { + if (pushPhiUses(bi->getArgForOperand(use), dfs)) + return; + } + // If we didn't find and process a phi, continue DFS. + dfsStack.emplace_back(dfs); } - SILValue visitNonAccess(SILValue value) { - setStorage(AccessedStorage()); - return SILValue(); + // Return true if this phi has been processed and does not need to be + // considered as a separate use. + bool pushPhiUses(const SILPhiArgument *phi, DFSEntry dfs) { + if (!visitedPhis.insert(phi).second) + return true; + + // If this phi has a common base, continue to follow the access path. This + // check is different for reference types vs pointer types. + if (dfs.isRef()) { + assert(!dfs.offset && "index_addr not allowed on reference roots"); + // When isRef is true, the address access hasn't been seen yet and + // we're still following the reference root's users. Check if all phi + // inputs have the same reference root before looking through it. + if (findReferenceRoot(phi) == storage.getObject()) { + pushUsers(phi, dfs); + return true; + } + // The branch will be pushed onto the normal user list. + return false; + } + // Check if all phi inputs have the same accessed storage before + // looking through it. If the phi input differ the its storage is invalid. + auto phiPath = AccessPath::compute(phi); + if (phiPath.isValid()) { + assert(phiPath.getStorage().hasIdenticalBase(storage) + && "inconsistent phi storage"); + // If the phi paths have different offsets, its path has unknown offset. + if (phiPath.getOffset() == AccessPath::UnknownOffset) { + if (!collectOverlappingUses) + return true; + dfs.offset = AccessPath::UnknownOffset; + } + pushUsers(phi, dfs); + return true; + } + // The branch will be pushed onto the normal user list. + return false; } - SILValue visitPhi(SILPhiArgument *phiArg) { - // Cycles involving phis are only handled within FindPhiStorageVisitor. - // Path components are not allowed in phi cycles. - if (visitedPhis.insert(phiArg).second) { - FindPhiStorageVisitor(this->asImpl()).findPhiStorage(phiArg); - return SILValue(); + // Return the offset at the current DFS path cursor, or zero. + int getPathOffset(const DFSEntry &dfs) const { + if (dfs.pathCursor == 0 + || pathIndices[dfs.pathCursor - 1].isSubObjectProjection()) { + return 0; } - // Cannot treat unresolved phis as "unidentified" because they may alias - // with global or class access. - return visitNonAccess(phiArg); + return pathIndices[dfs.pathCursor - 1].getOffset(); } - SILValue visitCast(SingleValueInstruction *projectedAddr, - Operand *parentAddr) { - return parentAddr->get(); + // Returns true as long as the useLimit is not reached. + bool visitUser(DFSEntry dfs) { + Operand *use = dfs.getUse(); + assert(!(dfs.isRef() && use->get()->getType().isAddress())); + if (auto *svi = dyn_cast(use->getUser())) { + if (use->getOperandNumber() == 0 + && visitSingleValueUser(svi, dfs) == IgnoredUse) { + return true; + } + } + // We weren't able to "see through" any more address conversions; so + // record this as a use. + + // Do the path offsets match? + if (!checkAndUpdateOffset(dfs)) + return true; + + // Is this a full or partial path match? + if (!collectOverlappingUses && dfs.pathCursor > 0) + return true; + + // Record the use if we haven't reached the limit. + if (uses.size() == useLimit) + return false; + + uses.push_back(use); + return true; } - SILValue visitPathComponent(SingleValueInstruction *projectedAddr, - Operand *parentAddr) { - return parentAddr->get(); + // Return true if the accumulated offset matches the current path index. + // Update the DFSEntry and pathCursor to skip remaining offsets. + bool checkAndUpdateOffset(DFSEntry &dfs) { + int pathOffset = getPathOffset(dfs); + if (pathOffset == 0) { + // No offset is on the expected path. + if (collectOverlappingUses && dfs.offset == AccessPath::UnknownOffset) { + dfs.offset = 0; + } + return dfs.offset == 0; + } + // pop the offset from the expected path; there should only be one. + --dfs.pathCursor; + assert(getPathOffset(dfs) == 0 && "only one offset index allowed"); + + int useOffset = dfs.offset; + dfs.offset = 0; + + // Ignore all uses on this path unless we're collecting containing uses. + // UnknownOffset appears to overlap with all offsets and subobject uses. + if (pathOffset == AccessPath::UnknownOffset + || useOffset == AccessPath::UnknownOffset) { + return collectOverlappingUses; + } + // A known offset must match regardless of collectOverlappingUses. + return pathOffset == useOffset; } -}; -struct FindAccessedStorageVisitor - : public FindAccessedStorageVisitorBase { + enum UseKind { LeafUse, IgnoredUse }; + UseKind visitSingleValueUser(SingleValueInstruction *svi, DFSEntry dfs); - SILValue visitNestedAccess(BeginAccessInst *access) { - return access->getSource(); + // Handle non-index_addr projections. + UseKind followProjection(SingleValueInstruction *svi, DFSEntry dfs) { + if (!checkAndUpdateOffset(dfs)) + return IgnoredUse; + + if (dfs.pathCursor == 0) + return LeafUse; + + AccessPath::Index pathIndex = pathIndices[dfs.pathCursor - 1]; + auto projIdx = ProjectionIndex(svi); + assert(projIdx.isValid()); + // Only subobjects indices are expected because offsets are handled above. + if (projIdx.Index == pathIndex.getSubObjectIndex()) { + --dfs.pathCursor; + pushUsers(svi, dfs); + } + return IgnoredUse; } }; -struct IdentifyAccessedStorageVisitor - : public FindAccessedStorageVisitorBase {}; +} // end anonymous namespace -} // namespace +// During the def-use traversal, visit a single-value instruction in which the +// used address is at operand zero. +// +// This must handle the def-use side of all operations that +// AccessUseDefChainVisitor::visit can handle. +// +// Return IgnoredUse if the def-use traversal either continues past \p +// svi or ignores this use. +// +// FIXME: Reuse getAccessProjectionOperand() instead of using special cases once +// the unchecked_take_enum_data_addr -> load -> project_box pattern is fixed. +CollectAccessPathUses::UseKind +CollectAccessPathUses::visitSingleValueUser(SingleValueInstruction *svi, + DFSEntry dfs) { + if (isAccessedStorageCast(svi)) { + pushUsers(svi, dfs); + return IgnoredUse; + } + switch (svi->getKind()) { + default: + return LeafUse; -AccessedStorage swift::findAccessedStorage(SILValue sourceAddr) { - return FindAccessedStorageVisitor().findStorage(sourceAddr); + case SILInstructionKind::BeginAccessInst: + pushUsers(svi, dfs); + return IgnoredUse; + + // Handle ref_element_addr since we start at the object root instead of + // the access base. + case SILInstructionKind::RefElementAddrInst: + assert(dfs.isRef()); + assert(dfs.pathCursor > 0 && "ref_element_addr cannot occur within access"); + dfs.useAndIsRef.setInt(false); + return followProjection(svi, dfs); + + case SILInstructionKind::RefTailAddrInst: { + assert(dfs.isRef()); + assert(dfs.pathCursor > 0 && "ref_tail_addr cannot occur within an access"); + dfs.useAndIsRef.setInt(false); + --dfs.pathCursor; + AccessPath::Index pathIndex = pathIndices[dfs.pathCursor]; + assert(pathIndex.isSubObjectProjection()); + if (pathIndex.getSubObjectIndex() == AccessedStorage::TailIndex) + pushUsers(svi, dfs); + + return IgnoredUse; + } + + // MARK: Access projections + + case SILInstructionKind::StructElementAddrInst: + case SILInstructionKind::TupleElementAddrInst: + return followProjection(svi, dfs); + + case SILInstructionKind::IndexAddrInst: + case SILInstructionKind::TailAddrInst: { + auto projIdx = ProjectionIndex(svi); + if (projIdx.isValid()) { + if (dfs.offset != AccessPath::UnknownOffset) + dfs.offset += projIdx.Index; + else + assert(collectOverlappingUses); + } else if (collectOverlappingUses) { + dfs.offset = AccessPath::UnknownOffset; + } else { + return IgnoredUse; + } + pushUsers(svi, dfs); + return IgnoredUse; + } + + // open_existential_addr and unchecked_take_enum_data_addr are classified as + // access projections, but they also modify memory. Both see through them and + // also report them as uses. + case SILInstructionKind::OpenExistentialAddrInst: + case SILInstructionKind::UncheckedTakeEnumDataAddrInst: + pushUsers(svi, dfs); + return LeafUse; + + case SILInstructionKind::StructExtractInst: + // Handle nested access to a KeyPath projection. The projection itself + // uses a Builtin. However, the returned UnsafeMutablePointer may be + // converted to an address and accessed via an inout argument. + if (isUnsafePointerExtraction(cast(svi))) { + pushUsers(svi, dfs); + return IgnoredUse; + } + return LeafUse; + + case SILInstructionKind::LoadInst: + // Load a box from an indirect payload of an opaque enum. See comments + // in AccessUseDefChainVisitor::visit. Record this load as a leaf-use even + // when we look through its project_box because anyone inspecting the load + // itself will see the same AccessPath. + // FIXME: if this doesn't go away with opaque values, add a new instruction + // for load+project_box. + if (svi->getType().is()) { + Operand *addrOper = &cast(svi)->getOperandRef(); + assert(isa(addrOper->get())); + // Push the project_box uses + for (auto *use : svi->getUses()) { + if (isa(use->getUser())) + pushUser(DFSEntry(use, dfs.isRef(), dfs.pathCursor, dfs.offset)); + } + } + return LeafUse; + } } -AccessedStorage swift::identifyAccessedStorageImpl(SILValue sourceAddr) { - return IdentifyAccessedStorageVisitor().findStorage(sourceAddr); +bool AccessPath::collectUses(SmallVectorImpl &uses, + bool collectOverlappingUses, + unsigned useLimit) const { + return CollectAccessPathUses(*this, uses, collectOverlappingUses, useLimit) + .collectUses(); +} + +bool AccessPathWithBase::collectUses(SmallVectorImpl &uses, + bool collectOverlappingUses, + unsigned useLimit) const { + CollectAccessPathUses collector(accessPath, uses, collectOverlappingUses, + useLimit); + if (accessPath.getRoot()) + return std::move(collector).collectUses(); + + if (!base) + return false; + + return std::move(collector).collectUses(base); +} + +void AccessPath::Index::print(raw_ostream &os) const { + if (isSubObjectProjection()) + os << '#' << getSubObjectIndex(); + else { + os << '@'; + if (getOffset() == AccessPath::UnknownOffset) + os << "Unknown"; + else + os << getOffset(); + } +} + +LLVM_ATTRIBUTE_USED void AccessPath::Index::dump() const { + print(llvm::dbgs()); +} + +static void recursivelyPrintPath(AccessPath::PathNode node, raw_ostream &os) { + AccessPath::PathNode parent = node.getParent(); + if (!parent.isRoot()) { + recursivelyPrintPath(parent, os); + os << ","; + } + node.getIndex().print(os); +} + +void AccessPath::printPath(raw_ostream &os) const { + os << "Path: "; + if (!isValid()) { + os << "INVALID\n"; + return; + } + os << "("; + PathNode node = getPathNode(); + if (offset != 0) { + Index::forOffset(offset).print(os); + if (!node.isRoot()) + os << ","; + } + if (!node.isRoot()) + recursivelyPrintPath(node, os); + os << ")\n"; +} + +void AccessPath::print(raw_ostream &os) const { + if (!isValid()) { + os << "INVALID\n"; + return; + } + os << "Storage: "; + getStorage().print(os); + printPath(os); +} + +LLVM_ATTRIBUTE_USED void AccessPath::dump() const { print(llvm::dbgs()); } + +void AccessPathWithBase::print(raw_ostream &os) const { + if (base) + os << "Base: " << base; + + accessPath.print(os); +} + +LLVM_ATTRIBUTE_USED void AccessPathWithBase::dump() const { + print(llvm::dbgs()); } //===----------------------------------------------------------------------===// -// MARK: Helper API +// MARK: Helper API for specific formal access patterns //===----------------------------------------------------------------------===// static bool isScratchBuffer(SILValue value) { @@ -556,7 +1471,9 @@ bool swift::isExternalGlobalAddressor(ApplyInst *AI) { // Return true if the given StructExtractInst extracts the RawPointer from // Unsafe[Mutable]Pointer. bool swift::isUnsafePointerExtraction(StructExtractInst *SEI) { - assert(isa(SEI->getType().getASTType())); + if (!isa(SEI->getType().getASTType())) + return false; + auto &C = SEI->getModule().getASTContext(); auto *decl = SEI->getStructDecl(); return decl == C.getUnsafeMutablePointerDecl() @@ -652,9 +1569,9 @@ SILBasicBlock::iterator swift::removeBeginAccess(BeginAccessInst *beginAccess) { // Verification //===----------------------------------------------------------------------===// -/// Helper for visitApplyAccesses that visits address-type call arguments, -/// including arguments to @noescape functions that are passed as closures to -/// the current call. +// Helper for visitApplyAccesses that visits address-type call arguments, +// including arguments to @noescape functions that are passed as closures to +// the current call. static void visitApplyAccesses(ApplySite apply, llvm::function_ref visitor) { for (Operand &oper : apply.getArgumentOperands()) { @@ -686,14 +1603,27 @@ static void visitBuiltinAddress(BuiltinInst *builtin, builtin->dump(); llvm_unreachable("unexpected builtin memory access."); - // WillThrow exists for the debugger, does nothing. + // Handle builtin "generic_add"($*V, $*V, $*V) and the like. +#define BUILTIN(Id, Name, Attrs) +#define BUILTIN_BINARY_OPERATION_POLYMORPHIC(Id, Name) \ + case BuiltinValueKind::Id: + +#include "swift/AST/Builtins.def" + + visitor(&builtin->getAllOperands()[1]); + visitor(&builtin->getAllOperands()[2]); + return; + + // WillThrow exists for the debugger, does nothing. case BuiltinValueKind::WillThrow: return; - // Buitins that affect memory but can't be formal accesses. + // Buitins that affect memory but can't be formal accesses. + case BuiltinValueKind::AssumeTrue: case BuiltinValueKind::UnexpectedError: case BuiltinValueKind::ErrorInMain: case BuiltinValueKind::IsOptionalType: + case BuiltinValueKind::CondFailMessage: case BuiltinValueKind::AllocRaw: case BuiltinValueKind::DeallocRaw: case BuiltinValueKind::Fence: @@ -703,14 +1633,15 @@ static void visitBuiltinAddress(BuiltinInst *builtin, case BuiltinValueKind::Unreachable: case BuiltinValueKind::CondUnreachable: case BuiltinValueKind::DestroyArray: - case BuiltinValueKind::COWBufferForReading: case BuiltinValueKind::UnsafeGuaranteed: case BuiltinValueKind::UnsafeGuaranteedEnd: case BuiltinValueKind::Swift3ImplicitObjCEntrypoint: + case BuiltinValueKind::PoundAssert: + case BuiltinValueKind::IntInstrprofIncrement: case BuiltinValueKind::TSanInoutAccess: return; - // General memory access to a pointer in first operand position. + // General memory access to a pointer in first operand position. case BuiltinValueKind::CmpXChg: case BuiltinValueKind::AtomicLoad: case BuiltinValueKind::AtomicStore: @@ -720,8 +1651,8 @@ static void visitBuiltinAddress(BuiltinInst *builtin, // visitor(&builtin->getAllOperands()[0]); return; - // Arrays: (T.Type, Builtin.RawPointer, Builtin.RawPointer, - // Builtin.Word) + // Arrays: (T.Type, Builtin.RawPointer, Builtin.RawPointer, + // Builtin.Word) case BuiltinValueKind::CopyArray: case BuiltinValueKind::TakeArrayNoAlias: case BuiltinValueKind::TakeArrayFrontToBack: @@ -811,11 +1742,11 @@ void swift::visitAccessedAddress(SILInstruction *I, visitor(&I->getAllOperands()[0]); return; + case SILInstructionKind::InitExistentialAddrInst: + case SILInstructionKind::InjectEnumAddrInst: #define NEVER_OR_SOMETIMES_LOADABLE_CHECKED_REF_STORAGE(Name, ...) \ case SILInstructionKind::Load##Name##Inst: #include "swift/AST/ReferenceStorage.def" - case SILInstructionKind::InitExistentialAddrInst: - case SILInstructionKind::InjectEnumAddrInst: case SILInstructionKind::LoadInst: case SILInstructionKind::LoadBorrowInst: case SILInstructionKind::OpenExistentialAddrInst: @@ -845,6 +1776,8 @@ void swift::visitAccessedAddress(SILInstruction *I, case SILInstructionKind::BeginAccessInst: case SILInstructionKind::BeginApplyInst: case SILInstructionKind::BeginBorrowInst: + case SILInstructionKind::BeginCOWMutationInst: + case SILInstructionKind::EndCOWMutationInst: case SILInstructionKind::BeginUnpairedAccessInst: case SILInstructionKind::BindMemoryInst: case SILInstructionKind::CheckedCastValueBranchInst: @@ -863,6 +1796,7 @@ void swift::visitAccessedAddress(SILInstruction *I, case SILInstructionKind::EndLifetimeInst: case SILInstructionKind::ExistentialMetatypeInst: case SILInstructionKind::FixLifetimeInst: + case SILInstructionKind::GlobalAddrInst: case SILInstructionKind::InitExistentialValueInst: case SILInstructionKind::IsUniqueInst: case SILInstructionKind::IsEscapingClosureInst: diff --git a/lib/SIL/Verifier/LoadBorrowInvalidationChecker.cpp b/lib/SIL/Verifier/LoadBorrowInvalidationChecker.cpp index 1f5e84803d5b9..b5aea5103bc67 100644 --- a/lib/SIL/Verifier/LoadBorrowInvalidationChecker.cpp +++ b/lib/SIL/Verifier/LoadBorrowInvalidationChecker.cpp @@ -453,12 +453,20 @@ bool LoadBorrowNeverInvalidatedAnalysis:: bool LoadBorrowNeverInvalidatedAnalysis::isNeverInvalidated( LoadBorrowInst *lbi) { - SILValue address = getAddressAccess(lbi->getOperand()); + + SILValue address = getAccessBegin(lbi->getOperand()); if (!address) return false; + auto storage = findAccessedStorage(address); + // If we couldn't find an access storage, return that we are assumed to write. + if (!storage) { + llvm::errs() << "Couldn't compute access storage?!\n"; + return false; + } + // If we have a let address, then we are already done. - if (isLetAddress(stripAccessMarkers(address))) + if (storage.isLetAccess(lbi->getFunction())) return true; // At this point, we know that we /may/ have writes. Now we go through various @@ -478,24 +486,34 @@ bool LoadBorrowNeverInvalidatedAnalysis::isNeverInvalidated( // Otherwise, validate that any writes to our begin_access is not when the // load_borrow's result is live. + // + // FIXME: do we verify that the load_borrow scope is always nested within + // the begin_access scope (to ensure no aliasing access)? return doesAddressHaveWriteThatInvalidatesLoadBorrow(lbi, endBorrowUses, bai); } + // FIXME: the subsequent checks appear to assume that 'address' is not aliased + // within the scope of the load_borrow. This can only be assumed when either + // the load_borrow is nested within an access scope or when the + // storage.isUniquelyIdentified() and all uses of storage.getRoot() have been + // analyzed. The later can be done with AccessPath::collectUses(). + // Check if our unidentified access storage is a project_box. In such a case, // validate that all uses of the project_box are not writes that overlap with // our load_borrow's result. These are things that may not be a formal access // base. // - // FIXME: we don't seem to verify anywhere that a pointer_to_address cannot - // itself be derived from another address that is accessible in the same - // function, either because it was returned from a call or directly - // address_to_pointer'd. + // FIXME: Remove this PointerToAddress check. It appears to be incorrect. we + // don't verify anywhere that a pointer_to_address cannot itself be derived + // from another address that is accessible in the same function, either + // because it was returned from a call or directly address_to_pointer'd. if (isa(address)) { return doesAddressHaveWriteThatInvalidatesLoadBorrow(lbi, endBorrowUses, address); } + // FIXME: This ProjectBoxInst // If we have a project_box, we need to see if our base, modulo begin_borrow, // copy_value have any other project_box that we need to analyze. if (auto *pbi = dyn_cast(address)) { @@ -503,13 +521,6 @@ bool LoadBorrowNeverInvalidatedAnalysis::isNeverInvalidated( pbi->getOperand()); } - auto storage = findAccessedStorage(address); - - // If we couldn't find an access storage, return that we are assumed to write. - if (!storage) { - llvm::errs() << "Couldn't compute access storage?!\n"; - return false; - } switch (storage.getKind()) { case AccessedStorage::Stack: { @@ -562,7 +573,8 @@ bool LoadBorrowNeverInvalidatedAnalysis::isNeverInvalidated( } case AccessedStorage::Unidentified: { // Otherwise, we didn't understand this, so bail. - llvm::errs() << "Unidentified access storage: " << storage; + llvm::errs() << "Unidentified access storage: "; + storage.dump(); return false; } case AccessedStorage::Nested: { diff --git a/lib/SILOptimizer/Analysis/AliasAnalysis.cpp b/lib/SILOptimizer/Analysis/AliasAnalysis.cpp index f9b28518454ca..44c40297e4b2c 100644 --- a/lib/SILOptimizer/Analysis/AliasAnalysis.cpp +++ b/lib/SILOptimizer/Analysis/AliasAnalysis.cpp @@ -334,7 +334,7 @@ static bool isAccessedAddressTBAASafe(SILValue V) { if (!V->getType().isAddress()) return false; - SILValue accessedAddress = getAccessedAddress(V); + SILValue accessedAddress = getAccessAddress(V); if (isa(accessedAddress)) return true; diff --git a/lib/SILOptimizer/Analysis/MemoryBehavior.cpp b/lib/SILOptimizer/Analysis/MemoryBehavior.cpp index b867c18b6300a..376c9a220313d 100644 --- a/lib/SILOptimizer/Analysis/MemoryBehavior.cpp +++ b/lib/SILOptimizer/Analysis/MemoryBehavior.cpp @@ -78,11 +78,13 @@ class MemoryBehaviorVisitor } /// If 'V' is an address projection within a formal access, return the - /// canonical address of the formal access. Otherwise, return 'V' itself, - /// which is either a reference or unknown pointer or address. + /// canonical address of the formal access if possible without looking past + /// any storage casts. Otherwise, a "best-effort" address + /// + /// If 'V' is an address, then the returned value is also an address. SILValue getValueAddress() { if (!cachedValueAddress) { - cachedValueAddress = V->getType().isAddress() ? getAccessedAddress(V) : V; + cachedValueAddress = V->getType().isAddress() ? getAccessAddress(V) : V; } return cachedValueAddress; } @@ -147,7 +149,7 @@ class MemoryBehaviorVisitor case SILAccessKind::Modify: if (isLetValue()) { - assert(stripAccessMarkers(beginAccess) != getValueAddress() + assert(getAccessBase(beginAccess) != getValueAddress() && "let modification not allowed"); return MemBehavior::None; } @@ -251,8 +253,7 @@ MemBehavior MemoryBehaviorVisitor::visitLoadInst(LoadInst *LI) { MemBehavior MemoryBehaviorVisitor::visitStoreInst(StoreInst *SI) { // No store besides the initialization of a "let"-variable // can have any effect on the value of this "let" variable. - if (isLetValue() - && (getAccessedAddress(SI->getDest()) != getValueAddress())) { + if (isLetValue() && (getAccessBase(SI->getDest()) != getValueAddress())) { return MemBehavior::None; } // If the store dest cannot alias the pointer in question, then the diff --git a/lib/SILOptimizer/LoopTransforms/LICM.cpp b/lib/SILOptimizer/LoopTransforms/LICM.cpp index 39c4628324e7c..6eb5e30a7f988 100644 --- a/lib/SILOptimizer/LoopTransforms/LICM.cpp +++ b/lib/SILOptimizer/LoopTransforms/LICM.cpp @@ -711,7 +711,7 @@ static bool analyzeBeginAccess(BeginAccessInst *BI, return true; } return BIAccessedStorageNonNested.isDistinctFrom( - findAccessedStorage(OtherBI)); + findAccessedStorage(OtherBI)); }; if (!std::all_of(BeginAccesses.begin(), BeginAccesses.end(), safeBeginPred)) diff --git a/lib/SILOptimizer/Mandatory/DiagnoseStaticExclusivity.cpp b/lib/SILOptimizer/Mandatory/DiagnoseStaticExclusivity.cpp index eace1f0eac6bb..674036973164d 100644 --- a/lib/SILOptimizer/Mandatory/DiagnoseStaticExclusivity.cpp +++ b/lib/SILOptimizer/Mandatory/DiagnoseStaticExclusivity.cpp @@ -964,20 +964,20 @@ static void checkStaticExclusivity(SILFunction &Fn, PostOrderFunctionInfo *PO, // Check that the given address-type operand is guarded by begin/end access // markers. static void checkAccessedAddress(Operand *memOper, StorageMap &Accesses) { - SILValue address = getAddressAccess(memOper->get()); + SILValue accessBegin = getAccessBegin(memOper->get()); SILInstruction *memInst = memOper->getUser(); - auto error = [address, memInst]() { + auto error = [accessBegin, memInst]() { llvm::dbgs() << "Memory access not protected by begin_access:\n"; memInst->printInContext(llvm::dbgs()); - llvm::dbgs() << "Accessing: " << address; + llvm::dbgs() << "Accessing: " << accessBegin; llvm::dbgs() << "In function:\n"; memInst->getFunction()->print(llvm::dbgs()); abort(); }; // Check if this address is guarded by an access. - if (auto *BAI = dyn_cast(address)) { + if (auto *BAI = dyn_cast(accessBegin)) { if (BAI->getEnforcement() == SILAccessEnforcement::Unsafe) return; @@ -1017,7 +1017,7 @@ static void checkAccessedAddress(Operand *memOper, StorageMap &Accesses) { return; } - const AccessedStorage &storage = findAccessedStorage(address); + const AccessedStorage &storage = findAccessedStorage(accessBegin); // findAccessedStorage may return an invalid storage object if the address // producer is not recognized by its allowlist. For the purpose of // verification, we assume that this can only happen for local From 1e9c4d13ad7caf62665ef996e898ec21f70e3fa1 Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Thu, 3 Sep 2020 13:02:48 -0700 Subject: [PATCH 07/20] Update LoadCopyToLoadBorrow for MemAccessUtils API. --- lib/SILOptimizer/SemanticARC/LoadCopyToLoadBorrowOpt.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/SILOptimizer/SemanticARC/LoadCopyToLoadBorrowOpt.cpp b/lib/SILOptimizer/SemanticARC/LoadCopyToLoadBorrowOpt.cpp index 157e21c3a9406..38d6304030741 100644 --- a/lib/SILOptimizer/SemanticARC/LoadCopyToLoadBorrowOpt.cpp +++ b/lib/SILOptimizer/SemanticARC/LoadCopyToLoadBorrowOpt.cpp @@ -266,8 +266,13 @@ class StorageGuaranteesLoadVisitor return next(parentAddr->get()); } - void visitPathComponent(SingleValueInstruction *projectedAddr, - Operand *parentAddr) { + void visitStorageCast(SingleValueInstruction *projectedAddr, + Operand *parentAddr) { + return next(parentAddr->get()); + } + + void visitAccessProjection(SingleValueInstruction *projectedAddr, + Operand *parentAddr) { return next(parentAddr->get()); } From 7e435fa2de512117028e202ea001c28724674a80 Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Thu, 27 Aug 2020 17:46:58 -0700 Subject: [PATCH 08/20] Add AccessPath support to the AccessedStorageDumper pass. Add a flag: enable-accessed-storage-dump-uses --- .../UtilityPasses/AccessedStorageDumper.cpp | 58 ++++++++++++++----- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/lib/SILOptimizer/UtilityPasses/AccessedStorageDumper.cpp b/lib/SILOptimizer/UtilityPasses/AccessedStorageDumper.cpp index ee5208a4bc6aa..31685d7a1c6bf 100644 --- a/lib/SILOptimizer/UtilityPasses/AccessedStorageDumper.cpp +++ b/lib/SILOptimizer/UtilityPasses/AccessedStorageDumper.cpp @@ -1,8 +1,8 @@ -//===--- AccessedStorageDumper.cpp - Dump accessed storage for functions ---===// +//===--- AccessedStorageDumper.cpp - Dump accessed storage ----------------===// // // This source file is part of the Swift.org open source project // -// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -17,24 +17,52 @@ #include "swift/SIL/SILValue.h" #include "swift/SILOptimizer/PassManager/Passes.h" #include "swift/SILOptimizer/PassManager/Transforms.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" using namespace swift; -static void dumpAccessedStorage(SILInstruction *inst) { - visitAccessedAddress( - inst, - [&](Operand *operand) { - inst->print(llvm::outs()); - findAccessedStorage(operand->get()).print(llvm::outs()); - } - ); -} +static llvm::cl::opt EnableDumpUses( + "enable-accessed-storage-dump-uses", llvm::cl::init(false), + llvm::cl::desc("With --sil-access-storage-dumper, dump all uses")); namespace { /// Dumps sorage information for each access. class AccessedStorageDumper : public SILModuleTransform { + llvm::SmallVector uses; + + void dumpAccessedStorage(Operand *operand) { + findAccessedStorage(operand->get()).print(llvm::outs()); + auto pathAndBase = AccessPathWithBase::compute(operand->get()); + pathAndBase.print(llvm::outs()); + + if (!pathAndBase.accessPath.isValid() || !EnableDumpUses) + return; + + uses.clear(); + pathAndBase.collectUses(uses, /*collectContainingUses*/ false); + llvm::outs() << "Exact Uses {\n"; + for (auto *useOperand : uses) { + llvm::outs() << *useOperand->getUser() << " "; + auto usePathAndBase = AccessPathWithBase::compute(useOperand->get()); + usePathAndBase.accessPath.printPath(llvm::outs()); + assert(pathAndBase.accessPath.contains(usePathAndBase.accessPath) + && "access path does not contain use access path"); + } + llvm::outs() << "}\n"; + uses.clear(); + pathAndBase.collectUses(uses, /*collectContainingUses*/ true); + llvm::outs() << "Overlapping Uses {\n"; + for (auto *useOperand : uses) { + llvm::outs() << *useOperand->getUser() << " "; + auto usePathAndBase = AccessPathWithBase::compute(useOperand->get()); + usePathAndBase.accessPath.printPath(llvm::outs()); + assert(pathAndBase.accessPath.mayOverlap(usePathAndBase.accessPath) + && "access path does not contain use access path"); + } + llvm::outs() << "}\n"; + } void run() override { for (auto &fn : *getModule()) { @@ -45,8 +73,12 @@ class AccessedStorageDumper : public SILModuleTransform { } for (auto &bb : fn) { for (auto &inst : bb) { - if (inst.mayReadOrWriteMemory()) - dumpAccessedStorage(&inst); + if (inst.mayReadOrWriteMemory()) { + llvm::outs() << "###For MemOp: " << inst; + visitAccessedAddress(&inst, [this](Operand *operand) { + dumpAccessedStorage(operand); + }); + } } } } From b2d1ac16315b2ecda356d0b873b29744eca675ce Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Thu, 27 Aug 2020 17:40:45 -0700 Subject: [PATCH 09/20] Add AccessPathVerification pass and run it in the pipeline. --- include/swift/SIL/MemAccessUtils.h | 2 + .../swift/SILOptimizer/PassManager/Passes.def | 2 + lib/SIL/Utils/MemAccessUtils.cpp | 2 +- lib/SILOptimizer/PassManager/PassPipeline.cpp | 10 ++ .../UtilityPasses/AccessPathVerification.cpp | 135 ++++++++++++++++++ lib/SILOptimizer/UtilityPasses/CMakeLists.txt | 1 + 6 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 lib/SILOptimizer/UtilityPasses/AccessPathVerification.cpp diff --git a/include/swift/SIL/MemAccessUtils.h b/include/swift/SIL/MemAccessUtils.h index a65effe525448..e6afa239c528e 100644 --- a/include/swift/SIL/MemAccessUtils.h +++ b/include/swift/SIL/MemAccessUtils.h @@ -844,6 +844,8 @@ class AccessPath { int getOffset() const { return offset; } + bool hasUnknownOffset() const { return offset == UnknownOffset; } + /// Return true if this path contains \p subPath. bool contains(AccessPath subPath) const; diff --git a/include/swift/SILOptimizer/PassManager/Passes.def b/include/swift/SILOptimizer/PassManager/Passes.def index 4c640d63981c9..79cf3644b3f8b 100644 --- a/include/swift/SILOptimizer/PassManager/Passes.def +++ b/include/swift/SILOptimizer/PassManager/Passes.def @@ -70,6 +70,8 @@ PASS(AccessSummaryDumper, "access-summary-dump", "Dump Address Parameter Access Summary") PASS(AccessedStorageAnalysisDumper, "accessed-storage-analysis-dump", "Dump Accessed Storage Analysis Summaries") +PASS(AccessPathVerification, "access-path-verification", + "Verify Access Paths (and Accessed Storage)") PASS(AccessedStorageDumper, "accessed-storage-dump", "Dump Accessed Storage Information") PASS(AccessMarkerElimination, "access-marker-elim", diff --git a/lib/SIL/Utils/MemAccessUtils.cpp b/lib/SIL/Utils/MemAccessUtils.cpp index c34ae23d3fc93..05470396d48ac 100644 --- a/lib/SIL/Utils/MemAccessUtils.cpp +++ b/lib/SIL/Utils/MemAccessUtils.cpp @@ -1274,7 +1274,7 @@ void AccessPath::Index::print(raw_ostream &os) const { os << '#' << getSubObjectIndex(); else { os << '@'; - if (getOffset() == AccessPath::UnknownOffset) + if (isUnknownOffset()) os << "Unknown"; else os << getOffset(); diff --git a/lib/SILOptimizer/PassManager/PassPipeline.cpp b/lib/SILOptimizer/PassManager/PassPipeline.cpp index 57038b5b5ac67..9ebe5b37eadf9 100644 --- a/lib/SILOptimizer/PassManager/PassPipeline.cpp +++ b/lib/SILOptimizer/PassManager/PassPipeline.cpp @@ -423,6 +423,11 @@ static void addPerfDebugSerializationPipeline(SILPassPipelinePlan &P) { static void addPrepareOptimizationsPipeline(SILPassPipelinePlan &P) { P.startPipeline("PrepareOptimizationPasses"); + // Verify AccessedStorage once in OSSA before optimizing. +#ifndef NDEBUG + P.addAccessPathVerification(); +#endif + P.addForEachLoopUnroll(); P.addMandatoryCombine(); P.addAccessMarkerElimination(); @@ -670,6 +675,11 @@ static void addLastChanceOptPassPipeline(SILPassPipelinePlan &P) { // A loop might have only one dynamic access now, i.e. hoistable P.addLICM(); + // Verify AccessedStorage once again after optimizing and lowering OSSA. +#ifndef NDEBUG + P.addAccessPathVerification(); +#endif + // Only has an effect if the -assume-single-thread option is specified. P.addAssumeSingleThreaded(); diff --git a/lib/SILOptimizer/UtilityPasses/AccessPathVerification.cpp b/lib/SILOptimizer/UtilityPasses/AccessPathVerification.cpp new file mode 100644 index 0000000000000..261115d957ab7 --- /dev/null +++ b/lib/SILOptimizer/UtilityPasses/AccessPathVerification.cpp @@ -0,0 +1,135 @@ +//===--- AccessPathVerification.cpp - verify access paths and storage -----===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +/// +/// Verify AccessPath computation. For the address of every memory operation in +/// the module, compute the access path, compute all the users of that path, +/// then verify that all users have the same access path. +/// +/// This is potentially expensive, so there is a fast mode that limits the +/// number of uses visited per path. +/// +/// During full verification, also check that all addresses that share an +/// AccessPath are covered when computed the use list of that AccessPath. This +/// is important because optimizations may assume that the use list is +/// complete. +/// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "access-path-verification" +#include "swift/SIL/MemAccessUtils.h" +#include "swift/SIL/PrettyStackTrace.h" +#include "swift/SIL/SILFunction.h" +#include "swift/SIL/SILInstruction.h" +#include "swift/SIL/SILValue.h" +#include "swift/SILOptimizer/PassManager/Passes.h" +#include "swift/SILOptimizer/PassManager/Transforms.h" +#include "llvm/Support/Debug.h" + +using namespace swift; + +namespace { + +/// Verify access path and uses of each access. +class AccessPathVerification : public SILModuleTransform { + llvm::DenseMap useToPathMap; + + // Transient uses + llvm::SmallVector uses; + +public: + void verifyAccessPath(Operand *operand) { + auto pathAndBase = AccessPathWithBase::compute(operand->get()); + auto accessPath = pathAndBase.accessPath; + if (!accessPath.isValid()) + return; + + auto iterAndInserted = useToPathMap.try_emplace(operand, accessPath); + // If this use was already computed from a previously visited path, make + // sure the path we just computed matches. + if (!iterAndInserted.second) { + auto collectedFromPath = iterAndInserted.first->second; + if (collectedFromPath != accessPath) { + llvm::errs() << "Address use: " << *operand->getUser() + << " collected from path\n "; + collectedFromPath.dump(); + llvm::errs() << " has different path\n "; + accessPath.dump(); + operand->getUser()->getFunction()->dump(); + assert(false && "computed path does not match collected path"); + } + return; + } + // This is a new path, so map all its uses. + assert(uses.empty()); + pathAndBase.collectUses(uses, /*collectContainingUses*/ false); + bool foundOperandUse = false; + for (Operand *use : uses) { + if (use == operand) { + foundOperandUse = true; + continue; + } + // (live) subobject projections within an access will be mapped later as a + // separate path. + switch (use->getUser()->getKind()) { + default: + break; + case SILInstructionKind::StructElementAddrInst: + case SILInstructionKind::TupleElementAddrInst: + case SILInstructionKind::IndexAddrInst: + continue; + } + auto iterAndInserted = useToPathMap.try_emplace(use, accessPath); + if (!iterAndInserted.second) { + llvm::errs() << "Address use: " << *operand->getUser() + << " with path...\n"; + pathAndBase.dump(); + llvm::errs() << " was not collected for: " << *use->getUser(); + llvm::errs() << " with path...\n"; + auto computedPath = iterAndInserted.first->second; + computedPath.dump(); + use->getUser()->getFunction()->dump(); + assert(false && "missing collected use"); + } + } + if (!foundOperandUse && !accessPath.hasUnknownOffset()) { + llvm::errs() << "Address use: " << *operand->getUser() + << " is not a use of path\n "; + accessPath.dump(); + assert(false && "not a user of its own computed path "); + } + uses.clear(); + } + + void run() override { + for (auto &fn : *getModule()) { + if (fn.empty()) + continue; + + PrettyStackTraceSILFunction functionDumper("...", &fn); + for (auto &bb : fn) { + for (auto &inst : bb) { + if (inst.mayReadOrWriteMemory()) + visitAccessedAddress(&inst, [this](Operand *operand) { + return verifyAccessPath(operand); + }); + } + } + useToPathMap.clear(); + } + } +}; + +} // end anonymous namespace + +SILTransform *swift::createAccessPathVerification() { + return new AccessPathVerification(); +} diff --git a/lib/SILOptimizer/UtilityPasses/CMakeLists.txt b/lib/SILOptimizer/UtilityPasses/CMakeLists.txt index 871ae9f29d3ec..0fbf885ed8e7b 100644 --- a/lib/SILOptimizer/UtilityPasses/CMakeLists.txt +++ b/lib/SILOptimizer/UtilityPasses/CMakeLists.txt @@ -1,6 +1,7 @@ target_sources(swiftSILOptimizer PRIVATE AADumper.cpp AccessSummaryDumper.cpp + AccessPathVerification.cpp AccessedStorageAnalysisDumper.cpp AccessedStorageDumper.cpp BasicCalleePrinter.cpp From 800c061d00d57f0afeb330e5585312a3d73971c2 Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Thu, 27 Aug 2020 17:47:41 -0700 Subject: [PATCH 10/20] accessed_storage.sil tests --- test/SILOptimizer/accessed_storage.sil | 318 ++++++++++++++++++++++++- 1 file changed, 316 insertions(+), 2 deletions(-) diff --git a/test/SILOptimizer/accessed_storage.sil b/test/SILOptimizer/accessed_storage.sil index 37e26e17b348b..26c9775556a1a 100644 --- a/test/SILOptimizer/accessed_storage.sil +++ b/test/SILOptimizer/accessed_storage.sil @@ -1,4 +1,7 @@ // RUN: %target-sil-opt %s -accessed-storage-dump -enable-sil-verify-all -o /dev/null | %FileCheck %s +// RUN: %target-sil-opt %s -access-path-verification -o /dev/null + +// REQUIRES: PTRSIZE=64 sil_stage canonical @@ -14,6 +17,9 @@ struct MyStruct { // CHECK-LABEL: @testStructPhiCommon // CHECK: store // CHECK: Argument %0 = argument of bb0 : $*MyStruct +// CHECK: Base: %0 = argument of bb0 : $*MyStruct +// CHECK: Storage: Argument %0 = argument of bb0 : $*MyStruct +// CHECK: Path: (#0) sil hidden @testStructPhiCommon : $@convention(thin) (@inout MyStruct) -> () { bb0(%0 : $*MyStruct): %2 = struct_element_addr %0 : $*MyStruct, #MyStruct.i @@ -42,6 +48,7 @@ bb3(%6 : $Builtin.RawPointer) : // CHECK-LABEL: @testStructPhiDivergent // CHECK: store // CHECK: INVALID +// CHECK: INVALID sil hidden @testStructPhiDivergent : $@convention(thin) (@inout MyStruct) -> () { bb0(%0 : $*MyStruct): cond_br undef, bb1, bb2 @@ -75,6 +82,7 @@ bb3(%6 : $Builtin.RawPointer) : // CHECK-LABEL: @testStructPhiChained // CHECK: store // CHECK: INVALID +// CHECK: INVALID sil hidden @testStructPhiChained : $@convention(thin) (@inout MyStruct, @inout Int64) -> () { bb0(%0 : $*MyStruct, %1 : $*Int64): cond_br undef, bb1, bb5 @@ -123,9 +131,15 @@ struct MyArray { // CHECK-LABEL: @arrayValue // CHECK: load [trivial] %{{.*}} : $*Builtin.Int64 -// CHECK: Tail %{{.*}} = unchecked_ref_cast [[REF:%[0-9]+]] : $Builtin.BridgeObject to $__ContiguousArrayStorageBase +// CHECK: Tail %{{.*}} = unchecked_ref_cast %{{.*}} : $Builtin.BridgeObject to $__ContiguousArrayStorageBase +// CHECK: Base: %{{.*}} = ref_tail_addr [immutable] %{{.*}} : $__ContiguousArrayStorageBase, $Int64 +// CHECK: Storage: Tail %{{.*}} = unchecked_ref_cast %{{.*}} : $Builtin.BridgeObject to $__ContiguousArrayStorageBase +// CHECK: Path: (@3,#0) // CHECK: load [trivial] %{{.*}} : $*Builtin.Int64 -// CHECK: Tail %{{.*}} = unchecked_ref_cast [[REF:%[0-9]+]] : $Builtin.BridgeObject to $__ContiguousArrayStorageBase +// CHECK: Tail %{{.*}} = unchecked_ref_cast %{{.*}} : $Builtin.BridgeObject to $__ContiguousArrayStorageBase +// CHECK: Base: %{{.*}} = ref_tail_addr [immutable] %{{.*}} : $__ContiguousArrayStorageBase, $Int64 +// CHECK: Storage: Tail %{{.*}} = unchecked_ref_cast %{{.*}} : $Builtin.BridgeObject to $__ContiguousArrayStorageBase +// CHECK: Path: (@4,#0) sil [ossa] @arrayValue : $@convention(thin) (@guaranteed MyArray) -> Int64 { bb0(%0 : @guaranteed $MyArray): %1 = integer_literal $Builtin.Word, 3 @@ -149,3 +163,303 @@ bb0(%0 : @guaranteed $MyArray): %19 = struct $Int64 (%16 : $Builtin.Int64) return %19 : $Int64 } + +// CHECK-LABEL: @staticIndexAddrChain +// CHECK: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Base: %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Storage: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Path: (@1,#0) +// CHECK: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK: Argument %{{.*}} = argument of bb0 : $Builtin.RawPointer +// CHECK: Base: %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Storage: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Path: (@2,#0) +sil [ossa] @staticIndexAddrChain : $@convention(thin) (Builtin.RawPointer) -> () { +bb0(%0 : $Builtin.RawPointer): + %1 = pointer_to_address %0 : $Builtin.RawPointer to $*MyStruct + %2 = integer_literal $Builtin.Word, 1 + %3 = index_addr %1 : $*MyStruct, %2 : $Builtin.Word + %4 = struct_element_addr %3 : $*MyStruct, #MyStruct.i + %5 = load [trivial] %4 : $*Int64 + %6 = index_addr %3 : $*MyStruct, %2 : $Builtin.Word + %7 = struct_element_addr %6 : $*MyStruct, #MyStruct.i + %8 = load [trivial] %7 : $*Int64 + %99 = tuple () + return %99 : $() +} + +// CHECK-LABEL: @staticIndexAddrSubobject +// CHECK: ###For MemOp: %5 = load [trivial] %4 : $*Int64 +// CHECK: Argument %0 = argument of bb0 : $Builtin.RawPointer // user: %1 +// CHECK: INVALID +sil [ossa] @staticIndexAddrSubobject : $@convention(thin) (Builtin.RawPointer) -> () { +bb0(%0 : $Builtin.RawPointer): + %1 = pointer_to_address %0 : $Builtin.RawPointer to $*MyStruct + %2 = struct_element_addr %1 : $*MyStruct, #MyStruct.i + %3 = integer_literal $Builtin.Word, 1 + %4 = index_addr %2 : $*Int64, %3 : $Builtin.Word + %5 = load [trivial] %4 : $*Int64 + %99 = tuple () + return %99 : $() +} + +// CHECK-LABEL: @dynamicIndexAddrChain +// CHECK: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Base: %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Storage: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Path: (@Unknown,#0) +// CHECK: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Base: %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Storage: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Path: (@Unknown,#0) +sil [ossa] @dynamicIndexAddrChain : $@convention(thin) (Builtin.RawPointer, Builtin.Word) -> () { +bb0(%0 : $Builtin.RawPointer, %1 : $Builtin.Word): + %2 = pointer_to_address %0 : $Builtin.RawPointer to $*MyStruct + %3 = index_addr %2 : $*MyStruct, %1 : $Builtin.Word + %4 = struct_element_addr %3 : $*MyStruct, #MyStruct.i + %5 = load [trivial] %4 : $*Int64 + %6 = integer_literal $Builtin.Word, 1 + %7 = index_addr %3 : $*MyStruct, %6 : $Builtin.Word + %8 = struct_element_addr %7 : $*MyStruct, #MyStruct.i + %9 = load [trivial] %8 : $*Int64 + %99 = tuple () + return %99 : $() +} + +// CHECK-LABEL: @staticIndexAddrCancel +// CHECK: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Base: %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Storage: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Path: (@1,#0) +// CHECK: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Base: %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Storage: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Path: (@2,#0) +sil [ossa] @staticIndexAddrCancel : $@convention(thin) (Builtin.RawPointer) -> () { +bb0(%0 : $Builtin.RawPointer): + %1 = pointer_to_address %0 : $Builtin.RawPointer to $*MyStruct + %2 = integer_literal $Builtin.Word, 1 + %3 = index_addr %1 : $*MyStruct, %2 : $Builtin.Word + %4 = struct_element_addr %3 : $*MyStruct, #MyStruct.i + %5 = load [trivial] %4 : $*Int64 + %6 = integer_literal $Builtin.Word, -1 + %7 = index_addr %3 : $*MyStruct, %2 : $Builtin.Word + %8 = struct_element_addr %7 : $*MyStruct, #MyStruct.i + %9 = load [trivial] %8 : $*Int64 + %99 = tuple () + return %99 : $() +} + +class A { + var prop0: Int64 +} +class B : A { + var prop1: Int64 +} + +// CHECK-LABEL: @testNonUniquePropertyIndex +// CHECK: store %0 to %{{.*}} : $*Int64 +// CHECK: Class %{{.*}} = alloc_ref $B +// CHECK: Field: var prop1: Int64 Index: 1 +// CHECK: Base: %{{.*}} = ref_element_addr %{{.*}} : $B, #B.prop1 +// CHECK: Storage: Class %{{.*}} = alloc_ref $B +// CHECK: Field: var prop1: Int64 Index: 1 +// CHECK: Path: () +// CHECK: store %0 to %{{.*}} : $*Int64 +// CHECK: Class %{{.*}} = upcast %{{.*}} : $B to $A +// CHECK: Field: var prop0: Int64 Index: 0 +// CHECK: Base: %{{.*}} = ref_element_addr %{{.*}} : $A, #A.prop0 +// CHECK: Storage: Class %{{.*}} = upcast %{{.*}} : $B to $A +// CHECK: Field: var prop0: Int64 Index: 0 +// CHECK: Path: () +sil @testNonUniquePropertyIndex : $@convention(thin) (Int64) -> () { +bb0(%0 : $Int64): + %1 = alloc_ref $B + %2 = ref_element_addr %1 : $B, #B.prop1 + store %0 to %2 : $*Int64 + %4 = upcast %1 : $B to $A + %5 = ref_element_addr %4 : $A, #A.prop0 + store %0 to %5 : $*Int64 + %99 = tuple () + return %99 : $() +} + +// CHECK-LABEL: @testRefTailAndStruct0 +// CHECK: %{{.*}} = load %{{.*}} : $*Builtin.Int64 +// CHECK: Class %{{.*}} = unchecked_ref_cast %{{.*}} : $Builtin.BridgeObject to $__ContiguousArrayStorageBase +// CHECK: Field: @usableFromInline final var countAndCapacity: _ArrayBody Index: 0 +// CHECK: Base: %{{.*}} = ref_element_addr [immutable] %{{.*}} : $__ContiguousArrayStorageBase, #__ContiguousArrayStorageBase.countAndCapacity +// CHECK: Storage: Class %{{.*}} = unchecked_ref_cast %{{.*}} : $Builtin.BridgeObject to $__ContiguousArrayStorageBase +// CHECK: Field: @usableFromInline final var countAndCapacity: _ArrayBody Index: 0 +// CHECK: Path: (#0,#0,#0) +// CHECK: %{{.*}} = load %{{.*}} : $*_StringGuts +// CHECK: Tail %{{.*}} = unchecked_ref_cast %{{.*}} : $Builtin.BridgeObject to $__ContiguousArrayStorageBase +// CHECK: Base: %{{.*}} = ref_tail_addr [immutable] %{{.*}} : $__ContiguousArrayStorageBase, $String +// CHECK: Storage: Tail %{{.*}} = unchecked_ref_cast %{{.*}} : $Builtin.BridgeObject to $__ContiguousArrayStorageBase +// CHECK: Path: (#0) +// CHECK: %{{.*}} = load %{{.*}} : $*String +// CHECK: Tail %{{.*}} = unchecked_ref_cast %{{.*}} : $Builtin.BridgeObject to $__ContiguousArrayStorageBase +// CHECK: Base: %{{.*}} = ref_tail_addr [immutable] %{{.*}} : $__ContiguousArrayStorageBase, $String +// CHECK: Storage: Tail %{{.*}} = unchecked_ref_cast %{{.*}} : $Builtin.BridgeObject to $__ContiguousArrayStorageBase +// CHECK: Path: () +sil hidden [noinline] @testRefTailAndStruct0 : $@convention(thin) (@owned MyArray) -> () { +bb0(%0 : $MyArray): + %1 = struct_extract %0 : $MyArray, #MyArray._buffer + %2 = struct_extract %1 : $_MyArrayBuffer, #_MyArrayBuffer._storage + %3 = struct_extract %2 : $_MyBridgeStorage, #_MyBridgeStorage.rawValue + %4 = unchecked_ref_cast %3 : $Builtin.BridgeObject to $__ContiguousArrayStorageBase + %5 = ref_element_addr [immutable] %4 : $__ContiguousArrayStorageBase, #__ContiguousArrayStorageBase.countAndCapacity + %6 = struct_element_addr %5 : $*_ArrayBody, #_ArrayBody._storage + %7 = struct_element_addr %6 : $*_SwiftArrayBodyStorage, #_SwiftArrayBodyStorage.count + %8 = struct_element_addr %7 : $*Int, #Int._value + %9 = load %8 : $*Builtin.Int64 + %10 = ref_tail_addr [immutable] %4 : $__ContiguousArrayStorageBase, $String + %11 = struct_element_addr %10 : $*String, #String._guts + %12 = load %11 : $*_StringGuts + %13 = load %10 : $*String + %14 = tuple () + return %14 : $() +} + +// CHECK-LABEL: @testPointerDynamicIndex +// CHECK: ###For MemOp: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct +// CHECK: Base: %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Storage: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Path: (@Unknown) +sil [serialized] [ossa] @testPointerDynamicIndex : $@convention(thin) (Builtin.RawPointer, Builtin.Word) -> () { +bb0(%0 : $Builtin.RawPointer, %1 : $Builtin.Word): + %2 = pointer_to_address %0 : $Builtin.RawPointer to [strict] $*MyStruct + %3 = index_addr %2 : $*MyStruct, %1 : $Builtin.Word + %4 = address_to_pointer %3 : $*MyStruct to $Builtin.RawPointer + %5 = pointer_to_address %4 : $Builtin.RawPointer to [strict] $*MyStruct + %6 = load [trivial] %5 : $*MyStruct + %7 = tuple () + return %7 : $() +} + +// CHECK-LABEL: @testAddressToPointer +// CHECK: ###For MemOp: %3 = load [trivial] %{{.*}} : $*MyStruct +// CHECK: Base: %0 = argument of bb0 : $*MyStruct +// CHECK: Storage: Argument %0 = argument of bb0 : $*MyStruct +// CHECK: Path: () +// CHECK: ###For MemOp: %5 = load [trivial] %4 : $*Int64 +// CHECK: Base: %0 = argument of bb0 : $*MyStruct +// CHECK: Storage: Argument %0 = argument of bb0 : $*MyStruct +// CHECK: Path: (#0) +sil [serialized] [ossa] @testAddressToPointer : $@convention(thin) (@in MyStruct) -> () { +bb18(%0 : $*MyStruct): + %1 = address_to_pointer %0 : $*MyStruct to $Builtin.RawPointer + %2 = pointer_to_address %1 : $Builtin.RawPointer to [strict] $*MyStruct + %3 = load [trivial] %2 : $*MyStruct + %4 = struct_element_addr %0 : $*MyStruct, #MyStruct.i + %5 = load [trivial] %4 : $*Int64 + %6 = tuple () + return %6 : $() +} + +// CHECK-LABEL: @testIndexLoop +// CHECK: ###For MemOp: %3 = load %2 : $*AnyObject +// CHECK: Base: %1 = struct_extract %0 : $UnsafeMutablePointer, #UnsafeMutablePointer._rawValue +// CHECK: Storage: Unidentified %1 = struct_extract %0 : $UnsafeMutablePointer, #UnsafeMutablePointer._rawValue +// CHECK: Path: () +// CHECK: ###For MemOp: %7 = load %6 : $*AnyObject +// CHECK: Base: %1 = struct_extract %0 : $UnsafeMutablePointer, #UnsafeMutablePointer._rawValue +// CHECK: Storage: Unidentified %1 = struct_extract %0 : $UnsafeMutablePointer, #UnsafeMutablePointer._rawValue +// CHECK: Path: (@Unknown) +// CHECK: ###For MemOp: store %7 to %6 : $*AnyObject // id: %8 +// CHECK: Base: %1 = struct_extract %0 : $UnsafeMutablePointer, #UnsafeMutablePointer._rawValue +// CHECK: Storage: Unidentified %1 = struct_extract %0 : $UnsafeMutablePointer, #UnsafeMutablePointer._rawValue +// CHECK: Path: (@Unknown) +sil shared @testIndexLoop : $@convention(thin) (UnsafeMutablePointer) -> UnsafeMutablePointer { +bb0(%0 : $UnsafeMutablePointer): + %1 = struct_extract %0 : $UnsafeMutablePointer, #UnsafeMutablePointer._rawValue + %2 = pointer_to_address %1 : $Builtin.RawPointer to [strict] $*AnyObject + %3 = load %2 : $*AnyObject + br bb1(%1 : $Builtin.RawPointer) + +bb1(%5 : $Builtin.RawPointer): + %6 = pointer_to_address %5 : $Builtin.RawPointer to [strict] $*AnyObject + %7 = load %6 : $*AnyObject + store %7 to %6 : $*AnyObject + %9 = integer_literal $Builtin.Word, 1 + %10 = index_addr %6 : $*AnyObject, %9 : $Builtin.Word + %11 = address_to_pointer %10 : $*AnyObject to $Builtin.RawPointer + cond_br undef, bb2, bb3(%11 : $Builtin.RawPointer) + +bb2: + br bb1(%11 : $Builtin.RawPointer) + +bb3(%14 : $Builtin.RawPointer): + %15 = struct $UnsafeMutablePointer (%14 : $Builtin.RawPointer) + return %15 : $UnsafeMutablePointer +} + +// Handle index_addr boundary conditions +// Most will end up as "unknown" offset, but we shouldn't crash. +// CHECK-LABEL: @indexAddrBoundaries +// CHECK: ###For MemOp: %4 = load %3 : $*Int +// CHECK: Path: (@Unknown) +// CHECK: ###For MemOp: %7 = load %6 : $*Int +// CHECK: Path: (@Unknown) +// CHECK: ###For MemOp: %10 = load %9 : $*Int +// CHECK: Path: (@Unknown) +// CHECK: ###For MemOp: %13 = load %12 : $*Int +// CHECK: Path: (@Unknown) +// CHECK: ###For MemOp: %16 = load %15 : $*Int +// CHECK: Path: (@1073741823) +// CHECK: ###For MemOp: %19 = load %18 : $*Int +// CHECK: Path: (@Unknown) +// CHECK: ###For MemOp: %22 = load %21 : $*Int +// CHECK: Path: (@Unknown) +// CHECK: ###For MemOp: %25 = load %24 : $*Int +// CHECK: Path: (@-1073741823) +// CHECK: ###For MemOp: %28 = load %27 : $*Int +// CHECK: Path: (@-1) +sil @indexAddrBoundaries : $@convention(thin) (Builtin.RawPointer) -> () { +bb0(%0 : $Builtin.RawPointer): + %1 = pointer_to_address %0 : $Builtin.RawPointer to [strict] $*Int + // INT_MIN (IndexTrie root) + %2 = integer_literal $Builtin.Word, 2147483648 // '0x80000000' + %3 = index_addr %1 : $*Int, %2 : $Builtin.Word + %4 = load %3 : $*Int + // INT_MIN (IndexTrie root) + %5 = integer_literal $Builtin.Word, -2147483648 // '0x80000000' + %6 = index_addr %1 : $*Int, %5 : $Builtin.Word + %7 = load %6 : $*Int + // INT_MAX (TailIndex) + %8 = integer_literal $Builtin.Word, 2147483647 // '0x7fffffff' + %9 = index_addr %1 : $*Int, %8 : $Builtin.Word + %10 = load %9 : $*Int + // Largest unsigned offset + 1 + %11 = integer_literal $Builtin.Word, 1073741824 // '0x40000000' + %12 = index_addr %1 : $*Int, %11 : $Builtin.Word + %13 = load %12 : $*Int + // Largest unsigned offset + %14 = integer_literal $Builtin.Word, 1073741823 // '0x3fffffff' + %15 = index_addr %1 : $*Int, %14 : $Builtin.Word + %16 = load %15 : $*Int + // Smallest signed offset - 1 + %17 = integer_literal $Builtin.Word, -1073741825 // '0xbfffffff' + %18 = index_addr %1 : $*Int, %17 : $Builtin.Word + %19 = load %18 : $*Int + // Smallest signed offset (Unknown offset) + %20 = integer_literal $Builtin.Word, -1073741824 // '0xc0000000' + %21 = index_addr %1 : $*Int, %20 : $Builtin.Word + %22 = load %21 : $*Int + // Smallest signed offset + 1 (concrete offset) + %23 = integer_literal $Builtin.Word, -1073741823 // '0xc0000001' + %24 = index_addr %1 : $*Int, %23 : $Builtin.Word + %25 = load %24 : $*Int + // Largest Negative + %26 = integer_literal $Builtin.Word, -1 // '0xffffffff' + %27 = index_addr %1 : $*Int, %26 : $Builtin.Word + %28 = load %27 : $*Int + // + %99 = tuple () + return %99 : $() +} From b8f3f9c84666301bb33ec68d7dea4305550b6d55 Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Thu, 27 Aug 2020 17:48:06 -0700 Subject: [PATCH 11/20] accesspath_uses.sil tests --- test/SILOptimizer/accesspath_uses.sil | 532 ++++++++++++++++++++++++++ 1 file changed, 532 insertions(+) create mode 100644 test/SILOptimizer/accesspath_uses.sil diff --git a/test/SILOptimizer/accesspath_uses.sil b/test/SILOptimizer/accesspath_uses.sil new file mode 100644 index 0000000000000..883f68314e1d1 --- /dev/null +++ b/test/SILOptimizer/accesspath_uses.sil @@ -0,0 +1,532 @@ +// RUN: %target-sil-opt %s -accessed-storage-dump -enable-accessed-storage-dump-uses -enable-sil-verify-all -o /dev/null | %FileCheck %s +// RUN: %target-sil-opt %s -access-path-verification -o /dev/null + +// REQUIRES: PTRSIZE=64 + +sil_stage canonical + +import Builtin +import Swift +import SwiftShims + +struct MyStruct { + @_hasStorage @_hasInitialValue var i: Int64 { get set } + @_hasStorage @_hasInitialValue var j: Int64 { get set } +} + +// unknown offset contains subobject indices +// CHECK-LABEL: @testDynamicIndexWithSubObject +// CHECK: ###For MemOp: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK: Path: (#0) +// CHECK: Exact Uses { +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK-NEXT: Path: (#0) +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK-NEXT: Path: (#0) +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: } +// CHECK: ###For MemOp: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct +// CHECK: Path: (@Unknown) +// CHECK-NEXT: Exact Uses { +// CHECK-NEXT: } +// CHECK-NEXT: Overlapping Uses { +// CHECK-NEXT: %{{.*}} = struct_element_addr %{{.*}} : $*MyStruct, #MyStruct.i +// CHECK-NEXT: Path: () +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: } +sil [serialized] [ossa] @testDynamicIndexWithSubObject : $@convention(thin) (Builtin.RawPointer, Builtin.Word) -> () { +bb0(%0 : $Builtin.RawPointer, %1 : $Builtin.Word): + %2 = pointer_to_address %0 : $Builtin.RawPointer to [strict] $*MyStruct + %3 = struct_element_addr %2 : $*MyStruct, #MyStruct.i + %4 = load [trivial] %3 : $*Int64 + %5 = index_addr %2 : $*MyStruct, %1 : $Builtin.Word + %6 = load [trivial] %5 : $*MyStruct + %7 = tuple () + return %7 : $() +} + +// An unknown offset contains known offsets. +// CHECK-LABEL: @testDynamicIndexWithStaticIndex +// CHECK: ###For MemOp: %5 = load [trivial] %4 : $*MyStruct +// CHECK: Path: (@1) +// CHECK: Exact Uses { +// CHECK-NEXT: %5 = load [trivial] %4 : $*MyStruct +// CHECK-NEXT: Path: (@1) +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: %5 = load [trivial] %4 : $*MyStruct +// CHECK-NEXT: Path: (@1) +// CHECK-NEXT: %7 = load [trivial] %6 : $*MyStruct +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: } +// CHECK: ###For MemOp: %7 = load [trivial] %6 : $*MyStruct +// CHECK: Path: (@Unknown) +// CHECK: Exact Uses { +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: %5 = load [trivial] %4 : $*MyStruct +// CHECK-NEXT: Path: (@1) +// CHECK-NEXT: %7 = load [trivial] %6 : $*MyStruct +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: } +sil [serialized] [ossa] @testDynamicIndexWithStaticIndex : $@convention(thin) (Builtin.RawPointer, Builtin.Word) -> () { +bb0(%0 : $Builtin.RawPointer, %1 : $Builtin.Word): + %2 = pointer_to_address %0 : $Builtin.RawPointer to [strict] $*MyStruct + %3 = integer_literal $Builtin.Word, 1 + %4 = index_addr %2 : $*MyStruct, %3 : $Builtin.Word + %5 = load [trivial] %4 : $*MyStruct + %6 = index_addr %2 : $*MyStruct, %1 : $Builtin.Word + %7 = load [trivial] %6 : $*MyStruct + %8 = tuple () + return %8 : $() +} + +// Even though the static offset does not match the first load, the +// dynamic offset should cancel it. +// CHECK-LABEL: @testChainedStaticDynamicIndex +// CHECK: ###For MemOp: %3 = load [trivial] %2 : $*MyStruct +// CHECK: Exact Uses { +// CHECK-NEXT: %3 = load [trivial] %2 : $*MyStruct +// CHECK-NEXT: Path: () +// CHECK-NEXT: } +// CHECK-NEXT: Overlapping Uses { +// CHECK-NEXT: %3 = load [trivial] %2 : $*MyStruct +// CHECK-NEXT: Path: () +// CHECK-NEXT: %8 = load [trivial] %7 : $*MyStruct +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: } +// CHECK: ###For MemOp: %6 = load [trivial] %5 : $*MyStruct +// CHECK: Storage: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Path: (@1) +// CHECK: Exact Uses { +// CHECK-NEXT: %6 = load [trivial] %5 : $*MyStruct +// CHECK-NEXT: Path: (@1) +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: %6 = load [trivial] %5 : $*MyStruct +// CHECK-NEXT: Path: (@1) +// CHECK-NEXT: %8 = load [trivial] %7 : $*MyStruct +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: } +// CHECK: ###For MemOp: %8 = load [trivial] %7 : $*MyStruct +// CHECK: Storage: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Path: (@Unknown) +// CHECK: Exact Uses { +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: %3 = load [trivial] %2 : $*MyStruct +// CHECK-NEXT: Path: () +// CHECK-NEXT: %6 = load [trivial] %5 : $*MyStruct +// CHECK-NEXT: Path: (@1) +// CHECK-NEXT: %8 = load [trivial] %7 : $*MyStruct +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: } +sil [serialized] [ossa] @testChainedStaticDynamicIndex : $@convention(thin) (Builtin.RawPointer, Builtin.Word) -> () { +bb0(%0 : $Builtin.RawPointer, %1 : $Builtin.Word): + %2 = pointer_to_address %0 : $Builtin.RawPointer to [strict] $*MyStruct + %3 = load [trivial] %2 : $*MyStruct + %4 = integer_literal $Builtin.Word, 1 + %5 = index_addr %2 : $*MyStruct, %4 : $Builtin.Word + %6 = load [trivial] %5 : $*MyStruct + %7 = index_addr %5 : $*MyStruct, %1 : $Builtin.Word + %8 = load [trivial] %7 : $*MyStruct + %9 = tuple () + return %9 : $() +} + +// Static indices cancel. +// Unknown offset overlaps with subobject projections. +// +// CHECK-LABEL: @staticIndexAddrCancel +// CHECK: ###For MemOp: %3 = load [trivial] %2 : $*Int64 +// CHECK: Storage: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Path: (#0) +// CHECK: Exact Uses { +// CHECK-NEXT: %3 = load [trivial] %2 : $*Int64 +// CHECK-NEXT: Path: (#0) +// CHECK-NEXT: %9 = load [trivial] %8 : $*Int64 +// CHECK-NEXT: Path: (#0) +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: %3 = load [trivial] %2 : $*Int64 +// CHECK-NEXT: Path: (#0) +// CHECK-NEXT: %9 = load [trivial] %8 : $*Int64 +// CHECK-NEXT: Path: (#0) +// CHECK-NEXT: } +// CHECK: ###For MemOp: %9 = load [trivial] %8 : $*Int64 +// CHECK: Storage: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Path: (#0) +// CHECK: Exact Uses { +// CHECK-NEXT: %3 = load [trivial] %2 : $*Int64 +// CHECK-NEXT: Path: (#0) +// CHECK-NEXT: %9 = load [trivial] %8 : $*Int64 +// CHECK-NEXT: Path: (#0) +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: %3 = load [trivial] %2 : $*Int64 +// CHECK-NEXT: Path: (#0) +// CHECK-NEXT: %9 = load [trivial] %8 : $*Int64 +// CHECK-NEXT: Path: (#0) +// CHECK-NEXT: } +sil [ossa] @staticIndexAddrCancel : $@convention(thin) (Builtin.RawPointer) -> () { +bb0(%0 : $Builtin.RawPointer): + %1 = pointer_to_address %0 : $Builtin.RawPointer to $*MyStruct + %2 = struct_element_addr %1 : $*MyStruct, #MyStruct.i + %3 = load [trivial] %2 : $*Int64 + %4 = integer_literal $Builtin.Word, 1 + %5 = index_addr %1 : $*MyStruct, %4 : $Builtin.Word + %6 = integer_literal $Builtin.Word, -1 + %7 = index_addr %5 : $*MyStruct, %6 : $Builtin.Word + %8 = struct_element_addr %7 : $*MyStruct, #MyStruct.i + %9 = load [trivial] %8 : $*Int64 + %99 = tuple () + return %99 : $() +} + +// Increment an indexable address by one in a loop, with subobject +// uses inside and outside the loop. +// +// CHECK-LABEL: @testIndexLoop +// CHECK: ###For MemOp: %3 = load %2 : $*AnyObject +// CHECK: Storage: Unidentified %1 = struct_extract %0 : $UnsafeMutablePointer, #UnsafeMutablePointer._rawValue +// CHECK: Path: () +// CHECK: Exact Uses { +// CHECK-NEXT: %3 = load %2 : $*AnyObject +// CHECK-NEXT: Path: () +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: %3 = load %2 : $*AnyObject +// CHECK-NEXT: Path: () +// CHECK-NEXT: %7 = load %6 : $*AnyObject +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: store %7 to %6 : $*AnyObject +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: store %7 to %10 : $*AnyObject +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: cond_br undef, bb2, bb3(%12 : $Builtin.RawPointer) +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: } +// CHECK: ###For MemOp: %7 = load %6 : $*AnyObject +// CHECK: Storage: Unidentified %1 = struct_extract %0 : $UnsafeMutablePointer, #UnsafeMutablePointer._rawValue +// CHECK: Path: (@Unknown) +// CHECK: Exact Uses { +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: %3 = load %2 : $*AnyObject +// CHECK-NEXT: Path: () +// CHECK-NEXT: %7 = load %6 : $*AnyObject +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: store %7 to %6 : $*AnyObject +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: store %7 to %10 : $*AnyObject +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: cond_br undef, bb2, bb3(%12 : $Builtin.RawPointer) +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: } +// CHECK: ###For MemOp: store %7 to %6 : $*AnyObject +// CHECK: Storage: Unidentified %1 = struct_extract %0 : $UnsafeMutablePointer, #UnsafeMutablePointer._rawValue +// CHECK: Path: (@Unknown) +// CHECK: Exact Uses { +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: %3 = load %2 : $*AnyObject +// CHECK-NEXT: Path: () +// CHECK-NEXT: %7 = load %6 : $*AnyObject +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: store %7 to %6 : $*AnyObject +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: store %7 to %10 : $*AnyObject +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: cond_br undef, bb2, bb3(%12 : $Builtin.RawPointer) +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: } +// CHECK: ###For MemOp: store %7 to %10 : $*AnyObject +// CHECK: Storage: Unidentified %1 = struct_extract %0 : $UnsafeMutablePointer, #UnsafeMutablePointer._rawValue +// CHECK: Path: (@Unknown) +// CHECK: Exact Uses { +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: %3 = load %2 : $*AnyObject +// CHECK-NEXT: Path: () +// CHECK-NEXT: %7 = load %6 : $*AnyObject +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: store %7 to %6 : $*AnyObject +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: store %7 to %10 : $*AnyObject +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: cond_br undef, bb2, bb3(%12 : $Builtin.RawPointer) +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: } +// CHECK: ###For MemOp: %17 = load %16 : $*AnyObject +// CHECK: Storage: Unidentified %1 = struct_extract %0 : $UnsafeMutablePointer, #UnsafeMutablePointer._rawValue +// CHECK: Path: (@Unknown) +// CHECK: Exact Uses { +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: %3 = load %2 : $*AnyObject +// CHECK-NEXT: Path: () +// CHECK-NEXT: %7 = load %6 : $*AnyObject +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: store %7 to %6 : $*AnyObject +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: store %7 to %10 : $*AnyObject +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: cond_br undef, bb2, bb3(%12 : $Builtin.RawPointer) +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: } +sil shared @testIndexLoop : $@convention(thin) (UnsafeMutablePointer) -> AnyObject { +bb0(%0 : $UnsafeMutablePointer): + %1 = struct_extract %0 : $UnsafeMutablePointer, #UnsafeMutablePointer._rawValue + %2 = pointer_to_address %1 : $Builtin.RawPointer to [strict] $*AnyObject + %3 = load %2 : $*AnyObject + br bb1(%1 : $Builtin.RawPointer) + +bb1(%5 : $Builtin.RawPointer): + %6 = pointer_to_address %5 : $Builtin.RawPointer to [strict] $*AnyObject + %7 = load %6 : $*AnyObject + store %7 to %6 : $*AnyObject + %9 = integer_literal $Builtin.Word, 1 + %10 = index_addr %6 : $*AnyObject, %9 : $Builtin.Word + store %7 to %10 : $*AnyObject + %12 = address_to_pointer %10 : $*AnyObject to $Builtin.RawPointer + cond_br undef, bb2, bb3(%12 : $Builtin.RawPointer) + +bb2: + br bb1(%12 : $Builtin.RawPointer) + +bb3(%15 : $Builtin.RawPointer): + %16 = pointer_to_address %15 : $Builtin.RawPointer to [strict] $*AnyObject + %17 = load %16 : $*AnyObject + return %17 : $AnyObject +} + +// CHECK-LABEL: @testEnumUses +// CHECK: ###For MemOp: copy_addr %0 to [initialization] %2 : $*IntTEnum +// CHECK: Storage: Argument %0 = argument of bb0 : $*IntTEnum +// CHECK: Path: () +// CHECK: Exact Uses { +// CHECK-NEXT: debug_value_addr %0 : $*IntTEnum, let, name "self", argno 1 +// CHECK-NEXT: Path: () +// CHECK-NEXT: copy_addr %0 to [initialization] %2 : $*IntTEnum +// CHECK-NEXT: Path: () +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: debug_value_addr %0 : $*IntTEnum, let, name "self", argno 1 +// CHECK-NEXT: Path: () +// CHECK-NEXT: copy_addr %0 to [initialization] %2 : $*IntTEnum +// CHECK-NEXT: Path: () +// CHECK-NEXT: } +// CHECK: Storage: Stack %2 = alloc_stack $IntTEnum +// CHECK: Path: () +// CHECK: Exact Uses { +// CHECK-NEXT: copy_addr %0 to [initialization] %2 : $*IntTEnum +// CHECK-NEXT: Path: () +// CHECK-NEXT: switch_enum_addr %2 : $*IntTEnum, case #IntTEnum.int!enumelt: bb1, case #IntTEnum.other!enumelt: bb2 +// CHECK-NEXT: Path: () +// CHECK-NEXT: %5 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.int!enumelt +// CHECK-NEXT: Path: () +// CHECK-NEXT: %6 = load [take] %5 : $*<Ï„_0_0> { var Int } +// CHECK-NEXT: Path: () +// CHECK-NEXT: %8 = load [trivial] %7 : $*Int +// CHECK-NEXT: Path: () +// CHECK-NEXT: dealloc_stack %2 : $*IntTEnum +// CHECK-NEXT: Path: () +// CHECK-NEXT: %13 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.other!enumelt +// CHECK-NEXT: Path: () +// CHECK-NEXT: destroy_addr %13 : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: dealloc_stack %2 : $*IntTEnum +// CHECK-NEXT: Path: () +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: copy_addr %0 to [initialization] %2 : $*IntTEnum +// CHECK-NEXT: Path: () +// CHECK-NEXT: switch_enum_addr %2 : $*IntTEnum, case #IntTEnum.int!enumelt: bb1, case #IntTEnum.other!enumelt: bb2 +// CHECK-NEXT: Path: () +// CHECK-NEXT: %5 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.int!enumelt +// CHECK-NEXT: Path: () +// CHECK-NEXT: %6 = load [take] %5 : $*<Ï„_0_0> { var Int } +// CHECK-NEXT: Path: () +// CHECK-NEXT: %8 = load [trivial] %7 : $*Int +// CHECK-NEXT: Path: () +// CHECK-NEXT: dealloc_stack %2 : $*IntTEnum +// CHECK-NEXT: Path: () +// CHECK-NEXT: %13 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.other!enumelt +// CHECK-NEXT: Path: () +// CHECK-NEXT: destroy_addr %13 : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: dealloc_stack %2 : $*IntTEnum +// CHECK-NEXT: Path: () +// CHECK-NEXT: } +// CHECK: ###For MemOp: switch_enum_addr %2 : $*IntTEnum, case #IntTEnum.int!enumelt: bb1, case #IntTEnum.other!enumelt: bb2 +// CHECK: Storage: Stack %2 = alloc_stack $IntTEnum +// CHECK: Path: () +// CHECK: Exact Uses { +// CHECK: copy_addr %0 to [initialization] %2 : $*IntTEnum +// CHECK: switch_enum_addr %2 : $*IntTEnum, case #IntTEnum.int!enumelt: bb1, case #IntTEnum.other!enumelt: bb2 +// CHECK: %5 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.int!enumelt +// CHECK: %6 = load [take] %5 : $*<Ï„_0_0> { var Int } +// CHECK: %8 = load [trivial] %7 : $*Int +// CHECK: %13 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.other!enumelt +// CHECK: } +// CHECK: Overlapping Uses { +// CHECK: copy_addr %0 to [initialization] %2 : $*IntTEnum +// CHECK: switch_enum_addr %2 : $*IntTEnum, case #IntTEnum.int!enumelt: bb1, case #IntTEnum.other!enumelt: bb2 +// CHECK: %5 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.int!enumelt +// CHECK: %6 = load [take] %5 : $*<Ï„_0_0> { var Int } +// CHECK: %8 = load [trivial] %7 : $*Int +// CHECK: %13 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.other!enumelt +// CHECK: } +// CHECK: ###For MemOp: %5 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.int!enumelt +// CHECK: Storage: Stack %2 = alloc_stack $IntTEnum +// CHECK: Path: () +// CHECK: Exact Uses { +// CHECK: copy_addr %0 to [initialization] %2 : $*IntTEnum +// CHECK: switch_enum_addr %2 : $*IntTEnum, case #IntTEnum.int!enumelt: bb1, case #IntTEnum.other!enumelt: bb2 +// CHECK: %5 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.int!enumelt +// CHECK: %6 = load [take] %5 : $*<Ï„_0_0> { var Int } +// CHECK: %8 = load [trivial] %7 : $*Int +// CHECK: %13 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.other!enumelt +// CHECK: } +// CHECK: Overlapping Uses { +// CHECK: copy_addr %0 to [initialization] %2 : $*IntTEnum +// CHECK: switch_enum_addr %2 : $*IntTEnum, case #IntTEnum.int!enumelt: bb1, case #IntTEnum.other!enumelt: bb2 +// CHECK: %5 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.int!enumelt +// CHECK: %6 = load [take] %5 : $*<Ï„_0_0> { var Int } +// CHECK: %8 = load [trivial] %7 : $*Int +// CHECK: %13 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.other!enumelt +// CHECK: } +// CHECK: ###For MemOp: %6 = load [take] %5 : $*<Ï„_0_0> { var Int } +// CHECK: Storage: Stack %2 = alloc_stack $IntTEnum +// CHECK: Path: () +// CHECK: Exact Uses { +// CHECK: copy_addr %0 to [initialization] %2 : $*IntTEnum +// CHECK: switch_enum_addr %2 : $*IntTEnum, case #IntTEnum.int!enumelt: bb1, case #IntTEnum.other!enumelt: bb2 +// CHECK: %5 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.int!enumelt +// CHECK: %6 = load [take] %5 : $*<Ï„_0_0> { var Int } +// CHECK: %8 = load [trivial] %7 : $*Int +// CHECK: %13 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.other!enumelt +// CHECK: } +// CHECK: Overlapping Uses { +// CHECK: copy_addr %0 to [initialization] %2 : $*IntTEnum +// CHECK: switch_enum_addr %2 : $*IntTEnum, case #IntTEnum.int!enumelt: bb1, case #IntTEnum.other!enumelt: bb2 +// CHECK: %5 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.int!enumelt +// CHECK: %6 = load [take] %5 : $*<Ï„_0_0> { var Int } +// CHECK: %8 = load [trivial] %7 : $*Int +// CHECK: %13 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.other!enumelt +// CHECK: } +// CHECK: ###For MemOp: %8 = load [trivial] %7 : $*Int +// CHECK: Storage: Stack %2 = alloc_stack $IntTEnum +// CHECK: Path: () +// CHECK: Exact Uses { +// CHECK: copy_addr %0 to [initialization] %2 : $*IntTEnum +// CHECK: switch_enum_addr %2 : $*IntTEnum, case #IntTEnum.int!enumelt: bb1, case #IntTEnum.other!enumelt: bb2 +// CHECK: %5 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.int!enumelt +// CHECK: %6 = load [take] %5 : $*<Ï„_0_0> { var Int } +// CHECK: %8 = load [trivial] %7 : $*Int +// CHECK: %13 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.other!enumelt +// CHECK: } +// CHECK: Overlapping Uses { +// CHECK: copy_addr %0 to [initialization] %2 : $*IntTEnum +// CHECK: switch_enum_addr %2 : $*IntTEnum, case #IntTEnum.int!enumelt: bb1, case #IntTEnum.other!enumelt: bb2 +// CHECK: %5 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.int!enumelt +// CHECK: %6 = load [take] %5 : $*<Ï„_0_0> { var Int } +// CHECK: %8 = load [trivial] %7 : $*Int +// CHECK: %13 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.other!enumelt +// CHECK: } +// CHECK: ###For MemOp: %13 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.other!enumelt +// CHECK: Storage: Stack %2 = alloc_stack $IntTEnum +// CHECK: Path: () +// CHECK: Exact Uses { +// CHECK: copy_addr %0 to [initialization] %2 : $*IntTEnum +// CHECK: switch_enum_addr %2 : $*IntTEnum, case #IntTEnum.int!enumelt: bb1, case #IntTEnum.other!enumelt: bb2 +// CHECK: %5 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.int!enumelt +// CHECK: %6 = load [take] %5 : $*<Ï„_0_0> { var Int } +// CHECK: %8 = load [trivial] %7 : $*Int +// CHECK: %13 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.other!enumelt +// CHECK: } +// CHECK: Overlapping Uses { +// CHECK: copy_addr %0 to [initialization] %2 : $*IntTEnum +// CHECK: switch_enum_addr %2 : $*IntTEnum, case #IntTEnum.int!enumelt: bb1, case #IntTEnum.other!enumelt: bb2 +// CHECK: %5 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.int!enumelt +// CHECK: %6 = load [take] %5 : $*<Ï„_0_0> { var Int } +// CHECK: %8 = load [trivial] %7 : $*Int +// CHECK: %13 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.other!enumelt +// CHECK: } +enum IntTEnum { + indirect case int(Int) + case other(T) + + var getValue: Int {get } +} + +sil hidden [ossa] @testEnumUses : $@convention(method) (@in_guaranteed IntTEnum) -> Int { +bb0(%0 : $*IntTEnum): + debug_value_addr %0 : $*IntTEnum, let, name "self", argno 1 + %2 = alloc_stack $IntTEnum + copy_addr %0 to [initialization] %2 : $*IntTEnum + switch_enum_addr %2 : $*IntTEnum, case #IntTEnum.int!enumelt: bb1, case #IntTEnum.other!enumelt: bb2 + +bb1: + %5 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.int!enumelt + %6 = load [take] %5 : $*<Ï„_0_0> { var Int } + %7 = project_box %6 : $<Ï„_0_0> { var Int } , 0 + %8 = load [trivial] %7 : $*Int + debug_value %8 : $Int, let, name "x" + destroy_value %6 : $<Ï„_0_0> { var Int } + dealloc_stack %2 : $*IntTEnum + br bb3(%8 : $Int) + +bb2: + %13 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.other!enumelt + %14 = integer_literal $Builtin.Int64, 0 + %15 = struct $Int (%14 : $Builtin.Int64) + destroy_addr %13 : $*T + dealloc_stack %2 : $*IntTEnum + br bb3(%15 : $Int) + +bb3(%19 : $Int): + return %19 : $Int +} + +class Storage {} + +// Test that tail_addr is always an unknown offset. +// +// CHECK-LABEL: @inlinedArrayProp +// CHECK: ###For MemOp: %{{.*}} = load [trivial] %{{.*}} : $*Int +// CHECK: Tail %0 = argument of bb0 : $Storage +// CHECK: Base: %{{.*}} = ref_tail_addr %0 : $Storage, $UInt +// CHECK: Storage: Tail %0 = argument of bb0 : $Storage +// CHECK: Path: (@Unknown) +// CHECK: Exact Uses { +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: %8 = load [trivial] %7 : $*Int +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: end_access %7 : $*Int +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: end_access %5 : $*Int +// CHECK-NEXT: Path: (@Unknown) +// CHECK-NEXT: end_access %2 : $*UInt +// CHECK-NEXT: Path: () +// CHECK-NEXT: } +sil hidden [ossa] @inlinedArrayProp : $@convention(thin) (@guaranteed Storage) -> Int { +bb0(%0 : @guaranteed $Storage): + %1 = ref_tail_addr %0 : $Storage, $UInt + %2 = begin_access [read] [static] %1 : $*UInt + %3 = integer_literal $Builtin.Word, 1 + %4 = tail_addr %2 : $*UInt, %3 : $Builtin.Word, $Int + %5 = begin_access [read] [static] %4 : $*Int + %6 = index_addr %5 : $*Int, %3 : $Builtin.Word + %7 = begin_access [read] [static] %6 : $*Int + %8 = load [trivial] %7 : $*Int + end_access %7 : $*Int + end_access %5 : $*Int + end_access %2 : $*UInt + return %8 : $Int +} From 0f428c2c06d5698aa750016ae60291424325c1bc Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Mon, 31 Aug 2020 14:30:01 -0700 Subject: [PATCH 12/20] alias-analysis test --- test/SILOptimizer/alias-analysis.sil | 60 ++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 test/SILOptimizer/alias-analysis.sil diff --git a/test/SILOptimizer/alias-analysis.sil b/test/SILOptimizer/alias-analysis.sil new file mode 100644 index 0000000000000..675211e426fb5 --- /dev/null +++ b/test/SILOptimizer/alias-analysis.sil @@ -0,0 +1,60 @@ +// RUN: %target-sil-opt %s -aa-kind=all -aa-dump -o /dev/null | %FileCheck %s + +// General black-box alias analysis tests. White-box unit tests are +// specific to the aa-kind, such as basic-aa.sil and typed-access-tb-aa.sil. + +// REQUIRES: asserts + +sil_stage canonical + +import Swift +import SwiftShims +import Builtin + +struct MyStruct { + @_hasStorage @_hasInitialValue var i: Int64 { get set } + @_hasStorage @_hasInitialValue var j: Int64 { get set } +} + +struct OuterStruct { + @_hasStorage @_hasInitialValue var s: MyStruct { get set } +} + +// Test overlaping access on a different path; the user has misused an index offset +// CHECK-LABEL: @testOffsetBehindProjection +// CHECK: PAIR #28. +// CHECK: %4 = load %3 : $*Int64 +// CHECK: %6 = load %5 : $*Int64 +// CHECK: MayAlias +sil shared @testOffsetBehindProjection : $@convention(thin) (@inout MyStruct) -> () { +bb0(%0 : $*MyStruct): + %1 = struct_element_addr %0 : $*MyStruct, #MyStruct.i + %2 = integer_literal $Builtin.Word, 1 + %3 = index_addr %1 : $*Int64, %2 : $Builtin.Word + %4 = load %3 : $*Int64 + // load from a different subobject overlaps + %5 = struct_element_addr %0 : $*MyStruct, #MyStruct.j + %6 = load %5 : $*Int64 + %99 = tuple () + return %99 : $() +} + +// CHECK-LABEL: @testOffsetsBehindProjectionOverlap +// CHECK: PAIR #28. +// CHECK: %3 = load %2 : $*Int64 +// CHECK: %7 = load %6 : $*Int64 +// CHECK: MayAlias +sil shared @testOffsetsBehindProjectionOverlap : $@convention(thin) (@inout OuterStruct) -> () { +bb0(%0 : $*OuterStruct): + %1 = struct_element_addr %0 : $*OuterStruct, #OuterStruct.s + %2 = struct_element_addr %1 : $*MyStruct, #MyStruct.i + %3 = load %2 : $*Int64 + // Loading from a different index within the same subobject still appears to overlap. + // Indexing from a subobject is always considered an unknown index. + %4 = integer_literal $Builtin.Word, 1 + %5 = index_addr %1 : $*MyStruct, %4 : $Builtin.Word + %6 = struct_element_addr %5 : $*MyStruct, #MyStruct.i + %7 = load %6 : $*Int64 + %99 = tuple () + return %99 : $() +} From 391b06d20b1bab6be5c289d66a3a56de793eef8d Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Wed, 2 Sep 2020 11:50:42 -0700 Subject: [PATCH 13/20] RLE tests for boundary Projection indices. --- test/SILOptimizer/redundant_load_elim.sil | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/test/SILOptimizer/redundant_load_elim.sil b/test/SILOptimizer/redundant_load_elim.sil index 633a72cd588ca..4549ec2c481f0 100644 --- a/test/SILOptimizer/redundant_load_elim.sil +++ b/test/SILOptimizer/redundant_load_elim.sil @@ -1176,14 +1176,27 @@ bb0(%0 : $*Builtin.Int64, %1 : $Builtin.NativeObject): return %6 : $(Builtin.Int64, Builtin.Int64) } -sil @dont_crash_on_index_addr_projection : $@convention(thin) (Builtin.RawPointer) -> Int { +sil @dont_crash_on_index_addr_projection : $@convention(thin) (Builtin.RawPointer) -> (Int, Int, Int, Int) { bb0(%0 : $Builtin.RawPointer): - %3 = integer_literal $Builtin.Word, 4294967295 + // Negative (valid constant index) + %3 = integer_literal $Builtin.Word, 4294967295 // '0xffffffff' %4 = pointer_to_address %0 : $Builtin.RawPointer to [strict] $*Int - // Just check if we can handle an index_addr projection with the special value of 0xffffffff %5 = index_addr %4 : $*Int, %3 : $Builtin.Word %6 = load %5 : $*Int - return %6 : $Int + // TailIndex (invalid constant index) + %7 = integer_literal $Builtin.Word, 2147483647 // '0x7fffffff' + %8 = index_addr %4 : $*Int, %7 : $Builtin.Word + %9 = load %8 : $*Int + // UnknownOffset (valid index) + %10 = integer_literal $Builtin.Word, 3221225472 // '0xC0000000' + %11 = index_addr %4 : $*Int, %10 : $Builtin.Word + %12 = load %11 : $*Int + // Root (unused/invalid index)) + %13 = integer_literal $Builtin.Word, 2147483648 // '0x80000000' + %14 = index_addr %4 : $*Int, %13 : $Builtin.Word + %15 = load %14 : $*Int + %99 = tuple (%6 : $Int, %9 : $Int, %12 : $Int, %15 : $Int) + return %99 : $(Int, Int, Int, Int) } sil @overwrite_int : $@convention(thin) (@inout Int, Int) -> () From 712e1abec67af6d33b7a7465d4c4c0c52fd1c55a Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Sun, 13 Sep 2020 23:38:13 -0700 Subject: [PATCH 14/20] AccessedStorage and AccessPath documentation. --- docs/SILProgrammersManual.md | 462 +++++++++++++++++++++++++++++ include/swift/SIL/MemAccessUtils.h | 3 +- 2 files changed, 464 insertions(+), 1 deletion(-) diff --git a/docs/SILProgrammersManual.md b/docs/SILProgrammersManual.md index b5cf4e9c7bd9e..7a43605f1855b 100644 --- a/docs/SILProgrammersManual.md +++ b/docs/SILProgrammersManual.md @@ -159,6 +159,468 @@ rules out PartialApplies is no excuse to conflate applied arguments with function arguments. Also, without consistent use of common idioms, it becomes overly burdensome to evolve these APIs over time. +## `AccessedStorage` and `AccessPath` + +The `AccessedStorage` and `AccessPath` types formalize memory access +in SIL. Given an address-typed SIL value, it is possible to +reliably identify the storage location of the accessed +memory. `AccessedStorage` identifies an accessed storage +location. `AccessPath` contains both a storage location and the +"access path" within that memory object. The relevant API details are +documented in MemAccessUtils.h + +### Formal access + +SIL preserves the language semantics of formal variable access in the +form of access markers. `begin_access` identifies the address of the +formal access and `end_access` delimits the scope of the access. At +the language level, a formal access is an access to a local variable +or class property. For details, see +[SE-0176: Enforce Exclusive Access to Memory](https://github.com/apple/swift-evolution/blob/master/proposals/0176-enforce-exclusive-access-to-memory.md) + +Access markers are preserved in SIL to: + +1. verify exclusivity enforcement + +2. optimize exclusivity checks following other transforms, such as + converting dynamic checks into static checks + +3. simplify and strengthen general analyses of memory access. For +example, `begin_access [read] %address` indicates that the accessed +address is immutable for the duration of its access scope + +### Access path def-use relationship + +Computing `AccessedStorage` and `AccessPath` for any given SIL address +involves a use-def traversal to determine the origin of the +address. It may traverse operations on address, pointer, box, and +reference types. The logic that formalizes which SIL operations may be +involved in the def-use chain is encapsulated with the +`AccessUseDefChainVisitor`. The traversal can be customized by +implementing this visitor. Customization is not expected to change the +meaning of AccessedStorage or AccessPath. Rather, it is intended for +additional pass-specific book-keeping or for higher-level convenience +APIs that operate on the use-def chain bypassing AccessedStorage +completely. + +Access def-use chains are divided by four points: the "root", the +access "base", the outer-most "access" scope, and the "address" of a +memory operation. For example: +``` + struct S { + var field: Int64 + } + class C { + var prop: S + } + + %root = alloc_ref $C + %base = ref_element_addr %root : $C, #C.prop + %access = begin_access [read] [static] %base : $*S + %address = struct_element_addr %access : $*S, #.field + %value = load [trivial] %address : $*Int64 + end_access %access : $*S +``` + +#### Reference root + +The first part of the def-use chain computes the formal access base +from the root of the object (e.g. `alloc_ref -> +ref_element_addr`). The reference root might be a locally allocated +object, a function argument, a function result, or a reference loaded +from storage. There is no enforcement on the type of operation that +can produce a reference; however, only reference types or +Builtin.BridgeObject types are only allowed in this part of the +def-use chain. The reference root is the greatest common ancestor in +the def-use graph that can identify an object by a single SILValue. If +the root as an `alloc_ref`, then it is *uniquely identified*. The +def-use chain from the root to the base may contain reference casts +(`isRCIdentityPreservingCast`) and phis. + +This example has an identifiable def-use chain from `%root` to `%base`: +``` +class A { + var prop0: Int64 +} +class B : A { +} + +bb0: + %root = alloc_ref $B + cond_br _, bb1, bb2 + +bb1: + %a1 = upcast %root : $B to $A + br bb3(%a1 : $A) + +bb2: + %a2 = upcast %root : $B to $A + br bb3(%a2 : $A) + +bb3(%a : $A): + %bridge = ref_to_bridge_object %a : $A, %bits : $Builtin.Word + %ref = bridge_object_to_ref %bridge : $Builtin.BridgeObject to $A + %base = ref_element_addr %ref : $A, #A.prop0 +``` + +Each object property and its tail storage is considered a separate +formal access base. The reference root is only one component of an +`AccessedStorage` location. AccessedStorage also identifies the class +property being accessed within that object. + +#### Access base + +The access base is the SILValue produced by an instruction that +directly identifies the kind of storage being accessed without further +use-def traversal. Common access bases are `alloc_box`, `alloc_stack`, +`global_addr`, `ref_element_addr`, and function arguments (see +`AccessedStorage::Kind`). + +The access base is the same as the "root" SILValue for all storage +kinds except global and class storage. Global storage has no root. For +class storage the root is the SILValue that identifies object, +described as the "reference root" above. + +"Box" storage is uniquely identified by an `alloc_box` +instruction. Therefore, we consider the `alloc_box` to be the base of +the access. Box storage does not apply to all box types or box +projections, which may instead originate from arguments or indirect +enums for example. + +Typically, the base is the address-type source operand of a +`begin_access`. However, the path from the access base to the +`begin_access` may include *storage casts* (see +`isAccessedStorageCast`). It may involve address, pointer, and box +types, and may traverse phis. For some kinds of storage, the base may +itself even be a non-address pointer. For phis that cannot be uniquely +resolved, the base may even be a box type. + +This example has an identifiable def-use chain from `%base` to `%access`: +``` +bb0: + %base = alloc_box $Int { var Int } + %boxadr = project_box %base : ${ var Int } + %p0 = address_to_pointer %boxadr : $*Int to $Builtin.RawPointer + cond_br _, bb1, bb2 + +bb1: + %p1 = copy_value %p0 : $Builtin.RawPointer + br bb3(%p1 : $Builtin.RawPointer) + +bb2: + br bb3(%p0 : $Builtin.RawPointer) + +bb3(%ptr : $Builtin.RawPointer): + %adr = pointer_to_address %ptr : $Builtin.RawPointer to $*Int + %access = begin_access [read] [static] %adr : $*Int +``` + +Note that address-type phis are illegal (full enforcement +pending). This is important for simplicity and efficiency, but also +allows for a class of storage optimizations, such as bitfields, in +which address storage is always uniquely determined. Currently, if a +(non-address) phi on the access path from `base` to `access` does not +have a common base, then it is considered an invalid access (the +AccessedStorage object is not valid). SIL verification ensures that a +formal access always has valid AccessedStorage (WIP). In other words, the +source of a `begin_access` marker must be a single, non-phi base. In +the future, for further simplicity, we may generally disallow box and +pointer phis unless they have a common base. + +Not all SIL memory access is part of a formal access, but the +`AccessedStorage` and `AccessPath` abstractions are universally +applicable. Non-formal access still has an access base, even though +the use-def search does not begin at a `begin_access` marker. For +non-formal access, SIL verification is not as strict. An invalid +access is allowed, but handled conservatively. This is safe as long as +those non-formal accesses can never alias with class and global +storage. Class and global access is always guarded by formal access +markers--at least until static markers are stripped from SIL. + +#### Nested access + +Nested access occurs when an access base is a function argument. The +caller always checks `@inout` arguments for exclusivity (an access +marker must exist in the caller). However, the argument itself is a +variable with its own formal access. Conflicts may occur in the callee +which were not evident in the caller. In this example, a conflict +occurs in `hasNestedAccess` but not in its caller: + +``` +func takesTwoInouts(_ : inout Int, _ : inout Int) -> () {} + +func hasNestedAccess(_ x : inout Int) -> () { + takesTwoInouts(&x, &x) +} + +var x = 0 +hasNestedAccess(&x) +``` + +Produces these access markers: +``` +sil @takesTwoInouts : $@convention(thin) (@inout Int, @inout Int) -> () + +sil @hasNestedAccess : $@convention(thin) (@inout Int) -> () { +bb0(%0 : $*Int): + %innerAccess = begin_access [modify] %0 : $*Int + %conflicting = begin_access [modify] %0 : $*Int + %f = function_ref @takesTwoInouts + apply %f(%innerAccess, %conflicting) + : $@convention(thin) (@inout Int, @inout Int) -> () + end_access %conflicting : $*Int + end_access %innerAccess : $*Int + //... +} + +%var = alloc_stack $Int +%outerAccess = begin_access [modify] %var : $*Int +%f = function_ref @hasNestedAccess +apply %f(%outerAccess) : $@convention(thin) (@inout Int) -> () { +end_access %outerAccess : $*Int +``` + +Nested accesses become part if the def-use chain after inlining. Here, +both `%innerAccess` and `%conflicting` are nested within +`%outerAccess`: + +``` +%var = alloc_stack $Int +%outerAccess = begin_access [modify] %var : $*Int +%innerAccess = begin_access [modify] %outerAccess : $*Int +%conflicting = begin_access [modify] %outerAccess : $*Int +%f = function_ref @takesTwoInouts +apply %f(%innerAccess, %conflicting) + : $@convention(thin) (@inout Int, @inout Int) -> () +end_access %conflicting : $*Int +end_access %innerAccess : $*Int +end_access %outerAccess : $*Int +``` + +For most purposes, the inner access scopes are irrelevant. When we ask +for the "accessed storage" for `%innerAccess`, we get an +`AccessedStorage` value of "Stack" kind with base `%var = +alloc_stack`. If instead of finding the original accessed storage, we +want to identify the enclosing formal access scope, we need to use a +different API that supports the special `Nested` storage kind. This is +typically only used for exclusivity diagnostics though. + +TODO: Nested static accesses that result from inlining could +potentially be removed, as long as DiagnoseStaticExclusivity has +already run. + +#### Access projections + +On the def-use chain between the *outermost* formal access scope within +the current function and a memory operation, *access projections* +identify subobjects laid out within the formally accessed +variable. The sequence of access projections between the base and the +memory address correspond to an access path. + +For example, there is no formal access for struct fields. Instead, +they are addressed using a `struct_element_addr` within the access +scope: + +``` +%access = begin_access [read] [static] %base : $*S +%memaddr = struct_element_addr %access : $*S, #.field +%value = load [trivial] %memaddr : $*Int64 +end_access %access : $*S +``` + +Note that is is possible to have a nested access scope on the address +of a struct field, which may show up as an access of +struct_element_addr after inlining. The rule is that access +projections cannot occur outside of the outermost access scope within +the function. + +Access projections are address projections--they take an address at +operand zero and produce a single address result. Other +straightforward access projections include `tuple_element_addr`, +`index_addr`, and `tail_addr` (an aligned form of `index_addr`). + +Enum payload extraction (`unchecked_take_enum_data_addr`) is also an +access projection, but it has no effect on the access path. + +Indirect enum payload extraction is a special two-instruction form of +address projection (`load : ${ var } -> project_box`). For simplicity, +and to avoid the appearance of box types on the access path, this +should eventually be encapsulated in a single SIL instruction. + +For example, the following complex def-use chain from `%base` to +`%load` actually has an empty access path: +``` +%boxadr = unchecked_take_enum_data_addr %base : $*Enum, #Enum.int!enumelt +%box = load [take] %boxadr : $*<Ï„_0_0> { var Int } +%valadr = project_box %box : $<Ï„_0_0> { var Int } , 0 +%load = load [trivial] %valadr : $*Int +``` + +Storage casts may also occur within an access. This typically results +from accessors, which perform address-to-pointer +conversion. Pointer-to-address conversion performs a type cast, and +could lead to different subobject types corresponding to the same base +and access path. Access paths still uniquely identify a memory +location because it is illegal to cast memory to non-layout-compatible +types on same execution path (without an intervening `bind_memory`). + +Address-type phis are prohibited, but because pointer and box types +may be on the def-use chain, phis may also occur on an access path. A +phi is only a valid part of an access path if it has no affect on the +path components. This means that pointer casting and unboxing may +occur on distinct phi paths, but index offsets and subobject +projections may not. These rules are currently enforced to a limited +extent, so it's possible for invalid access path to occur under +certain conditions. + +For example, the following is a valid def-use access chain, with an +access base defined in `bb0`, a memory operation in `bb3` and an +`index_addr` and `struct_element_addr` on the access path: + +``` +class A {} + +struct S { + var field0: Int64 + var field1: Int64 +} + +bb0: + %base = ref_tail_addr %ref : $A, $S + %idxproj = index_addr %tail : $*S, %idx : $Builtin.Word + %p0 = address_to_pointer %idxproj : $*S to $Builtin.RawPointer + cond_br _, bb1, bb2 + +bb1: + %pcopy = copy_value %p0 : $Builtin.RawPointer + %adr1 = pointer_to_address [strict] %pcopy : $Builtin.RawPointer to $*S + %p1 = address_to_pointer %adr1 : $*S to $Builtin.RawPointer + br bb3(%p1 : $Builtin.RawPointer) + +bb2: + br bb3(%p0 : $Builtin.RawPointer) + +bb3(%p3 : $Builtin.RawPointer): + %adr3 = pointer_to_address [strict] %p3 : $Builtin.RawPointer to $*S + %field = struct_element_addr %adr3 : $*S, $S.field0 + load %field : $*Int64 +``` + +### AccessedStorage + +`AccessedStorage` identifies an accessed storage location, be it a +box, stack location, class property, global variable, or argument. It +is implemented as a value object that requires no compile-time memory +allocation and can be used as the hash key for that location. Extra +bits are also available for information specific to a particular +optimization pass. Its API provides the kind of location being +accessed and information about the location's uniqueness or whether it +is distinct from other storage. + +Two __uniquely identified__ storage locations may only alias if their +AccessedStorage objects are identical. + +`AccessedStorage` records the "root" SILValue of the access. The root is +the same as the access base for all storage kinds except global and +class storage. For class properties, the storage root is the reference +root of the object, not the base of the property. Multiple +`ref_element_addr` projections may exist for the same property. Global +variable storage is always uniquely identified, but it is impossible +to find all uses from the def-use chain alone. Multiple `global_addr` +instructions may reference the same variable. To find all global uses, +the client must independently find all global variable references +within the function. Clients that need to know which SILValue base was +discovered during use-def traversal in all cases can make use of +`AccessedStorageWithBase` or `AccessPathWithBase`. + +### AccessPath + +`AccessPath` extends `AccessedStorage` to include the path components +that determine the address of a subobject within the access base. The +access path is a string of index offsets and subobject projection +indices. + +``` +struct S { + var field0: Int64 + var field1: Int64 +} + +%eltadr = struct_element_addr %access : $*S, #.field1 + +Path: (#1) +``` + +``` +class A {} + +%tail = ref_tail_addr %ref : $A, $S +%one = integer_literal $Builtin.Word, 1 +%elt = index_addr %tail : $*S, %one : $Builtin.Word +%field = struct_element_addr %elt : $*S, $S.field0 + +Path: (@1, #0) +``` + +Note that a projection from a reference type to the object's property +or tail storage is not part of the access path because it is already +identified by the storage location. + +Offset indices are all folded into a single index at the head of the +path (a missing offset implies offset zero). Offsets that are not +static constants are still valid but are labeled "@Unknown". Indexing +within a subobject is an ill-formed access, but is handled +conservatively since this rule cannot be fully enforced. + +For example, the following is an invalid access path, which just +happens to point to field1: +``` +%field0 = struct_element_addr %base : $*S, #field0 +%field1 = index_addr %elt : $*Int64, %one : $Builtin.Word + +Path: (INVALID) +``` + +The following APIs determine whether an access path contains another +or may overlap with another. + +`AccessPath::contains(AccessPath subPath)` + +`AccessPath::mayOverlap(AccessPath otherPath)` + +These are extremely light-weight APIs that, in the worst case, require +a trivial linked list traversal with single pointer comparison for the +length of subPath or otherPath. + +Subobjects are both contained with and overlap with their parent +storage. An unknown offset does not contain any known offsets but +overlaps with all offsets. + +### Access path uses + +For any accessed storage location and base, it must also be possible +to reliably identify all uses of that storage location within the +function for a particular access base. If the storage is uniquely +identified, then that also implies that all uses of that storage +within the function have been discovered. In other words, there are no +aliases to the same storage that aren't covered by this use set. + +The `AccessPath::collectUses()` API does this. It is possible to ask +for only the uses contained by the current path, or for all +potentially overlapping uses. It is guaranteed to return a complete +use set unless the client specifies a limit on the number of uses. + +As passes begin to adopt AccessPath::collectUses(), I expect it to +become a visitor pattern that allows the pass to perform custom +book-keeping for certain types of uses. + +The `AccessPathVerification` pass runs at key points in the pipeline +to ensure that all address uses are identified and have consistent +access paths. This pass ensures that the implementations of AccessPath +is internally consistent for all SIL patterns. Enforcing the validity +of the SIL itself, such as which operations are allowed on an access +def-use chain, is handled within the SIL verifier instead. + ## SILGen TBD: Possibly link to a separate document explaining the architecture of SILGen. diff --git a/include/swift/SIL/MemAccessUtils.h b/include/swift/SIL/MemAccessUtils.h index e6afa239c528e..bc6ad28457bbe 100644 --- a/include/swift/SIL/MemAccessUtils.h +++ b/include/swift/SIL/MemAccessUtils.h @@ -10,7 +10,8 @@ // //===----------------------------------------------------------------------===// /// -/// These utilities model the storage locations of memory access. +/// These utilities model the storage locations of memory access. See +/// ProgrammersGuide.md for high-level design. /// /// All memory operations that are part of a formal access, as defined by /// exclusivity rules, are marked by begin_access and end_access instructions. From 9c69d0242d4cf191e06ae8d2122e1265f9ddc4cc Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Thu, 8 Oct 2020 04:35:14 -0700 Subject: [PATCH 15/20] MemAccessUtils comment --- include/swift/SIL/MemAccessUtils.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/swift/SIL/MemAccessUtils.h b/include/swift/SIL/MemAccessUtils.h index bc6ad28457bbe..fdf5c68e90706 100644 --- a/include/swift/SIL/MemAccessUtils.h +++ b/include/swift/SIL/MemAccessUtils.h @@ -890,6 +890,7 @@ struct AccessPathWithBase { // // base may be invalid for global_addr -> address_to_pointer -> phi patterns. // FIXME: add a structural requirement to SIL so base is always valid in OSSA. + // !!! make this a PtrIntPair with a the access kind SILValue base; /// \p address identifies the object seen by any memory operation that From 6f2cda1390833cca96a8087a72fe348e74733713 Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Thu, 15 Oct 2020 00:04:19 -0700 Subject: [PATCH 16/20] Add AccessUseVisitor and cleanup related APIs. Add AccesssedStorage::compute and computeInScope to mirror AccessPath. Allow recovering the begin_access for Nested storage. Adds AccessedStorage.visitRoots(). --- include/swift/SIL/MemAccessUtils.h | 318 +++++--- .../SILOptimizer/Analysis/ValueTracking.h | 2 +- lib/IRGen/IRGenSIL.cpp | 2 +- lib/SIL/Utils/MemAccessUtils.cpp | 742 ++++++++++-------- lib/SILGen/SILGenLValue.cpp | 2 +- .../Analysis/AccessedStorageAnalysis.cpp | 8 +- lib/SILOptimizer/Analysis/AliasAnalysis.cpp | 2 +- lib/SILOptimizer/Analysis/MemoryBehavior.cpp | 3 +- lib/SILOptimizer/LoopTransforms/LICM.cpp | 6 +- .../Mandatory/DiagnoseStaticExclusivity.cpp | 21 +- .../Transforms/AccessEnforcementDom.cpp | 2 +- .../Transforms/AccessEnforcementOpts.cpp | 2 +- .../Transforms/AccessEnforcementWMO.cpp | 4 +- .../Transforms/PerformanceInliner.cpp | 2 +- .../UtilityPasses/AccessPathVerification.cpp | 18 +- .../UtilityPasses/AccessedStorageDumper.cpp | 42 +- test/SILOptimizer/accesspath_uses.sil | 210 ++++- test/SILOptimizer/licm.sil | 2 +- 18 files changed, 904 insertions(+), 484 deletions(-) diff --git a/include/swift/SIL/MemAccessUtils.h b/include/swift/SIL/MemAccessUtils.h index fdf5c68e90706..a1b302710acd1 100644 --- a/include/swift/SIL/MemAccessUtils.h +++ b/include/swift/SIL/MemAccessUtils.h @@ -27,16 +27,16 @@ /// memory operation, there are three levels of APIs that inspect the origin of /// that address: /// -/// 1. getAccessAddress(): Find the originating address as close as possible to -/// the address of the formal access *without* looking past any storage -/// casts. This is useful when the type of the returned access address must be -/// consistent with the memory operation's type (the same type or a parent -/// type). For a formal access, this typically returns the begin_access, but it -/// is not guaranteed to because some accesses contain storage casts. For +/// 1. getTypedAccessAddress(): Find the originating address as close as +/// possible to the address of the formal access *without* looking past any +/// storage casts. This is useful when the type of the returned access address +/// must be consistent with the memory operation's type (the same type or a +/// parent type). For a formal access, this typically returns the begin_access, +/// but it is not guaranteed to because some accesses contain storage casts. For /// non-formal access, it returns a best-effort address corresponding to the /// base of an access. /// -/// 2. getAccessBegin(): If the memory operation is part of a formal access, +/// 2. getAccessScope(): If the memory operation is part of a formal access, /// then this is guaranteed to return the begin_access marker. Otherwise, it /// returns the best-effort address or pointer corresponding to the base of an /// access. Useful to find the scope of a formal access. @@ -47,42 +47,43 @@ /// address type, pointer type, or box type, but never a reference type. /// Each object's property or its tail storage is separately accessed. /// -/// For better identification an access base, use findAccessedStorage(). It -/// returns an AccessedStorage value that identifies the storage location of a -/// memory access. It provides APIs for inspecting type of accessed storage and -/// allows for disambiguation between different types of storage and different -/// properties within a class. +/// For better identification an access base, use +/// AccessedStorage::compute(). It returns an AccessedStorage value +/// that identifies the storage location of a memory access. It provides APIs +/// for inspecting type of accessed storage and allows for disambiguation +/// between different types of storage and different properties within a class. /// -/// findAccessedStorage() follows the same logic as getAccessBase(), but if the -/// base is not recognized as a valid access, it returns invalid +/// AccessedStorage::compute() follows the same logic as getAccessBase(), but if +/// the base is not recognized as a valid access, it returns invalid /// AccessedStorage. It also performs further analysis to determine the root /// reference of an object access. /// -/// findAccessedStorage() returns the outermost AccessedStorage for any memory -/// address. It can be called on the address of a memory operation, the address -/// of a begin_access, or any other address value. If the address is from an -/// enforced begin_access or from any memory operation that is part of a formal -/// access, then it returns a valid AccessedStorage value. If the memory +/// AccessedStorage::compute() returns the outermost AccessedStorage for any +/// memory address. It can be called on the address of a memory operation, the +/// address of a begin_access, or any other address value. If the address is +/// from an enforced begin_access or from any memory operation that is part of a +/// formal access, then it returns a valid AccessedStorage value. If the memory /// operation is not part of a formal access, then it still identifies the /// accessed location as a best effort, but the result may be invalid storage. /// -/// An active goal is to require findAccessedStorage() to always return a +/// An active goal is to require compute() to always return a /// valid AccessedStorage value even for operations that aren't part of a /// formal access. /// /// The AccessEnforcementWMO pass is an example of an optimistic optimization -/// that relies on this requirement for correctness. If findAccessedStorage() -/// simply bailed out on an unrecognized memory address by returning an invalid -/// AccessedStorage, then the optimization could make incorrect assumptions -/// about the absence of access to globals or class properties. +/// that relies on this requirement for correctness. If +/// AccessedStorage::compute() simply bailed out on an unrecognized memory +/// address by returning an invalid AccessedStorage, then the optimization could +/// make incorrect assumptions about the absence of access to globals or class +/// properties. /// -/// identifyFormalAccess() is similar to findAccessedStorage(), but returns the -/// formally accessed storage of a begin_access instruction. This must return a -/// valid AccessedStorage value unless the access has "Unsafe" enforcement. The -/// formal access location may be nested within an outer begin_access. For the -/// purpose of exclusivity, nested accesses are considered distinct formal -/// accesses so they return distinct AccessedStorage values even though they may -/// access the same memory. +/// identifyFormalAccess() is similar to AccessedStorage::compute(), but returns +/// the formally accessed storage of a begin_access instruction. This must +/// return a valid AccessedStorage value unless the access has "Unsafe" +/// enforcement. The formal access location may be nested within an outer +/// begin_access. For the purpose of exclusivity, nested accesses are considered +/// distinct formal accesses so they return distinct AccessedStorage values even +/// though they may access the same memory. /// //===----------------------------------------------------------------------===// @@ -100,7 +101,7 @@ #include "llvm/ADT/DenseMap.h" //===----------------------------------------------------------------------===// -// MARK: General Helpers +// MARK: Standalone API //===----------------------------------------------------------------------===// namespace swift { @@ -127,7 +128,7 @@ inline SILValue stripAccessMarkers(SILValue v) { /// corresponding to the accessed variable. This never looks through /// pointer_to_address or other conversions that may change the address type /// other than via type-safe (TBAA-compatible) projection. -SILValue getAccessAddress(SILValue address); +SILValue getTypedAccessAddress(SILValue address); /// Return the source address or pointer after stripping all access projections /// and storage casts. @@ -138,7 +139,7 @@ SILValue getAccessAddress(SILValue address); /// If there is no access marker, then it returns a "best effort" address /// corresponding to the accessed variable. In this case, the returned value /// could be a non-address pointer type. -SILValue getAccessBegin(SILValue address); +SILValue getAccessScope(SILValue address); /// Return the source address or pointer after stripping access projections, /// access markers, and storage casts. @@ -174,6 +175,20 @@ inline bool accessKindMayConflict(SILAccessKind a, SILAccessKind b) { namespace swift { +/// Control def-use traversals, allowing them to remain with an access scope or +/// consider operations across scope boundaries. +enum class NestedAccessType { StopAtAccessBegin, IgnoreAccessBegin }; + +/// Exact uses only include uses whose AccessPath is identical to this one. +/// Inner uses have an AccessPath the same as or contained by this one. +/// Overlapping uses may contain, be contained by, or have an unknown +/// relationship with this one. An unknown relationship typically results from +/// a dynamic index_addr offset. +/// +/// The enum values are ordered. Each successive use type is a superset of the +/// previous. +enum class AccessUseType { Exact, Inner, Overlapping }; + /// Represents the identity of a storage object being accessed. /// /// Requirements: @@ -259,6 +274,32 @@ class AccessedStorage { return storage; } + /// Return an AccessedStorage value that best identifies a formally accessed + /// variable pointed to by \p sourceAddress, looking through any nested + /// formal accesses to find the underlying storage. + /// + /// \p sourceAddress may be an address, pointer, or box type. + /// + /// If \p sourceAddress is within a formal access scope, which does not have + /// "Unsafe" enforcement, then this always returns valid storage. + /// + /// If \p sourceAddress is not within a formal access scope, or within an + /// "Unsafe" scope, then this finds the formal storage if possible, otherwise + /// returning invalid storage. + static AccessedStorage compute(SILValue sourceAddress); + + /// Return an AccessedStorage object that identifies formal access scope that + /// immediately encloses \p sourceAddress. + /// + /// \p sourceAddress may be an address, pointer, or box type. + /// + /// If \p sourceAddress is within a formal access scope, this always returns a + /// valid "Nested" storage value. + /// + /// If \p sourceAddress is not within a formal access scope, then this finds + /// the formal storage if possible, otherwise returning invalid storage. + static AccessedStorage computeInScope(SILValue sourceAddress); + protected: // Checking the storage kind is far more common than other fields. Make sure // it can be byte load with no shift. @@ -394,6 +435,7 @@ class AccessedStorage { switch (getKind()) { case AccessedStorage::Box: case AccessedStorage::Stack: + case AccessedStorage::Nested: case AccessedStorage::Argument: case AccessedStorage::Yield: case AccessedStorage::Unidentified: @@ -403,12 +445,22 @@ class AccessedStorage { case AccessedStorage::Class: case AccessedStorage::Tail: return getObject(); - case AccessedStorage::Nested: - assert(false && "AccessPath cannot identify nested access"); - return SILValue(); } } + /// Visit all access roots. If any roots are visited then the original memory + /// operation access must be reachable from one of those roots. Unidentified + /// storage might not have any root. Identified storage always has at least + /// one root. Identified non-global storage always has a single root. For + /// Global storage, this visits all global_addr instructions in the function + /// that reference the same SILGlobalVariable. + /// + /// \p function must be non-null for Global storage (global_addr cannot + /// occur in a static initializer). + void + visitRoots(SILFunction *function, + llvm::function_ref visitor) const; + /// Return true if the given storage objects have identical storage locations. /// /// This compares only the AccessedStorage base class bits, ignoring the @@ -507,6 +559,21 @@ class AccessedStorage { /// checking via the ValueDecl if we are processing a `let` variable. const ValueDecl *getDecl() const; + /// Get all leaf uses of all address, pointer, or box values that have a this + /// AccessedStorage in common. Return true if all uses were found before + /// reaching the limit. + /// + /// The caller of 'collectUses' can determine the use type (exact, inner, or + /// overlapping) from the resulting \p uses list by checking 'accessPath == + /// usePath', accessPath.contains(usePath)', and + /// 'accessPath.mayOverlap(usePath)'. Alternatively, the client may call + /// 'visitAccessedStorageUses' with its own AccessUseVisitor subclass to + /// sort the use types. + bool + collectUses(SmallVectorImpl &uses, AccessUseType useTy, + SILFunction *function, + unsigned useLimit = std::numeric_limits::max()) const; + void print(raw_ostream &os) const; void dump() const; @@ -567,6 +634,7 @@ class AccessedStorage { } // end namespace swift namespace llvm { + /// Enable using AccessedStorage as a key in DenseMap. /// Do *not* include any extra pass data in key equality. /// @@ -617,54 +685,33 @@ template <> struct DenseMapInfo { return LHS.hasIdenticalBase(RHS); } }; + } // namespace llvm namespace swift { -/// Given an address used by an instruction that reads or writes memory, return -/// the AccessedStorage value that identifies the formally accessed memory, -/// looking through any nested formal accesses to find the underlying storage. -/// -/// This may return invalid storage for a memory operation that is not part of -/// a formal access or when the outermost formal access has Unsafe enforcement. -AccessedStorage findAccessedStorage(SILValue sourceAddr); - -// Helper for identifyFormalAccess. -AccessedStorage identifyAccessedStorageImpl(SILValue sourceAddr); - -/// Return an AccessedStorage object that identifies the formal access -/// represented by \p beginAccess. -/// -/// If the given access is nested within an outer access, return a Nested -/// AccessedStorage kind. This is useful for exclusivity checking to distinguish -/// between nested access vs. conflicting access on the same storage. +/// Return an AccessedStorage value that identifies formally accessed storage +/// for \p beginAccess, considering any outer access scope as having distinct +/// storage from this access scope. This is useful for exclusivity checking +/// to distinguish between nested access vs. conflicting access on the same +/// storage. /// /// May return an invalid storage for either: /// - A \p beginAccess with Unsafe enforcement /// - Non-OSSA form in which address-type block args are allowed inline AccessedStorage identifyFormalAccess(BeginAccessInst *beginAccess) { - return identifyAccessedStorageImpl(beginAccess->getSource()); + return AccessedStorage::computeInScope(beginAccess->getSource()); } inline AccessedStorage identifyFormalAccess(BeginUnpairedAccessInst *beginAccess) { - return identifyAccessedStorageImpl(beginAccess->getSource()); -} - -/// Return a valid AccessedStorage object for an address captured by a no-escape -/// closure. A no-escape closure may capture a regular storage address without -/// guarding it with an access marker. If the captured address does come from an -/// access marker, then this returns a Nested AccessedStorage kind. -inline AccessedStorage identifyCapturedStorage(SILValue capturedAddress) { - auto storage = identifyAccessedStorageImpl(capturedAddress); - assert(storage && "captured access has invalid storage"); - return storage; + return AccessedStorage::computeInScope(beginAccess->getSource()); } } // end namespace swift //===----------------------------------------------------------------------===// -// AccessPath +// MARK: AccessPath //===----------------------------------------------------------------------===// namespace swift { @@ -700,7 +747,7 @@ namespace swift { /// convert a subobject address into a pointer (for example, via implicit /// conversion), then advance that pointer. Since we can't absolutely prevent /// this, we instead consider it an invalid AccessPath. This is the only case in -/// which AccessPath::storage can differ from findAccessedStorage(). +/// which AccessPath::storage can differ from AccessedStorage::compute(). /// /// Storing an AccessPath ammortizes to constant space. To cache identification /// of address locations, AccessPath should be used rather than the @@ -712,9 +759,23 @@ namespace swift { /// compatible type. TODO: add enforcement for this rule. class AccessPath { public: - /// Create the AccessPath for any memory operation on the given address. + /// Compute the access path at \p address. This ignores begin_access markers, + /// returning the outermost AccessedStorage. + /// + /// The computed access path corresponds to the subobject for a memory + /// operation that directly operates on \p address; so, for an indexable + /// address, this implies an operation at index zero. static AccessPath compute(SILValue address); + /// Compute the access path at \p address. If \p address is within a formal + /// access, then AccessStorage will have a nested type and base will be a + /// begin_access marker. + /// + /// This is primarily useful for recovering the access scope. The original + /// storage kind will only be discovered when \p address is part of a formal + /// access, thus not within an access scope. + static AccessPath computeInScope(SILValue address); + // Encode a dynamic index_addr as an UnknownOffset. static constexpr int UnknownOffset = std::numeric_limits::min() >> 1; @@ -848,25 +909,34 @@ class AccessPath { bool hasUnknownOffset() const { return offset == UnknownOffset; } /// Return true if this path contains \p subPath. + /// + /// Identical AccessPath's contain each other. + /// + /// Returns false if either path is invalid. bool contains(AccessPath subPath) const; /// Return true if this path may overlap with \p otherPath. + /// + /// Returns true if either path is invalid. bool mayOverlap(AccessPath otherPath) const; /// Return the address root that the access path was based on. Returns /// an invalid SILValue for globals or invalid storage. SILValue getRoot() const { return storage.getRoot(); } - /// Get all uses of all address values that have a common AccessPath. Return - /// true if all uses were found before reaching the limit. - /// - /// This should find all uses for which calling AccessPath::compute() would - /// yield an identical AccessPath. + /// Get all leaf uses of all address, pointer, or box values that have a this + /// AccessedStorage in common. Return true if all uses were found before + /// reaching the limit. /// - /// This fails on global variables which have no root. To collect all uses, - /// including global variable uses, use AccessPathWithBase::collectUses. + /// The caller of 'collectUses' can determine the use type (exact, inner, or + /// overlapping) from the resulting \p uses list by checking 'accessPath == + /// usePath', accessPath.contains(usePath)', and + /// 'accessPath.mayOverlap(usePath)'. Alternatively, the client may call + /// 'visitAccessPathUses' with its own AccessUseVisitor subclass to + /// sort the use types. bool - collectUses(SmallVectorImpl &uses, bool collectOverlappingUses, + collectUses(SmallVectorImpl &uses, AccessUseType useTy, + SILFunction *function, unsigned useLimit = std::numeric_limits::max()) const; void printPath(raw_ostream &os) const; @@ -877,10 +947,7 @@ class AccessPath { // Encapsulate the result of computing an AccessPath. AccessPath does not store // the base address of the formal access because it does not always uniquely // indentify the access, but AccessPath users may use the base address to to -// recover the def-use chain. -// -// AccessPathWithBase::collectUses is guaranteed to be complete for all storage -// types, while AccessPath::collectUses cannot handle globals. +// recover the def-use chain for a specific global_addr or ref_element_addr. struct AccessPathWithBase { AccessPath accessPath; // The address-type value that is the base of the formal access. For @@ -888,16 +955,21 @@ struct AccessPathWithBase { // global_addr or initializer apply. For other storage, it is the same as // accessPath.getRoot(). // - // base may be invalid for global_addr -> address_to_pointer -> phi patterns. + // Note: base may be invalid for global_addr -> address_to_pointer -> phi + // patterns, while the accessPath is still valid. + // // FIXME: add a structural requirement to SIL so base is always valid in OSSA. - // !!! make this a PtrIntPair with a the access kind SILValue base; - /// \p address identifies the object seen by any memory operation that - /// directly operates on the address. For indexable addresses, this implies an - /// operation at index zero. + /// Compute the access path at \p address, and record the access base. This + /// ignores begin_access markers, returning the outermost AccessedStorage. static AccessPathWithBase compute(SILValue address); + /// Compute the access path at \p address, and record the access base. If \p + /// address is within a formal access, then AccessStorage will have a nested + /// type and base will be a begin_access marker. + static AccessPathWithBase computeInScope(SILValue address); + AccessPathWithBase(AccessPath accessPath, SILValue base) : accessPath(accessPath), base(base) {} @@ -906,16 +978,6 @@ struct AccessPathWithBase { } bool operator!=(AccessPathWithBase other) const { return !(*this == other); } - /// Get all uses of all address values that have a common AccessPath. Return - /// true if all uses were found before reaching the limit. - /// - /// This should find all uses for which calling AccessPath::compute() would - /// yield an identical AccessPath and, for global variables, have the same - /// access base (e.g. from the same global_addr instruction). - bool collectUses(SmallVectorImpl &uses, - bool collectOverlappingUses, - unsigned useLimit = std::numeric_limits::max()) const; - void print(raw_ostream &os) const; void dump() const; }; @@ -924,9 +986,14 @@ inline AccessPath AccessPath::compute(SILValue address) { return AccessPathWithBase::compute(address).accessPath; } +inline AccessPath AccessPath::computeInScope(SILValue address) { + return AccessPathWithBase::compute(address).accessPath; +} + } // end namespace swift namespace llvm { + /// Allow AccessPath to be used in DenseMap. template <> struct DenseMapInfo { static inline swift::AccessPath getEmptyKey() { @@ -972,8 +1039,64 @@ template <> struct DenseMapInfo { return lhs == rhs; } }; + } // end namespace llvm +//===----------------------------------------------------------------------===// +// MARK: Use visitors +//===----------------------------------------------------------------------===// + +namespace swift { + +/// Interface to the customizable use visitor. +struct AccessUseVisitor { + AccessUseType useTy; + NestedAccessType nestedAccessTy; + + AccessUseVisitor(AccessUseType useTy, NestedAccessType nestedTy) + : useTy(useTy), nestedAccessTy(nestedTy) {} + + virtual ~AccessUseVisitor() {} + + bool findInnerUses() const { return useTy >= AccessUseType::Inner; } + bool findOverlappingUses() const { + return useTy == AccessUseType::Overlapping; + } + + bool visitExactUse(Operand *use) { + return visitUse(use, AccessUseType::Exact); + } + bool visitInnerUse(Operand *use) { + return findInnerUses() ? visitUse(use, AccessUseType::Inner) : true; + } + bool visitOverlappingUse(Operand *use) { + return + findOverlappingUses() ? visitUse(use, AccessUseType::Overlapping) : true; + } + + virtual bool visitUse(Operand *use, AccessUseType useTy) = 0; +}; + +/// Visit all uses of \p storage. +/// +/// Return true if all uses were collected. This is always true as long the \p +/// visitor's visitUse method returns true. +bool visitAccessedStorageUses(AccessUseVisitor &visitor, + AccessedStorage storage, + SILFunction *function); + +/// Visit the uses of \p accessPath. +/// +/// If the storage kind is Global, then function must be non-null (global_addr +/// only occurs inside SILFunction). +/// +/// Return true if all uses were collected. This is always true as long the \p +/// visitor's visitUse method returns true. +bool visitAccessPathUses(AccessUseVisitor &visitor, AccessPath accessPath, + SILFunction *function); + +} // end namespace swift + //===----------------------------------------------------------------------===// // MARK: Helper API for specific formal access patterns //===----------------------------------------------------------------------===// @@ -1031,7 +1154,8 @@ void checkSwitchEnumBlockArg(SILPhiArgument *arg); /// This is not a member of AccessedStorage because it only makes sense to use /// in SILGen before access markers are emitted, or when verifying access /// markers. -bool isPossibleFormalAccessBase(const AccessedStorage &storage, SILFunction *F); +bool isPossibleFormalAccessBase(const AccessedStorage &storage, + SILFunction *F); /// Perform a RAUW operation on begin_access with it's own source operand. /// Then erase the begin_access and all associated end_access instructions. diff --git a/include/swift/SILOptimizer/Analysis/ValueTracking.h b/include/swift/SILOptimizer/Analysis/ValueTracking.h index be2ca665ae6b4..f98d90132e8d4 100644 --- a/include/swift/SILOptimizer/Analysis/ValueTracking.h +++ b/include/swift/SILOptimizer/Analysis/ValueTracking.h @@ -59,7 +59,7 @@ bool pointsToLocalObject(SILValue V); inline bool isUniquelyIdentified(SILValue V) { SILValue objectRef = V; if (V->getType().isAddress()) { - auto storage = findAccessedStorage(V); + auto storage = AccessedStorage::compute(V); if (!storage) return false; diff --git a/lib/IRGen/IRGenSIL.cpp b/lib/IRGen/IRGenSIL.cpp index 690708643afe8..bbf99ab5b65fb 100644 --- a/lib/IRGen/IRGenSIL.cpp +++ b/lib/IRGen/IRGenSIL.cpp @@ -4125,7 +4125,7 @@ void IRGenSILFunction::visitRefTailAddrInst(RefTailAddrInst *i) { } static bool isInvariantAddress(SILValue v) { - SILValue accessedAddress = getAccessAddress(v); + SILValue accessedAddress = getTypedAccessAddress(v); if (auto *ptrRoot = dyn_cast(accessedAddress)) { return ptrRoot->isInvariant(); } diff --git a/lib/SIL/Utils/MemAccessUtils.cpp b/lib/SIL/Utils/MemAccessUtils.cpp index 05470396d48ac..3673f095b6790 100644 --- a/lib/SIL/Utils/MemAccessUtils.cpp +++ b/lib/SIL/Utils/MemAccessUtils.cpp @@ -20,7 +20,7 @@ using namespace swift; //===----------------------------------------------------------------------===// -// MARK: General Helpers +// MARK: FindAccessVisitor //===----------------------------------------------------------------------===// namespace { @@ -146,8 +146,6 @@ class AccessPhiVisitor } }; -enum NestedAccessTy { StopAtAccessBegin, IgnoreAccessBegin }; - // Find the origin of an access while skipping projections and casts and // handling phis. template @@ -155,14 +153,14 @@ class FindAccessVisitorImpl : public AccessUseDefChainVisitor { using SuperTy = AccessUseDefChainVisitor; protected: - NestedAccessTy nestedAccessTy; + NestedAccessType nestedAccessTy; StorageCastTy storageCastTy; SmallPtrSet visitedPhis; bool hasUnknownOffset = false; public: - FindAccessVisitorImpl(NestedAccessTy nestedAccessTy, + FindAccessVisitorImpl(NestedAccessType nestedAccessTy, StorageCastTy storageCastTy) : nestedAccessTy(nestedAccessTy), storageCastTy(storageCastTy) {} @@ -188,7 +186,7 @@ class FindAccessVisitorImpl : public AccessUseDefChainVisitor { // Override AccessUseDefChainVisitor to ignore access markers and find the // outer access base. SILValue visitNestedAccess(BeginAccessInst *access) { - if (nestedAccessTy == IgnoreAccessBegin) + if (nestedAccessTy == NestedAccessType::IgnoreAccessBegin) return access->getSource(); return SuperTy::visitNestedAccess(access); @@ -245,10 +243,10 @@ class FindAccessVisitorImpl : public AccessUseDefChainVisitor { return; } llvm::errs() << "Visiting "; - sourceAddr->dump(); + sourceAddr->print(llvm::errs()); llvm::errs() << " not an address "; - nextAddr->dump(); - nextAddr->getFunction()->dump(); + nextAddr->print(llvm::errs()); + nextAddr->getFunction()->print(llvm::errs()); assert(false); } }; @@ -262,7 +260,7 @@ class FindAccessBaseVisitor Optional base; public: - FindAccessBaseVisitor(NestedAccessTy nestedAccessTy, + FindAccessBaseVisitor(NestedAccessType nestedAccessTy, StorageCastTy storageCastTy) : FindAccessVisitorImpl(nestedAccessTy, storageCastTy) {} @@ -315,10 +313,15 @@ class FindAccessBaseVisitor } // end anonymous namespace -SILValue swift::getAccessAddress(SILValue address) { +//===----------------------------------------------------------------------===// +// MARK: Standalone API +//===----------------------------------------------------------------------===// + +SILValue swift::getTypedAccessAddress(SILValue address) { assert(address->getType().isAddress()); SILValue accessAddress = - FindAccessBaseVisitor(StopAtAccessBegin, StopAtStorageCast) + FindAccessBaseVisitor(NestedAccessType::StopAtAccessBegin, + StopAtStorageCast) .findBase(address); assert(accessAddress->getType().isAddress()); return accessAddress; @@ -327,15 +330,17 @@ SILValue swift::getAccessAddress(SILValue address) { // TODO: When the optimizer stops stripping begin_access markers and SILGen // protects all memory operations with at least an "unsafe" access scope, then // we should be able to assert that this returns a BeginAccessInst. -SILValue swift::getAccessBegin(SILValue address) { +SILValue swift::getAccessScope(SILValue address) { assert(address->getType().isAddress()); - return FindAccessBaseVisitor(StopAtAccessBegin, IgnoreStorageCast) + return FindAccessBaseVisitor(NestedAccessType::StopAtAccessBegin, + IgnoreStorageCast) .findBase(address); } // This is allowed to be called on a non-address pointer type. SILValue swift::getAccessBase(SILValue address) { - return FindAccessBaseVisitor(IgnoreAccessBegin, IgnoreStorageCast) + return FindAccessBaseVisitor(NestedAccessType::IgnoreAccessBegin, + IgnoreStorageCast) .findBase(address); } @@ -357,7 +362,7 @@ bool swift::isLetAddress(SILValue address) { } //===----------------------------------------------------------------------===// -// MARK: FindReferenceRoot +// MARK: FindReferenceRoot //===----------------------------------------------------------------------===// namespace { @@ -432,6 +437,18 @@ static SILValue findReferenceRoot(SILValue ref) { // MARK: AccessedStorage //===----------------------------------------------------------------------===// +SILGlobalVariable *getReferencedGlobal(SILInstruction *inst) { + if (auto *gai = dyn_cast(inst)) { + return gai->getReferencedGlobal(); + } + if (auto apply = FullApplySite::isa(inst)) { + if (auto *funcRef = apply.getReferencedFunctionOrNull()) { + return getVariableOfGlobalInit(funcRef); + } + } + return nullptr; +} + constexpr unsigned AccessedStorage::TailIndex; AccessedStorage::AccessedStorage(SILValue base, Kind kind) { @@ -463,19 +480,13 @@ AccessedStorage::AccessedStorage(SILValue base, Kind kind) { setElementIndex(cast(base)->getIndex()); break; case Global: - if (auto *GAI = dyn_cast(base)) - global = GAI->getReferencedGlobal(); - else { - FullApplySite apply(cast(base)); - auto *funcRef = apply.getReferencedFunctionOrNull(); - assert(funcRef); - global = getVariableOfGlobalInit(funcRef); - assert(global); - // Require a decl for all formally accessed globals defined in this - // module. (Access of globals defined elsewhere has Unidentified storage). - // AccessEnforcementWMO requires this. - assert(global->getDecl()); - } + global = getReferencedGlobal(cast(base)); + // Require a decl for all formally accessed globals defined in this + // module. AccessEnforcementWMO requires this. Swift globals defined in + // another module either use an addressor, which has Unidentified + // storage. Imported non-Swift globals are accessed via global_addr but have + // no declaration. + assert(global->getDecl() || isa(base)); break; case Class: { // Do a best-effort to find the identity of the object being projected @@ -496,6 +507,27 @@ AccessedStorage::AccessedStorage(SILValue base, Kind kind) { } } +void AccessedStorage::visitRoots( + SILFunction *function, + llvm::function_ref visitor) const { + if (SILValue root = getRoot()) { + visitor(root); + return; + } + if (getKind() == Unidentified) { + return; + } + assert(getKind() == Global && function); + SILGlobalVariable *global = getGlobal(); + for (auto &block : *function) { + for (auto &instruction : block) { + if (global == getReferencedGlobal(&instruction)) { + visitor(cast(&instruction)); + } + } + } +} + // Return true if the given access is on a 'let' lvalue. bool AccessedStorage::isLetAccess(SILFunction *F) const { if (auto *decl = dyn_cast_or_null(getDecl())) @@ -633,7 +665,7 @@ class FindAccessedStorageVisitor } public: - FindAccessedStorageVisitor(NestedAccessTy nestedAccessTy) + FindAccessedStorageVisitor(NestedAccessType nestedAccessTy) : FindAccessVisitorImpl(nestedAccessTy, IgnoreStorageCast) {} // Main entry point @@ -676,41 +708,43 @@ class FindAccessedStorageVisitor } // end anonymous namespace -AccessedStorage swift::findAccessedStorage(SILValue sourceAddr) { - FindAccessedStorageVisitor visitor(IgnoreAccessBegin); - visitor.findStorage(sourceAddr); +AccessedStorage AccessedStorage::compute(SILValue sourceAddress) { + FindAccessedStorageVisitor visitor(NestedAccessType::IgnoreAccessBegin); + visitor.findStorage(sourceAddress); return visitor.getStorage(); } -AccessedStorage swift::identifyAccessedStorageImpl(SILValue sourceAddr) { - FindAccessedStorageVisitor visitor(StopAtAccessBegin); - visitor.findStorage(sourceAddr); +AccessedStorage AccessedStorage::computeInScope(SILValue sourceAddress) { + FindAccessedStorageVisitor visitor(NestedAccessType::StopAtAccessBegin); + visitor.findStorage(sourceAddress); return visitor.getStorage(); } //===----------------------------------------------------------------------===// -// AccessPath +// MARK: AccessPath //===----------------------------------------------------------------------===// bool AccessPath::contains(AccessPath subPath) const { - assert(isValid() && subPath.isValid()); - - if (!storage.hasIdenticalBase(subPath.storage)) + if (!isValid() || !subPath.isValid()) { return false; - + } + if (!storage.hasIdenticalBase(subPath.storage)) { + return false; + } // Does the offset index match? - if (offset != subPath.offset || offset == UnknownOffset) + if (offset != subPath.offset || offset == UnknownOffset) { return false; - + } return pathNode.node->isPrefixOf(subPath.pathNode.node); } bool AccessPath::mayOverlap(AccessPath otherPath) const { - assert(isValid() && otherPath.isValid()); + if (!isValid() || !otherPath.isValid()) + return true; - if (storage.isDistinctFrom(otherPath.storage)) + if (storage.isDistinctFrom(otherPath.storage)) { return false; - + } // If subpaths are disjoint, they do not overlap regardless of offset. if (!pathNode.node->isPrefixOf(otherPath.pathNode.node) && !otherPath.pathNode.node->isPrefixOf(pathNode.node)) { @@ -752,9 +786,9 @@ class AccessPathVisitor : public FindAccessVisitorImpl { int pendingOffset = 0; public: - AccessPathVisitor(SILModule *module) - : FindAccessVisitorImpl(IgnoreAccessBegin, IgnoreStorageCast), - module(module), storageVisitor(IgnoreAccessBegin) {} + AccessPathVisitor(SILModule *module, NestedAccessType nestedAccessTy) + : FindAccessVisitorImpl(nestedAccessTy, IgnoreStorageCast), + module(module), storageVisitor(NestedAccessType::IgnoreAccessBegin) {} // Main entry point. AccessPathWithBase findAccessPath(SILValue sourceAddr) && { @@ -862,37 +896,112 @@ class AccessPathVisitor : public FindAccessVisitorImpl { } // end anonymous namespace AccessPathWithBase AccessPathWithBase::compute(SILValue address) { - return AccessPathVisitor(address->getModule()).findAccessPath(address); + return AccessPathVisitor(address->getModule(), + NestedAccessType::IgnoreAccessBegin) + .findAccessPath(address); +} + +AccessPathWithBase AccessPathWithBase::computeInScope(SILValue address) { + return AccessPathVisitor(address->getModule(), + NestedAccessType::StopAtAccessBegin) + .findAccessPath(address); } +void AccessPath::Index::print(raw_ostream &os) const { + if (isSubObjectProjection()) + os << '#' << getSubObjectIndex(); + else { + os << '@'; + if (isUnknownOffset()) + os << "Unknown"; + else + os << getOffset(); + } +} + +LLVM_ATTRIBUTE_USED void AccessPath::Index::dump() const { + print(llvm::dbgs()); +} + +static void recursivelyPrintPath(AccessPath::PathNode node, raw_ostream &os) { + AccessPath::PathNode parent = node.getParent(); + if (!parent.isRoot()) { + recursivelyPrintPath(parent, os); + os << ","; + } + node.getIndex().print(os); +} + +void AccessPath::printPath(raw_ostream &os) const { + os << "Path: "; + if (!isValid()) { + os << "INVALID\n"; + return; + } + os << "("; + PathNode node = getPathNode(); + if (offset != 0) { + Index::forOffset(offset).print(os); + if (!node.isRoot()) + os << ","; + } + if (!node.isRoot()) + recursivelyPrintPath(node, os); + os << ")\n"; +} + +void AccessPath::print(raw_ostream &os) const { + if (!isValid()) { + os << "INVALID\n"; + return; + } + os << "Storage: "; + getStorage().print(os); + printPath(os); +} + +LLVM_ATTRIBUTE_USED void AccessPath::dump() const { print(llvm::dbgs()); } + +void AccessPathWithBase::print(raw_ostream &os) const { + if (base) + os << "Base: " << base; + + accessPath.print(os); +} + +LLVM_ATTRIBUTE_USED void AccessPathWithBase::dump() const { + print(llvm::dbgs()); +} + +//===----------------------------------------------------------------------===// +// MARK: AccessPathDefUseTraversal +//===----------------------------------------------------------------------===// + namespace { // Perform def-use DFS traversal along a given AccessPath. DFS terminates at // each discovered use. // -// If \p collectOverlappingUses is false, then the collected uses all have the -// same AccessPath. Uses that exactly match the AccessPath may either be exact -// uses, or may be subobject projections within that access path, including -// struct_element_addr and tuple_element_addr. The transitive uses of those -// subobject projections are not included. +// For useTy == Exact, the collected uses all have the same AccessPath. +// Subobject projections within that access path and their transitive uses are +// not included. // -// If \p collectOverlappingUses is true, then the collected uses also include -// uses that access an object that contains the given AccessPath. As before, -// overlapping uses do not include transitive uses of subobject projections -// contained by the current path; the def-use traversal stops at those -// projections regardless of collectOverlappingUses. However, overlapping uses -// may be at an unknown offset relative to the current path, so they don't -// necessarily contain the current path. +// For useTy == Inner, the collected uses to subobjects contained by the +// current access path. // -// Example: path = "(#2)" +// For useTy == Overlapping, the collected uses also include uses that +// access an object that contains the given AccessPath as well as uses at +// an unknown offset relative to the current path. +// +// Example, where AccessPath == (#2): // %base = ... // access base // load %base // containing use // %elt1 = struct_element_addr %base, #1 // non-use (ignored) // load %elt1 // non-use (unseen) -// %elt2 = struct_element_addr %base, #2 // chained use (ignored) +// %elt2 = struct_element_addr %base, #2 // outer projection (followed) // load %elt2 // exact use -// %sub = struct_element_addr %elt2, #i // projection use -// load %sub // interior use (ignored) +// %sub = struct_element_addr %elt2, #i // inner projection (followed) +// load %sub // inner use // // A use may be a BranchInst if the corresponding phi does not have common // AccessedStorage. @@ -903,18 +1012,15 @@ namespace { // %ref = ... // reference root // %base = ref_element_addr %refRoot // formal access address // load %base // use -class CollectAccessPathUses { - // Origin of the def-use traversal - AccessedStorage storage; - - // Result: Exact uses, projection uses, and containing uses. - SmallVectorImpl &uses; +class AccessPathDefUseTraversal { + AccessUseVisitor &visitor; - bool collectOverlappingUses; - unsigned useLimit; + // The origin of the def-use traversal. + AccessedStorage storage; - // Access path indices from storage to exact uses - SmallVector pathIndices; // in use-def order + // Remaining access path indices from the most recently visited def to any + // exact use in def-use order. + SmallVector pathIndices; // A point in the def-use traversal. isRef() is true only for object access // prior to reaching the base address. @@ -922,10 +1028,10 @@ class CollectAccessPathUses { // Next potential use to visit and flag indicating whether traversal has // reachaed the access base yet. llvm::PointerIntPair useAndIsRef; - unsigned pathCursor; // position within pathIndices - int offset; // index_addr offsets seen prior to this use + int pathCursor; // position within pathIndices + int offset; // index_addr offsets seen prior to this use - DFSEntry(Operand *use, bool isRef, unsigned pathCursor, int offset) + DFSEntry(Operand *use, bool isRef, int pathCursor, int offset) : useAndIsRef(use, isRef), pathCursor(pathCursor), offset(offset) {} Operand *getUse() const { return useAndIsRef.getPointer(); } @@ -936,57 +1042,31 @@ class CollectAccessPathUses { SmallPtrSet visitedPhis; + // Transient traversal data should not be copied. + AccessPathDefUseTraversal(const AccessPathDefUseTraversal &) = delete; + AccessPathDefUseTraversal & + operator=(const AccessPathDefUseTraversal &) = delete; + public: - CollectAccessPathUses(AccessPath accessPath, SmallVectorImpl &uses, - bool collectOverlappingUses, unsigned useLimit) - : storage(accessPath.getStorage()), uses(uses), - collectOverlappingUses(collectOverlappingUses), useLimit(useLimit) { + AccessPathDefUseTraversal(AccessUseVisitor &visitor, AccessPath accessPath, + SILFunction *function) + : visitor(visitor), storage(accessPath.getStorage()) { assert(accessPath.isValid()); - for (AccessPath::PathNode currentNode = accessPath.getPathNode(); - !currentNode.isRoot(); currentNode = currentNode.getParent()) { - assert(currentNode.getIndex().isSubObjectProjection() && - "a valid AccessPath does not contain any intermediate offsets"); - pathIndices.push_back(currentNode.getIndex()); - } - if (int offset = accessPath.getOffset()) - pathIndices.push_back(AccessPath::Index::forOffset(offset)); - - // The search will start from the object root, not the formal access base, - // so add the class index to the front. - auto storage = accessPath.getStorage(); - if (storage.getKind() == AccessedStorage::Class) { - pathIndices.push_back(AccessPath::Index::forSubObjectProjection( - storage.getPropertyIndex())); - } - if (storage.getKind() == AccessedStorage::Tail) { - pathIndices.push_back(AccessPath::Index::forSubObjectProjection( - ProjectionIndex::TailIndex)); - } - } - // Return true if all uses were collected. This is always true as long as the - // access has a single root, or globalBase is provided, and there is no - // useLimit. - // - // For Global storage \p globalBase must be provided as the head of the - // def-use search. - bool collectUses(SILValue globalBase = SILValue()) && { - SILValue root = storage.getRoot(); - if (!root) { - assert(storage.getKind() == AccessedStorage::Global); - if (!globalBase) - return false; + initializePathIndices(accessPath); - root = globalBase; - } - // If the expected path has an unknown offset, then none of the uses are - // exact. - if (!collectOverlappingUses && !pathIndices.empty() - && pathIndices.back().isUnknownOffset()) { + storage.visitRoots(function, [this](SILValue root) { + initializeDFS(root); return true; + }); + } + + // Return true is all uses have been visited. + bool visitUses() { + // Return false if initialization failed. + if (!storage) { + return false; } - pushUsers(root, - DFSEntry(nullptr, storage.isReference(), pathIndices.size(), 0)); while (!dfsStack.empty()) { if (!visitUser(dfsStack.pop_back_val())) return false; @@ -995,6 +1075,16 @@ class CollectAccessPathUses { } protected: + void initializeDFS(SILValue root) { + // If root is a phi, record it so that its uses aren't visited twice. + if (auto *phi = dyn_cast(root)) { + if (phi->isPhiArgument()) + visitedPhis.insert(phi); + } + pushUsers(root, + DFSEntry(nullptr, storage.isReference(), pathIndices.size(), 0)); + } + void pushUsers(SILValue def, const DFSEntry &dfs) { for (auto *use : def->getUses()) pushUser(DFSEntry(use, dfs.isRef(), dfs.pathCursor, dfs.offset)); @@ -1010,135 +1100,172 @@ class CollectAccessPathUses { dfsStack.emplace_back(dfs); } - // Return true if this phi has been processed and does not need to be - // considered as a separate use. - bool pushPhiUses(const SILPhiArgument *phi, DFSEntry dfs) { - if (!visitedPhis.insert(phi).second) - return true; + bool pushPhiUses(const SILPhiArgument *phi, DFSEntry dfs); - // If this phi has a common base, continue to follow the access path. This - // check is different for reference types vs pointer types. - if (dfs.isRef()) { - assert(!dfs.offset && "index_addr not allowed on reference roots"); - // When isRef is true, the address access hasn't been seen yet and - // we're still following the reference root's users. Check if all phi - // inputs have the same reference root before looking through it. - if (findReferenceRoot(phi) == storage.getObject()) { - pushUsers(phi, dfs); - return true; - } - // The branch will be pushed onto the normal user list. - return false; - } - // Check if all phi inputs have the same accessed storage before - // looking through it. If the phi input differ the its storage is invalid. - auto phiPath = AccessPath::compute(phi); - if (phiPath.isValid()) { - assert(phiPath.getStorage().hasIdenticalBase(storage) - && "inconsistent phi storage"); - // If the phi paths have different offsets, its path has unknown offset. - if (phiPath.getOffset() == AccessPath::UnknownOffset) { - if (!collectOverlappingUses) - return true; - dfs.offset = AccessPath::UnknownOffset; - } - pushUsers(phi, dfs); - return true; - } - // The branch will be pushed onto the normal user list. - return false; - } + void initializePathIndices(AccessPath accessPath); // Return the offset at the current DFS path cursor, or zero. - int getPathOffset(const DFSEntry &dfs) const { - if (dfs.pathCursor == 0 - || pathIndices[dfs.pathCursor - 1].isSubObjectProjection()) { - return 0; - } - return pathIndices[dfs.pathCursor - 1].getOffset(); - } + int getPathOffset(const DFSEntry &dfs) const; - // Returns true as long as the useLimit is not reached. - bool visitUser(DFSEntry dfs) { - Operand *use = dfs.getUse(); - assert(!(dfs.isRef() && use->get()->getType().isAddress())); - if (auto *svi = dyn_cast(use->getUser())) { - if (use->getOperandNumber() == 0 - && visitSingleValueUser(svi, dfs) == IgnoredUse) { - return true; - } - } - // We weren't able to "see through" any more address conversions; so - // record this as a use. + // Return true if the accumulated offset matches the current path index. + // Update the DFSEntry and pathCursor to skip remaining offsets. + bool checkAndUpdateOffset(DFSEntry &dfs); - // Do the path offsets match? - if (!checkAndUpdateOffset(dfs)) - return true; + // Handle non-index_addr projections. + void followProjection(SingleValueInstruction *svi, DFSEntry dfs); - // Is this a full or partial path match? - if (!collectOverlappingUses && dfs.pathCursor > 0) - return true; + enum UseKind { LeafUse, IgnoredUse }; + UseKind visitSingleValueUser(SingleValueInstruction *svi, DFSEntry dfs); - // Record the use if we haven't reached the limit. - if (uses.size() == useLimit) - return false; + // Returns true as long as the visitor returns true. + bool visitUser(DFSEntry dfs); +}; - uses.push_back(use); - return true; - } +} // end anonymous namespace - // Return true if the accumulated offset matches the current path index. - // Update the DFSEntry and pathCursor to skip remaining offsets. - bool checkAndUpdateOffset(DFSEntry &dfs) { - int pathOffset = getPathOffset(dfs); - if (pathOffset == 0) { - // No offset is on the expected path. - if (collectOverlappingUses && dfs.offset == AccessPath::UnknownOffset) { - dfs.offset = 0; - } - return dfs.offset == 0; - } - // pop the offset from the expected path; there should only be one. - --dfs.pathCursor; - assert(getPathOffset(dfs) == 0 && "only one offset index allowed"); +// Initialize the array of remaining path indices. +void AccessPathDefUseTraversal::initializePathIndices(AccessPath accessPath) { + for (AccessPath::PathNode currentNode = accessPath.getPathNode(); + !currentNode.isRoot(); currentNode = currentNode.getParent()) { + assert(currentNode.getIndex().isSubObjectProjection() + && "a valid AccessPath does not contain any intermediate offsets"); + pathIndices.push_back(currentNode.getIndex()); + } + if (int offset = accessPath.getOffset()) { + pathIndices.push_back(AccessPath::Index::forOffset(offset)); + } + // The search will start from the object root, not the formal access base, + // so add the class index to the front. + if (storage.getKind() == AccessedStorage::Class) { + pathIndices.push_back( + AccessPath::Index::forSubObjectProjection(storage.getPropertyIndex())); + } + if (storage.getKind() == AccessedStorage::Tail) { + pathIndices.push_back( + AccessPath::Index::forSubObjectProjection(ProjectionIndex::TailIndex)); + } + // If the expected path has an unknown offset, then none of the uses are + // exact. + if (!visitor.findOverlappingUses() && !pathIndices.empty() + && pathIndices.back().isUnknownOffset()) { + return; + } +} - int useOffset = dfs.offset; - dfs.offset = 0; +// Return true if this phi has been processed and does not need to be +// considered as a separate use. +bool AccessPathDefUseTraversal::pushPhiUses(const SILPhiArgument *phi, + DFSEntry dfs) { + if (!visitedPhis.insert(phi).second) + return true; - // Ignore all uses on this path unless we're collecting containing uses. - // UnknownOffset appears to overlap with all offsets and subobject uses. - if (pathOffset == AccessPath::UnknownOffset - || useOffset == AccessPath::UnknownOffset) { - return collectOverlappingUses; + // If this phi has a common base, continue to follow the access path. This + // check is different for reference types vs pointer types. + if (dfs.isRef()) { + assert(!dfs.offset && "index_addr not allowed on reference roots"); + // When isRef is true, the address access hasn't been seen yet and + // we're still following the reference root's users. Check if all phi + // inputs have the same reference root before looking through it. + if (findReferenceRoot(phi) == storage.getObject()) { + pushUsers(phi, dfs); + return true; } - // A known offset must match regardless of collectOverlappingUses. - return pathOffset == useOffset; + // The branch will be pushed onto the normal user list. + return false; } + // Check if all phi inputs have the same accessed storage before + // looking through it. If the phi input differ the its storage is invalid. + auto phiPath = AccessPath::compute(phi); + if (phiPath.isValid()) { + assert(phiPath.getStorage().hasIdenticalBase(storage) + && "inconsistent phi storage"); + // If the phi paths have different offsets, its path has unknown offset. + if (phiPath.getOffset() == AccessPath::UnknownOffset) { + if (!visitor.findOverlappingUses()) + return true; + dfs.offset = AccessPath::UnknownOffset; + } + pushUsers(phi, dfs); + return true; + } + // The branch will be pushed onto the normal user list. + return false; +} - enum UseKind { LeafUse, IgnoredUse }; - UseKind visitSingleValueUser(SingleValueInstruction *svi, DFSEntry dfs); - - // Handle non-index_addr projections. - UseKind followProjection(SingleValueInstruction *svi, DFSEntry dfs) { - if (!checkAndUpdateOffset(dfs)) - return IgnoredUse; - - if (dfs.pathCursor == 0) - return LeafUse; +// Return the offset at the current DFS path cursor, or zero. +int AccessPathDefUseTraversal::getPathOffset(const DFSEntry &dfs) const { + if (dfs.pathCursor <= 0 + || pathIndices[dfs.pathCursor - 1].isSubObjectProjection()) { + return 0; + } + return pathIndices[dfs.pathCursor - 1].getOffset(); +} - AccessPath::Index pathIndex = pathIndices[dfs.pathCursor - 1]; - auto projIdx = ProjectionIndex(svi); - assert(projIdx.isValid()); - // Only subobjects indices are expected because offsets are handled above. - if (projIdx.Index == pathIndex.getSubObjectIndex()) { +// Return true if the accumulated offset matches the current path index. +// Update the DFSEntry and pathCursor to skip remaining offsets. +bool AccessPathDefUseTraversal::checkAndUpdateOffset(DFSEntry &dfs) { + int pathOffset = getPathOffset(dfs); + if (dfs.offset == AccessPath::UnknownOffset) { + if (pathOffset > 0) { + // Pop the offset from the expected path; there should only be + // one. Continue matching subobject indices even after seeing an unknown + // offset. A subsequent mismatching subobject index is still considered + // non-overlapping. This is valid for aliasing since an offset from a + // subobject is considered an invalid access path. --dfs.pathCursor; - pushUsers(svi, dfs); + assert(getPathOffset(dfs) == 0 && "only one offset index allowed"); } - return IgnoredUse; - } -}; + // Continue searching only if we need to find overlapping uses. Preserve the + // unknown dfs offset so we don't consider any dependent operations to be + // exact or inner uses. + return visitor.findOverlappingUses(); + } + if (pathOffset == 0) { + return dfs.offset == 0; + } + // pop the offset from the expected path; there should only be one. + --dfs.pathCursor; + assert(getPathOffset(dfs) == 0 && "only one offset index allowed"); + + // Ignore all uses on this path unless we're collecting containing uses. + // UnknownOffset appears to overlap with all offsets and subobject uses. + if (pathOffset == AccessPath::UnknownOffset) { + // Set the dfs offset to unknown to avoid considering any dependent + // operations as exact or inner uses. + dfs.offset = AccessPath::UnknownOffset; + return visitor.findOverlappingUses(); + } + int useOffset = dfs.offset; + dfs.offset = 0; + // A known offset must match regardless of findOverlappingUses. + return pathOffset == useOffset; +} -} // end anonymous namespace +// Handle non-index_addr projections. +void AccessPathDefUseTraversal::followProjection(SingleValueInstruction *svi, + DFSEntry dfs) { + if (!checkAndUpdateOffset(dfs)) { + return; + } + if (dfs.pathCursor <= 0) { + if (visitor.useTy == AccessUseType::Exact) { + assert(dfs.pathCursor == 0); + return; + } + --dfs.pathCursor; + pushUsers(svi, dfs); + return; + } + AccessPath::Index pathIndex = pathIndices[dfs.pathCursor - 1]; + auto projIdx = ProjectionIndex(svi); + assert(projIdx.isValid()); + // Only subobjects indices are expected because offsets are handled above. + if (projIdx.Index == pathIndex.getSubObjectIndex()) { + --dfs.pathCursor; + pushUsers(svi, dfs); + } + return; +} // During the def-use traversal, visit a single-value instruction in which the // used address is at operand zero. @@ -1151,9 +1278,9 @@ class CollectAccessPathUses { // // FIXME: Reuse getAccessProjectionOperand() instead of using special cases once // the unchecked_take_enum_data_addr -> load -> project_box pattern is fixed. -CollectAccessPathUses::UseKind -CollectAccessPathUses::visitSingleValueUser(SingleValueInstruction *svi, - DFSEntry dfs) { +AccessPathDefUseTraversal::UseKind +AccessPathDefUseTraversal::visitSingleValueUser(SingleValueInstruction *svi, + DFSEntry dfs) { if (isAccessedStorageCast(svi)) { pushUsers(svi, dfs); return IgnoredUse; @@ -1163,6 +1290,9 @@ CollectAccessPathUses::visitSingleValueUser(SingleValueInstruction *svi, return LeafUse; case SILInstructionKind::BeginAccessInst: + if (visitor.nestedAccessTy == NestedAccessType::StopAtAccessBegin) { + return LeafUse; + } pushUsers(svi, dfs); return IgnoredUse; @@ -1172,7 +1302,8 @@ CollectAccessPathUses::visitSingleValueUser(SingleValueInstruction *svi, assert(dfs.isRef()); assert(dfs.pathCursor > 0 && "ref_element_addr cannot occur within access"); dfs.useAndIsRef.setInt(false); - return followProjection(svi, dfs); + followProjection(svi, dfs); + return IgnoredUse; case SILInstructionKind::RefTailAddrInst: { assert(dfs.isRef()); @@ -1191,7 +1322,8 @@ CollectAccessPathUses::visitSingleValueUser(SingleValueInstruction *svi, case SILInstructionKind::StructElementAddrInst: case SILInstructionKind::TupleElementAddrInst: - return followProjection(svi, dfs); + followProjection(svi, dfs); + return IgnoredUse; case SILInstructionKind::IndexAddrInst: case SILInstructionKind::TailAddrInst: { @@ -1200,8 +1332,8 @@ CollectAccessPathUses::visitSingleValueUser(SingleValueInstruction *svi, if (dfs.offset != AccessPath::UnknownOffset) dfs.offset += projIdx.Index; else - assert(collectOverlappingUses); - } else if (collectOverlappingUses) { + assert(visitor.findOverlappingUses()); + } else if (visitor.findOverlappingUses()) { dfs.offset = AccessPath::UnknownOffset; } else { return IgnoredUse; @@ -1248,91 +1380,71 @@ CollectAccessPathUses::visitSingleValueUser(SingleValueInstruction *svi, } } -bool AccessPath::collectUses(SmallVectorImpl &uses, - bool collectOverlappingUses, - unsigned useLimit) const { - return CollectAccessPathUses(*this, uses, collectOverlappingUses, useLimit) - .collectUses(); -} - -bool AccessPathWithBase::collectUses(SmallVectorImpl &uses, - bool collectOverlappingUses, - unsigned useLimit) const { - CollectAccessPathUses collector(accessPath, uses, collectOverlappingUses, - useLimit); - if (accessPath.getRoot()) - return std::move(collector).collectUses(); - - if (!base) - return false; - - return std::move(collector).collectUses(base); -} - -void AccessPath::Index::print(raw_ostream &os) const { - if (isSubObjectProjection()) - os << '#' << getSubObjectIndex(); - else { - os << '@'; - if (isUnknownOffset()) - os << "Unknown"; - else - os << getOffset(); +bool AccessPathDefUseTraversal::visitUser(DFSEntry dfs) { + Operand *use = dfs.getUse(); + assert(!(dfs.isRef() && use->get()->getType().isAddress())); + if (auto *svi = dyn_cast(use->getUser())) { + if (use->getOperandNumber() == 0 + && visitSingleValueUser(svi, dfs) == IgnoredUse) { + return true; + } } -} + // We weren't able to "see through" any more address conversions; so + // record this as a use. -LLVM_ATTRIBUTE_USED void AccessPath::Index::dump() const { - print(llvm::dbgs()); -} + // Do the path offsets match? + if (!checkAndUpdateOffset(dfs)) + return true; -static void recursivelyPrintPath(AccessPath::PathNode node, raw_ostream &os) { - AccessPath::PathNode parent = node.getParent(); - if (!parent.isRoot()) { - recursivelyPrintPath(parent, os); - os << ","; + // Is this a partial path match? + if (dfs.pathCursor > 0 || dfs.offset == AccessPath::UnknownOffset) { + return visitor.visitOverlappingUse(use); } - node.getIndex().print(os); + if (dfs.pathCursor < 0) { + return visitor.visitInnerUse(use); + } + return visitor.visitExactUse(use); } -void AccessPath::printPath(raw_ostream &os) const { - os << "Path: "; - if (!isValid()) { - os << "INVALID\n"; - return; - } - os << "("; - PathNode node = getPathNode(); - if (offset != 0) { - Index::forOffset(offset).print(os); - if (!node.isRoot()) - os << ","; - } - if (!node.isRoot()) - recursivelyPrintPath(node, os); - os << ")\n"; +bool swift::visitAccessPathUses(AccessUseVisitor &visitor, + AccessPath accessPath, SILFunction *function) { + return AccessPathDefUseTraversal(visitor, accessPath, function).visitUses(); } -void AccessPath::print(raw_ostream &os) const { - if (!isValid()) { - os << "INVALID\n"; - return; - } - os << "Storage: "; - getStorage().print(os); - printPath(os); +bool swift::visitAccessedStorageUses(AccessUseVisitor &visitor, + AccessedStorage storage, + SILFunction *function) { + IndexTrieNode *emptyPath = function->getModule().getIndexTrieRoot(); + return visitAccessPathUses(visitor, AccessPath(storage, emptyPath, 0), + function); } -LLVM_ATTRIBUTE_USED void AccessPath::dump() const { print(llvm::dbgs()); } +class CollectAccessPathUses : public AccessUseVisitor { + // Result: Exact uses, projection uses, and containing uses. + SmallVectorImpl &uses; -void AccessPathWithBase::print(raw_ostream &os) const { - if (base) - os << "Base: " << base; + unsigned useLimit; - accessPath.print(os); -} +public: + CollectAccessPathUses(SmallVectorImpl &uses, AccessUseType useTy, + unsigned useLimit) + : AccessUseVisitor(useTy, NestedAccessType::IgnoreAccessBegin), uses(uses), + useLimit(useLimit) {} -LLVM_ATTRIBUTE_USED void AccessPathWithBase::dump() const { - print(llvm::dbgs()); + bool visitUse(Operand *use, AccessUseType useTy) { + if (uses.size() == useLimit) { + return false; + } + uses.push_back(use); + return true; + } +}; + +bool AccessPath::collectUses(SmallVectorImpl &uses, + AccessUseType useTy, SILFunction *function, + unsigned useLimit) const { + CollectAccessPathUses collector(uses, useTy, useLimit); + return visitAccessPathUses(collector, *this, function); } //===----------------------------------------------------------------------===// @@ -1566,7 +1678,7 @@ SILBasicBlock::iterator swift::removeBeginAccess(BeginAccessInst *beginAccess) { } //===----------------------------------------------------------------------===// -// Verification +// MARK: Verification //===----------------------------------------------------------------------===// // Helper for visitApplyAccesses that visits address-type call arguments, diff --git a/lib/SILGen/SILGenLValue.cpp b/lib/SILGen/SILGenLValue.cpp index 443433584ca2f..fb2e6f0952347 100644 --- a/lib/SILGen/SILGenLValue.cpp +++ b/lib/SILGen/SILGenLValue.cpp @@ -603,7 +603,7 @@ SILValue UnenforcedAccess::beginAccess(SILGenFunction &SGF, SILLocation loc, if (!SGF.getOptions().VerifyExclusivity) return address; - const AccessedStorage &storage = findAccessedStorage(address); + auto storage = AccessedStorage::compute(address); // Unsafe access may have invalid storage (e.g. a RawPointer). if (storage && !isPossibleFormalAccessBase(storage, &SGF.F)) return address; diff --git a/lib/SILOptimizer/Analysis/AccessedStorageAnalysis.cpp b/lib/SILOptimizer/Analysis/AccessedStorageAnalysis.cpp index 90caebc1b3310..64a80796a3d19 100644 --- a/lib/SILOptimizer/Analysis/AccessedStorageAnalysis.cpp +++ b/lib/SILOptimizer/Analysis/AccessedStorageAnalysis.cpp @@ -256,8 +256,7 @@ transformCalleeStorage(const StorageAccessInfo &storage, SILValue argVal = getCallerArg(fullApply, storage.getParamIndex()); if (argVal) { // Remap the argument source value and inherit the old storage info. - auto calleeStorage = findAccessedStorage(argVal); - if (calleeStorage) + if (auto calleeStorage = AccessedStorage::compute(argVal)) return StorageAccessInfo(calleeStorage, storage); } // If the argument can't be transformed, demote it to an unidentified @@ -265,7 +264,7 @@ transformCalleeStorage(const StorageAccessInfo &storage, // // This is an untested bailout. It is only reachable if the call graph // contains an edge that getCallerArg is unable to analyze OR if - // findAccessedStorage returns an invalid SILValue, which won't + // AccessedStorage::compute returns an invalid SILValue, which won't // pass SIL verification. // // FIXME: In case argVal is invalid, support Unidentified access for invalid @@ -301,8 +300,7 @@ void AccessedStorageResult::visitBeginAccess(B *beginAccess) { if (beginAccess->getEnforcement() != SILAccessEnforcement::Dynamic) return; - const AccessedStorage &storage = - findAccessedStorage(beginAccess->getSource()); + auto storage = AccessedStorage::compute(beginAccess->getSource()); if (storage.getKind() == AccessedStorage::Unidentified) { // This also catches invalid storage. diff --git a/lib/SILOptimizer/Analysis/AliasAnalysis.cpp b/lib/SILOptimizer/Analysis/AliasAnalysis.cpp index 44c40297e4b2c..6e47743ba3310 100644 --- a/lib/SILOptimizer/Analysis/AliasAnalysis.cpp +++ b/lib/SILOptimizer/Analysis/AliasAnalysis.cpp @@ -334,7 +334,7 @@ static bool isAccessedAddressTBAASafe(SILValue V) { if (!V->getType().isAddress()) return false; - SILValue accessedAddress = getAccessAddress(V); + SILValue accessedAddress = getTypedAccessAddress(V); if (isa(accessedAddress)) return true; diff --git a/lib/SILOptimizer/Analysis/MemoryBehavior.cpp b/lib/SILOptimizer/Analysis/MemoryBehavior.cpp index 376c9a220313d..4c810fed69b0a 100644 --- a/lib/SILOptimizer/Analysis/MemoryBehavior.cpp +++ b/lib/SILOptimizer/Analysis/MemoryBehavior.cpp @@ -84,7 +84,8 @@ class MemoryBehaviorVisitor /// If 'V' is an address, then the returned value is also an address. SILValue getValueAddress() { if (!cachedValueAddress) { - cachedValueAddress = V->getType().isAddress() ? getAccessAddress(V) : V; + cachedValueAddress = + V->getType().isAddress() ? getTypedAccessAddress(V) : V; } return cachedValueAddress; } diff --git a/lib/SILOptimizer/LoopTransforms/LICM.cpp b/lib/SILOptimizer/LoopTransforms/LICM.cpp index 6eb5e30a7f988..bf489fe892e13 100644 --- a/lib/SILOptimizer/LoopTransforms/LICM.cpp +++ b/lib/SILOptimizer/LoopTransforms/LICM.cpp @@ -700,18 +700,18 @@ static bool analyzeBeginAccess(BeginAccessInst *BI, InstSet &SideEffectInsts, AccessedStorageAnalysis *ASA, DominanceInfo *DT) { - const AccessedStorage &storage = findAccessedStorage(BI->getSource()); + auto storage = AccessedStorage::compute(BI->getSource()); if (!storage) { return false; } - auto BIAccessedStorageNonNested = findAccessedStorage(BI); + auto BIAccessedStorageNonNested = AccessedStorage::compute(BI); auto safeBeginPred = [&](BeginAccessInst *OtherBI) { if (BI == OtherBI) { return true; } return BIAccessedStorageNonNested.isDistinctFrom( - findAccessedStorage(OtherBI)); + AccessedStorage::compute(OtherBI)); }; if (!std::all_of(BeginAccesses.begin(), BeginAccesses.end(), safeBeginPred)) diff --git a/lib/SILOptimizer/Mandatory/DiagnoseStaticExclusivity.cpp b/lib/SILOptimizer/Mandatory/DiagnoseStaticExclusivity.cpp index 674036973164d..f9e68af362c47 100644 --- a/lib/SILOptimizer/Mandatory/DiagnoseStaticExclusivity.cpp +++ b/lib/SILOptimizer/Mandatory/DiagnoseStaticExclusivity.cpp @@ -711,7 +711,8 @@ checkAccessSummary(ApplySite Apply, AccessState &State, // A valid AccessedStorage should always be found because Unsafe accesses // are not tracked by AccessSummaryAnalysis. - const AccessedStorage &Storage = identifyCapturedStorage(Argument); + auto Storage = AccessedStorage::computeInScope(Argument); + assert(Storage && "captured address must have valid storage"); auto AccessIt = State.Accesses->find(Storage); // Are there any accesses in progress at the time of the call? @@ -745,7 +746,8 @@ static void checkCaptureAccess(ApplySite Apply, AccessState &State) { // A valid AccessedStorage should always be found because Unsafe accesses // are not tracked by AccessSummaryAnalysis. - const AccessedStorage &Storage = identifyCapturedStorage(argOper.get()); + auto Storage = AccessedStorage::computeInScope(argOper.get()); + assert(Storage && "captured address must have valid storage"); // Are there any accesses in progress at the time of the call? auto AccessIt = State.Accesses->find(Storage); @@ -964,7 +966,7 @@ static void checkStaticExclusivity(SILFunction &Fn, PostOrderFunctionInfo *PO, // Check that the given address-type operand is guarded by begin/end access // markers. static void checkAccessedAddress(Operand *memOper, StorageMap &Accesses) { - SILValue accessBegin = getAccessBegin(memOper->get()); + SILValue accessBegin = getAccessScope(memOper->get()); SILInstruction *memInst = memOper->getUser(); auto error = [accessBegin, memInst]() { @@ -1017,13 +1019,12 @@ static void checkAccessedAddress(Operand *memOper, StorageMap &Accesses) { return; } - const AccessedStorage &storage = findAccessedStorage(accessBegin); - // findAccessedStorage may return an invalid storage object if the address - // producer is not recognized by its allowlist. For the purpose of - // verification, we assume that this can only happen for local - // initialization, not a formal memory access. The strength of - // verification rests on the completeness of the opcode list inside - // findAccessedStorage. + auto storage = AccessedStorage::compute(accessBegin); + // AccessedStorage::compute may return an invalid storage object if the + // address producer is not recognized by its allowlist. For the purpose of + // verification, we assume that this can only happen for local initialization, + // not a formal memory access. The strength of verification rests on the + // completeness of the opcode list inside AccessedStorage::compute. // // For the purpose of verification, an unidentified access is // unenforced. These occur in cases like global addressors and local buffers diff --git a/lib/SILOptimizer/Transforms/AccessEnforcementDom.cpp b/lib/SILOptimizer/Transforms/AccessEnforcementDom.cpp index 06fefeeac81c9..dd20d41da8e58 100644 --- a/lib/SILOptimizer/Transforms/AccessEnforcementDom.cpp +++ b/lib/SILOptimizer/Transforms/AccessEnforcementDom.cpp @@ -251,7 +251,7 @@ void DominatedAccessAnalysis::analyzeAccess(BeginAccessInst *BAI, // Only track dynamic access in the result. Static accesses still need to be // tracked by data flow, but they can't be optimized as "dominating". if (BAI->getEnforcement() == SILAccessEnforcement::Dynamic) { - AccessedStorage storage = findAccessedStorage(BAI->getSource()); + auto storage = AccessedStorage::compute(BAI->getSource()); // Copy the AccessStorage into DomAccessedStorage. All pass-specific bits // are initialized to zero. domStorage = DomAccessedStorage(storage); diff --git a/lib/SILOptimizer/Transforms/AccessEnforcementOpts.cpp b/lib/SILOptimizer/Transforms/AccessEnforcementOpts.cpp index 88e4ed0588e96..c0011d148b2e3 100644 --- a/lib/SILOptimizer/Transforms/AccessEnforcementOpts.cpp +++ b/lib/SILOptimizer/Transforms/AccessEnforcementOpts.cpp @@ -566,7 +566,7 @@ bool AccessConflictAndMergeAnalysis::identifyBeginAccesses() { // now, since this optimization runs at the end of the pipeline, we // gracefully ignore unrecognized source address patterns, which show up // here as an invalid `storage` value. - AccessedStorage storage = findAccessedStorage(beginAccess->getSource()); + auto storage = AccessedStorage::compute(beginAccess->getSource()); auto iterAndInserted = storageSet.insert(storage); diff --git a/lib/SILOptimizer/Transforms/AccessEnforcementWMO.cpp b/lib/SILOptimizer/Transforms/AccessEnforcementWMO.cpp index 36c82f0f676cc..9a16f7e22dac5 100644 --- a/lib/SILOptimizer/Transforms/AccessEnforcementWMO.cpp +++ b/lib/SILOptimizer/Transforms/AccessEnforcementWMO.cpp @@ -169,13 +169,13 @@ void GlobalAccessRemoval::perform() { void GlobalAccessRemoval::visitInstruction(SILInstruction *I) { if (auto *BAI = dyn_cast(I)) { - AccessedStorage storage = findAccessedStorage(BAI->getSource()); + auto storage = AccessedStorage::compute(BAI->getSource()); const VarDecl *decl = getDisjointAccessLocation(storage); recordAccess(BAI, decl, storage.getKind(), BAI->hasNoNestedConflict()); return; } if (auto *BUAI = dyn_cast(I)) { - AccessedStorage storage = findAccessedStorage(BUAI->getSource()); + auto storage = AccessedStorage::compute(BUAI->getSource()); const VarDecl *decl = getDisjointAccessLocation(storage); recordAccess(BUAI, decl, storage.getKind(), BUAI->hasNoNestedConflict()); return; diff --git a/lib/SILOptimizer/Transforms/PerformanceInliner.cpp b/lib/SILOptimizer/Transforms/PerformanceInliner.cpp index 077b851c167ac..435751fcf20e2 100644 --- a/lib/SILOptimizer/Transforms/PerformanceInliner.cpp +++ b/lib/SILOptimizer/Transforms/PerformanceInliner.cpp @@ -447,7 +447,7 @@ bool SILPerformanceInliner::isProfitableToInline( // The access is dynamic and has no nested conflict // See if the storage location is considered by // access enforcement optimizations - AccessedStorage storage = findAccessedStorage(BAI->getSource()); + auto storage = AccessedStorage::compute(BAI->getSource()); if (BAI->hasNoNestedConflict() && (storage.isFormalAccessBase())) { BlockW.updateBenefit(ExclusivityBenefitWeight, ExclusivityBenefitBase); diff --git a/lib/SILOptimizer/UtilityPasses/AccessPathVerification.cpp b/lib/SILOptimizer/UtilityPasses/AccessPathVerification.cpp index 261115d957ab7..b575dc2bd6c16 100644 --- a/lib/SILOptimizer/UtilityPasses/AccessPathVerification.cpp +++ b/lib/SILOptimizer/UtilityPasses/AccessPathVerification.cpp @@ -47,8 +47,7 @@ class AccessPathVerification : public SILModuleTransform { public: void verifyAccessPath(Operand *operand) { - auto pathAndBase = AccessPathWithBase::compute(operand->get()); - auto accessPath = pathAndBase.accessPath; + auto accessPath = AccessPath::compute(operand->get()); if (!accessPath.isValid()) return; @@ -70,28 +69,19 @@ class AccessPathVerification : public SILModuleTransform { } // This is a new path, so map all its uses. assert(uses.empty()); - pathAndBase.collectUses(uses, /*collectContainingUses*/ false); + accessPath.collectUses(uses, AccessUseType::Exact, + operand->getParentFunction()); bool foundOperandUse = false; for (Operand *use : uses) { if (use == operand) { foundOperandUse = true; continue; } - // (live) subobject projections within an access will be mapped later as a - // separate path. - switch (use->getUser()->getKind()) { - default: - break; - case SILInstructionKind::StructElementAddrInst: - case SILInstructionKind::TupleElementAddrInst: - case SILInstructionKind::IndexAddrInst: - continue; - } auto iterAndInserted = useToPathMap.try_emplace(use, accessPath); if (!iterAndInserted.second) { llvm::errs() << "Address use: " << *operand->getUser() << " with path...\n"; - pathAndBase.dump(); + accessPath.dump(); llvm::errs() << " was not collected for: " << *use->getUser(); llvm::errs() << " with path...\n"; auto computedPath = iterAndInserted.first->second; diff --git a/lib/SILOptimizer/UtilityPasses/AccessedStorageDumper.cpp b/lib/SILOptimizer/UtilityPasses/AccessedStorageDumper.cpp index 31685d7a1c6bf..f1bbb45d640b0 100644 --- a/lib/SILOptimizer/UtilityPasses/AccessedStorageDumper.cpp +++ b/lib/SILOptimizer/UtilityPasses/AccessedStorageDumper.cpp @@ -12,6 +12,7 @@ #define DEBUG_TYPE "sil-accessed-storage-dumper" #include "swift/SIL/MemAccessUtils.h" +#include "swift/SIL/PrettyStackTrace.h" #include "swift/SIL/SILFunction.h" #include "swift/SIL/SILInstruction.h" #include "swift/SIL/SILValue.h" @@ -33,33 +34,49 @@ class AccessedStorageDumper : public SILModuleTransform { llvm::SmallVector uses; void dumpAccessedStorage(Operand *operand) { - findAccessedStorage(operand->get()).print(llvm::outs()); + SILFunction *function = operand->getParentFunction(); + // Print storage itself first, for comparison against AccessPath. They can + // differ in rare cases of unidentified storage with phis. + AccessedStorage::compute(operand->get()).print(llvm::outs()); + // Now print the access path and base. auto pathAndBase = AccessPathWithBase::compute(operand->get()); pathAndBase.print(llvm::outs()); - - if (!pathAndBase.accessPath.isValid() || !EnableDumpUses) + // If enable-accessed-storage-dump-uses is set, dump all types of uses. + auto accessPath = pathAndBase.accessPath; + if (!accessPath.isValid() || !EnableDumpUses) return; uses.clear(); - pathAndBase.collectUses(uses, /*collectContainingUses*/ false); + accessPath.collectUses(uses, AccessUseType::Exact, function); llvm::outs() << "Exact Uses {\n"; for (auto *useOperand : uses) { llvm::outs() << *useOperand->getUser() << " "; - auto usePathAndBase = AccessPathWithBase::compute(useOperand->get()); - usePathAndBase.accessPath.printPath(llvm::outs()); - assert(pathAndBase.accessPath.contains(usePathAndBase.accessPath) + auto usePath = AccessPath::compute(useOperand->get()); + usePath.printPath(llvm::outs()); + assert(accessPath == usePath + && "access path does not match use access path"); + } + llvm::outs() << "}\n"; + uses.clear(); + accessPath.collectUses(uses, AccessUseType::Inner, function); + llvm::outs() << "Inner Uses {\n"; + for (auto *useOperand : uses) { + llvm::outs() << *useOperand->getUser() << " "; + auto usePath = AccessPath::compute(useOperand->get()); + usePath.printPath(llvm::outs()); + assert(accessPath.contains(usePath) && "access path does not contain use access path"); } llvm::outs() << "}\n"; uses.clear(); - pathAndBase.collectUses(uses, /*collectContainingUses*/ true); + accessPath.collectUses(uses, AccessUseType::Overlapping, function); llvm::outs() << "Overlapping Uses {\n"; for (auto *useOperand : uses) { llvm::outs() << *useOperand->getUser() << " "; - auto usePathAndBase = AccessPathWithBase::compute(useOperand->get()); - usePathAndBase.accessPath.printPath(llvm::outs()); - assert(pathAndBase.accessPath.mayOverlap(usePathAndBase.accessPath) - && "access path does not contain use access path"); + auto usePath = AccessPath::compute(useOperand->get()); + usePath.printPath(llvm::outs()); + assert(accessPath.mayOverlap(usePath) + && "access path does not overlap with use access path"); } llvm::outs() << "}\n"; } @@ -71,6 +88,7 @@ class AccessedStorageDumper : public SILModuleTransform { llvm::outs() << "\n"; continue; } + PrettyStackTraceSILFunction functionDumper("...", &fn); for (auto &bb : fn) { for (auto &inst : bb) { if (inst.mayReadOrWriteMemory()) { diff --git a/test/SILOptimizer/accesspath_uses.sil b/test/SILOptimizer/accesspath_uses.sil index 883f68314e1d1..73bf391e593c9 100644 --- a/test/SILOptimizer/accesspath_uses.sil +++ b/test/SILOptimizer/accesspath_uses.sil @@ -14,6 +14,55 @@ struct MyStruct { @_hasStorage @_hasInitialValue var j: Int64 { get set } } +// CHECK-LABEL: @testInnerUse +// CHECK: ###For MemOp: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct +// CHECK: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Storage: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Path: () +// CHECK: Exact Uses { +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct +// CHECK-NEXT: Path: () +// CHECK-NEXT: } +// CHECK: Inner Uses { +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct +// CHECK-NEXT: Path: () +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK-NEXT: Path: (#0) +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct +// CHECK-NEXT: Path: () +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK-NEXT: Path: (#0) +// CHECK: } +// CHECK: ###For MemOp: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Storage: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Path: (#0) +// CHECK: Exact Uses { +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK-NEXT: Path: (#0) +// CHECK-NEXT: } +// CHECK: Inner Uses { +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK-NEXT: Path: (#0) +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct +// CHECK-NEXT: Path: () +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK-NEXT: Path: (#0) +// CHECK: } +sil [serialized] [ossa] @testInnerUse : $@convention(thin) (Builtin.RawPointer, Builtin.Word) -> () { +bb0(%0 : $Builtin.RawPointer, %1 : $Builtin.Word): + %2 = pointer_to_address %0 : $Builtin.RawPointer to [strict] $*MyStruct + %3 = load [trivial] %2 : $*MyStruct + %4 = struct_element_addr %2 : $*MyStruct, #MyStruct.i + %5 = load [trivial] %4 : $*Int64 + %6 = tuple () + return %6 : $() +} + // unknown offset contains subobject indices // CHECK-LABEL: @testDynamicIndexWithSubObject // CHECK: ###For MemOp: %{{.*}} = load [trivial] %{{.*}} : $*Int64 @@ -22,6 +71,10 @@ struct MyStruct { // CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*Int64 // CHECK-NEXT: Path: (#0) // CHECK-NEXT: } +// CHECK: Inner Uses { +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK-NEXT: Path: (#0) +// CHECK-NEXT: } // CHECK: Overlapping Uses { // CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*Int64 // CHECK-NEXT: Path: (#0) @@ -32,9 +85,11 @@ struct MyStruct { // CHECK: Path: (@Unknown) // CHECK-NEXT: Exact Uses { // CHECK-NEXT: } +// CHECK-NEXT: Inner Uses { +// CHECK-NEXT: } // CHECK-NEXT: Overlapping Uses { -// CHECK-NEXT: %{{.*}} = struct_element_addr %{{.*}} : $*MyStruct, #MyStruct.i -// CHECK-NEXT: Path: () +// CHECK-NEXT: %{{.*}} = load [trivial] %3 : $*Int64 +// CHECK-NEXT: Path: (#0) // CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct // CHECK-NEXT: Path: (@Unknown) // CHECK-NEXT: } @@ -49,28 +104,69 @@ bb0(%0 : $Builtin.RawPointer, %1 : $Builtin.Word): return %7 : $() } +// The index load should be reported as an overlapping uses, not an +// exact or inner use. +// CHECK: ###For MemOp: %3 = load [trivial] %2 : $*MyStruct +// CHECK: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Base: %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Storage: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: Path: () +// CHECK: Exact Uses { +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct +// CHECK-NEXT: Path: () +// CHECK-NEXT: } +// CHECK-NEXT: Inner Uses { +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct +// CHECK-NEXT: Path: () +// CHECK-NEXT: } +// CHECK-NEXT: Overlapping Uses { +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct +// CHECK-NEXT: Path: () +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK-NEXT: Path: INVALID +// CHECK-NEXT: } +// CHECK: ###For MemOp: %{{.*}} = load [trivial] %{{.*}} : $*Int64 +// CHECK: Argument %0 = argument of bb0 : $Builtin.RawPointer +// CHECK: INVALID +sil [serialized] [ossa] @testDynamicIndexInsideSubObject : $@convention(thin) (Builtin.RawPointer, Builtin.Word) -> () { +bb0(%0 : $Builtin.RawPointer, %1 : $Builtin.Word): + %2 = pointer_to_address %0 : $Builtin.RawPointer to [strict] $*MyStruct + %3 = load [trivial] %2 : $*MyStruct + %4 = struct_element_addr %2 : $*MyStruct, #MyStruct.i + %5 = index_addr %4 : $*Int64, %1 : $Builtin.Word + %6 = load [trivial] %5 : $*Int64 + %7 = tuple () + return %7 : $() +} + // An unknown offset contains known offsets. // CHECK-LABEL: @testDynamicIndexWithStaticIndex -// CHECK: ###For MemOp: %5 = load [trivial] %4 : $*MyStruct +// CHECK: ###For MemOp: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct // CHECK: Path: (@1) // CHECK: Exact Uses { -// CHECK-NEXT: %5 = load [trivial] %4 : $*MyStruct +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct +// CHECK-NEXT: Path: (@1) +// CHECK-NEXT: } +// CHECK: Inner Uses { +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct // CHECK-NEXT: Path: (@1) // CHECK-NEXT: } // CHECK: Overlapping Uses { -// CHECK-NEXT: %5 = load [trivial] %4 : $*MyStruct +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct // CHECK-NEXT: Path: (@1) -// CHECK-NEXT: %7 = load [trivial] %6 : $*MyStruct +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct // CHECK-NEXT: Path: (@Unknown) // CHECK-NEXT: } -// CHECK: ###For MemOp: %7 = load [trivial] %6 : $*MyStruct +// CHECK: ###For MemOp: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct // CHECK: Path: (@Unknown) // CHECK: Exact Uses { // CHECK-NEXT: } +// CHECK: Inner Uses { +// CHECK-NEXT: } // CHECK: Overlapping Uses { -// CHECK-NEXT: %5 = load [trivial] %4 : $*MyStruct +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct // CHECK-NEXT: Path: (@1) -// CHECK-NEXT: %7 = load [trivial] %6 : $*MyStruct +// CHECK-NEXT: %{{.*}} = load [trivial] %{{.*}} : $*MyStruct // CHECK-NEXT: Path: (@Unknown) // CHECK-NEXT: } sil [serialized] [ossa] @testDynamicIndexWithStaticIndex : $@convention(thin) (Builtin.RawPointer, Builtin.Word) -> () { @@ -93,7 +189,7 @@ bb0(%0 : $Builtin.RawPointer, %1 : $Builtin.Word): // CHECK-NEXT: %3 = load [trivial] %2 : $*MyStruct // CHECK-NEXT: Path: () // CHECK-NEXT: } -// CHECK-NEXT: Overlapping Uses { +// CHECK: Overlapping Uses { // CHECK-NEXT: %3 = load [trivial] %2 : $*MyStruct // CHECK-NEXT: Path: () // CHECK-NEXT: %8 = load [trivial] %7 : $*MyStruct @@ -117,6 +213,8 @@ bb0(%0 : $Builtin.RawPointer, %1 : $Builtin.Word): // CHECK: Path: (@Unknown) // CHECK: Exact Uses { // CHECK-NEXT: } +// CHECK: Inner Uses { +// CHECK-NEXT: } // CHECK: Overlapping Uses { // CHECK-NEXT: %3 = load [trivial] %2 : $*MyStruct // CHECK-NEXT: Path: () @@ -304,6 +402,13 @@ bb3(%15 : $Builtin.RawPointer): return %17 : $AnyObject } +enum IntTEnum { + indirect case int(Int) + case other(T) + + var getValue: Int {get } +} + // CHECK-LABEL: @testEnumUses // CHECK: ###For MemOp: copy_addr %0 to [initialization] %2 : $*IntTEnum // CHECK: Storage: Argument %0 = argument of bb0 : $*IntTEnum @@ -457,13 +562,6 @@ bb3(%15 : $Builtin.RawPointer): // CHECK: %8 = load [trivial] %7 : $*Int // CHECK: %13 = unchecked_take_enum_data_addr %2 : $*IntTEnum, #IntTEnum.other!enumelt // CHECK: } -enum IntTEnum { - indirect case int(Int) - case other(T) - - var getValue: Int {get } -} - sil hidden [ossa] @testEnumUses : $@convention(method) (@in_guaranteed IntTEnum) -> Int { bb0(%0 : $*IntTEnum): debug_value_addr %0 : $*IntTEnum, let, name "self", argno 1 @@ -530,3 +628,81 @@ bb0(%0 : @guaranteed $Storage): end_access %2 : $*UInt return %8 : $Int } + +// CHECK-LABEL: @testBeginAccessUses +// CHECK: ###For MemOp: copy_addr [take] %1 to [initialization] %{{.*}} : $*T +// CHECK: Storage: Argument %1 = argument of bb0 : $*T +// CHECK: Path: () +// CHECK: Exact Uses { +// CHECK-NEXT: copy_addr [take] %1 to [initialization] %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: copy_addr [take] %1 to [initialization] %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: } +// CHECK: Storage: Stack %{{.*}} = alloc_stack $T, var, name "$value" +// CHECK: Path: () +// CHECK: Exact Uses { +// CHECK-NEXT: copy_addr [take] %1 to [initialization] %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: copy_addr %{{.*}} to [initialization] %0 : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: end_access %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: destroy_addr %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: dealloc_stack %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: copy_addr [take] %1 to [initialization] %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: copy_addr %{{.*}} to [initialization] %0 : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: end_access %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: destroy_addr %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: dealloc_stack %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: } +// CHECK: ###For MemOp: copy_addr %{{.*}} to [initialization] %0 : $*T +// CHECK: Storage: Stack %{{.*}} = alloc_stack $T, var, name "$value" +// CHECK: Path: () +// CHECK: Exact Uses { +// CHECK-NEXT: copy_addr [take] %1 to [initialization] %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: copy_addr %{{.*}} to [initialization] %0 : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: end_access %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: destroy_addr %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: dealloc_stack %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: } +// CHECK: Overlapping Uses { +// CHECK-NEXT: copy_addr [take] %1 to [initialization] %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: copy_addr %{{.*}} to [initialization] %0 : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: end_access %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: destroy_addr %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: dealloc_stack %{{.*}} : $*T +// CHECK-NEXT: Path: () +// CHECK-NEXT: } +sil [ossa] @testBeginAccessUses : $@convention(thin) (@in T) -> @out T { +bb0(%0 : $*T, %1 : $*T): + %2 = alloc_stack $T, var, name "$value" + copy_addr [take] %1 to [initialization] %2 : $*T + %4 = begin_access [modify] [static] %2 : $*T + copy_addr %4 to [initialization] %0 : $*T + end_access %4 : $*T + destroy_addr %2 : $*T + dealloc_stack %2 : $*T + %10 = tuple () + return %10 : $() +} diff --git a/test/SILOptimizer/licm.sil b/test/SILOptimizer/licm.sil index 9ac8ae2492cba..dcc8871611f91 100644 --- a/test/SILOptimizer/licm.sil +++ b/test/SILOptimizer/licm.sil @@ -756,7 +756,7 @@ bb3: } // ----------------------------------------------------------------------------- -// Reduced test case from rdar !!! +// Reduced test case from rdar://61246061 // // Test miscompilation of BidirectionalCollection._distance with // combined load/store hoisting/sinking with mutiple loads from From b272dc5e1a8357c7d882027e6bf4a7136de265ad Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Fri, 25 Sep 2020 23:43:13 -0700 Subject: [PATCH 17/20] Cache 'isLet' within AccessedStorage. Compute 'isLet' from the VarDecl that is available when constructing AccessedStorage so we don't need to recover the VarDecl for the base later. This generally makes more sense and is more efficient, but it will be necessary when we look past class casts when finding the reference root. --- include/swift/SIL/MemAccessUtils.h | 24 ++++++++----- include/swift/SIL/SILInstruction.h | 8 ++--- lib/SIL/IR/SILInstructions.cpp | 22 ++++++++---- lib/SIL/Utils/MemAccessUtils.cpp | 36 +++++++++++-------- .../LoadBorrowInvalidationChecker.cpp | 4 +-- .../SemanticARC/LoadCopyToLoadBorrowOpt.cpp | 2 +- 6 files changed, 57 insertions(+), 39 deletions(-) diff --git a/include/swift/SIL/MemAccessUtils.h b/include/swift/SIL/MemAccessUtils.h index a1b302710acd1..ba9ee554cd0bf 100644 --- a/include/swift/SIL/MemAccessUtils.h +++ b/include/swift/SIL/MemAccessUtils.h @@ -303,11 +303,11 @@ class AccessedStorage { protected: // Checking the storage kind is far more common than other fields. Make sure // it can be byte load with no shift. - static const int ReservedKindBits = 8; + static const int ReservedKindBits = 7; static_assert(ReservedKindBits >= NumKindBits, "Too many storage kinds."); static const unsigned InvalidElementIndex = - (1 << (32 - ReservedKindBits)) - 1; + (1 << (32 - (ReservedKindBits + 1))) - 1; // Form a bitfield that is effectively a union over any pass-specific data // with the fields used within this class as a common prefix. @@ -327,9 +327,10 @@ class AccessedStorage { // elementIndex can overflow while gracefully degrading analysis. For now, // reserve an absurd number of bits at a nice alignment boundary, but this // can be reduced. - SWIFT_INLINE_BITFIELD_BASE(AccessedStorage, 32, kind - : ReservedKindBits, - elementIndex : 32 - ReservedKindBits); + SWIFT_INLINE_BITFIELD_BASE(AccessedStorage, 32, + kind : ReservedKindBits, + isLet : 1, + elementIndex : 32 - (ReservedKindBits + 1)); // Define bits for use in AccessedStorageAnalysis. Each identified storage // object is mapped to one instance of this subclass. @@ -512,7 +513,7 @@ class AccessedStorage { } /// Return true if the given access is on a 'let' lvalue. - bool isLetAccess(SILFunction *F) const; + bool isLetAccess() const { return Bits.AccessedStorage.isLet; } /// If this is a uniquely identified formal access, then it cannot /// alias with any other uniquely identified access to different storage. @@ -555,9 +556,12 @@ class AccessedStorage { /// Returns the ValueDecl for the underlying storage, if it can be /// determined. Otherwise returns null. /// - /// WARNING: This is not a constant-time operation. It is for diagnostics and - /// checking via the ValueDecl if we are processing a `let` variable. - const ValueDecl *getDecl() const; + /// If \p base is provided, then it must be the accessed base for this + /// storage, as passed to the AccessedStorage constructor. What \p base is + /// provided, this is guaranteed to return a valid decl for class properties; + /// otherwise it is only a best effort based on the type of the object root + /// *before* the object is cast to the final accessed reference type. + const ValueDecl *getDecl(SILValue base = SILValue()) const; /// Get all leaf uses of all address, pointer, or box values that have a this /// AccessedStorage in common. Return true if all uses were found before @@ -629,6 +633,8 @@ class AccessedStorage { // nested/argument access. return false; } + + void setLetAccess(SILValue base); }; } // end namespace swift diff --git a/include/swift/SIL/SILInstruction.h b/include/swift/SIL/SILInstruction.h index c336ed5ab5d8a..913cb273e92a2 100644 --- a/include/swift/SIL/SILInstruction.h +++ b/include/swift/SIL/SILInstruction.h @@ -5822,13 +5822,11 @@ class TupleElementAddrInst /// object, including properties declared in a superclass. unsigned getFieldIndex(NominalTypeDecl *decl, VarDecl *property); -/// Get the property for a struct or class by its unique index. +/// Get the property for a struct or class by its unique index, or nullptr if +/// the index does not match a property declared in this struct or class or +/// one its superclasses. /// /// Precondition: \p decl must be a non-resilient struct or class. -/// -/// Precondition: \p index must be the index of a stored property -/// (as returned by getFieldIndex()) which is declared -/// in \p decl, not in a superclass. VarDecl *getIndexedField(NominalTypeDecl *decl, unsigned index); /// A common base for instructions that require a cached field index. diff --git a/lib/SIL/IR/SILInstructions.cpp b/lib/SIL/IR/SILInstructions.cpp index 5655e95219b8a..a752947b2f85f 100644 --- a/lib/SIL/IR/SILInstructions.cpp +++ b/lib/SIL/IR/SILInstructions.cpp @@ -1345,15 +1345,23 @@ unsigned swift::getFieldIndex(NominalTypeDecl *decl, VarDecl *field) { /// Get the property for a struct or class by its unique index. VarDecl *swift::getIndexedField(NominalTypeDecl *decl, unsigned index) { - if (auto *classDecl = dyn_cast(decl)) { - for (auto *superDecl = classDecl->getSuperclassDecl(); superDecl != nullptr; - superDecl = superDecl->getSuperclassDecl()) { - assert(index >= superDecl->getStoredProperties().size() - && "field index cannot refer to a superclass field"); - index -= superDecl->getStoredProperties().size(); + if (auto *structDecl = dyn_cast(decl)) { + return structDecl->getStoredProperties()[index]; + } + auto *classDecl = cast(decl); + SmallVector superclasses; + for (auto *superDecl = classDecl; superDecl != nullptr; + superDecl = superDecl->getSuperclassDecl()) { + superclasses.push_back(superDecl); + } + std::reverse(superclasses.begin(), superclasses.end()); + for (auto *superDecl : superclasses) { + if (index < superDecl->getStoredProperties().size()) { + return superDecl->getStoredProperties()[index]; } + index -= superDecl->getStoredProperties().size(); } - return decl->getStoredProperties()[index]; + return nullptr; } unsigned FieldIndexCacheBase::cacheFieldIndex() { diff --git a/lib/SIL/Utils/MemAccessUtils.cpp b/lib/SIL/Utils/MemAccessUtils.cpp index 3673f095b6790..e94accc2585d4 100644 --- a/lib/SIL/Utils/MemAccessUtils.cpp +++ b/lib/SIL/Utils/MemAccessUtils.cpp @@ -452,9 +452,9 @@ SILGlobalVariable *getReferencedGlobal(SILInstruction *inst) { constexpr unsigned AccessedStorage::TailIndex; AccessedStorage::AccessedStorage(SILValue base, Kind kind) { + // For kind==Unidentified, base may be an invalid empty or tombstone value. assert(base && "invalid storage base"); initKind(kind); - switch (kind) { case Box: assert(isa(base)); @@ -505,6 +505,7 @@ AccessedStorage::AccessedStorage(SILValue base, Kind kind) { break; } } + setLetAccess(base); } void AccessedStorage::visitRoots( @@ -528,21 +529,24 @@ void AccessedStorage::visitRoots( } } -// Return true if the given access is on a 'let' lvalue. -bool AccessedStorage::isLetAccess(SILFunction *F) const { - if (auto *decl = dyn_cast_or_null(getDecl())) - return decl->isLet(); - +// Set 'isLet' to true if this storage can be determined to be a 'let' variable. +// +// \p base must be the access base for this storage, as passed to the +// AccessedStorage constructor. +void AccessedStorage::setLetAccess(SILValue base) { // It's unclear whether a global will ever be missing it's varDecl, but // technically we only preserve it for debug info. So if we don't have a decl, // check the flag on SILGlobalVariable, which is guaranteed valid, - if (getKind() == AccessedStorage::Global) - return getGlobal()->isLet(); - - return false; + if (getKind() == AccessedStorage::Global) { + Bits.AccessedStorage.isLet = getGlobal()->isLet(); + return; + } + if (auto *decl = dyn_cast_or_null(getDecl(base))) { + Bits.AccessedStorage.isLet = decl->isLet(); + } } -const ValueDecl *AccessedStorage::getDecl() const { +const ValueDecl *AccessedStorage::getDecl(SILValue base) const { switch (getKind()) { case Box: return cast(value)->getLoc().getAsASTNode(); @@ -555,7 +559,7 @@ const ValueDecl *AccessedStorage::getDecl() const { case Class: { auto *decl = getObject()->getType().getNominalOrBoundGenericNominal(); - return getIndexedField(decl, getPropertyIndex()); + return decl ? getIndexedField(decl, getPropertyIndex()) : nullptr; } case Tail: return nullptr; @@ -621,8 +625,10 @@ void AccessedStorage::print(raw_ostream &os) const { break; case Class: os << getObject(); - os << " Field: "; - getDecl()->print(os); + if (auto *decl = getDecl()) { + os << " Field: "; + decl->print(os); + } os << " Index: " << getPropertyIndex() << "\n"; break; case Tail: @@ -1655,7 +1661,7 @@ bool swift::isPossibleFormalAccessBase(const AccessedStorage &storage, // Additional checks that apply to anything that may fall through. // Immutable values are only accessed for initialization. - if (storage.isLetAccess(F)) + if (storage.isLetAccess()) return false; return true; diff --git a/lib/SIL/Verifier/LoadBorrowInvalidationChecker.cpp b/lib/SIL/Verifier/LoadBorrowInvalidationChecker.cpp index b5aea5103bc67..fe549173fffdf 100644 --- a/lib/SIL/Verifier/LoadBorrowInvalidationChecker.cpp +++ b/lib/SIL/Verifier/LoadBorrowInvalidationChecker.cpp @@ -466,9 +466,9 @@ bool LoadBorrowNeverInvalidatedAnalysis::isNeverInvalidated( } // If we have a let address, then we are already done. - if (storage.isLetAccess(lbi->getFunction())) + if (storage.isLetAccess()) { return true; - + } // At this point, we know that we /may/ have writes. Now we go through various // cases to try and exhaustively identify if those writes overlap with our // load_borrow. diff --git a/lib/SILOptimizer/SemanticARC/LoadCopyToLoadBorrowOpt.cpp b/lib/SILOptimizer/SemanticARC/LoadCopyToLoadBorrowOpt.cpp index 38d6304030741..71323e34eac8f 100644 --- a/lib/SILOptimizer/SemanticARC/LoadCopyToLoadBorrowOpt.cpp +++ b/lib/SILOptimizer/SemanticARC/LoadCopyToLoadBorrowOpt.cpp @@ -193,7 +193,7 @@ class StorageGuaranteesLoadVisitor void visitGlobalAccess(SILValue global) { return answer( - !AccessedStorage(global, AccessedStorage::Global).isLetAccess(&ctx.fn)); + !AccessedStorage(global, AccessedStorage::Global).isLetAccess()); } void visitClassAccess(RefElementAddrInst *field) { From f6b32aedcd0afb9962645d1ae7a8b33cf35c5998 Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Sun, 27 Sep 2020 22:16:28 -0700 Subject: [PATCH 18/20] Add AccessedStorageWithBase to conviently recover the base's VarDecl --- include/swift/SIL/MemAccessUtils.h | 24 +++++++++++- lib/SIL/Utils/MemAccessUtils.cpp | 18 +++++++-- .../LoadBorrowInvalidationChecker.cpp | 4 +- .../Transforms/AccessEnforcementWMO.cpp | 39 ++++++++++--------- 4 files changed, 59 insertions(+), 26 deletions(-) diff --git a/include/swift/SIL/MemAccessUtils.h b/include/swift/SIL/MemAccessUtils.h index ba9ee554cd0bf..54f9ce67f3fdb 100644 --- a/include/swift/SIL/MemAccessUtils.h +++ b/include/swift/SIL/MemAccessUtils.h @@ -507,7 +507,7 @@ class AccessedStorage { llvm_unreachable("unhandled kind"); } - /// Return trye if the given access is guaranteed to be within a heap object. + /// Return true if the given access is guaranteed to be within a heap object. bool isObjectAccess() const { return getKind() == Class || getKind() == Tail; } @@ -696,6 +696,28 @@ template <> struct DenseMapInfo { namespace swift { +/// For convenience, encapsulate and AccessedStorage value along with its +/// accessed base address. +struct AccessedStorageWithBase { + AccessedStorage storage; + // The base of the formal access. For class storage, it is the + // ref_element_addr. For global storage it is the global_addr or initializer + // apply. For other storage, it is the same as accessPath.getRoot(). + // + // Base may be invalid for global_addr -> address_to_pointer -> phi patterns. + // FIXME: add a structural requirement to SIL so base is always valid in OSSA. + SILValue base; + + AccessedStorageWithBase(AccessedStorage storage, SILValue base) + : storage(storage), base(base) {} + + /// Identical to AccessedStorage::compute but preserves the access base. + static AccessedStorageWithBase compute(SILValue sourceAddress); + + /// Identical to AccessedStorage::computeInScope but preserves the base. + static AccessedStorageWithBase computeInScope(SILValue sourceAddress); +}; + /// Return an AccessedStorage value that identifies formally accessed storage /// for \p beginAccess, considering any outer access scope as having distinct /// storage from this access scope. This is useful for exclusivity checking diff --git a/lib/SIL/Utils/MemAccessUtils.cpp b/lib/SIL/Utils/MemAccessUtils.cpp index e94accc2585d4..95770fdf049bc 100644 --- a/lib/SIL/Utils/MemAccessUtils.cpp +++ b/lib/SIL/Utils/MemAccessUtils.cpp @@ -714,16 +714,26 @@ class FindAccessedStorageVisitor } // end anonymous namespace -AccessedStorage AccessedStorage::compute(SILValue sourceAddress) { +AccessedStorageWithBase +AccessedStorageWithBase::compute(SILValue sourceAddress) { FindAccessedStorageVisitor visitor(NestedAccessType::IgnoreAccessBegin); visitor.findStorage(sourceAddress); - return visitor.getStorage(); + return {visitor.getStorage(), visitor.getBase()}; } -AccessedStorage AccessedStorage::computeInScope(SILValue sourceAddress) { +AccessedStorageWithBase +AccessedStorageWithBase::computeInScope(SILValue sourceAddress) { FindAccessedStorageVisitor visitor(NestedAccessType::StopAtAccessBegin); visitor.findStorage(sourceAddress); - return visitor.getStorage(); + return {visitor.getStorage(), visitor.getBase()}; +} + +AccessedStorage AccessedStorage::compute(SILValue sourceAddress) { + return AccessedStorageWithBase::compute(sourceAddress).storage; +} + +AccessedStorage AccessedStorage::computeInScope(SILValue sourceAddress) { + return AccessedStorageWithBase::computeInScope(sourceAddress).storage; } //===----------------------------------------------------------------------===// diff --git a/lib/SIL/Verifier/LoadBorrowInvalidationChecker.cpp b/lib/SIL/Verifier/LoadBorrowInvalidationChecker.cpp index fe549173fffdf..6aea48c2fecac 100644 --- a/lib/SIL/Verifier/LoadBorrowInvalidationChecker.cpp +++ b/lib/SIL/Verifier/LoadBorrowInvalidationChecker.cpp @@ -454,11 +454,11 @@ bool LoadBorrowNeverInvalidatedAnalysis:: bool LoadBorrowNeverInvalidatedAnalysis::isNeverInvalidated( LoadBorrowInst *lbi) { - SILValue address = getAccessBegin(lbi->getOperand()); + SILValue address = getAccessScope(lbi->getOperand()); if (!address) return false; - auto storage = findAccessedStorage(address); + auto storage = AccessedStorage::compute(address); // If we couldn't find an access storage, return that we are assumed to write. if (!storage) { llvm::errs() << "Couldn't compute access storage?!\n"; diff --git a/lib/SILOptimizer/Transforms/AccessEnforcementWMO.cpp b/lib/SILOptimizer/Transforms/AccessEnforcementWMO.cpp index 9a16f7e22dac5..e8db5bc90fc90 100644 --- a/lib/SILOptimizer/Transforms/AccessEnforcementWMO.cpp +++ b/lib/SILOptimizer/Transforms/AccessEnforcementWMO.cpp @@ -67,22 +67,21 @@ using llvm::DenseMap; using llvm::SmallDenseSet; // Get the VarDecl that represents the DisjointAccessLocation for the given -// AccessedStorage. Returns nullptr for any storage that can't be partitioned -// into a disjoint location. +// storage and access base. Returns nullptr for any storage that can't be +// partitioned into a disjoint location. // -// identifyFormalAccess may only return Unidentified storage for a global -// variable access if the global is defined in a different module. -// -// WARNING: Retrieving VarDecl for Class access is not constant time. -const VarDecl *getDisjointAccessLocation(const AccessedStorage &storage) { +// Global storage is expected to be disjoint because identifyFormalAccess may +// only return Unidentified storage for a global variable access if the global +// is defined in a different module. +const VarDecl * +getDisjointAccessLocation(AccessedStorageWithBase storageAndBase) { + auto storage = storageAndBase.storage; switch (storage.getKind()) { case AccessedStorage::Global: - // A global variable may return a null decl. These variables are - // implementation details that aren't formally accessed. - return storage.getGlobal()->getDecl(); - case AccessedStorage::Class: { - return cast(storage.getDecl()); - } + case AccessedStorage::Class: + // Class and Globals are always a VarDecl, but the global decl may have a + // null value for global_addr -> phi. + return cast_or_null(storage.getDecl(storageAndBase.base)); case AccessedStorage::Box: case AccessedStorage::Stack: case AccessedStorage::Tail: @@ -169,15 +168,17 @@ void GlobalAccessRemoval::perform() { void GlobalAccessRemoval::visitInstruction(SILInstruction *I) { if (auto *BAI = dyn_cast(I)) { - auto storage = AccessedStorage::compute(BAI->getSource()); - const VarDecl *decl = getDisjointAccessLocation(storage); - recordAccess(BAI, decl, storage.getKind(), BAI->hasNoNestedConflict()); + auto storageAndBase = AccessedStorageWithBase::compute(BAI->getSource()); + const VarDecl *decl = getDisjointAccessLocation(storageAndBase); + recordAccess(BAI, decl, storageAndBase.storage.getKind(), + BAI->hasNoNestedConflict()); return; } if (auto *BUAI = dyn_cast(I)) { - auto storage = AccessedStorage::compute(BUAI->getSource()); - const VarDecl *decl = getDisjointAccessLocation(storage); - recordAccess(BUAI, decl, storage.getKind(), BUAI->hasNoNestedConflict()); + auto storageAndBase = AccessedStorageWithBase::compute(BUAI->getSource()); + const VarDecl *decl = getDisjointAccessLocation(storageAndBase); + recordAccess(BUAI, decl, storageAndBase.storage.getKind(), + BUAI->hasNoNestedConflict()); return; } if (auto *KPI = dyn_cast(I)) { From 53eebddbb2f63d3f141667ee4195d3e91478a179 Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Wed, 14 Oct 2020 00:09:02 -0700 Subject: [PATCH 19/20] Temporarily disable the load-borrow checker. A rewrite is ready and will be merged ASAP. --- lib/SIL/Verifier/LoadBorrowInvalidationChecker.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/SIL/Verifier/LoadBorrowInvalidationChecker.cpp b/lib/SIL/Verifier/LoadBorrowInvalidationChecker.cpp index 6aea48c2fecac..149c0e5d2aaa3 100644 --- a/lib/SIL/Verifier/LoadBorrowInvalidationChecker.cpp +++ b/lib/SIL/Verifier/LoadBorrowInvalidationChecker.cpp @@ -454,6 +454,9 @@ bool LoadBorrowNeverInvalidatedAnalysis:: bool LoadBorrowNeverInvalidatedAnalysis::isNeverInvalidated( LoadBorrowInst *lbi) { + // FIXME: To be reenabled separately in a follow-on commit. + return true; + SILValue address = getAccessScope(lbi->getOperand()); if (!address) return false; From 5e0c8f9b50d5d7b40f2fa1db19363ae1db6b1b4d Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Fri, 16 Oct 2020 13:14:15 -0700 Subject: [PATCH 20/20] The verifier should only output to llvm::errs() --- .../UtilityPasses/AccessPathVerification.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/SILOptimizer/UtilityPasses/AccessPathVerification.cpp b/lib/SILOptimizer/UtilityPasses/AccessPathVerification.cpp index b575dc2bd6c16..27c9ad9e61851 100644 --- a/lib/SILOptimizer/UtilityPasses/AccessPathVerification.cpp +++ b/lib/SILOptimizer/UtilityPasses/AccessPathVerification.cpp @@ -59,10 +59,10 @@ class AccessPathVerification : public SILModuleTransform { if (collectedFromPath != accessPath) { llvm::errs() << "Address use: " << *operand->getUser() << " collected from path\n "; - collectedFromPath.dump(); + collectedFromPath.print(llvm::errs()); llvm::errs() << " has different path\n "; - accessPath.dump(); - operand->getUser()->getFunction()->dump(); + accessPath.print(llvm::errs()); + operand->getUser()->getFunction()->print(llvm::errs()); assert(false && "computed path does not match collected path"); } return; @@ -81,19 +81,19 @@ class AccessPathVerification : public SILModuleTransform { if (!iterAndInserted.second) { llvm::errs() << "Address use: " << *operand->getUser() << " with path...\n"; - accessPath.dump(); + accessPath.print(llvm::errs()); llvm::errs() << " was not collected for: " << *use->getUser(); llvm::errs() << " with path...\n"; auto computedPath = iterAndInserted.first->second; - computedPath.dump(); - use->getUser()->getFunction()->dump(); + computedPath.print(llvm::errs()); + use->getUser()->getFunction()->print(llvm::errs()); assert(false && "missing collected use"); } } if (!foundOperandUse && !accessPath.hasUnknownOffset()) { llvm::errs() << "Address use: " << *operand->getUser() << " is not a use of path\n "; - accessPath.dump(); + accessPath.print(llvm::errs()); assert(false && "not a user of its own computed path "); } uses.clear();