From 9a8b0411448062e7231a0ee26bd14a3d1c097e9e Mon Sep 17 00:00:00 2001 From: Nathan James Date: Mon, 22 Jun 2020 18:25:44 +0100 Subject: [PATCH 01/13] [clang-tidy] llvm-twine-local ignores parameters Ignore paramater declarations of type `::llvm::Twine`, These don't suffer the same use after free risks as local twines. Reviewed By: aaron.ballman Differential Revision: https://reviews.llvm.org/D82281 --- clang-tools-extra/clang-tidy/llvm/TwineLocalCheck.cpp | 6 ++++-- .../test/clang-tidy/checkers/llvm-twine-local.cpp | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/clang-tools-extra/clang-tidy/llvm/TwineLocalCheck.cpp b/clang-tools-extra/clang-tidy/llvm/TwineLocalCheck.cpp index 63a9314a49e5e..907bdcb264bf3 100644 --- a/clang-tools-extra/clang-tidy/llvm/TwineLocalCheck.cpp +++ b/clang-tools-extra/clang-tidy/llvm/TwineLocalCheck.cpp @@ -19,8 +19,10 @@ namespace llvm_check { void TwineLocalCheck::registerMatchers(MatchFinder *Finder) { auto TwineType = - qualType(hasDeclaration(recordDecl(hasName("::llvm::Twine")))); - Finder->addMatcher(varDecl(hasType(TwineType)).bind("variable"), this); + qualType(hasDeclaration(cxxRecordDecl(hasName("::llvm::Twine")))); + Finder->addMatcher( + varDecl(unless(parmVarDecl()), hasType(TwineType)).bind("variable"), + this); } void TwineLocalCheck::check(const MatchFinder::MatchResult &Result) { diff --git a/clang-tools-extra/test/clang-tidy/checkers/llvm-twine-local.cpp b/clang-tools-extra/test/clang-tidy/checkers/llvm-twine-local.cpp index 06eb7613d439d..3dcf6abe0c22e 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/llvm-twine-local.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/llvm-twine-local.cpp @@ -13,6 +13,7 @@ class Twine { using namespace llvm; void foo(const Twine &x); +void bar(Twine x); static Twine Moo = Twine("bark") + "bah"; // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: twine variables are prone to use-after-free bugs From 6ae0f5f3e1d465e6a663a50f2cc077671bc6d097 Mon Sep 17 00:00:00 2001 From: Nathan James Date: Mon, 22 Jun 2020 18:26:17 +0100 Subject: [PATCH 02/13] [clang-tidy] RenamerClangTidy wont emit fixes in scratch space Prevent fixes being displayed if usages are found in the scratch buffer. See [[ https://bugs.llvm.org/show_bug.cgi?id=46219 | Fix-It hints are being generated in the ScratchBuffer ]]. It may be wise down the line to put in a general fix in clang-tidy to prevent ScratchBuffer replacements being applied, but for now this will help. Reviewed By: aaron.ballman Differential Revision: https://reviews.llvm.org/D82162 --- .../utils/RenamerClangTidyCheck.cpp | 39 ++++++++++++------- .../readability-identifier-naming.cpp | 16 ++++++++ 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp index e90ab1782b4b4..040378d980f1a 100644 --- a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp +++ b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp @@ -156,14 +156,17 @@ void RenamerClangTidyCheck::addUsage( // is already in there RenamerClangTidyCheck::NamingCheckFailure &Failure = NamingCheckFailures[Decl]; - if (!Failure.RawUsageLocs.insert(FixLocation.getRawEncoding()).second) - return; if (!Failure.ShouldFix()) return; + if (SourceMgr && SourceMgr->isWrittenInScratchSpace(FixLocation)) + Failure.FixStatus = RenamerClangTidyCheck::ShouldFixStatus::InsideMacro; + if (!utils::rangeCanBeFixed(Range, SourceMgr)) Failure.FixStatus = RenamerClangTidyCheck::ShouldFixStatus::InsideMacro; + + Failure.RawUsageLocs.insert(FixLocation.getRawEncoding()); } void RenamerClangTidyCheck::addUsage(const NamedDecl *Decl, SourceRange Range, @@ -248,13 +251,15 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) { if (const auto *Decl = Result.Nodes.getNodeAs("classRef")) { - addUsage(Decl->getParent(), Decl->getNameInfo().getSourceRange()); + addUsage(Decl->getParent(), Decl->getNameInfo().getSourceRange(), + Result.SourceManager); for (const auto *Init : Decl->inits()) { if (!Init->isWritten() || Init->isInClassMemberInitializer()) continue; if (const FieldDecl *FD = Init->getAnyMember()) - addUsage(FD, SourceRange(Init->getMemberLocation())); + addUsage(FD, SourceRange(Init->getMemberLocation()), + Result.SourceManager); // Note: delegating constructors and base class initializers are handled // via the "typeLoc" matcher. } @@ -271,7 +276,7 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) { // we want instead to replace the next token, that will be the identifier. Range.setBegin(CharSourceRange::getTokenRange(Range).getEnd()); - addUsage(Decl->getParent(), Range); + addUsage(Decl->getParent(), Range, Result.SourceManager); return; } @@ -289,7 +294,7 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) { // further TypeLocs handled below if (Decl) { - addUsage(Decl, Loc->getSourceRange()); + addUsage(Decl, Loc->getSourceRange(), Result.SourceManager); return; } @@ -300,7 +305,7 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) { SourceRange Range(Ref.getTemplateNameLoc(), Ref.getTemplateNameLoc()); if (const auto *ClassDecl = dyn_cast(Decl)) { if (const NamedDecl *TemplDecl = ClassDecl->getTemplatedDecl()) - addUsage(TemplDecl, Range); + addUsage(TemplDecl, Range, Result.SourceManager); return; } } @@ -308,7 +313,7 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) { if (const auto &Ref = Loc->getAs()) { if (const TagDecl *Decl = Ref.getTypePtr()->getAsTagDecl()) - addUsage(Decl, Loc->getSourceRange()); + addUsage(Decl, Loc->getSourceRange(), Result.SourceManager); return; } } @@ -317,7 +322,7 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) { Result.Nodes.getNodeAs("nestedNameLoc")) { if (const NestedNameSpecifier *Spec = Loc->getNestedNameSpecifier()) { if (const NamespaceDecl *Decl = Spec->getAsNamespace()) { - addUsage(Decl, Loc->getLocalSourceRange()); + addUsage(Decl, Loc->getLocalSourceRange(), Result.SourceManager); return; } } @@ -325,7 +330,8 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) { if (const auto *Decl = Result.Nodes.getNodeAs("using")) { for (const auto *Shadow : Decl->shadows()) - addUsage(Shadow->getTargetDecl(), Decl->getNameInfo().getSourceRange()); + addUsage(Shadow->getTargetDecl(), Decl->getNameInfo().getSourceRange(), + Result.SourceManager); return; } @@ -371,14 +377,14 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) { // Fix using namespace declarations. if (const auto *UsingNS = dyn_cast(Decl)) addUsage(UsingNS->getNominatedNamespaceAsWritten(), - UsingNS->getIdentLocation()); + UsingNS->getIdentLocation(), Result.SourceManager); if (!Decl->getIdentifier() || Decl->getName().empty() || Decl->isImplicit()) return; const auto *Canonical = cast(Decl->getCanonicalDecl()); if (Canonical != Decl) { - addUsage(Canonical, Decl->getLocation()); + addUsage(Canonical, Decl->getLocation(), Result.SourceManager); return; } @@ -386,7 +392,8 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) { if (const auto *Value = Result.Nodes.getNodeAs("decl")) { if (const Type *TypePtr = Value->getType().getTypePtrOrNull()) { if (const auto *Typedef = TypePtr->getAs()) - addUsage(Typedef->getDecl(), Value->getSourceRange()); + addUsage(Typedef->getDecl(), Value->getSourceRange(), + Result.SourceManager); } } @@ -394,11 +401,13 @@ void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) { if (const auto *Value = Result.Nodes.getNodeAs("decl")) { if (const auto *Typedef = Value->getReturnType().getTypePtr()->getAs()) - addUsage(Typedef->getDecl(), Value->getSourceRange()); + addUsage(Typedef->getDecl(), Value->getSourceRange(), + Result.SourceManager); for (const ParmVarDecl *Param : Value->parameters()) { if (const TypedefType *Typedef = Param->getType().getTypePtr()->getAs()) - addUsage(Typedef->getDecl(), Value->getSourceRange()); + addUsage(Typedef->getDecl(), Value->getSourceRange(), + Result.SourceManager); } } diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming.cpp index 3a3bfecc01fcc..24c1c4270dec8 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming.cpp @@ -562,3 +562,19 @@ void ReferenceBadNamedFunction() { } } // namespace redecls + +namespace scratchspace { +#define DUP(Tok) Tok +#define M1(Tok) DUP(badName##Tok()) + +// We don't want a warning here as the call to this in Foo is in a scratch +// buffer so its fix-it wouldn't be applied, resulting in invalid code. +void badNameWarn(); + +void Foo() { + M1(Warn); +} + +#undef M1 +#undef DUP +} // namespace scratchspace From 3a4feb1d53df68bed7748a0625b593ba4b5dae32 Mon Sep 17 00:00:00 2001 From: Mikhail Maltsev Date: Mon, 22 Jun 2020 17:15:11 +0000 Subject: [PATCH 03/13] [ARM][BFloat] Implement bf16 get/set_lane without casts to i16 vectors Currently, in order to extract an element from a bf16 vector, we cast the vector to an i16 vector, perform the extraction, and cast the result to bfloat. This behavior was copied from the old fp16 implementation. The goal of this patch is to achieve optimal code generation for lane copying intrinsics in a subsequent patch (LLVM fails to fold certain combinations of bitcast, insertelement, extractelement and shufflevector instructions leading to the generation of suboptimal code). Differential Revision: https://reviews.llvm.org/D82206 --- clang/include/clang/Basic/arm_neon.td | 58 +++++++------------ clang/lib/CodeGen/CGBuiltin.cpp | 16 +++++ .../test/CodeGen/arm-bf16-getset-intrinsics.c | 22 +++---- 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/clang/include/clang/Basic/arm_neon.td b/clang/include/clang/Basic/arm_neon.td index 289f5ea47b926..42d50f726393e 100644 --- a/clang/include/clang/Basic/arm_neon.td +++ b/clang/include/clang/Basic/arm_neon.td @@ -190,28 +190,20 @@ def OP_SCALAR_QRDMLAH_LN : Op<(call "vqadd", $p0, (call "vqrdmulh", $p1, def OP_SCALAR_QRDMLSH_LN : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1, (call "vget_lane", $p2, $p3)))>; -multiclass ScalarGetSetLaneOpsF16 { - def _GET_LN : Op<(bitcast scalarTy, - (call "vget_lane", - (bitcast "int16x4_t", $p0), $p1))>; - def _GET_LNQ : Op<(bitcast scalarTy, - (call "vget_lane", - (bitcast "int16x8_t", $p0), $p1))>; - def _SET_LN : Op<(bitcast vectorTy4, - (call "vset_lane", - (bitcast "int16_t", $p0), - (bitcast "int16x4_t", $p1), $p2))>; - def _SET_LNQ : Op<(bitcast vectorTy8, - (call "vset_lane", - (bitcast "int16_t", $p0), - (bitcast "int16x8_t", $p1), $p2))>; -} - -defm OP_SCALAR_HALF: ScalarGetSetLaneOpsF16<"float16_t", - "float16x4_t", "float16x8_t">; -defm OP_SCALAR_BF16: ScalarGetSetLaneOpsF16<"bfloat16_t", - "bfloat16x4_t", "bfloat16x8_t">; +def OP_SCALAR_HALF_GET_LN : Op<(bitcast "float16_t", + (call "vget_lane", + (bitcast "int16x4_t", $p0), $p1))>; +def OP_SCALAR_HALF_GET_LNQ : Op<(bitcast "float16_t", + (call "vget_lane", + (bitcast "int16x8_t", $p0), $p1))>; +def OP_SCALAR_HALF_SET_LN : Op<(bitcast "float16x4_t", + (call "vset_lane", + (bitcast "int16_t", $p0), + (bitcast "int16x4_t", $p1), $p2))>; +def OP_SCALAR_HALF_SET_LNQ : Op<(bitcast "float16x8_t", + (call "vset_lane", + (bitcast "int16_t", $p0), + (bitcast "int16x8_t", $p1), $p2))>; def OP_DOT_LN : Op<(call "vdot", $p0, $p1, @@ -1918,10 +1910,12 @@ let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)" in { def VGET_HIGH_BF : NoTestOpInst<"vget_high", ".Q", "b", OP_HI>; def VGET_LOW_BF : NoTestOpInst<"vget_low", ".Q", "b", OP_LO>; - def VGET_LANE_BF : IOpInst<"vget_lane", "1.I", "b", OP_SCALAR_BF16_GET_LN>; - def VSET_LANE_BF : IOpInst<"vset_lane", ".1.I", "b", OP_SCALAR_BF16_SET_LN>; - def VGET_LANEQ_BF : IOpInst<"vget_lane", "1.I", "Qb", OP_SCALAR_BF16_GET_LNQ>; - def VSET_LANEQ_BF : IOpInst<"vset_lane", ".1.I", "Qb", OP_SCALAR_BF16_SET_LNQ>; + def VGET_LANE_BF : IInst<"vget_lane", "1.I", "bQb">; + def VSET_LANE_BF : IInst<"vset_lane", ".1.I", "bQb">; + def SCALAR_VDUP_LANE_BF : IInst<"vdup_lane", "1.I", "Sb">; + def SCALAR_VDUP_LANEQ_BF : IInst<"vdup_laneq", "1QI", "Sb"> { + let isLaneQ = 1; + } def VLD1_BF : WInst<"vld1", ".(c*!)", "bQb">; def VLD2_BF : WInst<"vld2", "2(c*!)", "bQb">; @@ -1957,18 +1951,6 @@ let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)" in { } -let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC) && !defined(__aarch64__)" in { - def SCALAR_VDUP_LANE_BF_A32 : IOpInst<"vduph_lane", "1.I", "b", OP_SCALAR_BF16_GET_LN>; - def SCALAR_VDUP_LANEQ_BF_A32 : IOpInst<"vduph_laneq", "1.I", "Hb", OP_SCALAR_BF16_GET_LNQ>; -} - -let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC) && defined(__aarch64__)" in { - def SCALAR_VDUP_LANE_BF_A64 : IInst<"vdup_lane", "1.I", "Sb">; - def SCALAR_VDUP_LANEQ_BF_A64 : IInst<"vdup_laneq", "1QI", "Sb"> { - let isLaneQ = 1; - } -} - let ArchGuard = "defined(__ARM_FEATURE_BF16) && !defined(__aarch64__)" in { let BigEndianSafe = 1 in { defm VREINTERPRET_BF : REINTERPRET_CROSS_TYPES< diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index c18f01b75353c..8213ed8a39e1b 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -6416,21 +6416,27 @@ static bool HasExtraNeonArgument(unsigned BuiltinID) { default: break; case NEON::BI__builtin_neon_vget_lane_i8: case NEON::BI__builtin_neon_vget_lane_i16: + case NEON::BI__builtin_neon_vget_lane_bf16: case NEON::BI__builtin_neon_vget_lane_i32: case NEON::BI__builtin_neon_vget_lane_i64: case NEON::BI__builtin_neon_vget_lane_f32: case NEON::BI__builtin_neon_vgetq_lane_i8: case NEON::BI__builtin_neon_vgetq_lane_i16: + case NEON::BI__builtin_neon_vgetq_lane_bf16: case NEON::BI__builtin_neon_vgetq_lane_i32: case NEON::BI__builtin_neon_vgetq_lane_i64: case NEON::BI__builtin_neon_vgetq_lane_f32: + case NEON::BI__builtin_neon_vduph_lane_bf16: + case NEON::BI__builtin_neon_vduph_laneq_bf16: case NEON::BI__builtin_neon_vset_lane_i8: case NEON::BI__builtin_neon_vset_lane_i16: + case NEON::BI__builtin_neon_vset_lane_bf16: case NEON::BI__builtin_neon_vset_lane_i32: case NEON::BI__builtin_neon_vset_lane_i64: case NEON::BI__builtin_neon_vset_lane_f32: case NEON::BI__builtin_neon_vsetq_lane_i8: case NEON::BI__builtin_neon_vsetq_lane_i16: + case NEON::BI__builtin_neon_vsetq_lane_bf16: case NEON::BI__builtin_neon_vsetq_lane_i32: case NEON::BI__builtin_neon_vsetq_lane_i64: case NEON::BI__builtin_neon_vsetq_lane_f32: @@ -6876,12 +6882,16 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vget_lane_i16: case NEON::BI__builtin_neon_vget_lane_i32: case NEON::BI__builtin_neon_vget_lane_i64: + case NEON::BI__builtin_neon_vget_lane_bf16: case NEON::BI__builtin_neon_vget_lane_f32: case NEON::BI__builtin_neon_vgetq_lane_i8: case NEON::BI__builtin_neon_vgetq_lane_i16: case NEON::BI__builtin_neon_vgetq_lane_i32: case NEON::BI__builtin_neon_vgetq_lane_i64: + case NEON::BI__builtin_neon_vgetq_lane_bf16: case NEON::BI__builtin_neon_vgetq_lane_f32: + case NEON::BI__builtin_neon_vduph_lane_bf16: + case NEON::BI__builtin_neon_vduph_laneq_bf16: return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane"); case NEON::BI__builtin_neon_vrndns_f32: { @@ -6894,11 +6904,13 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vset_lane_i16: case NEON::BI__builtin_neon_vset_lane_i32: case NEON::BI__builtin_neon_vset_lane_i64: + case NEON::BI__builtin_neon_vset_lane_bf16: case NEON::BI__builtin_neon_vset_lane_f32: case NEON::BI__builtin_neon_vsetq_lane_i8: case NEON::BI__builtin_neon_vsetq_lane_i16: case NEON::BI__builtin_neon_vsetq_lane_i32: case NEON::BI__builtin_neon_vsetq_lane_i64: + case NEON::BI__builtin_neon_vsetq_lane_bf16: case NEON::BI__builtin_neon_vsetq_lane_f32: return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); @@ -9309,11 +9321,13 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vset_lane_i16: case NEON::BI__builtin_neon_vset_lane_i32: case NEON::BI__builtin_neon_vset_lane_i64: + case NEON::BI__builtin_neon_vset_lane_bf16: case NEON::BI__builtin_neon_vset_lane_f32: case NEON::BI__builtin_neon_vsetq_lane_i8: case NEON::BI__builtin_neon_vsetq_lane_i16: case NEON::BI__builtin_neon_vsetq_lane_i32: case NEON::BI__builtin_neon_vsetq_lane_i64: + case NEON::BI__builtin_neon_vsetq_lane_bf16: case NEON::BI__builtin_neon_vsetq_lane_f32: Ops.push_back(EmitScalarExpr(E->getArg(2))); return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); @@ -9592,11 +9606,13 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, : Intrinsic::aarch64_neon_sqsub; return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl"); } + case NEON::BI__builtin_neon_vget_lane_bf16: case NEON::BI__builtin_neon_vduph_lane_bf16: case NEON::BI__builtin_neon_vduph_lane_f16: { return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), "vget_lane"); } + case NEON::BI__builtin_neon_vgetq_lane_bf16: case NEON::BI__builtin_neon_vduph_laneq_bf16: case NEON::BI__builtin_neon_vduph_laneq_f16: { return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), diff --git a/clang/test/CodeGen/arm-bf16-getset-intrinsics.c b/clang/test/CodeGen/arm-bf16-getset-intrinsics.c index ed35f810d18fc..a1a5b7b54f3ed 100644 --- a/clang/test/CodeGen/arm-bf16-getset-intrinsics.c +++ b/clang/test/CodeGen/arm-bf16-getset-intrinsics.c @@ -1,6 +1,8 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // RUN: %clang_cc1 -triple armv8.6a-arm-none-eabi -target-feature +neon -target-feature +bf16 -mfloat-abi hard \ // RUN: -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg -instcombine | FileCheck %s +// RUN: %clang_cc1 -triple armv8.6a-arm-none-eabi -target-feature +neon -target-feature +bf16 -mfloat-abi soft \ +// RUN: -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg -instcombine | FileCheck %s #include @@ -98,8 +100,8 @@ bfloat16x4_t test_vget_low_bf16(bfloat16x8_t a) { // CHECK-LABEL: @test_vget_lane_bf16( // CHECK-NEXT: entry: -// CHECK-NEXT: [[DOTCAST1:%.*]] = extractelement <4 x bfloat> [[V:%.*]], i32 1 -// CHECK-NEXT: ret bfloat [[DOTCAST1]] +// CHECK-NEXT: [[VGET_LANE:%.*]] = extractelement <4 x bfloat> [[V:%.*]], i32 1 +// CHECK-NEXT: ret bfloat [[VGET_LANE]] // bfloat16_t test_vget_lane_bf16(bfloat16x4_t v) { return vget_lane_bf16(v, 1); @@ -107,8 +109,8 @@ bfloat16_t test_vget_lane_bf16(bfloat16x4_t v) { // CHECK-LABEL: @test_vgetq_lane_bf16( // CHECK-NEXT: entry: -// CHECK-NEXT: [[DOTCAST1:%.*]] = extractelement <8 x bfloat> [[V:%.*]], i32 7 -// CHECK-NEXT: ret bfloat [[DOTCAST1]] +// CHECK-NEXT: [[VGET_LANE:%.*]] = extractelement <8 x bfloat> [[V:%.*]], i32 7 +// CHECK-NEXT: ret bfloat [[VGET_LANE]] // bfloat16_t test_vgetq_lane_bf16(bfloat16x8_t v) { return vgetq_lane_bf16(v, 7); @@ -116,8 +118,8 @@ bfloat16_t test_vgetq_lane_bf16(bfloat16x8_t v) { // CHECK-LABEL: @test_vset_lane_bf16( // CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x bfloat> [[V:%.*]], bfloat [[A:%.*]], i32 1 -// CHECK-NEXT: ret <4 x bfloat> [[TMP0]] +// CHECK-NEXT: [[VSET_LANE:%.*]] = insertelement <4 x bfloat> [[V:%.*]], bfloat [[A:%.*]], i32 1 +// CHECK-NEXT: ret <4 x bfloat> [[VSET_LANE]] // bfloat16x4_t test_vset_lane_bf16(bfloat16_t a, bfloat16x4_t v) { return vset_lane_bf16(a, v, 1); @@ -125,8 +127,8 @@ bfloat16x4_t test_vset_lane_bf16(bfloat16_t a, bfloat16x4_t v) { // CHECK-LABEL: @test_vsetq_lane_bf16( // CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x bfloat> [[V:%.*]], bfloat [[A:%.*]], i32 7 -// CHECK-NEXT: ret <8 x bfloat> [[TMP0]] +// CHECK-NEXT: [[VSET_LANE:%.*]] = insertelement <8 x bfloat> [[V:%.*]], bfloat [[A:%.*]], i32 7 +// CHECK-NEXT: ret <8 x bfloat> [[VSET_LANE]] // bfloat16x8_t test_vsetq_lane_bf16(bfloat16_t a, bfloat16x8_t v) { return vsetq_lane_bf16(a, v, 7); @@ -143,8 +145,8 @@ bfloat16_t test_vduph_lane_bf16(bfloat16x4_t v) { // CHECK-LABEL: @test_vduph_laneq_bf16( // CHECK-NEXT: entry: -// CHECK-NEXT: [[VGETQ_LANE:%.*]] = extractelement <8 x bfloat> [[V:%.*]], i32 7 -// CHECK-NEXT: ret bfloat [[VGETQ_LANE]] +// CHECK-NEXT: [[VGET_LANE:%.*]] = extractelement <8 x bfloat> [[V:%.*]], i32 7 +// CHECK-NEXT: ret bfloat [[VGET_LANE]] // bfloat16_t test_vduph_laneq_bf16(bfloat16x8_t v) { return vduph_laneq_bf16(v, 7); From 30140cd24920a2c5469f503df7b39532c7af292d Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Mon, 22 Jun 2020 19:53:01 +0200 Subject: [PATCH 04/13] [llvm] Fix typo in test case comment (NFC) --- llvm/test/Transforms/Inline/always-inline.ll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/test/Transforms/Inline/always-inline.ll b/llvm/test/Transforms/Inline/always-inline.ll index 791eb94779b70..81cbb73fbf59d 100644 --- a/llvm/test/Transforms/Inline/always-inline.ll +++ b/llvm/test/Transforms/Inline/always-inline.ll @@ -212,7 +212,7 @@ entry: ; CHECK: ret void } -; The 'inner10' and 'outer10' functions test a frustrating consquence of the +; The 'inner10' and 'outer10' functions test a frustrating consequence of the ; current 'alwaysinline' semantic model. Because such functions are allowed to ; be external functions, it may be necessary to both inline all of their uses ; and leave them in the final output. These tests can be removed if and when From 809bcf4c87d01f74586a432594ac0e9c813182b7 Mon Sep 17 00:00:00 2001 From: Hanhan Wang Date: Mon, 22 Jun 2020 11:15:44 -0700 Subject: [PATCH 05/13] [mlir] Fix a doc link in LinalgBase.td Differential Revision: https://reviews.llvm.org/D82303 --- mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td index 3b52b0e639184..7955345f69668 100644 --- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td +++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td @@ -26,7 +26,8 @@ def Linalg_Dialect : Dialect { Additional [Linalg Dialect Documentation](https://mlir.llvm.org/docs/Dialects/Linalg) and a - [Rationale Document](https://mlir.llvm.org/docs/RationaleLinalgDialect) are + [Rationale + Document](https://mlir.llvm.org/docs/Rationale/RationaleLinalgDialect) are are also available and should be read first before going in the details of the op semantics. }]; From 663860f63e73518fc09e55a4a68b03f8027eafc8 Mon Sep 17 00:00:00 2001 From: Tim Keith Date: Mon, 22 Jun 2020 11:32:54 -0700 Subject: [PATCH 06/13] [flang] Fix bug checking SAVE attribute Treat function result like dummy argument: a SAVE statement without an entity-list does not make it saved. Differential Revision: https://reviews.llvm.org/D82309 --- flang/lib/Evaluate/tools.cpp | 2 +- flang/test/Semantics/resolve45.f90 | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp index 3538cd587f970..16cc39f78291c 100644 --- a/flang/lib/Evaluate/tools.cpp +++ b/flang/lib/Evaluate/tools.cpp @@ -1003,7 +1003,7 @@ bool IsSaved(const Symbol &original) { } else if (const Symbol * block{FindCommonBlockContaining(symbol)}; block && block->attrs().test(Attr::SAVE)) { return true; - } else if (IsDummy(symbol)) { + } else if (IsDummy(symbol) || IsFunctionResult(symbol)) { return false; } else { for (; !scope->IsGlobal(); scope = &scope->parent()) { diff --git a/flang/test/Semantics/resolve45.f90 b/flang/test/Semantics/resolve45.f90 index 8ab75595d4059..3e98ff662a171 100644 --- a/flang/test/Semantics/resolve45.f90 +++ b/flang/test/Semantics/resolve45.f90 @@ -15,8 +15,15 @@ function f2(x, y) !ERROR: SAVE attribute may not be applied to dummy argument 'x' complex, save :: x allocatable :: y + integer :: y !ERROR: SAVE attribute may not be applied to dummy argument 'y' - integer, save :: y + save :: y +end + +! SAVE statement should not trigger the above errors +function f2b(x, y) + real :: x, y + save end subroutine s3(x) From 5ac144fe64c6911b5b6011cac1d61e0f82562e7f Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Sun, 21 Jun 2020 17:30:10 -0700 Subject: [PATCH 07/13] [X86] Calculate the needed size of the feature arrays in _cpu_indicator_init and getHostCPUName using the size of the feature enum. Move 0 initialization up to the caller so we don't need to know the size. --- compiler-rt/lib/builtins/cpu_model.c | 10 ++++++---- llvm/include/llvm/Support/X86TargetParser.h | 2 +- llvm/lib/Support/Host.cpp | 11 ++++------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/compiler-rt/lib/builtins/cpu_model.c b/compiler-rt/lib/builtins/cpu_model.c index 59c6974207c79..3207f1449e00f 100644 --- a/compiler-rt/lib/builtins/cpu_model.c +++ b/compiler-rt/lib/builtins/cpu_model.c @@ -125,7 +125,8 @@ enum ProcessorFeatures { FEATURE_AVX512VNNI, FEATURE_AVX512BITALG, FEATURE_AVX512BF16, - FEATURE_AVX512VP2INTERSECT + FEATURE_AVX512VP2INTERSECT, + CPU_FEATURE_MAX }; // The check below for i386 was copied from clang's cpuid.h (__get_cpuid_max). @@ -493,8 +494,6 @@ static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model, static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, unsigned *Features) { - Features[0] = 0; - Features[1] = 0; unsigned EAX, EBX; #define setFeature(F) \ @@ -653,7 +652,8 @@ int CONSTRUCTOR_ATTRIBUTE __cpu_indicator_init(void) { unsigned MaxLeaf = 5; unsigned Vendor; unsigned Model, Family; - unsigned Features[2]; + const unsigned SIZE_OF_FEATURES = (CPU_FEATURE_MAX + 31) / 32; + unsigned Features[SIZE_OF_FEATURES] = {0}; // This function needs to run just once. if (__cpu_model.__cpu_vendor) @@ -670,6 +670,8 @@ int CONSTRUCTOR_ATTRIBUTE __cpu_indicator_init(void) { // Find available features. getAvailableFeatures(ECX, EDX, MaxLeaf, &Features[0]); + + assert(SIZE_OF_FEATURES == 2); __cpu_model.__cpu_features[0] = Features[0]; __cpu_features2 = Features[1]; diff --git a/llvm/include/llvm/Support/X86TargetParser.h b/llvm/include/llvm/Support/X86TargetParser.h index 7cdf1c5328146..e357d11a66158 100644 --- a/llvm/include/llvm/Support/X86TargetParser.h +++ b/llvm/include/llvm/Support/X86TargetParser.h @@ -56,7 +56,7 @@ enum ProcessorFeatures { #define X86_FEATURE(VAL, ENUM) \ ENUM = VAL, #include "llvm/Support/X86TargetParser.def" - + CPU_FEATURE_MAX }; enum CPUKind { diff --git a/llvm/lib/Support/Host.cpp b/llvm/lib/Support/Host.cpp index c6b12e6516a45..e4c6b879884ad 100644 --- a/llvm/lib/Support/Host.cpp +++ b/llvm/lib/Support/Host.cpp @@ -582,7 +582,7 @@ static void detectX86FamilyModel(unsigned EAX, unsigned *Family, static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, - const unsigned (&Features)[3], + const unsigned *Features, unsigned *Type, unsigned *Subtype) { auto testFeature = [&](unsigned F) { return (Features[F / 32] & (1U << (F % 32))) != 0; @@ -910,7 +910,7 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, } static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model, - const unsigned (&Features)[3], + const unsigned *Features, unsigned *Type, unsigned *Subtype) { auto testFeature = [&](unsigned F) { return (Features[F / 32] & (1U << (F % 32))) != 0; @@ -1012,10 +1012,7 @@ static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model, } static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, - unsigned (&Features)[3]) { - Features[0] = 0; - Features[1] = 0; - Features[2] = 0; + unsigned *Features) { unsigned EAX, EBX; auto setFeature = [&](unsigned F) { @@ -1157,7 +1154,7 @@ StringRef sys::getHostCPUName() { getX86CpuIDAndInfo(0x1, &EAX, &EBX, &ECX, &EDX); unsigned Family = 0, Model = 0; - unsigned Features[3] = {0, 0, 0}; + unsigned Features[(X86::CPU_FEATURE_MAX + 31) / 32] = {0}; detectX86FamilyModel(EAX, &Family, &Model); getAvailableFeatures(ECX, EDX, MaxLeaf, Features); From 0dfc8e1837e3e3ac06ab8c08fdc08b15e0ae1c2d Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Sun, 21 Jun 2020 20:20:04 -0700 Subject: [PATCH 08/13] [X86] Remove encoding value from the X86_FEATURE and X86_FEATURE_COMPAT macro. NFCI This was orignally done so we could separate the compatibility values and the llvm internal only features into a separate entries in the feature array. This was needed when we explicitly had to convert the feature into the proper 32-bit chunk at every reference and we didn't want things moving around. Now everything is in an array and we have helper funtions or macros to convert encoding to index. So we renumbering is no longer an issue. --- clang/lib/Basic/Targets/X86.cpp | 4 +- clang/lib/CodeGen/CGBuiltin.cpp | 2 +- llvm/include/llvm/Support/X86TargetParser.def | 91 ++++++++++--------- llvm/include/llvm/Support/X86TargetParser.h | 4 +- 4 files changed, 51 insertions(+), 50 deletions(-) diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp index 0092c52b2b231..a0eaf6d1faab9 100644 --- a/clang/lib/Basic/Targets/X86.cpp +++ b/clang/lib/Basic/Targets/X86.cpp @@ -1518,14 +1518,14 @@ bool X86TargetInfo::hasFeature(StringRef Feature) const { // X86TargetInfo::hasFeature for a somewhat comprehensive list). bool X86TargetInfo::validateCpuSupports(StringRef FeatureStr) const { return llvm::StringSwitch(FeatureStr) -#define X86_FEATURE_COMPAT(VAL, ENUM, STR) .Case(STR, true) +#define X86_FEATURE_COMPAT(ENUM, STR) .Case(STR, true) #include "llvm/Support/X86TargetParser.def" .Default(false); } static llvm::X86::ProcessorFeatures getFeature(StringRef Name) { return llvm::StringSwitch(Name) -#define X86_FEATURE_COMPAT(VAL, ENUM, STR) .Case(STR, llvm::X86::ENUM) +#define X86_FEATURE_COMPAT(ENUM, STR) .Case(STR, llvm::X86::ENUM) #include "llvm/Support/X86TargetParser.def" ; // Note, this function should only be used after ensuring the value is diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index 8213ed8a39e1b..32e43c295d23c 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -11662,7 +11662,7 @@ CodeGenFunction::GetX86CpuSupportsMask(ArrayRef FeatureStrs) { for (const StringRef &FeatureStr : FeatureStrs) { unsigned Feature = StringSwitch(FeatureStr) -#define X86_FEATURE_COMPAT(VAL, ENUM, STR) .Case(STR, VAL) +#define X86_FEATURE_COMPAT(ENUM, STR) .Case(STR, llvm::X86::ENUM) #include "llvm/Support/X86TargetParser.def" ; FeaturesMask |= (1ULL << Feature); diff --git a/llvm/include/llvm/Support/X86TargetParser.def b/llvm/include/llvm/Support/X86TargetParser.def index 3c2614b33bf32..e228cef760bc5 100644 --- a/llvm/include/llvm/Support/X86TargetParser.def +++ b/llvm/include/llvm/Support/X86TargetParser.def @@ -125,56 +125,57 @@ X86_CPU_SUBTYPE ("geode", AMDPENTIUM_GEODE) // This macro is used for cpu types present in compiler-rt/libgcc. #ifndef X86_FEATURE_COMPAT -#define X86_FEATURE_COMPAT(VAL, ENUM, STR) X86_FEATURE(VAL, ENUM) +#define X86_FEATURE_COMPAT(ENUM, STR) X86_FEATURE(ENUM) #endif #ifndef X86_FEATURE -#define X86_FEATURE(VAL, ENUM) +#define X86_FEATURE(ENUM) #endif -X86_FEATURE_COMPAT( 0, FEATURE_CMOV, "cmov") -X86_FEATURE_COMPAT( 1, FEATURE_MMX, "mmx") -X86_FEATURE_COMPAT( 2, FEATURE_POPCNT, "popcnt") -X86_FEATURE_COMPAT( 3, FEATURE_SSE, "sse") -X86_FEATURE_COMPAT( 4, FEATURE_SSE2, "sse2") -X86_FEATURE_COMPAT( 5, FEATURE_SSE3, "sse3") -X86_FEATURE_COMPAT( 6, FEATURE_SSSE3, "ssse3") -X86_FEATURE_COMPAT( 7, FEATURE_SSE4_1, "sse4.1") -X86_FEATURE_COMPAT( 8, FEATURE_SSE4_2, "sse4.2") -X86_FEATURE_COMPAT( 9, FEATURE_AVX, "avx") -X86_FEATURE_COMPAT(10, FEATURE_AVX2, "avx2") -X86_FEATURE_COMPAT(11, FEATURE_SSE4_A, "sse4a") -X86_FEATURE_COMPAT(12, FEATURE_FMA4, "fma4") -X86_FEATURE_COMPAT(13, FEATURE_XOP, "xop") -X86_FEATURE_COMPAT(14, FEATURE_FMA, "fma") -X86_FEATURE_COMPAT(15, FEATURE_AVX512F, "avx512f") -X86_FEATURE_COMPAT(16, FEATURE_BMI, "bmi") -X86_FEATURE_COMPAT(17, FEATURE_BMI2, "bmi2") -X86_FEATURE_COMPAT(18, FEATURE_AES, "aes") -X86_FEATURE_COMPAT(19, FEATURE_PCLMUL, "pclmul") -X86_FEATURE_COMPAT(20, FEATURE_AVX512VL, "avx512vl") -X86_FEATURE_COMPAT(21, FEATURE_AVX512BW, "avx512bw") -X86_FEATURE_COMPAT(22, FEATURE_AVX512DQ, "avx512dq") -X86_FEATURE_COMPAT(23, FEATURE_AVX512CD, "avx512cd") -X86_FEATURE_COMPAT(24, FEATURE_AVX512ER, "avx512er") -X86_FEATURE_COMPAT(25, FEATURE_AVX512PF, "avx512pf") -X86_FEATURE_COMPAT(26, FEATURE_AVX512VBMI, "avx512vbmi") -X86_FEATURE_COMPAT(27, FEATURE_AVX512IFMA, "avx512ifma") -X86_FEATURE_COMPAT(28, FEATURE_AVX5124VNNIW, "avx5124vnniw") -X86_FEATURE_COMPAT(29, FEATURE_AVX5124FMAPS, "avx5124fmaps") -X86_FEATURE_COMPAT(30, FEATURE_AVX512VPOPCNTDQ, "avx512vpopcntdq") -X86_FEATURE_COMPAT(31, FEATURE_AVX512VBMI2, "avx512vbmi2") -X86_FEATURE_COMPAT(32, FEATURE_GFNI, "gfni") -X86_FEATURE_COMPAT(33, FEATURE_VPCLMULQDQ, "vpclmulqdq") -X86_FEATURE_COMPAT(34, FEATURE_AVX512VNNI, "avx512vnni") -X86_FEATURE_COMPAT(35, FEATURE_AVX512BITALG, "avx512bitalg") -X86_FEATURE_COMPAT(36, FEATURE_AVX512BF16, "avx512bf16") -X86_FEATURE_COMPAT(37, FEATURE_AVX512VP2INTERSECT, "avx512vp2intersect") + +X86_FEATURE_COMPAT(FEATURE_CMOV, "cmov") +X86_FEATURE_COMPAT(FEATURE_MMX, "mmx") +X86_FEATURE_COMPAT(FEATURE_POPCNT, "popcnt") +X86_FEATURE_COMPAT(FEATURE_SSE, "sse") +X86_FEATURE_COMPAT(FEATURE_SSE2, "sse2") +X86_FEATURE_COMPAT(FEATURE_SSE3, "sse3") +X86_FEATURE_COMPAT(FEATURE_SSSE3, "ssse3") +X86_FEATURE_COMPAT(FEATURE_SSE4_1, "sse4.1") +X86_FEATURE_COMPAT(FEATURE_SSE4_2, "sse4.2") +X86_FEATURE_COMPAT(FEATURE_AVX, "avx") +X86_FEATURE_COMPAT(FEATURE_AVX2, "avx2") +X86_FEATURE_COMPAT(FEATURE_SSE4_A, "sse4a") +X86_FEATURE_COMPAT(FEATURE_FMA4, "fma4") +X86_FEATURE_COMPAT(FEATURE_XOP, "xop") +X86_FEATURE_COMPAT(FEATURE_FMA, "fma") +X86_FEATURE_COMPAT(FEATURE_AVX512F, "avx512f") +X86_FEATURE_COMPAT(FEATURE_BMI, "bmi") +X86_FEATURE_COMPAT(FEATURE_BMI2, "bmi2") +X86_FEATURE_COMPAT(FEATURE_AES, "aes") +X86_FEATURE_COMPAT(FEATURE_PCLMUL, "pclmul") +X86_FEATURE_COMPAT(FEATURE_AVX512VL, "avx512vl") +X86_FEATURE_COMPAT(FEATURE_AVX512BW, "avx512bw") +X86_FEATURE_COMPAT(FEATURE_AVX512DQ, "avx512dq") +X86_FEATURE_COMPAT(FEATURE_AVX512CD, "avx512cd") +X86_FEATURE_COMPAT(FEATURE_AVX512ER, "avx512er") +X86_FEATURE_COMPAT(FEATURE_AVX512PF, "avx512pf") +X86_FEATURE_COMPAT(FEATURE_AVX512VBMI, "avx512vbmi") +X86_FEATURE_COMPAT(FEATURE_AVX512IFMA, "avx512ifma") +X86_FEATURE_COMPAT(FEATURE_AVX5124VNNIW, "avx5124vnniw") +X86_FEATURE_COMPAT(FEATURE_AVX5124FMAPS, "avx5124fmaps") +X86_FEATURE_COMPAT(FEATURE_AVX512VPOPCNTDQ, "avx512vpopcntdq") +X86_FEATURE_COMPAT(FEATURE_AVX512VBMI2, "avx512vbmi2") +X86_FEATURE_COMPAT(FEATURE_GFNI, "gfni") +X86_FEATURE_COMPAT(FEATURE_VPCLMULQDQ, "vpclmulqdq") +X86_FEATURE_COMPAT(FEATURE_AVX512VNNI, "avx512vnni") +X86_FEATURE_COMPAT(FEATURE_AVX512BITALG, "avx512bitalg") +X86_FEATURE_COMPAT(FEATURE_AVX512BF16, "avx512bf16") +X86_FEATURE_COMPAT(FEATURE_AVX512VP2INTERSECT, "avx512vp2intersect") // Features below here are not in libgcc/compiler-rt. -X86_FEATURE (64, FEATURE_MOVBE) -X86_FEATURE (65, FEATURE_ADX) -X86_FEATURE (66, FEATURE_EM64T) -X86_FEATURE (67, FEATURE_CLFLUSHOPT) -X86_FEATURE (68, FEATURE_SHA) +X86_FEATURE (FEATURE_MOVBE) +X86_FEATURE (FEATURE_ADX) +X86_FEATURE (FEATURE_EM64T) +X86_FEATURE (FEATURE_CLFLUSHOPT) +X86_FEATURE (FEATURE_SHA) #undef X86_FEATURE_COMPAT #undef X86_FEATURE diff --git a/llvm/include/llvm/Support/X86TargetParser.h b/llvm/include/llvm/Support/X86TargetParser.h index e357d11a66158..79cb0af4ed06f 100644 --- a/llvm/include/llvm/Support/X86TargetParser.h +++ b/llvm/include/llvm/Support/X86TargetParser.h @@ -53,8 +53,8 @@ enum ProcessorSubtypes : unsigned { // This should be kept in sync with libcc/compiler-rt as it should be used // by clang as a proxy for what's in libgcc/compiler-rt. enum ProcessorFeatures { -#define X86_FEATURE(VAL, ENUM) \ - ENUM = VAL, +#define X86_FEATURE(ENUM) \ + ENUM, #include "llvm/Support/X86TargetParser.def" CPU_FEATURE_MAX }; From bebea4221d3d1db9adaf7c921c109b6c2af26b74 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Mon, 22 Jun 2020 12:20:40 -0700 Subject: [PATCH 09/13] Revert "[X86] Calculate the needed size of the feature arrays in _cpu_indicator_init and getHostCPUName using the size of the feature enum." Seems to breaking build. This reverts commit 5ac144fe64c6911b5b6011cac1d61e0f82562e7f. --- compiler-rt/lib/builtins/cpu_model.c | 10 ++++------ llvm/include/llvm/Support/X86TargetParser.h | 2 +- llvm/lib/Support/Host.cpp | 11 +++++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/compiler-rt/lib/builtins/cpu_model.c b/compiler-rt/lib/builtins/cpu_model.c index 3207f1449e00f..59c6974207c79 100644 --- a/compiler-rt/lib/builtins/cpu_model.c +++ b/compiler-rt/lib/builtins/cpu_model.c @@ -125,8 +125,7 @@ enum ProcessorFeatures { FEATURE_AVX512VNNI, FEATURE_AVX512BITALG, FEATURE_AVX512BF16, - FEATURE_AVX512VP2INTERSECT, - CPU_FEATURE_MAX + FEATURE_AVX512VP2INTERSECT }; // The check below for i386 was copied from clang's cpuid.h (__get_cpuid_max). @@ -494,6 +493,8 @@ static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model, static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, unsigned *Features) { + Features[0] = 0; + Features[1] = 0; unsigned EAX, EBX; #define setFeature(F) \ @@ -652,8 +653,7 @@ int CONSTRUCTOR_ATTRIBUTE __cpu_indicator_init(void) { unsigned MaxLeaf = 5; unsigned Vendor; unsigned Model, Family; - const unsigned SIZE_OF_FEATURES = (CPU_FEATURE_MAX + 31) / 32; - unsigned Features[SIZE_OF_FEATURES] = {0}; + unsigned Features[2]; // This function needs to run just once. if (__cpu_model.__cpu_vendor) @@ -670,8 +670,6 @@ int CONSTRUCTOR_ATTRIBUTE __cpu_indicator_init(void) { // Find available features. getAvailableFeatures(ECX, EDX, MaxLeaf, &Features[0]); - - assert(SIZE_OF_FEATURES == 2); __cpu_model.__cpu_features[0] = Features[0]; __cpu_features2 = Features[1]; diff --git a/llvm/include/llvm/Support/X86TargetParser.h b/llvm/include/llvm/Support/X86TargetParser.h index 79cb0af4ed06f..7b4ade866a9d6 100644 --- a/llvm/include/llvm/Support/X86TargetParser.h +++ b/llvm/include/llvm/Support/X86TargetParser.h @@ -56,7 +56,7 @@ enum ProcessorFeatures { #define X86_FEATURE(ENUM) \ ENUM, #include "llvm/Support/X86TargetParser.def" - CPU_FEATURE_MAX + }; enum CPUKind { diff --git a/llvm/lib/Support/Host.cpp b/llvm/lib/Support/Host.cpp index e4c6b879884ad..c6b12e6516a45 100644 --- a/llvm/lib/Support/Host.cpp +++ b/llvm/lib/Support/Host.cpp @@ -582,7 +582,7 @@ static void detectX86FamilyModel(unsigned EAX, unsigned *Family, static void getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, - const unsigned *Features, + const unsigned (&Features)[3], unsigned *Type, unsigned *Subtype) { auto testFeature = [&](unsigned F) { return (Features[F / 32] & (1U << (F % 32))) != 0; @@ -910,7 +910,7 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, } static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model, - const unsigned *Features, + const unsigned (&Features)[3], unsigned *Type, unsigned *Subtype) { auto testFeature = [&](unsigned F) { return (Features[F / 32] & (1U << (F % 32))) != 0; @@ -1012,7 +1012,10 @@ static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model, } static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, - unsigned *Features) { + unsigned (&Features)[3]) { + Features[0] = 0; + Features[1] = 0; + Features[2] = 0; unsigned EAX, EBX; auto setFeature = [&](unsigned F) { @@ -1154,7 +1157,7 @@ StringRef sys::getHostCPUName() { getX86CpuIDAndInfo(0x1, &EAX, &EBX, &ECX, &EDX); unsigned Family = 0, Model = 0; - unsigned Features[(X86::CPU_FEATURE_MAX + 31) / 32] = {0}; + unsigned Features[3] = {0, 0, 0}; detectX86FamilyModel(EAX, &Family, &Model); getAvailableFeatures(ECX, EDX, MaxLeaf, Features); From 5e2c736395cc44a00c8c2cc9fc916450bf780fbe Mon Sep 17 00:00:00 2001 From: Christopher Tetreault Date: Mon, 22 Jun 2020 10:46:53 -0700 Subject: [PATCH 10/13] [SVE] Remove calls to VectorType::getNumElements from WebASM Summary: The getNumElements in base VectorType is being deprecated. See: http://lists.llvm.org/pipermail/llvm-dev/2020-March/139811.html Reviewers: efriedma, tlively, fpetrogalli, c-rhodes, dschuff Reviewed By: tlively, dschuff Subscribers: dschuff, sbc100, tschuett, jgravelle-google, hiraditya, aheejin, rkruppe, psnobl, llvm-commits Tags: #llvm Differential Revision: https://reviews.llvm.org/D82217 --- .../WebAssembly/WebAssemblyTargetTransformInfo.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp index 9f767d391122a..28703a2787e0d 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp @@ -63,10 +63,11 @@ unsigned WebAssemblyTTIImpl::getArithmeticInstrCost( // approxmation. if (Opd2Info != TTI::OK_UniformValue && Opd2Info != TTI::OK_UniformConstantValue) - Cost = VTy->getNumElements() * - (TargetTransformInfo::TCC_Basic + - getArithmeticInstrCost(Opcode, VTy->getElementType(), CostKind) + - TargetTransformInfo::TCC_Basic); + Cost = + cast(VTy)->getNumElements() * + (TargetTransformInfo::TCC_Basic + + getArithmeticInstrCost(Opcode, VTy->getElementType(), CostKind) + + TargetTransformInfo::TCC_Basic); break; } } From 1357c065783e8d66c9db4be59aca389d1dc6c05f Mon Sep 17 00:00:00 2001 From: Hans Wennborg Date: Mon, 22 Jun 2020 20:58:11 +0200 Subject: [PATCH 11/13] Revert "[X86][SSE] MatchVectorAllZeroTest - handle OR vector reductions" This caused a Chromium test to miscompile. See discussion on the Phabricator review. > This patch extends MatchVectorAllZeroTest to handle OR vector reduction patterns where the result is compared against zero. > > Fixes PR45378 > > Differential Revision: https://reviews.llvm.org/D81547 This reverts 057c9c7ee00b7f7696065a3fc26a3df5ce3ebe96 --- llvm/lib/Target/X86/X86ISelLowering.cpp | 55 +- llvm/test/CodeGen/X86/pr45378.ll | 56 +- llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll | 1127 +++++++++++------ 3 files changed, 779 insertions(+), 459 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 31bdaabcfa95d..1c6164c56f502 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -21360,22 +21360,12 @@ static SDValue LowerVectorAllZero(const SDLoc &DL, SDValue V, ISD::CondCode CC, SelectionDAG &DAG, X86::CondCode &X86CC) { EVT VT = V.getValueType(); - assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode"); - X86CC = (CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE); - - // For sub-128-bit vector, cast to (legal) integer and compare with zero. - if (VT.getSizeInBits() < 128) { - EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); - if (!DAG.getTargetLoweringInfo().isTypeLegal(IntVT)) - return SDValue(); - return DAG.getNode(X86ISD::CMP, DL, MVT::i32, DAG.getBitcast(IntVT, V), - DAG.getConstant(0, DL, IntVT)); - } - - // Quit if not splittable to 128/256-bit vector. - if (!isPowerOf2_32(VT.getSizeInBits())) + // Quit if less than 128-bits or not splittable to 128/256-bit vector. + if (VT.getSizeInBits() < 128 || !isPowerOf2_32(VT.getSizeInBits())) return SDValue(); + X86CC = (CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE); + // Split down to 128/256-bit vector. unsigned TestSize = Subtarget.hasAVX() ? 256 : 128; while (VT.getSizeInBits() > TestSize) { @@ -21399,19 +21389,18 @@ static SDValue LowerVectorAllZero(const SDLoc &DL, SDValue V, ISD::CondCode CC, DAG.getConstant(0xFFFF, DL, MVT::i32)); } -// Check whether an OR'd reduction tree is PTEST-able, or if we can fallback to +// Check whether an OR'd tree is PTEST-able, or if we can fallback to // CMP(MOVMSK(PCMPEQB(X,0))). static SDValue MatchVectorAllZeroTest(SDValue Op, ISD::CondCode CC, - const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &X86CC) { - assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode"); + assert(Op.getOpcode() == ISD::OR && "Only check OR'd tree."); if (!Subtarget.hasSSE2() || !Op->hasOneUse()) return SDValue(); SmallVector VecIns; - if (Op.getOpcode() == ISD::OR && matchScalarReduction(Op, ISD::OR, VecIns)) { + if (matchScalarReduction(Op, ISD::OR, VecIns)) { EVT VT = VecIns[0].getValueType(); assert(llvm::all_of(VecIns, [VT](SDValue V) { return VT == V.getValueType(); }) && @@ -21421,6 +21410,8 @@ static SDValue MatchVectorAllZeroTest(SDValue Op, ISD::CondCode CC, if (VT.getSizeInBits() < 128 || !isPowerOf2_32(VT.getSizeInBits())) return SDValue(); + SDLoc DL(Op); + // If more than one full vector is evaluated, OR them first before PTEST. for (unsigned Slot = 0, e = VecIns.size(); e - Slot > 1; Slot += 2, e += 1) { @@ -21439,19 +21430,6 @@ static SDValue MatchVectorAllZeroTest(SDValue Op, ISD::CondCode CC, } } - if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { - ISD::NodeType BinOp; - if (SDValue Match = - DAG.matchBinOpReduction(Op.getNode(), BinOp, {ISD::OR})) { - X86::CondCode CCode; - if (SDValue V = - LowerVectorAllZero(DL, Match, CC, Subtarget, DAG, CCode)) { - X86CC = DAG.getTargetConstant(CCode, DL, MVT::i8); - return V; - } - } - } - return SDValue(); } @@ -22602,10 +22580,11 @@ SDValue X86TargetLowering::emitFlagsForSetcc(SDValue Op0, SDValue Op1, // Try to use PTEST/PMOVMSKB for a tree ORs equality compared with 0. // TODO: We could do AND tree with all 1s as well by using the C flag. - if (isNullConstant(Op1) && (CC == ISD::SETEQ || CC == ISD::SETNE)) - if (SDValue CmpZ = - MatchVectorAllZeroTest(Op0, CC, dl, Subtarget, DAG, X86CC)) + if (Op0.getOpcode() == ISD::OR && isNullConstant(Op1) && + (CC == ISD::SETEQ || CC == ISD::SETNE)) { + if (SDValue CmpZ = MatchVectorAllZeroTest(Op0, CC, Subtarget, DAG, X86CC)) return CmpZ; + } // Try to lower using KORTEST or KTEST. if (SDValue Test = EmitAVX512Test(Op0, Op1, CC, dl, DAG, Subtarget, X86CC)) @@ -46085,14 +46064,6 @@ static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG, if (CC == ISD::SETNE || CC == ISD::SETEQ) { if (SDValue V = combineVectorSizedSetCCEquality(N, DAG, Subtarget)) return V; - - if (VT == MVT::i1) { - SDValue X86CC; - if (SDValue V = - MatchVectorAllZeroTest(LHS, CC, DL, Subtarget, DAG, X86CC)) - return DAG.getNode(ISD::TRUNCATE, DL, VT, - DAG.getNode(X86ISD::SETCC, DL, MVT::i8, X86CC, V)); - } } if (VT.isVector() && VT.getVectorElementType() == MVT::i1 && diff --git a/llvm/test/CodeGen/X86/pr45378.ll b/llvm/test/CodeGen/X86/pr45378.ll index 681e06ada65d2..c092378a2a01c 100644 --- a/llvm/test/CodeGen/X86/pr45378.ll +++ b/llvm/test/CodeGen/X86/pr45378.ll @@ -9,29 +9,43 @@ declare i64 @llvm.experimental.vector.reduce.or.v2i64(<2 x i64>) define i1 @parseHeaders(i64 * %ptr) nounwind { -; SSE2-LABEL: parseHeaders: -; SSE2: # %bb.0: -; SSE2-NEXT: movdqu (%rdi), %xmm0 -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: pcmpeqb %xmm0, %xmm1 -; SSE2-NEXT: pmovmskb %xmm1, %eax -; SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF -; SSE2-NEXT: sete %al -; SSE2-NEXT: retq +; SSE-LABEL: parseHeaders: +; SSE: # %bb.0: +; SSE-NEXT: movdqu (%rdi), %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movq %xmm1, %rax +; SSE-NEXT: testq %rax, %rax +; SSE-NEXT: sete %al +; SSE-NEXT: retq ; -; SSE41-LABEL: parseHeaders: -; SSE41: # %bb.0: -; SSE41-NEXT: movdqu (%rdi), %xmm0 -; SSE41-NEXT: ptest %xmm0, %xmm0 -; SSE41-NEXT: sete %al -; SSE41-NEXT: retq +; AVX1-LABEL: parseHeaders: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqu (%rdi), %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vmovq %xmm0, %rax +; AVX1-NEXT: testq %rax, %rax +; AVX1-NEXT: sete %al +; AVX1-NEXT: retq ; -; AVX-LABEL: parseHeaders: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqu (%rdi), %xmm0 -; AVX-NEXT: vptest %xmm0, %xmm0 -; AVX-NEXT: sete %al -; AVX-NEXT: retq +; AVX2-LABEL: parseHeaders: +; AVX2: # %bb.0: +; AVX2-NEXT: vpbroadcastq 8(%rdi), %xmm0 +; AVX2-NEXT: vpor (%rdi), %xmm0, %xmm0 +; AVX2-NEXT: vmovq %xmm0, %rax +; AVX2-NEXT: testq %rax, %rax +; AVX2-NEXT: sete %al +; AVX2-NEXT: retq +; +; AVX512-LABEL: parseHeaders: +; AVX512: # %bb.0: +; AVX512-NEXT: vpbroadcastq 8(%rdi), %xmm0 +; AVX512-NEXT: vpor (%rdi), %xmm0, %xmm0 +; AVX512-NEXT: vmovq %xmm0, %rax +; AVX512-NEXT: testq %rax, %rax +; AVX512-NEXT: sete %al +; AVX512-NEXT: retq %vptr = bitcast i64 * %ptr to <2 x i64> * %vload = load <2 x i64>, <2 x i64> * %vptr, align 8 %vreduce = call i64 @llvm.experimental.vector.reduce.or.v2i64(<2 x i64> %vload) diff --git a/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll b/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll index 194c7402b0bfa..2cb2a07c1e568 100644 --- a/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll +++ b/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll @@ -11,24 +11,21 @@ ; define i1 @test_v2i64(<2 x i64> %a0) { -; SSE2-LABEL: test_v2i64: -; SSE2: # %bb.0: -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: pcmpeqb %xmm0, %xmm1 -; SSE2-NEXT: pmovmskb %xmm1, %eax -; SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF -; SSE2-NEXT: sete %al -; SSE2-NEXT: retq -; -; SSE41-LABEL: test_v2i64: -; SSE41: # %bb.0: -; SSE41-NEXT: ptest %xmm0, %xmm0 -; SSE41-NEXT: sete %al -; SSE41-NEXT: retq +; SSE-LABEL: test_v2i64: +; SSE: # %bb.0: +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movq %xmm1, %rax +; SSE-NEXT: testq %rax, %rax +; SSE-NEXT: sete %al +; SSE-NEXT: retq ; ; AVX-LABEL: test_v2i64: ; AVX: # %bb.0: -; AVX-NEXT: vptest %xmm0, %xmm0 +; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmovq %xmm0, %rax +; AVX-NEXT: testq %rax, %rax ; AVX-NEXT: sete %al ; AVX-NEXT: retq %1 = call i64 @llvm.experimental.vector.reduce.or.v2i64(<2 x i64> %a0) @@ -37,60 +34,78 @@ define i1 @test_v2i64(<2 x i64> %a0) { } define i1 @test_v4i64(<4 x i64> %a0) { -; SSE2-LABEL: test_v4i64: -; SSE2: # %bb.0: -; SSE2-NEXT: por %xmm1, %xmm0 -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: pcmpeqb %xmm0, %xmm1 -; SSE2-NEXT: pmovmskb %xmm1, %eax -; SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF -; SSE2-NEXT: setne %al -; SSE2-NEXT: retq -; -; SSE41-LABEL: test_v4i64: -; SSE41: # %bb.0: -; SSE41-NEXT: por %xmm1, %xmm0 -; SSE41-NEXT: ptest %xmm0, %xmm0 -; SSE41-NEXT: setne %al -; SSE41-NEXT: retq -; -; AVX-LABEL: test_v4i64: -; AVX: # %bb.0: -; AVX-NEXT: vptest %ymm0, %ymm0 -; AVX-NEXT: setne %al -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; SSE-LABEL: test_v4i64: +; SSE: # %bb.0: +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movq %xmm1, %rax +; SSE-NEXT: testq %rax, %rax +; SSE-NEXT: setne %al +; SSE-NEXT: retq +; +; AVX1-LABEL: test_v4i64: +; AVX1: # %bb.0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vmovq %xmm0, %rax +; AVX1-NEXT: testq %rax, %rax +; AVX1-NEXT: setne %al +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: test_v4i64: +; AVX2: # %bb.0: +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vmovq %xmm0, %rax +; AVX2-NEXT: testq %rax, %rax +; AVX2-NEXT: setne %al +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512-LABEL: test_v4i64: +; AVX512: # %bb.0: +; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vmovq %xmm0, %rax +; AVX512-NEXT: testq %rax, %rax +; AVX512-NEXT: setne %al +; AVX512-NEXT: vzeroupper +; AVX512-NEXT: retq %1 = call i64 @llvm.experimental.vector.reduce.or.v4i64(<4 x i64> %a0) %2 = icmp ne i64 %1, 0 ret i1 %2 } define i1 @test_v8i64(<8 x i64> %a0) { -; SSE2-LABEL: test_v8i64: -; SSE2: # %bb.0: -; SSE2-NEXT: por %xmm3, %xmm1 -; SSE2-NEXT: por %xmm2, %xmm1 -; SSE2-NEXT: por %xmm0, %xmm1 -; SSE2-NEXT: pxor %xmm0, %xmm0 -; SSE2-NEXT: pcmpeqb %xmm1, %xmm0 -; SSE2-NEXT: pmovmskb %xmm0, %eax -; SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF -; SSE2-NEXT: sete %al -; SSE2-NEXT: retq -; -; SSE41-LABEL: test_v8i64: -; SSE41: # %bb.0: -; SSE41-NEXT: por %xmm3, %xmm1 -; SSE41-NEXT: por %xmm2, %xmm1 -; SSE41-NEXT: por %xmm0, %xmm1 -; SSE41-NEXT: ptest %xmm1, %xmm1 -; SSE41-NEXT: sete %al -; SSE41-NEXT: retq +; SSE-LABEL: test_v8i64: +; SSE: # %bb.0: +; SSE-NEXT: por %xmm3, %xmm1 +; SSE-NEXT: por %xmm2, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: movq %xmm0, %rax +; SSE-NEXT: testq %rax, %rax +; SSE-NEXT: sete %al +; SSE-NEXT: retq ; ; AVX1-LABEL: test_v8i64: ; AVX1: # %bb.0: ; AVX1-NEXT: vorps %ymm1, %ymm0, %ymm0 -; AVX1-NEXT: vptest %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vmovq %xmm0, %rax +; AVX1-NEXT: testq %rax, %rax ; AVX1-NEXT: sete %al ; AVX1-NEXT: vzeroupper ; AVX1-NEXT: retq @@ -98,7 +113,12 @@ define i1 @test_v8i64(<8 x i64> %a0) { ; AVX2-LABEL: test_v8i64: ; AVX2: # %bb.0: ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: vptest %ymm0, %ymm0 +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vmovq %xmm0, %rax +; AVX2-NEXT: testq %rax, %rax ; AVX2-NEXT: sete %al ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq @@ -106,8 +126,13 @@ define i1 @test_v8i64(<8 x i64> %a0) { ; AVX512-LABEL: test_v8i64: ; AVX512: # %bb.0: ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: vptest %ymm0, %ymm0 +; AVX512-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vmovq %xmm0, %rax +; AVX512-NEXT: testq %rax, %rax ; AVX512-NEXT: sete %al ; AVX512-NEXT: vzeroupper ; AVX512-NEXT: retq @@ -117,41 +142,33 @@ define i1 @test_v8i64(<8 x i64> %a0) { } define i1 @test_v16i64(<16 x i64> %a0) { -; SSE2-LABEL: test_v16i64: -; SSE2: # %bb.0: -; SSE2-NEXT: por %xmm7, %xmm3 -; SSE2-NEXT: por %xmm5, %xmm3 -; SSE2-NEXT: por %xmm1, %xmm3 -; SSE2-NEXT: por %xmm6, %xmm2 -; SSE2-NEXT: por %xmm4, %xmm2 -; SSE2-NEXT: por %xmm3, %xmm2 -; SSE2-NEXT: por %xmm0, %xmm2 -; SSE2-NEXT: pxor %xmm0, %xmm0 -; SSE2-NEXT: pcmpeqb %xmm2, %xmm0 -; SSE2-NEXT: pmovmskb %xmm0, %eax -; SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF -; SSE2-NEXT: setne %al -; SSE2-NEXT: retq -; -; SSE41-LABEL: test_v16i64: -; SSE41: # %bb.0: -; SSE41-NEXT: por %xmm7, %xmm3 -; SSE41-NEXT: por %xmm5, %xmm3 -; SSE41-NEXT: por %xmm1, %xmm3 -; SSE41-NEXT: por %xmm6, %xmm2 -; SSE41-NEXT: por %xmm4, %xmm2 -; SSE41-NEXT: por %xmm3, %xmm2 -; SSE41-NEXT: por %xmm0, %xmm2 -; SSE41-NEXT: ptest %xmm2, %xmm2 -; SSE41-NEXT: setne %al -; SSE41-NEXT: retq +; SSE-LABEL: test_v16i64: +; SSE: # %bb.0: +; SSE-NEXT: por %xmm6, %xmm2 +; SSE-NEXT: por %xmm7, %xmm3 +; SSE-NEXT: por %xmm5, %xmm3 +; SSE-NEXT: por %xmm1, %xmm3 +; SSE-NEXT: por %xmm4, %xmm2 +; SSE-NEXT: por %xmm3, %xmm2 +; SSE-NEXT: por %xmm0, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1] +; SSE-NEXT: por %xmm2, %xmm0 +; SSE-NEXT: movq %xmm0, %rax +; SSE-NEXT: testq %rax, %rax +; SSE-NEXT: setne %al +; SSE-NEXT: retq ; ; AVX1-LABEL: test_v16i64: ; AVX1: # %bb.0: ; AVX1-NEXT: vorps %ymm3, %ymm1, %ymm1 ; AVX1-NEXT: vorps %ymm1, %ymm2, %ymm1 ; AVX1-NEXT: vorps %ymm1, %ymm0, %ymm0 -; AVX1-NEXT: vptest %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vmovq %xmm0, %rax +; AVX1-NEXT: testq %rax, %rax ; AVX1-NEXT: setne %al ; AVX1-NEXT: vzeroupper ; AVX1-NEXT: retq @@ -161,7 +178,12 @@ define i1 @test_v16i64(<16 x i64> %a0) { ; AVX2-NEXT: vpor %ymm3, %ymm1, %ymm1 ; AVX2-NEXT: vpor %ymm1, %ymm2, %ymm1 ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: vptest %ymm0, %ymm0 +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vmovq %xmm0, %rax +; AVX2-NEXT: testq %rax, %rax ; AVX2-NEXT: setne %al ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq @@ -170,8 +192,13 @@ define i1 @test_v16i64(<16 x i64> %a0) { ; AVX512: # %bb.0: ; AVX512-NEXT: vporq %zmm1, %zmm0, %zmm0 ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: vptest %ymm0, %ymm0 +; AVX512-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vmovq %xmm0, %rax +; AVX512-NEXT: testq %rax, %rax ; AVX512-NEXT: setne %al ; AVX512-NEXT: vzeroupper ; AVX512-NEXT: retq @@ -187,15 +214,19 @@ define i1 @test_v16i64(<16 x i64> %a0) { define i1 @test_v2i32(<2 x i32> %a0) { ; SSE-LABEL: test_v2i32: ; SSE: # %bb.0: -; SSE-NEXT: movq %xmm0, %rax -; SSE-NEXT: testq %rax, %rax +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movd %xmm1, %eax +; SSE-NEXT: testl %eax, %eax ; SSE-NEXT: sete %al ; SSE-NEXT: retq ; ; AVX-LABEL: test_v2i32: ; AVX: # %bb.0: -; AVX-NEXT: vmovq %xmm0, %rax -; AVX-NEXT: testq %rax, %rax +; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmovd %xmm0, %eax +; AVX-NEXT: testl %eax, %eax ; AVX-NEXT: sete %al ; AVX-NEXT: retq %1 = call i32 @llvm.experimental.vector.reduce.or.v2i32(<2 x i32> %a0) @@ -204,24 +235,25 @@ define i1 @test_v2i32(<2 x i32> %a0) { } define i1 @test_v4i32(<4 x i32> %a0) { -; SSE2-LABEL: test_v4i32: -; SSE2: # %bb.0: -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: pcmpeqb %xmm0, %xmm1 -; SSE2-NEXT: pmovmskb %xmm1, %eax -; SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF -; SSE2-NEXT: setne %al -; SSE2-NEXT: retq -; -; SSE41-LABEL: test_v4i32: -; SSE41: # %bb.0: -; SSE41-NEXT: ptest %xmm0, %xmm0 -; SSE41-NEXT: setne %al -; SSE41-NEXT: retq +; SSE-LABEL: test_v4i32: +; SSE: # %bb.0: +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3] +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: movd %xmm0, %eax +; SSE-NEXT: testl %eax, %eax +; SSE-NEXT: setne %al +; SSE-NEXT: retq ; ; AVX-LABEL: test_v4i32: ; AVX: # %bb.0: -; AVX-NEXT: vptest %xmm0, %xmm0 +; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmovd %xmm0, %eax +; AVX-NEXT: testl %eax, %eax ; AVX-NEXT: setne %al ; AVX-NEXT: retq %1 = call i32 @llvm.experimental.vector.reduce.or.v4i32(<4 x i32> %a0) @@ -230,60 +262,90 @@ define i1 @test_v4i32(<4 x i32> %a0) { } define i1 @test_v8i32(<8 x i32> %a0) { -; SSE2-LABEL: test_v8i32: -; SSE2: # %bb.0: -; SSE2-NEXT: por %xmm1, %xmm0 -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: pcmpeqb %xmm0, %xmm1 -; SSE2-NEXT: pmovmskb %xmm1, %eax -; SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF -; SSE2-NEXT: sete %al -; SSE2-NEXT: retq -; -; SSE41-LABEL: test_v8i32: -; SSE41: # %bb.0: -; SSE41-NEXT: por %xmm1, %xmm0 -; SSE41-NEXT: ptest %xmm0, %xmm0 -; SSE41-NEXT: sete %al -; SSE41-NEXT: retq -; -; AVX-LABEL: test_v8i32: -; AVX: # %bb.0: -; AVX-NEXT: vptest %ymm0, %ymm0 -; AVX-NEXT: sete %al -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; SSE-LABEL: test_v8i32: +; SSE: # %bb.0: +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3] +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: movd %xmm0, %eax +; SSE-NEXT: testl %eax, %eax +; SSE-NEXT: sete %al +; SSE-NEXT: retq +; +; AVX1-LABEL: test_v8i32: +; AVX1: # %bb.0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vmovd %xmm0, %eax +; AVX1-NEXT: testl %eax, %eax +; AVX1-NEXT: sete %al +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: test_v8i32: +; AVX2: # %bb.0: +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vmovd %xmm0, %eax +; AVX2-NEXT: testl %eax, %eax +; AVX2-NEXT: sete %al +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512-LABEL: test_v8i32: +; AVX512: # %bb.0: +; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vmovd %xmm0, %eax +; AVX512-NEXT: testl %eax, %eax +; AVX512-NEXT: sete %al +; AVX512-NEXT: vzeroupper +; AVX512-NEXT: retq %1 = call i32 @llvm.experimental.vector.reduce.or.v8i32(<8 x i32> %a0) %2 = icmp eq i32 %1, 0 ret i1 %2 } define i1 @test_v16i32(<16 x i32> %a0) { -; SSE2-LABEL: test_v16i32: -; SSE2: # %bb.0: -; SSE2-NEXT: por %xmm3, %xmm1 -; SSE2-NEXT: por %xmm2, %xmm1 -; SSE2-NEXT: por %xmm0, %xmm1 -; SSE2-NEXT: pxor %xmm0, %xmm0 -; SSE2-NEXT: pcmpeqb %xmm1, %xmm0 -; SSE2-NEXT: pmovmskb %xmm0, %eax -; SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF -; SSE2-NEXT: setne %al -; SSE2-NEXT: retq -; -; SSE41-LABEL: test_v16i32: -; SSE41: # %bb.0: -; SSE41-NEXT: por %xmm3, %xmm1 -; SSE41-NEXT: por %xmm2, %xmm1 -; SSE41-NEXT: por %xmm0, %xmm1 -; SSE41-NEXT: ptest %xmm1, %xmm1 -; SSE41-NEXT: setne %al -; SSE41-NEXT: retq +; SSE-LABEL: test_v16i32: +; SSE: # %bb.0: +; SSE-NEXT: por %xmm3, %xmm1 +; SSE-NEXT: por %xmm2, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movd %xmm1, %eax +; SSE-NEXT: testl %eax, %eax +; SSE-NEXT: setne %al +; SSE-NEXT: retq ; ; AVX1-LABEL: test_v16i32: ; AVX1: # %bb.0: ; AVX1-NEXT: vorps %ymm1, %ymm0, %ymm0 -; AVX1-NEXT: vptest %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vmovd %xmm0, %eax +; AVX1-NEXT: testl %eax, %eax ; AVX1-NEXT: setne %al ; AVX1-NEXT: vzeroupper ; AVX1-NEXT: retq @@ -291,7 +353,14 @@ define i1 @test_v16i32(<16 x i32> %a0) { ; AVX2-LABEL: test_v16i32: ; AVX2: # %bb.0: ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: vptest %ymm0, %ymm0 +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vmovd %xmm0, %eax +; AVX2-NEXT: testl %eax, %eax ; AVX2-NEXT: setne %al ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq @@ -299,8 +368,15 @@ define i1 @test_v16i32(<16 x i32> %a0) { ; AVX512-LABEL: test_v16i32: ; AVX512: # %bb.0: ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: vptest %ymm0, %ymm0 +; AVX512-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vmovd %xmm0, %eax +; AVX512-NEXT: testl %eax, %eax ; AVX512-NEXT: setne %al ; AVX512-NEXT: vzeroupper ; AVX512-NEXT: retq @@ -310,41 +386,37 @@ define i1 @test_v16i32(<16 x i32> %a0) { } define i1 @test_v32i32(<32 x i32> %a0) { -; SSE2-LABEL: test_v32i32: -; SSE2: # %bb.0: -; SSE2-NEXT: por %xmm7, %xmm3 -; SSE2-NEXT: por %xmm5, %xmm3 -; SSE2-NEXT: por %xmm1, %xmm3 -; SSE2-NEXT: por %xmm6, %xmm2 -; SSE2-NEXT: por %xmm4, %xmm2 -; SSE2-NEXT: por %xmm3, %xmm2 -; SSE2-NEXT: por %xmm0, %xmm2 -; SSE2-NEXT: pxor %xmm0, %xmm0 -; SSE2-NEXT: pcmpeqb %xmm2, %xmm0 -; SSE2-NEXT: pmovmskb %xmm0, %eax -; SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF -; SSE2-NEXT: sete %al -; SSE2-NEXT: retq -; -; SSE41-LABEL: test_v32i32: -; SSE41: # %bb.0: -; SSE41-NEXT: por %xmm7, %xmm3 -; SSE41-NEXT: por %xmm5, %xmm3 -; SSE41-NEXT: por %xmm1, %xmm3 -; SSE41-NEXT: por %xmm6, %xmm2 -; SSE41-NEXT: por %xmm4, %xmm2 -; SSE41-NEXT: por %xmm3, %xmm2 -; SSE41-NEXT: por %xmm0, %xmm2 -; SSE41-NEXT: ptest %xmm2, %xmm2 -; SSE41-NEXT: sete %al -; SSE41-NEXT: retq +; SSE-LABEL: test_v32i32: +; SSE: # %bb.0: +; SSE-NEXT: por %xmm6, %xmm2 +; SSE-NEXT: por %xmm7, %xmm3 +; SSE-NEXT: por %xmm5, %xmm3 +; SSE-NEXT: por %xmm1, %xmm3 +; SSE-NEXT: por %xmm4, %xmm2 +; SSE-NEXT: por %xmm3, %xmm2 +; SSE-NEXT: por %xmm0, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1] +; SSE-NEXT: por %xmm2, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movd %xmm1, %eax +; SSE-NEXT: testl %eax, %eax +; SSE-NEXT: sete %al +; SSE-NEXT: retq ; ; AVX1-LABEL: test_v32i32: ; AVX1: # %bb.0: ; AVX1-NEXT: vorps %ymm3, %ymm1, %ymm1 ; AVX1-NEXT: vorps %ymm1, %ymm2, %ymm1 ; AVX1-NEXT: vorps %ymm1, %ymm0, %ymm0 -; AVX1-NEXT: vptest %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vmovd %xmm0, %eax +; AVX1-NEXT: testl %eax, %eax ; AVX1-NEXT: sete %al ; AVX1-NEXT: vzeroupper ; AVX1-NEXT: retq @@ -354,7 +426,14 @@ define i1 @test_v32i32(<32 x i32> %a0) { ; AVX2-NEXT: vpor %ymm3, %ymm1, %ymm1 ; AVX2-NEXT: vpor %ymm1, %ymm2, %ymm1 ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: vptest %ymm0, %ymm0 +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vmovd %xmm0, %eax +; AVX2-NEXT: testl %eax, %eax ; AVX2-NEXT: sete %al ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq @@ -363,8 +442,15 @@ define i1 @test_v32i32(<32 x i32> %a0) { ; AVX512: # %bb.0: ; AVX512-NEXT: vpord %zmm1, %zmm0, %zmm0 ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: vptest %ymm0, %ymm0 +; AVX512-NEXT: vpord %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vmovd %xmm0, %eax +; AVX512-NEXT: testl %eax, %eax ; AVX512-NEXT: sete %al ; AVX512-NEXT: vzeroupper ; AVX512-NEXT: retq @@ -380,15 +466,20 @@ define i1 @test_v32i32(<32 x i32> %a0) { define i1 @test_v2i16(<2 x i16> %a0) { ; SSE-LABEL: test_v2i16: ; SSE: # %bb.0: -; SSE-NEXT: movd %xmm0, %eax -; SSE-NEXT: testl %eax, %eax +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psrld $16, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movd %xmm1, %eax +; SSE-NEXT: testw %ax, %ax ; SSE-NEXT: sete %al ; SSE-NEXT: retq ; ; AVX-LABEL: test_v2i16: ; AVX: # %bb.0: +; AVX-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX-NEXT: vmovd %xmm0, %eax -; AVX-NEXT: testl %eax, %eax +; AVX-NEXT: testw %ax, %ax ; AVX-NEXT: sete %al ; AVX-NEXT: retq %1 = call i16 @llvm.experimental.vector.reduce.or.v2i16(<2 x i16> %a0) @@ -399,15 +490,24 @@ define i1 @test_v2i16(<2 x i16> %a0) { define i1 @test_v4i16(<4 x i16> %a0) { ; SSE-LABEL: test_v4i16: ; SSE: # %bb.0: -; SSE-NEXT: movq %xmm0, %rax -; SSE-NEXT: testq %rax, %rax +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: psrld $16, %xmm0 +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: movd %xmm0, %eax +; SSE-NEXT: testw %ax, %ax ; SSE-NEXT: setne %al ; SSE-NEXT: retq ; ; AVX-LABEL: test_v4i16: ; AVX: # %bb.0: -; AVX-NEXT: vmovq %xmm0, %rax -; AVX-NEXT: testq %rax, %rax +; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmovd %xmm0, %eax +; AVX-NEXT: testw %ax, %ax ; AVX-NEXT: setne %al ; AVX-NEXT: retq %1 = call i16 @llvm.experimental.vector.reduce.or.v4i16(<4 x i16> %a0) @@ -416,24 +516,30 @@ define i1 @test_v4i16(<4 x i16> %a0) { } define i1 @test_v8i16(<8 x i16> %a0) { -; SSE2-LABEL: test_v8i16: -; SSE2: # %bb.0: -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: pcmpeqb %xmm0, %xmm1 -; SSE2-NEXT: pmovmskb %xmm1, %eax -; SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF -; SSE2-NEXT: sete %al -; SSE2-NEXT: retq -; -; SSE41-LABEL: test_v8i16: -; SSE41: # %bb.0: -; SSE41-NEXT: ptest %xmm0, %xmm0 -; SSE41-NEXT: sete %al -; SSE41-NEXT: retq +; SSE-LABEL: test_v8i16: +; SSE: # %bb.0: +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3] +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psrld $16, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movd %xmm1, %eax +; SSE-NEXT: testw %ax, %ax +; SSE-NEXT: sete %al +; SSE-NEXT: retq ; ; AVX-LABEL: test_v8i16: ; AVX: # %bb.0: -; AVX-NEXT: vptest %xmm0, %xmm0 +; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmovd %xmm0, %eax +; AVX-NEXT: testw %ax, %ax ; AVX-NEXT: sete %al ; AVX-NEXT: retq %1 = call i16 @llvm.experimental.vector.reduce.or.v8i16(<8 x i16> %a0) @@ -442,60 +548,104 @@ define i1 @test_v8i16(<8 x i16> %a0) { } define i1 @test_v16i16(<16 x i16> %a0) { -; SSE2-LABEL: test_v16i16: -; SSE2: # %bb.0: -; SSE2-NEXT: por %xmm1, %xmm0 -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: pcmpeqb %xmm0, %xmm1 -; SSE2-NEXT: pmovmskb %xmm1, %eax -; SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF -; SSE2-NEXT: setne %al -; SSE2-NEXT: retq -; -; SSE41-LABEL: test_v16i16: -; SSE41: # %bb.0: -; SSE41-NEXT: por %xmm1, %xmm0 -; SSE41-NEXT: ptest %xmm0, %xmm0 -; SSE41-NEXT: setne %al -; SSE41-NEXT: retq -; -; AVX-LABEL: test_v16i16: -; AVX: # %bb.0: -; AVX-NEXT: vptest %ymm0, %ymm0 -; AVX-NEXT: setne %al -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; SSE-LABEL: test_v16i16: +; SSE: # %bb.0: +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3] +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psrld $16, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movd %xmm1, %eax +; SSE-NEXT: testw %ax, %ax +; SSE-NEXT: setne %al +; SSE-NEXT: retq +; +; AVX1-LABEL: test_v16i16: +; AVX1: # %bb.0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vmovd %xmm0, %eax +; AVX1-NEXT: testw %ax, %ax +; AVX1-NEXT: setne %al +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: test_v16i16: +; AVX2: # %bb.0: +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vmovd %xmm0, %eax +; AVX2-NEXT: testw %ax, %ax +; AVX2-NEXT: setne %al +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512-LABEL: test_v16i16: +; AVX512: # %bb.0: +; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vmovd %xmm0, %eax +; AVX512-NEXT: testw %ax, %ax +; AVX512-NEXT: setne %al +; AVX512-NEXT: vzeroupper +; AVX512-NEXT: retq %1 = call i16 @llvm.experimental.vector.reduce.or.v16i16(<16 x i16> %a0) %2 = icmp ne i16 %1, 0 ret i1 %2 } define i1 @test_v32i16(<32 x i16> %a0) { -; SSE2-LABEL: test_v32i16: -; SSE2: # %bb.0: -; SSE2-NEXT: por %xmm3, %xmm1 -; SSE2-NEXT: por %xmm2, %xmm1 -; SSE2-NEXT: por %xmm0, %xmm1 -; SSE2-NEXT: pxor %xmm0, %xmm0 -; SSE2-NEXT: pcmpeqb %xmm1, %xmm0 -; SSE2-NEXT: pmovmskb %xmm0, %eax -; SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF -; SSE2-NEXT: sete %al -; SSE2-NEXT: retq -; -; SSE41-LABEL: test_v32i16: -; SSE41: # %bb.0: -; SSE41-NEXT: por %xmm3, %xmm1 -; SSE41-NEXT: por %xmm2, %xmm1 -; SSE41-NEXT: por %xmm0, %xmm1 -; SSE41-NEXT: ptest %xmm1, %xmm1 -; SSE41-NEXT: sete %al -; SSE41-NEXT: retq +; SSE-LABEL: test_v32i16: +; SSE: # %bb.0: +; SSE-NEXT: por %xmm3, %xmm1 +; SSE-NEXT: por %xmm2, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: psrld $16, %xmm0 +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: movd %xmm0, %eax +; SSE-NEXT: testw %ax, %ax +; SSE-NEXT: sete %al +; SSE-NEXT: retq ; ; AVX1-LABEL: test_v32i16: ; AVX1: # %bb.0: ; AVX1-NEXT: vorps %ymm1, %ymm0, %ymm0 -; AVX1-NEXT: vptest %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vmovd %xmm0, %eax +; AVX1-NEXT: testw %ax, %ax ; AVX1-NEXT: sete %al ; AVX1-NEXT: vzeroupper ; AVX1-NEXT: retq @@ -503,7 +653,16 @@ define i1 @test_v32i16(<32 x i16> %a0) { ; AVX2-LABEL: test_v32i16: ; AVX2: # %bb.0: ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: vptest %ymm0, %ymm0 +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vmovd %xmm0, %eax +; AVX2-NEXT: testw %ax, %ax ; AVX2-NEXT: sete %al ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq @@ -511,8 +670,17 @@ define i1 @test_v32i16(<32 x i16> %a0) { ; AVX512-LABEL: test_v32i16: ; AVX512: # %bb.0: ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: vptest %ymm0, %ymm0 +; AVX512-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vmovd %xmm0, %eax +; AVX512-NEXT: testw %ax, %ax ; AVX512-NEXT: sete %al ; AVX512-NEXT: vzeroupper ; AVX512-NEXT: retq @@ -522,41 +690,42 @@ define i1 @test_v32i16(<32 x i16> %a0) { } define i1 @test_v64i16(<64 x i16> %a0) { -; SSE2-LABEL: test_v64i16: -; SSE2: # %bb.0: -; SSE2-NEXT: por %xmm7, %xmm3 -; SSE2-NEXT: por %xmm5, %xmm3 -; SSE2-NEXT: por %xmm1, %xmm3 -; SSE2-NEXT: por %xmm6, %xmm2 -; SSE2-NEXT: por %xmm4, %xmm2 -; SSE2-NEXT: por %xmm3, %xmm2 -; SSE2-NEXT: por %xmm0, %xmm2 -; SSE2-NEXT: pxor %xmm0, %xmm0 -; SSE2-NEXT: pcmpeqb %xmm2, %xmm0 -; SSE2-NEXT: pmovmskb %xmm0, %eax -; SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF -; SSE2-NEXT: setne %al -; SSE2-NEXT: retq -; -; SSE41-LABEL: test_v64i16: -; SSE41: # %bb.0: -; SSE41-NEXT: por %xmm7, %xmm3 -; SSE41-NEXT: por %xmm5, %xmm3 -; SSE41-NEXT: por %xmm1, %xmm3 -; SSE41-NEXT: por %xmm6, %xmm2 -; SSE41-NEXT: por %xmm4, %xmm2 -; SSE41-NEXT: por %xmm3, %xmm2 -; SSE41-NEXT: por %xmm0, %xmm2 -; SSE41-NEXT: ptest %xmm2, %xmm2 -; SSE41-NEXT: setne %al -; SSE41-NEXT: retq +; SSE-LABEL: test_v64i16: +; SSE: # %bb.0: +; SSE-NEXT: por %xmm6, %xmm2 +; SSE-NEXT: por %xmm7, %xmm3 +; SSE-NEXT: por %xmm5, %xmm3 +; SSE-NEXT: por %xmm1, %xmm3 +; SSE-NEXT: por %xmm4, %xmm2 +; SSE-NEXT: por %xmm3, %xmm2 +; SSE-NEXT: por %xmm0, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1] +; SSE-NEXT: por %xmm2, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: psrld $16, %xmm0 +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: movd %xmm0, %eax +; SSE-NEXT: testw %ax, %ax +; SSE-NEXT: setne %al +; SSE-NEXT: retq ; ; AVX1-LABEL: test_v64i16: ; AVX1: # %bb.0: ; AVX1-NEXT: vorps %ymm3, %ymm1, %ymm1 ; AVX1-NEXT: vorps %ymm1, %ymm2, %ymm1 ; AVX1-NEXT: vorps %ymm1, %ymm0, %ymm0 -; AVX1-NEXT: vptest %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vmovd %xmm0, %eax +; AVX1-NEXT: testw %ax, %ax ; AVX1-NEXT: setne %al ; AVX1-NEXT: vzeroupper ; AVX1-NEXT: retq @@ -566,7 +735,16 @@ define i1 @test_v64i16(<64 x i16> %a0) { ; AVX2-NEXT: vpor %ymm3, %ymm1, %ymm1 ; AVX2-NEXT: vpor %ymm1, %ymm2, %ymm1 ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: vptest %ymm0, %ymm0 +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vmovd %xmm0, %eax +; AVX2-NEXT: testw %ax, %ax ; AVX2-NEXT: setne %al ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq @@ -575,8 +753,17 @@ define i1 @test_v64i16(<64 x i16> %a0) { ; AVX512: # %bb.0: ; AVX512-NEXT: vporq %zmm1, %zmm0, %zmm0 ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: vptest %ymm0, %ymm0 +; AVX512-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vmovd %xmm0, %eax +; AVX512-NEXT: testw %ax, %ax ; AVX512-NEXT: setne %al ; AVX512-NEXT: vzeroupper ; AVX512-NEXT: retq @@ -592,15 +779,20 @@ define i1 @test_v64i16(<64 x i16> %a0) { define i1 @test_v2i8(<2 x i8> %a0) { ; SSE-LABEL: test_v2i8: ; SSE: # %bb.0: -; SSE-NEXT: movd %xmm0, %eax -; SSE-NEXT: testw %ax, %ax +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psrlw $8, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movd %xmm1, %eax +; SSE-NEXT: testb %al, %al ; SSE-NEXT: sete %al ; SSE-NEXT: retq ; ; AVX-LABEL: test_v2i8: ; AVX: # %bb.0: +; AVX-NEXT: vpsrlw $8, %xmm0, %xmm1 +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX-NEXT: vmovd %xmm0, %eax -; AVX-NEXT: testw %ax, %ax +; AVX-NEXT: testb %al, %al ; AVX-NEXT: sete %al ; AVX-NEXT: retq %1 = call i8 @llvm.experimental.vector.reduce.or.v2i8(<2 x i8> %a0) @@ -611,15 +803,25 @@ define i1 @test_v2i8(<2 x i8> %a0) { define i1 @test_v4i8(<4 x i8> %a0) { ; SSE-LABEL: test_v4i8: ; SSE: # %bb.0: +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psrld $16, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: psrlw $8, %xmm0 +; SSE-NEXT: por %xmm1, %xmm0 ; SSE-NEXT: movd %xmm0, %eax -; SSE-NEXT: testl %eax, %eax +; SSE-NEXT: testb %al, %al ; SSE-NEXT: setne %al ; SSE-NEXT: retq ; ; AVX-LABEL: test_v4i8: ; AVX: # %bb.0: +; AVX-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpsrlw $8, %xmm0, %xmm1 +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX-NEXT: vmovd %xmm0, %eax -; AVX-NEXT: testl %eax, %eax +; AVX-NEXT: testb %al, %al ; AVX-NEXT: setne %al ; AVX-NEXT: retq %1 = call i8 @llvm.experimental.vector.reduce.or.v4i8(<4 x i8> %a0) @@ -630,15 +832,29 @@ define i1 @test_v4i8(<4 x i8> %a0) { define i1 @test_v8i8(<8 x i8> %a0) { ; SSE-LABEL: test_v8i8: ; SSE: # %bb.0: -; SSE-NEXT: movq %xmm0, %rax -; SSE-NEXT: testq %rax, %rax +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: psrld $16, %xmm0 +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psrlw $8, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movd %xmm1, %eax +; SSE-NEXT: testb %al, %al ; SSE-NEXT: sete %al ; SSE-NEXT: retq ; ; AVX-LABEL: test_v8i8: ; AVX: # %bb.0: -; AVX-NEXT: vmovq %xmm0, %rax -; AVX-NEXT: testq %rax, %rax +; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpsrlw $8, %xmm0, %xmm1 +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmovd %xmm0, %eax +; AVX-NEXT: testb %al, %al ; AVX-NEXT: sete %al ; AVX-NEXT: retq %1 = call i8 @llvm.experimental.vector.reduce.or.v8i8(<8 x i8> %a0) @@ -647,24 +863,35 @@ define i1 @test_v8i8(<8 x i8> %a0) { } define i1 @test_v16i8(<16 x i8> %a0) { -; SSE2-LABEL: test_v16i8: -; SSE2: # %bb.0: -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: pcmpeqb %xmm0, %xmm1 -; SSE2-NEXT: pmovmskb %xmm1, %eax -; SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF -; SSE2-NEXT: setne %al -; SSE2-NEXT: retq -; -; SSE41-LABEL: test_v16i8: -; SSE41: # %bb.0: -; SSE41-NEXT: ptest %xmm0, %xmm0 -; SSE41-NEXT: setne %al -; SSE41-NEXT: retq +; SSE-LABEL: test_v16i8: +; SSE: # %bb.0: +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3] +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psrld $16, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: psrlw $8, %xmm0 +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: movd %xmm0, %eax +; SSE-NEXT: testb %al, %al +; SSE-NEXT: setne %al +; SSE-NEXT: retq ; ; AVX-LABEL: test_v16i8: ; AVX: # %bb.0: -; AVX-NEXT: vptest %xmm0, %xmm0 +; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpsrlw $8, %xmm0, %xmm1 +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmovd %xmm0, %eax +; AVX-NEXT: testb %al, %al ; AVX-NEXT: setne %al ; AVX-NEXT: retq %1 = call i8 @llvm.experimental.vector.reduce.or.v16i8(<16 x i8> %a0) @@ -673,60 +900,118 @@ define i1 @test_v16i8(<16 x i8> %a0) { } define i1 @test_v32i8(<32 x i8> %a0) { -; SSE2-LABEL: test_v32i8: -; SSE2: # %bb.0: -; SSE2-NEXT: por %xmm1, %xmm0 -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: pcmpeqb %xmm0, %xmm1 -; SSE2-NEXT: pmovmskb %xmm1, %eax -; SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF -; SSE2-NEXT: sete %al -; SSE2-NEXT: retq -; -; SSE41-LABEL: test_v32i8: -; SSE41: # %bb.0: -; SSE41-NEXT: por %xmm1, %xmm0 -; SSE41-NEXT: ptest %xmm0, %xmm0 -; SSE41-NEXT: sete %al -; SSE41-NEXT: retq -; -; AVX-LABEL: test_v32i8: -; AVX: # %bb.0: -; AVX-NEXT: vptest %ymm0, %ymm0 -; AVX-NEXT: sete %al -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; SSE-LABEL: test_v32i8: +; SSE: # %bb.0: +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3] +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psrld $16, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: psrlw $8, %xmm0 +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: movd %xmm0, %eax +; SSE-NEXT: testb %al, %al +; SSE-NEXT: sete %al +; SSE-NEXT: retq +; +; AVX1-LABEL: test_v32i8: +; AVX1: # %bb.0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm1 +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vmovd %xmm0, %eax +; AVX1-NEXT: testb %al, %al +; AVX1-NEXT: sete %al +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: test_v32i8: +; AVX2: # %bb.0: +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vmovd %xmm0, %eax +; AVX2-NEXT: testb %al, %al +; AVX2-NEXT: sete %al +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512-LABEL: test_v32i8: +; AVX512: # %bb.0: +; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpsrlw $8, %xmm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vmovd %xmm0, %eax +; AVX512-NEXT: testb %al, %al +; AVX512-NEXT: sete %al +; AVX512-NEXT: vzeroupper +; AVX512-NEXT: retq %1 = call i8 @llvm.experimental.vector.reduce.or.v32i8(<32 x i8> %a0) %2 = icmp eq i8 %1, 0 ret i1 %2 } define i1 @test_v64i8(<64 x i8> %a0) { -; SSE2-LABEL: test_v64i8: -; SSE2: # %bb.0: -; SSE2-NEXT: por %xmm3, %xmm1 -; SSE2-NEXT: por %xmm2, %xmm1 -; SSE2-NEXT: por %xmm0, %xmm1 -; SSE2-NEXT: pxor %xmm0, %xmm0 -; SSE2-NEXT: pcmpeqb %xmm1, %xmm0 -; SSE2-NEXT: pmovmskb %xmm0, %eax -; SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF -; SSE2-NEXT: setne %al -; SSE2-NEXT: retq -; -; SSE41-LABEL: test_v64i8: -; SSE41: # %bb.0: -; SSE41-NEXT: por %xmm3, %xmm1 -; SSE41-NEXT: por %xmm2, %xmm1 -; SSE41-NEXT: por %xmm0, %xmm1 -; SSE41-NEXT: ptest %xmm1, %xmm1 -; SSE41-NEXT: setne %al -; SSE41-NEXT: retq +; SSE-LABEL: test_v64i8: +; SSE: # %bb.0: +; SSE-NEXT: por %xmm3, %xmm1 +; SSE-NEXT: por %xmm2, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: psrld $16, %xmm0 +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psrlw $8, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movd %xmm1, %eax +; SSE-NEXT: testb %al, %al +; SSE-NEXT: setne %al +; SSE-NEXT: retq ; ; AVX1-LABEL: test_v64i8: ; AVX1: # %bb.0: ; AVX1-NEXT: vorps %ymm1, %ymm0, %ymm0 -; AVX1-NEXT: vptest %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm1 +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vmovd %xmm0, %eax +; AVX1-NEXT: testb %al, %al ; AVX1-NEXT: setne %al ; AVX1-NEXT: vzeroupper ; AVX1-NEXT: retq @@ -734,7 +1019,18 @@ define i1 @test_v64i8(<64 x i8> %a0) { ; AVX2-LABEL: test_v64i8: ; AVX2: # %bb.0: ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: vptest %ymm0, %ymm0 +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vmovd %xmm0, %eax +; AVX2-NEXT: testb %al, %al ; AVX2-NEXT: setne %al ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq @@ -742,8 +1038,19 @@ define i1 @test_v64i8(<64 x i8> %a0) { ; AVX512-LABEL: test_v64i8: ; AVX512: # %bb.0: ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: vptest %ymm0, %ymm0 +; AVX512-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpsrlw $8, %xmm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vmovd %xmm0, %eax +; AVX512-NEXT: testb %al, %al ; AVX512-NEXT: setne %al ; AVX512-NEXT: vzeroupper ; AVX512-NEXT: retq @@ -753,41 +1060,47 @@ define i1 @test_v64i8(<64 x i8> %a0) { } define i1 @test_v128i8(<128 x i8> %a0) { -; SSE2-LABEL: test_v128i8: -; SSE2: # %bb.0: -; SSE2-NEXT: por %xmm7, %xmm3 -; SSE2-NEXT: por %xmm5, %xmm3 -; SSE2-NEXT: por %xmm1, %xmm3 -; SSE2-NEXT: por %xmm6, %xmm2 -; SSE2-NEXT: por %xmm4, %xmm2 -; SSE2-NEXT: por %xmm3, %xmm2 -; SSE2-NEXT: por %xmm0, %xmm2 -; SSE2-NEXT: pxor %xmm0, %xmm0 -; SSE2-NEXT: pcmpeqb %xmm2, %xmm0 -; SSE2-NEXT: pmovmskb %xmm0, %eax -; SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF -; SSE2-NEXT: sete %al -; SSE2-NEXT: retq -; -; SSE41-LABEL: test_v128i8: -; SSE41: # %bb.0: -; SSE41-NEXT: por %xmm7, %xmm3 -; SSE41-NEXT: por %xmm5, %xmm3 -; SSE41-NEXT: por %xmm1, %xmm3 -; SSE41-NEXT: por %xmm6, %xmm2 -; SSE41-NEXT: por %xmm4, %xmm2 -; SSE41-NEXT: por %xmm3, %xmm2 -; SSE41-NEXT: por %xmm0, %xmm2 -; SSE41-NEXT: ptest %xmm2, %xmm2 -; SSE41-NEXT: sete %al -; SSE41-NEXT: retq +; SSE-LABEL: test_v128i8: +; SSE: # %bb.0: +; SSE-NEXT: por %xmm6, %xmm2 +; SSE-NEXT: por %xmm7, %xmm3 +; SSE-NEXT: por %xmm5, %xmm3 +; SSE-NEXT: por %xmm1, %xmm3 +; SSE-NEXT: por %xmm4, %xmm2 +; SSE-NEXT: por %xmm3, %xmm2 +; SSE-NEXT: por %xmm0, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1] +; SSE-NEXT: por %xmm2, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: psrld $16, %xmm0 +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psrlw $8, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movd %xmm1, %eax +; SSE-NEXT: testb %al, %al +; SSE-NEXT: sete %al +; SSE-NEXT: retq ; ; AVX1-LABEL: test_v128i8: ; AVX1: # %bb.0: ; AVX1-NEXT: vorps %ymm3, %ymm1, %ymm1 ; AVX1-NEXT: vorps %ymm1, %ymm2, %ymm1 ; AVX1-NEXT: vorps %ymm1, %ymm0, %ymm0 -; AVX1-NEXT: vptest %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm1 +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vmovd %xmm0, %eax +; AVX1-NEXT: testb %al, %al ; AVX1-NEXT: sete %al ; AVX1-NEXT: vzeroupper ; AVX1-NEXT: retq @@ -797,7 +1110,18 @@ define i1 @test_v128i8(<128 x i8> %a0) { ; AVX2-NEXT: vpor %ymm3, %ymm1, %ymm1 ; AVX2-NEXT: vpor %ymm1, %ymm2, %ymm1 ; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: vptest %ymm0, %ymm0 +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vmovd %xmm0, %eax +; AVX2-NEXT: testb %al, %al ; AVX2-NEXT: sete %al ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq @@ -806,8 +1130,19 @@ define i1 @test_v128i8(<128 x i8> %a0) { ; AVX512: # %bb.0: ; AVX512-NEXT: vporq %zmm1, %zmm0, %zmm0 ; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1 -; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0 -; AVX512-NEXT: vptest %ymm0, %ymm0 +; AVX512-NEXT: vporq %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpsrlw $8, %xmm0, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vmovd %xmm0, %eax +; AVX512-NEXT: testb %al, %al ; AVX512-NEXT: sete %al ; AVX512-NEXT: vzeroupper ; AVX512-NEXT: retq From 315bd964377fe8a465e2f6c8c7c06ae9c900bb0f Mon Sep 17 00:00:00 2001 From: Lei Zhang Date: Mon, 22 Jun 2020 15:32:21 -0400 Subject: [PATCH 12/13] Use std::make_tuple instead initializer list Hopefully this pleases GCC-5 and fixes the build error: LowerExpectIntrinsic.cpp:62:53: error: converting to 'std::tuple' from initializer list would use explicit constructor 'constexpr std::tuple<_T1, _T2>::tuple(_U1&&, _U2&&) [with _U1 = llvm::cl::opt&; _U2 = llvm::cl::opt&; = void; _T1 = unsigned int; _T2 = unsigned int]' return {LikelyBranchWeight, UnlikelyBranchWeight}; Differential Revision: https://reviews.llvm.org/D82325 --- llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp index 4b98be28b862c..05db70c787bb6 100644 --- a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp +++ b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp @@ -59,7 +59,8 @@ std::tuple getBranchWeight(Intrinsic::ID IntrinsicID, CallInst *CI, int BranchCount) { if (IntrinsicID == Intrinsic::expect) { // __builtin_expect - return {LikelyBranchWeight, UnlikelyBranchWeight}; + return std::make_tuple(LikelyBranchWeight.getValue(), + UnlikelyBranchWeight.getValue()); } else { // __builtin_expect_with_probability assert(CI->getNumOperands() >= 3 && @@ -71,7 +72,7 @@ std::tuple getBranchWeight(Intrinsic::ID IntrinsicID, double FalseProb = (1.0 - TrueProb) / (BranchCount - 1); uint32_t LikelyBW = ceil((TrueProb * (double)(INT32_MAX - 1)) + 1.0); uint32_t UnlikelyBW = ceil((FalseProb * (double)(INT32_MAX - 1)) + 1.0); - return {LikelyBW, UnlikelyBW}; + return std::make_tuple(LikelyBW, UnlikelyBW); } } From 77df5a8283edbfc33ad3b12df3bd42d54d7ba4f4 Mon Sep 17 00:00:00 2001 From: Aaron En Ye Shi Date: Thu, 11 Jun 2020 18:36:10 +0000 Subject: [PATCH 13/13] [HIP] Move HIP Linking Logic into HIP ToolChain This patch is a follow up on https://reviews.llvm.org/D78759. Extract the HIP Linker script from generic GNU linker, and move it into HIP ToolChain. Update OffloadActionBuilder Link actions feature to apply device linking and host linking actions separately. Using MC Directives, embed the device images and define symbols. Reviewers: JonChesterfield, yaxunl Subscribers: tra, echristo, jdoerfert, msearles, scchan Differential Revision: https://reviews.llvm.org/D81963 --- clang/lib/Driver/Driver.cpp | 66 ++++++++-- clang/lib/Driver/ToolChains/CommonArgs.cpp | 117 +---------------- clang/lib/Driver/ToolChains/CommonArgs.h | 6 - clang/lib/Driver/ToolChains/Gnu.cpp | 4 - clang/lib/Driver/ToolChains/HIP.cpp | 74 +++++++++++ clang/lib/Driver/ToolChains/HIP.h | 7 + clang/test/Driver/hip-binding.hip | 11 +- clang/test/Driver/hip-link-save-temps.hip | 8 +- clang/test/Driver/hip-link-shared-library.hip | 11 +- clang/test/Driver/hip-phases.hip | 120 +++++++++++------- clang/test/Driver/hip-save-temps.hip | 47 ++++--- .../Driver/hip-toolchain-rdc-separate.hip | 21 ++- clang/test/Driver/hip-toolchain-rdc.hip | 45 ++++--- 13 files changed, 304 insertions(+), 233 deletions(-) diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index a48761af400fd..de732918fc763 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -2325,8 +2325,11 @@ class OffloadingActionBuilder final { /// Append top level actions generated by the builder. virtual void appendTopLevelActions(ActionList &AL) {} - /// Append linker actions generated by the builder. - virtual void appendLinkActions(ActionList &AL) {} + /// Append linker device actions generated by the builder. + virtual void appendLinkDeviceActions(ActionList &AL) {} + + /// Append linker host action generated by the builder. + virtual Action* appendLinkHostActions(ActionList &AL) { return nullptr; } /// Append linker actions generated by the builder. virtual void appendLinkDependences(OffloadAction::DeviceDependences &DA) {} @@ -2796,17 +2799,45 @@ class OffloadingActionBuilder final { : ABRT_Success; } - void appendLinkDependences(OffloadAction::DeviceDependences &DA) override { + void appendLinkDeviceActions(ActionList &AL) override { + if (DeviceLinkerInputs.size() == 0) + return; + + assert(DeviceLinkerInputs.size() == GpuArchList.size() && + "Linker inputs and GPU arch list sizes do not match."); + // Append a new link action for each device. unsigned I = 0; for (auto &LI : DeviceLinkerInputs) { + // Each entry in DeviceLinkerInputs corresponds to a GPU arch. auto *DeviceLinkAction = C.MakeAction(LI, types::TY_Image); - DA.add(*DeviceLinkAction, *ToolChains[0], - CudaArchToString(GpuArchList[I]), AssociatedOffloadKind); + // Linking all inputs for the current GPU arch. + // LI contains all the inputs for the linker. + OffloadAction::DeviceDependences DeviceLinkDeps; + DeviceLinkDeps.add(*DeviceLinkAction, *ToolChains[0], + CudaArchToString(GpuArchList[I]), AssociatedOffloadKind); + AL.push_back(C.MakeAction(DeviceLinkDeps, + DeviceLinkAction->getType())); ++I; } + DeviceLinkerInputs.clear(); + + // Create a host object from all the device images by embedding them + // in a fat binary. + OffloadAction::DeviceDependences DDeps; + auto *TopDeviceLinkAction = + C.MakeAction(AL, types::TY_Object); + DDeps.add(*TopDeviceLinkAction, *ToolChains[0], + nullptr, AssociatedOffloadKind); + + // Offload the host object to the host linker. + AL.push_back(C.MakeAction(DDeps, TopDeviceLinkAction->getType())); } + + Action* appendLinkHostActions(ActionList &AL) override { return AL.back(); } + + void appendLinkDependences(OffloadAction::DeviceDependences &DA) override {} }; /// OpenMP action builder. The host bitcode is passed to the device frontend @@ -2934,7 +2965,7 @@ class OffloadingActionBuilder final { OpenMPDeviceActions.clear(); } - void appendLinkActions(ActionList &AL) override { + void appendLinkDeviceActions(ActionList &AL) override { assert(ToolChains.size() == DeviceLinkerInputs.size() && "Toolchains and linker inputs sizes do not match."); @@ -2953,6 +2984,14 @@ class OffloadingActionBuilder final { DeviceLinkerInputs.clear(); } + Action* appendLinkHostActions(ActionList &AL) override { + // Create wrapper bitcode from the result of device link actions and compile + // it to an object which will be added to the host link command. + auto *BC = C.MakeAction(AL, types::TY_LLVM_BC); + auto *ASM = C.MakeAction(BC, types::TY_PP_Asm); + return C.MakeAction(ASM, types::TY_Object); + } + void appendLinkDependences(OffloadAction::DeviceDependences &DA) override {} bool initialize() override { @@ -3185,17 +3224,20 @@ class OffloadingActionBuilder final { for (DeviceActionBuilder *SB : SpecializedBuilders) { if (!SB->isValid()) continue; - SB->appendLinkActions(DeviceAL); + SB->appendLinkDeviceActions(DeviceAL); } if (DeviceAL.empty()) return nullptr; - // Create wrapper bitcode from the result of device link actions and compile - // it to an object which will be added to the host link command. - auto *BC = C.MakeAction(DeviceAL, types::TY_LLVM_BC); - auto *ASM = C.MakeAction(BC, types::TY_PP_Asm); - return C.MakeAction(ASM, types::TY_Object); + // Let builders add host linking actions. + Action* HA; + for (DeviceActionBuilder *SB : SpecializedBuilders) { + if (!SB->isValid()) + continue; + HA = SB->appendLinkHostActions(DeviceAL); + } + return HA; } /// Processes the host linker action. This currently consists of replacing it diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 7fdb3fdf00094..542fb67db7ef1 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -152,14 +152,12 @@ void tools::AddLinkerInputs(const ToolChain &TC, const InputInfoList &Inputs, addDirectoryList(Args, CmdArgs, "-L", "LIBRARY_PATH"); for (const auto &II : Inputs) { - // If the current tool chain refers to an OpenMP or HIP offloading host, we - // should ignore inputs that refer to OpenMP or HIP offloading devices - + // If the current tool chain refers to an OpenMP offloading host, we + // should ignore inputs that refer to OpenMP offloading devices - // they will be embedded according to a proper linker script. if (auto *IA = II.getAction()) if ((JA.isHostOffloading(Action::OFK_OpenMP) && - IA->isDeviceOffloading(Action::OFK_OpenMP)) || - (JA.isHostOffloading(Action::OFK_HIP) && - IA->isDeviceOffloading(Action::OFK_HIP))) + IA->isDeviceOffloading(Action::OFK_OpenMP))) continue; if (!TC.HasNativeLLVMSupport() && types::isLLVMIR(II.getType())) @@ -1298,115 +1296,6 @@ void tools::AddRunTimeLibs(const ToolChain &TC, const Driver &D, } } -/// Add HIP linker script arguments at the end of the argument list so that -/// the fat binary is built by embedding the device images into the host. The -/// linker script also defines a symbol required by the code generation so that -/// the image can be retrieved at runtime. This should be used only in tool -/// chains that support linker scripts. -void tools::AddHIPLinkerScript(const ToolChain &TC, Compilation &C, - const InputInfo &Output, - const InputInfoList &Inputs, const ArgList &Args, - ArgStringList &CmdArgs, const JobAction &JA, - const Tool &T) { - - // If this is not a HIP host toolchain, we don't need to do anything. - if (!JA.isHostOffloading(Action::OFK_HIP)) - return; - - InputInfoList DeviceInputs; - for (const auto &II : Inputs) { - const Action *A = II.getAction(); - // Is this a device linking action? - if (A && isa(A) && A->isDeviceOffloading(Action::OFK_HIP)) { - DeviceInputs.push_back(II); - } - } - - if (DeviceInputs.empty()) - return; - - // Create temporary linker script. Keep it if save-temps is enabled. - const char *LKS; - std::string Name = - std::string(llvm::sys::path::filename(Output.getFilename())); - if (C.getDriver().isSaveTempsEnabled()) { - LKS = C.getArgs().MakeArgString(Name + ".lk"); - } else { - auto TmpName = C.getDriver().GetTemporaryPath(Name, "lk"); - LKS = C.addTempFile(C.getArgs().MakeArgString(TmpName)); - } - - // Add linker script option to the command. - CmdArgs.push_back("-T"); - CmdArgs.push_back(LKS); - - // Create a buffer to write the contents of the linker script. - std::string LksBuffer; - llvm::raw_string_ostream LksStream(LksBuffer); - - // Get the HIP offload tool chain. - auto *HIPTC = static_cast( - C.getSingleOffloadToolChain()); - assert(HIPTC->getTriple().getArch() == llvm::Triple::amdgcn && - "Wrong platform"); - (void)HIPTC; - - const char *BundleFile; - if (C.getDriver().isSaveTempsEnabled()) { - BundleFile = C.getArgs().MakeArgString(Name + ".hipfb"); - } else { - auto TmpName = C.getDriver().GetTemporaryPath(Name, "hipfb"); - BundleFile = C.addTempFile(C.getArgs().MakeArgString(TmpName)); - } - AMDGCN::constructHIPFatbinCommand(C, JA, BundleFile, DeviceInputs, Args, T); - - // Add commands to embed target binaries. We ensure that each section and - // image is 16-byte aligned. This is not mandatory, but increases the - // likelihood of data to be aligned with a cache block in several main host - // machines. - LksStream << "/*\n"; - LksStream << " HIP Offload Linker Script\n"; - LksStream << " *** Automatically generated by Clang ***\n"; - LksStream << "*/\n"; - LksStream << "TARGET(binary)\n"; - LksStream << "INPUT(" << BundleFile << ")\n"; - LksStream << "SECTIONS\n"; - LksStream << "{\n"; - LksStream << " .hip_fatbin :\n"; - LksStream << " ALIGN(0x10)\n"; - LksStream << " {\n"; - LksStream << " PROVIDE_HIDDEN(__hip_fatbin = .);\n"; - LksStream << " " << BundleFile << "\n"; - LksStream << " }\n"; - LksStream << " /DISCARD/ :\n"; - LksStream << " {\n"; - LksStream << " * ( __CLANG_OFFLOAD_BUNDLE__* )\n"; - LksStream << " }\n"; - LksStream << "}\n"; - LksStream << "INSERT BEFORE .data\n"; - LksStream.flush(); - - // Dump the contents of the linker script if the user requested that. We - // support this option to enable testing of behavior with -###. - if (C.getArgs().hasArg(options::OPT_fhip_dump_offload_linker_script)) - llvm::errs() << LksBuffer; - - // If this is a dry run, do not create the linker script file. - if (C.getArgs().hasArg(options::OPT__HASH_HASH_HASH)) - return; - - // Open script file and write the contents. - std::error_code EC; - llvm::raw_fd_ostream Lksf(LKS, EC, llvm::sys::fs::OF_None); - - if (EC) { - C.getDriver().Diag(clang::diag::err_unable_to_make_temp) << EC.message(); - return; - } - - Lksf << LksBuffer; -} - SmallString<128> tools::getStatsFileName(const llvm::opt::ArgList &Args, const InputInfo &Output, const InputInfo &Input, diff --git a/clang/lib/Driver/ToolChains/CommonArgs.h b/clang/lib/Driver/ToolChains/CommonArgs.h index 58bc92c9b7569..c7d695ebf415f 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.h +++ b/clang/lib/Driver/ToolChains/CommonArgs.h @@ -45,12 +45,6 @@ void AddRunTimeLibs(const ToolChain &TC, const Driver &D, llvm::opt::ArgStringList &CmdArgs, const llvm::opt::ArgList &Args); -void AddHIPLinkerScript(const ToolChain &TC, Compilation &C, - const InputInfo &Output, const InputInfoList &Inputs, - const llvm::opt::ArgList &Args, - llvm::opt::ArgStringList &CmdArgs, const JobAction &JA, - const Tool &T); - const char *SplitDebugName(const llvm::opt::ArgList &Args, const InputInfo &Input, const InputInfo &Output); diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp index 93a285d42a08e..619407a7c57bf 100644 --- a/clang/lib/Driver/ToolChains/Gnu.cpp +++ b/clang/lib/Driver/ToolChains/Gnu.cpp @@ -625,10 +625,6 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA, } } - // Add HIP offloading linker script args if required. - AddHIPLinkerScript(getToolChain(), C, Output, Inputs, Args, CmdArgs, JA, - *this); - Args.AddAllArgs(CmdArgs, options::OPT_T); const char *Exec = Args.MakeArgString(ToolChain.GetLinkerPath()); diff --git a/clang/lib/Driver/ToolChains/HIP.cpp b/clang/lib/Driver/ToolChains/HIP.cpp index b9143f98f1526..f524951ce8375 100644 --- a/clang/lib/Driver/ToolChains/HIP.cpp +++ b/clang/lib/Driver/ToolChains/HIP.cpp @@ -104,6 +104,76 @@ void AMDGCN::constructHIPFatbinCommand(Compilation &C, const JobAction &JA, C.addCommand(std::make_unique(JA, T, Bundler, BundlerArgs, Inputs)); } +/// Add Generated HIP Object File which has device images embedded into the +/// host to the argument list for linking. Using MC directives, embed the +/// device code and also define symbols required by the code generation so that +/// the image can be retrieved at runtime. +void AMDGCN::Linker::constructGenerateObjFileFromHIPFatBinary( + Compilation &C, const InputInfo &Output, + const InputInfoList &Inputs, const ArgList &Args, + const JobAction &JA) const { + const ToolChain &TC = getToolChain(); + std::string Name = + std::string(llvm::sys::path::stem(Output.getFilename())); + + // Create Temp Object File Generator, + // Offload Bundled file and Bundled Object file. + // Keep them if save-temps is enabled. + const char *McinFile; + const char *BundleFile; + if (C.getDriver().isSaveTempsEnabled()) { + McinFile = C.getArgs().MakeArgString(Name + ".mcin"); + BundleFile = C.getArgs().MakeArgString(Name + ".hipfb"); + } else { + auto TmpNameMcin = C.getDriver().GetTemporaryPath(Name, "mcin"); + McinFile = C.addTempFile(C.getArgs().MakeArgString(TmpNameMcin)); + auto TmpNameFb = C.getDriver().GetTemporaryPath(Name, "hipfb"); + BundleFile = C.addTempFile(C.getArgs().MakeArgString(TmpNameFb)); + } + constructHIPFatbinCommand(C, JA, BundleFile, Inputs, Args, *this); + + // Create a buffer to write the contents of the temp obj generator. + std::string ObjBuffer; + llvm::raw_string_ostream ObjStream(ObjBuffer); + + // Add MC directives to embed target binaries. We ensure that each + // section and image is 16-byte aligned. This is not mandatory, but + // increases the likelihood of data to be aligned with a cache block + // in several main host machines. + ObjStream << "# HIP Object Generator\n"; + ObjStream << "# *** Automatically generated by Clang ***\n"; + ObjStream << " .type __hip_fatbin,@object\n"; + ObjStream << " .section .hip_fatbin,\"aMS\",@progbits,1\n"; + ObjStream << " .data\n"; + ObjStream << " .globl __hip_fatbin\n"; + ObjStream << " .p2align 3\n"; + ObjStream << "__hip_fatbin:\n"; + ObjStream << " .incbin \"" << BundleFile << "\"\n"; + ObjStream.flush(); + + // Dump the contents of the temp object file gen if the user requested that. + // We support this option to enable testing of behavior with -###. + if (C.getArgs().hasArg(options::OPT_fhip_dump_offload_linker_script)) + llvm::errs() << ObjBuffer; + + // Open script file and write the contents. + std::error_code EC; + llvm::raw_fd_ostream Objf(McinFile, EC, llvm::sys::fs::OF_None); + + if (EC) { + C.getDriver().Diag(clang::diag::err_unable_to_make_temp) << EC.message(); + return; + } + + Objf << ObjBuffer; + + ArgStringList McArgs{"-triple", Args.MakeArgString(TC.getTripleString()), + "-o", Output.getFilename(), + McinFile, "--filetype=obj"}; + const char *Mc = Args.MakeArgString(TC.GetProgramPath("llvm-mc")); + C.addCommand(std::make_unique(JA, *this, Mc, McArgs, Inputs)); +} + // For amdgcn the inputs of the linker job are device bitcode and output is // object file. It calls llvm-link, opt, llc, then lld steps. void AMDGCN::Linker::ConstructJob(Compilation &C, const JobAction &JA, @@ -111,6 +181,10 @@ void AMDGCN::Linker::ConstructJob(Compilation &C, const JobAction &JA, const InputInfoList &Inputs, const ArgList &Args, const char *LinkingOutput) const { + if (Inputs.size() > 0 && + Inputs[0].getType() == types::TY_Image && + JA.getType() == types::TY_Object) + return constructGenerateObjFileFromHIPFatBinary(C, Output, Inputs, Args, JA); if (JA.getType() == types::TY_HIP_FATBIN) return constructHIPFatbinCommand(C, JA, Output.getFilename(), Inputs, Args, *this); diff --git a/clang/lib/Driver/ToolChains/HIP.h b/clang/lib/Driver/ToolChains/HIP.h index fa6984ed9f52f..5e2be7138579a 100644 --- a/clang/lib/Driver/ToolChains/HIP.h +++ b/clang/lib/Driver/ToolChains/HIP.h @@ -42,6 +42,13 @@ class LLVM_LIBRARY_VISIBILITY Linker : public Tool { void constructLldCommand(Compilation &C, const JobAction &JA, const InputInfoList &Inputs, const InputInfo &Output, const llvm::opt::ArgList &Args) const; + + // Construct command for creating Object from HIP fatbin. + void constructGenerateObjFileFromHIPFatBinary(Compilation &C, + const InputInfo &Output, + const InputInfoList &Inputs, + const llvm::opt::ArgList &Args, + const JobAction &JA) const; }; } // end namespace AMDGCN diff --git a/clang/test/Driver/hip-binding.hip b/clang/test/Driver/hip-binding.hip index 53fe9af0e653c..3d83930223462 100644 --- a/clang/test/Driver/hip-binding.hip +++ b/clang/test/Driver/hip-binding.hip @@ -25,12 +25,15 @@ // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 -fgpu-rdc %t.o\ // RUN: 2>&1 | FileCheck %s -// CHECK: # "amdgcn-amd-amdhsa" - "offload bundler", inputs: ["[[IN:.*o]]"], outputs: ["[[OBJ1:.*o]]", "[[OBJ2:.*o]]", "[[OBJ3:.*o]]"] -// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[OBJ2]]"], output: "[[IMG2:.*out]]" +// CHECK: # "x86_64-unknown-linux-gnu" - "offload bundler", inputs: ["[[IN:.*o]]"], outputs: ["[[HOSTOBJ:.*o]]", "{{.*o}}", "{{.*o}}"] +// CHECK: # "amdgcn-amd-amdhsa" - "offload bundler", inputs: ["[[IN]]"], outputs: ["{{.*o}}", "[[DOBJ1:.*o]]", "[[DOBJ2:.*o]]"] +// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[DOBJ1]]"], output: "[[IMG1:.*out]]" // CHECK-NOT: offload bundler -// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[OBJ3]]"], output: "[[IMG3:.*out]]" +// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[DOBJ2]]"], output: "[[IMG2:.*out]]" // CHECK-NOT: offload bundler -// CHECK: # "x86_64-unknown-linux-gnu" - "GNU::Linker", inputs: ["[[OBJ1]]", "[[IMG2]]", "[[IMG3]]"], output: "a.out" +// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[IMG1]]", "[[IMG2]]"], output: "[[FATBINOBJ:.*o]]" +// CHECK-NOT: offload bundler +// CHECK: # "x86_64-unknown-linux-gnu" - "GNU::Linker", inputs: ["[[HOSTOBJ]]", "[[FATBINOBJ]]"], output: "a.out" // RUN: %clang --hip-link -ccc-print-bindings -target x86_64-linux-gnu \ // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t.o\ diff --git a/clang/test/Driver/hip-link-save-temps.hip b/clang/test/Driver/hip-link-save-temps.hip index 304be921ebc62..62d8d60af3c09 100644 --- a/clang/test/Driver/hip-link-save-temps.hip +++ b/clang/test/Driver/hip-link-save-temps.hip @@ -27,7 +27,7 @@ // CHECK-SAME: "-o" "a.out-hip-amdgcn-amd-amdhsa-gfx900" "obj1-hip-amdgcn-amd-amdhsa-gfx900.o" "obj2-hip-amdgcn-amd-amdhsa-gfx900.o" // CHECK: "{{.*lld.*}}" {{.*}} "-mllvm" "-amdgpu-internalize-symbols" // CHECK-SAME: "-o" "a.out-hip-amdgcn-amd-amdhsa-gfx906" "obj1-hip-amdgcn-amd-amdhsa-gfx906.o" "obj2-hip-amdgcn-amd-amdhsa-gfx906.o" -// OUT: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=executable.hipfb" -// OUT: "{{.*ld.*}}" {{.*}} "-o" "executable" {{.*}} "-T" "executable.lk" -// NOUT: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=a.out.hipfb" -// NOUT: "{{.*ld.*}}" {{.*}} "-o" "a.out" {{.*}} "-T" "a.out.lk" +// CHECK: {{".*llvm-mc.*"}} "-triple" "amdgcn-amd-amdhsa" "-o" +// CHECK-SAME: "[[OBJBUNDLE:.*.o]]" "{{.*}}.mcin" "--filetype=obj" +// OUT: "{{.*ld.*}}" {{.*}} "-o" "executable" {{.*}} "[[OBJBUNDLE]]" +// NOUT: "{{.*ld.*}}" {{.*}} "-o" "a.out" {{.*}} "[[OBJBUNDLE]]" diff --git a/clang/test/Driver/hip-link-shared-library.hip b/clang/test/Driver/hip-link-shared-library.hip index cb409d1a874d8..895bd53225e12 100644 --- a/clang/test/Driver/hip-link-shared-library.hip +++ b/clang/test/Driver/hip-link-shared-library.hip @@ -3,10 +3,13 @@ // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t.o %S/Inputs/in.so \ // RUN: -fgpu-rdc 2>&1 | FileCheck %s -// CHECK: # "amdgcn-amd-amdhsa" - "offload bundler", inputs: ["[[IN:.*o]]"], outputs: ["[[OBJ1:.*o]]", "[[OBJ2:.*o]]", "[[OBJ3:.*o]]"] -// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[OBJ2]]"], output: "[[IMG2:.*out]]" +// CHECK: # "x86_64-unknown-linux-gnu" - "offload bundler", inputs: ["[[IN:.*o]]"], outputs: ["[[HOSTOBJ:.*o]]", "{{.*o}}", "{{.*o}}"] +// CHECK: # "amdgcn-amd-amdhsa" - "offload bundler", inputs: ["[[IN]]"], outputs: ["{{.*o}}", "[[DOBJ1:.*o]]", "[[DOBJ2:.*o]]"] +// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[DOBJ1]]"], output: "[[IMG1:.*out]]" // CHECK-NOT: offload bundler -// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[OBJ3]]"], output: "[[IMG3:.*out]]" +// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[DOBJ2]]"], output: "[[IMG2:.*out]]" // CHECK-NOT: offload bundler -// CHECK: # "x86_64-unknown-linux-gnu" - "GNU::Linker", inputs: ["[[OBJ1]]", "{{.*}}/Inputs/in.so", "[[IMG2]]", "[[IMG3]]"], output: "a.out" +// CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[IMG1]]", "[[IMG2]]"], output: "[[FATBINOBJ:.*o]]" +// CHECK-NOT: offload bundler +// CHECK: # "x86_64-unknown-linux-gnu" - "GNU::Linker", inputs: ["[[HOSTOBJ]]", "{{.*}}/Inputs/in.so", "[[FATBINOBJ]]"], output: "a.out" diff --git a/clang/test/Driver/hip-phases.hip b/clang/test/Driver/hip-phases.hip index b27ae44167a21..7c2dc1384ccd4 100644 --- a/clang/test/Driver/hip-phases.hip +++ b/clang/test/Driver/hip-phases.hip @@ -21,25 +21,27 @@ // BIN-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]]) // BIN-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]]) // BIN-DAG: [[P2:[0-9]+]]: compiler, {[[P1]]}, ir, (host-[[T]]) +// RDC-DAG: [[P12:[0-9]+]]: backend, {[[P2]]}, assembler, (host-[[T]]) +// RDC-DAG: [[P13:[0-9]+]]: assembler, {[[P12]]}, object, (host-[[T]]) // BIN-DAG: [[P3:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T]], (device-[[T]], [[ARCH:gfx803]]) // BIN-DAG: [[P4:[0-9]+]]: preprocessor, {[[P3]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH]]) // BIN-DAG: [[P5:[0-9]+]]: compiler, {[[P4]]}, ir, (device-[[T]], [[ARCH]]) // NRD-DAG: [[P6:[0-9]+]]: backend, {[[P5]]}, assembler, (device-[[T]], [[ARCH]]) // NRD-DAG: [[P7:[0-9]+]]: assembler, {[[P6]]}, object, (device-[[T]], [[ARCH]]) -// NRD-DAG: [[P8:[0-9]+]]: linker, {[[P7]]}, image, (device-[[T]], [[ARCH]]) -// NRD-DAG: [[P9:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH]])" {[[P8]]}, image +// RDC-DAG: [[P7:[0-9]+]]: backend, {[[P5]]}, ir, (device-[[T]], [[ARCH]]) +// BIN-DAG: [[P8:[0-9]+]]: linker, {[[P7]]}, image, (device-[[T]], [[ARCH]]) +// BIN-DAG: [[P9:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH]])" {[[P8]]}, image // NRD-DAG: [[P10:[0-9]+]]: linker, {[[P9]]}, hip-fatbin, (device-[[T]]) -// RDC-DAG: [[P6:[0-9]+]]: backend, {[[P5]]}, ir, (device-[[T]], [[ARCH]]) -// RDC-DAG: [[P10:[0-9]+]]: linker, {[[P6]]}, image, (device-[[T]], [[ARCH]]) +// RDC-DAG: [[P10:[0-9]+]]: linker, {[[P9]]}, object, (device-[[T]]) -// NRD-DAG: [[P12:[0-9]+]]: offload, "host-[[T]] (x86_64-unknown-linux-gnu)" {[[P2]]}, "device-[[T]] (amdgcn-amd-amdhsa)" {[[P10]]}, ir +// NRD-DAG: [[P11:[0-9]+]]: offload, "host-[[T]] (x86_64-unknown-linux-gnu)" {[[P2]]}, "device-[[T]] (amdgcn-amd-amdhsa)" {[[P10]]}, ir +// RDC-DAG: [[P11:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa)" {[[P10]]}, object +// NRD-DAG: [[P12:[0-9]+]]: backend, {[[P11]]}, assembler, (host-[[T]]) +// NRD-DAG: [[P13:[0-9]+]]: assembler, {[[P12]]}, object, (host-[[T]]) +// NRD-DAG: [[P14:[0-9]+]]: linker, {[[P13]]}, image, (host-[[T]]) +// RDC-DAG: [[P14:[0-9]+]]: linker, {[[P13]], [[P11]]}, image, (host-[[T]]) -// NRD-DAG: [[P13:[0-9]+]]: backend, {[[P12]]}, assembler, (host-[[T]]) -// RDC-DAG: [[P13:[0-9]+]]: backend, {[[P2]]}, assembler, (host-[[T]]) -// BIN-DAG: [[P14:[0-9]+]]: assembler, {[[P13]]}, object, (host-[[T]]) -// BIN-DAG: [[P15:[0-9]+]]: linker, {[[P14]]}, image, (host-[[T]]) -// RDC-DAG: [[P16:[0-9]+]]: offload, "host-[[T]] (x86_64-unknown-linux-gnu)" {[[P15]]}, "device-[[T]] (amdgcn-amd-amdhsa:gfx803)" {[[P10]]}, image // // Test single gpu architecture up to the assemble phase. // @@ -56,59 +58,84 @@ // ASM-DAG: [[P8:[0-9]+]]: backend, {[[P7]]}, assembler, (host-[[T]]) // -// Test two gpu architectures with complete compilation. +// Test two gpu architectures with complete compilation with -fno-gpu-rdc. // // RUN: %clang -x hip -target x86_64-unknown-linux-gnu -ccc-print-phases \ // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s 2>&1 \ -// RUN: | FileCheck -check-prefixes=BIN2,NRD2,CL2 %s +// RUN: | FileCheck -check-prefixes=NRD2,NCL2 %s // RUN: %clang -x hip -target x86_64-unknown-linux-gnu -ccc-print-phases \ // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -c 2>&1 \ -// RUN: | FileCheck -check-prefixes=BIN2,NRD2 %s +// RUN: | FileCheck -check-prefixes=NRD2 %s +// NRD2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]]) +// NRD2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]]) +// NRD2-DAG: [[P2:[0-9]+]]: compiler, {[[P1]]}, ir, (host-[[T]]) + +// NRD2-DAG: [[P3:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T]], (device-[[T]], [[ARCH1:gfx803]]) +// NRD2-DAG: [[P4:[0-9]+]]: preprocessor, {[[P3]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH1]]) +// NRD2-DAG: [[P5:[0-9]+]]: compiler, {[[P4]]}, ir, (device-[[T]], [[ARCH1]]) +// NRD2-DAG: [[P6:[0-9]+]]: backend, {[[P5]]}, assembler, (device-[[T]], [[ARCH1]]) +// NRD2-DAG: [[P7:[0-9]+]]: assembler, {[[P6]]}, object, (device-[[T]], [[ARCH1]]) +// NRD2-DAG: [[P8:[0-9]+]]: linker, {[[P7]]}, image, (device-[[T]], [[ARCH1]]) +// NRD2-DAG: [[P9:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH1]])" {[[P8]]}, image + +// NRD2-DAG: [[P10:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T]], (device-[[T]], [[ARCH2:gfx900]]) +// NRD2-DAG: [[P11:[0-9]+]]: preprocessor, {[[P10]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH2]]) +// NRD2-DAG: [[P12:[0-9]+]]: compiler, {[[P11]]}, ir, (device-[[T]], [[ARCH2]]) +// NRD2-DAG: [[P13:[0-9]+]]: backend, {[[P12]]}, assembler, (device-[[T]], [[ARCH2]]) +// NRD2-DAG: [[P14:[0-9]+]]: assembler, {[[P13]]}, object, (device-[[T]], [[ARCH2]]) +// NRD2-DAG: [[P15:[0-9]+]]: linker, {[[P14]]}, image, (device-[[T]], [[ARCH2]]) +// NRD2-DAG: [[P16:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH2]])" {[[P15]]}, image +// NRD2-DAG: [[P17:[0-9]+]]: linker, {[[P9]], [[P16]]}, hip-fatbin, (device-[[T]]) +// NRD2-DAG: [[P18:[0-9]+]]: offload, "host-[[T]] (x86_64-unknown-linux-gnu)" {[[P2]]}, "device-[[T]] (amdgcn-amd-amdhsa)" {[[P17]]}, ir +// NRD2-DAG: [[P19:[0-9]+]]: backend, {[[P18]]}, assembler, (host-[[T]]) +// NRD2-DAG: [[P20:[0-9]+]]: assembler, {[[P19]]}, object, (host-[[T]]) +// NCL2-DAG: [[P21:[0-9]+]]: linker, {[[P20]]}, image, (host-[[T]]) + +// +// Test two gpu architectures with complete compilation with -fgpu-rdc. +// // RUN: %clang -x hip -target x86_64-unknown-linux-gnu -ccc-print-phases \ // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -fgpu-rdc 2>&1 \ -// RUN: | FileCheck -check-prefixes=BIN2,RDC2,CL2,RCL2 %s +// RUN: | FileCheck -check-prefixes=RDC2,CL2,RCL2 %s // RUN: %clang -x hip -target x86_64-unknown-linux-gnu -ccc-print-phases \ // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -fgpu-rdc -c 2>&1 \ -// RUN: | FileCheck -check-prefixes=BIN2,RDC2,RC2 %s +// RUN: | FileCheck -check-prefixes=RDC2,RC2 %s -// BIN2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]]) -// BIN2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]]) -// BIN2-DAG: [[P2:[0-9]+]]: compiler, {[[P1]]}, ir, (host-[[T]]) +// RCL2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]]) +// RCL2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]]) +// RCL2-DAG: [[P2:[0-9]+]]: compiler, {[[P1]]}, ir, (host-[[T]]) +// RCL2-DAG: [[P19:[0-9]+]]: backend, {[[P2]]}, assembler, (host-[[T]]) +// RCL2-DAG: [[P20:[0-9]+]]: assembler, {[[P19]]}, object, (host-[[T]]) -// BIN2-DAG: [[P3:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T]], (device-[[T]], [[ARCH1:gfx803]]) -// BIN2-DAG: [[P4:[0-9]+]]: preprocessor, {[[P3]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH1]]) -// BIN2-DAG: [[P5:[0-9]+]]: compiler, {[[P4]]}, ir, (device-[[T]], [[ARCH1]]) -// NRD2-DAG: [[P6:[0-9]+]]: backend, {[[P5]]}, assembler, (device-[[T]], [[ARCH1]]) +// RDC2-DAG: [[P3:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (device-[[T]], [[ARCH1:gfx803]]) +// RDC2-DAG: [[P4:[0-9]+]]: preprocessor, {[[P3]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH1]]) +// RDC2-DAG: [[P5:[0-9]+]]: compiler, {[[P4]]}, ir, (device-[[T]], [[ARCH1]]) // RDC2-DAG: [[P6:[0-9]+]]: backend, {[[P5]]}, ir, (device-[[T]], [[ARCH1]]) -// NRD2-DAG: [[P7:[0-9]+]]: assembler, {[[P6]]}, object, (device-[[T]], [[ARCH1]]) -// NRD2-DAG: [[P8:[0-9]+]]: linker, {[[P7]]}, image, (device-[[T]], [[ARCH1]]) // RCL2-DAG: [[P8:[0-9]+]]: linker, {[[P6]]}, image, (device-[[T]], [[ARCH1]]) -// NRD2-DAG: [[P9:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH1]])" {[[P8]]}, image +// RCL2-DAG: [[P9:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH1]])" {[[P8]]}, image // RC2-DAG: [[P9:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH1]])" {[[P6]]}, ir -// BIN2-DAG: [[P10:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T]], (device-[[T]], [[ARCH2:gfx900]]) -// BIN2-DAG: [[P11:[0-9]+]]: preprocessor, {[[P10]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH2]]) -// BIN2-DAG: [[P12:[0-9]+]]: compiler, {[[P11]]}, ir, (device-[[T]], [[ARCH2]]) -// NRD2-DAG: [[P13:[0-9]+]]: backend, {[[P12]]}, assembler, (device-[[T]], [[ARCH2]]) +// RDC2-DAG: [[P10:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T]], (device-[[T]], [[ARCH2:gfx900]]) +// RDC2-DAG: [[P11:[0-9]+]]: preprocessor, {[[P10]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH2]]) +// RDC2-DAG: [[P12:[0-9]+]]: compiler, {[[P11]]}, ir, (device-[[T]], [[ARCH2]]) // RDC2-DAG: [[P13:[0-9]+]]: backend, {[[P12]]}, ir, (device-[[T]], [[ARCH2]]) -// NRD2-DAG: [[P14:[0-9]+]]: assembler, {[[P13]]}, object, (device-[[T]], [[ARCH2]]) -// NRD2-DAG: [[P15:[0-9]+]]: linker, {[[P14]]}, image, (device-[[T]], [[ARCH2]]) // RCL2-DAG: [[P15:[0-9]+]]: linker, {[[P13]]}, image, (device-[[T]], [[ARCH2]]) -// NRD2-DAG: [[P16:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH2]])" {[[P15]]}, image +// RCL2-DAG: [[P16:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH2]])" {[[P15]]}, image // RC2-DAG: [[P16:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH2]])" {[[P13]]}, ir -// NRD2-DAG: [[P17:[0-9]+]]: linker, {[[P9]], [[P16]]}, hip-fatbin, (device-[[T]]) +// RC2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]]) +// RC2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]]) +// RC2-DAG: [[P2:[0-9]+]]: compiler, {[[P1]]}, ir, (host-[[T]]) +// RC2-DAG: [[P19:[0-9]+]]: backend, {[[P2]]}, assembler, (host-[[T]]) +// RC2-DAG: [[P20:[0-9]+]]: assembler, {[[P19]]}, object, (host-[[T]]) -// NRD2-DAG: [[P18:[0-9]+]]: offload, "host-[[T]] (x86_64-unknown-linux-gnu)" {[[P2]]}, "device-[[T]] (amdgcn-amd-amdhsa)" {[[P17]]}, ir -// NRD2-DAG: [[P19:[0-9]+]]: backend, {[[P18]]}, assembler, (host-[[T]]) -// RDC2-DAG: [[P19:[0-9]+]]: backend, {[[P2]]}, assembler, (host-[[T]]) -// BIN2-DAG: [[P20:[0-9]+]]: assembler, {[[P19]]}, object, (host-[[T]]) -// CL2-DAG: [[P21:[0-9]+]]: linker, {[[P20]]}, image, (host-[[T]]) -// RCL2-DAG: [[P22:[0-9]+]]: offload, "host-[[T]] (x86_64-unknown-linux-gnu)" {[[P21]]}, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH1]])" {[[P8]]}, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH2]])" {[[P15]]}, image -// RC2-DAG: [[P22:[0-9]+]]: clang-offload-bundler, {[[P9]], [[P16]], [[P20]]}, object, (host-[[T]]) +// RCL2-DAG: [[P17:[0-9]+]]: linker, {[[P9]], [[P16]]}, object, (device-[[T]]) +// RCL2-DAG: [[P22:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa)" {[[P17]]}, object +// RCL2-DAG: [[P23:[0-9]+]]: linker, {[[P20]], [[P22]]}, image, (host-[[T]]) +// RC2-DAG: [[P23:[0-9]+]]: clang-offload-bundler, {[[P9]], [[P16]], [[P20]]}, object, (host-[[T]]) // // Test two gpu architecturess up to the assemble phase. @@ -253,8 +280,13 @@ // RL2-DAG: [[P1:[0-9]+]]: clang-offload-unbundler, {[[P0]]}, object, (host-[[T]]) // L2-DAG: [[P2:[0-9]+]]: input, "{{.*}}obj2.o", object, (host-[[T]]) // RL2-DAG: [[P3:[0-9]+]]: clang-offload-unbundler, {[[P2]]}, object, (host-[[T]]) -// NL2-DAG: [[P4:[0-9]+]]: linker, {[[P0]], [[P2]]}, image, (host-[[T]]) -// RL2-DAG: [[P4:[0-9]+]]: linker, {[[P1]], [[P3]]}, image, (host-[[T]]) -// RL2-DAG: [[P5:[0-9]+]]: linker, {[[P1]], [[P3]]}, image, (device-[[T]], [[ARCH1:gfx803]]) + +// RL2-DAG: [[P4:[0-9]+]]: linker, {[[P1]], [[P3]]}, image, (device-[[T]], [[ARCH1:gfx803]]) +// RL2-DAG: [[P5:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH1]])" {[[P4]]}, image // RL2-DAG: [[P6:[0-9]+]]: linker, {[[P1]], [[P3]]}, image, (device-[[T]], [[ARCH2:gfx900]]) -// RL2-DAG: [[P7:[0-9]+]]: offload, "host-[[T]] (x86_64-unknown-linux-gnu)" {[[P4]]}, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH1]])" {[[P5]]}, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH2]])" {[[P6]]}, image +// RL2-DAG: [[P7:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa:[[ARCH2]])" {[[P6]]}, image +// RL2-DAG: [[P8:[0-9]+]]: linker, {[[P5]], [[P7]]}, object, (device-[[T]]) +// RL2-DAG: [[P9:[0-9]+]]: offload, "device-[[T]] (amdgcn-amd-amdhsa)" {[[P8]]}, object + +// NL2-DAG: [[P4:[0-9]+]]: linker, {[[P0]], [[P2]]}, image, (host-[[T]]) +// RL2-DAG: [[P4:[0-9]+]]: linker, {[[P1]], [[P3]], [[P9]]}, image, (host-[[T]]) diff --git a/clang/test/Driver/hip-save-temps.hip b/clang/test/Driver/hip-save-temps.hip index 1ac506aa6fba1..f326f4155a150 100644 --- a/clang/test/Driver/hip-save-temps.hip +++ b/clang/test/Driver/hip-save-temps.hip @@ -25,13 +25,20 @@ // -fgpu-rdc without -o // RUN: %clang -### -target x86_64-linux-gnu -nogpulib -save-temps \ // RUN: -fgpu-rdc --cuda-gpu-arch=gfx900 %s 2>&1 | \ -// RUN: FileCheck -check-prefixes=CHECK,RDC,RDCL,RDC-NOUT,NOUT %s +// RUN: FileCheck -check-prefixes=CHECK,RDC,RDCL,NOUT %s // -fgpu-rdc with -o -// RUN: %clang -### -target x86_64-linux-gnu -nogpulib -save-temps \ -// RUN: -o executable -fgpu-rdc --cuda-gpu-arch=gfx900 %s 2>&1 | \ -// RUN: FileCheck -check-prefixes=CHECK,RDC,RDCL,RDC-WOUT,WOUT %s +// UN: %clang -### -target x86_64-linux-gnu -nogpulib -save-temps \ +// UN: -o executable -fgpu-rdc --cuda-gpu-arch=gfx900 %s 2>&1 | \ +// UN: FileCheck -check-prefixes=CHECK,RDC,RDCL,WOUT %s + +// -fgpu-rdc host object path +// RDCL: "{{.*clang.*}}" "-cc1" {{.*}} "-E" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.cui" +// RDCL: "{{.*clang.*}}" "-cc1" {{.*}} "-emit-llvm-bc" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.bc" +// RDCL: "{{.*clang.*}}" "-cc1" {{.*}} "-S" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.s" +// RDCL: "{{.*clang.*}}" "-cc1as" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.o" +// device object paths // CHECK: {{".*clang.*"}} "-cc1" {{.*}} "-E" {{.*}} [[CPU:"-target-cpu" "gfx900"]] {{.*}} "-o" "hip-save-temps-hip-amdgcn-amd-amdhsa-gfx900.cui" // NORDC: {{".*clang.*"}} "-cc1" {{.*}} "-emit-llvm-bc" {{.*}} [[CPU]] {{.*}} "-disable-llvm-passes" {{.*}} "-o" "hip-save-temps-hip-amdgcn-amd-amdhsa-gfx900.bc" // RDC: {{".*clang.*"}} "-cc1" {{.*}} "-emit-llvm-bc" {{.*}} [[CPU]] {{.*}} "-disable-llvm-passes" {{.*}} "-o" "hip-save-temps-hip-amdgcn-amd-amdhsa-gfx900.tmp.bc" @@ -43,22 +50,26 @@ // RDC: {{".*clang.*"}} "-cc1" {{.*}} "-emit-llvm-bc" {{.*}} [[CPU]] {{.*}} "-o" "hip-save-temps-hip-amdgcn-amd-amdhsa-gfx900.bc" // NORDC: {{".*clang.*"}} "-cc1as" {{.*}} "-filetype" "obj" {{.*}} [[CPU]] {{.*}} "-o" "hip-save-temps-hip-amdgcn-amd-amdhsa-gfx900.o" -// CHECK-NOT: llvm-link -// CHECK-NOT: opt -// CHECK-NOT: llc +// CHECK-NOT: "{{.*}}llvm-link" +// CHECK-NOT: "{{.*}}opt" +// CHECK-NOT: "{{.*}}llc" // NORDC: {{.*lld.*}}"-o" "hip-save-temps-hip-amdgcn-amd-amdhsa-gfx900.out" - // RDCL: "{{.*lld.*}}" {{.*}} "-mllvm" "-amdgpu-internalize-symbols" // RDCL-SAME: "-o" "a.out-hip-amdgcn-amd-amdhsa-gfx900" +// RDCC: "{{.*clang.*}}" "-cc1" {{.*}} "-E" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.cui" +// RDCC: "{{.*clang.*}}" "-cc1" {{.*}} "-emit-llvm-bc" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.bc" +// RDCC: "{{.*clang.*}}" "-cc1" {{.*}} "-S" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.s" +// RDCC: "{{.*clang.*}}" "-cc1as" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.o" +// RDCC: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=hip-save-temps.o" +// RDCL: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=hip-save-temps-hip-amdgcn-amd-amdhsa.hipfb" +// RDCL: {{.*}}llvm-mc{{.*}}"-o" "hip-save-temps-hip-amdgcn-amd-amdhsa.o" "hip-save-temps-hip-amdgcn-amd-amdhsa.mcin" "--filetype=obj" -// NORDC: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=hip-save-temps.hip-hip-amdgcn-amd-amdhsa.hipfb" -// CHECK: "{{.*clang.*}}" "-cc1" {{.*}} "-E" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.cui" +// -fno-gpu-rdc host object path +// NORDC: "{{.*clang.*}}" "-cc1" {{.*}} "-E" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.cui" // NORDC: "{{.*clang.*}}" "-cc1" {{.*}} "-emit-llvm-bc" {{.*}} "-fcuda-include-gpubinary" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.bc" -// RDC: "{{.*clang.*}}" "-cc1" {{.*}} "-emit-llvm-bc" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.bc" -// CHECK: "{{.*clang.*}}" "-cc1" {{.*}} "-S" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.s" -// CHECK: "{{.*clang.*}}" "-cc1as" {{.*}} "-o" "hip-save-temps{{.*}}.o" -// RDCC: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=hip-save-temps.o" -// RDC-NOUT: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=a.out.hipfb" -// RDC-WOUT: "{{.*clang-offload-bundler.*}}" {{.*}} "-outputs=executable.hipfb" -// NOUT: "{{.*ld.*}}" {{.*}} "-o" "a.out" -// WOUT: "{{.*ld.*}}" {{.*}} "-o" "executable" \ No newline at end of file +// NORDC: "{{.*clang.*}}" "-cc1" {{.*}} "-S" {{.*}} "-o" "hip-save-temps-host-x86_64-unknown-linux-gnu.s" +// NORDC: "{{.*clang.*}}" "-cc1as" {{.*}} "-o" "hip-save-temps{{.*}}.o" + +// output to default a.out or -o specified file name +// NOUT: {{.*}}ld{{.*}}"-o" "a.out" +// WOUT: {{.*}}ld{{.*}}"-o" "executable" diff --git a/clang/test/Driver/hip-toolchain-rdc-separate.hip b/clang/test/Driver/hip-toolchain-rdc-separate.hip index 25661f399a82f..25fd44f2529b4 100644 --- a/clang/test/Driver/hip-toolchain-rdc-separate.hip +++ b/clang/test/Driver/hip-toolchain-rdc-separate.hip @@ -86,12 +86,22 @@ // LINK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o" // LINK-SAME: "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900" -// LINK-SAME: "-inputs=[[A_O:.*a.o]]" "-outputs=[[A_OBJ_HOST:.*o]],[[A_BC1:.*o]],[[A_BC2:.*o]]" +// LINK-SAME: "-inputs=[[A_O:.*a.o]]" "-outputs=[[A_OBJ_HOST:.*o]],{{.*o}},{{.*o}}" // LINK: "-unbundle" // LINK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o" // LINK-SAME: "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900" -// LINK-SAME: "-inputs=[[B_O:.*b.o]]" "-outputs=[[B_OBJ_HOST:.*o]],[[B_BC1:.*o]],[[B_BC2:.*o]]" +// LINK-SAME: "-inputs=[[B_O:.*b.o]]" "-outputs=[[B_OBJ_HOST:.*o]],{{.*o}},{{.*o}}" +// LINK: "-unbundle" + +// LINK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o" +// LINK-SAME: "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900" +// LINK-SAME: "-inputs=[[A_O]]" "-outputs={{.*o}},[[A_BC1:.*o]],[[A_BC2:.*o]]" +// LINK: "-unbundle" + +// LINK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o" +// LINK-SAME: "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900" +// LINK-SAME: "-inputs=[[B_O]]" "-outputs={{.*o}},[[B_BC1:.*o]],[[B_BC2:.*o]]" // LINK: "-unbundle" // LINK-NOT: "*.llvm-link" @@ -110,5 +120,8 @@ // LINK-SAME: "-targets={{.*}},hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900" // LINK-SAME: "-inputs={{.*}},[[IMG_DEV1]],[[IMG_DEV2]]" "-outputs=[[BUNDLE:.*hipfb]]" -// LINK: [[LD:".*ld.*"]] {{.*}} "[[A_OBJ_HOST]]" "[[B_OBJ_HOST]]" -// LINK-SAME: {{.*}} "-T" "{{.*}}.lk" +// LINK: {{".*llvm-mc.*"}} "-triple" "amdgcn-amd-amdhsa" "-o" +// LINK-SAME: "[[OBJBUNDLE:.*o]]" "{{.*}}.mcin" "--filetype=obj" + +// LINK: [[LD:".*ld.*"]] {{.*}} "-o" "a.out" {{.*}} "[[A_OBJ_HOST]]" +// LINK-SAME: "[[B_OBJ_HOST]]" "[[OBJBUNDLE]]" diff --git a/clang/test/Driver/hip-toolchain-rdc.hip b/clang/test/Driver/hip-toolchain-rdc.hip index f520236abcb9f..7148478b42859 100644 --- a/clang/test/Driver/hip-toolchain-rdc.hip +++ b/clang/test/Driver/hip-toolchain-rdc.hip @@ -12,7 +12,23 @@ // RUN: %S/Inputs/hip_multiple_inputs/b.hip \ // RUN: 2>&1 | FileCheck %s -// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-triple" "amdgcn-amd-amdhsa" +// emit objects for host side path +// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-triple" "x86_64-unknown-linux-gnu" +// CHECK-SAME: "-aux-triple" "amdgcn-amd-amdhsa" +// CHECK-SAME: "-emit-obj" +// CHECK-SAME: {{.*}} "-main-file-name" "a.cu" +// CHECK-SAME: {{.*}} "-o" [[A_OBJ_HOST:".*o"]] "-x" "hip" +// CHECK-SAME: {{.*}} [[A_SRC:".*a.cu"]] + +// CHECK: [[CLANG]] "-cc1" "-triple" "x86_64-unknown-linux-gnu" +// CHECK-SAME: "-aux-triple" "amdgcn-amd-amdhsa" +// CHECK-SAME: "-emit-obj" +// CHECK-SAME: {{.*}} "-main-file-name" "b.hip" +// CHECK-SAME: {{.*}} "-o" [[B_OBJ_HOST:".*o"]] "-x" "hip" +// CHECK-SAME: {{.*}} [[B_SRC:".*b.hip"]] + +// generate image for device side path on gfx803 +// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa" // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu" // CHECK-SAME: "-emit-llvm-bc" // CHECK-SAME: {{.*}} "-main-file-name" "a.cu" @@ -21,7 +37,7 @@ // CHECK-SAME: "{{.*}}lib1.bc" "{{.*}}lib2.bc" // CHECK-SAME: "-target-cpu" "gfx803" // CHECK-SAME: {{.*}} "-o" [[A_BC1:".*bc"]] "-x" "hip" -// CHECK-SAME: {{.*}} [[A_SRC:".*a.cu"]] +// CHECK-SAME: {{.*}} [[A_SRC]] // CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa" // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu" @@ -32,7 +48,7 @@ // CHECK-SAME: "{{.*}}lib1.bc" "{{.*}}lib2.bc" // CHECK-SAME: "-target-cpu" "gfx803" // CHECK-SAME: {{.*}} "-o" [[B_BC1:".*bc"]] "-x" "hip" -// CHECK-SAME: {{.*}} [[B_SRC:".*b.hip"]] +// CHECK-SAME: {{.*}} [[B_SRC]] // CHECK-NOT: "*.llvm-link" // CHECK-NOT: ".*opt" @@ -40,6 +56,7 @@ // CHECK: {{".*lld.*"}} {{.*}} "-mllvm" "-amdgpu-internalize-symbols" // CHECK-SAME: "-o" "[[IMG_DEV1:.*.out]]" [[A_BC1]] [[B_BC1]] +// generate image for device side path on gfx900 // CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa" // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu" // CHECK-SAME: "-emit-llvm-bc" @@ -66,23 +83,13 @@ // CHECK: {{".*lld.*"}} {{.*}} "-mllvm" "-amdgpu-internalize-symbols" // CHECK-SAME: "-o" "[[IMG_DEV2:.*.out]]" [[A_BC2]] [[B_BC2]] -// CHECK: [[CLANG]] "-cc1" "-triple" "x86_64-unknown-linux-gnu" -// CHECK-SAME: "-aux-triple" "amdgcn-amd-amdhsa" -// CHECK-SAME: "-emit-obj" -// CHECK-SAME: {{.*}} "-main-file-name" "a.cu" -// CHECK-SAME: {{.*}} "-o" [[A_OBJ_HOST:".*o"]] "-x" "hip" -// CHECK-SAME: {{.*}} [[A_SRC]] - -// CHECK: [[CLANG]] "-cc1" "-triple" "x86_64-unknown-linux-gnu" -// CHECK-SAME: "-aux-triple" "amdgcn-amd-amdhsa" -// CHECK-SAME: "-emit-obj" -// CHECK-SAME: {{.*}} "-main-file-name" "b.hip" -// CHECK-SAME: {{.*}} "-o" [[B_OBJ_HOST:".*o"]] "-x" "hip" -// CHECK-SAME: {{.*}} [[B_SRC]] - +// combine images generated into hip fat binary object // CHECK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o" // CHECK-SAME: "-targets={{.*}},hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900" // CHECK-SAME: "-inputs={{.*}},[[IMG_DEV1]],[[IMG_DEV2]]" "-outputs=[[BUNDLE:.*hipfb]]" -// CHECK: [[LD:".*ld.*"]] {{.*}} [[A_OBJ_HOST]] [[B_OBJ_HOST]] -// CHECK-SAME: {{.*}} "-T" "{{.*}}.lk" +// CHECK: [[MC:".*llvm-mc"]] "-triple" "amdgcn-amd-amdhsa" +// CHECK-SAME: "-o" [[OBJBUNDLE:".*o"]] "{{.*}}.mcin" "--filetype=obj" + +// output the executable +// CHECK: [[LD:".*ld.*"]] {{.*}}"-o" "a.out" {{.*}} [[A_OBJ_HOST]] [[B_OBJ_HOST]] [[OBJBUNDLE]]