diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index 31a2bdeb2d3e5e..3bc043b35e187b 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -1110,6 +1110,12 @@ def ReadCycleCounter : Builtin { let Prototype = "unsigned long long int()"; } +def ReadFixedTimer : Builtin { + let Spellings = ["__builtin_readfixedtimer"]; + let Attributes = [NoThrow]; + let Prototype = "unsigned long long int()"; +} + def Trap : Builtin { let Spellings = ["__builtin_trap"]; let Attributes = [NoThrow, NoReturn]; diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index a7a410dab1a018..8da8bbc56758d5 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -3443,6 +3443,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Function *F = CGM.getIntrinsic(Intrinsic::readcyclecounter); return RValue::get(Builder.CreateCall(F)); } + case Builtin::BI__builtin_readfixedtimer: { + Function *F = CGM.getIntrinsic(Intrinsic::readfixedtimer); + return RValue::get(Builder.CreateCall(F)); + } case Builtin::BI__builtin___clear_cache: { Value *Begin = EmitScalarExpr(E->getArg(0)); Value *End = EmitScalarExpr(E->getArg(1)); diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h index 349d1286c8dc4f..882e80c521e897 100644 --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -1179,6 +1179,12 @@ enum NodeType { /// counter-like register (or other high accuracy low latency clock source). READCYCLECOUNTER, + /// READFIXEDTIMER - This corresponds to the readfixedcounter intrinsic. + /// It has the same semantics as the READCYCLECOUNTER implementation except + /// that the result is the content of the architecture-specific fixed + /// frequency counter suitable for measuring elapsed time. + READFIXEDTIMER, + /// HANDLENODE node - Used as a handle for various purposes. HANDLENODE, diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index 3c19c7b063652c..4d7c57944f3778 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -870,6 +870,8 @@ def int_pcmarker : DefaultAttrsIntrinsic<[], [llvm_i32_ty]>; def int_readcyclecounter : DefaultAttrsIntrinsic<[llvm_i64_ty]>; +def int_readfixedtimer : DefaultAttrsIntrinsic<[llvm_i64_ty]>; + // The assume intrinsic is marked InaccessibleMemOnly so that proper control // dependencies will be maintained. def int_assume : DefaultAttrsIntrinsic< diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index abb237083d254e..29c6b6488ebb72 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -352,6 +352,9 @@ HANDLE_TARGET_OPCODE(G_INTRINSIC_ROUNDEVEN) /// INTRINSIC readcyclecounter HANDLE_TARGET_OPCODE(G_READCYCLECOUNTER) +/// INTRINSIC readfixedtimer +HANDLE_TARGET_OPCODE(G_READFIXEDTIMER) + /// Generic load (including anyext load) HANDLE_TARGET_OPCODE(G_LOAD) diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index 2c73b67f9e1af0..1b3f84c1d782bf 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -1101,6 +1101,12 @@ def G_READCYCLECOUNTER : GenericInstruction { let hasSideEffects = true; } +def G_READFIXEDTIMER : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins); + let hasSideEffects = true; +} + //------------------------------------------------------------------------------ // Memory ops //------------------------------------------------------------------------------ diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td index f792237203b431..e3e9622ae919d9 100644 --- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -167,6 +167,7 @@ def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; +def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index 22360353790dbc..bedfa6807f425a 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -657,6 +657,9 @@ def prefetch : SDNode<"ISD::PREFETCH" , SDTPrefetch, def readcyclecounter : SDNode<"ISD::READCYCLECOUNTER", SDTIntLeaf, [SDNPHasChain, SDNPSideEffect]>; +def readfixedtimer : SDNode<"ISD::READFIXEDTIMER", SDTIntLeaf, + [SDNPHasChain, SDNPSideEffect]>; + def membarrier : SDNode<"ISD::MEMBARRIER", SDTNone, [SDNPHasChain, SDNPSideEffect]>; diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index c1d8e890a66edb..2b2d03ab69c01c 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1885,6 +1885,8 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) { return TargetOpcode::G_INTRINSIC_TRUNC; case Intrinsic::readcyclecounter: return TargetOpcode::G_READCYCLECOUNTER; + case Intrinsic::readfixedtimer: + return TargetOpcode::G_READFIXEDTIMER; case Intrinsic::ptrmask: return TargetOpcode::G_PTRMASK; case Intrinsic::lrint: diff --git a/llvm/lib/CodeGen/IntrinsicLowering.cpp b/llvm/lib/CodeGen/IntrinsicLowering.cpp index 61920a0e04ab59..106799bc306850 100644 --- a/llvm/lib/CodeGen/IntrinsicLowering.cpp +++ b/llvm/lib/CodeGen/IntrinsicLowering.cpp @@ -312,6 +312,12 @@ void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) { CI->replaceAllUsesWith(ConstantInt::get(Type::getInt64Ty(Context), 0)); break; } + case Intrinsic::readfixedtimer: { + errs() << "WARNING: this target does not support the llvm.readfixedtimer" + << " intrinsic. It is being lowered to a constant 0\n"; + CI->replaceAllUsesWith(ConstantInt::get(Type::getInt64Ty(Context), 0)); + break; + } case Intrinsic::dbg_declare: case Intrinsic::dbg_label: diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 892bfbd62f0d02..29c2356cdc438c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -1127,8 +1127,9 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { Action = TargetLowering::Custom; break; case ISD::READCYCLECOUNTER: - // READCYCLECOUNTER returns an i64, even if type legalization might have - // expanded that to several smaller types. + case ISD::READFIXEDTIMER: + // READCYCLECOUNTER and READFIXEDTIMER return a i64, even if type + // legalization might have expanded that to several smaller types. Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64); break; case ISD::READ_REGISTER: @@ -3080,6 +3081,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { Results.push_back(Node->getOperand(0)); break; case ISD::READCYCLECOUNTER: + case ISD::READFIXEDTIMER: // If the target didn't expand this, just return 'zero' and preserve the // chain. Results.append(Node->getNumValues() - 1, diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index 39b7e061554141..90b1d87c17bb45 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -2648,7 +2648,12 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { case ISD::LLRINT: ExpandIntRes_XROUND_XRINT(N, Lo, Hi); break; case ISD::LOAD: ExpandIntRes_LOAD(cast(N), Lo, Hi); break; case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; - case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break; + case ISD::READCYCLECOUNTER: + ExpandIntRes_READCOUNTER(N, Lo, Hi); + break; + case ISD::READFIXEDTIMER: + ExpandIntRes_READCOUNTER(N, Lo, Hi); + break; case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; @@ -4026,8 +4031,8 @@ void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, Lo, Hi); } -void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo, - SDValue &Hi) { +void DAGTypeLegalizer::ExpandIntRes_READCOUNTER(SDNode *N, SDValue &Lo, + SDValue &Hi) { SDLoc DL(N); EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h index 09f0bca8b8611e..2b097030848618 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -439,7 +439,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer { void ExpandIntRes_CTPOP (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_CTTZ (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_LOAD (LoadSDNode *N, SDValue &Lo, SDValue &Hi); - void ExpandIntRes_READCYCLECOUNTER (SDNode *N, SDValue &Lo, SDValue &Hi); + void ExpandIntRes_READCOUNTER(SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_SIGN_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_SIGN_EXTEND_INREG (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_TRUNCATE (SDNode *N, SDValue &Lo, SDValue &Hi); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 5ce1013f30fd1b..ee1164c48140b3 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -6781,6 +6781,14 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, DAG.setRoot(Res.getValue(1)); return; } + case Intrinsic::readfixedtimer: { + SDValue Op = getRoot(); + Res = DAG.getNode(ISD::READFIXEDTIMER, sdl, + DAG.getVTList(MVT::i64, MVT::Other), Op); + setValue(&I, Res); + DAG.setRoot(Res.getValue(1)); + return; + } case Intrinsic::bitreverse: setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl, getValue(I.getArgOperand(0)).getValueType(), diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp index a28d834f0522f2..ebd49f130640c6 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -104,6 +104,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::ATOMIC_STORE: return "AtomicStore"; case ISD::PCMARKER: return "PCMarker"; case ISD::READCYCLECOUNTER: return "ReadCycleCounter"; + case ISD::READFIXEDTIMER: + return "ReadFixedTimer"; case ISD::SRCVALUE: return "SrcValue"; case ISD::MDNODE_SDNODE: return "MDNode"; case ISD::EntryToken: return "EntryToken"; diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp index d8302ba2b42608..e06b266ba3cc1d 100644 --- a/llvm/lib/CodeGen/TargetLoweringBase.cpp +++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -964,6 +964,9 @@ void TargetLoweringBase::initActions() { // Most targets also ignore the @llvm.readcyclecounter intrinsic. setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Expand); + // Most targets also ignore the @llvm.readfixedtimer intrinsic. + setOperationAction(ISD::READFIXEDTIMER, MVT::i64, Expand); + // ConstantFP nodes default to expand. Targets can either change this to // Legal, in which case all fp constants are legal, or use isFPImmLegal() // to optimize expansions for certain constants. diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index 97952de3e6a37b..63f843ea94fd71 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -1957,6 +1957,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, getActionDefinitionsBuilder(G_READCYCLECOUNTER) .legalFor({S64}); + getActionDefinitionsBuilder(G_READFIXEDTIMER).legalFor({S64}); + getActionDefinitionsBuilder(G_FENCE) .alwaysLegal(); diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp index 5323e4fc58de80..4a2c2ecdd8dd8b 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp @@ -4051,6 +4051,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case AMDGPU::G_CONSTANT: case AMDGPU::G_GLOBAL_VALUE: case AMDGPU::G_BLOCK_ADDR: + case AMDGPU::G_READFIXEDTIMER: case AMDGPU::G_READCYCLECOUNTER: { unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td index f3096962e2f3e8..21ef528bed2f8f 100644 --- a/llvm/lib/Target/AMDGPU/SMInstructions.td +++ b/llvm/lib/Target/AMDGPU/SMInstructions.td @@ -1068,6 +1068,20 @@ def : GCNPat < } } // let OtherPredicates = [HasShaderCyclesRegister] +let OtherPredicates = [HasSMemRealTime] in { +def : GCNPat < + (i64 (readfixedtimer)), + (S_MEMREALTIME) +>; +} // let OtherPredicates = [HasSMemRealTime] + +let SubtargetPredicate = isGFX11Plus in { +def : GCNPat < + (i64 (readfixedtimer)), + (S_SENDMSG_RTN_B64 (i32 0x83)) +>; +} // let SubtargetPredicate = [isGFX11Plus] + def i32imm_zero : TImmLeaf ;