Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DAG: Replace bitwidth with type in suffix in atomic tablegen ops #94845

Merged
merged 3 commits into from
Jun 13, 2024
Merged

DAG: Replace bitwidth with type in suffix in atomic tablegen ops #94845

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2f099ab
DAG: Replace bitwidth with type in suffix in atomic tablegen ops
arsenm Jun 7, 2024
2f1eda0
Remove AMDGPU custom v2f16 wrapper
arsenm Jun 10, 2024
fad8630
Indentation
arsenm Jun 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 26 additions & 48 deletions llvm/include/llvm/Target/TargetSelectionDAG.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1680,60 +1680,38 @@ multiclass ternary_atomic_op_ord {
}
}

multiclass binary_atomic_op<SDNode atomic_op, bit IsInt = 1> {
def _8 : PatFrag<(ops node:$ptr, node:$val),
(atomic_op node:$ptr, node:$val)> {
let IsAtomic = true;
let MemoryVT = !if(IsInt, i8, ?);
}
def _16 : PatFrag<(ops node:$ptr, node:$val),
(atomic_op node:$ptr, node:$val)> {
let IsAtomic = true;
let MemoryVT = !if(IsInt, i16, f16);
}
def _32 : PatFrag<(ops node:$ptr, node:$val),
(atomic_op node:$ptr, node:$val)> {
let IsAtomic = true;
let MemoryVT = !if(IsInt, i32, f32);
}
def _64 : PatFrag<(ops node:$ptr, node:$val),
(atomic_op node:$ptr, node:$val)> {
let IsAtomic = true;
let MemoryVT = !if(IsInt, i64, f64);
multiclass binary_atomic_op<SDNode atomic_op> {
foreach vt = [ i8, i16, i32, i64 ] in {
def _#vt : PatFrag<(ops node:$ptr, node:$val),
(atomic_op node:$ptr, node:$val)> {
let IsAtomic = true;
let MemoryVT = vt;
}

defm NAME#_#vt : binary_atomic_op_ord;
}
}

defm NAME#_8 : binary_atomic_op_ord;
defm NAME#_16 : binary_atomic_op_ord;
defm NAME#_32 : binary_atomic_op_ord;
defm NAME#_64 : binary_atomic_op_ord;
multiclass binary_atomic_op_fp<SDNode atomic_op> {
foreach vt = [ f16, bf16, v2f16, v2bf16, f32, f64 ] in {
def _#vt : PatFrag<(ops node:$ptr, node:$val),
(atomic_op node:$ptr, node:$val)> {
let IsAtomic = true;
let MemoryVT = vt;
}
}
}

multiclass ternary_atomic_op<SDNode atomic_op> {
def _8 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(atomic_op node:$ptr, node:$cmp, node:$val)> {
let IsAtomic = true;
let MemoryVT = i8;
}
def _16 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(atomic_op node:$ptr, node:$cmp, node:$val)> {
let IsAtomic = true;
let MemoryVT = i16;
}
def _32 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(atomic_op node:$ptr, node:$cmp, node:$val)> {
let IsAtomic = true;
let MemoryVT = i32;
}
def _64 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(atomic_op node:$ptr, node:$cmp, node:$val)> {
let IsAtomic = true;
let MemoryVT = i64;
foreach vt = [ i8, i16, i32, i64 ] in {
def _#vt : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(atomic_op node:$ptr, node:$cmp, node:$val)> {
let IsAtomic = true;
let MemoryVT = vt;
}

defm NAME#_#vt : ternary_atomic_op_ord;
}

defm NAME#_8 : ternary_atomic_op_ord;
defm NAME#_16 : ternary_atomic_op_ord;
defm NAME#_32 : ternary_atomic_op_ord;
defm NAME#_64 : ternary_atomic_op_ord;
}

defm atomic_load_add : binary_atomic_op<atomic_load_add>;
Expand Down
60 changes: 30 additions & 30 deletions llvm/lib/Target/AArch64/AArch64InstrFormats.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11887,79 +11887,79 @@ multiclass LDOPregister<bits<3> opc, string op, bits<1> Acq, bits<1> Rel,
// complex DAG for DstRHS.
let Predicates = [HasLSE] in
multiclass LDOPregister_patterns_ord_dag<string inst, string suffix, string op,
string size, dag SrcRHS, dag DstRHS> {
def : Pat<(!cast<PatFrag>(op#"_"#size#"_monotonic") GPR64sp:$Rn, SrcRHS),
ValueType vt, dag SrcRHS, dag DstRHS> {
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_monotonic") GPR64sp:$Rn, SrcRHS),
(!cast<Instruction>(inst # suffix) DstRHS, GPR64sp:$Rn)>;
def : Pat<(!cast<PatFrag>(op#"_"#size#"_acquire") GPR64sp:$Rn, SrcRHS),
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acquire") GPR64sp:$Rn, SrcRHS),
(!cast<Instruction>(inst # "A" # suffix) DstRHS, GPR64sp:$Rn)>;
def : Pat<(!cast<PatFrag>(op#"_"#size#"_release") GPR64sp:$Rn, SrcRHS),
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_release") GPR64sp:$Rn, SrcRHS),
(!cast<Instruction>(inst # "L" # suffix) DstRHS, GPR64sp:$Rn)>;
def : Pat<(!cast<PatFrag>(op#"_"#size#"_acq_rel") GPR64sp:$Rn, SrcRHS),
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acq_rel") GPR64sp:$Rn, SrcRHS),
(!cast<Instruction>(inst # "AL" # suffix) DstRHS, GPR64sp:$Rn)>;
def : Pat<(!cast<PatFrag>(op#"_"#size#"_seq_cst") GPR64sp:$Rn, SrcRHS),
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_seq_cst") GPR64sp:$Rn, SrcRHS),
(!cast<Instruction>(inst # "AL" # suffix) DstRHS, GPR64sp:$Rn)>;
}

multiclass LDOPregister_patterns_ord<string inst, string suffix, string op,
string size, dag RHS> {
defm : LDOPregister_patterns_ord_dag<inst, suffix, op, size, RHS, RHS>;
ValueType vt, dag RHS> {
defm : LDOPregister_patterns_ord_dag<inst, suffix, op, vt, RHS, RHS>;
}

multiclass LDOPregister_patterns_ord_mod<string inst, string suffix, string op,
string size, dag LHS, dag RHS> {
defm : LDOPregister_patterns_ord_dag<inst, suffix, op, size, LHS, RHS>;
ValueType vt, dag LHS, dag RHS> {
defm : LDOPregister_patterns_ord_dag<inst, suffix, op, vt, LHS, RHS>;
}

multiclass LDOPregister_patterns<string inst, string op> {
defm : LDOPregister_patterns_ord<inst, "X", op, "64", (i64 GPR64:$Rm)>;
defm : LDOPregister_patterns_ord<inst, "W", op, "32", (i32 GPR32:$Rm)>;
defm : LDOPregister_patterns_ord<inst, "H", op, "16", (i32 GPR32:$Rm)>;
defm : LDOPregister_patterns_ord<inst, "B", op, "8", (i32 GPR32:$Rm)>;
defm : LDOPregister_patterns_ord<inst, "X", op, i64, (i64 GPR64:$Rm)>;
defm : LDOPregister_patterns_ord<inst, "W", op, i32, (i32 GPR32:$Rm)>;
defm : LDOPregister_patterns_ord<inst, "H", op, i16, (i32 GPR32:$Rm)>;
defm : LDOPregister_patterns_ord<inst, "B", op, i8, (i32 GPR32:$Rm)>;
}

multiclass LDOPregister_patterns_mod<string inst, string op, string mod> {
defm : LDOPregister_patterns_ord_mod<inst, "X", op, "64",
defm : LDOPregister_patterns_ord_mod<inst, "X", op, i64,
(i64 GPR64:$Rm),
(i64 (!cast<Instruction>(mod#Xrr) XZR, GPR64:$Rm))>;
defm : LDOPregister_patterns_ord_mod<inst, "W", op, "32",
defm : LDOPregister_patterns_ord_mod<inst, "W", op, i32,
(i32 GPR32:$Rm),
(i32 (!cast<Instruction>(mod#Wrr) WZR, GPR32:$Rm))>;
defm : LDOPregister_patterns_ord_mod<inst, "H", op, "16",
defm : LDOPregister_patterns_ord_mod<inst, "H", op, i16,
(i32 GPR32:$Rm),
(i32 (!cast<Instruction>(mod#Wrr) WZR, GPR32:$Rm))>;
defm : LDOPregister_patterns_ord_mod<inst, "B", op, "8",
defm : LDOPregister_patterns_ord_mod<inst, "B", op, i8,
(i32 GPR32:$Rm),
(i32 (!cast<Instruction>(mod#Wrr) WZR, GPR32:$Rm))>;
}

let Predicates = [HasLSE] in
multiclass CASregister_patterns_ord_dag<string inst, string suffix, string op,
string size, dag OLD, dag NEW> {
def : Pat<(!cast<PatFrag>(op#"_"#size#"_monotonic") GPR64sp:$Rn, OLD, NEW),
ValueType vt, dag OLD, dag NEW> {
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_monotonic") GPR64sp:$Rn, OLD, NEW),
(!cast<Instruction>(inst # suffix) OLD, NEW, GPR64sp:$Rn)>;
def : Pat<(!cast<PatFrag>(op#"_"#size#"_acquire") GPR64sp:$Rn, OLD, NEW),
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acquire") GPR64sp:$Rn, OLD, NEW),
(!cast<Instruction>(inst # "A" # suffix) OLD, NEW, GPR64sp:$Rn)>;
def : Pat<(!cast<PatFrag>(op#"_"#size#"_release") GPR64sp:$Rn, OLD, NEW),
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_release") GPR64sp:$Rn, OLD, NEW),
(!cast<Instruction>(inst # "L" # suffix) OLD, NEW, GPR64sp:$Rn)>;
def : Pat<(!cast<PatFrag>(op#"_"#size#"_acq_rel") GPR64sp:$Rn, OLD, NEW),
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acq_rel") GPR64sp:$Rn, OLD, NEW),
(!cast<Instruction>(inst # "AL" # suffix) OLD, NEW, GPR64sp:$Rn)>;
def : Pat<(!cast<PatFrag>(op#"_"#size#"_seq_cst") GPR64sp:$Rn, OLD, NEW),
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_seq_cst") GPR64sp:$Rn, OLD, NEW),
(!cast<Instruction>(inst # "AL" # suffix) OLD, NEW, GPR64sp:$Rn)>;
}

multiclass CASregister_patterns_ord<string inst, string suffix, string op,
string size, dag OLD, dag NEW> {
defm : CASregister_patterns_ord_dag<inst, suffix, op, size, OLD, NEW>;
ValueType vt, dag OLD, dag NEW> {
defm : CASregister_patterns_ord_dag<inst, suffix, op, vt, OLD, NEW>;
}

multiclass CASregister_patterns<string inst, string op> {
defm : CASregister_patterns_ord<inst, "X", op, "64",
defm : CASregister_patterns_ord<inst, "X", op, i64,
(i64 GPR64:$Rold), (i64 GPR64:$Rnew)>;
defm : CASregister_patterns_ord<inst, "W", op, "32",
defm : CASregister_patterns_ord<inst, "W", op, i32,
(i32 GPR32:$Rold), (i32 GPR32:$Rnew)>;
defm : CASregister_patterns_ord<inst, "H", op, "16",
defm : CASregister_patterns_ord<inst, "H", op, i16,
(i32 GPR32:$Rold), (i32 GPR32:$Rnew)>;
defm : CASregister_patterns_ord<inst, "B", op, "8",
defm : CASregister_patterns_ord<inst, "B", op, i8,
(i32 GPR32:$Rold), (i32 GPR32:$Rnew)>;
}

Expand Down
8 changes: 4 additions & 4 deletions llvm/lib/Target/AArch64/AArch64InstrGISel.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -346,16 +346,16 @@ let Predicates = [HasNEON] in {
}

let Predicates = [HasNoLSE] in {
def : Pat<(atomic_cmp_swap_8 GPR64:$addr, GPR32:$desired, GPR32:$new),
def : Pat<(atomic_cmp_swap_i8 GPR64:$addr, GPR32:$desired, GPR32:$new),
(CMP_SWAP_8 GPR64:$addr, GPR32:$desired, GPR32:$new)>;

def : Pat<(atomic_cmp_swap_16 GPR64:$addr, GPR32:$desired, GPR32:$new),
def : Pat<(atomic_cmp_swap_i16 GPR64:$addr, GPR32:$desired, GPR32:$new),
(CMP_SWAP_16 GPR64:$addr, GPR32:$desired, GPR32:$new)>;

def : Pat<(atomic_cmp_swap_32 GPR64:$addr, GPR32:$desired, GPR32:$new),
def : Pat<(atomic_cmp_swap_i32 GPR64:$addr, GPR32:$desired, GPR32:$new),
(CMP_SWAP_32 GPR64:$addr, GPR32:$desired, GPR32:$new)>;

def : Pat<(atomic_cmp_swap_64 GPR64:$addr, GPR64:$desired, GPR64:$new),
def : Pat<(atomic_cmp_swap_i64 GPR64:$addr, GPR64:$desired, GPR64:$new),
(CMP_SWAP_64 GPR64:$addr, GPR64:$desired, GPR64:$new)>;
}

Expand Down
31 changes: 22 additions & 9 deletions llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -637,21 +637,36 @@ defm int_amdgcn_atomic_cond_sub_u32 : local_addr_space_atomic_op;
defm int_amdgcn_atomic_cond_sub_u32 : flat_addr_space_atomic_op;
defm int_amdgcn_atomic_cond_sub_u32 : global_addr_space_atomic_op;

multiclass noret_binary_atomic_op<SDNode atomic_op, bit IsInt = 1> {
multiclass noret_binary_atomic_op<SDNode atomic_op> {
let HasNoUse = true in
defm "_noret" : binary_atomic_op<atomic_op, IsInt>;
defm "_noret" : binary_atomic_op<atomic_op>;
}

multiclass noret_binary_atomic_op_fp<SDNode atomic_op> {
let HasNoUse = true in
defm "_noret" : binary_atomic_op_fp<atomic_op>;
}

multiclass noret_ternary_atomic_op<SDNode atomic_op> {
let HasNoUse = true in
defm "_noret" : ternary_atomic_op<atomic_op>;
}

multiclass binary_atomic_op_all_as<SDNode atomic_op, bit IsInt = 1> {
foreach as = [ "global", "flat", "constant", "local", "private", "region" ] in {
defvar atomic_addrspace_names = [ "global", "flat", "constant", "local", "private", "region" ];

multiclass binary_atomic_op_all_as<SDNode atomic_op> {
foreach as = atomic_addrspace_names in {
let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {
defm "_"#as : binary_atomic_op<atomic_op>;
defm "_"#as : noret_binary_atomic_op<atomic_op>;
}
}
}
multiclass binary_atomic_op_fp_all_as<SDNode atomic_op> {
foreach as = atomic_addrspace_names in {
let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {
defm "_"#as : binary_atomic_op<atomic_op, IsInt>;
defm "_"#as : noret_binary_atomic_op<atomic_op, IsInt>;
defm "_"#as : binary_atomic_op_fp<atomic_op>;
defm "_"#as : noret_binary_atomic_op_fp<atomic_op>;
}
}
}
Expand All @@ -666,11 +681,9 @@ defm atomic_load_sub : binary_atomic_op_all_as<atomic_load_sub>;
defm atomic_load_umax : binary_atomic_op_all_as<atomic_load_umax>;
defm atomic_load_umin : binary_atomic_op_all_as<atomic_load_umin>;
defm atomic_load_xor : binary_atomic_op_all_as<atomic_load_xor>;
defm atomic_load_fadd : binary_atomic_op_all_as<atomic_load_fadd, 0>;
defm atomic_load_fadd : binary_atomic_op_fp_all_as<atomic_load_fadd>;
defm atomic_load_uinc_wrap : binary_atomic_op_all_as<atomic_load_uinc_wrap>;
defm atomic_load_udec_wrap : binary_atomic_op_all_as<atomic_load_udec_wrap>;
let MemoryVT = v2f16 in
defm atomic_load_fadd_v2f16 : binary_atomic_op_all_as<atomic_load_fadd, 0>;
defm AMDGPUatomic_cmp_swap : binary_atomic_op_all_as<AMDGPUatomic_cmp_swap>;

def load_align8_local : PatFrag<(ops node:$ptr), (load_local node:$ptr)>,
Expand Down
4 changes: 2 additions & 2 deletions llvm/lib/Target/AMDGPU/BUFInstructions.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1545,7 +1545,7 @@ multiclass BufferAtomicPat_Common<string OpPrefix, ValueType vt, string Inst, bi

defvar Op = !cast<SDPatternOperator>(OpPrefix
# !if(!eq(RtnMode, "ret"), "", "_noret")
# !if(isIntr, "", "_" # vt.Size));
# !if(isIntr, "", "_" # vt));
defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", "");

let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in {
Expand Down Expand Up @@ -1582,7 +1582,7 @@ multiclass BufferAtomicCmpSwapPat_Common<ValueType vt, ValueType data_vt, string

defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global"
# !if(!eq(RtnMode, "ret"), "", "_noret")
# "_" # vt.Size);
# "_" # vt);
defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", "");
defvar data_vt_RC = getVregSrcForVT<data_vt>.ret.RegClass;

Expand Down
Loading
Loading