From 6c66638f33a81928a32081be430450701b698461 Mon Sep 17 00:00:00 2001 From: Tim Gymnich Date: Tue, 18 Jun 2024 05:09:17 +0200 Subject: [PATCH 1/9] [AArch64] Improve non-SVE popcount for 32bit and 64 bit using udot --- .../Target/AArch64/AArch64ISelLowering.cpp | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 0c834acacdca90..8f7a0bb0b56ced 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -9814,6 +9814,26 @@ SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op, Val = DAG.getBitcast(VT8Bit, Val); Val = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Val); + if (Subtarget->hasDotProd() && VT.getScalarSizeInBits() != 16) { + EVT DT = VT == MVT::v2i64 ? MVT::v4i32 : VT; + SDValue Zeros = DAG.getSplatBuildVector( + DT, DL, DAG.getConstant(0, DL, DT.getScalarType())); + SDValue Ones = + DAG.getSplatBuildVector(VT8Bit, DL, DAG.getConstant(1, DL, MVT::i8)); + + if (VT == MVT::v2i64) { + Val = DAG.getNode(AArch64ISD::UDOT, DL, DT, Zeros, Ones, Val); + Val = DAG.getNode(AArch64ISD::UADDLP, DL, VT, Val); + } else if (VT == MVT::v2i32) { + Val = DAG.getNode(AArch64ISD::UDOT, DL, DT, Zeros, Ones, Val); + } else if (VT == MVT::v4i32) { + Val = DAG.getNode(AArch64ISD::UDOT, DL, DT, Zeros, Ones, Val); + } else { + llvm_unreachable("Unexpected type for custom ctpop lowering"); + } + + return Val; + } // Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds. unsigned EltSize = 8; unsigned NumElts = VT.is64BitVector() ? 8 : 16; From 8482764a18c412751e29e236b6620f53cb793baa Mon Sep 17 00:00:00 2001 From: Tim Gymnich Date: Tue, 18 Jun 2024 06:46:42 +0200 Subject: [PATCH 2/9] add tests --- llvm/test/CodeGen/AArch64/popcount.ll | 295 ++++++++++++++++++++++++++ 1 file changed, 295 insertions(+) diff --git a/llvm/test/CodeGen/AArch64/popcount.ll b/llvm/test/CodeGen/AArch64/popcount.ll index b1231eeac1ea43..c041620fcc104d 100644 --- a/llvm/test/CodeGen/AArch64/popcount.ll +++ b/llvm/test/CodeGen/AArch64/popcount.ll @@ -1,5 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -O0 -mtriple=aarch64-unknown-unknown | FileCheck %s +; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon | FileCheck %s --check-prefix=NEON +; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon,+dotprod | FileCheck %s --check-prefix=DOT +; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+sve | FileCheck %s --check-prefix=SVE ; Function Attrs: nobuiltin nounwind readonly define i8 @popcount128(ptr nocapture nonnull readonly %0) { @@ -12,6 +15,36 @@ define i8 @popcount128(ptr nocapture nonnull readonly %0) { ; CHECK-NEXT: // kill: def $s0 killed $s0 killed $q0 ; CHECK-NEXT: fmov w0, s0 ; CHECK-NEXT: ret +; +; NEON-LABEL: popcount128: +; NEON: // %bb.0: // %Entry +; NEON-NEXT: ldr d0, [x0] +; NEON-NEXT: add x8, x0, #8 +; NEON-NEXT: ld1 { v0.d }[1], [x8] +; NEON-NEXT: cnt v0.16b, v0.16b +; NEON-NEXT: uaddlv h0, v0.16b +; NEON-NEXT: fmov w0, s0 +; NEON-NEXT: ret +; +; DOT-LABEL: popcount128: +; DOT: // %bb.0: // %Entry +; DOT-NEXT: ldr d0, [x0] +; DOT-NEXT: add x8, x0, #8 +; DOT-NEXT: ld1 { v0.d }[1], [x8] +; DOT-NEXT: cnt v0.16b, v0.16b +; DOT-NEXT: uaddlv h0, v0.16b +; DOT-NEXT: fmov w0, s0 +; DOT-NEXT: ret +; +; SVE-LABEL: popcount128: +; SVE: // %bb.0: // %Entry +; SVE-NEXT: ldr d0, [x0] +; SVE-NEXT: add x8, x0, #8 +; SVE-NEXT: ld1 { v0.d }[1], [x8] +; SVE-NEXT: cnt v0.16b, v0.16b +; SVE-NEXT: uaddlv h0, v0.16b +; SVE-NEXT: fmov w0, s0 +; SVE-NEXT: ret Entry: %1 = load i128, ptr %0, align 16 %2 = tail call i128 @llvm.ctpop.i128(i128 %1) @@ -56,6 +89,57 @@ define i16 @popcount256(ptr nocapture nonnull readonly %0) { ; CHECK-NEXT: adds x8, x8, x9 ; CHECK-NEXT: mov w0, w8 ; CHECK-NEXT: ret +; +; NEON-LABEL: popcount256: +; NEON: // %bb.0: // %Entry +; NEON-NEXT: ldr d0, [x0, #16] +; NEON-NEXT: ldr d1, [x0] +; NEON-NEXT: add x8, x0, #8 +; NEON-NEXT: add x9, x0, #24 +; NEON-NEXT: ld1 { v0.d }[1], [x9] +; NEON-NEXT: ld1 { v1.d }[1], [x8] +; NEON-NEXT: cnt v0.16b, v0.16b +; NEON-NEXT: cnt v1.16b, v1.16b +; NEON-NEXT: uaddlv h0, v0.16b +; NEON-NEXT: uaddlv h1, v1.16b +; NEON-NEXT: fmov w8, s0 +; NEON-NEXT: fmov w9, s1 +; NEON-NEXT: add w0, w9, w8 +; NEON-NEXT: ret +; +; DOT-LABEL: popcount256: +; DOT: // %bb.0: // %Entry +; DOT-NEXT: ldr d0, [x0, #16] +; DOT-NEXT: ldr d1, [x0] +; DOT-NEXT: add x8, x0, #8 +; DOT-NEXT: add x9, x0, #24 +; DOT-NEXT: ld1 { v0.d }[1], [x9] +; DOT-NEXT: ld1 { v1.d }[1], [x8] +; DOT-NEXT: cnt v0.16b, v0.16b +; DOT-NEXT: cnt v1.16b, v1.16b +; DOT-NEXT: uaddlv h0, v0.16b +; DOT-NEXT: uaddlv h1, v1.16b +; DOT-NEXT: fmov w8, s0 +; DOT-NEXT: fmov w9, s1 +; DOT-NEXT: add w0, w9, w8 +; DOT-NEXT: ret +; +; SVE-LABEL: popcount256: +; SVE: // %bb.0: // %Entry +; SVE-NEXT: ldr d0, [x0, #16] +; SVE-NEXT: ldr d1, [x0] +; SVE-NEXT: add x8, x0, #8 +; SVE-NEXT: add x9, x0, #24 +; SVE-NEXT: ld1 { v0.d }[1], [x9] +; SVE-NEXT: ld1 { v1.d }[1], [x8] +; SVE-NEXT: cnt v0.16b, v0.16b +; SVE-NEXT: cnt v1.16b, v1.16b +; SVE-NEXT: uaddlv h0, v0.16b +; SVE-NEXT: uaddlv h1, v1.16b +; SVE-NEXT: fmov w8, s0 +; SVE-NEXT: fmov w9, s1 +; SVE-NEXT: add w0, w9, w8 +; SVE-NEXT: ret Entry: %1 = load i256, ptr %0, align 16 %2 = tail call i256 @llvm.ctpop.i256(i256 %1) @@ -83,9 +167,220 @@ define <1 x i128> @popcount1x128(<1 x i128> %0) { ; CHECK-NEXT: // kill: def $x8 killed $w8 ; CHECK-NEXT: bfi x0, x8, #32, #32 ; CHECK-NEXT: ret +; +; NEON-LABEL: popcount1x128: +; NEON: // %bb.0: // %Entry +; NEON-NEXT: fmov d1, x0 +; NEON-NEXT: movi v0.2d, #0000000000000000 +; NEON-NEXT: mov v1.d[1], x1 +; NEON-NEXT: cnt v1.16b, v1.16b +; NEON-NEXT: uaddlv h1, v1.16b +; NEON-NEXT: mov v0.s[0], v1.s[0] +; NEON-NEXT: mov x1, v0.d[1] +; NEON-NEXT: fmov x0, d0 +; NEON-NEXT: ret +; +; DOT-LABEL: popcount1x128: +; DOT: // %bb.0: // %Entry +; DOT-NEXT: fmov d1, x0 +; DOT-NEXT: movi v0.2d, #0000000000000000 +; DOT-NEXT: mov v1.d[1], x1 +; DOT-NEXT: cnt v1.16b, v1.16b +; DOT-NEXT: uaddlv h1, v1.16b +; DOT-NEXT: mov v0.s[0], v1.s[0] +; DOT-NEXT: mov x1, v0.d[1] +; DOT-NEXT: fmov x0, d0 +; DOT-NEXT: ret +; +; SVE-LABEL: popcount1x128: +; SVE: // %bb.0: // %Entry +; SVE-NEXT: fmov d1, x0 +; SVE-NEXT: movi v0.2d, #0000000000000000 +; SVE-NEXT: mov v1.d[1], x1 +; SVE-NEXT: cnt v1.16b, v1.16b +; SVE-NEXT: uaddlv h1, v1.16b +; SVE-NEXT: mov v0.s[0], v1.s[0] +; SVE-NEXT: mov x1, v0.d[1] +; SVE-NEXT: fmov x0, d0 +; SVE-NEXT: ret Entry: %1 = tail call <1 x i128> @llvm.ctpop.v1.i128(<1 x i128> %0) ret <1 x i128> %1 } declare <1 x i128> @llvm.ctpop.v1.i128(<1 x i128>) + +define <2 x i64> @popcount2x64(<2 x i64> %0) { +; CHECK-LABEL: popcount2x64: +; CHECK: // %bb.0: // %Entry +; CHECK-NEXT: cnt v0.16b, v0.16b +; CHECK-NEXT: uaddlp v0.8h, v0.16b +; CHECK-NEXT: uaddlp v0.4s, v0.8h +; CHECK-NEXT: uaddlp v0.2d, v0.4s +; CHECK-NEXT: ret +; +; NEON-LABEL: popcount2x64: +; NEON: // %bb.0: // %Entry +; NEON-NEXT: cnt v0.16b, v0.16b +; NEON-NEXT: uaddlp v0.8h, v0.16b +; NEON-NEXT: uaddlp v0.4s, v0.8h +; NEON-NEXT: uaddlp v0.2d, v0.4s +; NEON-NEXT: ret +; +; DOT-LABEL: popcount2x64: +; DOT: // %bb.0: // %Entry +; DOT-NEXT: movi v1.16b, #1 +; DOT-NEXT: cnt v0.16b, v0.16b +; DOT-NEXT: movi v2.2d, #0000000000000000 +; DOT-NEXT: udot v2.4s, v1.16b, v0.16b +; DOT-NEXT: uaddlp v0.2d, v2.4s +; DOT-NEXT: ret +; +; SVE-LABEL: popcount2x64: +; SVE: // %bb.0: // %Entry +; SVE-NEXT: cnt v0.16b, v0.16b +; SVE-NEXT: uaddlp v0.8h, v0.16b +; SVE-NEXT: uaddlp v0.4s, v0.8h +; SVE-NEXT: uaddlp v0.2d, v0.4s +; SVE-NEXT: ret +Entry: + %1 = tail call <2 x i64> @llvm.ctpop.v2.i64(<2 x i64> %0) + ret <2 x i64> %1 +} + +declare <2 x i64> @llvm.ctpop.v2.i64(<2 x i64>) + +define <4 x i32> @popcount4x32(<4 x i32> %0) { +; CHECK-LABEL: popcount4x32: +; CHECK: // %bb.0: // %Entry +; CHECK-NEXT: cnt v0.16b, v0.16b +; CHECK-NEXT: uaddlp v0.8h, v0.16b +; CHECK-NEXT: uaddlp v0.4s, v0.8h +; CHECK-NEXT: ret +; +; NEON-LABEL: popcount4x32: +; NEON: // %bb.0: // %Entry +; NEON-NEXT: cnt v0.16b, v0.16b +; NEON-NEXT: uaddlp v0.8h, v0.16b +; NEON-NEXT: uaddlp v0.4s, v0.8h +; NEON-NEXT: ret +; +; DOT-LABEL: popcount4x32: +; DOT: // %bb.0: // %Entry +; DOT-NEXT: movi v1.16b, #1 +; DOT-NEXT: cnt v2.16b, v0.16b +; DOT-NEXT: movi v0.2d, #0000000000000000 +; DOT-NEXT: udot v0.4s, v1.16b, v2.16b +; DOT-NEXT: ret +; +; SVE-LABEL: popcount4x32: +; SVE: // %bb.0: // %Entry +; SVE-NEXT: cnt v0.16b, v0.16b +; SVE-NEXT: uaddlp v0.8h, v0.16b +; SVE-NEXT: uaddlp v0.4s, v0.8h +; SVE-NEXT: ret +Entry: + %1 = tail call <4 x i32> @llvm.ctpop.v4.i32(<4 x i32> %0) + ret <4 x i32> %1 +} + +declare <4 x i32> @llvm.ctpop.v4.i32(<4 x i32>) + +define <2 x i32> @popcount2x32(<2 x i32> %0) { +; CHECK-LABEL: popcount2x32: +; CHECK: // %bb.0: // %Entry +; CHECK-NEXT: cnt v0.8b, v0.8b +; CHECK-NEXT: uaddlp v0.4h, v0.8b +; CHECK-NEXT: uaddlp v0.2s, v0.4h +; CHECK-NEXT: ret +; +; NEON-LABEL: popcount2x32: +; NEON: // %bb.0: // %Entry +; NEON-NEXT: cnt v0.8b, v0.8b +; NEON-NEXT: uaddlp v0.4h, v0.8b +; NEON-NEXT: uaddlp v0.2s, v0.4h +; NEON-NEXT: ret +; +; DOT-LABEL: popcount2x32: +; DOT: // %bb.0: // %Entry +; DOT-NEXT: movi v1.2d, #0000000000000000 +; DOT-NEXT: cnt v0.8b, v0.8b +; DOT-NEXT: movi v2.8b, #1 +; DOT-NEXT: udot v1.2s, v2.8b, v0.8b +; DOT-NEXT: fmov d0, d1 +; DOT-NEXT: ret +; +; SVE-LABEL: popcount2x32: +; SVE: // %bb.0: // %Entry +; SVE-NEXT: cnt v0.8b, v0.8b +; SVE-NEXT: uaddlp v0.4h, v0.8b +; SVE-NEXT: uaddlp v0.2s, v0.4h +; SVE-NEXT: ret +Entry: + %1 = tail call <2 x i32> @llvm.ctpop.v2.i32(<2 x i32> %0) + ret <2 x i32> %1 +} + +declare <2 x i32> @llvm.ctpop.v2.i32(<2 x i32>) + +define <8 x i16> @popcount8x16(<8 x i16> %0) { +; CHECK-LABEL: popcount8x16: +; CHECK: // %bb.0: // %Entry +; CHECK-NEXT: cnt v0.16b, v0.16b +; CHECK-NEXT: uaddlp v0.8h, v0.16b +; CHECK-NEXT: ret +; +; NEON-LABEL: popcount8x16: +; NEON: // %bb.0: // %Entry +; NEON-NEXT: cnt v0.16b, v0.16b +; NEON-NEXT: uaddlp v0.8h, v0.16b +; NEON-NEXT: ret +; +; DOT-LABEL: popcount8x16: +; DOT: // %bb.0: // %Entry +; DOT-NEXT: cnt v0.16b, v0.16b +; DOT-NEXT: uaddlp v0.8h, v0.16b +; DOT-NEXT: ret +; +; SVE-LABEL: popcount8x16: +; SVE: // %bb.0: // %Entry +; SVE-NEXT: cnt v0.16b, v0.16b +; SVE-NEXT: uaddlp v0.8h, v0.16b +; SVE-NEXT: ret +Entry: + %1 = tail call <8 x i16> @llvm.ctpop.v8.i16(<8 x i16> %0) + ret <8 x i16> %1 +} + +declare <8 x i16> @llvm.ctpop.v8.i16(<8 x i16>) + +define <4 x i16> @popcount4x16(<4 x i16> %0) { +; CHECK-LABEL: popcount4x16: +; CHECK: // %bb.0: // %Entry +; CHECK-NEXT: cnt v0.8b, v0.8b +; CHECK-NEXT: uaddlp v0.4h, v0.8b +; CHECK-NEXT: ret +; +; NEON-LABEL: popcount4x16: +; NEON: // %bb.0: // %Entry +; NEON-NEXT: cnt v0.8b, v0.8b +; NEON-NEXT: uaddlp v0.4h, v0.8b +; NEON-NEXT: ret +; +; DOT-LABEL: popcount4x16: +; DOT: // %bb.0: // %Entry +; DOT-NEXT: cnt v0.8b, v0.8b +; DOT-NEXT: uaddlp v0.4h, v0.8b +; DOT-NEXT: ret +; +; SVE-LABEL: popcount4x16: +; SVE: // %bb.0: // %Entry +; SVE-NEXT: cnt v0.8b, v0.8b +; SVE-NEXT: uaddlp v0.4h, v0.8b +; SVE-NEXT: ret +Entry: + %1 = tail call <4 x i16> @llvm.ctpop.v4.i16(<4 x i16> %0) + ret <4 x i16> %1 +} + +declare <4 x i16> @llvm.ctpop.v4.i16(<4 x i16>) From a287e235a56a0bca8859a9c4f80445e50bd43990 Mon Sep 17 00:00:00 2001 From: Tim Gymnich Date: Tue, 18 Jun 2024 13:53:00 +0200 Subject: [PATCH 3/9] combine check prefixes --- llvm/test/CodeGen/AArch64/popcount.ll | 124 +------------------------- 1 file changed, 3 insertions(+), 121 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/popcount.ll b/llvm/test/CodeGen/AArch64/popcount.ll index c041620fcc104d..5329d619e58a9e 100644 --- a/llvm/test/CodeGen/AArch64/popcount.ll +++ b/llvm/test/CodeGen/AArch64/popcount.ll @@ -1,21 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -O0 -mtriple=aarch64-unknown-unknown | FileCheck %s -; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon | FileCheck %s --check-prefix=NEON -; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon,+dotprod | FileCheck %s --check-prefix=DOT -; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+sve | FileCheck %s --check-prefix=SVE +; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon | FileCheck %s --check-prefixes=CHECK,NEON +; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon,+dotprod | FileCheck %s --check-prefixes=CHECK,DOT +; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+sve | FileCheck %s --check-prefixes=CHECK,SVE ; Function Attrs: nobuiltin nounwind readonly define i8 @popcount128(ptr nocapture nonnull readonly %0) { -; CHECK-LABEL: popcount128: -; CHECK: // %bb.0: // %Entry -; CHECK-NEXT: ldr q0, [x0] -; CHECK-NEXT: cnt v0.16b, v0.16b -; CHECK-NEXT: uaddlv h0, v0.16b -; CHECK-NEXT: // kill: def $q0 killed $h0 -; CHECK-NEXT: // kill: def $s0 killed $s0 killed $q0 -; CHECK-NEXT: fmov w0, s0 -; CHECK-NEXT: ret -; ; NEON-LABEL: popcount128: ; NEON: // %bb.0: // %Entry ; NEON-NEXT: ldr d0, [x0] @@ -57,39 +47,6 @@ declare i128 @llvm.ctpop.i128(i128) ; Function Attrs: nobuiltin nounwind readonly define i16 @popcount256(ptr nocapture nonnull readonly %0) { -; CHECK-LABEL: popcount256: -; CHECK: // %bb.0: // %Entry -; CHECK-NEXT: ldr x11, [x0] -; CHECK-NEXT: ldr x10, [x0, #8] -; CHECK-NEXT: ldr x9, [x0, #16] -; CHECK-NEXT: ldr x8, [x0, #24] -; CHECK-NEXT: // implicit-def: $q1 -; CHECK-NEXT: mov v1.d[0], x11 -; CHECK-NEXT: mov v1.d[1], x10 -; CHECK-NEXT: // implicit-def: $q0 -; CHECK-NEXT: mov v0.d[0], x9 -; CHECK-NEXT: mov v0.d[1], x8 -; CHECK-NEXT: cnt v1.16b, v1.16b -; CHECK-NEXT: uaddlv h1, v1.16b -; CHECK-NEXT: // kill: def $q1 killed $h1 -; CHECK-NEXT: // kill: def $s1 killed $s1 killed $q1 -; CHECK-NEXT: fmov w0, s1 -; CHECK-NEXT: mov w10, wzr -; CHECK-NEXT: mov w9, w0 -; CHECK-NEXT: mov w8, w10 -; CHECK-NEXT: bfi x9, x8, #32, #32 -; CHECK-NEXT: cnt v0.16b, v0.16b -; CHECK-NEXT: uaddlv h0, v0.16b -; CHECK-NEXT: // kill: def $q0 killed $h0 -; CHECK-NEXT: // kill: def $s0 killed $s0 killed $q0 -; CHECK-NEXT: fmov w0, s0 -; CHECK-NEXT: mov w8, w0 -; CHECK-NEXT: // kill: def $x10 killed $w10 -; CHECK-NEXT: bfi x8, x10, #32, #32 -; CHECK-NEXT: adds x8, x8, x9 -; CHECK-NEXT: mov w0, w8 -; CHECK-NEXT: ret -; ; NEON-LABEL: popcount256: ; NEON: // %bb.0: // %Entry ; NEON-NEXT: ldr d0, [x0, #16] @@ -151,23 +108,6 @@ Entry: declare i256 @llvm.ctpop.i256(i256) define <1 x i128> @popcount1x128(<1 x i128> %0) { -; CHECK-LABEL: popcount1x128: -; CHECK: // %bb.0: // %Entry -; CHECK-NEXT: // implicit-def: $q0 -; CHECK-NEXT: mov v0.d[0], x0 -; CHECK-NEXT: mov v0.d[1], x1 -; CHECK-NEXT: cnt v0.16b, v0.16b -; CHECK-NEXT: uaddlv h0, v0.16b -; CHECK-NEXT: // kill: def $q0 killed $h0 -; CHECK-NEXT: mov x1, xzr -; CHECK-NEXT: // kill: def $s0 killed $s0 killed $q0 -; CHECK-NEXT: fmov w0, s0 -; CHECK-NEXT: mov w8, wzr -; CHECK-NEXT: // kill: def $x0 killed $w0 -; CHECK-NEXT: // kill: def $x8 killed $w8 -; CHECK-NEXT: bfi x0, x8, #32, #32 -; CHECK-NEXT: ret -; ; NEON-LABEL: popcount1x128: ; NEON: // %bb.0: // %Entry ; NEON-NEXT: fmov d1, x0 @@ -211,14 +151,6 @@ Entry: declare <1 x i128> @llvm.ctpop.v1.i128(<1 x i128>) define <2 x i64> @popcount2x64(<2 x i64> %0) { -; CHECK-LABEL: popcount2x64: -; CHECK: // %bb.0: // %Entry -; CHECK-NEXT: cnt v0.16b, v0.16b -; CHECK-NEXT: uaddlp v0.8h, v0.16b -; CHECK-NEXT: uaddlp v0.4s, v0.8h -; CHECK-NEXT: uaddlp v0.2d, v0.4s -; CHECK-NEXT: ret -; ; NEON-LABEL: popcount2x64: ; NEON: // %bb.0: // %Entry ; NEON-NEXT: cnt v0.16b, v0.16b @@ -251,13 +183,6 @@ Entry: declare <2 x i64> @llvm.ctpop.v2.i64(<2 x i64>) define <4 x i32> @popcount4x32(<4 x i32> %0) { -; CHECK-LABEL: popcount4x32: -; CHECK: // %bb.0: // %Entry -; CHECK-NEXT: cnt v0.16b, v0.16b -; CHECK-NEXT: uaddlp v0.8h, v0.16b -; CHECK-NEXT: uaddlp v0.4s, v0.8h -; CHECK-NEXT: ret -; ; NEON-LABEL: popcount4x32: ; NEON: // %bb.0: // %Entry ; NEON-NEXT: cnt v0.16b, v0.16b @@ -287,13 +212,6 @@ Entry: declare <4 x i32> @llvm.ctpop.v4.i32(<4 x i32>) define <2 x i32> @popcount2x32(<2 x i32> %0) { -; CHECK-LABEL: popcount2x32: -; CHECK: // %bb.0: // %Entry -; CHECK-NEXT: cnt v0.8b, v0.8b -; CHECK-NEXT: uaddlp v0.4h, v0.8b -; CHECK-NEXT: uaddlp v0.2s, v0.4h -; CHECK-NEXT: ret -; ; NEON-LABEL: popcount2x32: ; NEON: // %bb.0: // %Entry ; NEON-NEXT: cnt v0.8b, v0.8b @@ -329,24 +247,6 @@ define <8 x i16> @popcount8x16(<8 x i16> %0) { ; CHECK-NEXT: cnt v0.16b, v0.16b ; CHECK-NEXT: uaddlp v0.8h, v0.16b ; CHECK-NEXT: ret -; -; NEON-LABEL: popcount8x16: -; NEON: // %bb.0: // %Entry -; NEON-NEXT: cnt v0.16b, v0.16b -; NEON-NEXT: uaddlp v0.8h, v0.16b -; NEON-NEXT: ret -; -; DOT-LABEL: popcount8x16: -; DOT: // %bb.0: // %Entry -; DOT-NEXT: cnt v0.16b, v0.16b -; DOT-NEXT: uaddlp v0.8h, v0.16b -; DOT-NEXT: ret -; -; SVE-LABEL: popcount8x16: -; SVE: // %bb.0: // %Entry -; SVE-NEXT: cnt v0.16b, v0.16b -; SVE-NEXT: uaddlp v0.8h, v0.16b -; SVE-NEXT: ret Entry: %1 = tail call <8 x i16> @llvm.ctpop.v8.i16(<8 x i16> %0) ret <8 x i16> %1 @@ -360,24 +260,6 @@ define <4 x i16> @popcount4x16(<4 x i16> %0) { ; CHECK-NEXT: cnt v0.8b, v0.8b ; CHECK-NEXT: uaddlp v0.4h, v0.8b ; CHECK-NEXT: ret -; -; NEON-LABEL: popcount4x16: -; NEON: // %bb.0: // %Entry -; NEON-NEXT: cnt v0.8b, v0.8b -; NEON-NEXT: uaddlp v0.4h, v0.8b -; NEON-NEXT: ret -; -; DOT-LABEL: popcount4x16: -; DOT: // %bb.0: // %Entry -; DOT-NEXT: cnt v0.8b, v0.8b -; DOT-NEXT: uaddlp v0.4h, v0.8b -; DOT-NEXT: ret -; -; SVE-LABEL: popcount4x16: -; SVE: // %bb.0: // %Entry -; SVE-NEXT: cnt v0.8b, v0.8b -; SVE-NEXT: uaddlp v0.4h, v0.8b -; SVE-NEXT: ret Entry: %1 = tail call <4 x i16> @llvm.ctpop.v4.i16(<4 x i16> %0) ret <4 x i16> %1 From 0d57b39e425f2966ad95df952335c26558130371 Mon Sep 17 00:00:00 2001 From: Tim Gymnich Date: Tue, 18 Jun 2024 15:09:41 +0200 Subject: [PATCH 4/9] exclude v1i64 --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 8f7a0bb0b56ced..4dd0137d914b43 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -9814,7 +9814,8 @@ SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op, Val = DAG.getBitcast(VT8Bit, Val); Val = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Val); - if (Subtarget->hasDotProd() && VT.getScalarSizeInBits() != 16) { + if (Subtarget->hasDotProd() && VT.getScalarSizeInBits() != 16 && + VT.getVectorNumElements() >= 2) { EVT DT = VT == MVT::v2i64 ? MVT::v4i32 : VT; SDValue Zeros = DAG.getSplatBuildVector( DT, DL, DAG.getConstant(0, DL, DT.getScalarType())); @@ -9834,6 +9835,7 @@ SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op, return Val; } + // Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds. unsigned EltSize = 8; unsigned NumElts = VT.is64BitVector() ? 8 : 16; From 256b37a239e71dc56c42a451cd22c10d286e3902 Mon Sep 17 00:00:00 2001 From: Tim Gymnich Date: Tue, 18 Jun 2024 15:10:23 +0200 Subject: [PATCH 5/9] use getConstant --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 4dd0137d914b43..2b13154cdcbb3a 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -9817,10 +9817,8 @@ SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op, if (Subtarget->hasDotProd() && VT.getScalarSizeInBits() != 16 && VT.getVectorNumElements() >= 2) { EVT DT = VT == MVT::v2i64 ? MVT::v4i32 : VT; - SDValue Zeros = DAG.getSplatBuildVector( - DT, DL, DAG.getConstant(0, DL, DT.getScalarType())); - SDValue Ones = - DAG.getSplatBuildVector(VT8Bit, DL, DAG.getConstant(1, DL, MVT::i8)); + SDValue Zeros = DAG.getConstant(0, DL, DT); + SDValue Ones = DAG.getConstant(1, DL, VT8Bit); if (VT == MVT::v2i64) { Val = DAG.getNode(AArch64ISD::UDOT, DL, DT, Zeros, Ones, Val); From e6c5597cbf3d7e02708c4584cfa7a1f7597faef5 Mon Sep 17 00:00:00 2001 From: Tim Gymnich Date: Tue, 18 Jun 2024 19:56:15 +0200 Subject: [PATCH 6/9] fix check prefix --- llvm/test/CodeGen/AArch64/popcount.ll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/test/CodeGen/AArch64/popcount.ll b/llvm/test/CodeGen/AArch64/popcount.ll index 5329d619e58a9e..c0cd35258ee3ac 100644 --- a/llvm/test/CodeGen/AArch64/popcount.ll +++ b/llvm/test/CodeGen/AArch64/popcount.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -O0 -mtriple=aarch64-unknown-unknown | FileCheck %s +; RUN: llc < %s -O0 -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefix=CHECKO0 ; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon | FileCheck %s --check-prefixes=CHECK,NEON ; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon,+dotprod | FileCheck %s --check-prefixes=CHECK,DOT ; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+sve | FileCheck %s --check-prefixes=CHECK,SVE From 7bee852f6476ce41a35aa0d6a7287dcb8e96693b Mon Sep 17 00:00:00 2001 From: Tim Gymnich Date: Tue, 18 Jun 2024 19:56:54 +0200 Subject: [PATCH 7/9] fix intrinsic names --- llvm/test/CodeGen/AArch64/popcount.ll | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/popcount.ll b/llvm/test/CodeGen/AArch64/popcount.ll index c0cd35258ee3ac..d50c50d7aee291 100644 --- a/llvm/test/CodeGen/AArch64/popcount.ll +++ b/llvm/test/CodeGen/AArch64/popcount.ll @@ -144,11 +144,11 @@ define <1 x i128> @popcount1x128(<1 x i128> %0) { ; SVE-NEXT: fmov x0, d0 ; SVE-NEXT: ret Entry: - %1 = tail call <1 x i128> @llvm.ctpop.v1.i128(<1 x i128> %0) + %1 = tail call <1 x i128> @llvm.ctpop.v1i128(<1 x i128> %0) ret <1 x i128> %1 } -declare <1 x i128> @llvm.ctpop.v1.i128(<1 x i128>) +declare <1 x i128> @llvm.ctpop.v1i128(<1 x i128>) define <2 x i64> @popcount2x64(<2 x i64> %0) { ; NEON-LABEL: popcount2x64: @@ -176,11 +176,11 @@ define <2 x i64> @popcount2x64(<2 x i64> %0) { ; SVE-NEXT: uaddlp v0.2d, v0.4s ; SVE-NEXT: ret Entry: - %1 = tail call <2 x i64> @llvm.ctpop.v2.i64(<2 x i64> %0) + %1 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0) ret <2 x i64> %1 } -declare <2 x i64> @llvm.ctpop.v2.i64(<2 x i64>) +declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>) define <4 x i32> @popcount4x32(<4 x i32> %0) { ; NEON-LABEL: popcount4x32: @@ -205,11 +205,11 @@ define <4 x i32> @popcount4x32(<4 x i32> %0) { ; SVE-NEXT: uaddlp v0.4s, v0.8h ; SVE-NEXT: ret Entry: - %1 = tail call <4 x i32> @llvm.ctpop.v4.i32(<4 x i32> %0) + %1 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0) ret <4 x i32> %1 } -declare <4 x i32> @llvm.ctpop.v4.i32(<4 x i32>) +declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>) define <2 x i32> @popcount2x32(<2 x i32> %0) { ; NEON-LABEL: popcount2x32: @@ -235,11 +235,11 @@ define <2 x i32> @popcount2x32(<2 x i32> %0) { ; SVE-NEXT: uaddlp v0.2s, v0.4h ; SVE-NEXT: ret Entry: - %1 = tail call <2 x i32> @llvm.ctpop.v2.i32(<2 x i32> %0) + %1 = tail call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %0) ret <2 x i32> %1 } -declare <2 x i32> @llvm.ctpop.v2.i32(<2 x i32>) +declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>) define <8 x i16> @popcount8x16(<8 x i16> %0) { ; CHECK-LABEL: popcount8x16: @@ -248,11 +248,11 @@ define <8 x i16> @popcount8x16(<8 x i16> %0) { ; CHECK-NEXT: uaddlp v0.8h, v0.16b ; CHECK-NEXT: ret Entry: - %1 = tail call <8 x i16> @llvm.ctpop.v8.i16(<8 x i16> %0) + %1 = tail call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %0) ret <8 x i16> %1 } -declare <8 x i16> @llvm.ctpop.v8.i16(<8 x i16>) +declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>) define <4 x i16> @popcount4x16(<4 x i16> %0) { ; CHECK-LABEL: popcount4x16: @@ -261,8 +261,8 @@ define <4 x i16> @popcount4x16(<4 x i16> %0) { ; CHECK-NEXT: uaddlp v0.4h, v0.8b ; CHECK-NEXT: ret Entry: - %1 = tail call <4 x i16> @llvm.ctpop.v4.i16(<4 x i16> %0) + %1 = tail call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %0) ret <4 x i16> %1 } -declare <4 x i16> @llvm.ctpop.v4.i16(<4 x i16>) +declare <4 x i16> @llvm.ctpop.v4i16(<4 x i16>) From a78906bc29b6f7708ab01f1828381e3a75d8b1ec Mon Sep 17 00:00:00 2001 From: Tim Gymnich Date: Tue, 18 Jun 2024 20:05:54 +0200 Subject: [PATCH 8/9] update tests --- llvm/test/CodeGen/AArch64/popcount.ll | 238 ++++++++++++++------------ 1 file changed, 127 insertions(+), 111 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/popcount.ll b/llvm/test/CodeGen/AArch64/popcount.ll index d50c50d7aee291..9a0b2299e8d244 100644 --- a/llvm/test/CodeGen/AArch64/popcount.ll +++ b/llvm/test/CodeGen/AArch64/popcount.ll @@ -6,35 +6,25 @@ ; Function Attrs: nobuiltin nounwind readonly define i8 @popcount128(ptr nocapture nonnull readonly %0) { -; NEON-LABEL: popcount128: -; NEON: // %bb.0: // %Entry -; NEON-NEXT: ldr d0, [x0] -; NEON-NEXT: add x8, x0, #8 -; NEON-NEXT: ld1 { v0.d }[1], [x8] -; NEON-NEXT: cnt v0.16b, v0.16b -; NEON-NEXT: uaddlv h0, v0.16b -; NEON-NEXT: fmov w0, s0 -; NEON-NEXT: ret +; CHECKO0-LABEL: popcount128: +; CHECKO0: // %bb.0: // %Entry +; CHECKO0-NEXT: ldr q0, [x0] +; CHECKO0-NEXT: cnt v0.16b, v0.16b +; CHECKO0-NEXT: uaddlv h0, v0.16b +; CHECKO0-NEXT: // kill: def $q0 killed $h0 +; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q0 +; CHECKO0-NEXT: fmov w0, s0 +; CHECKO0-NEXT: ret ; -; DOT-LABEL: popcount128: -; DOT: // %bb.0: // %Entry -; DOT-NEXT: ldr d0, [x0] -; DOT-NEXT: add x8, x0, #8 -; DOT-NEXT: ld1 { v0.d }[1], [x8] -; DOT-NEXT: cnt v0.16b, v0.16b -; DOT-NEXT: uaddlv h0, v0.16b -; DOT-NEXT: fmov w0, s0 -; DOT-NEXT: ret -; -; SVE-LABEL: popcount128: -; SVE: // %bb.0: // %Entry -; SVE-NEXT: ldr d0, [x0] -; SVE-NEXT: add x8, x0, #8 -; SVE-NEXT: ld1 { v0.d }[1], [x8] -; SVE-NEXT: cnt v0.16b, v0.16b -; SVE-NEXT: uaddlv h0, v0.16b -; SVE-NEXT: fmov w0, s0 -; SVE-NEXT: ret +; CHECK-LABEL: popcount128: +; CHECK: // %bb.0: // %Entry +; CHECK-NEXT: ldr d0, [x0] +; CHECK-NEXT: add x8, x0, #8 +; CHECK-NEXT: ld1 { v0.d }[1], [x8] +; CHECK-NEXT: cnt v0.16b, v0.16b +; CHECK-NEXT: uaddlv h0, v0.16b +; CHECK-NEXT: fmov w0, s0 +; CHECK-NEXT: ret Entry: %1 = load i128, ptr %0, align 16 %2 = tail call i128 @llvm.ctpop.i128(i128 %1) @@ -47,56 +37,55 @@ declare i128 @llvm.ctpop.i128(i128) ; Function Attrs: nobuiltin nounwind readonly define i16 @popcount256(ptr nocapture nonnull readonly %0) { -; NEON-LABEL: popcount256: -; NEON: // %bb.0: // %Entry -; NEON-NEXT: ldr d0, [x0, #16] -; NEON-NEXT: ldr d1, [x0] -; NEON-NEXT: add x8, x0, #8 -; NEON-NEXT: add x9, x0, #24 -; NEON-NEXT: ld1 { v0.d }[1], [x9] -; NEON-NEXT: ld1 { v1.d }[1], [x8] -; NEON-NEXT: cnt v0.16b, v0.16b -; NEON-NEXT: cnt v1.16b, v1.16b -; NEON-NEXT: uaddlv h0, v0.16b -; NEON-NEXT: uaddlv h1, v1.16b -; NEON-NEXT: fmov w8, s0 -; NEON-NEXT: fmov w9, s1 -; NEON-NEXT: add w0, w9, w8 -; NEON-NEXT: ret +; CHECKO0-LABEL: popcount256: +; CHECKO0: // %bb.0: // %Entry +; CHECKO0-NEXT: ldr x11, [x0] +; CHECKO0-NEXT: ldr x10, [x0, #8] +; CHECKO0-NEXT: ldr x9, [x0, #16] +; CHECKO0-NEXT: ldr x8, [x0, #24] +; CHECKO0-NEXT: // implicit-def: $q1 +; CHECKO0-NEXT: mov v1.d[0], x11 +; CHECKO0-NEXT: mov v1.d[1], x10 +; CHECKO0-NEXT: // implicit-def: $q0 +; CHECKO0-NEXT: mov v0.d[0], x9 +; CHECKO0-NEXT: mov v0.d[1], x8 +; CHECKO0-NEXT: cnt v1.16b, v1.16b +; CHECKO0-NEXT: uaddlv h1, v1.16b +; CHECKO0-NEXT: // kill: def $q1 killed $h1 +; CHECKO0-NEXT: // kill: def $s1 killed $s1 killed $q1 +; CHECKO0-NEXT: fmov w0, s1 +; CHECKO0-NEXT: mov w10, wzr +; CHECKO0-NEXT: mov w9, w0 +; CHECKO0-NEXT: mov w8, w10 +; CHECKO0-NEXT: bfi x9, x8, #32, #32 +; CHECKO0-NEXT: cnt v0.16b, v0.16b +; CHECKO0-NEXT: uaddlv h0, v0.16b +; CHECKO0-NEXT: // kill: def $q0 killed $h0 +; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q0 +; CHECKO0-NEXT: fmov w0, s0 +; CHECKO0-NEXT: mov w8, w0 +; CHECKO0-NEXT: // kill: def $x10 killed $w10 +; CHECKO0-NEXT: bfi x8, x10, #32, #32 +; CHECKO0-NEXT: adds x8, x8, x9 +; CHECKO0-NEXT: mov w0, w8 +; CHECKO0-NEXT: ret ; -; DOT-LABEL: popcount256: -; DOT: // %bb.0: // %Entry -; DOT-NEXT: ldr d0, [x0, #16] -; DOT-NEXT: ldr d1, [x0] -; DOT-NEXT: add x8, x0, #8 -; DOT-NEXT: add x9, x0, #24 -; DOT-NEXT: ld1 { v0.d }[1], [x9] -; DOT-NEXT: ld1 { v1.d }[1], [x8] -; DOT-NEXT: cnt v0.16b, v0.16b -; DOT-NEXT: cnt v1.16b, v1.16b -; DOT-NEXT: uaddlv h0, v0.16b -; DOT-NEXT: uaddlv h1, v1.16b -; DOT-NEXT: fmov w8, s0 -; DOT-NEXT: fmov w9, s1 -; DOT-NEXT: add w0, w9, w8 -; DOT-NEXT: ret -; -; SVE-LABEL: popcount256: -; SVE: // %bb.0: // %Entry -; SVE-NEXT: ldr d0, [x0, #16] -; SVE-NEXT: ldr d1, [x0] -; SVE-NEXT: add x8, x0, #8 -; SVE-NEXT: add x9, x0, #24 -; SVE-NEXT: ld1 { v0.d }[1], [x9] -; SVE-NEXT: ld1 { v1.d }[1], [x8] -; SVE-NEXT: cnt v0.16b, v0.16b -; SVE-NEXT: cnt v1.16b, v1.16b -; SVE-NEXT: uaddlv h0, v0.16b -; SVE-NEXT: uaddlv h1, v1.16b -; SVE-NEXT: fmov w8, s0 -; SVE-NEXT: fmov w9, s1 -; SVE-NEXT: add w0, w9, w8 -; SVE-NEXT: ret +; CHECK-LABEL: popcount256: +; CHECK: // %bb.0: // %Entry +; CHECK-NEXT: ldr d0, [x0, #16] +; CHECK-NEXT: ldr d1, [x0] +; CHECK-NEXT: add x8, x0, #8 +; CHECK-NEXT: add x9, x0, #24 +; CHECK-NEXT: ld1 { v0.d }[1], [x9] +; CHECK-NEXT: ld1 { v1.d }[1], [x8] +; CHECK-NEXT: cnt v0.16b, v0.16b +; CHECK-NEXT: cnt v1.16b, v1.16b +; CHECK-NEXT: uaddlv h0, v0.16b +; CHECK-NEXT: uaddlv h1, v1.16b +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: fmov w9, s1 +; CHECK-NEXT: add w0, w9, w8 +; CHECK-NEXT: ret Entry: %1 = load i256, ptr %0, align 16 %2 = tail call i256 @llvm.ctpop.i256(i256 %1) @@ -108,41 +97,34 @@ Entry: declare i256 @llvm.ctpop.i256(i256) define <1 x i128> @popcount1x128(<1 x i128> %0) { -; NEON-LABEL: popcount1x128: -; NEON: // %bb.0: // %Entry -; NEON-NEXT: fmov d1, x0 -; NEON-NEXT: movi v0.2d, #0000000000000000 -; NEON-NEXT: mov v1.d[1], x1 -; NEON-NEXT: cnt v1.16b, v1.16b -; NEON-NEXT: uaddlv h1, v1.16b -; NEON-NEXT: mov v0.s[0], v1.s[0] -; NEON-NEXT: mov x1, v0.d[1] -; NEON-NEXT: fmov x0, d0 -; NEON-NEXT: ret -; -; DOT-LABEL: popcount1x128: -; DOT: // %bb.0: // %Entry -; DOT-NEXT: fmov d1, x0 -; DOT-NEXT: movi v0.2d, #0000000000000000 -; DOT-NEXT: mov v1.d[1], x1 -; DOT-NEXT: cnt v1.16b, v1.16b -; DOT-NEXT: uaddlv h1, v1.16b -; DOT-NEXT: mov v0.s[0], v1.s[0] -; DOT-NEXT: mov x1, v0.d[1] -; DOT-NEXT: fmov x0, d0 -; DOT-NEXT: ret +; CHECKO0-LABEL: popcount1x128: +; CHECKO0: // %bb.0: // %Entry +; CHECKO0-NEXT: // implicit-def: $q0 +; CHECKO0-NEXT: mov v0.d[0], x0 +; CHECKO0-NEXT: mov v0.d[1], x1 +; CHECKO0-NEXT: cnt v0.16b, v0.16b +; CHECKO0-NEXT: uaddlv h0, v0.16b +; CHECKO0-NEXT: // kill: def $q0 killed $h0 +; CHECKO0-NEXT: mov x1, xzr +; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q0 +; CHECKO0-NEXT: fmov w0, s0 +; CHECKO0-NEXT: mov w8, wzr +; CHECKO0-NEXT: // kill: def $x0 killed $w0 +; CHECKO0-NEXT: // kill: def $x8 killed $w8 +; CHECKO0-NEXT: bfi x0, x8, #32, #32 +; CHECKO0-NEXT: ret ; -; SVE-LABEL: popcount1x128: -; SVE: // %bb.0: // %Entry -; SVE-NEXT: fmov d1, x0 -; SVE-NEXT: movi v0.2d, #0000000000000000 -; SVE-NEXT: mov v1.d[1], x1 -; SVE-NEXT: cnt v1.16b, v1.16b -; SVE-NEXT: uaddlv h1, v1.16b -; SVE-NEXT: mov v0.s[0], v1.s[0] -; SVE-NEXT: mov x1, v0.d[1] -; SVE-NEXT: fmov x0, d0 -; SVE-NEXT: ret +; CHECK-LABEL: popcount1x128: +; CHECK: // %bb.0: // %Entry +; CHECK-NEXT: fmov d1, x0 +; CHECK-NEXT: movi v0.2d, #0000000000000000 +; CHECK-NEXT: mov v1.d[1], x1 +; CHECK-NEXT: cnt v1.16b, v1.16b +; CHECK-NEXT: uaddlv h1, v1.16b +; CHECK-NEXT: mov v0.s[0], v1.s[0] +; CHECK-NEXT: mov x1, v0.d[1] +; CHECK-NEXT: fmov x0, d0 +; CHECK-NEXT: ret Entry: %1 = tail call <1 x i128> @llvm.ctpop.v1i128(<1 x i128> %0) ret <1 x i128> %1 @@ -151,6 +133,14 @@ Entry: declare <1 x i128> @llvm.ctpop.v1i128(<1 x i128>) define <2 x i64> @popcount2x64(<2 x i64> %0) { +; CHECKO0-LABEL: popcount2x64: +; CHECKO0: // %bb.0: // %Entry +; CHECKO0-NEXT: cnt v0.16b, v0.16b +; CHECKO0-NEXT: uaddlp v0.8h, v0.16b +; CHECKO0-NEXT: uaddlp v0.4s, v0.8h +; CHECKO0-NEXT: uaddlp v0.2d, v0.4s +; CHECKO0-NEXT: ret +; ; NEON-LABEL: popcount2x64: ; NEON: // %bb.0: // %Entry ; NEON-NEXT: cnt v0.16b, v0.16b @@ -183,6 +173,13 @@ Entry: declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>) define <4 x i32> @popcount4x32(<4 x i32> %0) { +; CHECKO0-LABEL: popcount4x32: +; CHECKO0: // %bb.0: // %Entry +; CHECKO0-NEXT: cnt v0.16b, v0.16b +; CHECKO0-NEXT: uaddlp v0.8h, v0.16b +; CHECKO0-NEXT: uaddlp v0.4s, v0.8h +; CHECKO0-NEXT: ret +; ; NEON-LABEL: popcount4x32: ; NEON: // %bb.0: // %Entry ; NEON-NEXT: cnt v0.16b, v0.16b @@ -212,6 +209,13 @@ Entry: declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>) define <2 x i32> @popcount2x32(<2 x i32> %0) { +; CHECKO0-LABEL: popcount2x32: +; CHECKO0: // %bb.0: // %Entry +; CHECKO0-NEXT: cnt v0.8b, v0.8b +; CHECKO0-NEXT: uaddlp v0.4h, v0.8b +; CHECKO0-NEXT: uaddlp v0.2s, v0.4h +; CHECKO0-NEXT: ret +; ; NEON-LABEL: popcount2x32: ; NEON: // %bb.0: // %Entry ; NEON-NEXT: cnt v0.8b, v0.8b @@ -242,6 +246,12 @@ Entry: declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>) define <8 x i16> @popcount8x16(<8 x i16> %0) { +; CHECKO0-LABEL: popcount8x16: +; CHECKO0: // %bb.0: // %Entry +; CHECKO0-NEXT: cnt v0.16b, v0.16b +; CHECKO0-NEXT: uaddlp v0.8h, v0.16b +; CHECKO0-NEXT: ret +; ; CHECK-LABEL: popcount8x16: ; CHECK: // %bb.0: // %Entry ; CHECK-NEXT: cnt v0.16b, v0.16b @@ -255,6 +265,12 @@ Entry: declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>) define <4 x i16> @popcount4x16(<4 x i16> %0) { +; CHECKO0-LABEL: popcount4x16: +; CHECKO0: // %bb.0: // %Entry +; CHECKO0-NEXT: cnt v0.8b, v0.8b +; CHECKO0-NEXT: uaddlp v0.4h, v0.8b +; CHECKO0-NEXT: ret +; ; CHECK-LABEL: popcount4x16: ; CHECK: // %bb.0: // %Entry ; CHECK-NEXT: cnt v0.8b, v0.8b From 6fc26a5340655754399309908fe2ecf1b1a1e479 Mon Sep 17 00:00:00 2001 From: Tim Gymnich Date: Wed, 19 Jun 2024 01:32:18 +0200 Subject: [PATCH 9/9] add v1i64 test --- llvm/test/CodeGen/AArch64/popcount.ll | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/llvm/test/CodeGen/AArch64/popcount.ll b/llvm/test/CodeGen/AArch64/popcount.ll index 9a0b2299e8d244..f60d0db43c4a96 100644 --- a/llvm/test/CodeGen/AArch64/popcount.ll +++ b/llvm/test/CodeGen/AArch64/popcount.ll @@ -172,6 +172,33 @@ Entry: declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>) +define <1 x i64> @popcount1x64(<1 x i64> %0) { +; CHECKO0-LABEL: popcount1x64: +; CHECKO0: // %bb.0: // %Entry +; CHECKO0-NEXT: fmov x0, d0 +; CHECKO0-NEXT: fmov d0, x0 +; CHECKO0-NEXT: cnt v0.8b, v0.8b +; CHECKO0-NEXT: uaddlv h0, v0.8b +; CHECKO0-NEXT: // kill: def $q0 killed $h0 +; CHECKO0-NEXT: mov w8, v0.s[0] +; CHECKO0-NEXT: // kill: def $x8 killed $w8 +; CHECKO0-NEXT: fmov d0, x8 +; CHECKO0-NEXT: ret +; +; CHECK-LABEL: popcount1x64: +; CHECK: // %bb.0: // %Entry +; CHECK-NEXT: cnt v0.8b, v0.8b +; CHECK-NEXT: uaddlp v0.4h, v0.8b +; CHECK-NEXT: uaddlp v0.2s, v0.4h +; CHECK-NEXT: uaddlp v0.1d, v0.2s +; CHECK-NEXT: ret +Entry: + %1 = tail call <1 x i64> @llvm.ctpop.v1i64(<1 x i64> %0) + ret <1 x i64> %1 +} + +declare <1 x i64> @llvm.ctpop.v1i64(<1 x i64>) + define <4 x i32> @popcount4x32(<4 x i32> %0) { ; CHECKO0-LABEL: popcount4x32: ; CHECKO0: // %bb.0: // %Entry