Reapply "[LICM] Fold associative binary ops to promote code hoisting (#81608)" #100377

rj-jesus · 2024-07-24T13:49:38Z

This reverts commit b48819d.

This reapplies a more strict version of f2ccf80.

Perform the transformation

"(LV op C1) op C2" ==> "LV op (C1 op C2)"

where op is an associative binary op, LV is a loop variant, and C1 and C2 are loop invariants, and hoist (C1 op C2) into the preheader.

For now this fold is restricted to ADDs.

Perform the transformation "(LV op C1) op C2" ==> "LV op (C1 op C2)" where op is an associative binary op, LV is a loop variant, and C1 and C2 are loop invariants, and hoist (C1 op C2) into the preheader. For now this fold is restricted to ADDs.

llvmbot · 2024-07-24T13:50:14Z

@llvm/pr-subscribers-backend-powerpc

@llvm/pr-subscribers-llvm-transforms

Author: Ricardo Jesus (rj-jesus)

Changes

This reverts commit b48819d.

This reapplies a more strict version of f2ccf80.

Perform the transformation

"(LV op C1) op C2" ==> "LV op (C1 op C2)"

where op is an associative binary op, LV is a loop variant, and C1 and C2 are loop invariants, and hoist (C1 op C2) into the preheader.

For now this fold is restricted to ADDs.

Patch is 27.28 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/100377.diff

4 Files Affected:

(modified) llvm/lib/Transforms/Scalar/LICM.cpp (+74)
(modified) llvm/test/CodeGen/PowerPC/common-chain.ll (+160-155)
(added) llvm/test/Transforms/LICM/hoist-binop.ll (+229)
(modified) llvm/test/Transforms/LICM/sink-foldable.ll (+3-2)

diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index 91ef2b4b7c183..fe29fc36e2bb2 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -113,6 +113,8 @@ STATISTIC(NumFPAssociationsHoisted, "Number of invariant FP expressions "
 STATISTIC(NumIntAssociationsHoisted,
           "Number of invariant int expressions "
           "reassociated and hoisted out of the loop");
+STATISTIC(NumBOAssociationsHoisted, "Number of invariant BinaryOp expressions "
+                                    "reassociated and hoisted out of the loop");
 
 /// Memory promotion is enabled by default.
 static cl::opt<bool>
@@ -2779,6 +2781,72 @@ static bool hoistMulAddAssociation(Instruction &I, Loop &L,
   return true;
 }
 
+/// Reassociate associative binary expressions of the form
+///
+/// 1. "(LV op C1) op C2" ==> "LV op (C1 op C2)"
+///
+/// where op is an associative binary op, LV is a loop variant, and C1 and C2
+/// are loop invariants that we want to hoist.
+///
+/// TODO: This can be extended to more cases such as
+/// 2. "C1 op (C2 op LV)" ==> "(C1 op C2) op LV"
+/// 3. "(C1 op LV) op C2" ==> "LV op (C1 op C2)" if op is commutative
+/// 4. "C1 op (LV op C2)" ==> "(C1 op C2) op LV" if op is commutative
+static bool hoistBOAssociation(Instruction &I, Loop &L,
+                               ICFLoopSafetyInfo &SafetyInfo,
+                               MemorySSAUpdater &MSSAU, AssumptionCache *AC,
+                               DominatorTree *DT) {
+  BinaryOperator *BO = dyn_cast<BinaryOperator>(&I);
+  if (!BO || !BO->isAssociative())
+    return false;
+
+  // Only fold ADDs for now.
+  Instruction::BinaryOps Opcode = BO->getOpcode();
+  if (Opcode != Instruction::Add)
+    return false;
+
+  BinaryOperator *BO0 = dyn_cast<BinaryOperator>(BO->getOperand(0));
+
+  // Transform: "(LV op C1) op C2" ==> "LV op (C1 op C2)"
+  if (BO0 && BO0->getOpcode() == Opcode && BO0->isAssociative()) {
+    Value *LV = BO0->getOperand(0);
+    Value *C1 = BO0->getOperand(1);
+    Value *C2 = BO->getOperand(1);
+
+    if (L.isLoopInvariant(LV) || !L.isLoopInvariant(C1) ||
+        !L.isLoopInvariant(C2))
+      return false;
+
+    auto *Preheader = L.getLoopPreheader();
+    assert(Preheader && "Loop is not in simplify form?");
+
+    auto *Inv = BinaryOperator::Create(Opcode, C1, C2, "invariant.op",
+                                       Preheader->getTerminator());
+    auto *NewBO = BinaryOperator::Create(Opcode, LV, Inv,
+                                         BO->getName() + ".reass", BO);
+
+    // Copy NUW for ADDs if both instructions have it.
+    // https://alive2.llvm.org/ce/z/K9W3rk
+    if (Opcode == Instruction::Add && BO->hasNoUnsignedWrap() &&
+        BO0->hasNoUnsignedWrap()) {
+      Inv->setHasNoUnsignedWrap(true);
+      NewBO->setHasNoUnsignedWrap(true);
+    }
+
+    BO->replaceAllUsesWith(NewBO);
+    eraseInstruction(*BO, SafetyInfo, MSSAU);
+
+    // Note: (LV op C1) might not be erased if it has more uses than the one we
+    //       just replaced.
+    if (BO0->use_empty())
+      eraseInstruction(*BO0, SafetyInfo, MSSAU);
+
+    return true;
+  }
+
+  return false;
+}
+
 static bool hoistArithmetics(Instruction &I, Loop &L,
                              ICFLoopSafetyInfo &SafetyInfo,
                              MemorySSAUpdater &MSSAU, AssumptionCache *AC,
@@ -2816,6 +2884,12 @@ static bool hoistArithmetics(Instruction &I, Loop &L,
     return true;
   }
 
+  if (hoistBOAssociation(I, L, SafetyInfo, MSSAU, AC, DT)) {
+    ++NumHoisted;
+    ++NumBOAssociationsHoisted;
+    return true;
+  }
+
   return false;
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/common-chain.ll b/llvm/test/CodeGen/PowerPC/common-chain.ll
index 5f8c21e30f8fd..ccf0e4520f468 100644
--- a/llvm/test/CodeGen/PowerPC/common-chain.ll
+++ b/llvm/test/CodeGen/PowerPC/common-chain.ll
@@ -642,8 +642,8 @@ define i64 @two_chain_two_bases_succ(ptr %p, i64 %offset, i64 %base1, i64 %base2
 ; CHECK-NEXT:    cmpdi r7, 0
 ; CHECK-NEXT:    ble cr0, .LBB6_4
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
-; CHECK-NEXT:    add r6, r6, r4
 ; CHECK-NEXT:    add r5, r5, r4
+; CHECK-NEXT:    add r6, r6, r4
 ; CHECK-NEXT:    mtctr r7
 ; CHECK-NEXT:    sldi r4, r4, 1
 ; CHECK-NEXT:    add r5, r3, r5
@@ -743,214 +743,219 @@ define signext i32 @spill_reduce_succ(ptr %input1, ptr %input2, ptr %output, i64
 ; CHECK-NEXT:    std r9, -184(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r8, -176(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r7, -168(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    std r3, -160(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r4, -160(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    ble cr0, .LBB7_7
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
-; CHECK-NEXT:    sldi r6, r6, 2
-; CHECK-NEXT:    li r7, 1
-; CHECK-NEXT:    mr r30, r10
-; CHECK-NEXT:    cmpdi r6, 1
-; CHECK-NEXT:    iselgt r7, r6, r7
-; CHECK-NEXT:    addi r8, r7, -1
-; CHECK-NEXT:    clrldi r6, r7, 63
-; CHECK-NEXT:    cmpldi r8, 3
+; CHECK-NEXT:    sldi r4, r6, 2
+; CHECK-NEXT:    li r6, 1
+; CHECK-NEXT:    mr r0, r10
+; CHECK-NEXT:    std r10, -192(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    cmpdi r4, 1
+; CHECK-NEXT:    iselgt r4, r4, r6
+; CHECK-NEXT:    addi r7, r4, -1
+; CHECK-NEXT:    clrldi r6, r4, 63
+; CHECK-NEXT:    cmpldi r7, 3
 ; CHECK-NEXT:    blt cr0, .LBB7_4
 ; CHECK-NEXT:  # %bb.2: # %for.body.preheader.new
-; CHECK-NEXT:    ld r14, -168(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    mulli r24, r30, 24
-; CHECK-NEXT:    ld r16, -184(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld r15, -176(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld r3, -160(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    rldicl r0, r7, 62, 2
-; CHECK-NEXT:    sldi r11, r30, 5
-; CHECK-NEXT:    sldi r19, r30, 4
-; CHECK-NEXT:    sldi r7, r14, 3
-; CHECK-NEXT:    add r14, r30, r14
-; CHECK-NEXT:    sldi r10, r16, 3
-; CHECK-NEXT:    sldi r12, r15, 3
-; CHECK-NEXT:    add r16, r30, r16
-; CHECK-NEXT:    add r15, r30, r15
-; CHECK-NEXT:    add r27, r11, r7
-; CHECK-NEXT:    add r22, r24, r7
-; CHECK-NEXT:    add r17, r19, r7
-; CHECK-NEXT:    sldi r2, r14, 3
-; CHECK-NEXT:    add r26, r24, r10
-; CHECK-NEXT:    add r25, r24, r12
-; CHECK-NEXT:    add r21, r19, r10
-; CHECK-NEXT:    add r20, r19, r12
-; CHECK-NEXT:    add r8, r11, r10
-; CHECK-NEXT:    sldi r16, r16, 3
-; CHECK-NEXT:    add r29, r5, r27
-; CHECK-NEXT:    add r28, r4, r27
-; CHECK-NEXT:    add r27, r3, r27
-; CHECK-NEXT:    add r24, r5, r22
-; CHECK-NEXT:    add r23, r4, r22
-; CHECK-NEXT:    add r22, r3, r22
-; CHECK-NEXT:    add r19, r5, r17
-; CHECK-NEXT:    add r18, r4, r17
-; CHECK-NEXT:    add r17, r3, r17
-; CHECK-NEXT:    add r14, r5, r2
-; CHECK-NEXT:    add r31, r4, r2
-; CHECK-NEXT:    add r2, r3, r2
-; CHECK-NEXT:    add r9, r5, r8
-; CHECK-NEXT:    add r8, r11, r12
+; CHECK-NEXT:    ld r0, -192(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r30, -184(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r8, -176(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    rldicl r7, r4, 62, 2
+; CHECK-NEXT:    ld r9, -168(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    add r11, r0, r30
+; CHECK-NEXT:    add r4, r0, r0
+; CHECK-NEXT:    mulli r23, r0, 24
+; CHECK-NEXT:    add r14, r0, r8
+; CHECK-NEXT:    sldi r12, r0, 5
+; CHECK-NEXT:    add r31, r0, r9
+; CHECK-NEXT:    sldi r9, r9, 3
+; CHECK-NEXT:    sldi r18, r0, 4
+; CHECK-NEXT:    sldi r8, r8, 3
+; CHECK-NEXT:    add r10, r4, r4
+; CHECK-NEXT:    sldi r4, r30, 3
+; CHECK-NEXT:    sldi r11, r11, 3
+; CHECK-NEXT:    add r26, r12, r9
+; CHECK-NEXT:    add r16, r18, r9
+; CHECK-NEXT:    add r29, r12, r8
+; CHECK-NEXT:    add r19, r18, r8
+; CHECK-NEXT:    add r30, r12, r4
+; CHECK-NEXT:    mr r20, r4
+; CHECK-NEXT:    std r4, -200(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    ld r4, -160(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    add r15, r5, r11
+; CHECK-NEXT:    sldi r11, r14, 3
+; CHECK-NEXT:    add r29, r5, r29
+; CHECK-NEXT:    add r28, r3, r26
+; CHECK-NEXT:    add r19, r5, r19
+; CHECK-NEXT:    add r21, r23, r9
+; CHECK-NEXT:    add r24, r23, r8
+; CHECK-NEXT:    add r14, r5, r11
+; CHECK-NEXT:    sldi r11, r31, 3
+; CHECK-NEXT:    add r25, r23, r20
+; CHECK-NEXT:    add r20, r18, r20
+; CHECK-NEXT:    add r30, r5, r30
+; CHECK-NEXT:    add r18, r3, r16
+; CHECK-NEXT:    add r24, r5, r24
+; CHECK-NEXT:    add r23, r3, r21
+; CHECK-NEXT:    add r27, r4, r26
+; CHECK-NEXT:    add r22, r4, r21
+; CHECK-NEXT:    add r17, r4, r16
+; CHECK-NEXT:    add r2, r4, r11
+; CHECK-NEXT:    rldicl r4, r7, 2, 1
+; CHECK-NEXT:    sub r7, r8, r9
+; CHECK-NEXT:    ld r8, -200(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    add r26, r5, r26
 ; CHECK-NEXT:    add r25, r5, r25
 ; CHECK-NEXT:    add r21, r5, r21
 ; CHECK-NEXT:    add r20, r5, r20
 ; CHECK-NEXT:    add r16, r5, r16
-; CHECK-NEXT:    add r8, r5, r8
-; CHECK-NEXT:    rldicl r3, r0, 2, 1
-; CHECK-NEXT:    addi r3, r3, -4
-; CHECK-NEXT:    sub r0, r12, r7
-; CHECK-NEXT:    sub r12, r10, r7
-; CHECK-NEXT:    li r7, 0
-; CHECK-NEXT:    mr r10, r30
-; CHECK-NEXT:    sldi r15, r15, 3
-; CHECK-NEXT:    add r15, r5, r15
-; CHECK-NEXT:    rldicl r3, r3, 62, 2
-; CHECK-NEXT:    addi r3, r3, 1
-; CHECK-NEXT:    mtctr r3
+; CHECK-NEXT:    add r31, r5, r11
+; CHECK-NEXT:    add r11, r3, r11
+; CHECK-NEXT:    addi r4, r4, -4
+; CHECK-NEXT:    rldicl r4, r4, 62, 2
+; CHECK-NEXT:    sub r8, r8, r9
+; CHECK-NEXT:    li r9, 0
+; CHECK-NEXT:    addi r4, r4, 1
+; CHECK-NEXT:    mtctr r4
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB7_3: # %for.body
 ; CHECK-NEXT:    #
-; CHECK-NEXT:    lfd f0, 0(r2)
-; CHECK-NEXT:    lfd f1, 0(r31)
-; CHECK-NEXT:    add r3, r10, r30
-; CHECK-NEXT:    add r3, r3, r30
+; CHECK-NEXT:    lfd f0, 0(r11)
+; CHECK-NEXT:    lfd f1, 0(r2)
+; CHECK-NEXT:    add r0, r0, r10
 ; CHECK-NEXT:    xsmuldp f0, f0, f1
-; CHECK-NEXT:    lfd f1, 0(r14)
-; CHECK-NEXT:    add r3, r3, r30
-; CHECK-NEXT:    add r10, r3, r30
+; CHECK-NEXT:    lfd f1, 0(r31)
 ; CHECK-NEXT:    xsadddp f0, f1, f0
-; CHECK-NEXT:    stfd f0, 0(r14)
-; CHECK-NEXT:    add r14, r14, r11
-; CHECK-NEXT:    lfdx f0, r2, r0
-; CHECK-NEXT:    lfdx f1, r31, r0
+; CHECK-NEXT:    stfd f0, 0(r31)
+; CHECK-NEXT:    add r31, r31, r12
+; CHECK-NEXT:    lfdx f0, r11, r7
+; CHECK-NEXT:    lfdx f1, r2, r7
 ; CHECK-NEXT:    xsmuldp f0, f0, f1
-; CHECK-NEXT:    lfdx f1, r15, r7
+; CHECK-NEXT:    lfdx f1, r14, r9
 ; CHECK-NEXT:    xsadddp f0, f1, f0
-; CHECK-NEXT:    stfdx f0, r15, r7
-; CHECK-NEXT:    lfdx f0, r2, r12
-; CHECK-NEXT:    lfdx f1, r31, r12
-; CHECK-NEXT:    add r2, r2, r11
-; CHECK-NEXT:    add r31, r31, r11
+; CHECK-NEXT:    stfdx f0, r14, r9
+; CHECK-NEXT:    lfdx f0, r11, r8
+; CHECK-NEXT:    lfdx f1, r2, r8
+; CHECK-NEXT:    add r11, r11, r12
+; CHECK-NEXT:    add r2, r2, r12
 ; CHECK-NEXT:    xsmuldp f0, f0, f1
-; CHECK-NEXT:    lfdx f1, r16, r7
+; CHECK-NEXT:    lfdx f1, r15, r9
 ; CHECK-NEXT:    xsadddp f0, f1, f0
-; CHECK-NEXT:    stfdx f0, r16, r7
-; CHECK-NEXT:    lfd f0, 0(r17)
-; CHECK-NEXT:    lfd f1, 0(r18)
+; CHECK-NEXT:    stfdx f0, r15, r9
+; CHECK-NEXT:    lfd f0, 0(r18)
+; CHECK-NEXT:    lfd f1, 0(r17)
 ; CHECK-NEXT:    xsmuldp f0, f0, f1
-; CHECK-NEXT:    lfdx f1, r19, r7
+; CHECK-NEXT:    lfdx f1, r16, r9
 ; CHECK-NEXT:    xsadddp f0, f1, f0
-; CHECK-NEXT:    stfdx f0, r19, r7
-; CHECK-NEXT:    lfdx f0, r17, r0
-; CHECK-NEXT:    lfdx f1, r18, r0
+; CHECK-NEXT:    stfdx f0, r16, r9
+; CHECK-NEXT:    lfdx f0, r18, r7
+; CHECK-NEXT:    lfdx f1, r17, r7
 ; CHECK-NEXT:    xsmuldp f0, f0, f1
-; CHECK-NEXT:    lfdx f1, r20, r7
+; CHECK-NEXT:    lfdx f1, r19, r9
 ; CHECK-NEXT:    xsadddp f0, f1, f0
-; CHECK-NEXT:    stfdx f0, r20, r7
-; CHECK-NEXT:    lfdx f0, r17, r12
-; CHECK-NEXT:    lfdx f1, r18, r12
-; CHECK-NEXT:    add r17, r17, r11
-; CHECK-NEXT:    add r18, r18, r11
+; CHECK-NEXT:    stfdx f0, r19, r9
+; CHECK-NEXT:    lfdx f0, r18, r8
+; CHECK-NEXT:    lfdx f1, r17, r8
+; CHECK-NEXT:    add r18, r18, r12
+; CHECK-NEXT:    add r17, r17, r12
 ; CHECK-NEXT:    xsmuldp f0, f0, f1
-; CHECK-NEXT:    lfdx f1, r21, r7
+; CHECK-NEXT:    lfdx f1, r20, r9
 ; CHECK-NEXT:    xsadddp f0, f1, f0
-; CHECK-NEXT:    stfdx f0, r21, r7
-; CHECK-NEXT:    lfd f0, 0(r22)
-; CHECK-NEXT:    lfd f1, 0(r23)
+; CHECK-NEXT:    stfdx f0, r20, r9
+; CHECK-NEXT:    lfd f0, 0(r23)
+; CHECK-NEXT:    lfd f1, 0(r22)
 ; CHECK-NEXT:    xsmuldp f0, f0, f1
-; CHECK-NEXT:    lfdx f1, r24, r7
+; CHECK-NEXT:    lfdx f1, r21, r9
 ; CHECK-NEXT:    xsadddp f0, f1, f0
-; CHECK-NEXT:    stfdx f0, r24, r7
-; CHECK-NEXT:    lfdx f0, r22, r0
-; CHECK-NEXT:    lfdx f1, r23, r0
+; CHECK-NEXT:    stfdx f0, r21, r9
+; CHECK-NEXT:    lfdx f0, r23, r7
+; CHECK-NEXT:    lfdx f1, r22, r7
 ; CHECK-NEXT:    xsmuldp f0, f0, f1
-; CHECK-NEXT:    lfdx f1, r25, r7
+; CHECK-NEXT:    lfdx f1, r24, r9
 ; CHECK-NEXT:    xsadddp f0, f1, f0
-; CHECK-NEXT:    stfdx f0, r25, r7
-; CHECK-NEXT:    lfdx f0, r22, r12
-; CHECK-NEXT:    lfdx f1, r23, r12
-; CHECK-NEXT:    add r22, r22, r11
-; CHECK-NEXT:    add r23, r23, r11
+; CHECK-NEXT:    stfdx f0, r24, r9
+; CHECK-NEXT:    lfdx f0, r23, r8
+; CHECK-NEXT:    lfdx f1, r22, r8
+; CHECK-NEXT:    add r23, r23, r12
+; CHECK-NEXT:    add r22, r22, r12
 ; CHECK-NEXT:    xsmuldp f0, f0, f1
-; CHECK-NEXT:    lfdx f1, r26, r7
+; CHECK-NEXT:    lfdx f1, r25, r9
 ; CHECK-NEXT:    xsadddp f0, f1, f0
-; CHECK-NEXT:    stfdx f0, r26, r7
-; CHECK-NEXT:    lfd f0, 0(r27)
-; CHECK-NEXT:    lfd f1, 0(r28)
+; CHECK-NEXT:    stfdx f0, r25, r9
+; CHECK-NEXT:    lfd f0, 0(r28)
+; CHECK-NEXT:    lfd f1, 0(r27)
 ; CHECK-NEXT:    xsmuldp f0, f0, f1
-; CHECK-NEXT:    lfdx f1, r29, r7
+; CHECK-NEXT:    lfdx f1, r26, r9
 ; CHECK-NEXT:    xsadddp f0, f1, f0
-; CHECK-NEXT:    stfdx f0, r29, r7
-; CHECK-NEXT:    lfdx f0, r27, r0
-; CHECK-NEXT:    lfdx f1, r28, r0
+; CHECK-NEXT:    stfdx f0, r26, r9
+; CHECK-NEXT:    lfdx f0, r28, r7
+; CHECK-NEXT:    lfdx f1, r27, r7
 ; CHECK-NEXT:    xsmuldp f0, f0, f1
-; CHECK-NEXT:    lfdx f1, r8, r7
+; CHECK-NEXT:    lfdx f1, r29, r9
 ; CHECK-NEXT:    xsadddp f0, f1, f0
-; CHECK-NEXT:    stfdx f0, r8, r7
-; CHECK-NEXT:    lfdx f0, r27, r12
-; CHECK-NEXT:    lfdx f1, r28, r12
-; CHECK-NEXT:    add r27, r27, r11
-; CHECK-NEXT:    add r28, r28, r11
+; CHECK-NEXT:    stfdx f0, r29, r9
+; CHECK-NEXT:    lfdx f0, r28, r8
+; CHECK-NEXT:    lfdx f1, r27, r8
+; CHECK-NEXT:    add r28, r28, r12
+; CHECK-NEXT:    add r27, r27, r12
 ; CHECK-NEXT:    xsmuldp f0, f0, f1
-; CHECK-NEXT:    lfdx f1, r9, r7
+; CHECK-NEXT:    lfdx f1, r30, r9
 ; CHECK-NEXT:    xsadddp f0, f1, f0
-; CHECK-NEXT:    stfdx f0, r9, r7
-; CHECK-NEXT:    add r7, r7, r11
+; CHECK-NEXT:    stfdx f0, r30, r9
+; CHECK-NEXT:    add r9, r9, r12
 ; CHECK-NEXT:    bdnz .LBB7_3
 ; CHECK-NEXT:  .LBB7_4: # %for.cond.cleanup.loopexit.unr-lcssa
+; CHECK-NEXT:    ld r7, -192(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    cmpldi r6, 0
 ; CHECK-NEXT:    beq cr0, .LBB7_7
 ; CHECK-NEXT:  # %bb.5: # %for.body.epil.preheader
-; CHECK-NEXT:    ld r3, -184(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld r0, -160(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    sldi r8, r30, 3
-; CHECK-NEXT:    add r3, r10, r3
-; CHECK-NEXT:    sldi r3, r3, 3
-; CHECK-NEXT:    add r7, r5, r3
-; CHECK-NEXT:    add r9, r4, r3
-; CHECK-NEXT:    add r11, r0, r3
-; CHECK-NEXT:    ld r3, -176(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    add r3, r10, r3
-; CHECK-NEXT:    sldi r3, r3, 3
-; CHECK-NEXT:    add r12, r5, r3
-; CHECK-NEXT:    add r30, r4, r3
-; CHECK-NEXT:    add r29, r0, r3
-; CHECK-NEXT:    ld r3, -168(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    add r3, r10, r3
-; CHECK-NEXT:    li r10, 0
-; CHECK-NEXT:    sldi r3, r3, 3
-; CHECK-NEXT:    add r5, r5, r3
-; CHECK-NEXT:    add r4, r4, r3
-; CHECK-NEXT:    add r3, r0, r3
+; CHECK-NEXT:    ld r4, -184(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r29, -160(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    mr r30, r3
+; CHECK-NEXT:    sldi r7, r7, 3
+; CHECK-NEXT:    add r4, r0, r4
+; CHECK-NEXT:    sldi r4, r4, 3
+; CHECK-NEXT:    add r3, r5, r4
+; CHECK-NEXT:    add r8, r29, r4
+; CHECK-NEXT:    add r9, r30, r4
+; CHECK-NEXT:    ld r4, -176(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    add r4, r0, r4
+; CHECK-NEXT:    sldi r4, r4, 3
+; CHECK-NEXT:    add r10, r5, r4
+; CHECK-NEXT:    add r11, r29, r4
+; CHECK-NEXT:    add r12, r30, r4
+; CHECK-NEXT:    ld r4, -168(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    add r4, r0, r4
+; CHECK-NEXT:    sldi r0, r4, 3
+; CHECK-NEXT:    add r5, r5, r0
+; CHECK-NEXT:    add r4, r29, r0
+; CHECK-NEXT:    add r30, r30, r0
+; CHECK-NEXT:    li r0, 0
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB7_6: # %for.body.epil
 ; CHECK-NEXT:    #
-; CHECK-NEXT:    lfdx f0, r3, r10
-; CHECK-NEXT:    lfdx f1, r4, r10
+; CHECK-NEXT:    lfdx f0, r30, r0
+; CHECK-NEXT:    lfdx f1, r4, r0
 ; CHECK-NEXT:    addi r6, r6, -1
 ; CHECK-NEXT:    cmpldi r6, 0
 ; CHECK-NEXT:    xsmuldp f0, f0, f1
 ; CHECK-NEXT:    lfd f1, 0(r5)
 ; CHECK-NEXT:    xsadddp f0, f1, f0
 ; CHECK-NEXT:    stfd f0, 0(r5)
-; CHECK-NEXT:    add r5, r5, r8
-; CHECK-NEXT:    lfdx f0, r29, r10
-; CHECK-NEXT:    lfdx f1, r30, r10
+; CHECK-NEXT:    add r5, r5, r7
+; CHECK-NEXT:    lfdx f0, r12, r0
+; CHECK-NEXT:    lfdx f1, r11, r0
 ; CHECK-NEXT:    xsmuldp f0, f0, f1
-; CHECK-NEXT:    lfdx f1, r12, r10
+; CHECK-NEXT:    lfdx f1, r10, r0
 ; CHECK-NEXT:    xsadddp f0, f1, f0
-; CHECK-NEXT:    stfdx f0, r12, r10
-; CHECK-NEXT:    lfdx f0, r11, r10
-; CHECK-NEXT:    lfdx f1, r9, r10
+; CHECK-NEXT:    stfdx f0, r10, r0
+; CHECK-NEXT:    lfdx f0, r9, r0
+; CHECK-NEXT:    lfdx f1, r8, r0
 ; CHECK-NEXT:    xsmuldp f0, f0, f1
-; CHECK-NEXT:    lfdx f1, r7, r10
+; CHECK-NEXT:    lfdx f1, r3, r0
 ; CHECK-NEXT:    xsadddp f0, f1, f0
-; CHECK-NEXT:    stfdx f0, r7, r10
-; CHECK-NEXT:    add r10, r10, r8
+; CHECK-NEXT:    stfdx f0, r3, r0
+; CHECK-NEXT:    add r0, r0, r7
 ; CHECK-NEXT:    bne cr0, .LBB7_6
 ; CHECK-NEXT:  .LBB7_7: # %for.cond.cleanup
 ; CHECK-NEXT:    ld r2, -152(r1) # 8-byte Folded Reload
diff --git a/llvm/test/Transforms/LICM/hoist-binop.ll b/llvm/test/Transforms/LICM/hoist-binop.ll
new file mode 100644
index 0000000000000..9cdcc359c61ab
--- /dev/null
+++ b/llvm/test/Transforms/LICM/hoist-binop.ll
@@ -0,0 +1,229 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -passes=licm < %s | FileCheck %s
+
+; Fold ADD and remove old op if unused.
+; https://alive2.llvm.org/ce/z/wAY-Nd
+define void @add_one_use(i64 %c) {
+; CHECK-LABEL: @add_one_use(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = add i64 [[C:%.*]], [[C]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = add i64 [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
+  %step.add = add i64 %index, %c
+  %index.next = add i64 %step.add, %c
+  br label %loop
+}
+
+; Fold ADD and copy NUW if both ops have it.
+; https://alive2.llvm.org/ce/z/wAY-Nd
+define void @add_nuw(i64 %c) {
+; CHECK-LABEL: @add_nuw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = add nuw i64 [[C:%.*]], [[C]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = add nuw i64 [[INDEX]], [[C]]
+; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = add nuw i64 [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
+  %step.add = add nuw i64 %index, %c
+  call void @use(i64 %step.add)
+  %index.next = add nuw i64 %step.add, %c
+  br label %loop
+}
+
+; Fold ADD but don't copy NUW if only one op has it.
+; https://alive2.llvm.org/ce/z/6n95Gf
+define void @add_no_nuw(i64 %c) {
+; CHECK-LABEL: @add_no_nuw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = add i64 [[C:%.*]], [[C]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = add i64 [[INDEX]], [[C]]
+; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT_REASS...
[truncated]

github-actions · 2024-07-24T13:53:23Z

✅ With the latest revision this PR passed the C/C++ code formatter.

nikic

Looks reasonable, some nits.

nikic · 2024-07-25T12:54:21Z

llvm/lib/Transforms/Scalar/LICM.cpp

+                               ICFLoopSafetyInfo &SafetyInfo,
+                               MemorySSAUpdater &MSSAU, AssumptionCache *AC,
+                               DominatorTree *DT) {
+  BinaryOperator *BO = dyn_cast<BinaryOperator>(&I);


Suggested change

BinaryOperator *BO = dyn_cast<BinaryOperator>(&I);

auto *BO = dyn_cast<BinaryOperator>(&I);

nikic · 2024-07-25T12:54:28Z

llvm/lib/Transforms/Scalar/LICM.cpp

+  if (Opcode != Instruction::Add)
+    return false;
+
+  BinaryOperator *BO0 = dyn_cast<BinaryOperator>(BO->getOperand(0));


Suggested change

BinaryOperator *BO0 = dyn_cast<BinaryOperator>(BO->getOperand(0));

auto *BO0 = dyn_cast<BinaryOperator>(BO->getOperand(0));

nikic · 2024-07-25T12:55:54Z

llvm/lib/Transforms/Scalar/LICM.cpp

+                                         BO->getName() + ".reass", BO);
+
+    // Copy NUW for ADDs if both instructions have it.
+    // https://alive2.llvm.org/ce/z/K9W3rk


Better proof would be https://alive2.llvm.org/ce/z/ETuxM-. But we generally do not link alive proofs in source code.

Thanks, I've removed the link.

nikic · 2024-07-25T12:56:35Z

llvm/lib/Transforms/Scalar/LICM.cpp

+    // Note: (LV op C1) might not be erased if it has more uses than the one we
+    //       just replaced.


Suggested change

// Note: (LV op C1) might not be erased if it has more uses than the one we

// just replaced.

// (LV op C1) might not be erased if it has more uses than the one we

// just replaced.

nikic · 2024-07-25T12:56:42Z

llvm/lib/Transforms/Scalar/LICM.cpp

+  BinaryOperator *BO0 = dyn_cast<BinaryOperator>(BO->getOperand(0));
+
+  // Transform: "(LV op C1) op C2" ==> "LV op (C1 op C2)"
+  if (BO0 && BO0->getOpcode() == Opcode && BO0->isAssociative()) {


Early exit.

nikic · 2024-07-25T12:57:30Z

llvm/test/Transforms/LICM/hoist-binop.ll

+
+; Fold ADD and remove old op if unused.
+; https://alive2.llvm.org/ce/z/wAY-Nd
+define void @add_one_use(i64 %c) {


Prefer using two different %c1 and %c2, as the transform isn't limited to equal operands.

nikic · 2024-07-25T12:58:50Z

llvm/test/Transforms/LICM/hoist-binop.ll

+  br label %loop
+}
+
+; Don't fold floating-point ops, even if they are associative.


Maybe clarify here that the transform is valid, we just don't do it yet.

rj-jesus · 2024-07-25T15:19:15Z

@nikic Thanks for the comments, I've made the suggested changes.

nikic

LGTM

rj-jesus · 2024-07-26T09:14:19Z

Thanks!

dtcxzyw · 2024-07-26T13:00:50Z

Hi @rj-jesus @nikic

My CI reported that this patch caused significant performance regressions on SingleSource/Benchmarks/Stanford/Queens (+50% instructions) and SingleSource/Benchmarks/Misc/himenobmtxpa (+20% instructions). Can you have a look?
See also plctlab/llvm-ci#1176.

rj-jesus · 2024-07-26T16:17:02Z

Hi @dtcxzyw

Thanks for reporting this. I can't seem to reproduce the performance regression on SingleSource/Benchmarks/Stanford/Queens but do see a 5% increase in SingleSource/Benchmarks/Misc/himenobmtxpa and some increases in code size.

I think this can be addressed by bailing out early if BO0->getNumUses() > 2. This avoids cases where the intermediate op we are trying to avoid has many uses, and so performing the fold and hoist will likely not be beneficial or lead to a significant increase in code size. This seems to be what's happening with himenobmtxpa.

svs-quic · 2024-08-06T10:04:17Z

llvm/lib/Transforms/Scalar/LICM.cpp

+    NewBO->setHasNoUnsignedWrap(true);
+  }
+
+  BO->replaceAllUsesWith(NewBO);


Hi, do we need to preserve the debug information of the original binary operator by copying it over to the new one?

Something like:
NewBO->setDebugLoc(BO->getDebugLoc());

Hi, I think you're right, thanks. I'll have a look.

rj-jesus requested review from nikic and sjoerdmeijer July 24, 2024 13:49

llvmbot added backend:PowerPC llvm:transforms labels Jul 24, 2024

nikic reviewed Jul 25, 2024

View reviewed changes

Address comments

7c0175c

Fix formatting

b190814

nikic approved these changes Jul 25, 2024

View reviewed changes

rj-jesus merged commit 25da8e5 into llvm:main Jul 26, 2024
7 checks passed

rj-jesus deleted the rjj/licm-boassociation branch July 26, 2024 09:14

rj-jesus mentioned this pull request Aug 6, 2024

[LICM] Prevent fold and hoist of binary ops with over 2 uses #102114

Merged

svs-quic reviewed Aug 6, 2024

View reviewed changes

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Reapply "[LICM] Fold associative binary ops to promote code hoisting (#81608)" #100377

Reapply "[LICM] Fold associative binary ops to promote code hoisting (#81608)" #100377

rj-jesus commented Jul 24, 2024

llvmbot commented Jul 24, 2024 •

edited

Loading

github-actions bot commented Jul 24, 2024 •

edited

Loading

nikic left a comment

nikic Jul 25, 2024

nikic Jul 25, 2024

nikic Jul 25, 2024

rj-jesus Jul 25, 2024

nikic Jul 25, 2024

nikic Jul 25, 2024

nikic Jul 25, 2024

nikic Jul 25, 2024

rj-jesus commented Jul 25, 2024

nikic left a comment

rj-jesus commented Jul 26, 2024

dtcxzyw commented Jul 26, 2024

rj-jesus commented Jul 26, 2024

svs-quic Aug 6, 2024

rj-jesus Aug 7, 2024

	BinaryOperator *BO = dyn_cast<BinaryOperator>(&I);
	auto *BO = dyn_cast<BinaryOperator>(&I);

	BinaryOperator *BO0 = dyn_cast<BinaryOperator>(BO->getOperand(0));
	auto *BO0 = dyn_cast<BinaryOperator>(BO->getOperand(0));

		// Note: (LV op C1) might not be erased if it has more uses than the one we
		// just replaced.

Reapply "[LICM] Fold associative binary ops to promote code hoisting (#81608)" #100377

Reapply "[LICM] Fold associative binary ops to promote code hoisting (#81608)" #100377

Conversation

rj-jesus commented Jul 24, 2024

llvmbot commented Jul 24, 2024 • edited Loading

github-actions bot commented Jul 24, 2024 • edited Loading

nikic left a comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

rj-jesus commented Jul 25, 2024

nikic left a comment

Choose a reason for hiding this comment

rj-jesus commented Jul 26, 2024

dtcxzyw commented Jul 26, 2024

rj-jesus commented Jul 26, 2024

Choose a reason for hiding this comment

Choose a reason for hiding this comment

llvmbot commented Jul 24, 2024 •

edited

Loading

github-actions bot commented Jul 24, 2024 •

edited

Loading