From 4db93e5d56440be44d8ad2036e2579208c82ad37 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 20 Feb 2024 14:03:25 +0000 Subject: [PATCH] [IndVars] Recompute flags if needed in widenIVUse of IV increment. (#82352) widenIVUse may hoist a wide induction increment and introduce new uses, but does not recompute the wrap flags. In some cases this can make the new uses of the wide IV inc more poisonous. Update the code to recompute flags if needed when hoisting an IV. If both the narrow and wide IV increment's flags match and we can re-use the flags from the increments, there's no need to recompute the flags, as the replacement won't make the new uses of the wide IV's increment more poisonous. Note that this also updates a stale comment which claimed that the widen increment is only used if it dominates the new use. The helper should also be used to guard the code added in da437330be, which I am planning on doing separately once the helper lands. Fixes https://github.com/llvm/llvm-project/issues/82243. --- .../Utils/ScalarEvolutionExpander.h | 8 +++++ .../Utils/ScalarEvolutionExpander.cpp | 9 ++++++ llvm/lib/Transforms/Utils/SimplifyIndVar.cpp | 30 ++++++++++++------- ...wide-inc-for-narrow-use-recompute-flags.ll | 4 +-- 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h index 035705b7f4b78b..fa10443f14bb74 100644 --- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h +++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h @@ -258,6 +258,14 @@ class SCEVExpander : public SCEVVisitor { bool hoistIVInc(Instruction *IncV, Instruction *InsertPos, bool RecomputePoisonFlags = false); + /// Return true if both increments directly increment the corresponding IV PHI + /// nodes and have the same opcode. It is not safe to re-use the flags from + /// the original increment, if it is more complex and SCEV expansion may have + /// yielded a more simplified wider increment. + static bool canReuseFlagsFromOriginalIVInc(PHINode *OrigPhi, PHINode *WidePhi, + Instruction *OrigInc, + Instruction *WideInc); + /// replace congruent phis with their most canonical representative. Return /// the number of phis eliminated. unsigned replaceCongruentIVs(Loop *L, const DominatorTree *DT, diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp index 3a28909473d95f..fbe1dba5b8d4e4 100644 --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -771,6 +771,15 @@ bool SCEVExpander::hoistIVInc(Instruction *IncV, Instruction *InsertPos, return true; } +bool SCEVExpander::canReuseFlagsFromOriginalIVInc(PHINode *OrigPhi, + PHINode *WidePhi, + Instruction *OrigInc, + Instruction *WideInc) { + return match(OrigInc, m_c_BinOp(m_Specific(OrigPhi), m_Value())) && + match(WideInc, m_c_BinOp(m_Specific(WidePhi), m_Value())) && + OrigInc->getOpcode() == WideInc->getOpcode(); +} + /// Determine if this cyclic phi is in a form that would have been generated by /// LSR. We don't care if the phi was actually expanded in this pass, as long /// as it is in a low-cost form, for example, no implied multiplication. This diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp index 5aa6df49e7f676..66bba1ca2f1d7c 100644 --- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp @@ -1131,7 +1131,8 @@ class WidenIV { const SCEV *getSCEVByOpCode(const SCEV *LHS, const SCEV *RHS, unsigned OpCode) const; - Instruction *widenIVUse(NarrowIVDefUse DU, SCEVExpander &Rewriter); + Instruction *widenIVUse(NarrowIVDefUse DU, SCEVExpander &Rewriter, + PHINode *OrigPhi, PHINode *WidePhi); bool widenLoopCompare(NarrowIVDefUse DU); bool widenWithVariantUse(NarrowIVDefUse DU); @@ -1731,7 +1732,9 @@ bool WidenIV::widenWithVariantUse(WidenIV::NarrowIVDefUse DU) { /// Determine whether an individual user of the narrow IV can be widened. If so, /// return the wide clone of the user. -Instruction *WidenIV::widenIVUse(WidenIV::NarrowIVDefUse DU, SCEVExpander &Rewriter) { +Instruction *WidenIV::widenIVUse(WidenIV::NarrowIVDefUse DU, + SCEVExpander &Rewriter, PHINode *OrigPhi, + PHINode *WidePhi) { assert(ExtendKindMap.count(DU.NarrowDef) && "Should already know the kind of extension used to widen NarrowDef"); @@ -1825,11 +1828,18 @@ Instruction *WidenIV::widenIVUse(WidenIV::NarrowIVDefUse DU, SCEVExpander &Rewri if (!WideAddRec.first) return nullptr; - // Reuse the IV increment that SCEVExpander created as long as it dominates - // NarrowUse. + // Reuse the IV increment that SCEVExpander created. Recompute flags, unless + // the flags for both increments agree and it is safe to use the ones from + // the original inc. In that case, the new use of the wide increment won't + // be more poisonous. + bool NeedToRecomputeFlags = + !SCEVExpander::canReuseFlagsFromOriginalIVInc(OrigPhi, WidePhi, + DU.NarrowUse, WideInc) || + DU.NarrowUse->hasNoUnsignedWrap() != WideInc->hasNoUnsignedWrap() || + DU.NarrowUse->hasNoSignedWrap() != WideInc->hasNoSignedWrap(); Instruction *WideUse = nullptr; if (WideAddRec.first == WideIncExpr && - Rewriter.hoistIVInc(WideInc, DU.NarrowUse)) + Rewriter.hoistIVInc(WideInc, DU.NarrowUse, NeedToRecomputeFlags)) WideUse = WideInc; else { WideUse = cloneIVUser(DU, WideAddRec.first); @@ -1996,11 +2006,9 @@ PHINode *WidenIV::createWideIV(SCEVExpander &Rewriter) { // the same opcode. It is not safe to re-use the flags from the original // increment, if it is more complex and SCEV expansion may have yielded a // more simplified wider increment. - bool MatchingOps = - match(OrigInc, m_c_BinOp(m_Specific(OrigPhi), m_Value())) && - match(WideInc, m_c_BinOp(m_Specific(WidePhi), m_Value())) && - OrigInc->getOpcode() == WideInc->getOpcode(); - if (MatchingOps && isa(OrigInc) && + if (SCEVExpander::canReuseFlagsFromOriginalIVInc(OrigPhi, WidePhi, + OrigInc, WideInc) && + isa(OrigInc) && isa(WideInc)) { WideInc->setHasNoUnsignedWrap(WideInc->hasNoUnsignedWrap() || OrigInc->hasNoUnsignedWrap()); @@ -2024,7 +2032,7 @@ PHINode *WidenIV::createWideIV(SCEVExpander &Rewriter) { // Process a def-use edge. This may replace the use, so don't hold a // use_iterator across it. - Instruction *WideUse = widenIVUse(DU, Rewriter); + Instruction *WideUse = widenIVUse(DU, Rewriter, OrigPhi, WidePhi); // Follow all def-use edges from the previous narrow use. if (WideUse) diff --git a/llvm/test/Transforms/IndVarSimplify/hoist-wide-inc-for-narrow-use-recompute-flags.ll b/llvm/test/Transforms/IndVarSimplify/hoist-wide-inc-for-narrow-use-recompute-flags.ll index ddb3890f3bb360..cc99ee312ccb7f 100644 --- a/llvm/test/Transforms/IndVarSimplify/hoist-wide-inc-for-narrow-use-recompute-flags.ll +++ b/llvm/test/Transforms/IndVarSimplify/hoist-wide-inc-for-narrow-use-recompute-flags.ll @@ -4,7 +4,7 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" ; Test for https://github.com/llvm/llvm-project/issues/82243. -; FIXME: Currently no-wrap flags on hoisted wide IV are not dropped properly. +; Check that NUW flag on hoisted wide IV is dropped properly. define void @test_pr82243(ptr %f) { ; CHECK-LABEL: define void @test_pr82243( ; CHECK-SAME: ptr [[F:%.*]]) { @@ -14,7 +14,7 @@ define void @test_pr82243(ptr %f) { ; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ], [ 1, [[ENTRY:%.*]] ] ; CHECK-NEXT: [[GEP_IV_EXT:%.*]] = getelementptr i32, ptr [[F]], i64 [[INDVARS_IV]] ; CHECK-NEXT: store i32 1, ptr [[GEP_IV_EXT]], align 4 -; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], -1 +; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1 ; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 ; CHECK-NEXT: [[SHL:%.*]] = shl i32 123, [[TMP0]] ; CHECK-NEXT: [[GEP_SHL:%.*]] = getelementptr i32, ptr [[F]], i32 [[SHL]]