From 2e8d8155969f90b8f17634ce9a8e4541fb21dbab Mon Sep 17 00:00:00 2001 From: Graham Hunter Date: Fri, 10 May 2024 11:22:11 +0100 Subject: [PATCH] [TTI] Support scalable offsets in getScalingFactorCost (#88113) Part of the work to support vscale-relative immediates in LSR. --- llvm/include/llvm/Analysis/TargetTransformInfo.h | 6 +++--- llvm/include/llvm/Analysis/TargetTransformInfoImpl.h | 8 +++++--- llvm/include/llvm/CodeGen/BasicTTIImpl.h | 5 +++-- llvm/lib/Analysis/TargetTransformInfo.cpp | 2 +- llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp | 5 +++-- llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h | 2 +- llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp | 5 +++-- llvm/lib/Target/ARM/ARMTargetTransformInfo.h | 2 +- llvm/lib/Target/X86/X86TargetTransformInfo.cpp | 5 +++-- llvm/lib/Target/X86/X86TargetTransformInfo.h | 2 +- llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp | 6 ++++-- 11 files changed, 28 insertions(+), 20 deletions(-) diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index 1c76821fe5e4ab..f0eb83c143e2c8 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -834,7 +834,7 @@ class TargetTransformInfo { /// If the AM is not supported, it returns a negative value. /// TODO: Handle pre/postinc as well. InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, - int64_t BaseOffset, bool HasBaseReg, + StackOffset BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace = 0) const; @@ -1891,7 +1891,7 @@ class TargetTransformInfo::Concept { virtual bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) = 0; virtual bool prefersVectorizedAddressing() = 0; virtual InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, - int64_t BaseOffset, + StackOffset BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) = 0; virtual bool LSRWithInstrQueries() = 0; @@ -2403,7 +2403,7 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept { return Impl.prefersVectorizedAddressing(); } InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, - int64_t BaseOffset, bool HasBaseReg, + StackOffset BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) override { return Impl.getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg, Scale, diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index 4d5cd963e09267..262ebdb3cbef99 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -32,6 +32,7 @@ class Function; /// Base class for use as a mix-in that aids implementing /// a TargetTransformInfo-compatible class. class TargetTransformInfoImplBase { + protected: typedef TargetTransformInfo TTI; @@ -326,12 +327,13 @@ class TargetTransformInfoImplBase { bool prefersVectorizedAddressing() const { return true; } InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, - int64_t BaseOffset, bool HasBaseReg, + StackOffset BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) const { // Guess that all legal addressing mode are free. - if (isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg, Scale, - AddrSpace)) + if (isLegalAddressingMode(Ty, BaseGV, BaseOffset.getFixed(), HasBaseReg, + Scale, AddrSpace, /*I=*/nullptr, + BaseOffset.getScalable())) return 0; return -1; } diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index bcb60c65629670..fa481886b268a8 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -404,13 +404,14 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { } InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, - int64_t BaseOffset, bool HasBaseReg, + StackOffset BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) { TargetLoweringBase::AddrMode AM; AM.BaseGV = BaseGV; - AM.BaseOffs = BaseOffset; + AM.BaseOffs = BaseOffset.getFixed(); AM.HasBaseReg = HasBaseReg; AM.Scale = Scale; + AM.ScalableOffset = BaseOffset.getScalable(); if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AddrSpace)) return 0; return -1; diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp index 33c899fe889990..00443ace46f745 100644 --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -531,7 +531,7 @@ bool TargetTransformInfo::prefersVectorizedAddressing() const { } InstructionCost TargetTransformInfo::getScalingFactorCost( - Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, + Type *Ty, GlobalValue *BaseGV, StackOffset BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) const { InstructionCost Cost = TTIImpl->getScalingFactorCost( Ty, BaseGV, BaseOffset, HasBaseReg, Scale, AddrSpace); diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 4b410826f4bb1a..f49c73dc795192 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -4183,7 +4183,7 @@ bool AArch64TTIImpl::preferPredicateOverEpilogue(TailFoldingInfo *TFI) { InstructionCost AArch64TTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, - int64_t BaseOffset, bool HasBaseReg, + StackOffset BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) const { // Scaling factors are not free at all. // Operands | Rt Latency @@ -4194,9 +4194,10 @@ AArch64TTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, // Rt, [Xn, Wm, #imm] | TargetLoweringBase::AddrMode AM; AM.BaseGV = BaseGV; - AM.BaseOffs = BaseOffset; + AM.BaseOffs = BaseOffset.getFixed(); AM.HasBaseReg = HasBaseReg; AM.Scale = Scale; + AM.ScalableOffset = BaseOffset.getScalable(); if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AddrSpace)) // Scale represents reg2 * scale, thus account for 1 if // it is not equal to 0 or 1. diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h index 678c132e6a80a1..2f44aaa3e26ab8 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -407,7 +407,7 @@ class AArch64TTIImpl : public BasicTTIImplBase { /// If the AM is supported, the return value must be >= 0. /// If the AM is not supported, it returns a negative value. InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, - int64_t BaseOffset, bool HasBaseReg, + StackOffset BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) const; /// @} diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp index ee87f7f0e555ef..7db2e8ee7e6f92 100644 --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp @@ -2571,14 +2571,15 @@ bool ARMTTIImpl::preferPredicatedReductionSelect( } InstructionCost ARMTTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, - int64_t BaseOffset, + StackOffset BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) const { TargetLoweringBase::AddrMode AM; AM.BaseGV = BaseGV; - AM.BaseOffs = BaseOffset; + AM.BaseOffs = BaseOffset.getFixed(); AM.HasBaseReg = HasBaseReg; AM.Scale = Scale; + AM.ScalableOffset = BaseOffset.getScalable(); if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AddrSpace)) { if (ST->hasFPAO()) return AM.Scale < 0 ? 1 : 0; // positive offsets execute faster diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h index 58eab45b9641f5..8c4b92b8568882 100644 --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h @@ -303,7 +303,7 @@ class ARMTTIImpl : public BasicTTIImplBase { /// If the AM is supported, the return value must be >= 0. /// If the AM is not supported, the return value must be negative. InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, - int64_t BaseOffset, bool HasBaseReg, + StackOffset BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) const; bool maybeLoweredToCall(Instruction &I); diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index 6b7cddc6d72e40..d43480d0a0125b 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -6741,7 +6741,7 @@ InstructionCost X86TTIImpl::getInterleavedMemoryOpCost( } InstructionCost X86TTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, - int64_t BaseOffset, + StackOffset BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) const { // Scaling factors are not free at all. @@ -6764,9 +6764,10 @@ InstructionCost X86TTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, // vmovaps %ymm1, (%r8) can use port 2, 3, or 7. TargetLoweringBase::AddrMode AM; AM.BaseGV = BaseGV; - AM.BaseOffs = BaseOffset; + AM.BaseOffs = BaseOffset.getFixed(); AM.HasBaseReg = HasBaseReg; AM.Scale = Scale; + AM.ScalableOffset = BaseOffset.getScalable(); if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AddrSpace)) // Scale represents reg2 * scale, thus account for 1 // as soon as we use a second register. diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h index b5019307457398..d720cc136b8ae9 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.h +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h @@ -253,7 +253,7 @@ class X86TTIImpl : public BasicTTIImplBase { /// If the AM is supported, the return value must be >= 0. /// If the AM is not supported, it returns a negative value. InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, - int64_t BaseOffset, bool HasBaseReg, + StackOffset BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) const; bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1, diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp index ec42e2d6e193a6..eb1904ccaff352 100644 --- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp +++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp @@ -1817,10 +1817,12 @@ static InstructionCost getScalingFactorCost(const TargetTransformInfo &TTI, case LSRUse::Address: { // Check the scaling factor cost with both the min and max offsets. InstructionCost ScaleCostMinOffset = TTI.getScalingFactorCost( - LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MinOffset, F.HasBaseReg, + LU.AccessTy.MemTy, F.BaseGV, + StackOffset::getFixed(F.BaseOffset + LU.MinOffset), F.HasBaseReg, F.Scale, LU.AccessTy.AddrSpace); InstructionCost ScaleCostMaxOffset = TTI.getScalingFactorCost( - LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MaxOffset, F.HasBaseReg, + LU.AccessTy.MemTy, F.BaseGV, + StackOffset::getFixed(F.BaseOffset + LU.MaxOffset), F.HasBaseReg, F.Scale, LU.AccessTy.AddrSpace); assert(ScaleCostMinOffset.isValid() && ScaleCostMaxOffset.isValid() &&