From d9a685a9dd589486e882b722e513ee7b8c84870c Mon Sep 17 00:00:00 2001 From: Akira Hatanaka Date: Wed, 27 Mar 2024 12:24:49 -0700 Subject: [PATCH 01/54] [CodeGen][arm64e] Add methods and data members to Address, which are needed to authenticate signed pointers (#86721) To authenticate pointers, CodeGen needs access to the key and discriminators that were used to sign the pointer. That information is sometimes known from the context, but not always, which is why `Address` needs to hold that information. This patch adds methods and data members to `Address`, which will be needed in subsequent patches to authenticate signed pointers, and uses the newly added methods throughout CodeGen. Although this patch isn't strictly NFC as it causes CodeGen to use different code paths in some cases (e.g., `mergeAddressesInConditionalExpr`), it doesn't cause any changes in functionality as it doesn't add any information needed for authentication. In addition to the changes mentioned above, this patch introduces class `RawAddress`, which contains a pointer that we know is unsigned, and adds several new functions for creating `Address` and `LValue` objects. This reapplies 8bd1f9116aab879183f34707e6d21c7051d083b6. The commit broke msan bots because LValue::IsKnownNonNull was uninitialized. --- clang/lib/CodeGen/ABIInfoImpl.cpp | 10 +- clang/lib/CodeGen/Address.h | 195 ++++++++++++++--- clang/lib/CodeGen/CGAtomic.cpp | 53 ++--- clang/lib/CodeGen/CGBlocks.cpp | 34 +-- clang/lib/CodeGen/CGBlocks.h | 3 +- clang/lib/CodeGen/CGBuilder.h | 234 ++++++++++++++------- clang/lib/CodeGen/CGBuiltin.cpp | 173 +++++++-------- clang/lib/CodeGen/CGCUDANV.cpp | 19 +- clang/lib/CodeGen/CGCXXABI.cpp | 21 +- clang/lib/CodeGen/CGCXXABI.h | 14 +- clang/lib/CodeGen/CGCall.cpp | 171 ++++++++------- clang/lib/CodeGen/CGCall.h | 1 + clang/lib/CodeGen/CGClass.cpp | 76 ++++--- clang/lib/CodeGen/CGCleanup.cpp | 110 ++++------ clang/lib/CodeGen/CGCleanup.h | 2 +- clang/lib/CodeGen/CGCoroutine.cpp | 4 +- clang/lib/CodeGen/CGDecl.cpp | 28 +-- clang/lib/CodeGen/CGException.cpp | 19 +- clang/lib/CodeGen/CGExpr.cpp | 227 ++++++++++---------- clang/lib/CodeGen/CGExprAgg.cpp | 29 +-- clang/lib/CodeGen/CGExprCXX.cpp | 111 +++++----- clang/lib/CodeGen/CGExprConstant.cpp | 4 +- clang/lib/CodeGen/CGExprScalar.cpp | 23 +- clang/lib/CodeGen/CGNonTrivialStruct.cpp | 8 +- clang/lib/CodeGen/CGObjC.cpp | 43 ++-- clang/lib/CodeGen/CGObjCGNU.cpp | 42 ++-- clang/lib/CodeGen/CGObjCMac.cpp | 95 ++++----- clang/lib/CodeGen/CGObjCRuntime.cpp | 6 +- clang/lib/CodeGen/CGOpenMPRuntime.cpp | 194 +++++++++-------- clang/lib/CodeGen/CGOpenMPRuntime.h | 5 +- clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp | 76 +++---- clang/lib/CodeGen/CGStmt.cpp | 8 +- clang/lib/CodeGen/CGStmtOpenMP.cpp | 87 ++++---- clang/lib/CodeGen/CGVTables.cpp | 9 +- clang/lib/CodeGen/CGValue.h | 250 +++++++++++----------- clang/lib/CodeGen/CodeGenFunction.cpp | 71 ++++--- clang/lib/CodeGen/CodeGenFunction.h | 257 ++++++++++++++++------- clang/lib/CodeGen/CodeGenModule.cpp | 2 +- clang/lib/CodeGen/CodeGenPGO.cpp | 10 +- clang/lib/CodeGen/CodeGenPGO.h | 6 +- clang/lib/CodeGen/ItaniumCXXABI.cpp | 52 ++--- clang/lib/CodeGen/MicrosoftCXXABI.cpp | 58 ++--- clang/lib/CodeGen/TargetInfo.h | 5 + clang/lib/CodeGen/Targets/NVPTX.cpp | 2 +- clang/lib/CodeGen/Targets/PPC.cpp | 11 +- clang/lib/CodeGen/Targets/Sparc.cpp | 2 +- clang/lib/CodeGen/Targets/SystemZ.cpp | 9 +- clang/lib/CodeGen/Targets/XCore.cpp | 2 +- clang/utils/TableGen/MveEmitter.cpp | 2 +- llvm/include/llvm/IR/IRBuilder.h | 1 + 50 files changed, 1640 insertions(+), 1234 deletions(-) diff --git a/clang/lib/CodeGen/ABIInfoImpl.cpp b/clang/lib/CodeGen/ABIInfoImpl.cpp index dd59101ecc81b8..3e34d82cb399ba 100644 --- a/clang/lib/CodeGen/ABIInfoImpl.cpp +++ b/clang/lib/CodeGen/ABIInfoImpl.cpp @@ -187,7 +187,7 @@ CodeGen::emitVoidPtrDirectVAArg(CodeGenFunction &CGF, Address VAListAddr, CharUnits FullDirectSize = DirectSize.alignTo(SlotSize); Address NextPtr = CGF.Builder.CreateConstInBoundsByteGEP(Addr, FullDirectSize, "argp.next"); - CGF.Builder.CreateStore(NextPtr.getPointer(), VAListAddr); + CGF.Builder.CreateStore(NextPtr.emitRawPointer(CGF), VAListAddr); // If the argument is smaller than a slot, and this is a big-endian // target, the argument will be right-adjusted in its slot. @@ -239,8 +239,8 @@ Address CodeGen::emitMergePHI(CodeGenFunction &CGF, Address Addr1, const llvm::Twine &Name) { assert(Addr1.getType() == Addr2.getType()); llvm::PHINode *PHI = CGF.Builder.CreatePHI(Addr1.getType(), 2, Name); - PHI->addIncoming(Addr1.getPointer(), Block1); - PHI->addIncoming(Addr2.getPointer(), Block2); + PHI->addIncoming(Addr1.emitRawPointer(CGF), Block1); + PHI->addIncoming(Addr2.emitRawPointer(CGF), Block2); CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment()); return Address(PHI, Addr1.getElementType(), Align); } @@ -400,7 +400,7 @@ Address CodeGen::EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, llvm::Type *ElementTy = CGF.ConvertTypeForMem(Ty); llvm::Type *BaseTy = llvm::PointerType::getUnqual(ElementTy); llvm::Value *Addr = - CGF.Builder.CreateVAArg(VAListAddr.getPointer(), BaseTy); + CGF.Builder.CreateVAArg(VAListAddr.emitRawPointer(CGF), BaseTy); return Address(Addr, ElementTy, TyAlignForABI); } else { assert((AI.isDirect() || AI.isExtend()) && @@ -416,7 +416,7 @@ Address CodeGen::EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, "Unexpected CoerceToType seen in arginfo in generic VAArg emitter!"); Address Temp = CGF.CreateMemTemp(Ty, "varet"); - Val = CGF.Builder.CreateVAArg(VAListAddr.getPointer(), + Val = CGF.Builder.CreateVAArg(VAListAddr.emitRawPointer(CGF), CGF.ConvertTypeForMem(Ty)); CGF.Builder.CreateStore(Val, Temp); return Temp; diff --git a/clang/lib/CodeGen/Address.h b/clang/lib/CodeGen/Address.h index cf48df8f5e7367..35ec370a139c92 100644 --- a/clang/lib/CodeGen/Address.h +++ b/clang/lib/CodeGen/Address.h @@ -15,6 +15,7 @@ #define LLVM_CLANG_LIB_CODEGEN_ADDRESS_H #include "clang/AST/CharUnits.h" +#include "clang/AST/Type.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/IR/Constants.h" #include "llvm/Support/MathExtras.h" @@ -22,28 +23,41 @@ namespace clang { namespace CodeGen { +class Address; +class CGBuilderTy; +class CodeGenFunction; +class CodeGenModule; + // Indicates whether a pointer is known not to be null. enum KnownNonNull_t { NotKnownNonNull, KnownNonNull }; -/// An aligned address. -class Address { +/// An abstract representation of an aligned address. This is designed to be an +/// IR-level abstraction, carrying just the information necessary to perform IR +/// operations on an address like loads and stores. In particular, it doesn't +/// carry C type information or allow the representation of things like +/// bit-fields; clients working at that level should generally be using +/// `LValue`. +/// The pointer contained in this class is known to be unsigned. +class RawAddress { llvm::PointerIntPair PointerAndKnownNonNull; llvm::Type *ElementType; CharUnits Alignment; protected: - Address(std::nullptr_t) : ElementType(nullptr) {} + RawAddress(std::nullptr_t) : ElementType(nullptr) {} public: - Address(llvm::Value *Pointer, llvm::Type *ElementType, CharUnits Alignment, - KnownNonNull_t IsKnownNonNull = NotKnownNonNull) + RawAddress(llvm::Value *Pointer, llvm::Type *ElementType, CharUnits Alignment, + KnownNonNull_t IsKnownNonNull = NotKnownNonNull) : PointerAndKnownNonNull(Pointer, IsKnownNonNull), ElementType(ElementType), Alignment(Alignment) { assert(Pointer != nullptr && "Pointer cannot be null"); assert(ElementType != nullptr && "Element type cannot be null"); } - static Address invalid() { return Address(nullptr); } + inline RawAddress(Address Addr); + + static RawAddress invalid() { return RawAddress(nullptr); } bool isValid() const { return PointerAndKnownNonNull.getPointer() != nullptr; } @@ -80,6 +94,133 @@ class Address { return Alignment; } + /// Return address with different element type, but same pointer and + /// alignment. + RawAddress withElementType(llvm::Type *ElemTy) const { + return RawAddress(getPointer(), ElemTy, getAlignment(), isKnownNonNull()); + } + + KnownNonNull_t isKnownNonNull() const { + assert(isValid()); + return (KnownNonNull_t)PointerAndKnownNonNull.getInt(); + } +}; + +/// Like RawAddress, an abstract representation of an aligned address, but the +/// pointer contained in this class is possibly signed. +class Address { + friend class CGBuilderTy; + + // The boolean flag indicates whether the pointer is known to be non-null. + llvm::PointerIntPair Pointer; + + /// The expected IR type of the pointer. Carrying accurate element type + /// information in Address makes it more convenient to work with Address + /// values and allows frontend assertions to catch simple mistakes. + llvm::Type *ElementType = nullptr; + + CharUnits Alignment; + + /// Offset from the base pointer. + llvm::Value *Offset = nullptr; + + llvm::Value *emitRawPointerSlow(CodeGenFunction &CGF) const; + +protected: + Address(std::nullptr_t) : ElementType(nullptr) {} + +public: + Address(llvm::Value *pointer, llvm::Type *elementType, CharUnits alignment, + KnownNonNull_t IsKnownNonNull = NotKnownNonNull) + : Pointer(pointer, IsKnownNonNull), ElementType(elementType), + Alignment(alignment) { + assert(pointer != nullptr && "Pointer cannot be null"); + assert(elementType != nullptr && "Element type cannot be null"); + assert(!alignment.isZero() && "Alignment cannot be zero"); + } + + Address(llvm::Value *BasePtr, llvm::Type *ElementType, CharUnits Alignment, + llvm::Value *Offset, KnownNonNull_t IsKnownNonNull = NotKnownNonNull) + : Pointer(BasePtr, IsKnownNonNull), ElementType(ElementType), + Alignment(Alignment), Offset(Offset) {} + + Address(RawAddress RawAddr) + : Pointer(RawAddr.isValid() ? RawAddr.getPointer() : nullptr), + ElementType(RawAddr.isValid() ? RawAddr.getElementType() : nullptr), + Alignment(RawAddr.isValid() ? RawAddr.getAlignment() + : CharUnits::Zero()) {} + + static Address invalid() { return Address(nullptr); } + bool isValid() const { return Pointer.getPointer() != nullptr; } + + /// This function is used in situations where the caller is doing some sort of + /// opaque "laundering" of the pointer. + void replaceBasePointer(llvm::Value *P) { + assert(isValid() && "pointer isn't valid"); + assert(P->getType() == Pointer.getPointer()->getType() && + "Pointer's type changed"); + Pointer.setPointer(P); + assert(isValid() && "pointer is invalid after replacement"); + } + + CharUnits getAlignment() const { return Alignment; } + + void setAlignment(CharUnits Value) { Alignment = Value; } + + llvm::Value *getBasePointer() const { + assert(isValid() && "pointer isn't valid"); + return Pointer.getPointer(); + } + + /// Return the type of the pointer value. + llvm::PointerType *getType() const { + return llvm::PointerType::get( + ElementType, + llvm::cast(Pointer.getPointer()->getType()) + ->getAddressSpace()); + } + + /// Return the type of the values stored in this address. + llvm::Type *getElementType() const { + assert(isValid()); + return ElementType; + } + + /// Return the address space that this address resides in. + unsigned getAddressSpace() const { return getType()->getAddressSpace(); } + + /// Return the IR name of the pointer value. + llvm::StringRef getName() const { return Pointer.getPointer()->getName(); } + + // This function is called only in CGBuilderBaseTy::CreateElementBitCast. + void setElementType(llvm::Type *Ty) { + assert(hasOffset() && + "this funcion shouldn't be called when there is no offset"); + ElementType = Ty; + } + + /// Whether the pointer is known not to be null. + KnownNonNull_t isKnownNonNull() const { + assert(isValid()); + return (KnownNonNull_t)Pointer.getInt(); + } + + Address setKnownNonNull() { + assert(isValid()); + Pointer.setInt(KnownNonNull); + return *this; + } + + bool hasOffset() const { return Offset; } + + llvm::Value *getOffset() const { return Offset; } + + /// Return the pointer contained in this class after authenticating it and + /// adding offset to it if necessary. + llvm::Value *emitRawPointer(CodeGenFunction &CGF) const { + return getBasePointer(); + } + /// Return address with different pointer, but same element type and /// alignment. Address withPointer(llvm::Value *NewPointer, @@ -91,61 +232,59 @@ class Address { /// Return address with different alignment, but same pointer and element /// type. Address withAlignment(CharUnits NewAlignment) const { - return Address(getPointer(), getElementType(), NewAlignment, + return Address(Pointer.getPointer(), getElementType(), NewAlignment, isKnownNonNull()); } /// Return address with different element type, but same pointer and /// alignment. Address withElementType(llvm::Type *ElemTy) const { - return Address(getPointer(), ElemTy, getAlignment(), isKnownNonNull()); - } - - /// Whether the pointer is known not to be null. - KnownNonNull_t isKnownNonNull() const { - assert(isValid()); - return (KnownNonNull_t)PointerAndKnownNonNull.getInt(); - } - - /// Set the non-null bit. - Address setKnownNonNull() { - assert(isValid()); - PointerAndKnownNonNull.setInt(true); - return *this; + if (!hasOffset()) + return Address(getBasePointer(), ElemTy, getAlignment(), nullptr, + isKnownNonNull()); + Address A(*this); + A.ElementType = ElemTy; + return A; } }; +inline RawAddress::RawAddress(Address Addr) + : PointerAndKnownNonNull(Addr.isValid() ? Addr.getBasePointer() : nullptr, + Addr.isValid() ? Addr.isKnownNonNull() + : NotKnownNonNull), + ElementType(Addr.isValid() ? Addr.getElementType() : nullptr), + Alignment(Addr.isValid() ? Addr.getAlignment() : CharUnits::Zero()) {} + /// A specialization of Address that requires the address to be an /// LLVM Constant. -class ConstantAddress : public Address { - ConstantAddress(std::nullptr_t) : Address(nullptr) {} +class ConstantAddress : public RawAddress { + ConstantAddress(std::nullptr_t) : RawAddress(nullptr) {} public: ConstantAddress(llvm::Constant *pointer, llvm::Type *elementType, CharUnits alignment) - : Address(pointer, elementType, alignment) {} + : RawAddress(pointer, elementType, alignment) {} static ConstantAddress invalid() { return ConstantAddress(nullptr); } llvm::Constant *getPointer() const { - return llvm::cast(Address::getPointer()); + return llvm::cast(RawAddress::getPointer()); } ConstantAddress withElementType(llvm::Type *ElemTy) const { return ConstantAddress(getPointer(), ElemTy, getAlignment()); } - static bool isaImpl(Address addr) { + static bool isaImpl(RawAddress addr) { return llvm::isa(addr.getPointer()); } - static ConstantAddress castImpl(Address addr) { + static ConstantAddress castImpl(RawAddress addr) { return ConstantAddress(llvm::cast(addr.getPointer()), addr.getElementType(), addr.getAlignment()); } }; - } // Present a minimal LLVM-like casting interface. diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp index fb03d013e8afc7..56198385de9dcb 100644 --- a/clang/lib/CodeGen/CGAtomic.cpp +++ b/clang/lib/CodeGen/CGAtomic.cpp @@ -80,7 +80,7 @@ namespace { AtomicSizeInBits = C.toBits( C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) .alignTo(lvalue.getAlignment())); - llvm::Value *BitFieldPtr = lvalue.getBitFieldPointer(); + llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF); auto OffsetInChars = (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * lvalue.getAlignment(); @@ -139,13 +139,13 @@ namespace { const LValue &getAtomicLValue() const { return LVal; } llvm::Value *getAtomicPointer() const { if (LVal.isSimple()) - return LVal.getPointer(CGF); + return LVal.emitRawPointer(CGF); else if (LVal.isBitField()) - return LVal.getBitFieldPointer(); + return LVal.getRawBitFieldPointer(CGF); else if (LVal.isVectorElt()) - return LVal.getVectorPointer(); + return LVal.getRawVectorPointer(CGF); assert(LVal.isExtVectorElt()); - return LVal.getExtVectorPointer(); + return LVal.getRawExtVectorPointer(CGF); } Address getAtomicAddress() const { llvm::Type *ElTy; @@ -368,7 +368,7 @@ bool AtomicInfo::emitMemSetZeroIfNecessary() const { return false; CGF.Builder.CreateMemSet( - addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0), + addr.emitRawPointer(CGF), llvm::ConstantInt::get(CGF.Int8Ty, 0), CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), LVal.getAlignment().getAsAlign()); return true; @@ -1055,7 +1055,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { return getTargetHooks().performAddrSpaceCast( *this, V, AS, LangAS::opencl_generic, DestType, false); }; - Args.add(RValue::get(CastToGenericAddrSpace(Ptr.getPointer(), + + Args.add(RValue::get(CastToGenericAddrSpace(Ptr.emitRawPointer(*this), E->getPtr()->getType())), getContext().VoidPtrTy); @@ -1086,10 +1087,10 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { LibCallName = "__atomic_compare_exchange"; RetTy = getContext().BoolTy; HaveRetTy = true; - Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(), + Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), E->getVal1()->getType())), getContext().VoidPtrTy); - Args.add(RValue::get(CastToGenericAddrSpace(Val2.getPointer(), + Args.add(RValue::get(CastToGenericAddrSpace(Val2.emitRawPointer(*this), E->getVal2()->getType())), getContext().VoidPtrTy); Args.add(RValue::get(Order), getContext().IntTy); @@ -1105,7 +1106,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__scoped_atomic_exchange: case AtomicExpr::AO__scoped_atomic_exchange_n: LibCallName = "__atomic_exchange"; - Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(), + Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), E->getVal1()->getType())), getContext().VoidPtrTy); break; @@ -1120,7 +1121,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { LibCallName = "__atomic_store"; RetTy = getContext().VoidTy; HaveRetTy = true; - Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(), + Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), E->getVal1()->getType())), getContext().VoidPtrTy); break; @@ -1199,7 +1200,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { if (!HaveRetTy) { // Value is returned through parameter before the order. RetTy = getContext().VoidTy; - Args.add(RValue::get(CastToGenericAddrSpace(Dest.getPointer(), RetTy)), + Args.add(RValue::get( + CastToGenericAddrSpace(Dest.emitRawPointer(*this), RetTy)), getContext().VoidPtrTy); } // Order is always the last parameter. @@ -1513,7 +1515,7 @@ RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, } else TempAddr = CreateTempAlloca(); - EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); + EmitAtomicLoadLibcall(TempAddr.emitRawPointer(CGF), AO, IsVolatile); // Okay, turn that back into the original value or whole atomic (for // non-simple lvalues) type. @@ -1673,9 +1675,9 @@ std::pair AtomicInfo::EmitAtomicCompareExchange( if (shouldUseLibcall()) { // Produce a source address. Address ExpectedAddr = materializeRValue(Expected); - Address DesiredAddr = materializeRValue(Desired); - auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), - DesiredAddr.getPointer(), + llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); + llvm::Value *DesiredPtr = materializeRValue(Desired).emitRawPointer(CGF); + auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, Success, Failure); return std::make_pair( convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), @@ -1757,7 +1759,7 @@ void AtomicInfo::EmitAtomicUpdateLibcall( Address ExpectedAddr = CreateTempAlloca(); - EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); + EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile); auto *ContBB = CGF.createBasicBlock("atomic_cont"); auto *ExitBB = CGF.createBasicBlock("atomic_exit"); CGF.EmitBlock(ContBB); @@ -1771,10 +1773,10 @@ void AtomicInfo::EmitAtomicUpdateLibcall( AggValueSlot::ignored(), SourceLocation(), /*AsValue=*/false); EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); + llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); + llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF); auto *Res = - EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), - DesiredAddr.getPointer(), - AO, Failure); + EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure); CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); CGF.EmitBlock(ExitBB, /*IsFinished=*/true); } @@ -1843,7 +1845,7 @@ void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, Address ExpectedAddr = CreateTempAlloca(); - EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); + EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile); auto *ContBB = CGF.createBasicBlock("atomic_cont"); auto *ExitBB = CGF.createBasicBlock("atomic_exit"); CGF.EmitBlock(ContBB); @@ -1854,10 +1856,10 @@ void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, CGF.Builder.CreateStore(OldVal, DesiredAddr); } EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); + llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); + llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF); auto *Res = - EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), - DesiredAddr.getPointer(), - AO, Failure); + EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure); CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); CGF.EmitBlock(ExitBB, /*IsFinished=*/true); } @@ -1957,7 +1959,8 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, args.add(RValue::get(atomics.getAtomicSizeValue()), getContext().getSizeType()); args.add(RValue::get(atomics.getAtomicPointer()), getContext().VoidPtrTy); - args.add(RValue::get(srcAddr.getPointer()), getContext().VoidPtrTy); + args.add(RValue::get(srcAddr.emitRawPointer(*this)), + getContext().VoidPtrTy); args.add( RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), getContext().IntTy); diff --git a/clang/lib/CodeGen/CGBlocks.cpp b/clang/lib/CodeGen/CGBlocks.cpp index ad0b50d799618e..a01f2c7c979840 100644 --- a/clang/lib/CodeGen/CGBlocks.cpp +++ b/clang/lib/CodeGen/CGBlocks.cpp @@ -36,7 +36,8 @@ CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name) : Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false), NoEscape(false), HasCXXObject(false), UsesStret(false), HasCapturedVariableLayout(false), CapturesNonExternalType(false), - LocalAddress(Address::invalid()), StructureType(nullptr), Block(block) { + LocalAddress(RawAddress::invalid()), StructureType(nullptr), + Block(block) { // Skip asm prefix, if any. 'name' is usually taken directly from // the mangled name of the enclosing function. @@ -794,7 +795,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { // Otherwise, we have to emit this as a local block. - Address blockAddr = blockInfo.LocalAddress; + RawAddress blockAddr = blockInfo.LocalAddress; assert(blockAddr.isValid() && "block has no address!"); llvm::Constant *isa; @@ -939,7 +940,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { if (CI.isNested()) byrefPointer = Builder.CreateLoad(src, "byref.capture"); else - byrefPointer = src.getPointer(); + byrefPointer = src.emitRawPointer(*this); // Write that void* into the capture field. Builder.CreateStore(byrefPointer, blockField); @@ -961,10 +962,10 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { } // If it's a reference variable, copy the reference into the block field. - } else if (type->isReferenceType()) { - Builder.CreateStore(src.getPointer(), blockField); + } else if (auto refType = type->getAs()) { + Builder.CreateStore(src.emitRawPointer(*this), blockField); - // If type is const-qualified, copy the value into the block field. + // If type is const-qualified, copy the value into the block field. } else if (type.isConstQualified() && type.getObjCLifetime() == Qualifiers::OCL_Strong && CGM.getCodeGenOpts().OptimizationLevel != 0) { @@ -1377,7 +1378,7 @@ void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D, // Allocate a stack slot like for any local variable to guarantee optimal // debug info at -O0. The mem2reg pass will eliminate it when optimizing. - Address alloc = CreateMemTemp(D->getType(), D->getName() + ".addr"); + RawAddress alloc = CreateMemTemp(D->getType(), D->getName() + ".addr"); Builder.CreateStore(arg, alloc); if (CGDebugInfo *DI = getDebugInfo()) { if (CGM.getCodeGenOpts().hasReducedDebugInfo()) { @@ -1497,7 +1498,7 @@ llvm::Function *CodeGenFunction::GenerateBlockFunction( // frame setup instruction by llvm::DwarfDebug::beginFunction(). auto NL = ApplyDebugLocation::CreateEmpty(*this); Builder.CreateStore(BlockPointer, Alloca); - BlockPointerDbgLoc = Alloca.getPointer(); + BlockPointerDbgLoc = Alloca.emitRawPointer(*this); } // If we have a C++ 'this' reference, go ahead and force it into @@ -1557,8 +1558,8 @@ llvm::Function *CodeGenFunction::GenerateBlockFunction( const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable); if (capture.isConstant()) { auto addr = LocalDeclMap.find(variable)->second; - (void)DI->EmitDeclareOfAutoVariable(variable, addr.getPointer(), - Builder); + (void)DI->EmitDeclareOfAutoVariable( + variable, addr.emitRawPointer(*this), Builder); continue; } @@ -1662,7 +1663,7 @@ struct CallBlockRelease final : EHScopeStack::Cleanup { if (LoadBlockVarAddr) { BlockVarAddr = CGF.Builder.CreateLoad(Addr); } else { - BlockVarAddr = Addr.getPointer(); + BlockVarAddr = Addr.emitRawPointer(CGF); } CGF.BuildBlockRelease(BlockVarAddr, FieldFlags, CanThrow); @@ -1962,13 +1963,15 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) { // it. It's not quite worth the annoyance to avoid creating it in the // first place. if (!needsEHCleanup(captureType.isDestructedType())) - cast(dstField.getPointer())->eraseFromParent(); + if (auto *I = + cast_or_null(dstField.getBasePointer())) + I->eraseFromParent(); } break; } case BlockCaptureEntityKind::BlockObject: { llvm::Value *srcValue = Builder.CreateLoad(srcField, "blockcopy.src"); - llvm::Value *dstAddr = dstField.getPointer(); + llvm::Value *dstAddr = dstField.emitRawPointer(*this); llvm::Value *args[] = { dstAddr, srcValue, llvm::ConstantInt::get(Int32Ty, flags.getBitMask()) }; @@ -2139,7 +2142,7 @@ class ObjectByrefHelpers final : public BlockByrefHelpers { llvm::Value *flagsVal = llvm::ConstantInt::get(CGF.Int32Ty, flags); llvm::FunctionCallee fn = CGF.CGM.getBlockObjectAssign(); - llvm::Value *args[] = { destField.getPointer(), srcValue, flagsVal }; + llvm::Value *args[] = {destField.emitRawPointer(CGF), srcValue, flagsVal}; CGF.EmitNounwindRuntimeCall(fn, args); } @@ -2696,7 +2699,8 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) { storeHeaderField(V, getPointerSize(), "byref.isa"); // Store the address of the variable into its own forwarding pointer. - storeHeaderField(addr.getPointer(), getPointerSize(), "byref.forwarding"); + storeHeaderField(addr.emitRawPointer(*this), getPointerSize(), + "byref.forwarding"); // Blocks ABI: // c) the flags field is set to either 0 if no helper functions are diff --git a/clang/lib/CodeGen/CGBlocks.h b/clang/lib/CodeGen/CGBlocks.h index 4ef1ae9f33655c..8d10c4f69b2026 100644 --- a/clang/lib/CodeGen/CGBlocks.h +++ b/clang/lib/CodeGen/CGBlocks.h @@ -271,7 +271,8 @@ class CGBlockInfo { /// The block's captures. Non-constant captures are sorted by their offsets. llvm::SmallVector SortedCaptures; - Address LocalAddress; + // Currently we assume that block-pointer types are never signed. + RawAddress LocalAddress; llvm::StructType *StructureType; const BlockDecl *Block; const BlockExpr *BlockExpression; diff --git a/clang/lib/CodeGen/CGBuilder.h b/clang/lib/CodeGen/CGBuilder.h index bf5ab171d720d9..6dd9da7c4cadef 100644 --- a/clang/lib/CodeGen/CGBuilder.h +++ b/clang/lib/CodeGen/CGBuilder.h @@ -10,7 +10,9 @@ #define LLVM_CLANG_LIB_CODEGEN_CGBUILDER_H #include "Address.h" +#include "CGValue.h" #include "CodeGenTypeCache.h" +#include "llvm/Analysis/Utils/Local.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Type.h" @@ -18,12 +20,15 @@ namespace clang { namespace CodeGen { +class CGBuilderTy; class CodeGenFunction; /// This is an IRBuilder insertion helper that forwards to /// CodeGenFunction::InsertHelper, which adds necessary metadata to /// instructions. class CGBuilderInserter final : public llvm::IRBuilderDefaultInserter { + friend CGBuilderTy; + public: CGBuilderInserter() = default; explicit CGBuilderInserter(CodeGenFunction *CGF) : CGF(CGF) {} @@ -43,10 +48,42 @@ typedef llvm::IRBuilder CGBuilderBaseTy; class CGBuilderTy : public CGBuilderBaseTy { + friend class Address; + /// Storing a reference to the type cache here makes it a lot easier /// to build natural-feeling, target-specific IR. const CodeGenTypeCache &TypeCache; + CodeGenFunction *getCGF() const { return getInserter().CGF; } + + llvm::Value *emitRawPointerFromAddress(Address Addr) const { + return Addr.getBasePointer(); + } + + template + Address createConstGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1, + const llvm::Twine &Name) { + const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); + llvm::GetElementPtrInst *GEP; + if (IsInBounds) + GEP = cast(CreateConstInBoundsGEP2_32( + Addr.getElementType(), emitRawPointerFromAddress(Addr), Idx0, Idx1, + Name)); + else + GEP = cast(CreateConstGEP2_32( + Addr.getElementType(), emitRawPointerFromAddress(Addr), Idx0, Idx1, + Name)); + llvm::APInt Offset( + DL.getIndexSizeInBits(Addr.getType()->getPointerAddressSpace()), 0, + /*isSigned=*/true); + if (!GEP->accumulateConstantOffset(DL, Offset)) + llvm_unreachable("offset of GEP with constants is always computable"); + return Address(GEP, GEP->getResultElementType(), + Addr.getAlignment().alignmentAtOffset( + CharUnits::fromQuantity(Offset.getSExtValue())), + IsInBounds ? Addr.isKnownNonNull() : NotKnownNonNull); + } + public: CGBuilderTy(const CodeGenTypeCache &TypeCache, llvm::LLVMContext &C) : CGBuilderBaseTy(C), TypeCache(TypeCache) {} @@ -69,20 +106,22 @@ class CGBuilderTy : public CGBuilderBaseTy { // Note that we intentionally hide the CreateLoad APIs that don't // take an alignment. llvm::LoadInst *CreateLoad(Address Addr, const llvm::Twine &Name = "") { - return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(), + return CreateAlignedLoad(Addr.getElementType(), + emitRawPointerFromAddress(Addr), Addr.getAlignment().getAsAlign(), Name); } llvm::LoadInst *CreateLoad(Address Addr, const char *Name) { // This overload is required to prevent string literals from // ending up in the IsVolatile overload. - return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(), + return CreateAlignedLoad(Addr.getElementType(), + emitRawPointerFromAddress(Addr), Addr.getAlignment().getAsAlign(), Name); } llvm::LoadInst *CreateLoad(Address Addr, bool IsVolatile, const llvm::Twine &Name = "") { - return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(), - Addr.getAlignment().getAsAlign(), IsVolatile, - Name); + return CreateAlignedLoad( + Addr.getElementType(), emitRawPointerFromAddress(Addr), + Addr.getAlignment().getAsAlign(), IsVolatile, Name); } using CGBuilderBaseTy::CreateAlignedLoad; @@ -96,7 +135,7 @@ class CGBuilderTy : public CGBuilderBaseTy { // take an alignment. llvm::StoreInst *CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile = false) { - return CreateAlignedStore(Val, Addr.getPointer(), + return CreateAlignedStore(Val, emitRawPointerFromAddress(Addr), Addr.getAlignment().getAsAlign(), IsVolatile); } @@ -132,33 +171,41 @@ class CGBuilderTy : public CGBuilderBaseTy { llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID = llvm::SyncScope::System) { return CGBuilderBaseTy::CreateAtomicCmpXchg( - Addr.getPointer(), Cmp, New, Addr.getAlignment().getAsAlign(), - SuccessOrdering, FailureOrdering, SSID); + Addr.emitRawPointer(*getCGF()), Cmp, New, + Addr.getAlignment().getAsAlign(), SuccessOrdering, FailureOrdering, + SSID); } llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID = llvm::SyncScope::System) { - return CGBuilderBaseTy::CreateAtomicRMW(Op, Addr.getPointer(), Val, - Addr.getAlignment().getAsAlign(), - Ordering, SSID); + return CGBuilderBaseTy::CreateAtomicRMW( + Op, Addr.emitRawPointer(*getCGF()), Val, + Addr.getAlignment().getAsAlign(), Ordering, SSID); } using CGBuilderBaseTy::CreateAddrSpaceCast; Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, + llvm::Type *ElementTy, const llvm::Twine &Name = "") { - return Addr.withPointer(CreateAddrSpaceCast(Addr.getPointer(), Ty, Name), - Addr.isKnownNonNull()); + if (!Addr.hasOffset()) + return Address(CreateAddrSpaceCast(Addr.getBasePointer(), Ty, Name), + ElementTy, Addr.getAlignment(), nullptr, + Addr.isKnownNonNull()); + // Eagerly force a raw address if these is an offset. + return RawAddress( + CreateAddrSpaceCast(Addr.emitRawPointer(*getCGF()), Ty, Name), + ElementTy, Addr.getAlignment(), Addr.isKnownNonNull()); } using CGBuilderBaseTy::CreatePointerBitCastOrAddrSpaceCast; Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name = "") { - llvm::Value *Ptr = - CreatePointerBitCastOrAddrSpaceCast(Addr.getPointer(), Ty, Name); - return Address(Ptr, ElementTy, Addr.getAlignment(), Addr.isKnownNonNull()); + if (Addr.getType()->getAddressSpace() == Ty->getPointerAddressSpace()) + return Addr.withElementType(ElementTy); + return CreateAddrSpaceCast(Addr, Ty, ElementTy, Name); } /// Given @@ -176,10 +223,11 @@ class CGBuilderTy : public CGBuilderBaseTy { const llvm::StructLayout *Layout = DL.getStructLayout(ElTy); auto Offset = CharUnits::fromQuantity(Layout->getElementOffset(Index)); - return Address( - CreateStructGEP(Addr.getElementType(), Addr.getPointer(), Index, Name), - ElTy->getElementType(Index), - Addr.getAlignment().alignmentAtOffset(Offset), Addr.isKnownNonNull()); + return Address(CreateStructGEP(Addr.getElementType(), Addr.getBasePointer(), + Index, Name), + ElTy->getElementType(Index), + Addr.getAlignment().alignmentAtOffset(Offset), + Addr.isKnownNonNull()); } /// Given @@ -198,7 +246,7 @@ class CGBuilderTy : public CGBuilderBaseTy { CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy->getElementType())); return Address( - CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), + CreateInBoundsGEP(Addr.getElementType(), Addr.getBasePointer(), {getSize(CharUnits::Zero()), getSize(Index)}, Name), ElTy->getElementType(), Addr.getAlignment().alignmentAtOffset(Index * EltSize), @@ -216,10 +264,10 @@ class CGBuilderTy : public CGBuilderBaseTy { const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy)); - return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), - getSize(Index), Name), - ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize), - Addr.isKnownNonNull()); + return Address( + CreateInBoundsGEP(ElTy, Addr.getBasePointer(), getSize(Index), Name), + ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize), + Addr.isKnownNonNull()); } /// Given @@ -229,110 +277,133 @@ class CGBuilderTy : public CGBuilderBaseTy { /// where i64 is actually the target word size. Address CreateConstGEP(Address Addr, uint64_t Index, const llvm::Twine &Name = "") { + llvm::Type *ElTy = Addr.getElementType(); const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); - CharUnits EltSize = - CharUnits::fromQuantity(DL.getTypeAllocSize(Addr.getElementType())); + CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy)); - return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(), - getSize(Index), Name), + return Address(CreateGEP(ElTy, Addr.getBasePointer(), getSize(Index), Name), Addr.getElementType(), - Addr.getAlignment().alignmentAtOffset(Index * EltSize), - NotKnownNonNull); + Addr.getAlignment().alignmentAtOffset(Index * EltSize)); } /// Create GEP with single dynamic index. The address alignment is reduced /// according to the element size. using CGBuilderBaseTy::CreateGEP; - Address CreateGEP(Address Addr, llvm::Value *Index, + Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, const llvm::Twine &Name = "") { const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(Addr.getElementType())); return Address( - CreateGEP(Addr.getElementType(), Addr.getPointer(), Index, Name), + CreateGEP(Addr.getElementType(), Addr.emitRawPointer(CGF), Index, Name), Addr.getElementType(), - Addr.getAlignment().alignmentOfArrayElement(EltSize), NotKnownNonNull); + Addr.getAlignment().alignmentOfArrayElement(EltSize)); } /// Given a pointer to i8, adjust it by a given constant offset. Address CreateConstInBoundsByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name = "") { assert(Addr.getElementType() == TypeCache.Int8Ty); - return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), - getSize(Offset), Name), - Addr.getElementType(), - Addr.getAlignment().alignmentAtOffset(Offset), - Addr.isKnownNonNull()); + return Address( + CreateInBoundsGEP(Addr.getElementType(), Addr.getBasePointer(), + getSize(Offset), Name), + Addr.getElementType(), Addr.getAlignment().alignmentAtOffset(Offset), + Addr.isKnownNonNull()); } + Address CreateConstByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name = "") { assert(Addr.getElementType() == TypeCache.Int8Ty); - return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(), + return Address(CreateGEP(Addr.getElementType(), Addr.getBasePointer(), getSize(Offset), Name), Addr.getElementType(), - Addr.getAlignment().alignmentAtOffset(Offset), - NotKnownNonNull); + Addr.getAlignment().alignmentAtOffset(Offset)); } using CGBuilderBaseTy::CreateConstInBoundsGEP2_32; Address CreateConstInBoundsGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1, const llvm::Twine &Name = "") { - const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); + return createConstGEP2_32(Addr, Idx0, Idx1, Name); + } - auto *GEP = cast(CreateConstInBoundsGEP2_32( - Addr.getElementType(), Addr.getPointer(), Idx0, Idx1, Name)); - llvm::APInt Offset( - DL.getIndexSizeInBits(Addr.getType()->getPointerAddressSpace()), 0, - /*isSigned=*/true); - if (!GEP->accumulateConstantOffset(DL, Offset)) - llvm_unreachable("offset of GEP with constants is always computable"); - return Address(GEP, GEP->getResultElementType(), - Addr.getAlignment().alignmentAtOffset( - CharUnits::fromQuantity(Offset.getSExtValue())), - Addr.isKnownNonNull()); + using CGBuilderBaseTy::CreateConstGEP2_32; + Address CreateConstGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1, + const llvm::Twine &Name = "") { + return createConstGEP2_32(Addr, Idx0, Idx1, Name); + } + + Address CreateGEP(Address Addr, ArrayRef IdxList, + llvm::Type *ElementType, CharUnits Align, + const Twine &Name = "") { + llvm::Value *Ptr = emitRawPointerFromAddress(Addr); + return RawAddress(CreateGEP(Addr.getElementType(), Ptr, IdxList, Name), + ElementType, Align); + } + + using CGBuilderBaseTy::CreateInBoundsGEP; + Address CreateInBoundsGEP(Address Addr, ArrayRef IdxList, + llvm::Type *ElementType, CharUnits Align, + const Twine &Name = "") { + return RawAddress(CreateInBoundsGEP(Addr.getElementType(), + emitRawPointerFromAddress(Addr), + IdxList, Name), + ElementType, Align, Addr.isKnownNonNull()); + } + + using CGBuilderBaseTy::CreateIsNull; + llvm::Value *CreateIsNull(Address Addr, const Twine &Name = "") { + if (!Addr.hasOffset()) + return CreateIsNull(Addr.getBasePointer(), Name); + // The pointer isn't null if Addr has an offset since offsets can always + // be applied inbound. + return llvm::ConstantInt::getFalse(Context); } using CGBuilderBaseTy::CreateMemCpy; llvm::CallInst *CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile = false) { - return CreateMemCpy(Dest.getPointer(), Dest.getAlignment().getAsAlign(), - Src.getPointer(), Src.getAlignment().getAsAlign(), Size, - IsVolatile); + llvm::Value *DestPtr = emitRawPointerFromAddress(Dest); + llvm::Value *SrcPtr = emitRawPointerFromAddress(Src); + return CreateMemCpy(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr, + Src.getAlignment().getAsAlign(), Size, IsVolatile); } llvm::CallInst *CreateMemCpy(Address Dest, Address Src, uint64_t Size, bool IsVolatile = false) { - return CreateMemCpy(Dest.getPointer(), Dest.getAlignment().getAsAlign(), - Src.getPointer(), Src.getAlignment().getAsAlign(), Size, - IsVolatile); + llvm::Value *DestPtr = emitRawPointerFromAddress(Dest); + llvm::Value *SrcPtr = emitRawPointerFromAddress(Src); + return CreateMemCpy(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr, + Src.getAlignment().getAsAlign(), Size, IsVolatile); } using CGBuilderBaseTy::CreateMemCpyInline; llvm::CallInst *CreateMemCpyInline(Address Dest, Address Src, uint64_t Size) { - return CreateMemCpyInline( - Dest.getPointer(), Dest.getAlignment().getAsAlign(), Src.getPointer(), - Src.getAlignment().getAsAlign(), getInt64(Size)); + llvm::Value *DestPtr = emitRawPointerFromAddress(Dest); + llvm::Value *SrcPtr = emitRawPointerFromAddress(Src); + return CreateMemCpyInline(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr, + Src.getAlignment().getAsAlign(), getInt64(Size)); } using CGBuilderBaseTy::CreateMemMove; llvm::CallInst *CreateMemMove(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile = false) { - return CreateMemMove(Dest.getPointer(), Dest.getAlignment().getAsAlign(), - Src.getPointer(), Src.getAlignment().getAsAlign(), - Size, IsVolatile); + llvm::Value *DestPtr = emitRawPointerFromAddress(Dest); + llvm::Value *SrcPtr = emitRawPointerFromAddress(Src); + return CreateMemMove(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr, + Src.getAlignment().getAsAlign(), Size, IsVolatile); } using CGBuilderBaseTy::CreateMemSet; llvm::CallInst *CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile = false) { - return CreateMemSet(Dest.getPointer(), Value, Size, + return CreateMemSet(emitRawPointerFromAddress(Dest), Value, Size, Dest.getAlignment().getAsAlign(), IsVolatile); } using CGBuilderBaseTy::CreateMemSetInline; llvm::CallInst *CreateMemSetInline(Address Dest, llvm::Value *Value, uint64_t Size) { - return CreateMemSetInline(Dest.getPointer(), + return CreateMemSetInline(emitRawPointerFromAddress(Dest), Dest.getAlignment().getAsAlign(), Value, getInt64(Size)); } @@ -346,16 +417,31 @@ class CGBuilderTy : public CGBuilderBaseTy { const llvm::StructLayout *Layout = DL.getStructLayout(ElTy); auto Offset = CharUnits::fromQuantity(Layout->getElementOffset(Index)); - return Address(CreatePreserveStructAccessIndex(ElTy, Addr.getPointer(), - Index, FieldIndex, DbgInfo), - ElTy->getElementType(Index), - Addr.getAlignment().alignmentAtOffset(Offset)); + return Address( + CreatePreserveStructAccessIndex(ElTy, emitRawPointerFromAddress(Addr), + Index, FieldIndex, DbgInfo), + ElTy->getElementType(Index), + Addr.getAlignment().alignmentAtOffset(Offset)); + } + + using CGBuilderBaseTy::CreatePreserveUnionAccessIndex; + Address CreatePreserveUnionAccessIndex(Address Addr, unsigned FieldIndex, + llvm::MDNode *DbgInfo) { + Addr.replaceBasePointer(CreatePreserveUnionAccessIndex( + Addr.getBasePointer(), FieldIndex, DbgInfo)); + return Addr; } using CGBuilderBaseTy::CreateLaunderInvariantGroup; Address CreateLaunderInvariantGroup(Address Addr) { - return Addr.withPointer(CreateLaunderInvariantGroup(Addr.getPointer()), - Addr.isKnownNonNull()); + Addr.replaceBasePointer(CreateLaunderInvariantGroup(Addr.getBasePointer())); + return Addr; + } + + using CGBuilderBaseTy::CreateStripInvariantGroup; + Address CreateStripInvariantGroup(Address Addr) { + Addr.replaceBasePointer(CreateStripInvariantGroup(Addr.getBasePointer())); + return Addr; } }; diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index fdb517eb254d3b..5ab5917c0c8da7 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -2117,9 +2117,9 @@ llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction( auto AL = ApplyDebugLocation::CreateArtificial(*this); CharUnits Offset; - Address BufAddr = - Address(Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"), Int8Ty, - BufferAlignment); + Address BufAddr = makeNaturalAddressForPointer( + Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"), Ctx.VoidTy, + BufferAlignment); Builder.CreateStore(Builder.getInt8(Layout.getSummaryByte()), Builder.CreateConstByteGEP(BufAddr, Offset++, "summary")); Builder.CreateStore(Builder.getInt8(Layout.getNumArgsByte()), @@ -2162,7 +2162,7 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) { // Ignore argument 1, the format string. It is not currently used. CallArgList Args; - Args.add(RValue::get(BufAddr.getPointer()), Ctx.VoidPtrTy); + Args.add(RValue::get(BufAddr.emitRawPointer(*this)), Ctx.VoidPtrTy); for (const auto &Item : Layout.Items) { int Size = Item.getSizeByte(); @@ -2202,8 +2202,8 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) { if (!isa(ArgVal)) { CleanupKind Cleanup = getARCCleanupKind(); QualType Ty = TheExpr->getType(); - Address Alloca = Address::invalid(); - Address Addr = CreateMemTemp(Ty, "os.log.arg", &Alloca); + RawAddress Alloca = RawAddress::invalid(); + RawAddress Addr = CreateMemTemp(Ty, "os.log.arg", &Alloca); ArgVal = EmitARCRetain(Ty, ArgVal); Builder.CreateStore(ArgVal, Addr); pushLifetimeExtendedDestroy(Cleanup, Alloca, Ty, @@ -2236,7 +2236,7 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) { llvm::Function *F = CodeGenFunction(CGM).generateBuiltinOSLogHelperFunction( Layout, BufAddr.getAlignment()); EmitCall(FI, CGCallee::forDirect(F), ReturnValueSlot(), Args); - return RValue::get(BufAddr.getPointer()); + return RValue::get(BufAddr, *this); } static bool isSpecialUnsignedMultiplySignedResult( @@ -2984,7 +2984,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, // Check NonnullAttribute/NullabilityArg and Alignment. auto EmitArgCheck = [&](TypeCheckKind Kind, Address A, const Expr *Arg, unsigned ParmNum) { - Value *Val = A.getPointer(); + Value *Val = A.emitRawPointer(*this); EmitNonNullArgCheck(RValue::get(Val), Arg->getType(), Arg->getExprLoc(), FD, ParmNum); @@ -3013,12 +3013,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_va_end: EmitVAStartEnd(BuiltinID == Builtin::BI__va_start ? EmitScalarExpr(E->getArg(0)) - : EmitVAListRef(E->getArg(0)).getPointer(), + : EmitVAListRef(E->getArg(0)).emitRawPointer(*this), BuiltinID != Builtin::BI__builtin_va_end); return RValue::get(nullptr); case Builtin::BI__builtin_va_copy: { - Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer(); - Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer(); + Value *DstPtr = EmitVAListRef(E->getArg(0)).emitRawPointer(*this); + Value *SrcPtr = EmitVAListRef(E->getArg(1)).emitRawPointer(*this); Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy, {DstPtr->getType()}), {DstPtr, SrcPtr}); return RValue::get(nullptr); @@ -3849,13 +3849,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified(); Address Src = EmitPointerWithAlignment(E->getArg(0)); - EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(0)->getType(), - E->getArg(0)->getExprLoc(), FD, 0); + EmitNonNullArgCheck(RValue::get(Src.emitRawPointer(*this)), + E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, + 0); Value *Result = MB.CreateColumnMajorLoad( - Src.getElementType(), Src.getPointer(), + Src.getElementType(), Src.emitRawPointer(*this), Align(Src.getAlignment().getQuantity()), Stride, IsVolatile, - ResultTy->getNumRows(), ResultTy->getNumColumns(), - "matrix"); + ResultTy->getNumRows(), ResultTy->getNumColumns(), "matrix"); return RValue::get(Result); } @@ -3870,11 +3870,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, assert(PtrTy && "arg1 must be of pointer type"); bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified(); - EmitNonNullArgCheck(RValue::get(Dst.getPointer()), E->getArg(1)->getType(), - E->getArg(1)->getExprLoc(), FD, 0); + EmitNonNullArgCheck(RValue::get(Dst.emitRawPointer(*this)), + E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD, + 0); Value *Result = MB.CreateColumnMajorStore( - Matrix, Dst.getPointer(), Align(Dst.getAlignment().getQuantity()), - Stride, IsVolatile, MatrixTy->getNumRows(), MatrixTy->getNumColumns()); + Matrix, Dst.emitRawPointer(*this), + Align(Dst.getAlignment().getQuantity()), Stride, IsVolatile, + MatrixTy->getNumRows(), MatrixTy->getNumColumns()); return RValue::get(Result); } @@ -4033,7 +4035,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_bzero: { Address Dest = EmitPointerWithAlignment(E->getArg(0)); Value *SizeVal = EmitScalarExpr(E->getArg(1)); - EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), + EmitNonNullArgCheck(Dest, E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, 0); Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false); return RValue::get(nullptr); @@ -4044,10 +4046,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Address Src = EmitPointerWithAlignment(E->getArg(0)); Address Dest = EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = EmitScalarExpr(E->getArg(2)); - EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(0)->getType(), - E->getArg(0)->getExprLoc(), FD, 0); - EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(1)->getType(), - E->getArg(1)->getExprLoc(), FD, 0); + EmitNonNullArgCheck(RValue::get(Src.emitRawPointer(*this)), + E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, + 0); + EmitNonNullArgCheck(RValue::get(Dest.emitRawPointer(*this)), + E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD, + 0); Builder.CreateMemMove(Dest, Src, SizeVal, false); return RValue::get(nullptr); } @@ -4064,10 +4068,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Builder.CreateMemCpy(Dest, Src, SizeVal, false); if (BuiltinID == Builtin::BImempcpy || BuiltinID == Builtin::BI__builtin_mempcpy) - return RValue::get(Builder.CreateInBoundsGEP(Dest.getElementType(), - Dest.getPointer(), SizeVal)); + return RValue::get(Builder.CreateInBoundsGEP( + Dest.getElementType(), Dest.emitRawPointer(*this), SizeVal)); else - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BI__builtin_memcpy_inline: { @@ -4099,7 +4103,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Address Src = EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); Builder.CreateMemCpy(Dest, Src, SizeVal, false); - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BI__builtin_objc_memmove_collectable: { @@ -4108,7 +4112,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Value *SizeVal = EmitScalarExpr(E->getArg(2)); CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestAddr, SrcAddr, SizeVal); - return RValue::get(DestAddr.getPointer()); + return RValue::get(DestAddr, *this); } case Builtin::BI__builtin___memmove_chk: { @@ -4125,7 +4129,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Address Src = EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); Builder.CreateMemMove(Dest, Src, SizeVal, false); - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BImemmove: @@ -4136,7 +4140,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0); EmitArgCheck(TCK_Load, Src, E->getArg(1), 1); Builder.CreateMemMove(Dest, Src, SizeVal, false); - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BImemset: case Builtin::BI__builtin_memset: { @@ -4144,10 +4148,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty()); Value *SizeVal = EmitScalarExpr(E->getArg(2)); - EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), + EmitNonNullArgCheck(Dest, E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, 0); Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BI__builtin_memset_inline: { Address Dest = EmitPointerWithAlignment(E->getArg(0)); @@ -4155,8 +4159,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty()); uint64_t Size = E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue(); - EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), - E->getArg(0)->getExprLoc(), FD, 0); + EmitNonNullArgCheck(RValue::get(Dest.emitRawPointer(*this)), + E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, + 0); Builder.CreateMemSetInline(Dest, ByteVal, Size); return RValue::get(nullptr); } @@ -4175,7 +4180,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Builder.getInt8Ty()); Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BI__builtin_wmemchr: { // The MSVC runtime library does not provide a definition of wmemchr, so we @@ -4397,14 +4402,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, // Store the stack pointer to the setjmp buffer. Value *StackAddr = Builder.CreateStackSave(); - assert(Buf.getPointer()->getType() == StackAddr->getType()); + assert(Buf.emitRawPointer(*this)->getType() == StackAddr->getType()); Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Buf, 2); Builder.CreateStore(StackAddr, StackSaveSlot); // Call LLVM's EH setjmp, which is lightweight. Function *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp); - return RValue::get(Builder.CreateCall(F, Buf.getPointer())); + return RValue::get(Builder.CreateCall(F, Buf.emitRawPointer(*this))); } case Builtin::BI__builtin_longjmp: { Value *Buf = EmitScalarExpr(E->getArg(0)); @@ -5577,7 +5582,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, llvm::Value *Queue = EmitScalarExpr(E->getArg(0)); llvm::Value *Flags = EmitScalarExpr(E->getArg(1)); LValue NDRangeL = EmitAggExprToLValue(E->getArg(2)); - llvm::Value *Range = NDRangeL.getAddress(*this).getPointer(); + llvm::Value *Range = NDRangeL.getAddress(*this).emitRawPointer(*this); llvm::Type *RangeTy = NDRangeL.getAddress(*this).getType(); if (NumArgs == 4) { @@ -5686,9 +5691,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, getContext(), Expr::NPC_ValueDependentIsNotNull)) { EventWaitList = llvm::ConstantPointerNull::get(PtrTy); } else { - EventWaitList = E->getArg(4)->getType()->isArrayType() - ? EmitArrayToPointerDecay(E->getArg(4)).getPointer() - : EmitScalarExpr(E->getArg(4)); + EventWaitList = + E->getArg(4)->getType()->isArrayType() + ? EmitArrayToPointerDecay(E->getArg(4)).emitRawPointer(*this) + : EmitScalarExpr(E->getArg(4)); // Convert to generic address space. EventWaitList = Builder.CreatePointerCast(EventWaitList, PtrTy); } @@ -5784,7 +5790,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, llvm::Type *GenericVoidPtrTy = Builder.getPtrTy( getContext().getTargetAddressSpace(LangAS::opencl_generic)); LValue NDRangeL = EmitAggExprToLValue(E->getArg(0)); - llvm::Value *NDRange = NDRangeL.getAddress(*this).getPointer(); + llvm::Value *NDRange = NDRangeL.getAddress(*this).emitRawPointer(*this); auto Info = CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(1)); Value *Kernel = @@ -5869,7 +5875,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, auto PTy0 = FTy->getParamType(0); if (PTy0 != Arg0Val->getType()) { if (Arg0Ty->isArrayType()) - Arg0Val = EmitArrayToPointerDecay(Arg0).getPointer(); + Arg0Val = EmitArrayToPointerDecay(Arg0).emitRawPointer(*this); else Arg0Val = Builder.CreatePointerCast(Arg0Val, PTy0); } @@ -5907,7 +5913,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, auto PTy1 = FTy->getParamType(1); if (PTy1 != Arg1Val->getType()) { if (Arg1Ty->isArrayType()) - Arg1Val = EmitArrayToPointerDecay(Arg1).getPointer(); + Arg1Val = EmitArrayToPointerDecay(Arg1).emitRawPointer(*this); else Arg1Val = Builder.CreatePointerCast(Arg1Val, PTy1); } @@ -5921,7 +5927,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_ms_va_start: case Builtin::BI__builtin_ms_va_end: return RValue::get( - EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(), + EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).emitRawPointer(*this), BuiltinID == Builtin::BI__builtin_ms_va_start)); case Builtin::BI__builtin_ms_va_copy: { @@ -5963,8 +5969,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, // If this is a predefined lib function (e.g. malloc), emit the call // using exactly the normal call path. if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID)) - return emitLibraryCall(*this, FD, E, - cast(EmitScalarExpr(E->getCallee()))); + return emitLibraryCall( + *this, FD, E, cast(EmitScalarExpr(E->getCallee()))); // Check that a call to a target specific builtin has the correct target // features. @@ -6081,7 +6087,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, return RValue::get(nullptr); return RValue::get(V); case TEK_Aggregate: - return RValue::getAggregate(ReturnValue.getValue(), + return RValue::getAggregate(ReturnValue.getAddress(), ReturnValue.isVolatile()); case TEK_Complex: llvm_unreachable("No current target builtin returns complex"); @@ -8851,7 +8857,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, // Get the alignment for the argument in addition to the value; // we'll use it later. PtrOp0 = EmitPointerWithAlignment(E->getArg(0)); - Ops.push_back(PtrOp0.getPointer()); + Ops.push_back(PtrOp0.emitRawPointer(*this)); continue; } } @@ -8878,7 +8884,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, // Get the alignment for the argument in addition to the value; // we'll use it later. PtrOp1 = EmitPointerWithAlignment(E->getArg(1)); - Ops.push_back(PtrOp1.getPointer()); + Ops.push_back(PtrOp1.emitRawPointer(*this)); continue; } } @@ -9299,7 +9305,7 @@ Value *CodeGenFunction::EmitARMMVEBuiltinExpr(unsigned BuiltinID, if (ReturnValue.isNull()) return MvecOut; else - return Builder.CreateStore(MvecOut, ReturnValue.getValue()); + return Builder.CreateStore(MvecOut, ReturnValue.getAddress()); } case CustomCodeGen::VST24: { @@ -11479,7 +11485,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, // Get the alignment for the argument in addition to the value; // we'll use it later. PtrOp0 = EmitPointerWithAlignment(E->getArg(0)); - Ops.push_back(PtrOp0.getPointer()); + Ops.push_back(PtrOp0.emitRawPointer(*this)); continue; } } @@ -13345,15 +13351,15 @@ Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID, if (!getDebugInfo()) { CGM.Error(E->getExprLoc(), "using __builtin_preserve_field_info() without -g"); - return IsBitField ? EmitLValue(Arg).getBitFieldPointer() - : EmitLValue(Arg).getPointer(*this); + return IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*this) + : EmitLValue(Arg).emitRawPointer(*this); } // Enable underlying preserve_*_access_index() generation. bool OldIsInPreservedAIRegion = IsInPreservedAIRegion; IsInPreservedAIRegion = true; - Value *FieldAddr = IsBitField ? EmitLValue(Arg).getBitFieldPointer() - : EmitLValue(Arg).getPointer(*this); + Value *FieldAddr = IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*this) + : EmitLValue(Arg).emitRawPointer(*this); IsInPreservedAIRegion = OldIsInPreservedAIRegion; ConstantInt *C = cast(EmitScalarExpr(E->getArg(1))); @@ -14345,14 +14351,14 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, } case X86::BI_mm_setcsr: case X86::BI__builtin_ia32_ldmxcsr: { - Address Tmp = CreateMemTemp(E->getArg(0)->getType()); + RawAddress Tmp = CreateMemTemp(E->getArg(0)->getType()); Builder.CreateStore(Ops[0], Tmp); return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr), Tmp.getPointer()); } case X86::BI_mm_getcsr: case X86::BI__builtin_ia32_stmxcsr: { - Address Tmp = CreateMemTemp(E->getType()); + RawAddress Tmp = CreateMemTemp(E->getType()); Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr), Tmp.getPointer()); return Builder.CreateLoad(Tmp, "stmxcsr"); @@ -17627,7 +17633,8 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, SmallVector Ops; for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) if (E->getArg(i)->getType()->isArrayType()) - Ops.push_back(EmitArrayToPointerDecay(E->getArg(i)).getPointer()); + Ops.push_back( + EmitArrayToPointerDecay(E->getArg(i)).emitRawPointer(*this)); else Ops.push_back(EmitScalarExpr(E->getArg(i))); // The first argument of these two builtins is a pointer used to store their @@ -20089,14 +20096,14 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, // Save returned values. assert(II.NumResults); if (II.NumResults == 1) { - Builder.CreateAlignedStore(Result, Dst.getPointer(), + Builder.CreateAlignedStore(Result, Dst.emitRawPointer(*this), CharUnits::fromQuantity(4)); } else { for (unsigned i = 0; i < II.NumResults; ++i) { Builder.CreateAlignedStore( Builder.CreateBitCast(Builder.CreateExtractValue(Result, i), Dst.getElementType()), - Builder.CreateGEP(Dst.getElementType(), Dst.getPointer(), + Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); } @@ -20136,7 +20143,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < II.NumResults; ++i) { Value *V = Builder.CreateAlignedLoad( Src.getElementType(), - Builder.CreateGEP(Src.getElementType(), Src.getPointer(), + Builder.CreateGEP(Src.getElementType(), Src.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); Values.push_back(Builder.CreateBitCast(V, ParamType)); @@ -20208,7 +20215,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < MI.NumEltsA; ++i) { Value *V = Builder.CreateAlignedLoad( SrcA.getElementType(), - Builder.CreateGEP(SrcA.getElementType(), SrcA.getPointer(), + Builder.CreateGEP(SrcA.getElementType(), SrcA.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); Values.push_back(Builder.CreateBitCast(V, AType)); @@ -20218,7 +20225,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < MI.NumEltsB; ++i) { Value *V = Builder.CreateAlignedLoad( SrcB.getElementType(), - Builder.CreateGEP(SrcB.getElementType(), SrcB.getPointer(), + Builder.CreateGEP(SrcB.getElementType(), SrcB.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); Values.push_back(Builder.CreateBitCast(V, BType)); @@ -20229,7 +20236,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < MI.NumEltsC; ++i) { Value *V = Builder.CreateAlignedLoad( SrcC.getElementType(), - Builder.CreateGEP(SrcC.getElementType(), SrcC.getPointer(), + Builder.CreateGEP(SrcC.getElementType(), SrcC.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); Values.push_back(Builder.CreateBitCast(V, CType)); @@ -20239,7 +20246,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < MI.NumEltsD; ++i) Builder.CreateAlignedStore( Builder.CreateBitCast(Builder.CreateExtractValue(Result, i), DType), - Builder.CreateGEP(Dst.getElementType(), Dst.getPointer(), + Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); return Result; @@ -20497,7 +20504,7 @@ struct BuiltinAlignArgs { BuiltinAlignArgs(const CallExpr *E, CodeGenFunction &CGF) { QualType AstType = E->getArg(0)->getType(); if (AstType->isArrayType()) - Src = CGF.EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Src = CGF.EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(CGF); else Src = CGF.EmitScalarExpr(E->getArg(0)); SrcType = Src->getType(); @@ -21115,7 +21122,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_get: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); Value *Index = EmitScalarExpr(E->getArg(1)); Function *Callee; if (E->getType().isWebAssemblyExternrefType()) @@ -21129,7 +21136,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_set: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); Value *Index = EmitScalarExpr(E->getArg(1)); Value *Val = EmitScalarExpr(E->getArg(2)); Function *Callee; @@ -21144,13 +21151,13 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_size: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Value = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Value *Value = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_table_size); return Builder.CreateCall(Callee, Value); } case WebAssembly::BI__builtin_wasm_table_grow: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); Value *Val = EmitScalarExpr(E->getArg(1)); Value *NElems = EmitScalarExpr(E->getArg(2)); @@ -21167,7 +21174,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_fill: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); Value *Index = EmitScalarExpr(E->getArg(1)); Value *Val = EmitScalarExpr(E->getArg(2)); Value *NElems = EmitScalarExpr(E->getArg(3)); @@ -21185,8 +21192,8 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_copy: { assert(E->getArg(0)->getType()->isArrayType()); - Value *TableX = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); - Value *TableY = EmitArrayToPointerDecay(E->getArg(1)).getPointer(); + Value *TableX = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); + Value *TableY = EmitArrayToPointerDecay(E->getArg(1)).emitRawPointer(*this); Value *DstIdx = EmitScalarExpr(E->getArg(2)); Value *SrcIdx = EmitScalarExpr(E->getArg(3)); Value *NElems = EmitScalarExpr(E->getArg(4)); @@ -21265,7 +21272,7 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, auto MakeCircOp = [this, E](unsigned IntID, bool IsLoad) { // The base pointer is passed by address, so it needs to be loaded. Address A = EmitPointerWithAlignment(E->getArg(0)); - Address BP = Address(A.getPointer(), Int8PtrTy, A.getAlignment()); + Address BP = Address(A.emitRawPointer(*this), Int8PtrTy, A.getAlignment()); llvm::Value *Base = Builder.CreateLoad(BP); // The treatment of both loads and stores is the same: the arguments for // the builtin are the same as the arguments for the intrinsic. @@ -21306,8 +21313,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, // EmitPointerWithAlignment and EmitScalarExpr evaluates the expression // per call. Address DestAddr = EmitPointerWithAlignment(E->getArg(1)); - DestAddr = Address(DestAddr.getPointer(), Int8Ty, DestAddr.getAlignment()); - llvm::Value *DestAddress = DestAddr.getPointer(); + DestAddr = DestAddr.withElementType(Int8Ty); + llvm::Value *DestAddress = DestAddr.emitRawPointer(*this); // Operands are Base, Dest, Modifier. // The intrinsic format in LLVM IR is defined as @@ -21358,8 +21365,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn}); llvm::Value *PredOut = Builder.CreateExtractValue(Result, 1); - Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.getPointer(), - PredAddr.getAlignment()); + Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.emitRawPointer(*this), + PredAddr.getAlignment()); return Builder.CreateExtractValue(Result, 0); } // These are identical to the builtins above, except they don't consume @@ -21377,8 +21384,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))}); llvm::Value *PredOut = Builder.CreateExtractValue(Result, 1); - Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.getPointer(), - PredAddr.getAlignment()); + Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.emitRawPointer(*this), + PredAddr.getAlignment()); return Builder.CreateExtractValue(Result, 0); } diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp index b756318c46a900..0cb5b06a519c00 100644 --- a/clang/lib/CodeGen/CGCUDANV.cpp +++ b/clang/lib/CodeGen/CGCUDANV.cpp @@ -331,11 +331,11 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF, llvm::ConstantInt::get(SizeTy, std::max(1, Args.size()))); // Store pointers to the arguments in a locally allocated launch_args. for (unsigned i = 0; i < Args.size(); ++i) { - llvm::Value* VarPtr = CGF.GetAddrOfLocalVar(Args[i]).getPointer(); + llvm::Value *VarPtr = CGF.GetAddrOfLocalVar(Args[i]).emitRawPointer(CGF); llvm::Value *VoidVarPtr = CGF.Builder.CreatePointerCast(VarPtr, PtrTy); CGF.Builder.CreateDefaultAlignedStore( - VoidVarPtr, - CGF.Builder.CreateConstGEP1_32(PtrTy, KernelArgs.getPointer(), i)); + VoidVarPtr, CGF.Builder.CreateConstGEP1_32( + PtrTy, KernelArgs.emitRawPointer(CGF), i)); } llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end"); @@ -393,9 +393,10 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF, /*isVarArg=*/false), addUnderscoredPrefixToName("PopCallConfiguration")); - CGF.EmitRuntimeCallOrInvoke(cudaPopConfigFn, - {GridDim.getPointer(), BlockDim.getPointer(), - ShmemSize.getPointer(), Stream.getPointer()}); + CGF.EmitRuntimeCallOrInvoke(cudaPopConfigFn, {GridDim.emitRawPointer(CGF), + BlockDim.emitRawPointer(CGF), + ShmemSize.emitRawPointer(CGF), + Stream.emitRawPointer(CGF)}); // Emit the call to cudaLaunch llvm::Value *Kernel = @@ -405,7 +406,7 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF, cudaLaunchKernelFD->getParamDecl(0)->getType()); LaunchKernelArgs.add(RValue::getAggregate(GridDim), Dim3Ty); LaunchKernelArgs.add(RValue::getAggregate(BlockDim), Dim3Ty); - LaunchKernelArgs.add(RValue::get(KernelArgs.getPointer()), + LaunchKernelArgs.add(RValue::get(KernelArgs, CGF), cudaLaunchKernelFD->getParamDecl(3)->getType()); LaunchKernelArgs.add(RValue::get(CGF.Builder.CreateLoad(ShmemSize)), cudaLaunchKernelFD->getParamDecl(4)->getType()); @@ -438,8 +439,8 @@ void CGNVCUDARuntime::emitDeviceStubBodyLegacy(CodeGenFunction &CGF, auto TInfo = CGM.getContext().getTypeInfoInChars(A->getType()); Offset = Offset.alignTo(TInfo.Align); llvm::Value *Args[] = { - CGF.Builder.CreatePointerCast(CGF.GetAddrOfLocalVar(A).getPointer(), - PtrTy), + CGF.Builder.CreatePointerCast( + CGF.GetAddrOfLocalVar(A).emitRawPointer(CGF), PtrTy), llvm::ConstantInt::get(SizeTy, TInfo.Width.getQuantity()), llvm::ConstantInt::get(SizeTy, Offset.getQuantity()), }; diff --git a/clang/lib/CodeGen/CGCXXABI.cpp b/clang/lib/CodeGen/CGCXXABI.cpp index a8bf57a277e909..7c6dfc3e59d8c0 100644 --- a/clang/lib/CodeGen/CGCXXABI.cpp +++ b/clang/lib/CodeGen/CGCXXABI.cpp @@ -20,6 +20,12 @@ using namespace CodeGen; CGCXXABI::~CGCXXABI() { } +Address CGCXXABI::getThisAddress(CodeGenFunction &CGF) { + return CGF.makeNaturalAddressForPointer( + CGF.CXXABIThisValue, CGF.CXXABIThisDecl->getType()->getPointeeType(), + CGF.CXXABIThisAlignment); +} + void CGCXXABI::ErrorUnsupportedABI(CodeGenFunction &CGF, StringRef S) { DiagnosticsEngine &Diags = CGF.CGM.getDiags(); unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error, @@ -44,8 +50,12 @@ CGCallee CGCXXABI::EmitLoadOfMemberFunctionPointer( llvm::Value *MemPtr, const MemberPointerType *MPT) { ErrorUnsupportedABI(CGF, "calls through member pointers"); - ThisPtrForCall = This.getPointer(); - const auto *FPT = MPT->getPointeeType()->castAs(); + const auto *RD = + cast(MPT->getClass()->castAs()->getDecl()); + ThisPtrForCall = + CGF.getAsNaturalPointerTo(This, CGF.getContext().getRecordType(RD)); + const FunctionProtoType *FPT = + MPT->getPointeeType()->getAs(); llvm::Constant *FnPtr = llvm::Constant::getNullValue( llvm::PointerType::getUnqual(CGM.getLLVMContext())); return CGCallee::forDirect(FnPtr, FPT); @@ -251,16 +261,15 @@ void CGCXXABI::ReadArrayCookie(CodeGenFunction &CGF, Address ptr, // If we don't need an array cookie, bail out early. if (!requiresArrayCookie(expr, eltTy)) { - allocPtr = ptr.getPointer(); + allocPtr = ptr.emitRawPointer(CGF); numElements = nullptr; cookieSize = CharUnits::Zero(); return; } cookieSize = getArrayCookieSizeImpl(eltTy); - Address allocAddr = - CGF.Builder.CreateConstInBoundsByteGEP(ptr, -cookieSize); - allocPtr = allocAddr.getPointer(); + Address allocAddr = CGF.Builder.CreateConstInBoundsByteGEP(ptr, -cookieSize); + allocPtr = allocAddr.emitRawPointer(CGF); numElements = readArrayCookieImpl(CGF, allocAddr, cookieSize); } diff --git a/clang/lib/CodeGen/CGCXXABI.h b/clang/lib/CodeGen/CGCXXABI.h index ad1ad08d085688..c7eccbd0095a94 100644 --- a/clang/lib/CodeGen/CGCXXABI.h +++ b/clang/lib/CodeGen/CGCXXABI.h @@ -57,12 +57,8 @@ class CGCXXABI { llvm::Value *getThisValue(CodeGenFunction &CGF) { return CGF.CXXABIThisValue; } - Address getThisAddress(CodeGenFunction &CGF) { - return Address( - CGF.CXXABIThisValue, - CGF.ConvertTypeForMem(CGF.CXXABIThisDecl->getType()->getPointeeType()), - CGF.CXXABIThisAlignment); - } + + Address getThisAddress(CodeGenFunction &CGF); /// Issue a diagnostic about unsupported features in the ABI. void ErrorUnsupportedABI(CodeGenFunction &CGF, StringRef S); @@ -475,12 +471,6 @@ class CGCXXABI { BaseSubobject Base, const CXXRecordDecl *NearestVBase) = 0; - /// Get the address point of the vtable for the given base subobject while - /// building a constexpr. - virtual llvm::Constant * - getVTableAddressPointForConstExpr(BaseSubobject Base, - const CXXRecordDecl *VTableClass) = 0; - /// Get the address of the vtable for the given record decl which should be /// used for the vptr at the given offset in RD. virtual llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD, diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index b8adf5c26b3a35..fb0078214b07ff 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -1037,15 +1037,9 @@ static void forConstantArrayExpansion(CodeGenFunction &CGF, ConstantArrayExpansion *CAE, Address BaseAddr, llvm::function_ref Fn) { - CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy); - CharUnits EltAlign = - BaseAddr.getAlignment().alignmentOfArrayElement(EltSize); - llvm::Type *EltTy = CGF.ConvertTypeForMem(CAE->EltTy); - for (int i = 0, n = CAE->NumElts; i < n; i++) { - llvm::Value *EltAddr = CGF.Builder.CreateConstGEP2_32( - BaseAddr.getElementType(), BaseAddr.getPointer(), 0, i); - Fn(Address(EltAddr, EltTy, EltAlign)); + Address EltAddr = CGF.Builder.CreateConstGEP2_32(BaseAddr, 0, i); + Fn(EltAddr); } } @@ -1160,9 +1154,10 @@ void CodeGenFunction::ExpandTypeToArgs( } /// Create a temporary allocation for the purposes of coercion. -static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty, - CharUnits MinAlign, - const Twine &Name = "tmp") { +static RawAddress CreateTempAllocaForCoercion(CodeGenFunction &CGF, + llvm::Type *Ty, + CharUnits MinAlign, + const Twine &Name = "tmp") { // Don't use an alignment that's worse than what LLVM would prefer. auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty); CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign)); @@ -1332,11 +1327,11 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, } // Otherwise do coercion through memory. This is stupid, but simple. - Address Tmp = + RawAddress Tmp = CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName()); CGF.Builder.CreateMemCpy( - Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), Src.getPointer(), - Src.getAlignment().getAsAlign(), + Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), + Src.emitRawPointer(CGF), Src.getAlignment().getAsAlign(), llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinValue())); return CGF.Builder.CreateLoad(Tmp); } @@ -1420,11 +1415,12 @@ static void CreateCoercedStore(llvm::Value *Src, // // FIXME: Assert that we aren't truncating non-padding bits when have access // to that information. - Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment()); + RawAddress Tmp = + CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment()); CGF.Builder.CreateStore(Src, Tmp); CGF.Builder.CreateMemCpy( - Dst.getPointer(), Dst.getAlignment().getAsAlign(), Tmp.getPointer(), - Tmp.getAlignment().getAsAlign(), + Dst.emitRawPointer(CGF), Dst.getAlignment().getAsAlign(), + Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), llvm::ConstantInt::get(CGF.IntPtrTy, DstSize.getFixedValue())); } } @@ -3024,17 +3020,17 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, case ABIArgInfo::Indirect: case ABIArgInfo::IndirectAliased: { assert(NumIRArgs == 1); - Address ParamAddr = Address(Fn->getArg(FirstIRArg), ConvertTypeForMem(Ty), - ArgI.getIndirectAlign(), KnownNonNull); + Address ParamAddr = makeNaturalAddressForPointer( + Fn->getArg(FirstIRArg), Ty, ArgI.getIndirectAlign(), false, nullptr, + nullptr, KnownNonNull); if (!hasScalarEvaluationKind(Ty)) { // Aggregates and complex variables are accessed by reference. All we // need to do is realign the value, if requested. Also, if the address // may be aliased, copy it to ensure that the parameter variable is // mutable and has a unique adress, as C requires. - Address V = ParamAddr; if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) { - Address AlignedTemp = CreateMemTemp(Ty, "coerce"); + RawAddress AlignedTemp = CreateMemTemp(Ty, "coerce"); // Copy from the incoming argument pointer to the temporary with the // appropriate alignment. @@ -3044,11 +3040,12 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, CharUnits Size = getContext().getTypeSizeInChars(Ty); Builder.CreateMemCpy( AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(), - ParamAddr.getPointer(), ParamAddr.getAlignment().getAsAlign(), + ParamAddr.emitRawPointer(*this), + ParamAddr.getAlignment().getAsAlign(), llvm::ConstantInt::get(IntPtrTy, Size.getQuantity())); - V = AlignedTemp; + ParamAddr = AlignedTemp; } - ArgVals.push_back(ParamValue::forIndirect(V)); + ArgVals.push_back(ParamValue::forIndirect(ParamAddr)); } else { // Load scalar value from indirect argument. llvm::Value *V = @@ -3162,10 +3159,10 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, == ParameterABI::SwiftErrorResult) { QualType pointeeTy = Ty->getPointeeType(); assert(pointeeTy->isPointerType()); - Address temp = - CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp"); - Address arg(V, ConvertTypeForMem(pointeeTy), - getContext().getTypeAlignInChars(pointeeTy)); + RawAddress temp = + CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp"); + Address arg = makeNaturalAddressForPointer( + V, pointeeTy, getContext().getTypeAlignInChars(pointeeTy)); llvm::Value *incomingErrorValue = Builder.CreateLoad(arg); Builder.CreateStore(incomingErrorValue, temp); V = temp.getPointer(); @@ -3502,7 +3499,7 @@ static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF, llvm::LoadInst *load = dyn_cast(retainedValue->stripPointerCasts()); if (!load || load->isAtomic() || load->isVolatile() || - load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer()) + load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getBasePointer()) return nullptr; // Okay! Burn it all down. This relies for correctness on the @@ -3539,12 +3536,15 @@ static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF, /// Heuristically search for a dominating store to the return-value slot. static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { + llvm::Value *ReturnValuePtr = CGF.ReturnValue.getBasePointer(); + // Check if a User is a store which pointerOperand is the ReturnValue. // We are looking for stores to the ReturnValue, not for stores of the // ReturnValue to some other location. - auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * { + auto GetStoreIfValid = [&CGF, + ReturnValuePtr](llvm::User *U) -> llvm::StoreInst * { auto *SI = dyn_cast(U); - if (!SI || SI->getPointerOperand() != CGF.ReturnValue.getPointer() || + if (!SI || SI->getPointerOperand() != ReturnValuePtr || SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType()) return nullptr; // These aren't actually possible for non-coerced returns, and we @@ -3558,7 +3558,7 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { // for something immediately preceding the IP. Sometimes this can // happen with how we generate implicit-returns; it can also happen // with noreturn cleanups. - if (!CGF.ReturnValue.getPointer()->hasOneUse()) { + if (!ReturnValuePtr->hasOneUse()) { llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock(); if (IP->empty()) return nullptr; @@ -3576,8 +3576,7 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { return nullptr; } - llvm::StoreInst *store = - GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back()); + llvm::StoreInst *store = GetStoreIfValid(ReturnValuePtr->user_back()); if (!store) return nullptr; // Now do a first-and-dirty dominance check: just walk up the @@ -4121,7 +4120,11 @@ void CodeGenFunction::EmitDelegateCallArg(CallArgList &args, } static bool isProvablyNull(llvm::Value *addr) { - return isa(addr); + return llvm::isa_and_nonnull(addr); +} + +static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) { + return llvm::isKnownNonZero(Addr.getBasePointer(), CGF.CGM.getDataLayout()); } /// Emit the actual writing-back of a writeback. @@ -4129,21 +4132,20 @@ static void emitWriteback(CodeGenFunction &CGF, const CallArgList::Writeback &writeback) { const LValue &srcLV = writeback.Source; Address srcAddr = srcLV.getAddress(CGF); - assert(!isProvablyNull(srcAddr.getPointer()) && + assert(!isProvablyNull(srcAddr.getBasePointer()) && "shouldn't have writeback for provably null argument"); llvm::BasicBlock *contBB = nullptr; // If the argument wasn't provably non-null, we need to null check // before doing the store. - bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(), - CGF.CGM.getDataLayout()); + bool provablyNonNull = isProvablyNonNull(srcAddr, CGF); + if (!provablyNonNull) { llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback"); contBB = CGF.createBasicBlock("icr.done"); - llvm::Value *isNull = - CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull"); + llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull"); CGF.Builder.CreateCondBr(isNull, contBB, writebackBB); CGF.EmitBlock(writebackBB); } @@ -4247,7 +4249,7 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, CGF.ConvertTypeForMem(CRE->getType()->getPointeeType()); // If the address is a constant null, just pass the appropriate null. - if (isProvablyNull(srcAddr.getPointer())) { + if (isProvablyNull(srcAddr.getBasePointer())) { args.add(RValue::get(llvm::ConstantPointerNull::get(destType)), CRE->getType()); return; @@ -4276,17 +4278,16 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, // If the address is *not* known to be non-null, we need to switch. llvm::Value *finalArgument; - bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(), - CGF.CGM.getDataLayout()); + bool provablyNonNull = isProvablyNonNull(srcAddr, CGF); + if (provablyNonNull) { - finalArgument = temp.getPointer(); + finalArgument = temp.emitRawPointer(CGF); } else { - llvm::Value *isNull = - CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull"); + llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull"); - finalArgument = CGF.Builder.CreateSelect(isNull, - llvm::ConstantPointerNull::get(destType), - temp.getPointer(), "icr.argument"); + finalArgument = CGF.Builder.CreateSelect( + isNull, llvm::ConstantPointerNull::get(destType), + temp.emitRawPointer(CGF), "icr.argument"); // If we need to copy, then the load has to be conditional, which // means we need control flow. @@ -4410,6 +4411,16 @@ void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType, EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, std::nullopt); } +void CodeGenFunction::EmitNonNullArgCheck(Address Addr, QualType ArgType, + SourceLocation ArgLoc, + AbstractCallee AC, unsigned ParmNum) { + if (!AC.getDecl() || !(SanOpts.has(SanitizerKind::NonnullAttribute) || + SanOpts.has(SanitizerKind::NullabilityArg))) + return; + + EmitNonNullArgCheck(RValue::get(Addr, *this), ArgType, ArgLoc, AC, ParmNum); +} + // Check if the call is going to use the inalloca convention. This needs to // agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged // later, so we can't check it directly. @@ -4750,10 +4761,20 @@ CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) { llvm::CallInst * CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const llvm::Twine &name) { - return EmitNounwindRuntimeCall(callee, std::nullopt, name); + return EmitNounwindRuntimeCall(callee, ArrayRef(), name); } /// Emits a call to the given nounwind runtime function. +llvm::CallInst * +CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee, + ArrayRef
args, + const llvm::Twine &name) { + SmallVector values; + for (auto arg : args) + values.push_back(arg.emitRawPointer(*this)); + return EmitNounwindRuntimeCall(callee, values, name); +} + llvm::CallInst * CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee, ArrayRef args, @@ -5032,7 +5053,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // If we're using inalloca, insert the allocation after the stack save. // FIXME: Do this earlier rather than hacking it in here! - Address ArgMemory = Address::invalid(); + RawAddress ArgMemory = RawAddress::invalid(); if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) { const llvm::DataLayout &DL = CGM.getDataLayout(); llvm::Instruction *IP = CallArgs.getStackBase(); @@ -5048,7 +5069,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, AI->setAlignment(Align.getAsAlign()); AI->setUsedWithInAlloca(true); assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca()); - ArgMemory = Address(AI, ArgStruct, Align); + ArgMemory = RawAddress(AI, ArgStruct, Align); } ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo); @@ -5057,11 +5078,11 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // If the call returns a temporary with struct return, create a temporary // alloca to hold the result, unless one is given to us. Address SRetPtr = Address::invalid(); - Address SRetAlloca = Address::invalid(); + RawAddress SRetAlloca = RawAddress::invalid(); llvm::Value *UnusedReturnSizePtr = nullptr; if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) { if (!ReturnValue.isNull()) { - SRetPtr = ReturnValue.getValue(); + SRetPtr = ReturnValue.getAddress(); } else { SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca); if (HaveInsertPoint() && ReturnValue.isUnused()) { @@ -5071,15 +5092,16 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, } } if (IRFunctionArgs.hasSRetArg()) { - IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer(); + IRCallArgs[IRFunctionArgs.getSRetArgNo()] = + getAsNaturalPointerTo(SRetPtr, RetTy); } else if (RetAI.isInAlloca()) { Address Addr = Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex()); - Builder.CreateStore(SRetPtr.getPointer(), Addr); + Builder.CreateStore(getAsNaturalPointerTo(SRetPtr, RetTy), Addr); } } - Address swiftErrorTemp = Address::invalid(); + RawAddress swiftErrorTemp = RawAddress::invalid(); Address swiftErrorArg = Address::invalid(); // When passing arguments using temporary allocas, we need to add the @@ -5112,9 +5134,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, assert(NumIRArgs == 0); assert(getTarget().getTriple().getArch() == llvm::Triple::x86); if (I->isAggregate()) { - Address Addr = I->hasLValue() - ? I->getKnownLValue().getAddress(*this) - : I->getKnownRValue().getAggregateAddress(); + RawAddress Addr = I->hasLValue() + ? I->getKnownLValue().getAddress(*this) + : I->getKnownRValue().getAggregateAddress(); llvm::Instruction *Placeholder = cast(Addr.getPointer()); @@ -5138,7 +5160,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, } else if (ArgInfo.getInAllocaIndirect()) { // Make a temporary alloca and store the address of it into the argument // struct. - Address Addr = CreateMemTempWithoutCast( + RawAddress Addr = CreateMemTempWithoutCast( I->Ty, getContext().getTypeAlignInChars(I->Ty), "indirect-arg-temp"); I->copyInto(*this, Addr); @@ -5160,12 +5182,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, assert(NumIRArgs == 1); if (!I->isAggregate()) { // Make a temporary alloca to pass the argument. - Address Addr = CreateMemTempWithoutCast( + RawAddress Addr = CreateMemTempWithoutCast( I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp"); - llvm::Value *Val = Addr.getPointer(); + llvm::Value *Val = getAsNaturalPointerTo(Addr, I->Ty); if (ArgHasMaybeUndefAttr) - Val = Builder.CreateFreeze(Addr.getPointer()); + Val = Builder.CreateFreeze(Val); IRCallArgs[FirstIRArg] = Val; I->copyInto(*this, Addr); @@ -5181,7 +5203,6 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, Address Addr = I->hasLValue() ? I->getKnownLValue().getAddress(*this) : I->getKnownRValue().getAggregateAddress(); - llvm::Value *V = Addr.getPointer(); CharUnits Align = ArgInfo.getIndirectAlign(); const llvm::DataLayout *TD = &CGM.getDataLayout(); @@ -5192,8 +5213,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, bool NeedCopy = false; if (Addr.getAlignment() < Align && - llvm::getOrEnforceKnownAlignment(V, Align.getAsAlign(), *TD) < - Align.getAsAlign()) { + llvm::getOrEnforceKnownAlignment(Addr.emitRawPointer(*this), + Align.getAsAlign(), + *TD) < Align.getAsAlign()) { NeedCopy = true; } else if (I->hasLValue()) { auto LV = I->getKnownLValue(); @@ -5224,11 +5246,11 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, if (NeedCopy) { // Create an aligned temporary, and copy to it. - Address AI = CreateMemTempWithoutCast( + RawAddress AI = CreateMemTempWithoutCast( I->Ty, ArgInfo.getIndirectAlign(), "byval-temp"); - llvm::Value *Val = AI.getPointer(); + llvm::Value *Val = getAsNaturalPointerTo(AI, I->Ty); if (ArgHasMaybeUndefAttr) - Val = Builder.CreateFreeze(AI.getPointer()); + Val = Builder.CreateFreeze(Val); IRCallArgs[FirstIRArg] = Val; // Emit lifetime markers for the temporary alloca. @@ -5245,6 +5267,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, I->copyInto(*this, AI); } else { // Skip the extra memcpy call. + llvm::Value *V = getAsNaturalPointerTo(Addr, I->Ty); auto *T = llvm::PointerType::get( CGM.getLLVMContext(), CGM.getDataLayout().getAllocaAddrSpace()); @@ -5284,8 +5307,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, assert(!swiftErrorTemp.isValid() && "multiple swifterror args"); QualType pointeeTy = I->Ty->getPointeeType(); - swiftErrorArg = Address(V, ConvertTypeForMem(pointeeTy), - getContext().getTypeAlignInChars(pointeeTy)); + swiftErrorArg = makeNaturalAddressForPointer( + V, pointeeTy, getContext().getTypeAlignInChars(pointeeTy)); swiftErrorTemp = CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp"); @@ -5422,7 +5445,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, llvm::Value *tempSize = nullptr; Address addr = Address::invalid(); - Address AllocaAddr = Address::invalid(); + RawAddress AllocaAddr = RawAddress::invalid(); if (I->isAggregate()) { addr = I->hasLValue() ? I->getKnownLValue().getAddress(*this) : I->getKnownRValue().getAggregateAddress(); @@ -5856,7 +5879,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, return RValue::getComplex(std::make_pair(Real, Imag)); } case TEK_Aggregate: { - Address DestPtr = ReturnValue.getValue(); + Address DestPtr = ReturnValue.getAddress(); bool DestIsVolatile = ReturnValue.isVolatile(); if (!DestPtr.isValid()) { diff --git a/clang/lib/CodeGen/CGCall.h b/clang/lib/CodeGen/CGCall.h index 1bd48a07259307..6b676ac196db2a 100644 --- a/clang/lib/CodeGen/CGCall.h +++ b/clang/lib/CodeGen/CGCall.h @@ -377,6 +377,7 @@ class ReturnValueSlot { Address getValue() const { return Addr; } bool isUnused() const { return IsUnused; } bool isExternallyDestructed() const { return IsExternallyDestructed; } + Address getAddress() const { return Addr; } }; /// Adds attributes to \p F according to our \p CodeGenOpts and \p LangOpts, as diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp index 34319381901af6..8c1c8ee455d2e6 100644 --- a/clang/lib/CodeGen/CGClass.cpp +++ b/clang/lib/CodeGen/CGClass.cpp @@ -139,8 +139,9 @@ Address CodeGenFunction::LoadCXXThisAddress() { CXXThisAlignment = CGM.getClassPointerAlignment(MD->getParent()); } - llvm::Type *Ty = ConvertType(MD->getFunctionObjectParameterType()); - return Address(LoadCXXThis(), Ty, CXXThisAlignment, KnownNonNull); + return makeNaturalAddressForPointer( + LoadCXXThis(), MD->getFunctionObjectParameterType(), CXXThisAlignment, + false, nullptr, nullptr, KnownNonNull); } /// Emit the address of a field using a member data pointer. @@ -270,7 +271,7 @@ ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr, } // Apply the base offset. - llvm::Value *ptr = addr.getPointer(); + llvm::Value *ptr = addr.emitRawPointer(CGF); ptr = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ptr, baseOffset, "add.ptr"); // If we have a virtual component, the alignment of the result will @@ -338,8 +339,8 @@ Address CodeGenFunction::GetAddressOfBaseClass( if (sanitizePerformTypeCheck()) { SanitizerSet SkippedChecks; SkippedChecks.set(SanitizerKind::Null, !NullCheckValue); - EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(), - DerivedTy, DerivedAlign, SkippedChecks); + EmitTypeCheck(TCK_Upcast, Loc, Value.emitRawPointer(*this), DerivedTy, + DerivedAlign, SkippedChecks); } return Value.withElementType(BaseValueTy); } @@ -354,7 +355,7 @@ Address CodeGenFunction::GetAddressOfBaseClass( llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull"); endBB = createBasicBlock("cast.end"); - llvm::Value *isNull = Builder.CreateIsNull(Value.getPointer()); + llvm::Value *isNull = Builder.CreateIsNull(Value); Builder.CreateCondBr(isNull, endBB, notNullBB); EmitBlock(notNullBB); } @@ -363,14 +364,15 @@ Address CodeGenFunction::GetAddressOfBaseClass( SanitizerSet SkippedChecks; SkippedChecks.set(SanitizerKind::Null, true); EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc, - Value.getPointer(), DerivedTy, DerivedAlign, SkippedChecks); + Value.emitRawPointer(*this), DerivedTy, DerivedAlign, + SkippedChecks); } // Compute the virtual offset. llvm::Value *VirtualOffset = nullptr; if (VBase) { VirtualOffset = - CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase); + CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase); } // Apply both offsets. @@ -387,7 +389,7 @@ Address CodeGenFunction::GetAddressOfBaseClass( EmitBlock(endBB); llvm::PHINode *PHI = Builder.CreatePHI(PtrTy, 2, "cast.result"); - PHI->addIncoming(Value.getPointer(), notNullBB); + PHI->addIncoming(Value.emitRawPointer(*this), notNullBB); PHI->addIncoming(llvm::Constant::getNullValue(PtrTy), origBB); Value = Value.withPointer(PHI, NotKnownNonNull); } @@ -424,15 +426,19 @@ CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr, CastNotNull = createBasicBlock("cast.notnull"); CastEnd = createBasicBlock("cast.end"); - llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr.getPointer()); + llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr); Builder.CreateCondBr(IsNull, CastNull, CastNotNull); EmitBlock(CastNotNull); } // Apply the offset. - llvm::Value *Value = BaseAddr.getPointer(); - Value = Builder.CreateInBoundsGEP( - Int8Ty, Value, Builder.CreateNeg(NonVirtualOffset), "sub.ptr"); + Address Addr = BaseAddr.withElementType(Int8Ty); + Addr = Builder.CreateInBoundsGEP( + Addr, Builder.CreateNeg(NonVirtualOffset), Int8Ty, + CGM.getClassPointerAlignment(Derived), "sub.ptr"); + + // Just cast. + Addr = Addr.withElementType(DerivedValueTy); // Produce a PHI if we had a null-check. if (NullCheckValue) { @@ -441,13 +447,15 @@ CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr, Builder.CreateBr(CastEnd); EmitBlock(CastEnd); + llvm::Value *Value = Addr.emitRawPointer(*this); llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2); PHI->addIncoming(Value, CastNotNull); PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull); - Value = PHI; + return Address(PHI, Addr.getElementType(), + CGM.getClassPointerAlignment(Derived)); } - return Address(Value, DerivedValueTy, CGM.getClassPointerAlignment(Derived)); + return Addr; } llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD, @@ -1719,7 +1727,7 @@ namespace { // Use the base class declaration location as inline DebugLocation. All // fields of the class are destroyed. DeclAsInlineDebugLocation InlineHere(CGF, *BaseClass); - EmitSanitizerDtorFieldsCallback(CGF, Addr.getPointer(), + EmitSanitizerDtorFieldsCallback(CGF, Addr.emitRawPointer(CGF), BaseSize.getQuantity()); // Prevent the current stack frame from disappearing from the stack trace. @@ -2022,7 +2030,7 @@ void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor, // Find the end of the array. llvm::Type *elementType = arrayBase.getElementType(); - llvm::Value *arrayBegin = arrayBase.getPointer(); + llvm::Value *arrayBegin = arrayBase.emitRawPointer(*this); llvm::Value *arrayEnd = Builder.CreateInBoundsGEP( elementType, arrayBegin, numElements, "arrayctor.end"); @@ -2118,14 +2126,15 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, Address This = ThisAVS.getAddress(); LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace(); LangAS ThisAS = D->getFunctionObjectParameterType().getAddressSpace(); - llvm::Value *ThisPtr = This.getPointer(); + llvm::Value *ThisPtr = + getAsNaturalPointerTo(This, D->getThisType()->getPointeeType()); if (SlotAS != ThisAS) { unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS); llvm::Type *NewType = llvm::PointerType::get(getLLVMContext(), TargetThisAS); - ThisPtr = getTargetHooks().performAddrSpaceCast(*this, This.getPointer(), - ThisAS, SlotAS, NewType); + ThisPtr = getTargetHooks().performAddrSpaceCast(*this, ThisPtr, ThisAS, + SlotAS, NewType); } // Push the this ptr. @@ -2194,7 +2203,7 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, const CXXRecordDecl *ClassDecl = D->getParent(); if (!NewPointerIsChecked) - EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This.getPointer(), + EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This, getContext().getRecordType(ClassDecl), CharUnits::Zero()); if (D->isTrivial() && D->isDefaultConstructor()) { @@ -2207,10 +2216,9 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, // model that copy. if (isMemcpyEquivalentSpecialMember(D)) { assert(Args.size() == 2 && "unexpected argcount for trivial ctor"); - QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType(); - Address Src = Address(Args[1].getRValue(*this).getScalarVal(), ConvertTypeForMem(SrcTy), - CGM.getNaturalTypeAlignment(SrcTy)); + Address Src = makeNaturalAddressForPointer( + Args[1].getRValue(*this).getScalarVal(), SrcTy); LValue SrcLVal = MakeAddrLValue(Src, SrcTy); QualType DestTy = getContext().getTypeDeclType(ClassDecl); LValue DestLVal = MakeAddrLValue(This, DestTy); @@ -2263,7 +2271,9 @@ void CodeGenFunction::EmitInheritedCXXConstructorCall( const CXXConstructorDecl *D, bool ForVirtualBase, Address This, bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) { CallArgList Args; - CallArg ThisArg(RValue::get(This.getPointer()), D->getThisType()); + CallArg ThisArg(RValue::get(getAsNaturalPointerTo( + This, D->getThisType()->getPointeeType())), + D->getThisType()); // Forward the parameters. if (InheritedFromVBase && @@ -2388,12 +2398,14 @@ CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, CallArgList Args; // Push the this ptr. - Args.add(RValue::get(This.getPointer()), D->getThisType()); + Args.add(RValue::get(getAsNaturalPointerTo(This, D->getThisType())), + D->getThisType()); // Push the src ptr. QualType QT = *(FPT->param_type_begin()); llvm::Type *t = CGM.getTypes().ConvertType(QT); - llvm::Value *SrcVal = Builder.CreateBitCast(Src.getPointer(), t); + llvm::Value *Val = getAsNaturalPointerTo(Src, D->getThisType()); + llvm::Value *SrcVal = Builder.CreateBitCast(Val, t); Args.add(RValue::get(SrcVal), QT); // Skip over first argument (Src). @@ -2418,7 +2430,9 @@ CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor, // this Address This = LoadCXXThisAddress(); - DelegateArgs.add(RValue::get(This.getPointer()), (*I)->getType()); + DelegateArgs.add(RValue::get(getAsNaturalPointerTo( + This, (*I)->getType()->getPointeeType())), + (*I)->getType()); ++I; // FIXME: The location of the VTT parameter in the parameter list is @@ -2775,7 +2789,7 @@ void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, Address Derived, if (MayBeNull) { llvm::Value *DerivedNotNull = - Builder.CreateIsNotNull(Derived.getPointer(), "cast.nonnull"); + Builder.CreateIsNotNull(Derived.emitRawPointer(*this), "cast.nonnull"); llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check"); ContBlock = createBasicBlock("cast.cont"); @@ -2976,7 +2990,7 @@ void CodeGenFunction::EmitLambdaBlockInvokeBody() { QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda)); Address ThisPtr = GetAddrOfBlockDecl(variable); - CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType); + CallArgs.add(RValue::get(getAsNaturalPointerTo(ThisPtr, ThisType)), ThisType); // Add the rest of the parameters. for (auto *param : BD->parameters()) @@ -3004,7 +3018,7 @@ void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) { QualType LambdaType = getContext().getRecordType(Lambda); QualType ThisType = getContext().getPointerType(LambdaType); Address ThisPtr = CreateMemTemp(LambdaType, "unused.capture"); - CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType); + CallArgs.add(RValue::get(ThisPtr.emitRawPointer(*this)), ThisType); EmitLambdaDelegatingInvokeBody(MD, CallArgs); } diff --git a/clang/lib/CodeGen/CGCleanup.cpp b/clang/lib/CodeGen/CGCleanup.cpp index f87caf050eeaa7..e6f8e6873004f2 100644 --- a/clang/lib/CodeGen/CGCleanup.cpp +++ b/clang/lib/CodeGen/CGCleanup.cpp @@ -27,7 +27,7 @@ bool DominatingValue::saved_type::needsSaving(RValue rv) { if (rv.isScalar()) return DominatingLLVMValue::needsSaving(rv.getScalarVal()); if (rv.isAggregate()) - return DominatingLLVMValue::needsSaving(rv.getAggregatePointer()); + return DominatingValue
::needsSaving(rv.getAggregateAddress()); return true; } @@ -35,69 +35,40 @@ DominatingValue::saved_type DominatingValue::saved_type::save(CodeGenFunction &CGF, RValue rv) { if (rv.isScalar()) { llvm::Value *V = rv.getScalarVal(); - - // These automatically dominate and don't need to be saved. - if (!DominatingLLVMValue::needsSaving(V)) - return saved_type(V, nullptr, ScalarLiteral); - - // Everything else needs an alloca. - Address addr = - CGF.CreateDefaultAlignTempAlloca(V->getType(), "saved-rvalue"); - CGF.Builder.CreateStore(V, addr); - return saved_type(addr.getPointer(), nullptr, ScalarAddress); + return saved_type(DominatingLLVMValue::save(CGF, V), + DominatingLLVMValue::needsSaving(V) ? ScalarAddress + : ScalarLiteral); } if (rv.isComplex()) { CodeGenFunction::ComplexPairTy V = rv.getComplexVal(); - llvm::Type *ComplexTy = - llvm::StructType::get(V.first->getType(), V.second->getType()); - Address addr = CGF.CreateDefaultAlignTempAlloca(ComplexTy, "saved-complex"); - CGF.Builder.CreateStore(V.first, CGF.Builder.CreateStructGEP(addr, 0)); - CGF.Builder.CreateStore(V.second, CGF.Builder.CreateStructGEP(addr, 1)); - return saved_type(addr.getPointer(), nullptr, ComplexAddress); + return saved_type(DominatingLLVMValue::save(CGF, V.first), + DominatingLLVMValue::save(CGF, V.second)); } assert(rv.isAggregate()); - Address V = rv.getAggregateAddress(); // TODO: volatile? - if (!DominatingLLVMValue::needsSaving(V.getPointer())) - return saved_type(V.getPointer(), V.getElementType(), AggregateLiteral, - V.getAlignment().getQuantity()); - - Address addr = - CGF.CreateTempAlloca(V.getType(), CGF.getPointerAlign(), "saved-rvalue"); - CGF.Builder.CreateStore(V.getPointer(), addr); - return saved_type(addr.getPointer(), V.getElementType(), AggregateAddress, - V.getAlignment().getQuantity()); + Address V = rv.getAggregateAddress(); + return saved_type( + DominatingValue
::save(CGF, V), rv.isVolatileQualified(), + DominatingValue
::needsSaving(V) ? AggregateAddress + : AggregateLiteral); } /// Given a saved r-value produced by SaveRValue, perform the code /// necessary to restore it to usability at the current insertion /// point. RValue DominatingValue::saved_type::restore(CodeGenFunction &CGF) { - auto getSavingAddress = [&](llvm::Value *value) { - auto *AI = cast(value); - return Address(value, AI->getAllocatedType(), - CharUnits::fromQuantity(AI->getAlign().value())); - }; switch (K) { case ScalarLiteral: - return RValue::get(Value); case ScalarAddress: - return RValue::get(CGF.Builder.CreateLoad(getSavingAddress(Value))); + return RValue::get(DominatingLLVMValue::restore(CGF, Vals.first)); case AggregateLiteral: + case AggregateAddress: return RValue::getAggregate( - Address(Value, ElementType, CharUnits::fromQuantity(Align))); - case AggregateAddress: { - auto addr = CGF.Builder.CreateLoad(getSavingAddress(Value)); - return RValue::getAggregate( - Address(addr, ElementType, CharUnits::fromQuantity(Align))); - } + DominatingValue
::restore(CGF, AggregateAddr), IsVolatile); case ComplexAddress: { - Address address = getSavingAddress(Value); - llvm::Value *real = - CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(address, 0)); - llvm::Value *imag = - CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(address, 1)); + llvm::Value *real = DominatingLLVMValue::restore(CGF, Vals.first); + llvm::Value *imag = DominatingLLVMValue::restore(CGF, Vals.second); return RValue::getComplex(real, imag); } } @@ -294,14 +265,14 @@ void EHScopeStack::popNullFixups() { BranchFixups.pop_back(); } -Address CodeGenFunction::createCleanupActiveFlag() { +RawAddress CodeGenFunction::createCleanupActiveFlag() { // Create a variable to decide whether the cleanup needs to be run. - Address active = CreateTempAllocaWithoutCast( + RawAddress active = CreateTempAllocaWithoutCast( Builder.getInt1Ty(), CharUnits::One(), "cleanup.cond"); // Initialize it to false at a site that's guaranteed to be run // before each evaluation. - setBeforeOutermostConditional(Builder.getFalse(), active); + setBeforeOutermostConditional(Builder.getFalse(), active, *this); // Initialize it to true at the current location. Builder.CreateStore(Builder.getTrue(), active); @@ -309,7 +280,7 @@ Address CodeGenFunction::createCleanupActiveFlag() { return active; } -void CodeGenFunction::initFullExprCleanupWithFlag(Address ActiveFlag) { +void CodeGenFunction::initFullExprCleanupWithFlag(RawAddress ActiveFlag) { // Set that as the active flag in the cleanup. EHCleanupScope &cleanup = cast(*EHStack.begin()); assert(!cleanup.hasActiveFlag() && "cleanup already has active flag?"); @@ -322,15 +293,17 @@ void CodeGenFunction::initFullExprCleanupWithFlag(Address ActiveFlag) { void EHScopeStack::Cleanup::anchor() {} static void createStoreInstBefore(llvm::Value *value, Address addr, - llvm::Instruction *beforeInst) { - auto store = new llvm::StoreInst(value, addr.getPointer(), beforeInst); + llvm::Instruction *beforeInst, + CodeGenFunction &CGF) { + auto store = new llvm::StoreInst(value, addr.emitRawPointer(CGF), beforeInst); store->setAlignment(addr.getAlignment().getAsAlign()); } static llvm::LoadInst *createLoadInstBefore(Address addr, const Twine &name, - llvm::Instruction *beforeInst) { - return new llvm::LoadInst(addr.getElementType(), addr.getPointer(), name, - false, addr.getAlignment().getAsAlign(), + llvm::Instruction *beforeInst, + CodeGenFunction &CGF) { + return new llvm::LoadInst(addr.getElementType(), addr.emitRawPointer(CGF), + name, false, addr.getAlignment().getAsAlign(), beforeInst); } @@ -357,8 +330,8 @@ static void ResolveAllBranchFixups(CodeGenFunction &CGF, // entry which we're currently popping. if (Fixup.OptimisticBranchBlock == nullptr) { createStoreInstBefore(CGF.Builder.getInt32(Fixup.DestinationIndex), - CGF.getNormalCleanupDestSlot(), - Fixup.InitialBranch); + CGF.getNormalCleanupDestSlot(), Fixup.InitialBranch, + CGF); Fixup.InitialBranch->setSuccessor(0, CleanupEntry); } @@ -385,7 +358,7 @@ static llvm::SwitchInst *TransitionToCleanupSwitch(CodeGenFunction &CGF, if (llvm::BranchInst *Br = dyn_cast(Term)) { assert(Br->isUnconditional()); auto Load = createLoadInstBefore(CGF.getNormalCleanupDestSlot(), - "cleanup.dest", Term); + "cleanup.dest", Term, CGF); llvm::SwitchInst *Switch = llvm::SwitchInst::Create(Load, Br->getSuccessor(0), 4, Block); Br->eraseFromParent(); @@ -513,8 +486,8 @@ void CodeGenFunction::PopCleanupBlocks( I += Header.getSize(); if (Header.isConditional()) { - Address ActiveFlag = - reinterpret_cast
(LifetimeExtendedCleanupStack[I]); + RawAddress ActiveFlag = + reinterpret_cast(LifetimeExtendedCleanupStack[I]); initFullExprCleanupWithFlag(ActiveFlag); I += sizeof(ActiveFlag); } @@ -888,7 +861,7 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { if (NormalCleanupDestSlot->hasOneUse()) { NormalCleanupDestSlot->user_back()->eraseFromParent(); NormalCleanupDestSlot->eraseFromParent(); - NormalCleanupDest = Address::invalid(); + NormalCleanupDest = RawAddress::invalid(); } llvm::BasicBlock *BranchAfter = Scope.getBranchAfterBlock(0); @@ -912,9 +885,8 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { // pass the abnormal exit flag to Fn (SEH cleanup) cleanupFlags.setHasExitSwitch(); - llvm::LoadInst *Load = - createLoadInstBefore(getNormalCleanupDestSlot(), "cleanup.dest", - nullptr); + llvm::LoadInst *Load = createLoadInstBefore( + getNormalCleanupDestSlot(), "cleanup.dest", nullptr, *this); llvm::SwitchInst *Switch = llvm::SwitchInst::Create(Load, Default, SwitchCapacity); @@ -961,8 +933,8 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { if (!Fixup.Destination) continue; if (!Fixup.OptimisticBranchBlock) { createStoreInstBefore(Builder.getInt32(Fixup.DestinationIndex), - getNormalCleanupDestSlot(), - Fixup.InitialBranch); + getNormalCleanupDestSlot(), Fixup.InitialBranch, + *this); Fixup.InitialBranch->setSuccessor(0, NormalEntry); } Fixup.OptimisticBranchBlock = NormalExit; @@ -1135,7 +1107,7 @@ void CodeGenFunction::EmitBranchThroughCleanup(JumpDest Dest) { // Store the index at the start. llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex()); - createStoreInstBefore(Index, getNormalCleanupDestSlot(), BI); + createStoreInstBefore(Index, getNormalCleanupDestSlot(), BI, *this); // Adjust BI to point to the first cleanup block. { @@ -1269,9 +1241,9 @@ static void SetupCleanupBlockActivation(CodeGenFunction &CGF, // If we're in a conditional block, ignore the dominating IP and // use the outermost conditional branch. if (CGF.isInConditionalBranch()) { - CGF.setBeforeOutermostConditional(value, var); + CGF.setBeforeOutermostConditional(value, var, CGF); } else { - createStoreInstBefore(value, var, dominatingIP); + createStoreInstBefore(value, var, dominatingIP, CGF); } } @@ -1321,7 +1293,7 @@ void CodeGenFunction::DeactivateCleanupBlock(EHScopeStack::stable_iterator C, Scope.setActive(false); } -Address CodeGenFunction::getNormalCleanupDestSlot() { +RawAddress CodeGenFunction::getNormalCleanupDestSlot() { if (!NormalCleanupDest.isValid()) NormalCleanupDest = CreateDefaultAlignTempAlloca(Builder.getInt32Ty(), "cleanup.dest.slot"); diff --git a/clang/lib/CodeGen/CGCleanup.h b/clang/lib/CodeGen/CGCleanup.h index 7a7344c07160db..03e4a29d7b3dbf 100644 --- a/clang/lib/CodeGen/CGCleanup.h +++ b/clang/lib/CodeGen/CGCleanup.h @@ -333,7 +333,7 @@ class alignas(8) EHCleanupScope : public EHScope { Address getActiveFlag() const { return ActiveFlag; } - void setActiveFlag(Address Var) { + void setActiveFlag(RawAddress Var) { assert(Var.getAlignment().isOne()); ActiveFlag = Var; } diff --git a/clang/lib/CodeGen/CGCoroutine.cpp b/clang/lib/CodeGen/CGCoroutine.cpp index b7142ec08af986..93ca711f716fce 100644 --- a/clang/lib/CodeGen/CGCoroutine.cpp +++ b/clang/lib/CodeGen/CGCoroutine.cpp @@ -867,8 +867,8 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) { EmitStmt(S.getPromiseDeclStmt()); Address PromiseAddr = GetAddrOfLocalVar(S.getPromiseDecl()); - auto *PromiseAddrVoidPtr = - new llvm::BitCastInst(PromiseAddr.getPointer(), VoidPtrTy, "", CoroId); + auto *PromiseAddrVoidPtr = new llvm::BitCastInst( + PromiseAddr.emitRawPointer(*this), VoidPtrTy, "", CoroId); // Update CoroId to refer to the promise. We could not do it earlier because // promise local variable was not emitted yet. CoroId->setArgOperand(1, PromiseAddrVoidPtr); diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp index 2ef5ed04af30b6..267f2e40a7bbaa 100644 --- a/clang/lib/CodeGen/CGDecl.cpp +++ b/clang/lib/CodeGen/CGDecl.cpp @@ -1461,7 +1461,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { bool EmitDebugInfo = DI && CGM.getCodeGenOpts().hasReducedDebugInfo(); Address address = Address::invalid(); - Address AllocaAddr = Address::invalid(); + RawAddress AllocaAddr = RawAddress::invalid(); Address OpenMPLocalAddr = Address::invalid(); if (CGM.getLangOpts().OpenMPIRBuilder) OpenMPLocalAddr = OMPBuilderCBHelpers::getAddressOfLocalVariable(*this, &D); @@ -1524,7 +1524,10 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { // return slot, so that we can elide the copy when returning this // variable (C++0x [class.copy]p34). address = ReturnValue; - AllocaAddr = ReturnValue; + AllocaAddr = + RawAddress(ReturnValue.emitRawPointer(*this), + ReturnValue.getElementType(), ReturnValue.getAlignment()); + ; if (const RecordType *RecordTy = Ty->getAs()) { const auto *RD = RecordTy->getDecl(); @@ -1535,7 +1538,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { // to this variable. Set it to zero to indicate that NRVO was not // applied. llvm::Value *Zero = Builder.getFalse(); - Address NRVOFlag = + RawAddress NRVOFlag = CreateTempAlloca(Zero->getType(), CharUnits::One(), "nrvo"); EnsureInsertPoint(); Builder.CreateStore(Zero, NRVOFlag); @@ -1678,7 +1681,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { } if (D.hasAttr() && HaveInsertPoint()) - EmitVarAnnotations(&D, address.getPointer()); + EmitVarAnnotations(&D, address.emitRawPointer(*this)); // Make sure we call @llvm.lifetime.end. if (emission.useLifetimeMarkers()) @@ -1851,12 +1854,13 @@ void CodeGenFunction::emitZeroOrPatternForAutoVarInit(QualType type, llvm::Value *BaseSizeInChars = llvm::ConstantInt::get(IntPtrTy, EltSize.getQuantity()); Address Begin = Loc.withElementType(Int8Ty); - llvm::Value *End = Builder.CreateInBoundsGEP( - Begin.getElementType(), Begin.getPointer(), SizeVal, "vla.end"); + llvm::Value *End = Builder.CreateInBoundsGEP(Begin.getElementType(), + Begin.emitRawPointer(*this), + SizeVal, "vla.end"); llvm::BasicBlock *OriginBB = Builder.GetInsertBlock(); EmitBlock(LoopBB); llvm::PHINode *Cur = Builder.CreatePHI(Begin.getType(), 2, "vla.cur"); - Cur->addIncoming(Begin.getPointer(), OriginBB); + Cur->addIncoming(Begin.emitRawPointer(*this), OriginBB); CharUnits CurAlign = Loc.getAlignment().alignmentOfArrayElement(EltSize); auto *I = Builder.CreateMemCpy(Address(Cur, Int8Ty, CurAlign), @@ -2283,7 +2287,7 @@ void CodeGenFunction::emitDestroy(Address addr, QualType type, checkZeroLength = false; } - llvm::Value *begin = addr.getPointer(); + llvm::Value *begin = addr.emitRawPointer(*this); llvm::Value *end = Builder.CreateInBoundsGEP(addr.getElementType(), begin, length); emitArrayDestroy(begin, end, type, elementAlign, destroyer, @@ -2543,7 +2547,7 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg, } Address DeclPtr = Address::invalid(); - Address AllocaPtr = Address::invalid(); + RawAddress AllocaPtr = Address::invalid(); bool DoStore = false; bool IsScalar = hasScalarEvaluationKind(Ty); bool UseIndirectDebugAddress = false; @@ -2555,8 +2559,8 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg, // Indirect argument is in alloca address space, which may be different // from the default address space. auto AllocaAS = CGM.getASTAllocaAddressSpace(); - auto *V = DeclPtr.getPointer(); - AllocaPtr = DeclPtr; + auto *V = DeclPtr.emitRawPointer(*this); + AllocaPtr = RawAddress(V, DeclPtr.getElementType(), DeclPtr.getAlignment()); // For truly ABI indirect arguments -- those that are not `byval` -- store // the address of the argument on the stack to preserve debug information. @@ -2695,7 +2699,7 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg, } if (D.hasAttr()) - EmitVarAnnotations(&D, DeclPtr.getPointer()); + EmitVarAnnotations(&D, DeclPtr.emitRawPointer(*this)); // We can only check return value nullability if all arguments to the // function satisfy their nullability preconditions. This makes it necessary diff --git a/clang/lib/CodeGen/CGException.cpp b/clang/lib/CodeGen/CGException.cpp index 5a9d06da12de57..34f289334a7df9 100644 --- a/clang/lib/CodeGen/CGException.cpp +++ b/clang/lib/CodeGen/CGException.cpp @@ -397,7 +397,7 @@ namespace { void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) { // Make sure the exception object is cleaned up if there's an // exception during initialization. - pushFullExprCleanup(EHCleanup, addr.getPointer()); + pushFullExprCleanup(EHCleanup, addr.emitRawPointer(*this)); EHScopeStack::stable_iterator cleanup = EHStack.stable_begin(); // __cxa_allocate_exception returns a void*; we need to cast this @@ -416,8 +416,8 @@ void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) { /*IsInit*/ true); // Deactivate the cleanup block. - DeactivateCleanupBlock(cleanup, - cast(typedAddr.getPointer())); + DeactivateCleanupBlock( + cleanup, cast(typedAddr.emitRawPointer(*this))); } Address CodeGenFunction::getExceptionSlot() { @@ -1834,7 +1834,8 @@ Address CodeGenFunction::recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF, llvm::Value *ParentFP) { llvm::CallInst *RecoverCall = nullptr; CGBuilderTy Builder(*this, AllocaInsertPt); - if (auto *ParentAlloca = dyn_cast(ParentVar.getPointer())) { + if (auto *ParentAlloca = + dyn_cast_or_null(ParentVar.getBasePointer())) { // Mark the variable escaped if nobody else referenced it and compute the // localescape index. auto InsertPair = ParentCGF.EscapedLocals.insert( @@ -1851,8 +1852,8 @@ Address CodeGenFunction::recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF, // If the parent didn't have an alloca, we're doing some nested outlining. // Just clone the existing localrecover call, but tweak the FP argument to // use our FP value. All other arguments are constants. - auto *ParentRecover = - cast(ParentVar.getPointer()->stripPointerCasts()); + auto *ParentRecover = cast( + ParentVar.emitRawPointer(*this)->stripPointerCasts()); assert(ParentRecover->getIntrinsicID() == llvm::Intrinsic::localrecover && "expected alloca or localrecover in parent LocalDeclMap"); RecoverCall = cast(ParentRecover->clone()); @@ -1925,7 +1926,8 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF, if (isa(D) && D->getType() == getContext().VoidPtrTy) { assert(D->getName().starts_with("frame_pointer")); - FramePtrAddrAlloca = cast(I.second.getPointer()); + FramePtrAddrAlloca = + cast(I.second.getBasePointer()); break; } } @@ -1986,7 +1988,8 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF, LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField); if (!LambdaThisCaptureField->getType()->isPointerType()) { - CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer(); + CXXThisValue = + ThisFieldLValue.getAddress(*this).emitRawPointer(*this); } else { CXXThisValue = EmitLoadOfLValue(ThisFieldLValue, SourceLocation()) .getScalarVal(); diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp index 6491835cf8ef17..36872c0fedb76e 100644 --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -65,21 +65,21 @@ static llvm::cl::opt ClSanitizeDebugDeoptimization( /// CreateTempAlloca - This creates a alloca and inserts it into the entry /// block. -Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, - CharUnits Align, - const Twine &Name, - llvm::Value *ArraySize) { +RawAddress +CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits Align, + const Twine &Name, + llvm::Value *ArraySize) { auto Alloca = CreateTempAlloca(Ty, Name, ArraySize); Alloca->setAlignment(Align.getAsAlign()); - return Address(Alloca, Ty, Align, KnownNonNull); + return RawAddress(Alloca, Ty, Align, KnownNonNull); } /// CreateTempAlloca - This creates a alloca and inserts it into the entry /// block. The alloca is casted to default address space if necessary. -Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, - const Twine &Name, - llvm::Value *ArraySize, - Address *AllocaAddr) { +RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, + const Twine &Name, + llvm::Value *ArraySize, + RawAddress *AllocaAddr) { auto Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize); if (AllocaAddr) *AllocaAddr = Alloca; @@ -101,7 +101,7 @@ Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, Ty->getPointerTo(DestAddrSpace), /*non-null*/ true); } - return Address(V, Ty, Align, KnownNonNull); + return RawAddress(V, Ty, Align, KnownNonNull); } /// CreateTempAlloca - This creates an alloca and inserts it into the entry @@ -120,28 +120,29 @@ llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, /// default alignment of the corresponding LLVM type, which is *not* /// guaranteed to be related in any way to the expected alignment of /// an AST type that might have been lowered to Ty. -Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty, - const Twine &Name) { +RawAddress CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty, + const Twine &Name) { CharUnits Align = CharUnits::fromQuantity(CGM.getDataLayout().getPrefTypeAlign(Ty)); return CreateTempAlloca(Ty, Align, Name); } -Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) { +RawAddress CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) { CharUnits Align = getContext().getTypeAlignInChars(Ty); return CreateTempAlloca(ConvertType(Ty), Align, Name); } -Address CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name, - Address *Alloca) { +RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name, + RawAddress *Alloca) { // FIXME: Should we prefer the preferred type alignment here? return CreateMemTemp(Ty, getContext().getTypeAlignInChars(Ty), Name, Alloca); } -Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align, - const Twine &Name, Address *Alloca) { - Address Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name, - /*ArraySize=*/nullptr, Alloca); +RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align, + const Twine &Name, + RawAddress *Alloca) { + RawAddress Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name, + /*ArraySize=*/nullptr, Alloca); if (Ty->isConstantMatrixType()) { auto *ArrayTy = cast(Result.getElementType()); @@ -154,13 +155,14 @@ Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align, return Result; } -Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, CharUnits Align, - const Twine &Name) { +RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, + CharUnits Align, + const Twine &Name) { return CreateTempAllocaWithoutCast(ConvertTypeForMem(Ty), Align, Name); } -Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, - const Twine &Name) { +RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, + const Twine &Name) { return CreateMemTempWithoutCast(Ty, getContext().getTypeAlignInChars(Ty), Name); } @@ -359,7 +361,7 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, } else { CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor( GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete)); - CleanupArg = cast(ReferenceTemporary.getPointer()); + CleanupArg = cast(ReferenceTemporary.emitRawPointer(CGF)); } CGF.CGM.getCXXABI().registerGlobalDtor( CGF, *cast(M->getExtendingDecl()), CleanupFn, CleanupArg); @@ -384,10 +386,10 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, } } -static Address createReferenceTemporary(CodeGenFunction &CGF, - const MaterializeTemporaryExpr *M, - const Expr *Inner, - Address *Alloca = nullptr) { +static RawAddress createReferenceTemporary(CodeGenFunction &CGF, + const MaterializeTemporaryExpr *M, + const Expr *Inner, + RawAddress *Alloca = nullptr) { auto &TCG = CGF.getTargetHooks(); switch (M->getStorageDuration()) { case SD_FullExpression: @@ -416,7 +418,7 @@ static Address createReferenceTemporary(CodeGenFunction &CGF, GV->getValueType()->getPointerTo( CGF.getContext().getTargetAddressSpace(LangAS::Default))); // FIXME: Should we put the new global into a COMDAT? - return Address(C, GV->getValueType(), alignment); + return RawAddress(C, GV->getValueType(), alignment); } return CGF.CreateMemTemp(Ty, "ref.tmp", Alloca); } @@ -448,7 +450,7 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { auto ownership = M->getType().getObjCLifetime(); if (ownership != Qualifiers::OCL_None && ownership != Qualifiers::OCL_ExplicitNone) { - Address Object = createReferenceTemporary(*this, M, E); + RawAddress Object = createReferenceTemporary(*this, M, E); if (auto *Var = dyn_cast(Object.getPointer())) { llvm::Type *Ty = ConvertTypeForMem(E->getType()); Object = Object.withElementType(Ty); @@ -502,8 +504,8 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { } // Create and initialize the reference temporary. - Address Alloca = Address::invalid(); - Address Object = createReferenceTemporary(*this, M, E, &Alloca); + RawAddress Alloca = Address::invalid(); + RawAddress Object = createReferenceTemporary(*this, M, E, &Alloca); if (auto *Var = dyn_cast( Object.getPointer()->stripPointerCasts())) { llvm::Type *TemporaryType = ConvertTypeForMem(E->getType()); @@ -1111,12 +1113,12 @@ llvm::Value *CodeGenFunction::EmitCountedByFieldExpr( } else if (const MemberExpr *ME = dyn_cast(StructBase)) { LValue LV = EmitMemberExpr(ME); Address Addr = LV.getAddress(*this); - Res = Addr.getPointer(); + Res = Addr.emitRawPointer(*this); } else if (StructBase->getType()->isPointerType()) { LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; Address Addr = EmitPointerWithAlignment(StructBase, &BaseInfo, &TBAAInfo); - Res = Addr.getPointer(); + Res = Addr.emitRawPointer(*this); } else { return nullptr; } @@ -1282,8 +1284,7 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) { if (BaseInfo) BaseInfo->mergeForCast(TargetTypeBaseInfo); - Addr = Address(Addr.getPointer(), Addr.getElementType(), Align, - IsKnownNonNull); + Addr.setAlignment(Align); } } @@ -1300,8 +1301,8 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, CGF.ConvertTypeForMem(E->getType()->getPointeeType()); Addr = Addr.withElementType(ElemTy); if (CE->getCastKind() == CK_AddressSpaceConversion) - Addr = CGF.Builder.CreateAddrSpaceCast(Addr, - CGF.ConvertType(E->getType())); + Addr = CGF.Builder.CreateAddrSpaceCast( + Addr, CGF.ConvertType(E->getType()), ElemTy); return Addr; } break; @@ -1364,10 +1365,9 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, // TODO: conditional operators, comma. // Otherwise, use the alignment of the type. - CharUnits Align = - CGF.CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo); - llvm::Type *ElemTy = CGF.ConvertTypeForMem(E->getType()->getPointeeType()); - return Address(CGF.EmitScalarExpr(E), ElemTy, Align, IsKnownNonNull); + return CGF.makeNaturalAddressForPointer( + CGF.EmitScalarExpr(E), E->getType()->getPointeeType(), CharUnits(), + /*ForPointeeType=*/true, BaseInfo, TBAAInfo, IsKnownNonNull); } /// EmitPointerWithAlignment - Given an expression of pointer type, try to @@ -1468,8 +1468,7 @@ LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) { if (IsBaseCXXThis || isa(ME->getBase())) SkippedChecks.set(SanitizerKind::Null, true); } - EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(*this), E->getType(), - LV.getAlignment(), SkippedChecks); + EmitTypeCheck(TCK, E->getExprLoc(), LV, E->getType(), SkippedChecks); } return LV; } @@ -1581,11 +1580,11 @@ LValue CodeGenFunction::EmitLValueHelper(const Expr *E, // Defend against branches out of gnu statement expressions surrounded by // cleanups. Address Addr = LV.getAddress(*this); - llvm::Value *V = Addr.getPointer(); + llvm::Value *V = Addr.getBasePointer(); Scope.ForceCleanup({&V}); - return LValue::MakeAddr(Addr.withPointer(V, Addr.isKnownNonNull()), - LV.getType(), getContext(), LV.getBaseInfo(), - LV.getTBAAInfo()); + Addr.replaceBasePointer(V); + return LValue::MakeAddr(Addr, LV.getType(), getContext(), + LV.getBaseInfo(), LV.getTBAAInfo()); } // FIXME: Is it possible to create an ExprWithCleanups that produces a // bitfield lvalue or some other non-simple lvalue? @@ -1929,7 +1928,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo, bool isNontemporal) { - if (auto *GV = dyn_cast(Addr.getPointer())) + if (auto *GV = dyn_cast(Addr.getBasePointer())) if (GV->isThreadLocal()) Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV), NotKnownNonNull); @@ -2039,8 +2038,9 @@ llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) { // Convert the pointer of \p Addr to a pointer to a vector (the value type of // MatrixType), if it points to a array (the memory type of MatrixType). -static Address MaybeConvertMatrixAddress(Address Addr, CodeGenFunction &CGF, - bool IsVector = true) { +static RawAddress MaybeConvertMatrixAddress(RawAddress Addr, + CodeGenFunction &CGF, + bool IsVector = true) { auto *ArrayTy = dyn_cast(Addr.getElementType()); if (ArrayTy && IsVector) { auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(), @@ -2077,7 +2077,7 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo, bool isInit, bool isNontemporal) { - if (auto *GV = dyn_cast(Addr.getPointer())) + if (auto *GV = dyn_cast(Addr.getBasePointer())) if (GV->isThreadLocal()) Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV), NotKnownNonNull); @@ -2432,14 +2432,12 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL"); llvm::Type *ResultType = IntPtrTy; Address dst = EmitPointerWithAlignment(Dst.getBaseIvarExp()); - llvm::Value *RHS = dst.getPointer(); + llvm::Value *RHS = dst.emitRawPointer(*this); RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast"); - llvm::Value *LHS = - Builder.CreatePtrToInt(LvalueDst.getPointer(), ResultType, - "sub.ptr.lhs.cast"); + llvm::Value *LHS = Builder.CreatePtrToInt(LvalueDst.emitRawPointer(*this), + ResultType, "sub.ptr.lhs.cast"); llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset"); - CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst, - BytesBetween); + CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst, BytesBetween); } else if (Dst.isGlobalObjCRef()) { CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst, Dst.isThreadLocalRef()); @@ -2770,12 +2768,9 @@ CodeGenFunction::EmitLoadOfReference(LValue RefLVal, llvm::LoadInst *Load = Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile()); CGM.DecorateInstructionWithTBAA(Load, RefLVal.getTBAAInfo()); - - QualType PointeeType = RefLVal.getType()->getPointeeType(); - CharUnits Align = CGM.getNaturalTypeAlignment( - PointeeType, PointeeBaseInfo, PointeeTBAAInfo, - /* forPointeeType= */ true); - return Address(Load, ConvertTypeForMem(PointeeType), Align); + return makeNaturalAddressForPointer(Load, RefLVal.getType()->getPointeeType(), + CharUnits(), /*ForPointeeType=*/true, + PointeeBaseInfo, PointeeTBAAInfo); } LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) { @@ -2792,10 +2787,9 @@ Address CodeGenFunction::EmitLoadOfPointer(Address Ptr, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo) { llvm::Value *Addr = Builder.CreateLoad(Ptr); - return Address(Addr, ConvertTypeForMem(PtrTy->getPointeeType()), - CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(), BaseInfo, - TBAAInfo, - /*forPointeeType=*/true)); + return makeNaturalAddressForPointer(Addr, PtrTy->getPointeeType(), + CharUnits(), /*ForPointeeType=*/true, + BaseInfo, TBAAInfo); } LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr, @@ -2991,7 +2985,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { /* BaseInfo= */ nullptr, /* TBAAInfo= */ nullptr, /* forPointeeType= */ true); - Addr = Address(Val, ConvertTypeForMem(E->getType()), Alignment); + Addr = makeNaturalAddressForPointer(Val, T, Alignment); } return MakeAddrLValue(Addr, T, AlignmentSource::Decl); } @@ -3023,11 +3017,12 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD), CapturedStmtInfo->getContextValue()); Address LValueAddress = CapLVal.getAddress(*this); - CapLVal = MakeAddrLValue( - Address(LValueAddress.getPointer(), LValueAddress.getElementType(), - getContext().getDeclAlign(VD)), - CapLVal.getType(), LValueBaseInfo(AlignmentSource::Decl), - CapLVal.getTBAAInfo()); + CapLVal = MakeAddrLValue(Address(LValueAddress.emitRawPointer(*this), + LValueAddress.getElementType(), + getContext().getDeclAlign(VD)), + CapLVal.getType(), + LValueBaseInfo(AlignmentSource::Decl), + CapLVal.getTBAAInfo()); // Mark lvalue as nontemporal if the variable is marked as nontemporal // in simd context. if (getLangOpts().OpenMP && @@ -3083,7 +3078,8 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { // Handle threadlocal function locals. if (VD->getTLSKind() != VarDecl::TLS_None) addr = addr.withPointer( - Builder.CreateThreadLocalAddress(addr.getPointer()), NotKnownNonNull); + Builder.CreateThreadLocalAddress(addr.getBasePointer()), + NotKnownNonNull); // Check for OpenMP threadprivate variables. if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd && @@ -3351,7 +3347,7 @@ llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) { // Pointers are passed directly, everything else is passed by address. if (!V->getType()->isPointerTy()) { - Address Ptr = CreateDefaultAlignTempAlloca(V->getType()); + RawAddress Ptr = CreateDefaultAlignTempAlloca(V->getType()); Builder.CreateStore(V, Ptr); V = Ptr.getPointer(); } @@ -3924,6 +3920,21 @@ static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF, } } +static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, + ArrayRef indices, + llvm::Type *elementType, bool inbounds, + bool signedIndices, SourceLocation loc, + CharUnits align, + const llvm::Twine &name = "arrayidx") { + if (inbounds) { + return CGF.EmitCheckedInBoundsGEP(addr, indices, elementType, signedIndices, + CodeGenFunction::NotSubtraction, loc, + align, name); + } else { + return CGF.Builder.CreateGEP(addr, indices, elementType, align, name); + } +} + static CharUnits getArrayElementAlign(CharUnits arrayAlign, llvm::Value *idx, CharUnits eltSize) { @@ -3971,7 +3982,7 @@ static Address wrapWithBPFPreserveStaticOffset(CodeGenFunction &CGF, llvm::Function *Fn = CGF.CGM.getIntrinsic(llvm::Intrinsic::preserve_static_offset); - llvm::CallInst *Call = CGF.Builder.CreateCall(Fn, {Addr.getPointer()}); + llvm::CallInst *Call = CGF.Builder.CreateCall(Fn, {Addr.emitRawPointer(CGF)}); return Address(Call, Addr.getElementType(), Addr.getAlignment()); } @@ -4034,7 +4045,7 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, // We can use that to compute the best alignment of the element. CharUnits eltSize = CGF.getContext().getTypeSizeInChars(eltType); CharUnits eltAlign = - getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize); + getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize); if (hasBPFPreserveStaticOffset(Base)) addr = wrapWithBPFPreserveStaticOffset(CGF, addr); @@ -4043,19 +4054,19 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, auto LastIndex = dyn_cast(indices.back()); if (!LastIndex || (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, Base))) { - eltPtr = emitArraySubscriptGEP( - CGF, addr.getElementType(), addr.getPointer(), indices, inbounds, - signedIndices, loc, name); + addr = emitArraySubscriptGEP(CGF, addr, indices, + CGF.ConvertTypeForMem(eltType), inbounds, + signedIndices, loc, eltAlign, name); + return addr; } else { // Remember the original array subscript for bpf target unsigned idx = LastIndex->getZExtValue(); llvm::DIType *DbgInfo = nullptr; if (arrayType) DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(*arrayType, loc); - eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(addr.getElementType(), - addr.getPointer(), - indices.size() - 1, - idx, DbgInfo); + eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex( + addr.getElementType(), addr.emitRawPointer(CGF), indices.size() - 1, + idx, DbgInfo); } return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign); @@ -4224,8 +4235,8 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, CharUnits EltAlign = getArrayElementAlign(Addr.getAlignment(), Idx, InterfaceSize); llvm::Value *EltPtr = - emitArraySubscriptGEP(*this, Int8Ty, Addr.getPointer(), ScaledIdx, - false, SignedIndices, E->getExprLoc()); + emitArraySubscriptGEP(*this, Int8Ty, Addr.emitRawPointer(*this), + ScaledIdx, false, SignedIndices, E->getExprLoc()); Addr = Address(EltPtr, OrigBaseElemTy, EltAlign); } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) { // If this is A[i] where A is an array, the frontend will have decayed the @@ -4271,7 +4282,7 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, llvm::Type *CountTy = ConvertType(CountFD->getType()); llvm::Value *Res = Builder.CreateInBoundsGEP( - Int8Ty, Addr.getPointer(), + Int8Ty, Addr.emitRawPointer(*this), Builder.getInt32(OffsetDiff.getQuantity()), ".counted_by.gep"); Res = Builder.CreateAlignedLoad(CountTy, Res, getIntAlign(), ".counted_by.load"); @@ -4517,9 +4528,9 @@ LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, BaseInfo = ArrayLV.getBaseInfo(); TBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, ResultExprTy); } else { - Address Base = emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, - TBAAInfo, BaseTy, ResultExprTy, - IsLowerBound); + Address Base = + emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, TBAAInfo, BaseTy, + ResultExprTy, IsLowerBound); EltPtr = emitArraySubscriptGEP(*this, Base, Idx, ResultExprTy, !getLangOpts().isSignedOverflowDefined(), /*signedIndices=*/false, E->getExprLoc()); @@ -4606,7 +4617,7 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) { SkippedChecks.set(SanitizerKind::Alignment, true); if (IsBaseCXXThis || isa(BaseExpr)) SkippedChecks.set(SanitizerKind::Null, true); - EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr.getPointer(), PtrTy, + EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr, PtrTy, /*Alignment=*/CharUnits::Zero(), SkippedChecks); BaseLV = MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo); } else @@ -4655,8 +4666,8 @@ LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field, LambdaLV = EmitLoadOfReferenceLValue(AddrOfExplicitObject, D->getType(), AlignmentSource::Decl); else - LambdaLV = MakeNaturalAlignAddrLValue(AddrOfExplicitObject.getPointer(), - D->getType().getNonReferenceType()); + LambdaLV = MakeAddrLValue(AddrOfExplicitObject, + D->getType().getNonReferenceType()); } else { QualType LambdaTagType = getContext().getTagDeclType(Field->getParent()); LambdaLV = MakeNaturalAlignAddrLValue(ThisValue, LambdaTagType); @@ -4846,7 +4857,8 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, // information provided by invariant.group. This is because accessing // fields may leak the real address of dynamic object, which could result // in miscompilation when leaked pointer would be compared. - auto *stripped = Builder.CreateStripInvariantGroup(addr.getPointer()); + auto *stripped = + Builder.CreateStripInvariantGroup(addr.emitRawPointer(*this)); addr = Address(stripped, addr.getElementType(), addr.getAlignment()); } } @@ -4865,10 +4877,11 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, // Remember the original union field index llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(base.getType(), rec->getLocation()); - addr = Address( - Builder.CreatePreserveUnionAccessIndex( - addr.getPointer(), getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo), - addr.getElementType(), addr.getAlignment()); + addr = + Address(Builder.CreatePreserveUnionAccessIndex( + addr.emitRawPointer(*this), + getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo), + addr.getElementType(), addr.getAlignment()); } if (FieldType->isReferenceType()) @@ -5105,11 +5118,9 @@ LValue CodeGenFunction::EmitConditionalOperatorLValue( if (Info.LHS && Info.RHS) { Address lhsAddr = Info.LHS->getAddress(*this); Address rhsAddr = Info.RHS->getAddress(*this); - llvm::PHINode *phi = Builder.CreatePHI(lhsAddr.getType(), 2, "cond-lvalue"); - phi->addIncoming(lhsAddr.getPointer(), Info.lhsBlock); - phi->addIncoming(rhsAddr.getPointer(), Info.rhsBlock); - Address result(phi, lhsAddr.getElementType(), - std::min(lhsAddr.getAlignment(), rhsAddr.getAlignment())); + Address result = mergeAddressesInConditionalExpr( + lhsAddr, rhsAddr, Info.lhsBlock, Info.rhsBlock, + Builder.GetInsertBlock(), expr->getType()); AlignmentSource alignSource = std::max(Info.LHS->getBaseInfo().getAlignmentSource(), Info.RHS->getBaseInfo().getAlignmentSource()); @@ -5196,7 +5207,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { LValue LV = EmitLValue(E->getSubExpr()); Address V = LV.getAddress(*this); const auto *DCE = cast(E); - return MakeNaturalAlignAddrLValue(EmitDynamicCast(V, DCE), E->getType()); + return MakeNaturalAlignRawAddrLValue(EmitDynamicCast(V, DCE), E->getType()); } case CK_ConstructorConversion: @@ -5261,8 +5272,8 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is // performed and the object is not of the derived type. if (sanitizePerformTypeCheck()) - EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(), - Derived.getPointer(), E->getType()); + EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(), Derived, + E->getType()); if (SanOpts.has(SanitizerKind::CFIDerivedCast)) EmitVTablePtrCheckForCast(E->getType(), Derived, @@ -5618,7 +5629,7 @@ LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) { LValue CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) { - return MakeNaturalAlignAddrLValue(EmitCXXTypeidExpr(E), E->getType()); + return MakeNaturalAlignRawAddrLValue(EmitCXXTypeidExpr(E), E->getType()); } Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) { diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp index 5190b22bcc1622..143855aa84ca3f 100644 --- a/clang/lib/CodeGen/CGExprAgg.cpp +++ b/clang/lib/CodeGen/CGExprAgg.cpp @@ -294,10 +294,10 @@ void AggExprEmitter::withReturnValueSlot( // Otherwise, EmitCall will emit its own, notice that it's "unused", and end // its lifetime before we have the chance to emit a proper destructor call. bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() || - (RequiresDestruction && !Dest.getAddress().isValid()); + (RequiresDestruction && Dest.isIgnored()); Address RetAddr = Address::invalid(); - Address RetAllocaAddr = Address::invalid(); + RawAddress RetAllocaAddr = RawAddress::invalid(); EHScopeStack::stable_iterator LifetimeEndBlock; llvm::Value *LifetimeSizePtr = nullptr; @@ -329,7 +329,8 @@ void AggExprEmitter::withReturnValueSlot( if (!UseTemp) return; - assert(Dest.isIgnored() || Dest.getPointer() != Src.getAggregatePointer()); + assert(Dest.isIgnored() || Dest.emitRawPointer(CGF) != + Src.getAggregatePointer(E->getType(), CGF)); EmitFinalDestCopy(E->getType(), Src); if (!RequiresDestruction && LifetimeStartInst) { @@ -448,7 +449,8 @@ AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) { llvm::Value *Zero = llvm::ConstantInt::get(CGF.PtrDiffTy, 0); llvm::Value *IdxStart[] = { Zero, Zero }; llvm::Value *ArrayStart = Builder.CreateInBoundsGEP( - ArrayPtr.getElementType(), ArrayPtr.getPointer(), IdxStart, "arraystart"); + ArrayPtr.getElementType(), ArrayPtr.emitRawPointer(CGF), IdxStart, + "arraystart"); CGF.EmitStoreThroughLValue(RValue::get(ArrayStart), Start); ++Field; @@ -465,7 +467,8 @@ AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) { // End pointer. llvm::Value *IdxEnd[] = { Zero, Size }; llvm::Value *ArrayEnd = Builder.CreateInBoundsGEP( - ArrayPtr.getElementType(), ArrayPtr.getPointer(), IdxEnd, "arrayend"); + ArrayPtr.getElementType(), ArrayPtr.emitRawPointer(CGF), IdxEnd, + "arrayend"); CGF.EmitStoreThroughLValue(RValue::get(ArrayEnd), EndOrLength); } else if (Ctx.hasSameType(Field->getType(), Ctx.getSizeType())) { // Length. @@ -516,9 +519,9 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, // down a level. llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0); llvm::Value *indices[] = { zero, zero }; - llvm::Value *begin = Builder.CreateInBoundsGEP( - DestPtr.getElementType(), DestPtr.getPointer(), indices, - "arrayinit.begin"); + llvm::Value *begin = Builder.CreateInBoundsGEP(DestPtr.getElementType(), + DestPtr.emitRawPointer(CGF), + indices, "arrayinit.begin"); CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType); CharUnits elementAlign = @@ -1059,7 +1062,7 @@ void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) { if (RV.isScalar()) return {RV.getScalarVal(), nullptr}; if (RV.isAggregate()) - return {RV.getAggregatePointer(), nullptr}; + return {RV.getAggregatePointer(E->getType(), CGF), nullptr}; assert(RV.isComplex()); return RV.getComplexVal(); }; @@ -1818,7 +1821,7 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr( // else, clean it up for -O0 builds and general tidiness. if (!pushedCleanup && LV.isSimple()) if (llvm::GetElementPtrInst *GEP = - dyn_cast(LV.getPointer(CGF))) + dyn_cast(LV.emitRawPointer(CGF))) if (GEP->use_empty()) GEP->eraseFromParent(); } @@ -1849,9 +1852,9 @@ void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E, // destPtr is an array*. Construct an elementType* by drilling down a level. llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0); llvm::Value *indices[] = {zero, zero}; - llvm::Value *begin = Builder.CreateInBoundsGEP( - destPtr.getElementType(), destPtr.getPointer(), indices, - "arrayinit.begin"); + llvm::Value *begin = Builder.CreateInBoundsGEP(destPtr.getElementType(), + destPtr.emitRawPointer(CGF), + indices, "arrayinit.begin"); // Prepare to special-case multidimensional array initialization: we avoid // emitting multiple destructor loops in that case. diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp index 35da0f1a89bc3f..5c16a48d3ee6d6 100644 --- a/clang/lib/CodeGen/CGExprCXX.cpp +++ b/clang/lib/CodeGen/CGExprCXX.cpp @@ -280,7 +280,8 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; Address ThisValue = EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo); - This = MakeAddrLValue(ThisValue, Base->getType(), BaseInfo, TBAAInfo); + This = MakeAddrLValue(ThisValue, Base->getType()->getPointeeType(), + BaseInfo, TBAAInfo); } else { This = EmitLValue(Base); } @@ -353,10 +354,8 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( if (IsImplicitObjectCXXThis || isa(IOA)) SkippedChecks.set(SanitizerKind::Null, true); } - EmitTypeCheck(CodeGenFunction::TCK_MemberCall, CallLoc, - This.getPointer(*this), - C.getRecordType(CalleeDecl->getParent()), - /*Alignment=*/CharUnits::Zero(), SkippedChecks); + EmitTypeCheck(CodeGenFunction::TCK_MemberCall, CallLoc, This, + C.getRecordType(CalleeDecl->getParent()), SkippedChecks); // C++ [class.virtual]p12: // Explicit qualification with the scope operator (5.1) suppresses the @@ -455,7 +454,7 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E, else This = EmitLValue(BaseExpr, KnownNonNull).getAddress(*this); - EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.getPointer(), + EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.emitRawPointer(*this), QualType(MPT->getClass(), 0)); // Get the member function pointer. @@ -1109,9 +1108,10 @@ void CodeGenFunction::EmitNewArrayInitializer( // alloca. EndOfInit = CreateTempAlloca(BeginPtr.getType(), getPointerAlign(), "array.init.end"); - CleanupDominator = Builder.CreateStore(BeginPtr.getPointer(), EndOfInit); - pushIrregularPartialArrayCleanup(BeginPtr.getPointer(), EndOfInit, - ElementType, ElementAlign, + CleanupDominator = + Builder.CreateStore(BeginPtr.emitRawPointer(*this), EndOfInit); + pushIrregularPartialArrayCleanup(BeginPtr.emitRawPointer(*this), + EndOfInit, ElementType, ElementAlign, getDestroyer(DtorKind)); Cleanup = EHStack.stable_begin(); } @@ -1123,16 +1123,17 @@ void CodeGenFunction::EmitNewArrayInitializer( // element. TODO: some of these stores can be trivially // observed to be unnecessary. if (EndOfInit.isValid()) { - Builder.CreateStore(CurPtr.getPointer(), EndOfInit); + Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit); } // FIXME: If the last initializer is an incomplete initializer list for // an array, and we have an array filler, we can fold together the two // initialization loops. StoreAnyExprIntoOneUnit(*this, IE, IE->getType(), CurPtr, AggValueSlot::DoesNotOverlap); - CurPtr = Address(Builder.CreateInBoundsGEP( - CurPtr.getElementType(), CurPtr.getPointer(), - Builder.getSize(1), "array.exp.next"), + CurPtr = Address(Builder.CreateInBoundsGEP(CurPtr.getElementType(), + CurPtr.emitRawPointer(*this), + Builder.getSize(1), + "array.exp.next"), CurPtr.getElementType(), StartAlign.alignmentAtOffset((++i) * ElementSize)); } @@ -1186,7 +1187,7 @@ void CodeGenFunction::EmitNewArrayInitializer( // FIXME: Share this cleanup with the constructor call emission rather than // having it create a cleanup of its own. if (EndOfInit.isValid()) - Builder.CreateStore(CurPtr.getPointer(), EndOfInit); + Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit); // Emit a constructor call loop to initialize the remaining elements. if (InitListElements) @@ -1249,15 +1250,15 @@ void CodeGenFunction::EmitNewArrayInitializer( llvm::BasicBlock *ContBB = createBasicBlock("new.loop.end"); // Find the end of the array, hoisted out of the loop. - llvm::Value *EndPtr = - Builder.CreateInBoundsGEP(BeginPtr.getElementType(), BeginPtr.getPointer(), - NumElements, "array.end"); + llvm::Value *EndPtr = Builder.CreateInBoundsGEP( + BeginPtr.getElementType(), BeginPtr.emitRawPointer(*this), NumElements, + "array.end"); // If the number of elements isn't constant, we have to now check if there is // anything left to initialize. if (!ConstNum) { - llvm::Value *IsEmpty = - Builder.CreateICmpEQ(CurPtr.getPointer(), EndPtr, "array.isempty"); + llvm::Value *IsEmpty = Builder.CreateICmpEQ(CurPtr.emitRawPointer(*this), + EndPtr, "array.isempty"); Builder.CreateCondBr(IsEmpty, ContBB, LoopBB); } @@ -1267,19 +1268,20 @@ void CodeGenFunction::EmitNewArrayInitializer( // Set up the current-element phi. llvm::PHINode *CurPtrPhi = Builder.CreatePHI(CurPtr.getType(), 2, "array.cur"); - CurPtrPhi->addIncoming(CurPtr.getPointer(), EntryBB); + CurPtrPhi->addIncoming(CurPtr.emitRawPointer(*this), EntryBB); CurPtr = Address(CurPtrPhi, CurPtr.getElementType(), ElementAlign); // Store the new Cleanup position for irregular Cleanups. if (EndOfInit.isValid()) - Builder.CreateStore(CurPtr.getPointer(), EndOfInit); + Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit); // Enter a partial-destruction Cleanup if necessary. if (!CleanupDominator && needsEHCleanup(DtorKind)) { - pushRegularPartialArrayCleanup(BeginPtr.getPointer(), CurPtr.getPointer(), - ElementType, ElementAlign, - getDestroyer(DtorKind)); + llvm::Value *BeginPtrRaw = BeginPtr.emitRawPointer(*this); + llvm::Value *CurPtrRaw = CurPtr.emitRawPointer(*this); + pushRegularPartialArrayCleanup(BeginPtrRaw, CurPtrRaw, ElementType, + ElementAlign, getDestroyer(DtorKind)); Cleanup = EHStack.stable_begin(); CleanupDominator = Builder.CreateUnreachable(); } @@ -1295,9 +1297,8 @@ void CodeGenFunction::EmitNewArrayInitializer( } // Advance to the next element by adjusting the pointer type as necessary. - llvm::Value *NextPtr = - Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr.getPointer(), 1, - "array.next"); + llvm::Value *NextPtr = Builder.CreateConstInBoundsGEP1_32( + ElementTy, CurPtr.emitRawPointer(*this), 1, "array.next"); // Check whether we've gotten to the end of the array and, if so, // exit the loop. @@ -1523,14 +1524,9 @@ static void EnterNewDeleteCleanup(CodeGenFunction &CGF, typedef CallDeleteDuringNew DirectCleanup; - DirectCleanup *Cleanup = CGF.EHStack - .pushCleanupWithExtra(EHCleanup, - E->getNumPlacementArgs(), - E->getOperatorDelete(), - NewPtr.getPointer(), - AllocSize, - E->passAlignment(), - AllocAlign); + DirectCleanup *Cleanup = CGF.EHStack.pushCleanupWithExtra( + EHCleanup, E->getNumPlacementArgs(), E->getOperatorDelete(), + NewPtr.emitRawPointer(CGF), AllocSize, E->passAlignment(), AllocAlign); for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) { auto &Arg = NewArgs[I + NumNonPlacementArgs]; Cleanup->setPlacementArg(I, Arg.getRValue(CGF), Arg.Ty); @@ -1541,7 +1537,7 @@ static void EnterNewDeleteCleanup(CodeGenFunction &CGF, // Otherwise, we need to save all this stuff. DominatingValue::saved_type SavedNewPtr = - DominatingValue::save(CGF, RValue::get(NewPtr.getPointer())); + DominatingValue::save(CGF, RValue::get(NewPtr, CGF)); DominatingValue::saved_type SavedAllocSize = DominatingValue::save(CGF, RValue::get(AllocSize)); @@ -1618,14 +1614,14 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { // In these cases, discard the computed alignment and use the // formal alignment of the allocated type. if (BaseInfo.getAlignmentSource() != AlignmentSource::Decl) - allocation = allocation.withAlignment(allocAlign); + allocation.setAlignment(allocAlign); // Set up allocatorArgs for the call to operator delete if it's not // the reserved global operator. if (E->getOperatorDelete() && !E->getOperatorDelete()->isReservedGlobalPlacementOperator()) { allocatorArgs.add(RValue::get(allocSize), getContext().getSizeType()); - allocatorArgs.add(RValue::get(allocation.getPointer()), arg->getType()); + allocatorArgs.add(RValue::get(allocation, *this), arg->getType()); } } else { @@ -1713,8 +1709,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { llvm::BasicBlock *notNullBB = createBasicBlock("new.notnull"); contBB = createBasicBlock("new.cont"); - llvm::Value *isNull = - Builder.CreateIsNull(allocation.getPointer(), "new.isnull"); + llvm::Value *isNull = Builder.CreateIsNull(allocation, "new.isnull"); Builder.CreateCondBr(isNull, contBB, notNullBB); EmitBlock(notNullBB); } @@ -1760,12 +1755,12 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { SkippedChecks.set(SanitizerKind::Null, nullCheck); EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, E->getAllocatedTypeSourceInfo()->getTypeLoc().getBeginLoc(), - result.getPointer(), allocType, result.getAlignment(), - SkippedChecks, numElements); + result, allocType, result.getAlignment(), SkippedChecks, + numElements); EmitNewInitializer(*this, E, allocType, elementTy, result, numElements, allocSizeWithoutCookie); - llvm::Value *resultPtr = result.getPointer(); + llvm::Value *resultPtr = result.emitRawPointer(*this); if (E->isArray()) { // NewPtr is a pointer to the base element type. If we're // allocating an array of arrays, we'll need to cast back to the @@ -1909,7 +1904,8 @@ static void EmitDestroyingObjectDelete(CodeGenFunction &CGF, CGF.CGM.getCXXABI().emitVirtualObjectDelete(CGF, DE, Ptr, ElementType, Dtor); else - CGF.EmitDeleteCall(DE->getOperatorDelete(), Ptr.getPointer(), ElementType); + CGF.EmitDeleteCall(DE->getOperatorDelete(), Ptr.emitRawPointer(CGF), + ElementType); } /// Emit the code for deleting a single object. @@ -1925,8 +1921,7 @@ static bool EmitObjectDelete(CodeGenFunction &CGF, // dynamic type, the static type shall be a base class of the dynamic type // of the object to be deleted and the static type shall have a virtual // destructor or the behavior is undefined. - CGF.EmitTypeCheck(CodeGenFunction::TCK_MemberCall, - DE->getExprLoc(), Ptr.getPointer(), + CGF.EmitTypeCheck(CodeGenFunction::TCK_MemberCall, DE->getExprLoc(), Ptr, ElementType); const FunctionDecl *OperatorDelete = DE->getOperatorDelete(); @@ -1975,9 +1970,8 @@ static bool EmitObjectDelete(CodeGenFunction &CGF, // Make sure that we call delete even if the dtor throws. // This doesn't have to a conditional cleanup because we're going // to pop it off in a second. - CGF.EHStack.pushCleanup(NormalAndEHCleanup, - Ptr.getPointer(), - OperatorDelete, ElementType); + CGF.EHStack.pushCleanup( + NormalAndEHCleanup, Ptr.emitRawPointer(CGF), OperatorDelete, ElementType); if (Dtor) CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, @@ -2064,7 +2058,7 @@ static void EmitArrayDelete(CodeGenFunction &CGF, CharUnits elementAlign = deletedPtr.getAlignment().alignmentOfArrayElement(elementSize); - llvm::Value *arrayBegin = deletedPtr.getPointer(); + llvm::Value *arrayBegin = deletedPtr.emitRawPointer(CGF); llvm::Value *arrayEnd = CGF.Builder.CreateInBoundsGEP( deletedPtr.getElementType(), arrayBegin, numElements, "delete.end"); @@ -2095,7 +2089,7 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) { llvm::BasicBlock *DeleteNotNull = createBasicBlock("delete.notnull"); llvm::BasicBlock *DeleteEnd = createBasicBlock("delete.end"); - llvm::Value *IsNull = Builder.CreateIsNull(Ptr.getPointer(), "isnull"); + llvm::Value *IsNull = Builder.CreateIsNull(Ptr, "isnull"); Builder.CreateCondBr(IsNull, DeleteEnd, DeleteNotNull); EmitBlock(DeleteNotNull); @@ -2130,10 +2124,8 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) { GEP.push_back(Zero); } - Ptr = Address(Builder.CreateInBoundsGEP(Ptr.getElementType(), - Ptr.getPointer(), GEP, "del.first"), - ConvertTypeForMem(DeleteTy), Ptr.getAlignment(), - Ptr.isKnownNonNull()); + Ptr = Builder.CreateInBoundsGEP(Ptr, GEP, ConvertTypeForMem(DeleteTy), + Ptr.getAlignment(), "del.first"); } assert(ConvertTypeForMem(DeleteTy) == Ptr.getElementType()); @@ -2191,7 +2183,7 @@ static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E, // destruction and the static type of the operand is neither the constructor // or destructor’s class nor one of its bases, the behavior is undefined. CGF.EmitTypeCheck(CodeGenFunction::TCK_DynamicOperation, E->getExprLoc(), - ThisPtr.getPointer(), SrcRecordTy); + ThisPtr, SrcRecordTy); // C++ [expr.typeid]p2: // If the glvalue expression is obtained by applying the unary * operator to @@ -2207,7 +2199,7 @@ static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E, CGF.createBasicBlock("typeid.bad_typeid"); llvm::BasicBlock *EndBlock = CGF.createBasicBlock("typeid.end"); - llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr.getPointer()); + llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr); CGF.Builder.CreateCondBr(IsNull, BadTypeidBlock, EndBlock); CGF.EmitBlock(BadTypeidBlock); @@ -2293,8 +2285,7 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr, // construction or destruction and the static type of the operand is not a // pointer to or object of the constructor or destructor’s own class or one // of its bases, the dynamic_cast results in undefined behavior. - EmitTypeCheck(TCK_DynamicOperation, DCE->getExprLoc(), ThisAddr.getPointer(), - SrcRecordTy); + EmitTypeCheck(TCK_DynamicOperation, DCE->getExprLoc(), ThisAddr, SrcRecordTy); if (DCE->isAlwaysNull()) { if (llvm::Value *T = EmitDynamicCastToNull(*this, DestTy)) { @@ -2329,7 +2320,7 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr, CastNull = createBasicBlock("dynamic_cast.null"); CastNotNull = createBasicBlock("dynamic_cast.notnull"); - llvm::Value *IsNull = Builder.CreateIsNull(ThisAddr.getPointer()); + llvm::Value *IsNull = Builder.CreateIsNull(ThisAddr); Builder.CreateCondBr(IsNull, CastNull, CastNotNull); EmitBlock(CastNotNull); } diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp index 67a3cdc770426e..36d7493d9a6baf 100644 --- a/clang/lib/CodeGen/CGExprConstant.cpp +++ b/clang/lib/CodeGen/CGExprConstant.cpp @@ -800,8 +800,8 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD, // Add a vtable pointer, if we need one and it hasn't already been added. if (Layout.hasOwnVFPtr()) { llvm::Constant *VTableAddressPoint = - CGM.getCXXABI().getVTableAddressPointForConstExpr( - BaseSubobject(CD, Offset), VTableClass); + CGM.getCXXABI().getVTableAddressPoint(BaseSubobject(CD, Offset), + VTableClass); if (!AppendBytes(Offset, VTableAddressPoint)) return false; } diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp index 8536570087ad0f..83247aa48f8609 100644 --- a/clang/lib/CodeGen/CGExprScalar.cpp +++ b/clang/lib/CodeGen/CGExprScalar.cpp @@ -2250,7 +2250,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { // performed and the object is not of the derived type. if (CGF.sanitizePerformTypeCheck()) CGF.EmitTypeCheck(CodeGenFunction::TCK_DowncastPointer, CE->getExprLoc(), - Derived.getPointer(), DestTy->getPointeeType()); + Derived, DestTy->getPointeeType()); if (CGF.SanOpts.has(SanitizerKind::CFIDerivedCast)) CGF.EmitVTablePtrCheckForCast(DestTy->getPointeeType(), Derived, @@ -2258,13 +2258,14 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { CodeGenFunction::CFITCK_DerivedCast, CE->getBeginLoc()); - return Derived.getPointer(); + return CGF.getAsNaturalPointerTo(Derived, CE->getType()->getPointeeType()); } case CK_UncheckedDerivedToBase: case CK_DerivedToBase: { // The EmitPointerWithAlignment path does this fine; just discard // the alignment. - return CGF.EmitPointerWithAlignment(CE).getPointer(); + return CGF.getAsNaturalPointerTo(CGF.EmitPointerWithAlignment(CE), + CE->getType()->getPointeeType()); } case CK_Dynamic: { @@ -2274,7 +2275,8 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { } case CK_ArrayToPointerDecay: - return CGF.EmitArrayToPointerDecay(E).getPointer(); + return CGF.getAsNaturalPointerTo(CGF.EmitArrayToPointerDecay(E), + CE->getType()->getPointeeType()); case CK_FunctionToPointerDecay: return EmitLValue(E).getPointer(CGF); @@ -5588,3 +5590,16 @@ CodeGenFunction::EmitCheckedInBoundsGEP(llvm::Type *ElemTy, Value *Ptr, return GEPVal; } + +Address CodeGenFunction::EmitCheckedInBoundsGEP( + Address Addr, ArrayRef IdxList, llvm::Type *elementType, + bool SignedIndices, bool IsSubtraction, SourceLocation Loc, CharUnits Align, + const Twine &Name) { + if (!SanOpts.has(SanitizerKind::PointerOverflow)) + return Builder.CreateInBoundsGEP(Addr, IdxList, elementType, Align, Name); + + return RawAddress( + EmitCheckedInBoundsGEP(Addr.getElementType(), Addr.emitRawPointer(*this), + IdxList, SignedIndices, IsSubtraction, Loc, Name), + elementType, Align); +} diff --git a/clang/lib/CodeGen/CGNonTrivialStruct.cpp b/clang/lib/CodeGen/CGNonTrivialStruct.cpp index 75c1d7fbea8406..8fade0fac21e93 100644 --- a/clang/lib/CodeGen/CGNonTrivialStruct.cpp +++ b/clang/lib/CodeGen/CGNonTrivialStruct.cpp @@ -366,7 +366,7 @@ template struct GenFuncBase { llvm::Value *SizeInBytes = CGF.Builder.CreateNUWMul(BaseEltSizeVal, NumElts); llvm::Value *DstArrayEnd = CGF.Builder.CreateInBoundsGEP( - CGF.Int8Ty, DstAddr.getPointer(), SizeInBytes); + CGF.Int8Ty, DstAddr.emitRawPointer(CGF), SizeInBytes); llvm::BasicBlock *PreheaderBB = CGF.Builder.GetInsertBlock(); // Create the header block and insert the phi instructions. @@ -376,7 +376,7 @@ template struct GenFuncBase { for (unsigned I = 0; I < N; ++I) { PHIs[I] = CGF.Builder.CreatePHI(CGF.CGM.Int8PtrPtrTy, 2, "addr.cur"); - PHIs[I]->addIncoming(StartAddrs[I].getPointer(), PreheaderBB); + PHIs[I]->addIncoming(StartAddrs[I].emitRawPointer(CGF), PreheaderBB); } // Create the exit and loop body blocks. @@ -410,7 +410,7 @@ template struct GenFuncBase { // Instrs to update the destination and source addresses. // Update phi instructions. NewAddrs[I] = getAddrWithOffset(NewAddrs[I], EltSize); - PHIs[I]->addIncoming(NewAddrs[I].getPointer(), LoopBB); + PHIs[I]->addIncoming(NewAddrs[I].emitRawPointer(CGF), LoopBB); } // Insert an unconditional branch to the header block. @@ -488,7 +488,7 @@ template struct GenFuncBase { for (unsigned I = 0; I < N; ++I) { Alignments[I] = Addrs[I].getAlignment(); - Ptrs[I] = Addrs[I].getPointer(); + Ptrs[I] = Addrs[I].emitRawPointer(CallerCGF); } if (llvm::Function *F = diff --git a/clang/lib/CodeGen/CGObjC.cpp b/clang/lib/CodeGen/CGObjC.cpp index f3a948cf13f9c9..c7f497a7c8451b 100644 --- a/clang/lib/CodeGen/CGObjC.cpp +++ b/clang/lib/CodeGen/CGObjC.cpp @@ -94,8 +94,8 @@ CodeGenFunction::EmitObjCBoxedExpr(const ObjCBoxedExpr *E) { // and cast value to correct type Address Temporary = CreateMemTemp(SubExpr->getType()); EmitAnyExprToMem(SubExpr, Temporary, Qualifiers(), /*isInit*/ true); - llvm::Value *BitCast = - Builder.CreateBitCast(Temporary.getPointer(), ConvertType(ArgQT)); + llvm::Value *BitCast = Builder.CreateBitCast( + Temporary.emitRawPointer(*this), ConvertType(ArgQT)); Args.add(RValue::get(BitCast), ArgQT); // Create char array to store type encoding @@ -204,11 +204,11 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E, ObjCMethodDecl::param_const_iterator PI = MethodWithObjects->param_begin(); const ParmVarDecl *argDecl = *PI++; QualType ArgQT = argDecl->getType().getUnqualifiedType(); - Args.add(RValue::get(Objects.getPointer()), ArgQT); + Args.add(RValue::get(Objects, *this), ArgQT); if (DLE) { argDecl = *PI++; ArgQT = argDecl->getType().getUnqualifiedType(); - Args.add(RValue::get(Keys.getPointer()), ArgQT); + Args.add(RValue::get(Keys, *this), ArgQT); } argDecl = *PI; ArgQT = argDecl->getType().getUnqualifiedType(); @@ -827,7 +827,7 @@ static void emitStructGetterCall(CodeGenFunction &CGF, ObjCIvarDecl *ivar, // sizeof (Type of Ivar), isAtomic, false); CallArgList args; - llvm::Value *dest = CGF.ReturnValue.getPointer(); + llvm::Value *dest = CGF.ReturnValue.emitRawPointer(CGF); args.add(RValue::get(dest), Context.VoidPtrTy); args.add(RValue::get(src), Context.VoidPtrTy); @@ -1147,8 +1147,8 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, callCStructCopyConstructor(Dst, Src); } else { ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl(); - emitCPPObjectAtomicGetterCall(*this, ReturnValue.getPointer(), ivar, - AtomicHelperFn); + emitCPPObjectAtomicGetterCall(*this, ReturnValue.emitRawPointer(*this), + ivar, AtomicHelperFn); } return; } @@ -1163,7 +1163,7 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, } else { ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl(); - emitCPPObjectAtomicGetterCall(*this, ReturnValue.getPointer(), + emitCPPObjectAtomicGetterCall(*this, ReturnValue.emitRawPointer(*this), ivar, AtomicHelperFn); } return; @@ -1287,7 +1287,7 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, case TEK_Scalar: { llvm::Value *value; if (propType->isReferenceType()) { - value = LV.getAddress(*this).getPointer(); + value = LV.getAddress(*this).emitRawPointer(*this); } else { // We want to load and autoreleaseReturnValue ARC __weak ivars. if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) { @@ -1821,16 +1821,14 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ CallArgList Args; // The first argument is a temporary of the enumeration-state type. - Args.add(RValue::get(StatePtr.getPointer()), - getContext().getPointerType(StateTy)); + Args.add(RValue::get(StatePtr, *this), getContext().getPointerType(StateTy)); // The second argument is a temporary array with space for NumItems // pointers. We'll actually be loading elements from the array // pointer written into the control state; this buffer is so that // collections that *aren't* backed by arrays can still queue up // batches of elements. - Args.add(RValue::get(ItemsPtr.getPointer()), - getContext().getPointerType(ItemsTy)); + Args.add(RValue::get(ItemsPtr, *this), getContext().getPointerType(ItemsTy)); // The third argument is the capacity of that temporary array. llvm::Type *NSUIntegerTy = ConvertType(getContext().getNSUIntegerType()); @@ -2198,7 +2196,7 @@ static llvm::Value *emitARCLoadOperation(CodeGenFunction &CGF, Address addr, if (!fn) fn = getARCIntrinsic(IntID, CGF.CGM); - return CGF.EmitNounwindRuntimeCall(fn, addr.getPointer()); + return CGF.EmitNounwindRuntimeCall(fn, addr.emitRawPointer(CGF)); } /// Perform an operation having the following signature: @@ -2216,9 +2214,8 @@ static llvm::Value *emitARCStoreOperation(CodeGenFunction &CGF, Address addr, llvm::Type *origType = value->getType(); llvm::Value *args[] = { - CGF.Builder.CreateBitCast(addr.getPointer(), CGF.Int8PtrPtrTy), - CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy) - }; + CGF.Builder.CreateBitCast(addr.emitRawPointer(CGF), CGF.Int8PtrPtrTy), + CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy)}; llvm::CallInst *result = CGF.EmitNounwindRuntimeCall(fn, args); if (ignored) return nullptr; @@ -2237,9 +2234,8 @@ static void emitARCCopyOperation(CodeGenFunction &CGF, Address dst, Address src, fn = getARCIntrinsic(IntID, CGF.CGM); llvm::Value *args[] = { - CGF.Builder.CreateBitCast(dst.getPointer(), CGF.Int8PtrPtrTy), - CGF.Builder.CreateBitCast(src.getPointer(), CGF.Int8PtrPtrTy) - }; + CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), CGF.Int8PtrPtrTy), + CGF.Builder.CreateBitCast(src.emitRawPointer(CGF), CGF.Int8PtrPtrTy)}; CGF.EmitNounwindRuntimeCall(fn, args); } @@ -2490,9 +2486,8 @@ llvm::Value *CodeGenFunction::EmitARCStoreStrongCall(Address addr, fn = getARCIntrinsic(llvm::Intrinsic::objc_storeStrong, CGM); llvm::Value *args[] = { - Builder.CreateBitCast(addr.getPointer(), Int8PtrPtrTy), - Builder.CreateBitCast(value, Int8PtrTy) - }; + Builder.CreateBitCast(addr.emitRawPointer(*this), Int8PtrPtrTy), + Builder.CreateBitCast(value, Int8PtrTy)}; EmitNounwindRuntimeCall(fn, args); if (ignored) return nullptr; @@ -2643,7 +2638,7 @@ void CodeGenFunction::EmitARCDestroyWeak(Address addr) { if (!fn) fn = getARCIntrinsic(llvm::Intrinsic::objc_destroyWeak, CGM); - EmitNounwindRuntimeCall(fn, addr.getPointer()); + EmitNounwindRuntimeCall(fn, addr.emitRawPointer(*this)); } /// void \@objc_moveWeak(i8** %dest, i8** %src) diff --git a/clang/lib/CodeGen/CGObjCGNU.cpp b/clang/lib/CodeGen/CGObjCGNU.cpp index a36b0cdddaf0af..4e7f777ba1d916 100644 --- a/clang/lib/CodeGen/CGObjCGNU.cpp +++ b/clang/lib/CodeGen/CGObjCGNU.cpp @@ -706,7 +706,8 @@ class CGObjCGCC : public CGObjCGNU { llvm::Value *cmd, MessageSendInfo &MSI) override { CGBuilderTy &Builder = CGF.Builder; llvm::Value *lookupArgs[] = { - EnforceType(Builder, ObjCSuper.getPointer(), PtrToObjCSuperTy), cmd}; + EnforceType(Builder, ObjCSuper.emitRawPointer(CGF), PtrToObjCSuperTy), + cmd}; return CGF.EmitNounwindRuntimeCall(MsgLookupSuperFn, lookupArgs); } @@ -761,8 +762,8 @@ class CGObjCGNUstep : public CGObjCGNU { llvm::FunctionCallee LookupFn = SlotLookupFn; // Store the receiver on the stack so that we can reload it later - Address ReceiverPtr = - CGF.CreateTempAlloca(Receiver->getType(), CGF.getPointerAlign()); + RawAddress ReceiverPtr = + CGF.CreateTempAlloca(Receiver->getType(), CGF.getPointerAlign()); Builder.CreateStore(Receiver, ReceiverPtr); llvm::Value *self; @@ -778,9 +779,9 @@ class CGObjCGNUstep : public CGObjCGNU { LookupFn2->addParamAttr(0, llvm::Attribute::NoCapture); llvm::Value *args[] = { - EnforceType(Builder, ReceiverPtr.getPointer(), PtrToIdTy), - EnforceType(Builder, cmd, SelectorTy), - EnforceType(Builder, self, IdTy) }; + EnforceType(Builder, ReceiverPtr.getPointer(), PtrToIdTy), + EnforceType(Builder, cmd, SelectorTy), + EnforceType(Builder, self, IdTy)}; llvm::CallBase *slot = CGF.EmitRuntimeCallOrInvoke(LookupFn, args); slot->setOnlyReadsMemory(); slot->setMetadata(msgSendMDKind, node); @@ -800,7 +801,7 @@ class CGObjCGNUstep : public CGObjCGNU { llvm::Value *cmd, MessageSendInfo &MSI) override { CGBuilderTy &Builder = CGF.Builder; - llvm::Value *lookupArgs[] = {ObjCSuper.getPointer(), cmd}; + llvm::Value *lookupArgs[] = {ObjCSuper.emitRawPointer(CGF), cmd}; llvm::CallInst *slot = CGF.EmitNounwindRuntimeCall(SlotLookupSuperFn, lookupArgs); @@ -1221,10 +1222,10 @@ class CGObjCGNUstep2 : public CGObjCGNUstep { llvm::Value *cmd, MessageSendInfo &MSI) override { // Don't access the slot unless we're trying to cache the result. CGBuilderTy &Builder = CGF.Builder; - llvm::Value *lookupArgs[] = {CGObjCGNU::EnforceType(Builder, - ObjCSuper.getPointer(), - PtrToObjCSuperTy), - cmd}; + llvm::Value *lookupArgs[] = { + CGObjCGNU::EnforceType(Builder, ObjCSuper.emitRawPointer(CGF), + PtrToObjCSuperTy), + cmd}; return CGF.EmitNounwindRuntimeCall(MsgLookupSuperFn, lookupArgs); } @@ -2186,7 +2187,8 @@ class CGObjCObjFW: public CGObjCGNU { llvm::Value *cmd, MessageSendInfo &MSI) override { CGBuilderTy &Builder = CGF.Builder; llvm::Value *lookupArgs[] = { - EnforceType(Builder, ObjCSuper.getPointer(), PtrToObjCSuperTy), cmd, + EnforceType(Builder, ObjCSuper.emitRawPointer(CGF), PtrToObjCSuperTy), + cmd, }; if (CGM.ReturnTypeUsesSRet(MSI.CallInfo)) @@ -4201,15 +4203,15 @@ void CGObjCGNU::EmitThrowStmt(CodeGenFunction &CGF, llvm::Value * CGObjCGNU::EmitObjCWeakRead(CodeGenFunction &CGF, Address AddrWeakObj) { CGBuilderTy &B = CGF.Builder; - return B.CreateCall(WeakReadFn, - EnforceType(B, AddrWeakObj.getPointer(), PtrToIdTy)); + return B.CreateCall( + WeakReadFn, EnforceType(B, AddrWeakObj.emitRawPointer(CGF), PtrToIdTy)); } void CGObjCGNU::EmitObjCWeakAssign(CodeGenFunction &CGF, llvm::Value *src, Address dst) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); - llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy); + llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy); B.CreateCall(WeakAssignFn, {src, dstVal}); } @@ -4218,7 +4220,7 @@ void CGObjCGNU::EmitObjCGlobalAssign(CodeGenFunction &CGF, bool threadlocal) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); - llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy); + llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy); // FIXME. Add threadloca assign API assert(!threadlocal && "EmitObjCGlobalAssign - Threal Local API NYI"); B.CreateCall(GlobalAssignFn, {src, dstVal}); @@ -4229,7 +4231,7 @@ void CGObjCGNU::EmitObjCIvarAssign(CodeGenFunction &CGF, llvm::Value *ivarOffset) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); - llvm::Value *dstVal = EnforceType(B, dst.getPointer(), IdTy); + llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), IdTy); B.CreateCall(IvarAssignFn, {src, dstVal, ivarOffset}); } @@ -4237,7 +4239,7 @@ void CGObjCGNU::EmitObjCStrongCastAssign(CodeGenFunction &CGF, llvm::Value *src, Address dst) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); - llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy); + llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy); B.CreateCall(StrongCastAssignFn, {src, dstVal}); } @@ -4246,8 +4248,8 @@ void CGObjCGNU::EmitGCMemmoveCollectable(CodeGenFunction &CGF, Address SrcPtr, llvm::Value *Size) { CGBuilderTy &B = CGF.Builder; - llvm::Value *DestPtrVal = EnforceType(B, DestPtr.getPointer(), PtrTy); - llvm::Value *SrcPtrVal = EnforceType(B, SrcPtr.getPointer(), PtrTy); + llvm::Value *DestPtrVal = EnforceType(B, DestPtr.emitRawPointer(CGF), PtrTy); + llvm::Value *SrcPtrVal = EnforceType(B, SrcPtr.emitRawPointer(CGF), PtrTy); B.CreateCall(MemMoveFn, {DestPtrVal, SrcPtrVal, Size}); } diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp index ed8d7b9a065d7b..8a599c10e1caf1 100644 --- a/clang/lib/CodeGen/CGObjCMac.cpp +++ b/clang/lib/CodeGen/CGObjCMac.cpp @@ -1310,7 +1310,7 @@ class CGObjCMac : public CGObjCCommonMac { /// EmitSelector - Return a Value*, of type ObjCTypes.SelectorPtrTy, /// for the given selector. llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel); - Address EmitSelectorAddr(Selector Sel); + ConstantAddress EmitSelectorAddr(Selector Sel); public: CGObjCMac(CodeGen::CodeGenModule &cgm); @@ -1538,7 +1538,7 @@ class CGObjCNonFragileABIMac : public CGObjCCommonMac { /// EmitSelector - Return a Value*, of type ObjCTypes.SelectorPtrTy, /// for the given selector. llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel); - Address EmitSelectorAddr(Selector Sel); + ConstantAddress EmitSelectorAddr(Selector Sel); /// GetInterfaceEHType - Get the cached ehtype for the given Objective-C /// interface. The return value has type EHTypePtrTy. @@ -2064,9 +2064,8 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, const ObjCMethodDecl *Method) { // Create and init a super structure; this is a (receiver, class) // pair we will pass to objc_msgSendSuper. - Address ObjCSuper = - CGF.CreateTempAlloca(ObjCTypes.SuperTy, CGF.getPointerAlign(), - "objc_super"); + RawAddress ObjCSuper = CGF.CreateTempAlloca( + ObjCTypes.SuperTy, CGF.getPointerAlign(), "objc_super"); llvm::Value *ReceiverAsObject = CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy); CGF.Builder.CreateStore(ReceiverAsObject, @@ -4259,7 +4258,7 @@ namespace { CGF.EmitBlock(FinallyCallExit); CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryExitFn(), - ExceptionData.getPointer()); + ExceptionData.emitRawPointer(CGF)); CGF.EmitBlock(FinallyNoCallExit); @@ -4425,7 +4424,9 @@ void FragileHazards::emitHazardsInNewBlocks() { } static void addIfPresent(llvm::DenseSet &S, Address V) { - if (V.isValid()) S.insert(V.getPointer()); + if (V.isValid()) + if (llvm::Value *Ptr = V.getBasePointer()) + S.insert(Ptr); } void FragileHazards::collectLocals() { @@ -4628,13 +4629,13 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // - Call objc_exception_try_enter to push ExceptionData on top of // the EH stack. CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(), - ExceptionData.getPointer()); + ExceptionData.emitRawPointer(CGF)); // - Call setjmp on the exception data buffer. llvm::Constant *Zero = llvm::ConstantInt::get(CGF.Builder.getInt32Ty(), 0); llvm::Value *GEPIndexes[] = { Zero, Zero, Zero }; llvm::Value *SetJmpBuffer = CGF.Builder.CreateGEP( - ObjCTypes.ExceptionDataTy, ExceptionData.getPointer(), GEPIndexes, + ObjCTypes.ExceptionDataTy, ExceptionData.emitRawPointer(CGF), GEPIndexes, "setjmp_buffer"); llvm::CallInst *SetJmpResult = CGF.EmitNounwindRuntimeCall( ObjCTypes.getSetJmpFn(), SetJmpBuffer, "setjmp_result"); @@ -4673,9 +4674,9 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, } else { // Retrieve the exception object. We may emit multiple blocks but // nothing can cross this so the value is already in SSA form. - llvm::CallInst *Caught = - CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(), - ExceptionData.getPointer(), "caught"); + llvm::CallInst *Caught = CGF.EmitNounwindRuntimeCall( + ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF), + "caught"); // Push the exception to rethrow onto the EH value stack for the // benefit of any @throws in the handlers. @@ -4698,7 +4699,7 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // Enter a new exception try block (in case a @catch block // throws an exception). CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(), - ExceptionData.getPointer()); + ExceptionData.emitRawPointer(CGF)); llvm::CallInst *SetJmpResult = CGF.EmitNounwindRuntimeCall(ObjCTypes.getSetJmpFn(), @@ -4829,9 +4830,9 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // Extract the new exception and save it to the // propagating-exception slot. assert(PropagatingExnVar.isValid()); - llvm::CallInst *NewCaught = - CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(), - ExceptionData.getPointer(), "caught"); + llvm::CallInst *NewCaught = CGF.EmitNounwindRuntimeCall( + ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF), + "caught"); CGF.Builder.CreateStore(NewCaught, PropagatingExnVar); // Don't pop the catch handler; the throw already did. @@ -4861,9 +4862,8 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // Otherwise, just look in the buffer for the exception to throw. } else { - llvm::CallInst *Caught = - CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(), - ExceptionData.getPointer()); + llvm::CallInst *Caught = CGF.EmitNounwindRuntimeCall( + ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF)); PropagatingExn = Caught; } @@ -4906,7 +4906,7 @@ llvm::Value * CGObjCMac::EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF, Address AddrWeakObj) { llvm::Type* DestTy = AddrWeakObj.getElementType(); llvm::Value *AddrWeakObjVal = CGF.Builder.CreateBitCast( - AddrWeakObj.getPointer(), ObjCTypes.PtrObjectPtrTy); + AddrWeakObj.emitRawPointer(CGF), ObjCTypes.PtrObjectPtrTy); llvm::Value *read_weak = CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(), AddrWeakObjVal, "weakread"); @@ -4928,8 +4928,8 @@ void CGObjCMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = { src, dstVal }; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(), args, "weakassign"); @@ -4950,8 +4950,8 @@ void CGObjCMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; if (!threadlocal) CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(), @@ -4977,8 +4977,8 @@ void CGObjCMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal, ivarOffset}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args); } @@ -4997,8 +4997,8 @@ void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(), args, "strongassign"); @@ -5007,7 +5007,8 @@ void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF, void CGObjCMac::EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *size) { - llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), size }; + llvm::Value *args[] = {DestPtr.emitRawPointer(CGF), + SrcPtr.emitRawPointer(CGF), size}; CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args); } @@ -5243,7 +5244,7 @@ llvm::Value *CGObjCMac::EmitSelector(CodeGenFunction &CGF, Selector Sel) { return CGF.Builder.CreateLoad(EmitSelectorAddr(Sel)); } -Address CGObjCMac::EmitSelectorAddr(Selector Sel) { +ConstantAddress CGObjCMac::EmitSelectorAddr(Selector Sel) { CharUnits Align = CGM.getPointerAlign(); llvm::GlobalVariable *&Entry = SelectorReferences[Sel]; @@ -5254,7 +5255,7 @@ Address CGObjCMac::EmitSelectorAddr(Selector Sel) { Entry->setExternallyInitialized(true); } - return Address(Entry, ObjCTypes.SelectorPtrTy, Align); + return ConstantAddress(Entry, ObjCTypes.SelectorPtrTy, Align); } llvm::Constant *CGObjCCommonMac::GetClassName(StringRef RuntimeName) { @@ -7323,7 +7324,7 @@ CGObjCNonFragileABIMac::EmitVTableMessageSend(CodeGenFunction &CGF, ObjCTypes.MessageRefTy, CGF.getPointerAlign()); // Update the message ref argument. - args[1].setRValue(RValue::get(mref.getPointer())); + args[1].setRValue(RValue::get(mref, CGF)); // Load the function to call from the message ref table. Address calleeAddr = CGF.Builder.CreateStructGEP(mref, 0); @@ -7552,9 +7553,8 @@ CGObjCNonFragileABIMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, // ... // Create and init a super structure; this is a (receiver, class) // pair we will pass to objc_msgSendSuper. - Address ObjCSuper = - CGF.CreateTempAlloca(ObjCTypes.SuperTy, CGF.getPointerAlign(), - "objc_super"); + RawAddress ObjCSuper = CGF.CreateTempAlloca( + ObjCTypes.SuperTy, CGF.getPointerAlign(), "objc_super"); llvm::Value *ReceiverAsObject = CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy); @@ -7594,7 +7594,7 @@ llvm::Value *CGObjCNonFragileABIMac::EmitSelector(CodeGenFunction &CGF, return LI; } -Address CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) { +ConstantAddress CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) { llvm::GlobalVariable *&Entry = SelectorReferences[Sel]; CharUnits Align = CGM.getPointerAlign(); if (!Entry) { @@ -7610,7 +7610,7 @@ Address CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) { CGM.addCompilerUsedGlobal(Entry); } - return Address(Entry, ObjCTypes.SelectorPtrTy, Align); + return ConstantAddress(Entry, ObjCTypes.SelectorPtrTy, Align); } /// EmitObjCIvarAssign - Code gen for assigning to a __strong object. @@ -7629,8 +7629,8 @@ void CGObjCNonFragileABIMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal, ivarOffset}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args); } @@ -7650,8 +7650,8 @@ void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign( src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(), args, "weakassign"); @@ -7660,7 +7660,8 @@ void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign( void CGObjCNonFragileABIMac::EmitGCMemmoveCollectable( CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *Size) { - llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), Size }; + llvm::Value *args[] = {DestPtr.emitRawPointer(CGF), + SrcPtr.emitRawPointer(CGF), Size}; CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args); } @@ -7672,7 +7673,7 @@ llvm::Value * CGObjCNonFragileABIMac::EmitObjCWeakRead( Address AddrWeakObj) { llvm::Type *DestTy = AddrWeakObj.getElementType(); llvm::Value *AddrWeakObjVal = CGF.Builder.CreateBitCast( - AddrWeakObj.getPointer(), ObjCTypes.PtrObjectPtrTy); + AddrWeakObj.emitRawPointer(CGF), ObjCTypes.PtrObjectPtrTy); llvm::Value *read_weak = CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(), AddrWeakObjVal, "weakread"); @@ -7694,8 +7695,8 @@ void CGObjCNonFragileABIMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(), args, "weakassign"); @@ -7716,8 +7717,8 @@ void CGObjCNonFragileABIMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; if (!threadlocal) CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(), diff --git a/clang/lib/CodeGen/CGObjCRuntime.cpp b/clang/lib/CodeGen/CGObjCRuntime.cpp index 424564f9759995..01d0f35da19643 100644 --- a/clang/lib/CodeGen/CGObjCRuntime.cpp +++ b/clang/lib/CodeGen/CGObjCRuntime.cpp @@ -67,7 +67,7 @@ LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF, V = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, V, Offset, "add.ptr"); if (!Ivar->isBitField()) { - LValue LV = CGF.MakeNaturalAlignAddrLValue(V, IvarTy); + LValue LV = CGF.MakeNaturalAlignRawAddrLValue(V, IvarTy); return LV; } @@ -233,7 +233,7 @@ void CGObjCRuntime::EmitTryCatchStmt(CodeGenFunction &CGF, llvm::Instruction *CPICandidate = Handler.Block->getFirstNonPHI(); if (auto *CPI = dyn_cast_or_null(CPICandidate)) { CGF.CurrentFuncletPad = CPI; - CPI->setOperand(2, CGF.getExceptionSlot().getPointer()); + CPI->setOperand(2, CGF.getExceptionSlot().emitRawPointer(CGF)); CGF.EHStack.pushCleanup(NormalCleanup, CPI); } } @@ -405,7 +405,7 @@ bool CGObjCRuntime::canMessageReceiverBeNull(CodeGenFunction &CGF, auto self = curMethod->getSelfDecl(); if (self->getType().isConstQualified()) { if (auto LI = dyn_cast(receiver->stripPointerCasts())) { - llvm::Value *selfAddr = CGF.GetAddrOfLocalVar(self).getPointer(); + llvm::Value *selfAddr = CGF.GetAddrOfLocalVar(self).emitRawPointer(CGF); if (selfAddr == LI->getPointerOperand()) { return false; } diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 00e395c2a207f9..bc363313dec6f8 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -622,7 +622,7 @@ static void emitInitWithReductionInitializer(CodeGenFunction &CGF, auto *GV = new llvm::GlobalVariable( CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, Init, Name); - LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty); + LValue LV = CGF.MakeNaturalAlignRawAddrLValue(GV, Ty); RValue InitRVal; switch (CGF.getEvaluationKind(Ty)) { case TEK_Scalar: @@ -668,8 +668,8 @@ static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, llvm::Value *SrcBegin = nullptr; if (DRD) - SrcBegin = SrcAddr.getPointer(); - llvm::Value *DestBegin = DestAddr.getPointer(); + SrcBegin = SrcAddr.emitRawPointer(CGF); + llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF); // Cast from pointer to array type to pointer to single element. llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestAddr.getElementType(), DestBegin, NumElements); @@ -912,7 +912,7 @@ static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, Address OriginalBaseAddress, llvm::Value *Addr) { - Address Tmp = Address::invalid(); + RawAddress Tmp = RawAddress::invalid(); Address TopTmp = Address::invalid(); Address MostTopTmp = Address::invalid(); BaseTy = BaseTy.getNonReferenceType(); @@ -971,10 +971,10 @@ Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address SharedAddr = SharedAddresses[N].first.getAddress(CGF); llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff( SharedAddr.getElementType(), BaseLValue.getPointer(CGF), - SharedAddr.getPointer()); + SharedAddr.emitRawPointer(CGF)); llvm::Value *PrivatePointer = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - PrivateAddr.getPointer(), SharedAddr.getType()); + PrivateAddr.emitRawPointer(CGF), SharedAddr.getType()); llvm::Value *Ptr = CGF.Builder.CreateGEP( SharedAddr.getElementType(), PrivatePointer, Adjustment); return castToBase(CGF, OrigVD->getType(), @@ -1557,7 +1557,7 @@ static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc( return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack, ParentName); } -Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) { +ConstantAddress CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) { auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(VD); }; auto LinkageForVariable = [&VD, this]() { @@ -1579,8 +1579,8 @@ Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) { LinkageForVariable); if (!addr) - return Address::invalid(); - return Address(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(VD)); + return ConstantAddress::invalid(); + return ConstantAddress(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(VD)); } llvm::Constant * @@ -1604,7 +1604,7 @@ Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, llvm::Type *VarTy = VDAddr.getElementType(); llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), - CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy), + CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy), CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)), getOrCreateThreadPrivateCache(VD)}; return Address( @@ -1627,7 +1627,8 @@ void CGOpenMPRuntime::emitThreadPrivateVarInit( // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor) // to register constructor/destructor for variable. llvm::Value *Args[] = { - OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy), + OMPLoc, + CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.VoidPtrTy), Ctor, CopyCtor, Dtor}; CGF.EmitRuntimeCall( OMPBuilder.getOrCreateRuntimeFunction( @@ -1900,13 +1901,13 @@ void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, // OutlinedFn(>id, &zero_bound, CapturedStruct); Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc); - Address ZeroAddrBound = + RawAddress ZeroAddrBound = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, /*Name=*/".bound.zero.addr"); CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddrBound); llvm::SmallVector OutlinedFnArgs; // ThreadId for serialized parallels is 0. - OutlinedFnArgs.push_back(ThreadIDAddr.getPointer()); + OutlinedFnArgs.push_back(ThreadIDAddr.emitRawPointer(CGF)); OutlinedFnArgs.push_back(ZeroAddrBound.getPointer()); OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end()); @@ -2272,7 +2273,7 @@ void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, emitUpdateLocation(CGF, Loc), // ident_t * getThreadID(CGF, Loc), // i32 BufSize, // size_t - CL.getPointer(), // void * + CL.emitRawPointer(CGF), // void * CpyFn, // void (*) (void *, void *) DidItVal // i32 did_it }; @@ -2591,10 +2592,10 @@ static void emitForStaticInitCall( ThreadId, CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1, M2)), // Schedule type - Values.IL.getPointer(), // &isLastIter - Values.LB.getPointer(), // &LB - Values.UB.getPointer(), // &UB - Values.ST.getPointer(), // &Stride + Values.IL.emitRawPointer(CGF), // &isLastIter + Values.LB.emitRawPointer(CGF), // &LB + Values.UB.emitRawPointer(CGF), // &UB + Values.ST.emitRawPointer(CGF), // &Stride CGF.Builder.getIntN(Values.IVSize, 1), // Incr Chunk // Chunk }; @@ -2697,12 +2698,11 @@ llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF, // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper, // kmp_int[32|64] *p_stride); llvm::Value *Args[] = { - emitUpdateLocation(CGF, Loc), - getThreadID(CGF, Loc), - IL.getPointer(), // &isLastIter - LB.getPointer(), // &Lower - UB.getPointer(), // &Upper - ST.getPointer() // &Stride + emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), + IL.emitRawPointer(CGF), // &isLastIter + LB.emitRawPointer(CGF), // &Lower + UB.emitRawPointer(CGF), // &Upper + ST.emitRawPointer(CGF) // &Stride }; llvm::Value *Call = CGF.EmitRuntimeCall( OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args); @@ -3047,7 +3047,7 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, CGF.Builder .CreatePointerBitCastOrAddrSpaceCast(TDBase.getAddress(CGF), CGF.VoidPtrTy, CGF.Int8Ty) - .getPointer()}; + .emitRawPointer(CGF)}; SmallVector CallArgs(std::begin(CommonArgs), std::end(CommonArgs)); if (isOpenMPTaskLoopDirective(Kind)) { @@ -3574,7 +3574,8 @@ getPointerAndSize(CodeGenFunction &CGF, const Expr *E) { CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false); Address UpAddrAddress = UpAddrLVal.getAddress(CGF); llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32( - UpAddrAddress.getElementType(), UpAddrAddress.getPointer(), /*Idx0=*/1); + UpAddrAddress.getElementType(), UpAddrAddress.emitRawPointer(CGF), + /*Idx0=*/1); llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy); llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy); SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr); @@ -3888,8 +3889,9 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *Size; std::tie(Addr, Size) = getPointerAndSize(CGF, E); llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); - LValue Base = CGF.MakeAddrLValue( - CGF.Builder.CreateGEP(AffinitiesArray, Idx), KmpTaskAffinityInfoTy); + LValue Base = + CGF.MakeAddrLValue(CGF.Builder.CreateGEP(CGF, AffinitiesArray, Idx), + KmpTaskAffinityInfoTy); // affs[i].base_addr = &; LValue BaseAddrLVal = CGF.EmitLValueForField( Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr)); @@ -3910,7 +3912,7 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *LocRef = emitUpdateLocation(CGF, Loc); llvm::Value *GTid = getThreadID(CGF, Loc); llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - AffinitiesArray.getPointer(), CGM.VoidPtrTy); + AffinitiesArray.emitRawPointer(CGF), CGM.VoidPtrTy); // FIXME: Emit the function and ignore its result for now unless the // runtime function is properly implemented. (void)CGF.EmitRuntimeCall( @@ -3921,8 +3923,8 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *NewTaskNewTaskTTy = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( NewTask, KmpTaskTWithPrivatesPtrTy); - LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy, - KmpTaskTWithPrivatesQTy); + LValue Base = CGF.MakeNaturalAlignRawAddrLValue(NewTaskNewTaskTTy, + KmpTaskTWithPrivatesQTy); LValue TDBase = CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin()); // Fill the data in the resulting kmp_task_t record. @@ -4047,7 +4049,7 @@ CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, CGF.ConvertTypeForMem(KmpDependInfoPtrTy)), KmpDependInfoPtrTy->castAs()); Address DepObjAddr = CGF.Builder.CreateGEP( - Base.getAddress(CGF), + CGF, Base.getAddress(CGF), llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); LValue NumDepsBase = CGF.MakeAddrLValue( DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo()); @@ -4097,7 +4099,7 @@ static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, LValue &PosLVal = *Pos.get(); llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); Base = CGF.MakeAddrLValue( - CGF.Builder.CreateGEP(DependenciesArray, Idx), KmpDependInfoTy); + CGF.Builder.CreateGEP(CGF, DependenciesArray, Idx), KmpDependInfoTy); } // deps[i].base_addr = &; LValue BaseAddrLVal = CGF.EmitLValueForField( @@ -4195,7 +4197,7 @@ void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF, ElSize, CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false)); llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); - Address DepAddr = CGF.Builder.CreateGEP(DependenciesArray, Pos); + Address DepAddr = CGF.Builder.CreateGEP(CGF, DependenciesArray, Pos); CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(CGF), Size); // Increase pos. @@ -4430,7 +4432,7 @@ void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy), CGF.ConvertTypeForMem(KmpDependInfoTy)); llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( - Addr.getElementType(), Addr.getPointer(), + Addr.getElementType(), Addr.emitRawPointer(CGF), llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr, CGF.VoidPtrTy); @@ -4460,8 +4462,8 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, Address Begin = Base.getAddress(CGF); // Cast from pointer to array type to pointer to single element. - llvm::Value *End = CGF.Builder.CreateGEP( - Begin.getElementType(), Begin.getPointer(), NumDeps); + llvm::Value *End = CGF.Builder.CreateGEP(Begin.getElementType(), + Begin.emitRawPointer(CGF), NumDeps); // The basic structure here is a while-do loop. llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body"); llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done"); @@ -4469,7 +4471,7 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, CGF.EmitBlock(BodyBB); llvm::PHINode *ElementPHI = CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast"); - ElementPHI->addIncoming(Begin.getPointer(), EntryBB); + ElementPHI->addIncoming(Begin.emitRawPointer(CGF), EntryBB); Begin = Begin.withPointer(ElementPHI, KnownNonNull); Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo()); @@ -4483,12 +4485,12 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, FlagsLVal); // Shift the address forward by one element. - Address ElementNext = - CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext"); - ElementPHI->addIncoming(ElementNext.getPointer(), - CGF.Builder.GetInsertBlock()); + llvm::Value *ElementNext = + CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext") + .emitRawPointer(CGF); + ElementPHI->addIncoming(ElementNext, CGF.Builder.GetInsertBlock()); llvm::Value *IsEmpty = - CGF.Builder.CreateICmpEQ(ElementNext.getPointer(), End, "omp.isempty"); + CGF.Builder.CreateICmpEQ(ElementNext, End, "omp.isempty"); CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); // Done. CGF.EmitBlock(DoneBB, /*IsFinished=*/true); @@ -4531,7 +4533,7 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, DepTaskArgs[1] = ThreadID; DepTaskArgs[2] = NewTask; DepTaskArgs[3] = NumOfElements; - DepTaskArgs[4] = DependenciesArray.getPointer(); + DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF); DepTaskArgs[5] = CGF.Builder.getInt32(0); DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); } @@ -4563,7 +4565,7 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, DepWaitTaskArgs[0] = UpLoc; DepWaitTaskArgs[1] = ThreadID; DepWaitTaskArgs[2] = NumOfElements; - DepWaitTaskArgs[3] = DependenciesArray.getPointer(); + DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF); DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); DepWaitTaskArgs[6] = @@ -4725,8 +4727,8 @@ static void EmitOMPAggregateReduction( const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe(); llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr); - llvm::Value *RHSBegin = RHSAddr.getPointer(); - llvm::Value *LHSBegin = LHSAddr.getPointer(); + llvm::Value *RHSBegin = RHSAddr.emitRawPointer(CGF); + llvm::Value *LHSBegin = LHSAddr.emitRawPointer(CGF); // Cast from pointer to array type to pointer to single element. llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSAddr.getElementType(), LHSBegin, NumElements); @@ -4990,7 +4992,7 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, QualType ReductionArrayTy = C.getConstantArrayType( C.VoidPtrTy, ArraySize, nullptr, ArraySizeModifier::Normal, /*IndexTypeQuals=*/0); - Address ReductionList = + RawAddress ReductionList = CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); const auto *IPriv = Privates.begin(); unsigned Idx = 0; @@ -5462,7 +5464,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit( C.getConstantArrayType(RDType, ArraySize, nullptr, ArraySizeModifier::Normal, /*IndexTypeQuals=*/0); // kmp_task_red_input_t .rd_input.[Size]; - Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input."); + RawAddress TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input."); ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionOrigs, Data.ReductionCopies, Data.ReductionOps); for (unsigned Cnt = 0; Cnt < Size; ++Cnt) { @@ -5473,7 +5475,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit( TaskRedInput.getElementType(), TaskRedInput.getPointer(), Idxs, /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc, ".rd_input.gep."); - LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType); + LValue ElemLVal = CGF.MakeNaturalAlignRawAddrLValue(GEP, RDType); // ElemLVal.reduce_shar = &Shareds[Cnt]; LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD); RCG.emitSharedOrigLValue(CGF, Cnt); @@ -5629,7 +5631,7 @@ void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, DepWaitTaskArgs[0] = UpLoc; DepWaitTaskArgs[1] = ThreadID; DepWaitTaskArgs[2] = NumOfElements; - DepWaitTaskArgs[3] = DependenciesArray.getPointer(); + DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF); DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); DepWaitTaskArgs[6] = @@ -5852,7 +5854,7 @@ void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF, AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, CGF.getContext().VoidPtrTy, AllocatorTraitsLVal.getBaseInfo(), AllocatorTraitsLVal.getTBAAInfo()); - llvm::Value *Traits = Addr.getPointer(); + llvm::Value *Traits = Addr.emitRawPointer(CGF); llvm::Value *AllocatorVal = CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( @@ -7312,17 +7314,19 @@ class MappableExprsHandler { CGF.EmitOMPSharedLValue(MC.getAssociatedExpression()) .getAddress(CGF); } - Size = CGF.Builder.CreatePtrDiff( - CGF.Int8Ty, ComponentLB.getPointer(), LB.getPointer()); + llvm::Value *ComponentLBPtr = ComponentLB.emitRawPointer(CGF); + llvm::Value *LBPtr = LB.emitRawPointer(CGF); + Size = CGF.Builder.CreatePtrDiff(CGF.Int8Ty, ComponentLBPtr, + LBPtr); break; } } assert(Size && "Failed to determine structure size"); CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - CombinedInfo.BasePointers.push_back(BP.getPointer()); + CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF)); CombinedInfo.DevicePtrDecls.push_back(nullptr); CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - CombinedInfo.Pointers.push_back(LB.getPointer()); + CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( Size, CGF.Int64Ty, /*isSigned=*/true)); CombinedInfo.Types.push_back(Flags); @@ -7332,13 +7336,14 @@ class MappableExprsHandler { LB = CGF.Builder.CreateConstGEP(ComponentLB, 1); } CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - CombinedInfo.BasePointers.push_back(BP.getPointer()); + CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF)); CombinedInfo.DevicePtrDecls.push_back(nullptr); CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - CombinedInfo.Pointers.push_back(LB.getPointer()); + CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); + llvm::Value *LBPtr = LB.emitRawPointer(CGF); Size = CGF.Builder.CreatePtrDiff( - CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).getPointer(), - LB.getPointer()); + CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).emitRawPointer(CGF), + LBPtr); CombinedInfo.Sizes.push_back( CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true)); CombinedInfo.Types.push_back(Flags); @@ -7356,20 +7361,21 @@ class MappableExprsHandler { (Next == CE && MapType != OMPC_MAP_unknown)) { if (!IsMappingWholeStruct) { CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - CombinedInfo.BasePointers.push_back(BP.getPointer()); + CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF)); CombinedInfo.DevicePtrDecls.push_back(nullptr); CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - CombinedInfo.Pointers.push_back(LB.getPointer()); + CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( Size, CGF.Int64Ty, /*isSigned=*/true)); CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize : 1); } else { StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - StructBaseCombinedInfo.BasePointers.push_back(BP.getPointer()); + StructBaseCombinedInfo.BasePointers.push_back( + BP.emitRawPointer(CGF)); StructBaseCombinedInfo.DevicePtrDecls.push_back(nullptr); StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - StructBaseCombinedInfo.Pointers.push_back(LB.getPointer()); + StructBaseCombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); StructBaseCombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( Size, CGF.Int64Ty, /*isSigned=*/true)); StructBaseCombinedInfo.NonContigInfo.Dims.push_back( @@ -8211,11 +8217,11 @@ class MappableExprsHandler { } CombinedInfo.Exprs.push_back(VD); // Base is the base of the struct - CombinedInfo.BasePointers.push_back(PartialStruct.Base.getPointer()); + CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF)); CombinedInfo.DevicePtrDecls.push_back(nullptr); CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); // Pointer is the address of the lowest element - llvm::Value *LB = LBAddr.getPointer(); + llvm::Value *LB = LBAddr.emitRawPointer(CGF); const CXXMethodDecl *MD = CGF.CurFuncDecl ? dyn_cast(CGF.CurFuncDecl) : nullptr; const CXXRecordDecl *RD = MD ? MD->getParent() : nullptr; @@ -8229,7 +8235,7 @@ class MappableExprsHandler { // if the this[:1] expression had appeared in a map clause with a map-type // of tofrom. // Emit this[:1] - CombinedInfo.Pointers.push_back(PartialStruct.Base.getPointer()); + CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF)); QualType Ty = MD->getFunctionObjectParameterType(); llvm::Value *Size = CGF.Builder.CreateIntCast(CGF.getTypeSize(Ty), CGF.Int64Ty, @@ -8238,7 +8244,7 @@ class MappableExprsHandler { } else { CombinedInfo.Pointers.push_back(LB); // Size is (addr of {highest+1} element) - (addr of lowest element) - llvm::Value *HB = HBAddr.getPointer(); + llvm::Value *HB = HBAddr.emitRawPointer(CGF); llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32( HBAddr.getElementType(), HB, /*Idx0=*/1); llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy); @@ -8747,7 +8753,7 @@ class MappableExprsHandler { Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue( CV, ElementType, CGF.getContext().getDeclAlign(VD), AlignmentSource::Decl)); - CombinedInfo.Pointers.push_back(PtrAddr.getPointer()); + CombinedInfo.Pointers.push_back(PtrAddr.emitRawPointer(CGF)); } else { CombinedInfo.Pointers.push_back(CV); } @@ -9558,10 +9564,11 @@ static void emitTargetCallKernelLaunch( bool HasNoWait = D.hasClausesOfKind(); unsigned NumTargetItems = InputInfo.NumberOfTargetItems; - llvm::Value *BasePointersArray = InputInfo.BasePointersArray.getPointer(); - llvm::Value *PointersArray = InputInfo.PointersArray.getPointer(); - llvm::Value *SizesArray = InputInfo.SizesArray.getPointer(); - llvm::Value *MappersArray = InputInfo.MappersArray.getPointer(); + llvm::Value *BasePointersArray = + InputInfo.BasePointersArray.emitRawPointer(CGF); + llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF); + llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF); + llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF); auto &&EmitTargetCallFallbackCB = [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS, @@ -10309,15 +10316,16 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall( // Source location for the ident struct llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); - llvm::Value *OffloadingArgs[] = {RTLoc, - DeviceID, - PointerNum, - InputInfo.BasePointersArray.getPointer(), - InputInfo.PointersArray.getPointer(), - InputInfo.SizesArray.getPointer(), - MapTypesArray, - MapNamesArray, - InputInfo.MappersArray.getPointer()}; + llvm::Value *OffloadingArgs[] = { + RTLoc, + DeviceID, + PointerNum, + InputInfo.BasePointersArray.emitRawPointer(CGF), + InputInfo.PointersArray.emitRawPointer(CGF), + InputInfo.SizesArray.emitRawPointer(CGF), + MapTypesArray, + MapNamesArray, + InputInfo.MappersArray.emitRawPointer(CGF)}; // Select the right runtime function call for each standalone // directive. @@ -11128,7 +11136,7 @@ void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF, getThreadID(CGF, D.getBeginLoc()), llvm::ConstantInt::getSigned(CGM.Int32Ty, NumIterations.size()), CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).getPointer(), + CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).emitRawPointer(CGF), CGM.VoidPtrTy)}; llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction( @@ -11162,7 +11170,8 @@ static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM, /*Volatile=*/false, Int64Ty); } llvm::Value *Args[] = { - ULoc, ThreadID, CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()}; + ULoc, ThreadID, + CGF.Builder.CreateConstArrayGEP(CntAddr, 0).emitRawPointer(CGF)}; llvm::FunctionCallee RTLFn; llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder(); OMPDoacrossKind ODK; @@ -11332,7 +11341,7 @@ Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF, Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID( CGF, SourceLocation::getFromRawEncoding(LocEncoding)); Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - Addr.getPointer(), CGF.VoidPtrTy); + Addr.emitRawPointer(CGF), CGF.VoidPtrTy); llvm::Value *AllocVal = getAllocatorVal(CGF, AllocExpr); Args[2] = AllocVal; CGF.EmitRuntimeCall(RTLFn, Args); @@ -11690,15 +11699,17 @@ void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LLIVTy, getName({UniqueDeclName, "iv"})); cast(LastIV)->setAlignment( IVLVal.getAlignment().getAsAlign()); - LValue LastIVLVal = CGF.MakeNaturalAlignAddrLValue(LastIV, IVLVal.getType()); + LValue LastIVLVal = + CGF.MakeNaturalAlignRawAddrLValue(LastIV, IVLVal.getType()); // Last value of the lastprivate conditional. // decltype(priv_a) last_a; llvm::GlobalVariable *Last = OMPBuilder.getOrCreateInternalVariable( CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName); - Last->setAlignment(LVal.getAlignment().getAsAlign()); - LValue LastLVal = CGF.MakeAddrLValue( - Address(Last, Last->getValueType(), LVal.getAlignment()), LVal.getType()); + cast(Last)->setAlignment( + LVal.getAlignment().getAsAlign()); + LValue LastLVal = + CGF.MakeRawAddrLValue(Last, LVal.getType(), LVal.getAlignment()); // Global loop counter. Required to handle inner parallel-for regions. // iv @@ -11871,9 +11882,8 @@ void CGOpenMPRuntime::emitLastprivateConditionalFinalUpdate( // The variable was not updated in the region - exit. if (!GV) return; - LValue LPLVal = CGF.MakeAddrLValue( - Address(GV, GV->getValueType(), PrivLVal.getAlignment()), - PrivLVal.getType().getNonReferenceType()); + LValue LPLVal = CGF.MakeRawAddrLValue( + GV, PrivLVal.getType().getNonReferenceType(), PrivLVal.getAlignment()); llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc); CGF.EmitStoreOfScalar(Res, PrivLVal); } diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h index c3206427b143e1..522ae3d35d22d7 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.h +++ b/clang/lib/CodeGen/CGOpenMPRuntime.h @@ -1068,13 +1068,12 @@ class CGOpenMPRuntime { /// \param Loc Location of the reference to threadprivate var. /// \return Address of the threadprivate variable for the current thread. virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, - const VarDecl *VD, - Address VDAddr, + const VarDecl *VD, Address VDAddr, SourceLocation Loc); /// Returns the address of the variable marked as declare target with link /// clause OR as declare target with to clause and unified memory. - virtual Address getAddrOfDeclareTargetVar(const VarDecl *VD); + virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD); /// Emit a code for initialization of threadprivate variable. It emits /// a call to runtime library which adds initial value to the newly created diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp index 299ee1460b3db0..5baac8f0e3e268 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp @@ -1096,7 +1096,8 @@ void CGOpenMPRuntimeGPU::emitGenericVarsProlog(CodeGenFunction &CGF, llvm::PointerType *VarPtrTy = CGF.ConvertTypeForMem(VarTy)->getPointerTo(); llvm::Value *CastedVoidPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( VoidPtr, VarPtrTy, VD->getName() + "_on_stack"); - LValue VarAddr = CGF.MakeNaturalAlignAddrLValue(CastedVoidPtr, VarTy); + LValue VarAddr = + CGF.MakeNaturalAlignPointeeRawAddrLValue(CastedVoidPtr, VarTy); Rec.second.PrivateAddr = VarAddr.getAddress(CGF); Rec.second.GlobalizedVal = VoidPtr; @@ -1206,8 +1207,8 @@ void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF, bool IsBareKernel = D.getSingleClause(); - Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, - /*Name=*/".zero.addr"); + RawAddress ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, + /*Name=*/".zero.addr"); CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddr); llvm::SmallVector OutlinedFnArgs; // We don't emit any thread id function call in bare kernel, but because the @@ -1215,7 +1216,7 @@ void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF, if (IsBareKernel) OutlinedFnArgs.push_back(llvm::ConstantPointerNull::get(CGM.VoidPtrTy)); else - OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).getPointer()); + OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).emitRawPointer(CGF)); OutlinedFnArgs.push_back(ZeroAddr.getPointer()); OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end()); emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); @@ -1289,7 +1290,7 @@ void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF, llvm::ConstantInt::get(CGF.Int32Ty, -1), FnPtr, ID, - Bld.CreateBitOrPointerCast(CapturedVarsAddrs.getPointer(), + Bld.CreateBitOrPointerCast(CapturedVarsAddrs.emitRawPointer(CGF), CGF.VoidPtrPtrTy), llvm::ConstantInt::get(CGM.SizeTy, CapturedVars.size())}; CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( @@ -1503,17 +1504,18 @@ static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr, CGF.EmitBlock(PreCondBB); llvm::PHINode *PhiSrc = Bld.CreatePHI(Ptr.getType(), /*NumReservedValues=*/2); - PhiSrc->addIncoming(Ptr.getPointer(), CurrentBB); + PhiSrc->addIncoming(Ptr.emitRawPointer(CGF), CurrentBB); llvm::PHINode *PhiDest = Bld.CreatePHI(ElemPtr.getType(), /*NumReservedValues=*/2); - PhiDest->addIncoming(ElemPtr.getPointer(), CurrentBB); + PhiDest->addIncoming(ElemPtr.emitRawPointer(CGF), CurrentBB); Ptr = Address(PhiSrc, Ptr.getElementType(), Ptr.getAlignment()); ElemPtr = Address(PhiDest, ElemPtr.getElementType(), ElemPtr.getAlignment()); + llvm::Value *PtrEndRaw = PtrEnd.emitRawPointer(CGF); + llvm::Value *PtrRaw = Ptr.emitRawPointer(CGF); llvm::Value *PtrDiff = Bld.CreatePtrDiff( - CGF.Int8Ty, PtrEnd.getPointer(), - Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr.getPointer(), - CGF.VoidPtrTy)); + CGF.Int8Ty, PtrEndRaw, + Bld.CreatePointerBitCastOrAddrSpaceCast(PtrRaw, CGF.VoidPtrTy)); Bld.CreateCondBr(Bld.CreateICmpSGT(PtrDiff, Bld.getInt64(IntSize - 1)), ThenBB, ExitBB); CGF.EmitBlock(ThenBB); @@ -1528,8 +1530,8 @@ static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr, TBAAAccessInfo()); Address LocalPtr = Bld.CreateConstGEP(Ptr, 1); Address LocalElemPtr = Bld.CreateConstGEP(ElemPtr, 1); - PhiSrc->addIncoming(LocalPtr.getPointer(), ThenBB); - PhiDest->addIncoming(LocalElemPtr.getPointer(), ThenBB); + PhiSrc->addIncoming(LocalPtr.emitRawPointer(CGF), ThenBB); + PhiDest->addIncoming(LocalElemPtr.emitRawPointer(CGF), ThenBB); CGF.EmitBranch(PreCondBB); CGF.EmitBlock(ExitBB); } else { @@ -1676,10 +1678,10 @@ static void emitReductionListCopy( // scope and that of functions it invokes (i.e., reduce_function). // RemoteReduceData[i] = (void*)&RemoteElem if (UpdateDestListPtr) { - CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast( - DestElementAddr.getPointer(), CGF.VoidPtrTy), - DestElementPtrAddr, /*Volatile=*/false, - C.VoidPtrTy); + CGF.EmitStoreOfScalar( + Bld.CreatePointerBitCastOrAddrSpaceCast( + DestElementAddr.emitRawPointer(CGF), CGF.VoidPtrTy), + DestElementPtrAddr, /*Volatile=*/false, C.VoidPtrTy); } ++Idx; @@ -1830,7 +1832,7 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM, // elemptr = ((CopyType*)(elemptrptr)) + I Address ElemPtr(ElemPtrPtr, CopyType, Align); if (NumIters > 1) - ElemPtr = Bld.CreateGEP(ElemPtr, Cnt); + ElemPtr = Bld.CreateGEP(CGF, ElemPtr, Cnt); // Get pointer to location in transfer medium. // MediumPtr = &medium[warp_id] @@ -1894,7 +1896,7 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM, TargetElemPtrPtr, /*Volatile=*/false, C.VoidPtrTy, Loc); Address TargetElemPtr(TargetElemPtrVal, CopyType, Align); if (NumIters > 1) - TargetElemPtr = Bld.CreateGEP(TargetElemPtr, Cnt); + TargetElemPtr = Bld.CreateGEP(CGF, TargetElemPtr, Cnt); // *TargetElemPtr = SrcMediumVal; llvm::Value *SrcMediumValue = @@ -2105,9 +2107,9 @@ static llvm::Function *emitShuffleAndReduceFunction( CGF.EmitBlock(ThenBB); // reduce_function(LocalReduceList, RemoteReduceList) llvm::Value *LocalReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( - LocalReduceList.getPointer(), CGF.VoidPtrTy); + LocalReduceList.emitRawPointer(CGF), CGF.VoidPtrTy); llvm::Value *RemoteReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( - RemoteReduceList.getPointer(), CGF.VoidPtrTy); + RemoteReduceList.emitRawPointer(CGF), CGF.VoidPtrTy); CGM.getOpenMPRuntime().emitOutlinedFunctionCall( CGF, Loc, ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr}); Bld.CreateBr(MergeBB); @@ -2218,9 +2220,9 @@ static llvm::Value *emitListToGlobalCopyFunction( llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); LValue GlobLVal = CGF.EmitLValueForField( - CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); + CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD); Address GlobAddr = GlobLVal.getAddress(CGF); - GlobLVal.setAddress(Address(GlobAddr.getPointer(), + GlobLVal.setAddress(Address(GlobAddr.emitRawPointer(CGF), CGF.ConvertTypeForMem(Private->getType()), GlobAddr.getAlignment())); switch (CGF.getEvaluationKind(Private->getType())) { @@ -2304,7 +2306,7 @@ static llvm::Value *emitListToGlobalReduceFunction( // 1. Build a list of reduction variables. // void *RedList[] = {[0], ..., [-1]}; - Address ReductionList = + RawAddress ReductionList = CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); auto IPriv = Privates.begin(); llvm::Value *Idxs[] = {CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), @@ -2319,10 +2321,10 @@ static llvm::Value *emitListToGlobalReduceFunction( llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); LValue GlobLVal = CGF.EmitLValueForField( - CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); + CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD); Address GlobAddr = GlobLVal.getAddress(CGF); - CGF.EmitStoreOfScalar(GlobAddr.getPointer(), Elem, /*Volatile=*/false, - C.VoidPtrTy); + CGF.EmitStoreOfScalar(GlobAddr.emitRawPointer(CGF), Elem, + /*Volatile=*/false, C.VoidPtrTy); if ((*IPriv)->getType()->isVariablyModifiedType()) { // Store array size. ++Idx; @@ -2425,9 +2427,9 @@ static llvm::Value *emitGlobalToListCopyFunction( llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); LValue GlobLVal = CGF.EmitLValueForField( - CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); + CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD); Address GlobAddr = GlobLVal.getAddress(CGF); - GlobLVal.setAddress(Address(GlobAddr.getPointer(), + GlobLVal.setAddress(Address(GlobAddr.emitRawPointer(CGF), CGF.ConvertTypeForMem(Private->getType()), GlobAddr.getAlignment())); switch (CGF.getEvaluationKind(Private->getType())) { @@ -2526,10 +2528,10 @@ static llvm::Value *emitGlobalToListReduceFunction( llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); LValue GlobLVal = CGF.EmitLValueForField( - CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); + CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD); Address GlobAddr = GlobLVal.getAddress(CGF); - CGF.EmitStoreOfScalar(GlobAddr.getPointer(), Elem, /*Volatile=*/false, - C.VoidPtrTy); + CGF.EmitStoreOfScalar(GlobAddr.emitRawPointer(CGF), Elem, + /*Volatile=*/false, C.VoidPtrTy); if ((*IPriv)->getType()->isVariablyModifiedType()) { // Store array size. ++Idx; @@ -2545,7 +2547,7 @@ static llvm::Value *emitGlobalToListReduceFunction( } // Call reduce_function(ReduceList, GlobalReduceList) - llvm::Value *GlobalReduceList = ReductionList.getPointer(); + llvm::Value *GlobalReduceList = ReductionList.emitRawPointer(CGF); Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar( AddrReduceListArg, /*Volatile=*/false, C.VoidPtrTy, Loc); @@ -2876,7 +2878,7 @@ void CGOpenMPRuntimeGPU::emitReduction( } llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - ReductionList.getPointer(), CGF.VoidPtrTy); + ReductionList.emitRawPointer(CGF), CGF.VoidPtrTy); llvm::Function *ReductionFn = emitReductionFunction( CGF.CurFn->getName(), Loc, CGF.ConvertTypeForMem(ReductionArrayTy), Privates, LHSExprs, RHSExprs, ReductionOps); @@ -3106,15 +3108,15 @@ llvm::Function *CGOpenMPRuntimeGPU::createParallelDataSharingWrapper( // Get the array of arguments. SmallVector Args; - Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).getPointer()); - Args.emplace_back(ZeroAddr.getPointer()); + Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).emitRawPointer(CGF)); + Args.emplace_back(ZeroAddr.emitRawPointer(CGF)); CGBuilderTy &Bld = CGF.Builder; auto CI = CS.capture_begin(); // Use global memory for data sharing. // Handle passing of global args to workers. - Address GlobalArgs = + RawAddress GlobalArgs = CGF.CreateDefaultAlignTempAlloca(CGF.VoidPtrPtrTy, "global_args"); llvm::Value *GlobalArgsPtr = GlobalArgs.getPointer(); llvm::Value *DataSharingArgs[] = {GlobalArgsPtr}; @@ -3400,7 +3402,7 @@ void CGOpenMPRuntimeGPU::adjustTargetSpecificDataForLambdas( VDAddr = CGF.EmitLoadOfReferenceLValue(VDAddr, VD->getType().getCanonicalType()) .getAddress(CGF); - CGF.EmitStoreOfScalar(VDAddr.getPointer(), VarLVal); + CGF.EmitStoreOfScalar(VDAddr.emitRawPointer(CGF), VarLVal); } } } diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp index cb5a004e4f4a6c..576fe2f7a2d46f 100644 --- a/clang/lib/CodeGen/CGStmt.cpp +++ b/clang/lib/CodeGen/CGStmt.cpp @@ -2294,7 +2294,7 @@ std::pair CodeGenFunction::EmitAsmInputLValue( Address Addr = InputValue.getAddress(*this); ConstraintStr += '*'; - return {Addr.getPointer(), Addr.getElementType()}; + return {InputValue.getPointer(*this), Addr.getElementType()}; } std::pair @@ -2701,7 +2701,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { ArgTypes.push_back(DestAddr.getType()); ArgElemTypes.push_back(DestAddr.getElementType()); - Args.push_back(DestAddr.getPointer()); + Args.push_back(DestAddr.emitRawPointer(*this)); Constraints += "=*"; Constraints += OutputConstraint; ReadOnly = ReadNone = false; @@ -3076,8 +3076,8 @@ CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) { CapturedStmtInfo->setContextValue(Builder.CreateLoad(DeclPtr)); // Initialize variable-length arrays. - LValue Base = MakeNaturalAlignAddrLValue(CapturedStmtInfo->getContextValue(), - Ctx.getTagDeclType(RD)); + LValue Base = MakeNaturalAlignRawAddrLValue( + CapturedStmtInfo->getContextValue(), Ctx.getTagDeclType(RD)); for (auto *FD : RD->fields()) { if (FD->hasCapturedVLAType()) { auto *ExprArg = diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp index f37ac549d10a82..e6d504bcdeca5b 100644 --- a/clang/lib/CodeGen/CGStmtOpenMP.cpp +++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp @@ -350,7 +350,8 @@ void CodeGenFunction::GenerateOpenMPCapturedVars( LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType()); llvm::Value *SrcAddrVal = EmitScalarConversion( - DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()), + DstAddr.emitRawPointer(*this), + Ctx.getPointerType(Ctx.getUIntPtrType()), Ctx.getPointerType(CurField->getType()), CurCap->getLocation()); LValue SrcLV = MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType()); @@ -364,7 +365,8 @@ void CodeGenFunction::GenerateOpenMPCapturedVars( CapturedVars.push_back(CV); } else { assert(CurCap->capturesVariable() && "Expected capture by reference."); - CapturedVars.push_back(EmitLValue(*I).getAddress(*this).getPointer()); + CapturedVars.push_back( + EmitLValue(*I).getAddress(*this).emitRawPointer(*this)); } } } @@ -375,8 +377,9 @@ static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, ASTContext &Ctx = CGF.getContext(); llvm::Value *CastedPtr = CGF.EmitScalarConversion( - AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(), + AddrLV.getAddress(CGF).emitRawPointer(CGF), Ctx.getUIntPtrType(), Ctx.getPointerType(DstType), Loc); + // FIXME: should the pointee type (DstType) be passed? Address TmpAddr = CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress(CGF); return TmpAddr; @@ -702,8 +705,8 @@ void CodeGenFunction::EmitOMPAggregateAssign( llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr); SrcAddr = SrcAddr.withElementType(DestAddr.getElementType()); - llvm::Value *SrcBegin = SrcAddr.getPointer(); - llvm::Value *DestBegin = DestAddr.getPointer(); + llvm::Value *SrcBegin = SrcAddr.emitRawPointer(*this); + llvm::Value *DestBegin = DestAddr.emitRawPointer(*this); // Cast from pointer to array type to pointer to single element. llvm::Value *DestEnd = Builder.CreateInBoundsGEP(DestAddr.getElementType(), DestBegin, NumElements); @@ -1007,10 +1010,10 @@ bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) { CopyBegin = createBasicBlock("copyin.not.master"); CopyEnd = createBasicBlock("copyin.not.master.end"); // TODO: Avoid ptrtoint conversion. - auto *MasterAddrInt = - Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy); - auto *PrivateAddrInt = - Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy); + auto *MasterAddrInt = Builder.CreatePtrToInt( + MasterAddr.emitRawPointer(*this), CGM.IntPtrTy); + auto *PrivateAddrInt = Builder.CreatePtrToInt( + PrivateAddr.emitRawPointer(*this), CGM.IntPtrTy); Builder.CreateCondBr( Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin, CopyEnd); @@ -1666,7 +1669,7 @@ Address CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate( llvm::Type *VarTy = VDAddr.getElementType(); llvm::Value *Data = - CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy); + CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy); llvm::ConstantInt *Size = CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)); std::string Suffix = getNameWithSeparators({"cache", ""}); llvm::Twine CacheName = Twine(CGM.getMangledName(VD)).concat(Suffix); @@ -2045,7 +2048,7 @@ void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) { ->getParam(0) ->getType() .getNonReferenceType(); - Address CountAddr = CreateMemTemp(LogicalTy, ".count.addr"); + RawAddress CountAddr = CreateMemTemp(LogicalTy, ".count.addr"); emitCapturedStmtCall(*this, DistanceClosure, {CountAddr.getPointer()}); llvm::Value *DistVal = Builder.CreateLoad(CountAddr, ".count"); @@ -2061,7 +2064,7 @@ void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) { LValue LCVal = EmitLValue(LoopVarRef); Address LoopVarAddress = LCVal.getAddress(*this); emitCapturedStmtCall(*this, LoopVarClosure, - {LoopVarAddress.getPointer(), IndVar}); + {LoopVarAddress.emitRawPointer(*this), IndVar}); RunCleanupsScope BodyScope(*this); EmitStmt(BodyStmt); @@ -4795,7 +4798,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( ParamTypes.push_back(PrivatesPtr->getType()); for (const Expr *E : Data.PrivateVars) { const auto *VD = cast(cast(E)->getDecl()); - Address PrivatePtr = CGF.CreateMemTemp( + RawAddress PrivatePtr = CGF.CreateMemTemp( CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); CallArgs.push_back(PrivatePtr.getPointer()); @@ -4803,7 +4806,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( } for (const Expr *E : Data.FirstprivateVars) { const auto *VD = cast(cast(E)->getDecl()); - Address PrivatePtr = + RawAddress PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".firstpriv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); @@ -4813,7 +4816,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( } for (const Expr *E : Data.LastprivateVars) { const auto *VD = cast(cast(E)->getDecl()); - Address PrivatePtr = + RawAddress PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".lastpriv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); @@ -4826,7 +4829,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( Ty = CGF.getContext().getPointerType(Ty); if (isAllocatableDecl(VD)) Ty = CGF.getContext().getPointerType(Ty); - Address PrivatePtr = CGF.CreateMemTemp( + RawAddress PrivatePtr = CGF.CreateMemTemp( CGF.getContext().getPointerType(Ty), ".local.ptr.addr"); auto Result = UntiedLocalVars.insert( std::make_pair(VD, std::make_pair(PrivatePtr, Address::invalid()))); @@ -4859,7 +4862,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( if (auto *DI = CGF.getDebugInfo()) if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo()) (void)DI->EmitDeclareOfAutoVariable( - Pair.first, Pair.second.getPointer(), CGF.Builder, + Pair.first, Pair.second.getBasePointer(), CGF.Builder, /*UsePointerValue*/ true); } // Adjust mapping for internal locals by mapping actual memory instead of @@ -4912,14 +4915,14 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( RedCG, Cnt); Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); - Replacement = - Address(CGF.EmitScalarConversion( - Replacement.getPointer(), CGF.getContext().VoidPtrTy, - CGF.getContext().getPointerType( - Data.ReductionCopies[Cnt]->getType()), - Data.ReductionCopies[Cnt]->getExprLoc()), - CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), - Replacement.getAlignment()); + Replacement = Address( + CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF), + CGF.getContext().VoidPtrTy, + CGF.getContext().getPointerType( + Data.ReductionCopies[Cnt]->getType()), + Data.ReductionCopies[Cnt]->getExprLoc()), + CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), + Replacement.getAlignment()); Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement); } @@ -4970,7 +4973,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); Replacement = Address( CGF.EmitScalarConversion( - Replacement.getPointer(), CGF.getContext().VoidPtrTy, + Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy, CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), InRedPrivs[Cnt]->getExprLoc()), CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()), @@ -5089,7 +5092,7 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective( // If there is no user-defined mapper, the mapper array will be nullptr. In // this case, we don't need to privatize it. if (!isa_and_nonnull( - InputInfo.MappersArray.getPointer())) { + InputInfo.MappersArray.emitRawPointer(*this))) { MVD = createImplicitFirstprivateForType( getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc()); TargetScope.addPrivate(MVD, InputInfo.MappersArray); @@ -5115,7 +5118,7 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective( ParamTypes.push_back(PrivatesPtr->getType()); for (const Expr *E : Data.FirstprivateVars) { const auto *VD = cast(cast(E)->getDecl()); - Address PrivatePtr = + RawAddress PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".firstpriv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); @@ -5194,14 +5197,14 @@ void CodeGenFunction::processInReduction(const OMPExecutableDirective &S, RedCG, Cnt); Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); - Replacement = - Address(CGF.EmitScalarConversion( - Replacement.getPointer(), CGF.getContext().VoidPtrTy, - CGF.getContext().getPointerType( - Data.ReductionCopies[Cnt]->getType()), - Data.ReductionCopies[Cnt]->getExprLoc()), - CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), - Replacement.getAlignment()); + Replacement = Address( + CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF), + CGF.getContext().VoidPtrTy, + CGF.getContext().getPointerType( + Data.ReductionCopies[Cnt]->getType()), + Data.ReductionCopies[Cnt]->getExprLoc()), + CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), + Replacement.getAlignment()); Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement); } @@ -5247,7 +5250,7 @@ void CodeGenFunction::processInReduction(const OMPExecutableDirective &S, CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); Replacement = Address( CGF.EmitScalarConversion( - Replacement.getPointer(), CGF.getContext().VoidPtrTy, + Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy, CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), InRedPrivs[Cnt]->getExprLoc()), CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()), @@ -5394,7 +5397,7 @@ void CodeGenFunction::EmitOMPDepobjDirective(const OMPDepobjDirective &S) { Dependencies.DepExprs.append(DC->varlist_begin(), DC->varlist_end()); Address DepAddr = CGM.getOpenMPRuntime().emitDepobjDependClause( *this, Dependencies, DC->getBeginLoc()); - EmitStoreOfScalar(DepAddr.getPointer(), DOLVal); + EmitStoreOfScalar(DepAddr.emitRawPointer(*this), DOLVal); return; } if (const auto *DC = S.getSingleClause()) { @@ -6471,21 +6474,21 @@ static void emitOMPAtomicCompareExpr( D->getType()->hasSignedIntegerRepresentation()); llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{ - XAddr.getPointer(), XAddr.getElementType(), + XAddr.emitRawPointer(CGF), XAddr.getElementType(), X->getType()->hasSignedIntegerRepresentation(), X->getType().isVolatileQualified()}; llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal; if (V) { LValue LV = CGF.EmitLValue(V); Address Addr = LV.getAddress(CGF); - VOpVal = {Addr.getPointer(), Addr.getElementType(), + VOpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(), V->getType()->hasSignedIntegerRepresentation(), V->getType().isVolatileQualified()}; } if (R) { LValue LV = CGF.EmitLValue(R); Address Addr = LV.getAddress(CGF); - ROpVal = {Addr.getPointer(), Addr.getElementType(), + ROpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(), R->getType()->hasSignedIntegerRepresentation(), R->getType().isVolatileQualified()}; } @@ -7029,7 +7032,7 @@ void CodeGenFunction::EmitOMPInteropDirective(const OMPInteropDirective &S) { std::tie(NumDependences, DependenciesArray) = CGM.getOpenMPRuntime().emitDependClause(*this, Data.Dependences, S.getBeginLoc()); - DependenceList = DependenciesArray.getPointer(); + DependenceList = DependenciesArray.emitRawPointer(*this); } Data.HasNowaitClause = S.hasClausesOfKind(); diff --git a/clang/lib/CodeGen/CGVTables.cpp b/clang/lib/CodeGen/CGVTables.cpp index 8dee3f74b44b4e..862369ae009f48 100644 --- a/clang/lib/CodeGen/CGVTables.cpp +++ b/clang/lib/CodeGen/CGVTables.cpp @@ -201,14 +201,13 @@ CodeGenFunction::GenerateVarArgsThunk(llvm::Function *Fn, // Find the first store of "this", which will be to the alloca associated // with "this". - Address ThisPtr = - Address(&*AI, ConvertTypeForMem(MD->getFunctionObjectParameterType()), - CGM.getClassPointerAlignment(MD->getParent())); + Address ThisPtr = makeNaturalAddressForPointer( + &*AI, MD->getFunctionObjectParameterType(), + CGM.getClassPointerAlignment(MD->getParent())); llvm::BasicBlock *EntryBB = &Fn->front(); llvm::BasicBlock::iterator ThisStore = llvm::find_if(*EntryBB, [&](llvm::Instruction &I) { - return isa(I) && - I.getOperand(0) == ThisPtr.getPointer(); + return isa(I) && I.getOperand(0) == &*AI; }); assert(ThisStore != EntryBB->end() && "Store of this should be in entry block?"); diff --git a/clang/lib/CodeGen/CGValue.h b/clang/lib/CodeGen/CGValue.h index 1e6f67250583d6..cc9ad10ae5969d 100644 --- a/clang/lib/CodeGen/CGValue.h +++ b/clang/lib/CodeGen/CGValue.h @@ -14,12 +14,13 @@ #ifndef LLVM_CLANG_LIB_CODEGEN_CGVALUE_H #define LLVM_CLANG_LIB_CODEGEN_CGVALUE_H +#include "Address.h" +#include "CodeGenTBAA.h" +#include "EHScopeStack.h" #include "clang/AST/ASTContext.h" #include "clang/AST/Type.h" -#include "llvm/IR/Value.h" #include "llvm/IR/Type.h" -#include "Address.h" -#include "CodeGenTBAA.h" +#include "llvm/IR/Value.h" namespace llvm { class Constant; @@ -28,57 +29,64 @@ namespace llvm { namespace clang { namespace CodeGen { - class AggValueSlot; - class CodeGenFunction; - struct CGBitFieldInfo; +class AggValueSlot; +class CGBuilderTy; +class CodeGenFunction; +struct CGBitFieldInfo; /// RValue - This trivial value class is used to represent the result of an /// expression that is evaluated. It can be one of three things: either a /// simple LLVM SSA value, a pair of SSA values for complex numbers, or the /// address of an aggregate value in memory. class RValue { - enum Flavor { Scalar, Complex, Aggregate }; + friend struct DominatingValue; - // The shift to make to an aggregate's alignment to make it look - // like a pointer. - enum { AggAlignShift = 4 }; + enum FlavorEnum { Scalar, Complex, Aggregate }; - // Stores first value and flavor. - llvm::PointerIntPair V1; - // Stores second value and volatility. - llvm::PointerIntPair V2; - // Stores element type for aggregate values. - llvm::Type *ElementType; + union { + // Stores first and second value. + struct { + llvm::Value *first; + llvm::Value *second; + } Vals; + + // Stores aggregate address. + Address AggregateAddr; + }; + + unsigned IsVolatile : 1; + unsigned Flavor : 2; public: - bool isScalar() const { return V1.getInt() == Scalar; } - bool isComplex() const { return V1.getInt() == Complex; } - bool isAggregate() const { return V1.getInt() == Aggregate; } + RValue() : Vals{nullptr, nullptr}, Flavor(Scalar) {} + + bool isScalar() const { return Flavor == Scalar; } + bool isComplex() const { return Flavor == Complex; } + bool isAggregate() const { return Flavor == Aggregate; } - bool isVolatileQualified() const { return V2.getInt(); } + bool isVolatileQualified() const { return IsVolatile; } /// getScalarVal() - Return the Value* of this scalar value. llvm::Value *getScalarVal() const { assert(isScalar() && "Not a scalar!"); - return V1.getPointer(); + return Vals.first; } /// getComplexVal - Return the real/imag components of this complex value. /// std::pair getComplexVal() const { - return std::make_pair(V1.getPointer(), V2.getPointer()); + return std::make_pair(Vals.first, Vals.second); } /// getAggregateAddr() - Return the Value* of the address of the aggregate. Address getAggregateAddress() const { assert(isAggregate() && "Not an aggregate!"); - auto align = reinterpret_cast(V2.getPointer()) >> AggAlignShift; - return Address( - V1.getPointer(), ElementType, CharUnits::fromQuantity(align)); + return AggregateAddr; } - llvm::Value *getAggregatePointer() const { - assert(isAggregate() && "Not an aggregate!"); - return V1.getPointer(); + + llvm::Value *getAggregatePointer(QualType PointeeType, + CodeGenFunction &CGF) const { + return getAggregateAddress().getBasePointer(); } static RValue getIgnored() { @@ -88,17 +96,19 @@ class RValue { static RValue get(llvm::Value *V) { RValue ER; - ER.V1.setPointer(V); - ER.V1.setInt(Scalar); - ER.V2.setInt(false); + ER.Vals.first = V; + ER.Flavor = Scalar; + ER.IsVolatile = false; return ER; } + static RValue get(Address Addr, CodeGenFunction &CGF) { + return RValue::get(Addr.emitRawPointer(CGF)); + } static RValue getComplex(llvm::Value *V1, llvm::Value *V2) { RValue ER; - ER.V1.setPointer(V1); - ER.V2.setPointer(V2); - ER.V1.setInt(Complex); - ER.V2.setInt(false); + ER.Vals = {V1, V2}; + ER.Flavor = Complex; + ER.IsVolatile = false; return ER; } static RValue getComplex(const std::pair &C) { @@ -107,15 +117,15 @@ class RValue { // FIXME: Aggregate rvalues need to retain information about whether they are // volatile or not. Remove default to find all places that probably get this // wrong. + + /// Convert an Address to an RValue. If the Address is not + /// signed, create an RValue using the unsigned address. Otherwise, resign the + /// address using the provided type. static RValue getAggregate(Address addr, bool isVolatile = false) { RValue ER; - ER.V1.setPointer(addr.getPointer()); - ER.V1.setInt(Aggregate); - ER.ElementType = addr.getElementType(); - - auto align = static_cast(addr.getAlignment().getQuantity()); - ER.V2.setPointer(reinterpret_cast(align << AggAlignShift)); - ER.V2.setInt(isVolatile); + ER.AggregateAddr = addr; + ER.Flavor = Aggregate; + ER.IsVolatile = isVolatile; return ER; } }; @@ -178,8 +188,10 @@ class LValue { MatrixElt // This is a matrix element, use getVector* } LVType; - llvm::Value *V; - llvm::Type *ElementType; + union { + Address Addr = Address::invalid(); + llvm::Value *V; + }; union { // Index into a vector subscript: V[i] @@ -197,10 +209,6 @@ class LValue { // 'const' is unused here Qualifiers Quals; - // The alignment to use when accessing this lvalue. (For vector elements, - // this is the alignment of the whole vector.) - unsigned Alignment; - // objective-c's ivar bool Ivar:1; @@ -234,23 +242,19 @@ class LValue { Expr *BaseIvarExp; private: - void Initialize(QualType Type, Qualifiers Quals, CharUnits Alignment, + void Initialize(QualType Type, Qualifiers Quals, Address Addr, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) { - assert((!Alignment.isZero() || Type->isIncompleteType()) && - "initializing l-value with zero alignment!"); - if (isGlobalReg()) - assert(ElementType == nullptr && "Global reg does not store elem type"); - else - assert(ElementType != nullptr && "Must have elem type"); - this->Type = Type; this->Quals = Quals; const unsigned MaxAlign = 1U << 31; - this->Alignment = Alignment.getQuantity() <= MaxAlign - ? Alignment.getQuantity() - : MaxAlign; - assert(this->Alignment == Alignment.getQuantity() && - "Alignment exceeds allowed max!"); + CharUnits Alignment = Addr.getAlignment(); + assert((isGlobalReg() || !Alignment.isZero() || Type->isIncompleteType()) && + "initializing l-value with zero alignment!"); + if (Alignment.getQuantity() > MaxAlign) { + assert(false && "Alignment exceeds allowed max!"); + Alignment = CharUnits::fromQuantity(MaxAlign); + } + this->Addr = Addr; this->BaseInfo = BaseInfo; this->TBAAInfo = TBAAInfo; @@ -259,9 +263,20 @@ class LValue { this->ImpreciseLifetime = false; this->Nontemporal = false; this->ThreadLocalRef = false; + this->IsKnownNonNull = false; this->BaseIvarExp = nullptr; } + void initializeSimpleLValue(Address Addr, QualType Type, + LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo, + ASTContext &Context) { + Qualifiers QS = Type.getQualifiers(); + QS.setObjCGCAttr(Context.getObjCGCAttrKind(Type)); + LVType = Simple; + Initialize(Type, QS, Addr, BaseInfo, TBAAInfo); + assert(Addr.getBasePointer()->getType()->isPointerTy()); + } + public: bool isSimple() const { return LVType == Simple; } bool isVectorElt() const { return LVType == VectorElt; } @@ -328,8 +343,8 @@ class LValue { LangAS getAddressSpace() const { return Quals.getAddressSpace(); } - CharUnits getAlignment() const { return CharUnits::fromQuantity(Alignment); } - void setAlignment(CharUnits A) { Alignment = A.getQuantity(); } + CharUnits getAlignment() const { return Addr.getAlignment(); } + void setAlignment(CharUnits A) { Addr.setAlignment(A); } LValueBaseInfo getBaseInfo() const { return BaseInfo; } void setBaseInfo(LValueBaseInfo Info) { BaseInfo = Info; } @@ -345,28 +360,32 @@ class LValue { // simple lvalue llvm::Value *getPointer(CodeGenFunction &CGF) const { assert(isSimple()); - return V; + return Addr.getBasePointer(); } - Address getAddress(CodeGenFunction &CGF) const { - return Address(getPointer(CGF), ElementType, getAlignment(), - isKnownNonNull()); - } - void setAddress(Address address) { + llvm::Value *emitRawPointer(CodeGenFunction &CGF) const { assert(isSimple()); - V = address.getPointer(); - ElementType = address.getElementType(); - Alignment = address.getAlignment().getQuantity(); - IsKnownNonNull = address.isKnownNonNull(); + return Addr.isValid() ? Addr.emitRawPointer(CGF) : nullptr; } + Address getAddress(CodeGenFunction &CGF) const { + // FIXME: remove parameter. + return Addr; + } + + void setAddress(Address address) { Addr = address; } + // vector elt lvalue Address getVectorAddress() const { - return Address(getVectorPointer(), ElementType, getAlignment(), - (KnownNonNull_t)isKnownNonNull()); + assert(isVectorElt()); + return Addr; + } + llvm::Value *getRawVectorPointer(CodeGenFunction &CGF) const { + assert(isVectorElt()); + return Addr.emitRawPointer(CGF); } llvm::Value *getVectorPointer() const { assert(isVectorElt()); - return V; + return Addr.getBasePointer(); } llvm::Value *getVectorIdx() const { assert(isVectorElt()); @@ -374,12 +393,12 @@ class LValue { } Address getMatrixAddress() const { - return Address(getMatrixPointer(), ElementType, getAlignment(), - (KnownNonNull_t)isKnownNonNull()); + assert(isMatrixElt()); + return Addr; } llvm::Value *getMatrixPointer() const { assert(isMatrixElt()); - return V; + return Addr.getBasePointer(); } llvm::Value *getMatrixIdx() const { assert(isMatrixElt()); @@ -388,12 +407,12 @@ class LValue { // extended vector elements. Address getExtVectorAddress() const { - return Address(getExtVectorPointer(), ElementType, getAlignment(), - (KnownNonNull_t)isKnownNonNull()); + assert(isExtVectorElt()); + return Addr; } - llvm::Value *getExtVectorPointer() const { + llvm::Value *getRawExtVectorPointer(CodeGenFunction &CGF) const { assert(isExtVectorElt()); - return V; + return Addr.emitRawPointer(CGF); } llvm::Constant *getExtVectorElts() const { assert(isExtVectorElt()); @@ -402,10 +421,14 @@ class LValue { // bitfield lvalue Address getBitFieldAddress() const { - return Address(getBitFieldPointer(), ElementType, getAlignment(), - (KnownNonNull_t)isKnownNonNull()); + assert(isBitField()); + return Addr; + } + llvm::Value *getRawBitFieldPointer(CodeGenFunction &CGF) const { + assert(isBitField()); + return Addr.emitRawPointer(CGF); } - llvm::Value *getBitFieldPointer() const { assert(isBitField()); return V; } + const CGBitFieldInfo &getBitFieldInfo() const { assert(isBitField()); return *BitFieldInfo; @@ -414,18 +437,13 @@ class LValue { // global register lvalue llvm::Value *getGlobalReg() const { assert(isGlobalReg()); return V; } - static LValue MakeAddr(Address address, QualType type, ASTContext &Context, + static LValue MakeAddr(Address Addr, QualType type, ASTContext &Context, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) { - Qualifiers qs = type.getQualifiers(); - qs.setObjCGCAttr(Context.getObjCGCAttrKind(type)); - LValue R; R.LVType = Simple; - assert(address.getPointer()->getType()->isPointerTy()); - R.V = address.getPointer(); - R.ElementType = address.getElementType(); - R.IsKnownNonNull = address.isKnownNonNull(); - R.Initialize(type, qs, address.getAlignment(), BaseInfo, TBAAInfo); + R.initializeSimpleLValue(Addr, type, BaseInfo, TBAAInfo, Context); + R.Addr = Addr; + assert(Addr.getType()->isPointerTy()); return R; } @@ -434,26 +452,18 @@ class LValue { TBAAAccessInfo TBAAInfo) { LValue R; R.LVType = VectorElt; - R.V = vecAddress.getPointer(); - R.ElementType = vecAddress.getElementType(); R.VectorIdx = Idx; - R.IsKnownNonNull = vecAddress.isKnownNonNull(); - R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(), - BaseInfo, TBAAInfo); + R.Initialize(type, type.getQualifiers(), vecAddress, BaseInfo, TBAAInfo); return R; } - static LValue MakeExtVectorElt(Address vecAddress, llvm::Constant *Elts, + static LValue MakeExtVectorElt(Address Addr, llvm::Constant *Elts, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) { LValue R; R.LVType = ExtVectorElt; - R.V = vecAddress.getPointer(); - R.ElementType = vecAddress.getElementType(); R.VectorElts = Elts; - R.IsKnownNonNull = vecAddress.isKnownNonNull(); - R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(), - BaseInfo, TBAAInfo); + R.Initialize(type, type.getQualifiers(), Addr, BaseInfo, TBAAInfo); return R; } @@ -468,12 +478,8 @@ class LValue { TBAAAccessInfo TBAAInfo) { LValue R; R.LVType = BitField; - R.V = Addr.getPointer(); - R.ElementType = Addr.getElementType(); R.BitFieldInfo = &Info; - R.IsKnownNonNull = Addr.isKnownNonNull(); - R.Initialize(type, type.getQualifiers(), Addr.getAlignment(), BaseInfo, - TBAAInfo); + R.Initialize(type, type.getQualifiers(), Addr, BaseInfo, TBAAInfo); return R; } @@ -481,11 +487,9 @@ class LValue { QualType type) { LValue R; R.LVType = GlobalReg; - R.V = V; - R.ElementType = nullptr; - R.IsKnownNonNull = true; - R.Initialize(type, type.getQualifiers(), alignment, + R.Initialize(type, type.getQualifiers(), Address::invalid(), LValueBaseInfo(AlignmentSource::Decl), TBAAAccessInfo()); + R.V = V; return R; } @@ -494,12 +498,8 @@ class LValue { TBAAAccessInfo TBAAInfo) { LValue R; R.LVType = MatrixElt; - R.V = matAddress.getPointer(); - R.ElementType = matAddress.getElementType(); R.VectorIdx = Idx; - R.IsKnownNonNull = matAddress.isKnownNonNull(); - R.Initialize(type, type.getQualifiers(), matAddress.getAlignment(), - BaseInfo, TBAAInfo); + R.Initialize(type, type.getQualifiers(), matAddress, BaseInfo, TBAAInfo); return R; } @@ -643,17 +643,17 @@ class AggValueSlot { return NeedsGCBarriers_t(ObjCGCFlag); } - llvm::Value *getPointer() const { - return Addr.getPointer(); + llvm::Value *getPointer(QualType PointeeTy, CodeGenFunction &CGF) const; + + llvm::Value *emitRawPointer(CodeGenFunction &CGF) const { + return Addr.isValid() ? Addr.emitRawPointer(CGF) : nullptr; } Address getAddress() const { return Addr; } - bool isIgnored() const { - return !Addr.isValid(); - } + bool isIgnored() const { return !Addr.isValid(); } CharUnits getAlignment() const { return Addr.getAlignment(); diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp index f2ebaf7674523c..44103884940fd9 100644 --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -193,26 +193,35 @@ CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() { CGF.Builder.setDefaultConstrainedRounding(OldRounding); } -LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) { +static LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T, + bool ForPointeeType, + CodeGenFunction &CGF) { LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; - CharUnits Alignment = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo); - Address Addr(V, ConvertTypeForMem(T), Alignment); - return LValue::MakeAddr(Addr, T, getContext(), BaseInfo, TBAAInfo); + CharUnits Alignment = + CGF.CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo, ForPointeeType); + Address Addr = Address(V, CGF.ConvertTypeForMem(T), Alignment); + return CGF.MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo); +} + +LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) { + return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false, *this); } -/// Given a value of type T* that may not be to a complete object, -/// construct an l-value with the natural pointee alignment of T. LValue CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) { - LValueBaseInfo BaseInfo; - TBAAAccessInfo TBAAInfo; - CharUnits Align = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo, - /* forPointeeType= */ true); - Address Addr(V, ConvertTypeForMem(T), Align); - return MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo); + return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true, *this); +} + +LValue CodeGenFunction::MakeNaturalAlignRawAddrLValue(llvm::Value *V, + QualType T) { + return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false, *this); } +LValue CodeGenFunction::MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, + QualType T) { + return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true, *this); +} llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) { return CGM.getTypes().ConvertTypeForMem(T); @@ -525,7 +534,8 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) { ReturnBlock.getBlock()->eraseFromParent(); } if (ReturnValue.isValid()) { - auto *RetAlloca = dyn_cast(ReturnValue.getPointer()); + auto *RetAlloca = + dyn_cast(ReturnValue.emitRawPointer(*this)); if (RetAlloca && RetAlloca->use_empty()) { RetAlloca->eraseFromParent(); ReturnValue = Address::invalid(); @@ -1122,13 +1132,14 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, auto AI = CurFn->arg_begin(); if (CurFnInfo->getReturnInfo().isSRetAfterThis()) ++AI; - ReturnValue = - Address(&*AI, ConvertType(RetTy), - CurFnInfo->getReturnInfo().getIndirectAlign(), KnownNonNull); + ReturnValue = makeNaturalAddressForPointer( + &*AI, RetTy, CurFnInfo->getReturnInfo().getIndirectAlign(), false, + nullptr, nullptr, KnownNonNull); if (!CurFnInfo->getReturnInfo().getIndirectByVal()) { - ReturnValuePointer = CreateDefaultAlignTempAlloca( - ReturnValue.getPointer()->getType(), "result.ptr"); - Builder.CreateStore(ReturnValue.getPointer(), ReturnValuePointer); + ReturnValuePointer = + CreateDefaultAlignTempAlloca(ReturnValue.getType(), "result.ptr"); + Builder.CreateStore(ReturnValue.emitRawPointer(*this), + ReturnValuePointer); } } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca && !hasScalarEvaluationKind(CurFnInfo->getReturnType())) { @@ -1189,8 +1200,9 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, // or contains the address of the enclosing object). LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField); if (!LambdaThisCaptureField->getType()->isPointerType()) { - // If the enclosing object was captured by value, just use its address. - CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer(); + // If the enclosing object was captured by value, just use its + // address. Sign this pointer. + CXXThisValue = ThisFieldLValue.getPointer(*this); } else { // Load the lvalue pointed to by the field, since '*this' was captured // by reference. @@ -2012,8 +2024,9 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity()); Address begin = dest.withElementType(CGF.Int8Ty); - llvm::Value *end = Builder.CreateInBoundsGEP( - begin.getElementType(), begin.getPointer(), sizeInChars, "vla.end"); + llvm::Value *end = Builder.CreateInBoundsGEP(begin.getElementType(), + begin.emitRawPointer(CGF), + sizeInChars, "vla.end"); llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock(); llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop"); @@ -2024,7 +2037,7 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, CGF.EmitBlock(loopBB); llvm::PHINode *cur = Builder.CreatePHI(begin.getType(), 2, "vla.cur"); - cur->addIncoming(begin.getPointer(), originBB); + cur->addIncoming(begin.emitRawPointer(CGF), originBB); CharUnits curAlign = dest.getAlignment().alignmentOfArrayElement(baseSize); @@ -2217,10 +2230,10 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType, addr = addr.withElementType(baseType); } else { // Create the actual GEP. - addr = Address(Builder.CreateInBoundsGEP( - addr.getElementType(), addr.getPointer(), gepIndices, "array.begin"), - ConvertTypeForMem(eltType), - addr.getAlignment()); + addr = Address(Builder.CreateInBoundsGEP(addr.getElementType(), + addr.emitRawPointer(*this), + gepIndices, "array.begin"), + ConvertTypeForMem(eltType), addr.getAlignment()); } baseType = eltType; @@ -2561,7 +2574,7 @@ void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) { Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D, Address Addr) { assert(D->hasAttr() && "no annotate attribute"); - llvm::Value *V = Addr.getPointer(); + llvm::Value *V = Addr.emitRawPointer(*this); llvm::Type *VTy = V->getType(); auto *PTy = dyn_cast(VTy); unsigned AS = PTy ? PTy->getAddressSpace() : 0; diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h index e8f8aa601ed017..8dd6da5f85f11d 100644 --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -151,6 +151,9 @@ struct DominatingLLVMValue { /// Answer whether the given value needs extra work to be saved. static bool needsSaving(llvm::Value *value) { + if (!value) + return false; + // If it's not an instruction, we don't need to save. if (!isa(value)) return false; @@ -177,21 +180,28 @@ template <> struct DominatingValue
{ typedef Address type; struct saved_type { - DominatingLLVMValue::saved_type SavedValue; + DominatingLLVMValue::saved_type BasePtr; llvm::Type *ElementType; CharUnits Alignment; + DominatingLLVMValue::saved_type Offset; + llvm::PointerType *EffectiveType; }; static bool needsSaving(type value) { - return DominatingLLVMValue::needsSaving(value.getPointer()); + if (DominatingLLVMValue::needsSaving(value.getBasePointer()) || + DominatingLLVMValue::needsSaving(value.getOffset())) + return true; + return false; } static saved_type save(CodeGenFunction &CGF, type value) { - return { DominatingLLVMValue::save(CGF, value.getPointer()), - value.getElementType(), value.getAlignment() }; + return {DominatingLLVMValue::save(CGF, value.getBasePointer()), + value.getElementType(), value.getAlignment(), + DominatingLLVMValue::save(CGF, value.getOffset()), value.getType()}; } static type restore(CodeGenFunction &CGF, saved_type value) { - return Address(DominatingLLVMValue::restore(CGF, value.SavedValue), - value.ElementType, value.Alignment); + return Address(DominatingLLVMValue::restore(CGF, value.BasePtr), + value.ElementType, value.Alignment, + DominatingLLVMValue::restore(CGF, value.Offset)); } }; @@ -201,14 +211,26 @@ template <> struct DominatingValue { class saved_type { enum Kind { ScalarLiteral, ScalarAddress, AggregateLiteral, AggregateAddress, ComplexAddress }; - - llvm::Value *Value; - llvm::Type *ElementType; + union { + struct { + DominatingLLVMValue::saved_type first, second; + } Vals; + DominatingValue
::saved_type AggregateAddr; + }; LLVM_PREFERRED_TYPE(Kind) unsigned K : 3; - unsigned Align : 29; - saved_type(llvm::Value *v, llvm::Type *e, Kind k, unsigned a = 0) - : Value(v), ElementType(e), K(k), Align(a) {} + unsigned IsVolatile : 1; + + saved_type(DominatingLLVMValue::saved_type Val1, unsigned K) + : Vals{Val1, DominatingLLVMValue::saved_type()}, K(K) {} + + saved_type(DominatingLLVMValue::saved_type Val1, + DominatingLLVMValue::saved_type Val2) + : Vals{Val1, Val2}, K(ComplexAddress) {} + + saved_type(DominatingValue
::saved_type AggregateAddr, + bool IsVolatile, unsigned K) + : AggregateAddr(AggregateAddr), K(K) {} public: static bool needsSaving(RValue value); @@ -659,7 +681,7 @@ class CodeGenFunction : public CodeGenTypeCache { llvm::Value *Size; public: - CallLifetimeEnd(Address addr, llvm::Value *size) + CallLifetimeEnd(RawAddress addr, llvm::Value *size) : Addr(addr.getPointer()), Size(size) {} void Emit(CodeGenFunction &CGF, Flags flags) override { @@ -684,7 +706,7 @@ class CodeGenFunction : public CodeGenTypeCache { }; /// i32s containing the indexes of the cleanup destinations. - Address NormalCleanupDest = Address::invalid(); + RawAddress NormalCleanupDest = RawAddress::invalid(); unsigned NextCleanupDestIndex = 1; @@ -819,10 +841,10 @@ class CodeGenFunction : public CodeGenTypeCache { template void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) { if (!isInConditionalBranch()) - return pushCleanupAfterFullExprWithActiveFlag(Kind, Address::invalid(), - A...); + return pushCleanupAfterFullExprWithActiveFlag( + Kind, RawAddress::invalid(), A...); - Address ActiveFlag = createCleanupActiveFlag(); + RawAddress ActiveFlag = createCleanupActiveFlag(); assert(!DominatingValue
::needsSaving(ActiveFlag) && "cleanup active flag should never need saving"); @@ -835,7 +857,7 @@ class CodeGenFunction : public CodeGenTypeCache { template void pushCleanupAfterFullExprWithActiveFlag(CleanupKind Kind, - Address ActiveFlag, As... A) { + RawAddress ActiveFlag, As... A) { LifetimeExtendedCleanupHeader Header = {sizeof(T), Kind, ActiveFlag.isValid()}; @@ -850,7 +872,7 @@ class CodeGenFunction : public CodeGenTypeCache { new (Buffer) LifetimeExtendedCleanupHeader(Header); new (Buffer + sizeof(Header)) T(A...); if (Header.IsConditional) - new (Buffer + sizeof(Header) + sizeof(T)) Address(ActiveFlag); + new (Buffer + sizeof(Header) + sizeof(T)) RawAddress(ActiveFlag); } /// Set up the last cleanup that was pushed as a conditional @@ -859,8 +881,8 @@ class CodeGenFunction : public CodeGenTypeCache { initFullExprCleanupWithFlag(createCleanupActiveFlag()); } - void initFullExprCleanupWithFlag(Address ActiveFlag); - Address createCleanupActiveFlag(); + void initFullExprCleanupWithFlag(RawAddress ActiveFlag); + RawAddress createCleanupActiveFlag(); /// PushDestructorCleanup - Push a cleanup to call the /// complete-object destructor of an object of the given type at the @@ -1048,7 +1070,7 @@ class CodeGenFunction : public CodeGenTypeCache { QualType VarTy = LocalVD->getType(); if (VarTy->isReferenceType()) { Address Temp = CGF.CreateMemTemp(VarTy); - CGF.Builder.CreateStore(TempAddr.getPointer(), Temp); + CGF.Builder.CreateStore(TempAddr.emitRawPointer(CGF), Temp); TempAddr = Temp; } SavedTempAddresses.try_emplace(LocalVD, TempAddr); @@ -1243,10 +1265,12 @@ class CodeGenFunction : public CodeGenTypeCache { /// one branch or the other of a conditional expression. bool isInConditionalBranch() const { return OutermostConditional != nullptr; } - void setBeforeOutermostConditional(llvm::Value *value, Address addr) { + void setBeforeOutermostConditional(llvm::Value *value, Address addr, + CodeGenFunction &CGF) { assert(isInConditionalBranch()); llvm::BasicBlock *block = OutermostConditional->getStartingBlock(); - auto store = new llvm::StoreInst(value, addr.getPointer(), &block->back()); + auto store = + new llvm::StoreInst(value, addr.emitRawPointer(CGF), &block->back()); store->setAlignment(addr.getAlignment().getAsAlign()); } @@ -1601,7 +1625,7 @@ class CodeGenFunction : public CodeGenTypeCache { /// If \p StepV is null, the default increment is 1. void maybeUpdateMCDCTestVectorBitmap(const Expr *E) { if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) { - PGO.emitMCDCTestVectorBitmapUpdate(Builder, E, MCDCCondBitmapAddr); + PGO.emitMCDCTestVectorBitmapUpdate(Builder, E, MCDCCondBitmapAddr, *this); PGO.setCurrentStmt(E); } } @@ -1609,7 +1633,7 @@ class CodeGenFunction : public CodeGenTypeCache { /// Update the MCDC temp value with the condition's evaluated result. void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val) { if (isMCDCCoverageEnabled()) { - PGO.emitMCDCCondBitmapUpdate(Builder, E, MCDCCondBitmapAddr, Val); + PGO.emitMCDCCondBitmapUpdate(Builder, E, MCDCCondBitmapAddr, Val, *this); PGO.setCurrentStmt(E); } } @@ -1704,7 +1728,7 @@ class CodeGenFunction : public CodeGenTypeCache { : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue), OldCXXThisAlignment(CGF.CXXThisAlignment), SourceLocScope(E, CGF.CurSourceLocExprScope) { - CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getPointer(); + CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getBasePointer(); CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment(); } ~CXXDefaultInitExprScope() { @@ -2090,7 +2114,7 @@ class CodeGenFunction : public CodeGenTypeCache { llvm::Value *getExceptionFromSlot(); llvm::Value *getSelectorFromSlot(); - Address getNormalCleanupDestSlot(); + RawAddress getNormalCleanupDestSlot(); llvm::BasicBlock *getUnreachableBlock() { if (!UnreachableBlock) { @@ -2579,10 +2603,40 @@ class CodeGenFunction : public CodeGenTypeCache { // Helpers //===--------------------------------------------------------------------===// + Address mergeAddressesInConditionalExpr(Address LHS, Address RHS, + llvm::BasicBlock *LHSBlock, + llvm::BasicBlock *RHSBlock, + llvm::BasicBlock *MergeBlock, + QualType MergedType) { + Builder.SetInsertPoint(MergeBlock); + llvm::PHINode *PtrPhi = Builder.CreatePHI(LHS.getType(), 2, "cond"); + PtrPhi->addIncoming(LHS.getBasePointer(), LHSBlock); + PtrPhi->addIncoming(RHS.getBasePointer(), RHSBlock); + LHS.replaceBasePointer(PtrPhi); + LHS.setAlignment(std::min(LHS.getAlignment(), RHS.getAlignment())); + return LHS; + } + + /// Construct an address with the natural alignment of T. If a pointer to T + /// is expected to be signed, the pointer passed to this function must have + /// been signed, and the returned Address will have the pointer authentication + /// information needed to authenticate the signed pointer. + Address makeNaturalAddressForPointer( + llvm::Value *Ptr, QualType T, CharUnits Alignment = CharUnits::Zero(), + bool ForPointeeType = false, LValueBaseInfo *BaseInfo = nullptr, + TBAAAccessInfo *TBAAInfo = nullptr, + KnownNonNull_t IsKnownNonNull = NotKnownNonNull) { + if (Alignment.isZero()) + Alignment = + CGM.getNaturalTypeAlignment(T, BaseInfo, TBAAInfo, ForPointeeType); + return Address(Ptr, ConvertTypeForMem(T), Alignment, nullptr, + IsKnownNonNull); + } + LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source = AlignmentSource::Type) { - return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source), - CGM.getTBAAAccessInfo(T)); + return MakeAddrLValue(Addr, T, LValueBaseInfo(Source), + CGM.getTBAAAccessInfo(T)); } LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo, @@ -2592,6 +2646,14 @@ class CodeGenFunction : public CodeGenTypeCache { LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, AlignmentSource Source = AlignmentSource::Type) { + return MakeAddrLValue(makeNaturalAddressForPointer(V, T, Alignment), T, + LValueBaseInfo(Source), CGM.getTBAAAccessInfo(T)); + } + + /// Same as MakeAddrLValue above except that the pointer is known to be + /// unsigned. + LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, + AlignmentSource Source = AlignmentSource::Type) { Address Addr(V, ConvertTypeForMem(T), Alignment); return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source), CGM.getTBAAAccessInfo(T)); @@ -2604,9 +2666,18 @@ class CodeGenFunction : public CodeGenTypeCache { TBAAAccessInfo()); } + /// Given a value of type T* that may not be to a complete object, construct + /// an l-value with the natural pointee alignment of T. LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T); + LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T); + /// Same as MakeNaturalAlignPointeeAddrLValue except that the pointer is known + /// to be unsigned. + LValue MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, QualType T); + + LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T); + Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo = nullptr, TBAAAccessInfo *PointeeTBAAInfo = nullptr); @@ -2655,13 +2726,13 @@ class CodeGenFunction : public CodeGenTypeCache { /// more efficient if the caller knows that the address will not be exposed. llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp", llvm::Value *ArraySize = nullptr); - Address CreateTempAlloca(llvm::Type *Ty, CharUnits align, - const Twine &Name = "tmp", - llvm::Value *ArraySize = nullptr, - Address *Alloca = nullptr); - Address CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align, - const Twine &Name = "tmp", - llvm::Value *ArraySize = nullptr); + RawAddress CreateTempAlloca(llvm::Type *Ty, CharUnits align, + const Twine &Name = "tmp", + llvm::Value *ArraySize = nullptr, + RawAddress *Alloca = nullptr); + RawAddress CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align, + const Twine &Name = "tmp", + llvm::Value *ArraySize = nullptr); /// CreateDefaultAlignedTempAlloca - This creates an alloca with the /// default ABI alignment of the given LLVM type. @@ -2673,8 +2744,8 @@ class CodeGenFunction : public CodeGenTypeCache { /// not hand this address off to arbitrary IRGen routines, and especially /// do not pass it as an argument to a function that might expect a /// properly ABI-aligned value. - Address CreateDefaultAlignTempAlloca(llvm::Type *Ty, - const Twine &Name = "tmp"); + RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty, + const Twine &Name = "tmp"); /// CreateIRTemp - Create a temporary IR object of the given type, with /// appropriate alignment. This routine should only be used when an temporary @@ -2684,32 +2755,31 @@ class CodeGenFunction : public CodeGenTypeCache { /// /// That is, this is exactly equivalent to CreateMemTemp, but calling /// ConvertType instead of ConvertTypeForMem. - Address CreateIRTemp(QualType T, const Twine &Name = "tmp"); + RawAddress CreateIRTemp(QualType T, const Twine &Name = "tmp"); /// CreateMemTemp - Create a temporary memory object of the given type, with /// appropriate alignmen and cast it to the default address space. Returns /// the original alloca instruction by \p Alloca if it is not nullptr. - Address CreateMemTemp(QualType T, const Twine &Name = "tmp", - Address *Alloca = nullptr); - Address CreateMemTemp(QualType T, CharUnits Align, const Twine &Name = "tmp", - Address *Alloca = nullptr); + RawAddress CreateMemTemp(QualType T, const Twine &Name = "tmp", + RawAddress *Alloca = nullptr); + RawAddress CreateMemTemp(QualType T, CharUnits Align, + const Twine &Name = "tmp", + RawAddress *Alloca = nullptr); /// CreateMemTemp - Create a temporary memory object of the given type, with /// appropriate alignmen without casting it to the default address space. - Address CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp"); - Address CreateMemTempWithoutCast(QualType T, CharUnits Align, - const Twine &Name = "tmp"); + RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp"); + RawAddress CreateMemTempWithoutCast(QualType T, CharUnits Align, + const Twine &Name = "tmp"); /// CreateAggTemp - Create a temporary memory object for the given /// aggregate type. AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp", - Address *Alloca = nullptr) { - return AggValueSlot::forAddr(CreateMemTemp(T, Name, Alloca), - T.getQualifiers(), - AggValueSlot::IsNotDestructed, - AggValueSlot::DoesNotNeedGCBarriers, - AggValueSlot::IsNotAliased, - AggValueSlot::DoesNotOverlap); + RawAddress *Alloca = nullptr) { + return AggValueSlot::forAddr( + CreateMemTemp(T, Name, Alloca), T.getQualifiers(), + AggValueSlot::IsNotDestructed, AggValueSlot::DoesNotNeedGCBarriers, + AggValueSlot::IsNotAliased, AggValueSlot::DoesNotOverlap); } /// EvaluateExprAsBool - Perform the usual unary conversions on the specified @@ -3083,6 +3153,25 @@ class CodeGenFunction : public CodeGenTypeCache { /// calls to EmitTypeCheck can be skipped. bool sanitizePerformTypeCheck() const; + void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV, + QualType Type, SanitizerSet SkippedChecks = SanitizerSet(), + llvm::Value *ArraySize = nullptr) { + if (!sanitizePerformTypeCheck()) + return; + EmitTypeCheck(TCK, Loc, LV.emitRawPointer(*this), Type, LV.getAlignment(), + SkippedChecks, ArraySize); + } + + void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, Address Addr, + QualType Type, CharUnits Alignment = CharUnits::Zero(), + SanitizerSet SkippedChecks = SanitizerSet(), + llvm::Value *ArraySize = nullptr) { + if (!sanitizePerformTypeCheck()) + return; + EmitTypeCheck(TCK, Loc, Addr.emitRawPointer(*this), Type, Alignment, + SkippedChecks, ArraySize); + } + /// Emit a check that \p V is the address of storage of the /// appropriate size and alignment for an object of type \p Type /// (or if ArraySize is provided, for an array of that bound). @@ -3183,17 +3272,17 @@ class CodeGenFunction : public CodeGenTypeCache { /// Address with original alloca instruction. Invalid if the variable was /// emitted as a global constant. - Address AllocaAddr; + RawAddress AllocaAddr; struct Invalid {}; AutoVarEmission(Invalid) : Variable(nullptr), Addr(Address::invalid()), - AllocaAddr(Address::invalid()) {} + AllocaAddr(RawAddress::invalid()) {} AutoVarEmission(const VarDecl &variable) : Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr), IsEscapingByRef(false), IsConstantAggregate(false), - SizeForLifetimeMarkers(nullptr), AllocaAddr(Address::invalid()) {} + SizeForLifetimeMarkers(nullptr), AllocaAddr(RawAddress::invalid()) {} bool wasEmittedAsGlobal() const { return !Addr.isValid(); } @@ -3216,7 +3305,7 @@ class CodeGenFunction : public CodeGenTypeCache { } /// Returns the address for the original alloca instruction. - Address getOriginalAllocatedAddress() const { return AllocaAddr; } + RawAddress getOriginalAllocatedAddress() const { return AllocaAddr; } /// Returns the address of the object within this declaration. /// Note that this does not chase the forwarding pointer for @@ -3246,23 +3335,32 @@ class CodeGenFunction : public CodeGenTypeCache { llvm::GlobalValue::LinkageTypes Linkage); class ParamValue { - llvm::Value *Value; - llvm::Type *ElementType; - unsigned Alignment; - ParamValue(llvm::Value *V, llvm::Type *T, unsigned A) - : Value(V), ElementType(T), Alignment(A) {} + union { + Address Addr; + llvm::Value *Value; + }; + + bool IsIndirect; + + ParamValue(llvm::Value *V) : Value(V), IsIndirect(false) {} + ParamValue(Address A) : Addr(A), IsIndirect(true) {} + public: static ParamValue forDirect(llvm::Value *value) { - return ParamValue(value, nullptr, 0); + return ParamValue(value); } static ParamValue forIndirect(Address addr) { assert(!addr.getAlignment().isZero()); - return ParamValue(addr.getPointer(), addr.getElementType(), - addr.getAlignment().getQuantity()); + return ParamValue(addr); } - bool isIndirect() const { return Alignment != 0; } - llvm::Value *getAnyValue() const { return Value; } + bool isIndirect() const { return IsIndirect; } + llvm::Value *getAnyValue() const { + if (!isIndirect()) + return Value; + assert(!Addr.hasOffset() && "unexpected offset"); + return Addr.getBasePointer(); + } llvm::Value *getDirectValue() const { assert(!isIndirect()); @@ -3271,8 +3369,7 @@ class CodeGenFunction : public CodeGenTypeCache { Address getIndirectAddress() const { assert(isIndirect()); - return Address(Value, ElementType, CharUnits::fromQuantity(Alignment), - KnownNonNull); + return Addr; } }; @@ -4182,6 +4279,9 @@ class CodeGenFunction : public CodeGenTypeCache { const Twine &name = ""); llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name = ""); + llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee, + ArrayRef
args, + const Twine &name = ""); llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee, ArrayRef args, const Twine &name = ""); @@ -4208,6 +4308,12 @@ class CodeGenFunction : public CodeGenTypeCache { CXXDtorType Type, const CXXRecordDecl *RD); + llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) { + return Addr.getBasePointer(); + } + + bool isPointerKnownNonNull(const Expr *E); + // Return the copy constructor name with the prefix "__copy_constructor_" // removed. static std::string getNonTrivialCopyConstructorStr(QualType QT, @@ -4780,6 +4886,11 @@ class CodeGenFunction : public CodeGenTypeCache { SourceLocation Loc, const Twine &Name = ""); + Address EmitCheckedInBoundsGEP(Address Addr, ArrayRef IdxList, + llvm::Type *elementType, bool SignedIndices, + bool IsSubtraction, SourceLocation Loc, + CharUnits Align, const Twine &Name = ""); + /// Specifies which type of sanitizer check to apply when handling a /// particular builtin. enum BuiltinCheckKind { @@ -4842,6 +4953,10 @@ class CodeGenFunction : public CodeGenTypeCache { void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc, AbstractCallee AC, unsigned ParmNum); + void EmitNonNullArgCheck(Address Addr, QualType ArgType, + SourceLocation ArgLoc, AbstractCallee AC, + unsigned ParmNum); + /// EmitCallArg - Emit a single call argument. void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType); @@ -5050,7 +5165,7 @@ DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) { CGF.CreateTempAlloca(value->getType(), align, "cond-cleanup.save"); CGF.Builder.CreateStore(value, alloca); - return saved_type(alloca.getPointer(), true); + return saved_type(alloca.emitRawPointer(CGF), true); } inline llvm::Value *DominatingLLVMValue::restore(CodeGenFunction &CGF, diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index e3ed5e90f2d36b..00b3bfcaa0bc25 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -7267,7 +7267,7 @@ void CodeGenFunction::EmitDeclMetadata() { for (auto &I : LocalDeclMap) { const Decl *D = I.first; - llvm::Value *Addr = I.second.getPointer(); + llvm::Value *Addr = I.second.emitRawPointer(*this); if (auto *Alloca = dyn_cast(Addr)) { llvm::Value *DAddr = GetPointerConstant(getLLVMContext(), D); Alloca->setMetadata( diff --git a/clang/lib/CodeGen/CodeGenPGO.cpp b/clang/lib/CodeGen/CodeGenPGO.cpp index 2619edfeb7dc76..76704c4d7be4a4 100644 --- a/clang/lib/CodeGen/CodeGenPGO.cpp +++ b/clang/lib/CodeGen/CodeGenPGO.cpp @@ -1239,7 +1239,8 @@ void CodeGenPGO::emitMCDCParameters(CGBuilderTy &Builder) { void CodeGenPGO::emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, const Expr *S, - Address MCDCCondBitmapAddr) { + Address MCDCCondBitmapAddr, + CodeGenFunction &CGF) { if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState) return; @@ -1262,7 +1263,7 @@ void CodeGenPGO::emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, Builder.getInt64(FunctionHash), Builder.getInt32(RegionMCDCState->BitmapBytes), Builder.getInt32(MCDCTestVectorBitmapOffset), - MCDCCondBitmapAddr.getPointer()}; + MCDCCondBitmapAddr.emitRawPointer(CGF)}; Builder.CreateCall( CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_tvbitmap_update), Args); } @@ -1283,7 +1284,8 @@ void CodeGenPGO::emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S, void CodeGenPGO::emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S, Address MCDCCondBitmapAddr, - llvm::Value *Val) { + llvm::Value *Val, + CodeGenFunction &CGF) { if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState) return; @@ -1312,7 +1314,7 @@ void CodeGenPGO::emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S, llvm::Value *Args[5] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), Builder.getInt64(FunctionHash), Builder.getInt32(Branch.ID), - MCDCCondBitmapAddr.getPointer(), Val}; + MCDCCondBitmapAddr.emitRawPointer(CGF), Val}; Builder.CreateCall( CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_condbitmap_update), Args); diff --git a/clang/lib/CodeGen/CodeGenPGO.h b/clang/lib/CodeGen/CodeGenPGO.h index 036fbf6815a49d..9d66ffad6f4350 100644 --- a/clang/lib/CodeGen/CodeGenPGO.h +++ b/clang/lib/CodeGen/CodeGenPGO.h @@ -113,12 +113,14 @@ class CodeGenPGO { void emitCounterSetOrIncrement(CGBuilderTy &Builder, const Stmt *S, llvm::Value *StepV); void emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, const Expr *S, - Address MCDCCondBitmapAddr); + Address MCDCCondBitmapAddr, + CodeGenFunction &CGF); void emitMCDCParameters(CGBuilderTy &Builder); void emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S, Address MCDCCondBitmapAddr); void emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S, - Address MCDCCondBitmapAddr, llvm::Value *Val); + Address MCDCCondBitmapAddr, llvm::Value *Val, + CodeGenFunction &CGF); /// Return the region count for the counter at the given index. uint64_t getRegionCount(const Stmt *S) { diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp index bdd53a192f828a..fd71317572f0c9 100644 --- a/clang/lib/CodeGen/ItaniumCXXABI.cpp +++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -307,10 +307,6 @@ class ItaniumCXXABI : public CodeGen::CGCXXABI { CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base, const CXXRecordDecl *NearestVBase); - llvm::Constant * - getVTableAddressPointForConstExpr(BaseSubobject Base, - const CXXRecordDecl *VTableClass) override; - llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) override; @@ -646,7 +642,7 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer( // Apply the adjustment and cast back to the original struct type // for consistency. - llvm::Value *This = ThisAddr.getPointer(); + llvm::Value *This = ThisAddr.emitRawPointer(CGF); This = Builder.CreateInBoundsGEP(Builder.getInt8Ty(), This, Adj); ThisPtrForCall = This; @@ -850,7 +846,7 @@ llvm::Value *ItaniumCXXABI::EmitMemberDataPointerAddress( CGBuilderTy &Builder = CGF.Builder; // Apply the offset, which we assume is non-null. - return Builder.CreateInBoundsGEP(CGF.Int8Ty, Base.getPointer(), MemPtr, + return Builder.CreateInBoundsGEP(CGF.Int8Ty, Base.emitRawPointer(CGF), MemPtr, "memptr.offset"); } @@ -1245,7 +1241,7 @@ void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF, CGF.getPointerAlign()); // Apply the offset. - llvm::Value *CompletePtr = Ptr.getPointer(); + llvm::Value *CompletePtr = Ptr.emitRawPointer(CGF); CompletePtr = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, CompletePtr, Offset); @@ -1482,7 +1478,8 @@ llvm::Value *ItaniumCXXABI::emitDynamicCastCall( computeOffsetHint(CGF.getContext(), SrcDecl, DestDecl).getQuantity()); // Emit the call to __dynamic_cast. - llvm::Value *Args[] = {ThisAddr.getPointer(), SrcRTTI, DestRTTI, OffsetHint}; + llvm::Value *Args[] = {ThisAddr.emitRawPointer(CGF), SrcRTTI, DestRTTI, + OffsetHint}; llvm::Value *Value = CGF.EmitNounwindRuntimeCall(getItaniumDynamicCastFn(CGF), Args); @@ -1571,7 +1568,7 @@ llvm::Value *ItaniumCXXABI::emitExactDynamicCast( VPtr, CGM.getTBAAVTablePtrAccessInfo(CGF.VoidPtrPtrTy)); llvm::Value *Success = CGF.Builder.CreateICmpEQ( VPtr, getVTableAddressPoint(BaseSubobject(SrcDecl, *Offset), DestDecl)); - llvm::Value *Result = ThisAddr.getPointer(); + llvm::Value *Result = ThisAddr.emitRawPointer(CGF); if (!Offset->isZero()) Result = CGF.Builder.CreateInBoundsGEP( CGF.CharTy, Result, @@ -1611,7 +1608,7 @@ llvm::Value *ItaniumCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF, PtrDiffLTy, OffsetToTop, CGF.getPointerAlign(), "offset.to.top"); } // Finally, add the offset to the pointer. - return CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ThisAddr.getPointer(), + return CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ThisAddr.emitRawPointer(CGF), OffsetToTop); } @@ -1792,8 +1789,8 @@ void ItaniumCXXABI::EmitDestructorCall(CodeGenFunction &CGF, else Callee = CGCallee::forDirect(CGM.getAddrOfCXXStructor(GD), GD); - CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, VTT, VTTTy, - nullptr); + CGF.EmitCXXDestructorCall(GD, Callee, CGF.getAsNaturalPointerTo(This, ThisTy), + ThisTy, VTT, VTTTy, nullptr); } void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, @@ -1952,11 +1949,6 @@ llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructorWithVTT( CGF.getPointerAlign()); } -llvm::Constant *ItaniumCXXABI::getVTableAddressPointForConstExpr( - BaseSubobject Base, const CXXRecordDecl *VTableClass) { - return getVTableAddressPoint(Base, VTableClass); -} - llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) { assert(VPtrOffset.isZero() && "Itanium ABI only supports zero vptr offsets"); @@ -2088,8 +2080,8 @@ llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall( ThisTy = D->getDestroyedType(); } - CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, nullptr, - QualType(), nullptr); + CGF.EmitCXXDestructorCall(GD, Callee, This.emitRawPointer(CGF), ThisTy, + nullptr, QualType(), nullptr); return nullptr; } @@ -2162,7 +2154,7 @@ static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF, int64_t VirtualAdjustment, bool IsReturnAdjustment) { if (!NonVirtualAdjustment && !VirtualAdjustment) - return InitialPtr.getPointer(); + return InitialPtr.emitRawPointer(CGF); Address V = InitialPtr.withElementType(CGF.Int8Ty); @@ -2195,10 +2187,10 @@ static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF, CGF.getPointerAlign()); } // Adjust our pointer. - ResultPtr = CGF.Builder.CreateInBoundsGEP( - V.getElementType(), V.getPointer(), Offset); + ResultPtr = CGF.Builder.CreateInBoundsGEP(V.getElementType(), + V.emitRawPointer(CGF), Offset); } else { - ResultPtr = V.getPointer(); + ResultPtr = V.emitRawPointer(CGF); } // In a derived-to-base conversion, the non-virtual adjustment is @@ -2284,7 +2276,7 @@ Address ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF, llvm::FunctionType::get(CGM.VoidTy, NumElementsPtr.getType(), false); llvm::FunctionCallee F = CGM.CreateRuntimeFunction(FTy, "__asan_poison_cxx_array_cookie"); - CGF.Builder.CreateCall(F, NumElementsPtr.getPointer()); + CGF.Builder.CreateCall(F, NumElementsPtr.emitRawPointer(CGF)); } // Finally, compute a pointer to the actual data buffer by skipping @@ -2315,7 +2307,7 @@ llvm::Value *ItaniumCXXABI::readArrayCookieImpl(CodeGenFunction &CGF, llvm::FunctionType::get(CGF.SizeTy, CGF.UnqualPtrTy, false); llvm::FunctionCallee F = CGM.CreateRuntimeFunction(FTy, "__asan_load_cxx_array_cookie"); - return CGF.Builder.CreateCall(F, numElementsPtr.getPointer()); + return CGF.Builder.CreateCall(F, numElementsPtr.emitRawPointer(CGF)); } CharUnits ARMCXXABI::getArrayCookieSizeImpl(QualType elementType) { @@ -2627,7 +2619,7 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF, // Call __cxa_guard_release. This cannot throw. CGF.EmitNounwindRuntimeCall(getGuardReleaseFn(CGM, guardPtrTy), - guardAddr.getPointer()); + guardAddr.emitRawPointer(CGF)); } else if (D.isLocalVarDecl()) { // For local variables, store 1 into the first byte of the guard variable // after the object initialization completes so that initialization is @@ -3120,10 +3112,10 @@ LValue ItaniumCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, LValue LV; if (VD->getType()->isReferenceType()) - LV = CGF.MakeNaturalAlignAddrLValue(CallVal, LValType); + LV = CGF.MakeNaturalAlignRawAddrLValue(CallVal, LValType); else - LV = CGF.MakeAddrLValue(CallVal, LValType, - CGF.getContext().getDeclAlign(VD)); + LV = CGF.MakeRawAddrLValue(CallVal, LValType, + CGF.getContext().getDeclAlign(VD)); // FIXME: need setObjCGCLValueClass? return LV; } @@ -4604,7 +4596,7 @@ static void InitCatchParam(CodeGenFunction &CGF, CGF.Builder.CreateStore(Casted, ExnPtrTmp); // Bind the reference to the temporary. - AdjustedExn = ExnPtrTmp.getPointer(); + AdjustedExn = ExnPtrTmp.emitRawPointer(CGF); } } diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp index 172c4c937b9728..d38a26940a3cb6 100644 --- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp +++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp @@ -327,10 +327,6 @@ class MicrosoftCXXABI : public CGCXXABI { CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base, const CXXRecordDecl *NearestVBase) override; - llvm::Constant * - getVTableAddressPointForConstExpr(BaseSubobject Base, - const CXXRecordDecl *VTableClass) override; - llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) override; @@ -937,7 +933,7 @@ void MicrosoftCXXABI::emitBeginCatch(CodeGenFunction &CGF, } CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(*CatchParam); - CPI->setArgOperand(2, var.getObjectAddress(CGF).getPointer()); + CPI->setArgOperand(2, var.getObjectAddress(CGF).emitRawPointer(CGF)); CGF.EHStack.pushCleanup(NormalCleanup, CPI); CGF.EmitAutoVarCleanups(var); } @@ -974,7 +970,7 @@ MicrosoftCXXABI::performBaseAdjustment(CodeGenFunction &CGF, Address Value, llvm::Value *Offset = GetVirtualBaseClassOffset(CGF, Value, SrcDecl, PolymorphicBase); llvm::Value *Ptr = CGF.Builder.CreateInBoundsGEP( - Value.getElementType(), Value.getPointer(), Offset); + Value.getElementType(), Value.emitRawPointer(CGF), Offset); CharUnits VBaseAlign = CGF.CGM.getVBaseAlignment(Value.getAlignment(), SrcDecl, PolymorphicBase); return std::make_tuple(Address(Ptr, CGF.Int8Ty, VBaseAlign), Offset, @@ -1011,7 +1007,7 @@ llvm::Value *MicrosoftCXXABI::EmitTypeid(CodeGenFunction &CGF, llvm::Type *StdTypeInfoPtrTy) { std::tie(ThisPtr, std::ignore, std::ignore) = performBaseAdjustment(CGF, ThisPtr, SrcRecordTy); - llvm::CallBase *Typeid = emitRTtypeidCall(CGF, ThisPtr.getPointer()); + llvm::CallBase *Typeid = emitRTtypeidCall(CGF, ThisPtr.emitRawPointer(CGF)); return CGF.Builder.CreateBitCast(Typeid, StdTypeInfoPtrTy); } @@ -1033,7 +1029,7 @@ llvm::Value *MicrosoftCXXABI::emitDynamicCastCall( llvm::Value *Offset; std::tie(This, Offset, std::ignore) = performBaseAdjustment(CGF, This, SrcRecordTy); - llvm::Value *ThisPtr = This.getPointer(); + llvm::Value *ThisPtr = This.emitRawPointer(CGF); Offset = CGF.Builder.CreateTrunc(Offset, CGF.Int32Ty); // PVOID __RTDynamicCast( @@ -1065,7 +1061,7 @@ llvm::Value *MicrosoftCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF, llvm::FunctionCallee Function = CGF.CGM.CreateRuntimeFunction( llvm::FunctionType::get(CGF.Int8PtrTy, ArgTypes, false), "__RTCastToVoid"); - llvm::Value *Args[] = {Value.getPointer()}; + llvm::Value *Args[] = {Value.emitRawPointer(CGF)}; return CGF.EmitRuntimeCall(Function, Args); } @@ -1493,7 +1489,7 @@ Address MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall( llvm::Value *VBaseOffset = GetVirtualBaseClassOffset(CGF, Result, Derived, VBase); llvm::Value *VBasePtr = CGF.Builder.CreateInBoundsGEP( - Result.getElementType(), Result.getPointer(), VBaseOffset); + Result.getElementType(), Result.emitRawPointer(CGF), VBaseOffset); CharUnits VBaseAlign = CGF.CGM.getVBaseAlignment(Result.getAlignment(), Derived, VBase); Result = Address(VBasePtr, CGF.Int8Ty, VBaseAlign); @@ -1660,7 +1656,8 @@ void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF, llvm::Value *Implicit = getCXXDestructorImplicitParam(CGF, DD, Type, ForVirtualBase, Delegating); // = nullptr - CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, + CGF.EmitCXXDestructorCall(GD, Callee, CGF.getAsNaturalPointerTo(This, ThisTy), + ThisTy, /*ImplicitParam=*/Implicit, /*ImplicitParamTy=*/QualType(), nullptr); if (BaseDtorEndBB) { @@ -1791,13 +1788,6 @@ MicrosoftCXXABI::getVTableAddressPoint(BaseSubobject Base, return VFTablesMap[ID]; } -llvm::Constant *MicrosoftCXXABI::getVTableAddressPointForConstExpr( - BaseSubobject Base, const CXXRecordDecl *VTableClass) { - llvm::Constant *VFTable = getVTableAddressPoint(Base, VTableClass); - assert(VFTable && "Couldn't find a vftable for the given base?"); - return VFTable; -} - llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) { // getAddrOfVTable may return 0 if asked to get an address of a vtable which @@ -2013,8 +2003,9 @@ llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall( } This = adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, true); - RValue RV = CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, - ImplicitParam, Context.IntTy, CE); + RValue RV = + CGF.EmitCXXDestructorCall(GD, Callee, This.emitRawPointer(CGF), ThisTy, + ImplicitParam, Context.IntTy, CE); return RV.getScalarVal(); } @@ -2212,13 +2203,13 @@ llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF, Address This, const ThisAdjustment &TA) { if (TA.isEmpty()) - return This.getPointer(); + return This.emitRawPointer(CGF); This = This.withElementType(CGF.Int8Ty); llvm::Value *V; if (TA.Virtual.isEmpty()) { - V = This.getPointer(); + V = This.emitRawPointer(CGF); } else { assert(TA.Virtual.Microsoft.VtordispOffset < 0); // Adjust the this argument based on the vtordisp value. @@ -2227,7 +2218,7 @@ llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF, CharUnits::fromQuantity(TA.Virtual.Microsoft.VtordispOffset)); VtorDispPtr = VtorDispPtr.withElementType(CGF.Int32Ty); llvm::Value *VtorDisp = CGF.Builder.CreateLoad(VtorDispPtr, "vtordisp"); - V = CGF.Builder.CreateGEP(This.getElementType(), This.getPointer(), + V = CGF.Builder.CreateGEP(This.getElementType(), This.emitRawPointer(CGF), CGF.Builder.CreateNeg(VtorDisp)); // Unfortunately, having applied the vtordisp means that we no @@ -2264,11 +2255,11 @@ llvm::Value * MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret, const ReturnAdjustment &RA) { if (RA.isEmpty()) - return Ret.getPointer(); + return Ret.emitRawPointer(CGF); Ret = Ret.withElementType(CGF.Int8Ty); - llvm::Value *V = Ret.getPointer(); + llvm::Value *V = Ret.emitRawPointer(CGF); if (RA.Virtual.Microsoft.VBIndex) { assert(RA.Virtual.Microsoft.VBIndex > 0); int32_t IntSize = CGF.getIntSize().getQuantity(); @@ -2583,7 +2574,7 @@ struct ResetGuardBit final : EHScopeStack::Cleanup { struct CallInitThreadAbort final : EHScopeStack::Cleanup { llvm::Value *Guard; - CallInitThreadAbort(Address Guard) : Guard(Guard.getPointer()) {} + CallInitThreadAbort(RawAddress Guard) : Guard(Guard.getPointer()) {} void Emit(CodeGenFunction &CGF, Flags flags) override { // Calling _Init_thread_abort will reset the guard's state. @@ -3123,8 +3114,8 @@ MicrosoftCXXABI::GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF, llvm::Value **VBPtrOut) { CGBuilderTy &Builder = CGF.Builder; // Load the vbtable pointer from the vbptr in the instance. - llvm::Value *VBPtr = Builder.CreateInBoundsGEP(CGM.Int8Ty, This.getPointer(), - VBPtrOffset, "vbptr"); + llvm::Value *VBPtr = Builder.CreateInBoundsGEP( + CGM.Int8Ty, This.emitRawPointer(CGF), VBPtrOffset, "vbptr"); if (VBPtrOut) *VBPtrOut = VBPtr; @@ -3203,7 +3194,7 @@ llvm::Value *MicrosoftCXXABI::AdjustVirtualBase( Builder.CreateBr(SkipAdjustBB); CGF.EmitBlock(SkipAdjustBB); llvm::PHINode *Phi = Builder.CreatePHI(CGM.Int8PtrTy, 2, "memptr.base"); - Phi->addIncoming(Base.getPointer(), OriginalBB); + Phi->addIncoming(Base.emitRawPointer(CGF), OriginalBB); Phi->addIncoming(AdjustedBase, VBaseAdjustBB); return Phi; } @@ -3238,7 +3229,7 @@ llvm::Value *MicrosoftCXXABI::EmitMemberDataPointerAddress( Addr = AdjustVirtualBase(CGF, E, RD, Base, VirtualBaseAdjustmentOffset, VBPtrOffset); } else { - Addr = Base.getPointer(); + Addr = Base.emitRawPointer(CGF); } // Apply the offset, which we assume is non-null. @@ -3526,7 +3517,7 @@ CGCallee MicrosoftCXXABI::EmitLoadOfMemberFunctionPointer( ThisPtrForCall = AdjustVirtualBase(CGF, E, RD, This, VirtualBaseAdjustmentOffset, VBPtrOffset); } else { - ThisPtrForCall = This.getPointer(); + ThisPtrForCall = This.emitRawPointer(CGF); } if (NonVirtualBaseAdjustment) @@ -4445,10 +4436,7 @@ void MicrosoftCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) { llvm::GlobalVariable *TI = getThrowInfo(ThrowType); // Call into the runtime to throw the exception. - llvm::Value *Args[] = { - AI.getPointer(), - TI - }; + llvm::Value *Args[] = {AI.emitRawPointer(CGF), TI}; CGF.EmitNoreturnRuntimeCallOrInvoke(getThrowFn(), Args); } diff --git a/clang/lib/CodeGen/TargetInfo.h b/clang/lib/CodeGen/TargetInfo.h index 6893b50a3cfe90..b1dfe5bf8f274d 100644 --- a/clang/lib/CodeGen/TargetInfo.h +++ b/clang/lib/CodeGen/TargetInfo.h @@ -295,6 +295,11 @@ class TargetCodeGenInfo { /// Get the AST address space for alloca. virtual LangAS getASTAllocaAddressSpace() const { return LangAS::Default; } + Address performAddrSpaceCast(CodeGen::CodeGenFunction &CGF, Address Addr, + LangAS SrcAddr, LangAS DestAddr, + llvm::Type *DestTy, + bool IsNonNull = false) const; + /// Perform address space cast of an expression of pointer type. /// \param V is the LLVM value to be casted to another address space. /// \param SrcAddr is the language address space of \p V. diff --git a/clang/lib/CodeGen/Targets/NVPTX.cpp b/clang/lib/CodeGen/Targets/NVPTX.cpp index 8718f1ecf3a7e0..7dce5042c3dc20 100644 --- a/clang/lib/CodeGen/Targets/NVPTX.cpp +++ b/clang/lib/CodeGen/Targets/NVPTX.cpp @@ -85,7 +85,7 @@ class NVPTXTargetCodeGenInfo : public TargetCodeGenInfo { LValue Src) { llvm::Value *Handle = nullptr; llvm::Constant *C = - llvm::dyn_cast(Src.getAddress(CGF).getPointer()); + llvm::dyn_cast(Src.getAddress(CGF).emitRawPointer(CGF)); // Lookup `addrspacecast` through the constant pointer if any. if (auto *ASC = llvm::dyn_cast_or_null(C)) C = llvm::cast(ASC->getPointerOperand()); diff --git a/clang/lib/CodeGen/Targets/PPC.cpp b/clang/lib/CodeGen/Targets/PPC.cpp index 00b04723f17dd2..362add30b43500 100644 --- a/clang/lib/CodeGen/Targets/PPC.cpp +++ b/clang/lib/CodeGen/Targets/PPC.cpp @@ -513,9 +513,10 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, CharUnits RegSize = CharUnits::fromQuantity((isInt || IsSoftFloatABI) ? 4 : 8); llvm::Value *RegOffset = Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity())); - RegAddr = Address( - Builder.CreateInBoundsGEP(CGF.Int8Ty, RegAddr.getPointer(), RegOffset), - DirectTy, RegAddr.getAlignment().alignmentOfArrayElement(RegSize)); + RegAddr = Address(Builder.CreateInBoundsGEP( + CGF.Int8Ty, RegAddr.emitRawPointer(CGF), RegOffset), + DirectTy, + RegAddr.getAlignment().alignmentOfArrayElement(RegSize)); // Increase the used-register count. NumRegs = @@ -551,7 +552,7 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, // Round up address of argument to alignment CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty); if (Align > OverflowAreaAlign) { - llvm::Value *Ptr = OverflowArea.getPointer(); + llvm::Value *Ptr = OverflowArea.emitRawPointer(CGF); OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align), OverflowArea.getElementType(), Align); } @@ -560,7 +561,7 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, // Increase the overflow area. OverflowArea = Builder.CreateConstInBoundsByteGEP(OverflowArea, Size); - Builder.CreateStore(OverflowArea.getPointer(), OverflowAreaAddr); + Builder.CreateStore(OverflowArea.emitRawPointer(CGF), OverflowAreaAddr); CGF.EmitBranch(Cont); } diff --git a/clang/lib/CodeGen/Targets/Sparc.cpp b/clang/lib/CodeGen/Targets/Sparc.cpp index a337a52a94eca9..9025a633f328e2 100644 --- a/clang/lib/CodeGen/Targets/Sparc.cpp +++ b/clang/lib/CodeGen/Targets/Sparc.cpp @@ -326,7 +326,7 @@ Address SparcV9ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, // Update VAList. Address NextPtr = Builder.CreateConstInBoundsByteGEP(Addr, Stride, "ap.next"); - Builder.CreateStore(NextPtr.getPointer(), VAListAddr); + Builder.CreateStore(NextPtr.emitRawPointer(CGF), VAListAddr); return ArgAddr.withElementType(ArgTy); } diff --git a/clang/lib/CodeGen/Targets/SystemZ.cpp b/clang/lib/CodeGen/Targets/SystemZ.cpp index 6eb0c6ef2f7d63..deaafc85a31576 100644 --- a/clang/lib/CodeGen/Targets/SystemZ.cpp +++ b/clang/lib/CodeGen/Targets/SystemZ.cpp @@ -306,7 +306,7 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, // Update overflow_arg_area_ptr pointer llvm::Value *NewOverflowArgArea = CGF.Builder.CreateGEP( - OverflowArgArea.getElementType(), OverflowArgArea.getPointer(), + OverflowArgArea.getElementType(), OverflowArgArea.emitRawPointer(CGF), PaddedSizeV, "overflow_arg_area"); CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr); @@ -382,10 +382,9 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, Address MemAddr = RawMemAddr.withElementType(DirectTy); // Update overflow_arg_area_ptr pointer - llvm::Value *NewOverflowArgArea = - CGF.Builder.CreateGEP(OverflowArgArea.getElementType(), - OverflowArgArea.getPointer(), PaddedSizeV, - "overflow_arg_area"); + llvm::Value *NewOverflowArgArea = CGF.Builder.CreateGEP( + OverflowArgArea.getElementType(), OverflowArgArea.emitRawPointer(CGF), + PaddedSizeV, "overflow_arg_area"); CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr); CGF.EmitBranch(ContBlock); diff --git a/clang/lib/CodeGen/Targets/XCore.cpp b/clang/lib/CodeGen/Targets/XCore.cpp index aeb48f851e1693..88edb781a947b1 100644 --- a/clang/lib/CodeGen/Targets/XCore.cpp +++ b/clang/lib/CodeGen/Targets/XCore.cpp @@ -180,7 +180,7 @@ Address XCoreABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, // Increment the VAList. if (!ArgSize.isZero()) { Address APN = Builder.CreateConstInBoundsByteGEP(AP, ArgSize); - Builder.CreateStore(APN.getPointer(), VAListAddr); + Builder.CreateStore(APN.emitRawPointer(CGF), VAListAddr); } return Val; diff --git a/clang/utils/TableGen/MveEmitter.cpp b/clang/utils/TableGen/MveEmitter.cpp index 3a90eee5f1c921..aa20c758d84a60 100644 --- a/clang/utils/TableGen/MveEmitter.cpp +++ b/clang/utils/TableGen/MveEmitter.cpp @@ -575,7 +575,7 @@ class BuiltinArgResult : public Result { // Emit code to generate this result as a Value *. std::string asValue() override { if (AddressType) - return "(" + varname() + ".getPointer())"; + return "(" + varname() + ".emitRawPointer(*this))"; return Result::asValue(); } bool hasIntegerValue() const override { return Immediate; } diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h index 2a0c1e9e8c446b..2e2ec9a1c83026 100644 --- a/llvm/include/llvm/IR/IRBuilder.h +++ b/llvm/include/llvm/IR/IRBuilder.h @@ -2708,6 +2708,7 @@ class IRBuilder : public IRBuilderBase { IRBuilder(const IRBuilder &) = delete; InserterTy &getInserter() { return Inserter; } + const InserterTy &getInserter() const { return Inserter; } }; template From cd17082b24079a31eff0057abe407da5cfb7b0fc Mon Sep 17 00:00:00 2001 From: OverMighty Date: Wed, 27 Mar 2024 19:28:27 +0000 Subject: [PATCH 02/54] [libc][math][c23] Add remaining linux/* entrypoints for {,u}fromfp{,x}* (#86692) --- libc/config/linux/aarch64/entrypoints.txt | 16 ++++++++++++ libc/config/linux/arm/entrypoints.txt | 12 +++++++++ libc/config/linux/riscv/entrypoints.txt | 16 ++++++++++++ libc/docs/math/index.rst | 32 +++++++++++------------ 4 files changed, 60 insertions(+), 16 deletions(-) diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt index 78da7f0b334b1f..ab0be7e35dced2 100644 --- a/libc/config/linux/aarch64/entrypoints.txt +++ b/libc/config/linux/aarch64/entrypoints.txt @@ -396,6 +396,12 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.frexp libc.src.math.frexpf libc.src.math.frexpl + libc.src.math.fromfp + libc.src.math.fromfpf + libc.src.math.fromfpl + libc.src.math.fromfpx + libc.src.math.fromfpxf + libc.src.math.fromfpxl libc.src.math.hypot libc.src.math.hypotf libc.src.math.ilogb @@ -478,6 +484,12 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.trunc libc.src.math.truncf libc.src.math.truncl + libc.src.math.ufromfp + libc.src.math.ufromfpf + libc.src.math.ufromfpl + libc.src.math.ufromfpx + libc.src.math.ufromfpxf + libc.src.math.ufromfpxl ) if(LIBC_TYPES_HAS_FLOAT128) @@ -500,6 +512,8 @@ if(LIBC_TYPES_HAS_FLOAT128) libc.src.math.fminimum_mag_numf128 libc.src.math.fmodf128 libc.src.math.frexpf128 + libc.src.math.fromfpf128 + libc.src.math.fromfpxf128 libc.src.math.ilogbf128 libc.src.math.ldexpf128 libc.src.math.llogbf128 @@ -517,6 +531,8 @@ if(LIBC_TYPES_HAS_FLOAT128) libc.src.math.roundf128 libc.src.math.sqrtf128 libc.src.math.truncf128 + libc.src.math.ufromfpf128 + libc.src.math.ufromfpxf128 ) endif() diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt index 6e63e270280e7a..1d9d5ed67200c1 100644 --- a/libc/config/linux/arm/entrypoints.txt +++ b/libc/config/linux/arm/entrypoints.txt @@ -263,6 +263,12 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.frexp libc.src.math.frexpf libc.src.math.frexpl + libc.src.math.fromfp + libc.src.math.fromfpf + libc.src.math.fromfpl + libc.src.math.fromfpx + libc.src.math.fromfpxf + libc.src.math.fromfpxl libc.src.math.hypot libc.src.math.hypotf libc.src.math.ilogb @@ -345,6 +351,12 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.trunc libc.src.math.truncf libc.src.math.truncl + libc.src.math.ufromfp + libc.src.math.ufromfpf + libc.src.math.ufromfpl + libc.src.math.ufromfpx + libc.src.math.ufromfpxf + libc.src.math.ufromfpxl ) set(TARGET_LLVMLIBC_ENTRYPOINTS diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt index 5aae4e246cfb3c..96acd999efda4d 100644 --- a/libc/config/linux/riscv/entrypoints.txt +++ b/libc/config/linux/riscv/entrypoints.txt @@ -404,6 +404,12 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.frexp libc.src.math.frexpf libc.src.math.frexpl + libc.src.math.fromfp + libc.src.math.fromfpf + libc.src.math.fromfpl + libc.src.math.fromfpx + libc.src.math.fromfpxf + libc.src.math.fromfpxl libc.src.math.hypot libc.src.math.hypotf libc.src.math.ilogb @@ -486,6 +492,12 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.trunc libc.src.math.truncf libc.src.math.truncl + libc.src.math.ufromfp + libc.src.math.ufromfpf + libc.src.math.ufromfpl + libc.src.math.ufromfpx + libc.src.math.ufromfpxf + libc.src.math.ufromfpxl ) if(LIBC_TYPES_HAS_FLOAT128) @@ -508,6 +520,8 @@ if(LIBC_TYPES_HAS_FLOAT128) libc.src.math.fminimum_mag_numf128 libc.src.math.fmodf128 libc.src.math.frexpf128 + libc.src.math.fromfpf128 + libc.src.math.fromfpxf128 libc.src.math.ilogbf128 libc.src.math.ldexpf128 libc.src.math.llogbf128 @@ -525,6 +539,8 @@ if(LIBC_TYPES_HAS_FLOAT128) libc.src.math.roundf128 libc.src.math.sqrtf128 libc.src.math.truncf128 + libc.src.math.ufromfpf128 + libc.src.math.ufromfpxf128 ) endif() diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst index 080b6a4427f511..d8dee3c9061908 100644 --- a/libc/docs/math/index.rst +++ b/libc/docs/math/index.rst @@ -190,21 +190,21 @@ Basic Operations +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ | frexpf128 | |check| | |check| | | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| fromfp | |check| | | | | | | | | | | | | +| fromfp | |check| | |check| | |check| | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| fromfpf | |check| | | | | | | | | | | | | +| fromfpf | |check| | |check| | |check| | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| fromfpl | |check| | | | | | | | | | | | | +| fromfpl | |check| | |check| | |check| | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| fromfpf128 | |check| | | | | | | | | | | | | +| fromfpf128 | |check| | |check| | | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| fromfpx | |check| | | | | | | | | | | | | +| fromfpx | |check| | |check| | |check| | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| fromfpxf | |check| | | | | | | | | | | | | +| fromfpxf | |check| | |check| | |check| | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| fromfpxl | |check| | | | | | | | | | | | | +| fromfpxl | |check| | |check| | |check| | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| fromfpxf128 | |check| | | | | | | | | | | | | +| fromfpxf128 | |check| | |check| | | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ | ilogb | |check| | |check| | |check| | |check| | |check| | | | |check| | |check| | |check| | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ @@ -364,21 +364,21 @@ Basic Operations +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ | truncf128 | |check| | |check| | | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| ufromfp | |check| | | | | | | | | | | | | +| ufromfp | |check| | |check| | |check| | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| ufromfpf | |check| | | | | | | | | | | | | +| ufromfpf | |check| | |check| | |check| | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| ufromfpl | |check| | | | | | | | | | | | | +| ufromfpl | |check| | |check| | |check| | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| ufromfpf128 | |check| | | | | | | | | | | | | +| ufromfpf128 | |check| | |check| | | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| ufromfpx | |check| | | | | | | | | | | | | +| ufromfpx | |check| | |check| | |check| | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| ufromfpxf | |check| | | | | | | | | | | | | +| ufromfpxf | |check| | |check| | |check| | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| ufromfpxl | |check| | | | | | | | | | | | | +| ufromfpxl | |check| | |check| | |check| | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| ufromfpxf128 | |check| | | | | | | | | | | | | +| ufromfpxf128 | |check| | |check| | | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ From 4a5056be2e38bdf27125a41f3f1b7ae4233f9713 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Wed, 27 Mar 2024 14:37:55 -0500 Subject: [PATCH 03/54] [Libomptarget][NFC] Remove trivially true checks on function pointers (#86804) Summary: Previously we had an interface that checked these functions pointers to see if they are implemented by the plugin. This was removed as currently every single function is implemented as a part of the common interface. These checks are now always true and do nothing. --- openmp/libomptarget/src/device.cpp | 61 ++++++--------------------- openmp/libomptarget/src/interface.cpp | 6 +-- openmp/libomptarget/src/omptarget.cpp | 14 +++--- 3 files changed, 19 insertions(+), 62 deletions(-) diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp index 3345277d91d3a9..44a2facc8d3ddd 100644 --- a/openmp/libomptarget/src/device.cpp +++ b/openmp/libomptarget/src/device.cpp @@ -79,8 +79,7 @@ DeviceTy::~DeviceTy() { llvm::Error DeviceTy::init() { // Make call to init_requires if it exists for this plugin. int32_t Ret = 0; - if (RTL->init_requires) - Ret = RTL->init_requires(PM->getRequirements()); + Ret = RTL->init_requires(PM->getRequirements()); if (Ret != OFFLOAD_SUCCESS) return llvm::createStringError( llvm::inconvertibleErrorCode(), @@ -154,8 +153,6 @@ int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size, omp_get_initial_device(), HstPtrBegin, DeviceID, TgtPtrBegin, Size, /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);) - if (!AsyncInfo || !RTL->data_submit_async || !RTL->synchronize) - return RTL->data_submit(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size); return RTL->data_submit_async(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size, AsyncInfo); } @@ -176,8 +173,6 @@ int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin, DeviceID, TgtPtrBegin, omp_get_initial_device(), HstPtrBegin, Size, /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);) - if (!RTL->data_retrieve_async || !RTL->synchronize) - return RTL->data_retrieve(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size); return RTL->data_retrieve_async(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size, AsyncInfo); } @@ -196,7 +191,7 @@ int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr, RegionInterface.getCallbacks(), RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr, Size, /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);) - if (!AsyncInfo || !RTL->data_exchange_async || !RTL->synchronize) { + if (!AsyncInfo) { assert(RTL->data_exchange && "RTL->data_exchange is nullptr"); return RTL->data_exchange(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr, Size); @@ -206,9 +201,6 @@ int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr, } int32_t DeviceTy::notifyDataMapped(void *HstPtr, int64_t Size) { - if (!RTL->data_notify_mapped) - return OFFLOAD_SUCCESS; - DP("Notifying about new mapping: HstPtr=" DPxMOD ", Size=%" PRId64 "\n", DPxPTR(HstPtr), Size); @@ -220,9 +212,6 @@ int32_t DeviceTy::notifyDataMapped(void *HstPtr, int64_t Size) { } int32_t DeviceTy::notifyDataUnmapped(void *HstPtr) { - if (!RTL->data_notify_unmapped) - return OFFLOAD_SUCCESS; - DP("Notifying about an unmapping: HstPtr=" DPxMOD "\n", DPxPTR(HstPtr)); if (RTL->data_notify_unmapped(RTLDeviceID, HstPtr)) { @@ -242,70 +231,46 @@ int32_t DeviceTy::launchKernel(void *TgtEntryPtr, void **TgtVarsPtr, // Run region on device bool DeviceTy::printDeviceInfo() { - if (!RTL->print_device_info) - return false; RTL->print_device_info(RTLDeviceID); return true; } // Whether data can be copied to DstDevice directly bool DeviceTy::isDataExchangable(const DeviceTy &DstDevice) { - if (RTL != DstDevice.RTL || !RTL->is_data_exchangable) + if (RTL != DstDevice.RTL) return false; if (RTL->is_data_exchangable(RTLDeviceID, DstDevice.RTLDeviceID)) - return (RTL->data_exchange != nullptr) || - (RTL->data_exchange_async != nullptr); - + return true; return false; } int32_t DeviceTy::synchronize(AsyncInfoTy &AsyncInfo) { - if (RTL->synchronize) - return RTL->synchronize(RTLDeviceID, AsyncInfo); - return OFFLOAD_SUCCESS; + return RTL->synchronize(RTLDeviceID, AsyncInfo); } int32_t DeviceTy::queryAsync(AsyncInfoTy &AsyncInfo) { - if (RTL->query_async) - return RTL->query_async(RTLDeviceID, AsyncInfo); - - return synchronize(AsyncInfo); + return RTL->query_async(RTLDeviceID, AsyncInfo); } int32_t DeviceTy::createEvent(void **Event) { - if (RTL->create_event) - return RTL->create_event(RTLDeviceID, Event); - - return OFFLOAD_SUCCESS; + return RTL->create_event(RTLDeviceID, Event); } int32_t DeviceTy::recordEvent(void *Event, AsyncInfoTy &AsyncInfo) { - if (RTL->record_event) - return RTL->record_event(RTLDeviceID, Event, AsyncInfo); - - return OFFLOAD_SUCCESS; + return RTL->record_event(RTLDeviceID, Event, AsyncInfo); } int32_t DeviceTy::waitEvent(void *Event, AsyncInfoTy &AsyncInfo) { - if (RTL->wait_event) - return RTL->wait_event(RTLDeviceID, Event, AsyncInfo); - - return OFFLOAD_SUCCESS; + return RTL->wait_event(RTLDeviceID, Event, AsyncInfo); } int32_t DeviceTy::syncEvent(void *Event) { - if (RTL->sync_event) - return RTL->sync_event(RTLDeviceID, Event); - - return OFFLOAD_SUCCESS; + return RTL->sync_event(RTLDeviceID, Event); } int32_t DeviceTy::destroyEvent(void *Event) { - if (RTL->create_event) - return RTL->destroy_event(RTLDeviceID, Event); - - return OFFLOAD_SUCCESS; + return RTL->destroy_event(RTLDeviceID, Event); } void DeviceTy::dumpOffloadEntries() { @@ -321,7 +286,5 @@ void DeviceTy::dumpOffloadEntries() { } bool DeviceTy::useAutoZeroCopy() { - if (RTL->use_auto_zero_copy) - return RTL->use_auto_zero_copy(RTLDeviceID); - return false; + return RTL->use_auto_zero_copy(RTLDeviceID); } diff --git a/openmp/libomptarget/src/interface.cpp b/openmp/libomptarget/src/interface.cpp index b7f547f1ec3d5c..b562ba8818c39b 100644 --- a/openmp/libomptarget/src/interface.cpp +++ b/openmp/libomptarget/src/interface.cpp @@ -456,10 +456,8 @@ EXTERN void __tgt_set_info_flag(uint32_t NewInfoLevel) { assert(PM && "Runtime not initialized"); std::atomic &InfoLevel = getInfoLevelInternal(); InfoLevel.store(NewInfoLevel); - for (auto &R : PM->pluginAdaptors()) { - if (R.set_info_flag) - R.set_info_flag(NewInfoLevel); - } + for (auto &R : PM->pluginAdaptors()) + R.set_info_flag(NewInfoLevel); } EXTERN int __tgt_print_device_info(int64_t DeviceId) { diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp index 5bbf3a455c72ad..803e941fe83822 100644 --- a/openmp/libomptarget/src/omptarget.cpp +++ b/openmp/libomptarget/src/omptarget.cpp @@ -481,12 +481,10 @@ void *targetLockExplicit(void *HostPtr, size_t Size, int DeviceNum, FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str()); int32_t Err = 0; - if (!DeviceOrErr->RTL->data_lock) { - Err = DeviceOrErr->RTL->data_lock(DeviceNum, HostPtr, Size, &RC); - if (Err) { - DP("Could not lock ptr %p\n", HostPtr); - return nullptr; - } + Err = DeviceOrErr->RTL->data_lock(DeviceNum, HostPtr, Size, &RC); + if (Err) { + DP("Could not lock ptr %p\n", HostPtr); + return nullptr; } DP("%s returns device ptr " DPxMOD "\n", Name, DPxPTR(RC)); return RC; @@ -499,9 +497,7 @@ void targetUnlockExplicit(void *HostPtr, int DeviceNum, const char *Name) { if (!DeviceOrErr) FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str()); - if (!DeviceOrErr->RTL->data_unlock) - DeviceOrErr->RTL->data_unlock(DeviceNum, HostPtr); - + DeviceOrErr->RTL->data_unlock(DeviceNum, HostPtr); DP("%s returns\n", Name); } From 685d7855acb28f89aa948e0056d2807bf30d3971 Mon Sep 17 00:00:00 2001 From: Amy Kwan Date: Wed, 27 Mar 2024 14:19:00 -0500 Subject: [PATCH 04/54] Fix the -Wmissing-designated-field-initializers on the clang-ppc64le-rhel bot --- compiler-rt/lib/scudo/standalone/tests/strings_test.cpp | 2 +- compiler-rt/lib/scudo/standalone/tests/vector_test.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp b/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp index 3e41f67ba922b7..17a596d712d0ca 100644 --- a/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp @@ -136,7 +136,7 @@ TEST(ScudoStringsTest, CapacityIncreaseFails) { rlimit Limit = {}; EXPECT_EQ(0, getrlimit(RLIMIT_AS, &Limit)); - rlimit EmptyLimit = {.rlim_max = Limit.rlim_max}; + rlimit EmptyLimit = {.rlim_cur = 0, .rlim_max = Limit.rlim_max}; EXPECT_EQ(0, setrlimit(RLIMIT_AS, &EmptyLimit)); // Test requires that the default length is at least 6 characters. diff --git a/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp b/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp index b7678678d8a294..add62c5a42a3e4 100644 --- a/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp @@ -55,7 +55,7 @@ TEST(ScudoVectorTest, ReallocateFails) { rlimit Limit = {}; EXPECT_EQ(0, getrlimit(RLIMIT_AS, &Limit)); - rlimit EmptyLimit = {.rlim_max = Limit.rlim_max}; + rlimit EmptyLimit = {.rlim_cur = 0, .rlim_max = Limit.rlim_max}; EXPECT_EQ(0, setrlimit(RLIMIT_AS, &EmptyLimit)); V.resize(capacity); From abc270ae00cd991bf1b2125e880b9eb73d8d6727 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Wed, 27 Mar 2024 12:39:07 -0700 Subject: [PATCH 05/54] Finish revert "[SystemZ][z/OS] TXT records in the GOFF reader (#74526)" This finishes the revert started in aeb8628c218f8224e08dddcdd3199a445d8607a8 which didn't completely back out the original patch. --- llvm/lib/Object/GOFFObjectFile.cpp | 148 ----------------------------- 1 file changed, 148 deletions(-) diff --git a/llvm/lib/Object/GOFFObjectFile.cpp b/llvm/lib/Object/GOFFObjectFile.cpp index d8e1ddf2aef864..76a13559ebfe35 100644 --- a/llvm/lib/Object/GOFFObjectFile.cpp +++ b/llvm/lib/Object/GOFFObjectFile.cpp @@ -394,154 +394,6 @@ GOFFObjectFile::getSectionPrEsdRecord(uint32_t SectionIndex) const { return EsdRecord; } -uint32_t GOFFObjectFile::getSectionDefEsdId(DataRefImpl &Sec) const { - const uint8_t *EsdRecord = getSectionEdEsdRecord(Sec); - uint32_t Length; - ESDRecord::getLength(EsdRecord, Length); - if (Length == 0) { - const uint8_t *PrEsdRecord = getSectionPrEsdRecord(Sec); - if (PrEsdRecord) - EsdRecord = PrEsdRecord; - } - - uint32_t DefEsdId; - ESDRecord::getEsdId(EsdRecord, DefEsdId); - LLVM_DEBUG(dbgs() << "Got def EsdId: " << DefEsdId << '\n'); - return DefEsdId; -} - -void GOFFObjectFile::moveSectionNext(DataRefImpl &Sec) const { - Sec.d.a++; - if ((Sec.d.a) >= SectionList.size()) - Sec.d.a = 0; -} - -Expected GOFFObjectFile::getSectionName(DataRefImpl Sec) const { - DataRefImpl EdSym; - SectionEntryImpl EsdIds = SectionList[Sec.d.a]; - EdSym.d.a = EsdIds.d.a; - Expected Name = getSymbolName(EdSym); - if (Name) { - StringRef Res = *Name; - LLVM_DEBUG(dbgs() << "Got section: " << Res << '\n'); - LLVM_DEBUG(dbgs() << "Final section name: " << Res << '\n'); - Name = Res; - } - return Name; -} - -uint64_t GOFFObjectFile::getSectionAddress(DataRefImpl Sec) const { - uint32_t Offset; - const uint8_t *EsdRecord = getSectionEdEsdRecord(Sec); - ESDRecord::getOffset(EsdRecord, Offset); - return Offset; -} - -uint64_t GOFFObjectFile::getSectionSize(DataRefImpl Sec) const { - uint32_t Length; - uint32_t DefEsdId = getSectionDefEsdId(Sec); - const uint8_t *EsdRecord = EsdPtrs[DefEsdId]; - ESDRecord::getLength(EsdRecord, Length); - LLVM_DEBUG(dbgs() << "Got section size: " << Length << '\n'); - return static_cast(Length); -} - -// Unravel TXT records and expand fill characters to produce -// a contiguous sequence of bytes. -Expected> -GOFFObjectFile::getSectionContents(DataRefImpl Sec) const { - if (SectionDataCache.count(Sec.d.a)) { - auto &Buf = SectionDataCache[Sec.d.a]; - return ArrayRef(Buf); - } - uint64_t SectionSize = getSectionSize(Sec); - uint32_t DefEsdId = getSectionDefEsdId(Sec); - - const uint8_t *EdEsdRecord = getSectionEdEsdRecord(Sec); - bool FillBytePresent; - ESDRecord::getFillBytePresent(EdEsdRecord, FillBytePresent); - uint8_t FillByte = '\0'; - if (FillBytePresent) - ESDRecord::getFillByteValue(EdEsdRecord, FillByte); - - // Initialize section with fill byte. - SmallVector Data(SectionSize, FillByte); - - // Replace section with content from text records. - for (const uint8_t *TxtRecordInt : TextPtrs) { - const uint8_t *TxtRecordPtr = TxtRecordInt; - uint32_t TxtEsdId; - TXTRecord::getElementEsdId(TxtRecordPtr, TxtEsdId); - LLVM_DEBUG(dbgs() << "Got txt EsdId: " << TxtEsdId << '\n'); - - if (TxtEsdId != DefEsdId) - continue; - - uint32_t TxtDataOffset; - TXTRecord::getOffset(TxtRecordPtr, TxtDataOffset); - - uint16_t TxtDataSize; - TXTRecord::getDataLength(TxtRecordPtr, TxtDataSize); - - LLVM_DEBUG(dbgs() << "Record offset " << TxtDataOffset << ", data size " - << TxtDataSize << "\n"); - - SmallString<256> CompleteData; - CompleteData.reserve(TxtDataSize); - if (Error Err = TXTRecord::getData(TxtRecordPtr, CompleteData)) - return std::move(Err); - assert(CompleteData.size() == TxtDataSize && "Wrong length of data"); - std::copy(CompleteData.data(), CompleteData.data() + TxtDataSize, - Data.begin() + TxtDataOffset); - } - SectionDataCache[Sec.d.a] = Data; - return ArrayRef(Data); -} - -uint64_t GOFFObjectFile::getSectionAlignment(DataRefImpl Sec) const { - const uint8_t *EsdRecord = getSectionEdEsdRecord(Sec); - GOFF::ESDAlignment Pow2Alignment; - ESDRecord::getAlignment(EsdRecord, Pow2Alignment); - return 1ULL << static_cast(Pow2Alignment); -} - -bool GOFFObjectFile::isSectionText(DataRefImpl Sec) const { - const uint8_t *EsdRecord = getSectionEdEsdRecord(Sec); - GOFF::ESDExecutable Executable; - ESDRecord::getExecutable(EsdRecord, Executable); - return Executable == GOFF::ESD_EXE_CODE; -} - -bool GOFFObjectFile::isSectionData(DataRefImpl Sec) const { - const uint8_t *EsdRecord = getSectionEdEsdRecord(Sec); - GOFF::ESDExecutable Executable; - ESDRecord::getExecutable(EsdRecord, Executable); - return Executable == GOFF::ESD_EXE_DATA; -} - -bool GOFFObjectFile::isSectionNoLoad(DataRefImpl Sec) const { - const uint8_t *EsdRecord = getSectionEdEsdRecord(Sec); - GOFF::ESDLoadingBehavior LoadingBehavior; - ESDRecord::getLoadingBehavior(EsdRecord, LoadingBehavior); - return LoadingBehavior == GOFF::ESD_LB_NoLoad; -} - -bool GOFFObjectFile::isSectionReadOnlyData(DataRefImpl Sec) const { - if (!isSectionData(Sec)) - return false; - - const uint8_t *EsdRecord = getSectionEdEsdRecord(Sec); - GOFF::ESDLoadingBehavior LoadingBehavior; - ESDRecord::getLoadingBehavior(EsdRecord, LoadingBehavior); - return LoadingBehavior == GOFF::ESD_LB_Initial; -} - -bool GOFFObjectFile::isSectionZeroInit(DataRefImpl Sec) const { - // GOFF uses fill characters and fill characters are applied - // on getSectionContents() - so we say false to zero init. - return false; -} - section_iterator GOFFObjectFile::section_begin() const { DataRefImpl Sec; moveSectionNext(Sec); From 0d7ea50d20414e2da3019c45a0a3de804167fa47 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 27 Mar 2024 12:15:17 -0700 Subject: [PATCH 06/54] [AArch64] Pre-commit test for #86717. NFC --- llvm/test/CodeGen/AArch64/pr86717.ll | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 llvm/test/CodeGen/AArch64/pr86717.ll diff --git a/llvm/test/CodeGen/AArch64/pr86717.ll b/llvm/test/CodeGen/AArch64/pr86717.ll new file mode 100644 index 00000000000000..aac0c0df5f51a1 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/pr86717.ll @@ -0,0 +1,22 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=aarch64 | FileCheck %s + +define <16 x i8> @f(i32 %0) { +; CHECK-LABEL: f: +; CHECK: // %bb.0: +; CHECK-NEXT: sub sp, sp, #16 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: movi v0.2d, #0000000000000000 +; CHECK-NEXT: mov w8, #1 // =0x1 +; CHECK-NEXT: mov x9, sp +; CHECK-NEXT: sub w8, w8, w0 +; CHECK-NEXT: mov w10, #3 // =0x3 +; CHECK-NEXT: orr x8, x9, x8 +; CHECK-NEXT: str q0, [sp] +; CHECK-NEXT: strb w10, [x8] +; CHECK-NEXT: ldr q0, [sp], #16 +; CHECK-NEXT: ret + %2 = sub nuw i32 1, %0 + %3 = insertelement <16 x i8> zeroinitializer, i8 3, i32 %2 + ret <16 x i8> %3 +} From acab142751d3498c6d0aa63253dc1c9f3f83f268 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 27 Mar 2024 11:35:12 -0700 Subject: [PATCH 07/54] [LegalizeDAG] Freeze index when converting insert_elt/insert_subvector to load/store on stack. We try clamp the index to be within the bounds of the stack object we create, but if we don't freeze it, poison can propagate into the clamp code. This can cause the access to leave the bounds of the stack object. We have other instances of this issue in type legalization and extract_elt/subvector, but posting this patch first for direction check. Fixes #86717 --- llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 3 ++ llvm/test/CodeGen/AArch64/pr86717.ll | 6 +-- .../lasx/ir-instruction/insertelement.ll | 42 ++++++++++-------- .../lsx/ir-instruction/insertelement.ll | 42 ++++++++++-------- .../X86/2009-06-05-VariableIndexInsert.ll | 8 ++-- .../CodeGen/X86/insertelement-var-index.ll | 44 ++++++++++--------- 6 files changed, 82 insertions(+), 63 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index e10b8bc8c5e2eb..24f69ea1b742a6 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -1455,6 +1455,9 @@ SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { // First store the whole vector. SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo); + // Freeze the index so we don't poison the clamping code we're about to emit. + Idx = DAG.getFreeze(Idx); + // Then store the inserted part. if (PartVT.isVector()) { SDValue SubStackPtr = diff --git a/llvm/test/CodeGen/AArch64/pr86717.ll b/llvm/test/CodeGen/AArch64/pr86717.ll index aac0c0df5f51a1..aa8be954be72d0 100644 --- a/llvm/test/CodeGen/AArch64/pr86717.ll +++ b/llvm/test/CodeGen/AArch64/pr86717.ll @@ -10,10 +10,10 @@ define <16 x i8> @f(i32 %0) { ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: sub w8, w8, w0 -; CHECK-NEXT: mov w10, #3 // =0x3 -; CHECK-NEXT: orr x8, x9, x8 +; CHECK-NEXT: bfxil x9, x8, #0, #4 +; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: str q0, [sp] -; CHECK-NEXT: strb w10, [x8] +; CHECK-NEXT: strb w8, [x9] ; CHECK-NEXT: ldr q0, [sp], #16 ; CHECK-NEXT: ret %2 = sub nuw i32 1, %0 diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll index 25106b456d2f7a..6629d34405492c 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll @@ -123,9 +123,10 @@ define void @insert_32xi8_idx(ptr %src, ptr %dst, i8 %in, i32 %idx) nounwind { ; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0 ; CHECK-NEXT: xvld $xr0, $a0, 0 ; CHECK-NEXT: xvst $xr0, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a3, 4, 0 -; CHECK-NEXT: st.b $a2, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a3, 31, 0 +; CHECK-NEXT: addi.d $a3, $sp, 0 +; CHECK-NEXT: bstrins.d $a3, $a0, 4, 0 +; CHECK-NEXT: st.b $a2, $a3, 0 ; CHECK-NEXT: xvld $xr0, $sp, 0 ; CHECK-NEXT: xvst $xr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $fp, -64 @@ -149,9 +150,10 @@ define void @insert_16xi16_idx(ptr %src, ptr %dst, i16 %in, i32 %idx) nounwind { ; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0 ; CHECK-NEXT: xvld $xr0, $a0, 0 ; CHECK-NEXT: xvst $xr0, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a3, 4, 1 -; CHECK-NEXT: st.h $a2, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a3, 31, 0 +; CHECK-NEXT: addi.d $a3, $sp, 0 +; CHECK-NEXT: bstrins.d $a3, $a0, 4, 1 +; CHECK-NEXT: st.h $a2, $a3, 0 ; CHECK-NEXT: xvld $xr0, $sp, 0 ; CHECK-NEXT: xvst $xr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $fp, -64 @@ -175,9 +177,10 @@ define void @insert_8xi32_idx(ptr %src, ptr %dst, i32 %in, i32 %idx) nounwind { ; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0 ; CHECK-NEXT: xvld $xr0, $a0, 0 ; CHECK-NEXT: xvst $xr0, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a3, 4, 2 -; CHECK-NEXT: st.w $a2, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a3, 31, 0 +; CHECK-NEXT: addi.d $a3, $sp, 0 +; CHECK-NEXT: bstrins.d $a3, $a0, 4, 2 +; CHECK-NEXT: st.w $a2, $a3, 0 ; CHECK-NEXT: xvld $xr0, $sp, 0 ; CHECK-NEXT: xvst $xr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $fp, -64 @@ -201,9 +204,10 @@ define void @insert_4xi64_idx(ptr %src, ptr %dst, i64 %in, i32 %idx) nounwind { ; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0 ; CHECK-NEXT: xvld $xr0, $a0, 0 ; CHECK-NEXT: xvst $xr0, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a3, 4, 3 -; CHECK-NEXT: st.d $a2, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a3, 31, 0 +; CHECK-NEXT: addi.d $a3, $sp, 0 +; CHECK-NEXT: bstrins.d $a3, $a0, 4, 3 +; CHECK-NEXT: st.d $a2, $a3, 0 ; CHECK-NEXT: xvld $xr0, $sp, 0 ; CHECK-NEXT: xvst $xr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $fp, -64 @@ -227,9 +231,10 @@ define void @insert_8xfloat_idx(ptr %src, ptr %dst, float %in, i32 %idx) nounwin ; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0 ; CHECK-NEXT: xvld $xr1, $a0, 0 ; CHECK-NEXT: xvst $xr1, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a2, 4, 2 -; CHECK-NEXT: fst.s $fa0, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a2, 31, 0 +; CHECK-NEXT: addi.d $a2, $sp, 0 +; CHECK-NEXT: bstrins.d $a2, $a0, 4, 2 +; CHECK-NEXT: fst.s $fa0, $a2, 0 ; CHECK-NEXT: xvld $xr0, $sp, 0 ; CHECK-NEXT: xvst $xr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $fp, -64 @@ -253,9 +258,10 @@ define void @insert_4xdouble_idx(ptr %src, ptr %dst, double %in, i32 %idx) nounw ; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0 ; CHECK-NEXT: xvld $xr1, $a0, 0 ; CHECK-NEXT: xvst $xr1, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a2, 4, 3 -; CHECK-NEXT: fst.d $fa0, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a2, 31, 0 +; CHECK-NEXT: addi.d $a2, $sp, 0 +; CHECK-NEXT: bstrins.d $a2, $a0, 4, 3 +; CHECK-NEXT: fst.d $fa0, $a2, 0 ; CHECK-NEXT: xvld $xr0, $sp, 0 ; CHECK-NEXT: xvst $xr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $fp, -64 diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insertelement.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insertelement.ll index 7f232073ae129c..19171b7d8ed784 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insertelement.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insertelement.ll @@ -87,9 +87,10 @@ define void @insert_16xi8_idx(ptr %src, ptr %dst, i8 %ins, i32 %idx) nounwind { ; CHECK-NEXT: addi.d $sp, $sp, -16 ; CHECK-NEXT: vld $vr0, $a0, 0 ; CHECK-NEXT: vst $vr0, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a3, 3, 0 -; CHECK-NEXT: st.b $a2, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a3, 31, 0 +; CHECK-NEXT: addi.d $a3, $sp, 0 +; CHECK-NEXT: bstrins.d $a3, $a0, 3, 0 +; CHECK-NEXT: st.b $a2, $a3, 0 ; CHECK-NEXT: vld $vr0, $sp, 0 ; CHECK-NEXT: vst $vr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $sp, 16 @@ -106,9 +107,10 @@ define void @insert_8xi16_idx(ptr %src, ptr %dst, i16 %ins, i32 %idx) nounwind { ; CHECK-NEXT: addi.d $sp, $sp, -16 ; CHECK-NEXT: vld $vr0, $a0, 0 ; CHECK-NEXT: vst $vr0, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a3, 3, 1 -; CHECK-NEXT: st.h $a2, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a3, 31, 0 +; CHECK-NEXT: addi.d $a3, $sp, 0 +; CHECK-NEXT: bstrins.d $a3, $a0, 3, 1 +; CHECK-NEXT: st.h $a2, $a3, 0 ; CHECK-NEXT: vld $vr0, $sp, 0 ; CHECK-NEXT: vst $vr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $sp, 16 @@ -125,9 +127,10 @@ define void @insert_4xi32_idx(ptr %src, ptr %dst, i32 %ins, i32 %idx) nounwind { ; CHECK-NEXT: addi.d $sp, $sp, -16 ; CHECK-NEXT: vld $vr0, $a0, 0 ; CHECK-NEXT: vst $vr0, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a3, 3, 2 -; CHECK-NEXT: st.w $a2, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a3, 31, 0 +; CHECK-NEXT: addi.d $a3, $sp, 0 +; CHECK-NEXT: bstrins.d $a3, $a0, 3, 2 +; CHECK-NEXT: st.w $a2, $a3, 0 ; CHECK-NEXT: vld $vr0, $sp, 0 ; CHECK-NEXT: vst $vr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $sp, 16 @@ -144,9 +147,10 @@ define void @insert_2xi64_idx(ptr %src, ptr %dst, i64 %ins, i32 %idx) nounwind { ; CHECK-NEXT: addi.d $sp, $sp, -16 ; CHECK-NEXT: vld $vr0, $a0, 0 ; CHECK-NEXT: vst $vr0, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a3, 3, 3 -; CHECK-NEXT: st.d $a2, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a3, 31, 0 +; CHECK-NEXT: addi.d $a3, $sp, 0 +; CHECK-NEXT: bstrins.d $a3, $a0, 3, 3 +; CHECK-NEXT: st.d $a2, $a3, 0 ; CHECK-NEXT: vld $vr0, $sp, 0 ; CHECK-NEXT: vst $vr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $sp, 16 @@ -163,9 +167,10 @@ define void @insert_4xfloat_idx(ptr %src, ptr %dst, float %ins, i32 %idx) nounwi ; CHECK-NEXT: addi.d $sp, $sp, -16 ; CHECK-NEXT: vld $vr1, $a0, 0 ; CHECK-NEXT: vst $vr1, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a2, 3, 2 -; CHECK-NEXT: fst.s $fa0, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a2, 31, 0 +; CHECK-NEXT: addi.d $a2, $sp, 0 +; CHECK-NEXT: bstrins.d $a2, $a0, 3, 2 +; CHECK-NEXT: fst.s $fa0, $a2, 0 ; CHECK-NEXT: vld $vr0, $sp, 0 ; CHECK-NEXT: vst $vr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $sp, 16 @@ -182,9 +187,10 @@ define void @insert_2xdouble_idx(ptr %src, ptr %dst, double %ins, i32 %idx) noun ; CHECK-NEXT: addi.d $sp, $sp, -16 ; CHECK-NEXT: vld $vr1, $a0, 0 ; CHECK-NEXT: vst $vr1, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a2, 3, 3 -; CHECK-NEXT: fst.d $fa0, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a2, 31, 0 +; CHECK-NEXT: addi.d $a2, $sp, 0 +; CHECK-NEXT: bstrins.d $a2, $a0, 3, 3 +; CHECK-NEXT: fst.d $fa0, $a2, 0 ; CHECK-NEXT: vld $vr0, $sp, 0 ; CHECK-NEXT: vst $vr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $sp, 16 diff --git a/llvm/test/CodeGen/X86/2009-06-05-VariableIndexInsert.ll b/llvm/test/CodeGen/X86/2009-06-05-VariableIndexInsert.ll index 535450a52ff60e..695a2d0cd806e0 100644 --- a/llvm/test/CodeGen/X86/2009-06-05-VariableIndexInsert.ll +++ b/llvm/test/CodeGen/X86/2009-06-05-VariableIndexInsert.ll @@ -9,11 +9,11 @@ define <2 x i64> @_mm_insert_epi16(<2 x i64> %a, i32 %b, i32 %imm) nounwind read ; X86-NEXT: movl %esp, %ebp ; X86-NEXT: andl $-16, %esp ; X86-NEXT: subl $32, %esp -; X86-NEXT: movzwl 8(%ebp), %eax -; X86-NEXT: movl 12(%ebp), %ecx -; X86-NEXT: andl $7, %ecx +; X86-NEXT: movl 12(%ebp), %eax +; X86-NEXT: movzwl 8(%ebp), %ecx +; X86-NEXT: andl $7, %eax ; X86-NEXT: movaps %xmm0, (%esp) -; X86-NEXT: movw %ax, (%esp,%ecx,2) +; X86-NEXT: movw %cx, (%esp,%eax,2) ; X86-NEXT: movaps (%esp), %xmm0 ; X86-NEXT: movl %ebp, %esp ; X86-NEXT: popl %ebp diff --git a/llvm/test/CodeGen/X86/insertelement-var-index.ll b/llvm/test/CodeGen/X86/insertelement-var-index.ll index 8ed8495d7a4614..5420e6b5ce86f3 100644 --- a/llvm/test/CodeGen/X86/insertelement-var-index.ll +++ b/llvm/test/CodeGen/X86/insertelement-var-index.ll @@ -1009,18 +1009,19 @@ define <2 x i64> @arg_i64_v2i64(<2 x i64> %v, i64 %x, i32 %y) nounwind { ; X86AVX2-NEXT: pushl %esi ; X86AVX2-NEXT: andl $-16, %esp ; X86AVX2-NEXT: subl $48, %esp -; X86AVX2-NEXT: movl 8(%ebp), %eax -; X86AVX2-NEXT: movl 12(%ebp), %ecx -; X86AVX2-NEXT: movl 16(%ebp), %edx +; X86AVX2-NEXT: movl 8(%ebp), %edx +; X86AVX2-NEXT: movl 12(%ebp), %eax +; X86AVX2-NEXT: movl 16(%ebp), %ecx ; X86AVX2-NEXT: vmovaps %xmm0, (%esp) -; X86AVX2-NEXT: leal (%edx,%edx), %esi +; X86AVX2-NEXT: addl %ecx, %ecx +; X86AVX2-NEXT: movl %ecx, %esi ; X86AVX2-NEXT: andl $3, %esi -; X86AVX2-NEXT: movl %eax, (%esp,%esi,4) +; X86AVX2-NEXT: movl %edx, (%esp,%esi,4) ; X86AVX2-NEXT: vmovaps (%esp), %xmm0 ; X86AVX2-NEXT: vmovaps %xmm0, {{[0-9]+}}(%esp) -; X86AVX2-NEXT: leal 1(%edx,%edx), %eax -; X86AVX2-NEXT: andl $3, %eax -; X86AVX2-NEXT: movl %ecx, 16(%esp,%eax,4) +; X86AVX2-NEXT: incl %ecx +; X86AVX2-NEXT: andl $3, %ecx +; X86AVX2-NEXT: movl %eax, 16(%esp,%ecx,4) ; X86AVX2-NEXT: vmovaps {{[0-9]+}}(%esp), %xmm0 ; X86AVX2-NEXT: leal -4(%ebp), %esp ; X86AVX2-NEXT: popl %esi @@ -1362,12 +1363,13 @@ define <2 x i64> @load_i64_v2i64(<2 x i64> %v, ptr %p, i32 %y) nounwind { ; X86AVX2-NEXT: movl (%ecx), %edx ; X86AVX2-NEXT: movl 4(%ecx), %ecx ; X86AVX2-NEXT: vmovaps %xmm0, (%esp) -; X86AVX2-NEXT: leal (%eax,%eax), %esi +; X86AVX2-NEXT: addl %eax, %eax +; X86AVX2-NEXT: movl %eax, %esi ; X86AVX2-NEXT: andl $3, %esi ; X86AVX2-NEXT: movl %edx, (%esp,%esi,4) ; X86AVX2-NEXT: vmovaps (%esp), %xmm0 ; X86AVX2-NEXT: vmovaps %xmm0, {{[0-9]+}}(%esp) -; X86AVX2-NEXT: leal 1(%eax,%eax), %eax +; X86AVX2-NEXT: incl %eax ; X86AVX2-NEXT: andl $3, %eax ; X86AVX2-NEXT: movl %ecx, 16(%esp,%eax,4) ; X86AVX2-NEXT: vmovaps {{[0-9]+}}(%esp), %xmm0 @@ -1742,18 +1744,19 @@ define <4 x i64> @arg_i64_v4i64(<4 x i64> %v, i64 %x, i32 %y) nounwind { ; X86AVX2-NEXT: pushl %esi ; X86AVX2-NEXT: andl $-32, %esp ; X86AVX2-NEXT: subl $96, %esp -; X86AVX2-NEXT: movl 8(%ebp), %eax -; X86AVX2-NEXT: movl 12(%ebp), %ecx -; X86AVX2-NEXT: movl 16(%ebp), %edx +; X86AVX2-NEXT: movl 8(%ebp), %edx +; X86AVX2-NEXT: movl 12(%ebp), %eax +; X86AVX2-NEXT: movl 16(%ebp), %ecx ; X86AVX2-NEXT: vmovaps %ymm0, (%esp) -; X86AVX2-NEXT: leal (%edx,%edx), %esi +; X86AVX2-NEXT: addl %ecx, %ecx +; X86AVX2-NEXT: movl %ecx, %esi ; X86AVX2-NEXT: andl $7, %esi -; X86AVX2-NEXT: movl %eax, (%esp,%esi,4) +; X86AVX2-NEXT: movl %edx, (%esp,%esi,4) ; X86AVX2-NEXT: vmovaps (%esp), %ymm0 ; X86AVX2-NEXT: vmovaps %ymm0, {{[0-9]+}}(%esp) -; X86AVX2-NEXT: leal 1(%edx,%edx), %eax -; X86AVX2-NEXT: andl $7, %eax -; X86AVX2-NEXT: movl %ecx, 32(%esp,%eax,4) +; X86AVX2-NEXT: incl %ecx +; X86AVX2-NEXT: andl $7, %ecx +; X86AVX2-NEXT: movl %eax, 32(%esp,%ecx,4) ; X86AVX2-NEXT: vmovaps {{[0-9]+}}(%esp), %ymm0 ; X86AVX2-NEXT: leal -4(%ebp), %esp ; X86AVX2-NEXT: popl %esi @@ -2128,12 +2131,13 @@ define <4 x i64> @load_i64_v4i64(<4 x i64> %v, ptr %p, i32 %y) nounwind { ; X86AVX2-NEXT: movl (%ecx), %edx ; X86AVX2-NEXT: movl 4(%ecx), %ecx ; X86AVX2-NEXT: vmovaps %ymm0, (%esp) -; X86AVX2-NEXT: leal (%eax,%eax), %esi +; X86AVX2-NEXT: addl %eax, %eax +; X86AVX2-NEXT: movl %eax, %esi ; X86AVX2-NEXT: andl $7, %esi ; X86AVX2-NEXT: movl %edx, (%esp,%esi,4) ; X86AVX2-NEXT: vmovaps (%esp), %ymm0 ; X86AVX2-NEXT: vmovaps %ymm0, {{[0-9]+}}(%esp) -; X86AVX2-NEXT: leal 1(%eax,%eax), %eax +; X86AVX2-NEXT: incl %eax ; X86AVX2-NEXT: andl $7, %eax ; X86AVX2-NEXT: movl %ecx, 32(%esp,%eax,4) ; X86AVX2-NEXT: vmovaps {{[0-9]+}}(%esp), %ymm0 From 4d4626d9d59bce9f4d19bd4a1b384f1122904419 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Wed, 27 Mar 2024 13:10:42 -0700 Subject: [PATCH 08/54] [TEST][HWASAN] Fix test after #86771 --- .../test/Instrumentation/HWAddressSanitizer/globals-access.ll | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/globals-access.ll b/llvm/test/Instrumentation/HWAddressSanitizer/globals-access.ll index c83911f60149a7..184f84d2859aff 100644 --- a/llvm/test/Instrumentation/HWAddressSanitizer/globals-access.ll +++ b/llvm/test/Instrumentation/HWAddressSanitizer/globals-access.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --global-value-regex "x" --version 4 -; RUN: opt < %s -S -passes=hwasan -mtriple=aarch64 -hwasan-globals=0 | FileCheck %s --check-prefixes=NOGLOB -; RUN: opt < %s -S -passes=hwasan -mtriple=aarch64 -hwasan-globals=1 | FileCheck %s +; RUN: opt < %s -S -passes=hwasan -mtriple=aarch64-linux-gnu -hwasan-globals=0 | FileCheck %s --check-prefixes=NOGLOB +; RUN: opt < %s -S -passes=hwasan -mtriple=aarch64-linux-gnu -hwasan-globals=1 | FileCheck %s @x = dso_local global i32 0, align 4 From 3522de9e412fbaa6d7344cc31bd43d1be6c95d59 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Wed, 27 Mar 2024 13:19:00 -0700 Subject: [PATCH 09/54] [TEST][HWASAN] Fix test after #86771 --- llvm/test/Instrumentation/HWAddressSanitizer/globals-access.ll | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/globals-access.ll b/llvm/test/Instrumentation/HWAddressSanitizer/globals-access.ll index 184f84d2859aff..f8d13fc4237e56 100644 --- a/llvm/test/Instrumentation/HWAddressSanitizer/globals-access.ll +++ b/llvm/test/Instrumentation/HWAddressSanitizer/globals-access.ll @@ -2,6 +2,8 @@ ; RUN: opt < %s -S -passes=hwasan -mtriple=aarch64-linux-gnu -hwasan-globals=0 | FileCheck %s --check-prefixes=NOGLOB ; RUN: opt < %s -S -passes=hwasan -mtriple=aarch64-linux-gnu -hwasan-globals=1 | FileCheck %s +target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" + @x = dso_local global i32 0, align 4 ;. From fa90a0aa0e12aaf657d0f6f7626e05f90ba6f999 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Wed, 27 Mar 2024 13:24:53 -0700 Subject: [PATCH 10/54] [llvm-exegesis] Improve error handling for shm_open calls This patch adds error handling for shm_open failures in one case where they were not handled before and also makes an error handler in another case report the value of errno for diagnosis. --- llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp index 1fd81bd407becb..0a947f6e206fef 100644 --- a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp +++ b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp @@ -63,6 +63,10 @@ Error SubprocessMemory::addMemoryDefinition( SharedMemoryNames.push_back(SharedMemoryName); int SharedMemoryFD = shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); + if (SharedMemoryFD == -1) + return make_error( + "Failed to create shared memory object for memory definition: " + + Twine(strerror(errno))); if (ftruncate(SharedMemoryFD, MemVal.SizeBytes) != 0) { return make_error("Truncating a memory definiton failed: " + Twine(strerror(errno))); @@ -100,7 +104,8 @@ Expected SubprocessMemory::setupAuxiliaryMemoryInSubprocess( shm_open(AuxiliaryMemoryName.c_str(), O_RDWR, S_IRUSR | S_IWUSR); if (AuxiliaryMemoryFileDescriptor == -1) return make_error( - "Getting file descriptor for auxiliary memory failed"); + "Getting file descriptor for auxiliary memory failed: " + + Twine(strerror(errno))); // set up memory value file descriptors int *AuxiliaryMemoryMapping = (int *)mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, From 8a071678a9091d536eae29912ca7be6238105956 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Wed, 27 Mar 2024 13:28:26 -0700 Subject: [PATCH 11/54] Revert "[libc][math][c23] Add remaining linux/* entrypoints for {,u}fromfp{,x}* (#86692)" This reverts commit cd17082b24079a31eff0057abe407da5cfb7b0fc because the newly added tests fail on 32b ARM. Link: #86692 Link: https://lab.llvm.org/buildbot/#/builders/229/builds/24458 --- libc/config/linux/aarch64/entrypoints.txt | 16 ------------ libc/config/linux/arm/entrypoints.txt | 12 --------- libc/config/linux/riscv/entrypoints.txt | 16 ------------ libc/docs/math/index.rst | 32 +++++++++++------------ 4 files changed, 16 insertions(+), 60 deletions(-) diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt index ab0be7e35dced2..78da7f0b334b1f 100644 --- a/libc/config/linux/aarch64/entrypoints.txt +++ b/libc/config/linux/aarch64/entrypoints.txt @@ -396,12 +396,6 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.frexp libc.src.math.frexpf libc.src.math.frexpl - libc.src.math.fromfp - libc.src.math.fromfpf - libc.src.math.fromfpl - libc.src.math.fromfpx - libc.src.math.fromfpxf - libc.src.math.fromfpxl libc.src.math.hypot libc.src.math.hypotf libc.src.math.ilogb @@ -484,12 +478,6 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.trunc libc.src.math.truncf libc.src.math.truncl - libc.src.math.ufromfp - libc.src.math.ufromfpf - libc.src.math.ufromfpl - libc.src.math.ufromfpx - libc.src.math.ufromfpxf - libc.src.math.ufromfpxl ) if(LIBC_TYPES_HAS_FLOAT128) @@ -512,8 +500,6 @@ if(LIBC_TYPES_HAS_FLOAT128) libc.src.math.fminimum_mag_numf128 libc.src.math.fmodf128 libc.src.math.frexpf128 - libc.src.math.fromfpf128 - libc.src.math.fromfpxf128 libc.src.math.ilogbf128 libc.src.math.ldexpf128 libc.src.math.llogbf128 @@ -531,8 +517,6 @@ if(LIBC_TYPES_HAS_FLOAT128) libc.src.math.roundf128 libc.src.math.sqrtf128 libc.src.math.truncf128 - libc.src.math.ufromfpf128 - libc.src.math.ufromfpxf128 ) endif() diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt index 1d9d5ed67200c1..6e63e270280e7a 100644 --- a/libc/config/linux/arm/entrypoints.txt +++ b/libc/config/linux/arm/entrypoints.txt @@ -263,12 +263,6 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.frexp libc.src.math.frexpf libc.src.math.frexpl - libc.src.math.fromfp - libc.src.math.fromfpf - libc.src.math.fromfpl - libc.src.math.fromfpx - libc.src.math.fromfpxf - libc.src.math.fromfpxl libc.src.math.hypot libc.src.math.hypotf libc.src.math.ilogb @@ -351,12 +345,6 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.trunc libc.src.math.truncf libc.src.math.truncl - libc.src.math.ufromfp - libc.src.math.ufromfpf - libc.src.math.ufromfpl - libc.src.math.ufromfpx - libc.src.math.ufromfpxf - libc.src.math.ufromfpxl ) set(TARGET_LLVMLIBC_ENTRYPOINTS diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt index 96acd999efda4d..5aae4e246cfb3c 100644 --- a/libc/config/linux/riscv/entrypoints.txt +++ b/libc/config/linux/riscv/entrypoints.txt @@ -404,12 +404,6 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.frexp libc.src.math.frexpf libc.src.math.frexpl - libc.src.math.fromfp - libc.src.math.fromfpf - libc.src.math.fromfpl - libc.src.math.fromfpx - libc.src.math.fromfpxf - libc.src.math.fromfpxl libc.src.math.hypot libc.src.math.hypotf libc.src.math.ilogb @@ -492,12 +486,6 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.trunc libc.src.math.truncf libc.src.math.truncl - libc.src.math.ufromfp - libc.src.math.ufromfpf - libc.src.math.ufromfpl - libc.src.math.ufromfpx - libc.src.math.ufromfpxf - libc.src.math.ufromfpxl ) if(LIBC_TYPES_HAS_FLOAT128) @@ -520,8 +508,6 @@ if(LIBC_TYPES_HAS_FLOAT128) libc.src.math.fminimum_mag_numf128 libc.src.math.fmodf128 libc.src.math.frexpf128 - libc.src.math.fromfpf128 - libc.src.math.fromfpxf128 libc.src.math.ilogbf128 libc.src.math.ldexpf128 libc.src.math.llogbf128 @@ -539,8 +525,6 @@ if(LIBC_TYPES_HAS_FLOAT128) libc.src.math.roundf128 libc.src.math.sqrtf128 libc.src.math.truncf128 - libc.src.math.ufromfpf128 - libc.src.math.ufromfpxf128 ) endif() diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst index d8dee3c9061908..080b6a4427f511 100644 --- a/libc/docs/math/index.rst +++ b/libc/docs/math/index.rst @@ -190,21 +190,21 @@ Basic Operations +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ | frexpf128 | |check| | |check| | | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| fromfp | |check| | |check| | |check| | |check| | | | | | | | | | +| fromfp | |check| | | | | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| fromfpf | |check| | |check| | |check| | |check| | | | | | | | | | +| fromfpf | |check| | | | | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| fromfpl | |check| | |check| | |check| | |check| | | | | | | | | | +| fromfpl | |check| | | | | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| fromfpf128 | |check| | |check| | | |check| | | | | | | | | | +| fromfpf128 | |check| | | | | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| fromfpx | |check| | |check| | |check| | |check| | | | | | | | | | +| fromfpx | |check| | | | | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| fromfpxf | |check| | |check| | |check| | |check| | | | | | | | | | +| fromfpxf | |check| | | | | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| fromfpxl | |check| | |check| | |check| | |check| | | | | | | | | | +| fromfpxl | |check| | | | | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| fromfpxf128 | |check| | |check| | | |check| | | | | | | | | | +| fromfpxf128 | |check| | | | | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ | ilogb | |check| | |check| | |check| | |check| | |check| | | | |check| | |check| | |check| | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ @@ -364,21 +364,21 @@ Basic Operations +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ | truncf128 | |check| | |check| | | |check| | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| ufromfp | |check| | |check| | |check| | |check| | | | | | | | | | +| ufromfp | |check| | | | | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| ufromfpf | |check| | |check| | |check| | |check| | | | | | | | | | +| ufromfpf | |check| | | | | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| ufromfpl | |check| | |check| | |check| | |check| | | | | | | | | | +| ufromfpl | |check| | | | | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| ufromfpf128 | |check| | |check| | | |check| | | | | | | | | | +| ufromfpf128 | |check| | | | | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| ufromfpx | |check| | |check| | |check| | |check| | | | | | | | | | +| ufromfpx | |check| | | | | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| ufromfpxf | |check| | |check| | |check| | |check| | | | | | | | | | +| ufromfpxf | |check| | | | | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| ufromfpxl | |check| | |check| | |check| | |check| | | | | | | | | | +| ufromfpxl | |check| | | | | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| ufromfpxf128 | |check| | |check| | | |check| | | | | | | | | | +| ufromfpxf128 | |check| | | | | | | | | | | | | +------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ From 453a63caebc35378bcdb31da9d0f358a3998e51b Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Wed, 27 Mar 2024 13:38:21 -0700 Subject: [PATCH 12/54] [NFC][HWASAN] Promote InstrumentGlobals to member (#86773) --- llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp index 5d366e3d6dee0a..96fd530be33318 100644 --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -422,6 +422,7 @@ class HWAddressSanitizer { bool InstrumentLandingPads; bool InstrumentWithCalls; bool InstrumentStack; + bool InstrumentGlobals; bool DetectUseAfterScope; bool UsePageAliases; bool UseMatchAllCallback; @@ -639,11 +640,13 @@ void HWAddressSanitizer::initializeModule() { // If we don't have personality function support, fall back to landing pads. InstrumentLandingPads = optOr(ClInstrumentLandingPads, !NewRuntime); + InstrumentGlobals = + !CompileKernel && !UsePageAliases && optOr(ClGlobals, NewRuntime); + if (!CompileKernel) { createHwasanCtorComdat(); - bool InstrumentGlobals = optOr(ClGlobals, NewRuntime); - if (InstrumentGlobals && !UsePageAliases) + if (InstrumentGlobals) instrumentGlobals(); bool InstrumentPersonalityFunctions = From 70e8cf0c31493071c50c8112c6e0c7903abf6ca6 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Wed, 27 Mar 2024 13:43:18 -0700 Subject: [PATCH 13/54] [HWASAN] Don't instrument loads from global if globals are not tagged (#86774) --- .../Instrumentation/HWAddressSanitizer.cpp | 7 +++++++ .../HWAddressSanitizer/globals-access.ll | 16 ---------------- .../HWAddressSanitizer/use-after-scope-setjmp.ll | 1 - 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp index 96fd530be33318..f7d4803ded155a 100644 --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -790,6 +790,13 @@ bool HWAddressSanitizer::ignoreAccess(Instruction *Inst, Value *Ptr) { if (SSI && SSI->stackAccessIsSafe(*Inst)) return true; } + + if (isa(getUnderlyingObject(Ptr))) { + if (!InstrumentGlobals) + return true; + // TODO: Optimize inbound global accesses, like Asan `instrumentMop`. + } + return false; } diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/globals-access.ll b/llvm/test/Instrumentation/HWAddressSanitizer/globals-access.ll index f8d13fc4237e56..f9040afd1c0166 100644 --- a/llvm/test/Instrumentation/HWAddressSanitizer/globals-access.ll +++ b/llvm/test/Instrumentation/HWAddressSanitizer/globals-access.ll @@ -15,22 +15,6 @@ define dso_local noundef i32 @_Z3tmpv() sanitize_hwaddress { ; NOGLOB-LABEL: define dso_local noundef i32 @_Z3tmpv( ; NOGLOB-SAME: ) #[[ATTR0:[0-9]+]] { ; NOGLOB-NEXT: entry: -; NOGLOB-NEXT: [[TMP12:%.*]] = load i64, ptr @__hwasan_tls, align 8 -; NOGLOB-NEXT: [[TMP1:%.*]] = or i64 [[TMP12]], 4294967295 -; NOGLOB-NEXT: [[HWASAN_SHADOW:%.*]] = add i64 [[TMP1]], 1 -; NOGLOB-NEXT: [[TMP2:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr -; NOGLOB-NEXT: [[TMP3:%.*]] = lshr i64 ptrtoint (ptr @x to i64), 56 -; NOGLOB-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i8 -; NOGLOB-NEXT: [[TMP5:%.*]] = and i64 ptrtoint (ptr @x to i64), 72057594037927935 -; NOGLOB-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP5]], 4 -; NOGLOB-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP2]], i64 [[TMP6]] -; NOGLOB-NEXT: [[TMP8:%.*]] = load i8, ptr [[TMP7]], align 1 -; NOGLOB-NEXT: [[TMP9:%.*]] = icmp ne i8 [[TMP4]], [[TMP8]] -; NOGLOB-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1:![0-9]+]] -; NOGLOB: 10: -; NOGLOB-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP2]], ptr @x, i32 2) -; NOGLOB-NEXT: br label [[TMP11]] -; NOGLOB: 11: ; NOGLOB-NEXT: [[TMP0:%.*]] = load i32, ptr @x, align 4 ; NOGLOB-NEXT: ret i32 [[TMP0]] ; diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll b/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll index 079d7224128301..62fd7a16715693 100644 --- a/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll +++ b/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll @@ -54,7 +54,6 @@ define dso_local noundef i1 @_Z6targetv() sanitize_hwaddress { ; CHECK: sw.bb1: ; CHECK-NEXT: br label [[RETURN]] ; CHECK: while.body: -; CHECK-NEXT: call void @llvm.hwasan.check.memaccess(ptr [[TMP16]], ptr @stackbuf, i32 19) ; CHECK-NEXT: store ptr [[BUF_HWASAN]], ptr @stackbuf, align 8 ; CHECK-NEXT: call void @may_jump() ; CHECK-NEXT: br label [[RETURN]] From f18600c87404eab8d0a279b0286f8add8b4a1bb8 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 27 Mar 2024 13:50:17 -0700 Subject: [PATCH 14/54] [Driver] Avoid repeated ToolChain.getTriple() calls. NFC --- clang/lib/Driver/ToolChains/CommonArgs.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 6b1fbba7abd031..ace4fb99581e38 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -758,15 +758,15 @@ bool tools::isTLSDESCEnabled(const ToolChain &TC, void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args, ArgStringList &CmdArgs, const InputInfo &Output, const InputInfo &Input, bool IsThinLTO) { - const bool IsOSAIX = ToolChain.getTriple().isOSAIX(); - const bool IsAMDGCN = ToolChain.getTriple().isAMDGCN(); + const llvm::Triple &Triple = ToolChain.getTriple(); + const bool IsOSAIX = Triple.isOSAIX(); + const bool IsAMDGCN = Triple.isAMDGCN(); const char *Linker = Args.MakeArgString(ToolChain.GetLinkerPath()); const Driver &D = ToolChain.getDriver(); const bool IsFatLTO = Args.hasArg(options::OPT_ffat_lto_objects); const bool IsUnifiedLTO = Args.hasArg(options::OPT_funified_lto); if (llvm::sys::path::filename(Linker) != "ld.lld" && - llvm::sys::path::stem(Linker) != "ld.lld" && - !ToolChain.getTriple().isOSOpenBSD()) { + llvm::sys::path::stem(Linker) != "ld.lld" && !Triple.isOSOpenBSD()) { // Tell the linker to load the plugin. This has to come before // AddLinkerInputs as gold requires -plugin and AIX ld requires -bplugin to // come before any -plugin-opt/-bplugin_opt that -Wl might forward. @@ -835,7 +835,7 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args, // the plugin. // Handle flags for selecting CPU variants. - std::string CPU = getCPUName(D, Args, ToolChain.getTriple()); + std::string CPU = getCPUName(D, Args, Triple); if (!CPU.empty()) CmdArgs.push_back( Args.MakeArgString(Twine(PluginOptPrefix) + ExtraDash + "mcpu=" + CPU)); @@ -966,10 +966,9 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args, bool HasRoptr = Args.hasFlag(options::OPT_mxcoff_roptr, options::OPT_mno_xcoff_roptr, false); StringRef OptStr = HasRoptr ? "-mxcoff-roptr" : "-mno-xcoff-roptr"; - if (!IsOSAIX) D.Diag(diag::err_drv_unsupported_opt_for_target) - << OptStr << ToolChain.getTriple().str(); + << OptStr << Triple.str(); if (HasRoptr) { // The data sections option is on by default on AIX. We only need to error @@ -1032,7 +1031,7 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args, } if (Args.hasFlag(options::OPT_femulated_tls, options::OPT_fno_emulated_tls, - ToolChain.getTriple().hasDefaultEmulatedTLS())) { + Triple.hasDefaultEmulatedTLS())) { CmdArgs.push_back( Args.MakeArgString(Twine(PluginOptPrefix) + "-emulated-tls")); } From c7d947f5e6c966bdba4db26097c3b433fcbe7841 Mon Sep 17 00:00:00 2001 From: Jason Molenda Date: Wed, 27 Mar 2024 13:59:35 -0700 Subject: [PATCH 15/54] [lldb] [ObjC runtime] Don't cast to signed when left shifting (#86605) This is fixing a report from ubsan which I don't think is super high value, but our testsuite hits it on TestDataFormatterObjCNSContainer.py so I'd like to work around it. We are getting ``` runtime error: left shift of negative value -8827055269646171913 3159 int64_t data_payload_signed = 3160 ((int64_t)((int64_t)unobfuscated -> 3161 << m_objc_debug_taggedpointer_ext_payload_lshift) >> 3162 m_objc_debug_taggedpointer_ext_payload_rshift); ``` At this point `unobfuscated` is 0x85800000000000f7 and `m_objc_debug_taggedpointer_ext_payload_lshift` is 9, so `(int64_t)0x85800000000000f7<<9` shifts off the "sign" bit and then some zeroes etc, and that's how we get this error. We're only trying to extract some bits in the middle of the doubleword, so the fact that we're "losing" the sign is not a bug. Change the inner cast to (uint64_t). --- .../ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp index 3e5ee6f6637303..d3fc487aed4333 100644 --- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp +++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp @@ -3154,7 +3154,7 @@ AppleObjCRuntimeV2::TaggedPointerVendorExtended::GetClassDescriptor( << m_objc_debug_taggedpointer_ext_payload_lshift) >> m_objc_debug_taggedpointer_ext_payload_rshift); int64_t data_payload_signed = - ((int64_t)((int64_t)unobfuscated + ((int64_t)((uint64_t)unobfuscated << m_objc_debug_taggedpointer_ext_payload_lshift) >> m_objc_debug_taggedpointer_ext_payload_rshift); From acdba090f2724335cf6cf6ecbfb68f67f0cd2def Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 27 Mar 2024 14:08:59 -0700 Subject: [PATCH 16/54] [ELF] Change duplicate symbol errors to use errorOrWarn so that --noinhibit-exec downgrades the error to a warning, which matches GNU ld. Most recoverable errors should use errorOrWarn. --- lld/ELF/Symbols.cpp | 6 +++--- lld/test/ELF/allow-multiple-definition.s | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lld/ELF/Symbols.cpp b/lld/ELF/Symbols.cpp index cd2b9e22ab3224..93653def328f82 100644 --- a/lld/ELF/Symbols.cpp +++ b/lld/ELF/Symbols.cpp @@ -539,8 +539,8 @@ void elf::reportDuplicate(const Symbol &sym, const InputFile *newFile, if (!d->section && !errSec && errOffset && d->value == errOffset) return; if (!d->section || !errSec) { - error("duplicate symbol: " + toString(sym) + "\n>>> defined in " + - toString(sym.file) + "\n>>> defined in " + toString(newFile)); + errorOrWarn("duplicate symbol: " + toString(sym) + "\n>>> defined in " + + toString(sym.file) + "\n>>> defined in " + toString(newFile)); return; } @@ -564,7 +564,7 @@ void elf::reportDuplicate(const Symbol &sym, const InputFile *newFile, if (!src2.empty()) msg += src2 + "\n>>> "; msg += obj2; - error(msg); + errorOrWarn(msg); } void Symbol::checkDuplicate(const Defined &other) const { diff --git a/lld/test/ELF/allow-multiple-definition.s b/lld/test/ELF/allow-multiple-definition.s index 492784a3601df1..96fa2627e1bf88 100644 --- a/lld/test/ELF/allow-multiple-definition.s +++ b/lld/test/ELF/allow-multiple-definition.s @@ -9,6 +9,9 @@ # RUN: llvm-objdump --no-print-imm-hex -d %t3 | FileCheck %s # RUN: llvm-objdump --no-print-imm-hex -d %t4 | FileCheck --check-prefix=REVERT %s +# RUN: ld.lld --noinhibit-exec %t2 %t1 -o /dev/null 2>&1 | FileCheck %s --check-prefix=WARN +# WARN: warning: duplicate symbol: _bar + # RUN: ld.lld -z muldefs --fatal-warnings %t1 %t2 -o %t3 # RUN: ld.lld -z muldefs --fatal-warnings %t2 %t1 -o %t4 # RUN: llvm-objdump --no-print-imm-hex -d %t3 | FileCheck %s From b9cd48f96acdd07c627ccafbf4386a1f3dcd6c51 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 27 Mar 2024 21:22:15 +0000 Subject: [PATCH 17/54] Revert "[TBAA] Add verifier for tbaa.struct metadata (#86709)" This reverts commit df75183d70e029352a49c93f275db703c81a65c1. Revert for now as this appears to cause failures on some buildbots, e.g.: https://lab.llvm.org/buildbot/#/builders/93/builds/19428/steps/10/logs/stdio --- llvm/include/llvm/IR/Verifier.h | 1 - llvm/lib/IR/Verifier.cpp | 32 ------------------- llvm/test/CodeGen/AArch64/arm64-abi_align.ll | 4 +-- .../AMDGPU/mem-intrinsics.ll | 2 +- .../InstCombine/struct-assign-tbaa.ll | 2 +- llvm/test/Transforms/SROA/tbaa-struct3.ll | 2 +- .../Scalarizer/basic-inseltpoison.ll | 3 +- llvm/test/Transforms/Scalarizer/basic.ll | 3 +- llvm/test/Verifier/tbaa-struct.ll | 14 ++------ 9 files changed, 9 insertions(+), 54 deletions(-) diff --git a/llvm/include/llvm/IR/Verifier.h b/llvm/include/llvm/IR/Verifier.h index b7db6e0bbfb71c..b25f8eb77ee38b 100644 --- a/llvm/include/llvm/IR/Verifier.h +++ b/llvm/include/llvm/IR/Verifier.h @@ -77,7 +77,6 @@ class TBAAVerifier { /// Visit an instruction and return true if it is valid, return false if an /// invalid TBAA is attached. bool visitTBAAMetadata(Instruction &I, const MDNode *MD); - bool visitTBAAStructMetadata(Instruction &I, const MDNode *MD); }; /// Check a function for errors, useful for use when debugging a diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index e16572540f96c2..33f358440a312d 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -5096,9 +5096,6 @@ void Verifier::visitInstruction(Instruction &I) { if (MDNode *TBAA = I.getMetadata(LLVMContext::MD_tbaa)) TBAAVerifyHelper.visitTBAAMetadata(I, TBAA); - if (MDNode *TBAA = I.getMetadata(LLVMContext::MD_tbaa_struct)) - TBAAVerifyHelper.visitTBAAStructMetadata(I, TBAA); - if (MDNode *MD = I.getMetadata(LLVMContext::MD_noalias)) visitAliasScopeListMetadata(MD); if (MDNode *MD = I.getMetadata(LLVMContext::MD_alias_scope)) @@ -7422,35 +7419,6 @@ bool TBAAVerifier::visitTBAAMetadata(Instruction &I, const MDNode *MD) { return true; } -bool TBAAVerifier::visitTBAAStructMetadata(Instruction &I, const MDNode *MD) { - CheckTBAA(MD->getNumOperands() % 3 == 0, - "tbaa.struct operands must occur in groups of three", &I, MD); - - // Each group of three operands must consist of two integers and a - // tbaa node. Moreover, the regions described by the offset and size - // operands must be non-overlapping. - std::optional NextFree; - for (unsigned int Idx = 0; Idx < MD->getNumOperands(); Idx += 3) { - auto *OffsetCI = - mdconst::dyn_extract_or_null(MD->getOperand(Idx)); - CheckTBAA(OffsetCI, "Offset must be a constant integer", &I, MD); - - auto *SizeCI = - mdconst::dyn_extract_or_null(MD->getOperand(Idx + 1)); - CheckTBAA(SizeCI, "Size must be a constant integer", &I, MD); - - MDNode *TBAA = dyn_cast_or_null(MD->getOperand(Idx + 2)); - CheckTBAA(TBAA, "TBAA tag missing", &I, MD); - visitTBAAMetadata(I, TBAA); - - bool NonOverlapping = !NextFree || NextFree->ule(OffsetCI->getValue()); - CheckTBAA(NonOverlapping, "Overlapping tbaa.struct regions", &I, MD); - - NextFree = OffsetCI->getValue() + SizeCI->getValue(); - } - return true; -} - char VerifierLegacyPass::ID = 0; INITIALIZE_PASS(VerifierLegacyPass, "verify", "Module Verifier", false, false) diff --git a/llvm/test/CodeGen/AArch64/arm64-abi_align.ll b/llvm/test/CodeGen/AArch64/arm64-abi_align.ll index c9fd2d38e27acd..089e171e5a4a79 100644 --- a/llvm/test/CodeGen/AArch64/arm64-abi_align.ll +++ b/llvm/test/CodeGen/AArch64/arm64-abi_align.ll @@ -518,6 +518,4 @@ attributes #5 = { nobuiltin } !1 = !{!"omnipotent char", !2} !2 = !{!"Simple C/C++ TBAA"} !3 = !{!"short", !1} -!4 = !{i64 0, i64 4, !5, i64 4, i64 2, !6, i64 8, i64 4, !5, i64 12, i64 2, !6, i64 16, i64 4, !5, i64 20, i64 2, !6} -!5 = !{!0, !0, i64 0} -!6 = !{!3, !3, i64 0} +!4 = !{i64 0, i64 4, !0, i64 4, i64 2, !3, i64 8, i64 4, !0, i64 12, i64 2, !3, i64 16, i64 4, !0, i64 20, i64 2, !3} diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/mem-intrinsics.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/mem-intrinsics.ll index 2f264a2432fc3d..50b0e7a0f5471b 100644 --- a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/mem-intrinsics.ll +++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/mem-intrinsics.ll @@ -141,4 +141,4 @@ attributes #1 = { argmemonly nounwind } !5 = distinct !{!5, !"some domain"} !6 = !{!7} !7 = distinct !{!7, !5, !"some scope 2"} -!8 = !{i64 0, i64 8, !0} +!8 = !{i64 0, i64 8, null} diff --git a/llvm/test/Transforms/InstCombine/struct-assign-tbaa.ll b/llvm/test/Transforms/InstCombine/struct-assign-tbaa.ll index d079c03f1dcb93..996d2c0e67e165 100644 --- a/llvm/test/Transforms/InstCombine/struct-assign-tbaa.ll +++ b/llvm/test/Transforms/InstCombine/struct-assign-tbaa.ll @@ -75,7 +75,7 @@ entry: !1 = !{!"omnipotent char", !0} !2 = !{!5, !5, i64 0} !3 = !{i64 0, i64 4, !2} -!4 = !{i64 0, i64 8, !2} +!4 = !{i64 0, i64 8, null} !5 = !{!"float", !0} !6 = !{i64 0, i64 4, !2, i64 4, i64 4, !2} !7 = !{i64 0, i64 2, !2, i64 4, i64 6, !2} diff --git a/llvm/test/Transforms/SROA/tbaa-struct3.ll b/llvm/test/Transforms/SROA/tbaa-struct3.ll index 61034de81e4b27..0fcd787fef9769 100644 --- a/llvm/test/Transforms/SROA/tbaa-struct3.ll +++ b/llvm/test/Transforms/SROA/tbaa-struct3.ll @@ -539,7 +539,7 @@ declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias !6 = !{!5, !5, i64 0} !7 = !{i64 0, i64 8, !6, i64 8, i64 4, !1} !8 = !{i64 0, i64 4, !1, i64 4, i64 8, !6} -!9 = !{i64 0, i64 8, !6, i64 8, i64 8, !1} +!9 = !{i64 0, i64 8, !6, i64 4, i64 8, !1} !10 = !{i64 0, i64 2, !1, i64 2, i64 2, !1} !11 = !{i64 0, i64 1, !1, i64 1, i64 3, !1} !12 = !{i64 0, i64 2, !1, i64 2, i64 6, !1} diff --git a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll index 73ae66dd76c66e..bbcdcb6f586742 100644 --- a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll +++ b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll @@ -836,6 +836,5 @@ define <2 x i32> @f23_crash(<2 x i32> %srcvec, i32 %v1) { !2 = !{ !"set2", !0 } !3 = !{ !3, !{!"llvm.loop.parallel_accesses", !13} } !4 = !{ float 4.0 } -!5 = !{ i64 0, i64 8, !6 } -!6 = !{ !1, !1, i64 0 } +!5 = !{ i64 0, i64 8, null } !13 = distinct !{} diff --git a/llvm/test/Transforms/Scalarizer/basic.ll b/llvm/test/Transforms/Scalarizer/basic.ll index 87a70ccd3fc7c5..db7c5f535f7e9d 100644 --- a/llvm/test/Transforms/Scalarizer/basic.ll +++ b/llvm/test/Transforms/Scalarizer/basic.ll @@ -870,6 +870,5 @@ define <2 x float> @f25(<2 x float> %src) { !2 = !{ !"set2", !0 } !3 = !{ !3, !{!"llvm.loop.parallel_accesses", !13} } !4 = !{ float 4.0 } -!5 = !{ i64 0, i64 8, !6 } -!6 = !{ !1, !1, i64 0 } +!5 = !{ i64 0, i64 8, null } !13 = distinct !{} diff --git a/llvm/test/Verifier/tbaa-struct.ll b/llvm/test/Verifier/tbaa-struct.ll index 14c19a19d5ae89..b8ddc7cee496a9 100644 --- a/llvm/test/Verifier/tbaa-struct.ll +++ b/llvm/test/Verifier/tbaa-struct.ll @@ -1,36 +1,28 @@ -; RUN: not llvm-as < %s 2>&1 | FileCheck %s +; RUN: llvm-as < %s 2>&1 + +; FIXME: The verifer should reject the invalid !tbaa.struct nodes below. define void @test_overlapping_regions(ptr %a1) { -; CHECK: Overlapping tbaa.struct regions -; CHECK-NEXT: %ld = load i8, ptr %a1, align 1, !tbaa.struct !0 %ld = load i8, ptr %a1, align 1, !tbaa.struct !0 ret void } define void @test_size_not_integer(ptr %a1) { -; CHECK: Size must be a constant integer -; CHECK-NEXT: store i8 1, ptr %a1, align 1, !tbaa.struct !5 store i8 1, ptr %a1, align 1, !tbaa.struct !5 ret void } define void @test_offset_not_integer(ptr %a1, ptr %a2) { -; CHECK: Offset must be a constant integer -; CHECK-NEXT: tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a1, ptr align 8 %a2, i64 16, i1 false), !tbaa.struct !6 tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a1, ptr align 8 %a2, i64 16, i1 false), !tbaa.struct !6 ret void } define void @test_tbaa_missing(ptr %a1, ptr %a2) { -; CHECK: TBAA tag missing -; CHECK-NEXT: tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a1, ptr align 8 %a2, i64 16, i1 false), !tbaa.struct !7 tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a1, ptr align 8 %a2, i64 16, i1 false), !tbaa.struct !7 ret void } define void @test_tbaa_invalid(ptr %a1) { -; CHECK: Old-style TBAA is no longer allowed, use struct-path TBAA instead -; CHECK-NEXT: store i8 1, ptr %a1, align 1, !tbaa.struct !8 store i8 1, ptr %a1, align 1, !tbaa.struct !8 ret void } From 552c8eb731a1fabef4d81e2a69911506adf39e22 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Wed, 27 Mar 2024 14:25:30 -0700 Subject: [PATCH 18/54] [SLP][NFC]Add a test with the wrong result extension after reduction, NFC. --- .../reduction-extension-after-bitwidth.ll | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/reduction-extension-after-bitwidth.ll diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reduction-extension-after-bitwidth.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reduction-extension-after-bitwidth.ll new file mode 100644 index 00000000000000..611003a55d735b --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reduction-extension-after-bitwidth.ll @@ -0,0 +1,33 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -mtriple=riscv64-unknown-linux-gnu -mattr="+v" --passes=slp-vectorizer < %s | FileCheck %s + +define i32 @test(ptr %0, ptr %1) { +; CHECK-LABEL: define i32 @test( +; CHECK-SAME: ptr [[TMP0:%.*]], ptr [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOAD_5:%.*]] = load i32, ptr [[TMP1]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> ) +; CHECK-NEXT: [[TMP3:%.*]] = sext i1 [[TMP2]] to i32 +; CHECK-NEXT: [[OP_RDX:%.*]] = and i32 [[TMP3]], [[LOAD_5]] +; CHECK-NEXT: ret i32 [[OP_RDX]] +; +entry: + %zext.0 = zext i8 1 to i32 + %zext.1 = zext i8 1 to i32 + %zext.2 = zext i8 1 to i32 + %zext.3 = zext i8 1 to i32 + %select.zext.0 = select i1 false, i32 -1, i32 %zext.0 + %select.zext.1 = select i1 false, i32 0, i32 %zext.1 + %select.zext.2 = select i1 false, i32 0, i32 %zext.2 + %select.zext.3 = select i1 false, i32 0, i32 %zext.3 + + %load.5 = load i32, ptr %1, align 4 + + %and.0 = and i32 %load.5, %select.zext.0 + %and.1 = and i32 %and.0, %select.zext.1 + %and.2 = and i32 %and.1, %select.zext.2 + %and.3 = and i32 %and.2, %select.zext.3 + + ret i32 %and.3 +} + From 742a82a729925dc79641beb649f492003be40725 Mon Sep 17 00:00:00 2001 From: alx32 <103613512+alx32@users.noreply.github.com> Date: Wed, 27 Mar 2024 14:34:27 -0700 Subject: [PATCH 19/54] [lld-macho] Implement support for ObjC relative method lists (#86231) The MachO format supports relative offsets for ObjC method lists. This support is present already in ld64. With this change we implement this support in lld also. Relative method lists can be identified by a specific flag (0x80000000) in the method list header. When this flag is present, the method list will contain 32-bit relative offsets to the current Program Counter (PC), instead of absolute pointers. Additionally, when relative method lists are used, the offset to the selector name will now be relative and point to the selector reference (selref) instead of the name itself. --- lld/MachO/Config.h | 1 + lld/MachO/Driver.cpp | 17 ++ lld/MachO/InputSection.cpp | 8 + lld/MachO/InputSection.h | 1 + lld/MachO/MapFile.cpp | 22 +- lld/MachO/ObjC.h | 2 + lld/MachO/Options.td | 6 + lld/MachO/SyntheticSections.cpp | 236 +++++++++++++++++ lld/MachO/SyntheticSections.h | 49 ++++ lld/MachO/Writer.cpp | 3 + .../MachO/objc-relative-method-lists-simple.s | 245 ++++++++++++++++++ 11 files changed, 583 insertions(+), 7 deletions(-) create mode 100644 lld/test/MachO/objc-relative-method-lists-simple.s diff --git a/lld/MachO/Config.h b/lld/MachO/Config.h index f820513a111ea3..7b45f7f4c39a1b 100644 --- a/lld/MachO/Config.h +++ b/lld/MachO/Config.h @@ -135,6 +135,7 @@ struct Configuration { bool emitEncryptionInfo = false; bool emitInitOffsets = false; bool emitChainedFixups = false; + bool emitRelativeMethodLists = false; bool thinLTOEmitImportsFiles; bool thinLTOEmitIndexFiles; bool thinLTOIndexOnly; diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp index 14c111ce9685c9..65de531db04b75 100644 --- a/lld/MachO/Driver.cpp +++ b/lld/MachO/Driver.cpp @@ -1086,6 +1086,22 @@ static bool shouldEmitChainedFixups(const InputArgList &args) { return isRequested; } +static bool shouldEmitRelativeMethodLists(const InputArgList &args) { + const Arg *arg = args.getLastArg(OPT_objc_relative_method_lists, + OPT_no_objc_relative_method_lists); + if (arg && arg->getOption().getID() == OPT_objc_relative_method_lists) + return true; + if (arg && arg->getOption().getID() == OPT_no_objc_relative_method_lists) + return false; + + // TODO: If no flag is specified, don't default to false, but instead: + // - default false on < ios14 + // - default true on >= ios14 + // For now, until this feature is confirmed stable, default to false if no + // flag is explicitly specified + return false; +} + void SymbolPatterns::clear() { literals.clear(); globs.clear(); @@ -1630,6 +1646,7 @@ bool link(ArrayRef argsArr, llvm::raw_ostream &stdoutOS, config->emitChainedFixups = shouldEmitChainedFixups(args); config->emitInitOffsets = config->emitChainedFixups || args.hasArg(OPT_init_offsets); + config->emitRelativeMethodLists = shouldEmitRelativeMethodLists(args); config->icfLevel = getICFLevel(args); config->dedupStrings = args.hasFlag(OPT_deduplicate_strings, OPT_no_deduplicate_strings, true); diff --git a/lld/MachO/InputSection.cpp b/lld/MachO/InputSection.cpp index 22930d52dd1db2..e3d7a400e4ce92 100644 --- a/lld/MachO/InputSection.cpp +++ b/lld/MachO/InputSection.cpp @@ -46,6 +46,14 @@ void lld::macho::addInputSection(InputSection *inputSection) { if (auto *isec = dyn_cast(inputSection)) { if (isec->isCoalescedWeak()) return; + if (config->emitRelativeMethodLists && + ObjCMethListSection::isMethodList(isec)) { + if (in.objcMethList->inputOrder == UnspecifiedInputOrder) + in.objcMethList->inputOrder = inputSectionsOrder++; + in.objcMethList->addInput(isec); + isec->parent = in.objcMethList; + return; + } if (config->emitInitOffsets && sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS) { in.initOffsets->addInput(isec); diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h index 694bdf734907ba..a0e6afef92cb82 100644 --- a/lld/MachO/InputSection.h +++ b/lld/MachO/InputSection.h @@ -342,6 +342,7 @@ constexpr const char moduleTermFunc[] = "__mod_term_func"; constexpr const char nonLazySymbolPtr[] = "__nl_symbol_ptr"; constexpr const char objcCatList[] = "__objc_catlist"; constexpr const char objcClassList[] = "__objc_classlist"; +constexpr const char objcMethList[] = "__objc_methlist"; constexpr const char objcClassRefs[] = "__objc_classrefs"; constexpr const char objcConst[] = "__objc_const"; constexpr const char objCImageInfo[] = "__objc_imageinfo"; diff --git a/lld/MachO/MapFile.cpp b/lld/MachO/MapFile.cpp index f736360624ebd1..2a31a5c09cdd22 100644 --- a/lld/MachO/MapFile.cpp +++ b/lld/MachO/MapFile.cpp @@ -197,18 +197,24 @@ void macho::writeMapFile() { seg->name.str().c_str(), osec->name.str().c_str()); } + // Shared function to print an array of symbols. + auto printIsecArrSyms = [&](const std::vector &arr) { + for (const ConcatInputSection *isec : arr) { + for (Defined *sym : isec->symbols) { + if (!(isPrivateLabel(sym->getName()) && sym->size == 0)) + os << format("0x%08llX\t0x%08llX\t[%3u] %s\n", sym->getVA(), + sym->size, readerToFileOrdinal[sym->getFile()], + sym->getName().str().data()); + } + } + }; + os << "# Symbols:\n"; os << "# Address\tSize \tFile Name\n"; for (const OutputSegment *seg : outputSegments) { for (const OutputSection *osec : seg->getSections()) { if (auto *concatOsec = dyn_cast(osec)) { - for (const InputSection *isec : concatOsec->inputs) { - for (Defined *sym : isec->symbols) - if (!(isPrivateLabel(sym->getName()) && sym->size == 0)) - os << format("0x%08llX\t0x%08llX\t[%3u] %s\n", sym->getVA(), - sym->size, readerToFileOrdinal[sym->getFile()], - sym->getName().str().data()); - } + printIsecArrSyms(concatOsec->inputs); } else if (osec == in.cStringSection || osec == in.objcMethnameSection) { const auto &liveCStrings = info.liveCStringsForSection.lookup(osec); uint64_t lastAddr = 0; // strings will never start at address 0, so this @@ -237,6 +243,8 @@ void macho::writeMapFile() { printNonLazyPointerSection(os, in.got); } else if (osec == in.tlvPointers) { printNonLazyPointerSection(os, in.tlvPointers); + } else if (osec == in.objcMethList) { + printIsecArrSyms(in.objcMethList->getInputs()); } // TODO print other synthetic sections } diff --git a/lld/MachO/ObjC.h b/lld/MachO/ObjC.h index 9fbe984e6223ec..8081605670c519 100644 --- a/lld/MachO/ObjC.h +++ b/lld/MachO/ObjC.h @@ -22,6 +22,8 @@ constexpr const char klassPropList[] = "__OBJC_$_CLASS_PROP_LIST_"; constexpr const char metaclass[] = "_OBJC_METACLASS_$_"; constexpr const char ehtype[] = "_OBJC_EHTYPE_$_"; constexpr const char ivar[] = "_OBJC_IVAR_$_"; +constexpr const char instanceMethods[] = "__OBJC_$_INSTANCE_METHODS_"; +constexpr const char classMethods[] = "__OBJC_$_CLASS_METHODS_"; constexpr const char listProprieties[] = "__OBJC_$_PROP_LIST_"; constexpr const char category[] = "__OBJC_$_CATEGORY_"; diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td index 0d8ee2a0926be2..19f8509ba714bd 100644 --- a/lld/MachO/Options.td +++ b/lld/MachO/Options.td @@ -1284,6 +1284,12 @@ def fixup_chains_section : Flag<["-"], "fixup_chains_section">, HelpText<"This option is undocumented in ld64">, Flags<[HelpHidden]>, Group; +def objc_relative_method_lists : Flag<["-"], "objc_relative_method_lists">, + HelpText<"Emit relative method lists (more compact representation)">, + Group; +def no_objc_relative_method_lists : Flag<["-"], "no_objc_relative_method_lists">, + HelpText<"Don't emit relative method lists (use traditional representation)">, + Group; def flto_codegen_only : Flag<["-"], "flto-codegen-only">, HelpText<"This option is undocumented in ld64">, Flags<[HelpHidden]>, diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp index 0afbbd478bb9fd..808ea8eac6eb59 100644 --- a/lld/MachO/SyntheticSections.cpp +++ b/lld/MachO/SyntheticSections.cpp @@ -12,6 +12,7 @@ #include "ExportTrie.h" #include "InputFiles.h" #include "MachOStructs.h" +#include "ObjC.h" #include "OutputSegment.h" #include "SymbolTable.h" #include "Symbols.h" @@ -1975,6 +1976,241 @@ void InitOffsetsSection::setUp() { } } +ObjCMethListSection::ObjCMethListSection() + : SyntheticSection(segment_names::text, section_names::objcMethList) { + flags = S_ATTR_NO_DEAD_STRIP; + align = relativeOffsetSize; +} + +// Go through all input method lists and ensure that we have selrefs for all +// their method names. The selrefs will be needed later by ::writeTo. We need to +// create them early on here to ensure they are processed correctly by the lld +// pipeline. +void ObjCMethListSection::setUp() { + for (const ConcatInputSection *isec : inputs) { + uint32_t structSizeAndFlags = 0, structCount = 0; + readMethodListHeader(isec->data.data(), structSizeAndFlags, structCount); + uint32_t originalStructSize = structSizeAndFlags & structSizeMask; + // Method name is immediately after header + uint32_t methodNameOff = methodListHeaderSize; + + // Loop through all methods, and ensure a selref for each of them exists. + while (methodNameOff < isec->data.size()) { + const Reloc *reloc = isec->getRelocAt(methodNameOff); + assert(reloc && "Relocation expected at method list name slot"); + auto *def = dyn_cast_or_null(reloc->referent.get()); + assert(def && "Expected valid Defined at method list name slot"); + auto *cisec = cast(def->isec); + assert(cisec && "Expected method name to be in a CStringInputSection"); + auto methname = cisec->getStringRefAtOffset(def->value); + if (!ObjCSelRefsHelper::getSelRef(methname)) + ObjCSelRefsHelper::makeSelRef(methname); + + // Jump to method name offset in next struct + methodNameOff += originalStructSize; + } + } +} + +// Calculate section size and final offsets for where InputSection's need to be +// written. +void ObjCMethListSection::finalize() { + // sectionSize will be the total size of the __objc_methlist section + sectionSize = 0; + for (ConcatInputSection *isec : inputs) { + // We can also use sectionSize as write offset for isec + assert(sectionSize == alignToPowerOf2(sectionSize, relativeOffsetSize) && + "expected __objc_methlist to be aligned by default with the " + "required section alignment"); + isec->outSecOff = sectionSize; + + isec->isFinal = true; + uint32_t relativeListSize = + computeRelativeMethodListSize(isec->data.size()); + sectionSize += relativeListSize; + + // If encoding the method list in relative offset format shrinks the size, + // then we also need to adjust symbol sizes to match the new size. Note that + // on 32bit platforms the size of the method list will remain the same when + // encoded in relative offset format. + if (relativeListSize != isec->data.size()) { + for (Symbol *sym : isec->symbols) { + assert(isa(sym) && + "Unexpected undefined symbol in ObjC method list"); + auto *def = cast(sym); + // There can be 0-size symbols, check if this is the case and ignore + // them. + if (def->size) { + assert( + def->size == isec->data.size() && + "Invalid ObjC method list symbol size: expected symbol size to " + "match isec size"); + def->size = relativeListSize; + } + } + } + } +} + +void ObjCMethListSection::writeTo(uint8_t *bufStart) const { + uint8_t *buf = bufStart; + for (const ConcatInputSection *isec : inputs) { + assert(buf - bufStart == long(isec->outSecOff) && + "Writing at unexpected offset"); + uint32_t writtenSize = writeRelativeMethodList(isec, buf); + buf += writtenSize; + } + assert(buf - bufStart == sectionSize && + "Written size does not match expected section size"); +} + +// Check if an InputSection is a method list. To do this we scan the +// InputSection for any symbols who's names match the patterns we expect clang +// to generate for method lists. +bool ObjCMethListSection::isMethodList(const ConcatInputSection *isec) { + const char *symPrefixes[] = {objc::symbol_names::classMethods, + objc::symbol_names::instanceMethods, + objc::symbol_names::categoryInstanceMethods, + objc::symbol_names::categoryClassMethods}; + if (!isec) + return false; + for (const Symbol *sym : isec->symbols) { + auto *def = dyn_cast_or_null(sym); + if (!def) + continue; + for (const char *prefix : symPrefixes) { + if (def->getName().starts_with(prefix)) { + assert(def->size == isec->data.size() && + "Invalid ObjC method list symbol size: expected symbol size to " + "match isec size"); + assert(def->value == 0 && + "Offset of ObjC method list symbol must be 0"); + return true; + } + } + } + + return false; +} + +// Encode a single relative offset value. The input is the data/symbol at +// (&isec->data[inSecOff]). The output is written to (&buf[outSecOff]). +// 'createSelRef' indicates that we should not directly use the specified +// symbol, but instead get the selRef for the symbol and use that instead. +void ObjCMethListSection::writeRelativeOffsetForIsec( + const ConcatInputSection *isec, uint8_t *buf, uint32_t &inSecOff, + uint32_t &outSecOff, bool useSelRef) const { + const Reloc *reloc = isec->getRelocAt(inSecOff); + assert(reloc && "Relocation expected at __objc_methlist Offset"); + auto *def = dyn_cast_or_null(reloc->referent.get()); + assert(def && "Expected all syms in __objc_methlist to be defined"); + uint32_t symVA = def->getVA(); + + if (useSelRef) { + auto *cisec = cast(def->isec); + auto methname = cisec->getStringRefAtOffset(def->value); + ConcatInputSection *selRef = ObjCSelRefsHelper::getSelRef(methname); + assert(selRef && "Expected all selector names to already be already be " + "present in __objc_selrefs"); + symVA = selRef->getVA(); + assert(selRef->data.size() == sizeof(target->wordSize) && + "Expected one selref per ConcatInputSection"); + } + + uint32_t currentVA = isec->getVA() + outSecOff; + uint32_t delta = symVA - currentVA; + write32le(buf + outSecOff, delta); + + // Move one pointer forward in the absolute method list + inSecOff += target->wordSize; + // Move one relative offset forward in the relative method list (32 bits) + outSecOff += relativeOffsetSize; +} + +// Write a relative method list to buf, return the size of the written +// information +uint32_t +ObjCMethListSection::writeRelativeMethodList(const ConcatInputSection *isec, + uint8_t *buf) const { + // Copy over the header, and add the "this is a relative method list" magic + // value flag + uint32_t structSizeAndFlags = 0, structCount = 0; + readMethodListHeader(isec->data.data(), structSizeAndFlags, structCount); + // Set the struct size for the relative method list + uint32_t relativeStructSizeAndFlags = + (relativeOffsetSize * pointersPerStruct) & structSizeMask; + // Carry over the old flags from the input struct + relativeStructSizeAndFlags |= structSizeAndFlags & structFlagsMask; + // Set the relative method list flag + relativeStructSizeAndFlags |= relMethodHeaderFlag; + + writeMethodListHeader(buf, relativeStructSizeAndFlags, structCount); + + assert(methodListHeaderSize + + (structCount * pointersPerStruct * target->wordSize) == + isec->data.size() && + "Invalid computed ObjC method list size"); + + uint32_t inSecOff = methodListHeaderSize; + uint32_t outSecOff = methodListHeaderSize; + + // Go through the method list and encode input absolute pointers as relative + // offsets. writeRelativeOffsetForIsec will be incrementing inSecOff and + // outSecOff + for (uint32_t i = 0; i < structCount; i++) { + // Write the name of the method + writeRelativeOffsetForIsec(isec, buf, inSecOff, outSecOff, true); + // Write the type of the method + writeRelativeOffsetForIsec(isec, buf, inSecOff, outSecOff, false); + // Write reference to the selector of the method + writeRelativeOffsetForIsec(isec, buf, inSecOff, outSecOff, false); + } + + // Expecting to have read all the data in the isec + assert(inSecOff == isec->data.size() && + "Invalid actual ObjC method list size"); + assert( + outSecOff == computeRelativeMethodListSize(inSecOff) && + "Mismatch between input & output size when writing relative method list"); + return outSecOff; +} + +// Given the size of an ObjC method list InputSection, return the size of the +// method list when encoded in relative offsets format. We can do this without +// decoding the actual data, as it can be directly inferred from the size of the +// isec. +uint32_t ObjCMethListSection::computeRelativeMethodListSize( + uint32_t absoluteMethodListSize) const { + uint32_t oldPointersSize = absoluteMethodListSize - methodListHeaderSize; + uint32_t pointerCount = oldPointersSize / target->wordSize; + assert(((pointerCount % pointersPerStruct) == 0) && + "__objc_methlist expects method lists to have multiple-of-3 pointers"); + + uint32_t newPointersSize = pointerCount * relativeOffsetSize; + uint32_t newTotalSize = methodListHeaderSize + newPointersSize; + + assert((newTotalSize <= absoluteMethodListSize) && + "Expected relative method list size to be smaller or equal than " + "original size"); + return newTotalSize; +} + +// Read a method list header from buf +void ObjCMethListSection::readMethodListHeader(const uint8_t *buf, + uint32_t &structSizeAndFlags, + uint32_t &structCount) const { + structSizeAndFlags = read32le(buf); + structCount = read32le(buf + sizeof(uint32_t)); +} + +// Write a method list header to buf +void ObjCMethListSection::writeMethodListHeader(uint8_t *buf, + uint32_t structSizeAndFlags, + uint32_t structCount) const { + write32le(buf, structSizeAndFlags); + write32le(buf + sizeof(structSizeAndFlags), structCount); +} + void macho::createSyntheticSymbols() { auto addHeaderSymbol = [](const char *name) { symtab->addSynthetic(name, in.header->isec, /*value=*/0, diff --git a/lld/MachO/SyntheticSections.h b/lld/MachO/SyntheticSections.h index 4586a4a0bf4361..e8fadfef56d4b2 100644 --- a/lld/MachO/SyntheticSections.h +++ b/lld/MachO/SyntheticSections.h @@ -684,6 +684,54 @@ class InitOffsetsSection final : public SyntheticSection { std::vector sections; }; +// This SyntheticSection is for the __objc_methlist section, which contains +// relative method lists if the -objc_relative_method_lists option is enabled. +class ObjCMethListSection final : public SyntheticSection { +public: + ObjCMethListSection(); + + static bool isMethodList(const ConcatInputSection *isec); + void addInput(ConcatInputSection *isec) { inputs.push_back(isec); } + std::vector getInputs() { return inputs; } + + void setUp(); + void finalize() override; + bool isNeeded() const override { return !inputs.empty(); } + uint64_t getSize() const override { return sectionSize; } + void writeTo(uint8_t *bufStart) const override; + +private: + void readMethodListHeader(const uint8_t *buf, uint32_t &structSizeAndFlags, + uint32_t &structCount) const; + void writeMethodListHeader(uint8_t *buf, uint32_t structSizeAndFlags, + uint32_t structCount) const; + uint32_t computeRelativeMethodListSize(uint32_t absoluteMethodListSize) const; + void writeRelativeOffsetForIsec(const ConcatInputSection *isec, uint8_t *buf, + uint32_t &inSecOff, uint32_t &outSecOff, + bool useSelRef) const; + uint32_t writeRelativeMethodList(const ConcatInputSection *isec, + uint8_t *buf) const; + + static constexpr uint32_t methodListHeaderSize = + /*structSizeAndFlags*/ sizeof(uint32_t) + + /*structCount*/ sizeof(uint32_t); + // Relative method lists are supported only for 3-pointer method lists + static constexpr uint32_t pointersPerStruct = 3; + // The runtime identifies relative method lists via this magic value + static constexpr uint32_t relMethodHeaderFlag = 0x80000000; + // In the method list header, the first 2 bytes are the size of struct + static constexpr uint32_t structSizeMask = 0x0000FFFF; + // In the method list header, the last 2 bytes are the flags for the struct + static constexpr uint32_t structFlagsMask = 0xFFFF0000; + // Relative method lists have 4 byte alignment as all data in the InputSection + // is 4 byte + static constexpr uint32_t relativeOffsetSize = sizeof(uint32_t); + + // The output size of the __objc_methlist section, computed during finalize() + uint32_t sectionSize = 0; + std::vector inputs; +}; + // Chained fixups are a replacement for classic dyld opcodes. In this format, // most of the metadata necessary for binding symbols and rebasing addresses is // stored directly in the memory location that will have the fixup applied. @@ -810,6 +858,7 @@ struct InStruct { ObjCImageInfoSection *objCImageInfo = nullptr; ConcatInputSection *imageLoaderCache = nullptr; InitOffsetsSection *initOffsets = nullptr; + ObjCMethListSection *objcMethList = nullptr; ChainedFixupsSection *chainedFixups = nullptr; }; diff --git a/lld/MachO/Writer.cpp b/lld/MachO/Writer.cpp index a18b5268fd42aa..fe989de648d789 100644 --- a/lld/MachO/Writer.cpp +++ b/lld/MachO/Writer.cpp @@ -1292,6 +1292,8 @@ template void Writer::run() { scanSymbols(); if (in.objcStubs->isNeeded()) in.objcStubs->setUp(); + if (in.objcMethList->isNeeded()) + in.objcMethList->setUp(); scanRelocations(); if (in.initOffsets->isNeeded()) in.initOffsets->setUp(); @@ -1363,6 +1365,7 @@ void macho::createSyntheticSections() { in.unwindInfo = makeUnwindInfoSection(); in.objCImageInfo = make(); in.initOffsets = make(); + in.objcMethList = make(); // This section contains space for just a single word, and will be used by // dyld to cache an address to the image loader it uses. diff --git a/lld/test/MachO/objc-relative-method-lists-simple.s b/lld/test/MachO/objc-relative-method-lists-simple.s new file mode 100644 index 00000000000000..1ffec3c6241cf4 --- /dev/null +++ b/lld/test/MachO/objc-relative-method-lists-simple.s @@ -0,0 +1,245 @@ +# REQUIRES: aarch64 +# RUN: rm -rf %t; split-file %s %t && cd %t + +## Compile a64_rel_dylib.o +# RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o a64_rel_dylib.o a64_simple_class.s + +## Test arm64 + relative method lists +# RUN: %no-lsystem-lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64 -objc_relative_method_lists +# RUN: llvm-objdump --macho --objc-meta-data a64_rel_dylib.dylib | FileCheck %s --check-prefix=CHK_REL + +## Test arm64 + traditional method lists (no relative offsets) +# RUN: %no-lsystem-lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64 -no_objc_relative_method_lists +# RUN: llvm-objdump --macho --objc-meta-data a64_rel_dylib.dylib | FileCheck %s --check-prefix=CHK_NO_REL + + +CHK_REL: Contents of (__DATA_CONST,__objc_classlist) section +CHK_REL-NEXT: _OBJC_CLASS_$_MyClass +CHK_REL: baseMethods +CHK_REL-NEXT: entsize 12 (relative) +CHK_REL-NEXT: count 3 +CHK_REL-NEXT: name 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) instance_method_00 +CHK_REL-NEXT: types 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) v16@0:8 +CHK_REL-NEXT: imp 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) -[MyClass instance_method_00] +CHK_REL-NEXT: name 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) instance_method_01 +CHK_REL-NEXT: types 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) v16@0:8 +CHK_REL-NEXT: imp 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) -[MyClass instance_method_01] +CHK_REL-NEXT: name 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) instance_method_02 +CHK_REL-NEXT: types 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) v16@0:8 +CHK_REL-NEXT: imp 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) -[MyClass instance_method_02] + +CHK_REL: Meta Class +CHK_REL-NEXT: isa 0x{{[0-9a-f]*}} _OBJC_METACLASS_$_MyClass +CHK_REL: baseMethods 0x{{[0-9a-f]*}} (struct method_list_t *) +CHK_REL-NEXT: entsize 12 (relative) +CHK_REL-NEXT: count 3 +CHK_REL-NEXT: name 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) class_method_00 +CHK_REL-NEXT: types 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) v16@0:8 +CHK_REL-NEXT: imp 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) +[MyClass class_method_00] +CHK_REL-NEXT: name 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) class_method_01 +CHK_REL-NEXT: types 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) v16@0:8 +CHK_REL-NEXT: imp 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) +[MyClass class_method_01] +CHK_REL-NEXT: name 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) class_method_02 +CHK_REL-NEXT: types 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) v16@0:8 +CHK_REL-NEXT: imp 0x{{[0-9a-f]*}} (0x{{[0-9a-f]*}}) +[MyClass class_method_02] + + +CHK_NO_REL-NOT: (relative) + +CHK_NO_REL: Contents of (__DATA_CONST,__objc_classlist) section +CHK_NO_REL-NEXT: _OBJC_CLASS_$_MyClass + +CHK_NO_REL: baseMethods 0x{{[0-9a-f]*}} (struct method_list_t *) +CHK_NO_REL-NEXT: entsize 24 +CHK_NO_REL-NEXT: count 3 +CHK_NO_REL-NEXT: name 0x{{[0-9a-f]*}} instance_method_00 +CHK_NO_REL-NEXT: types 0x{{[0-9a-f]*}} v16@0:8 +CHK_NO_REL-NEXT: imp -[MyClass instance_method_00] +CHK_NO_REL-NEXT: name 0x{{[0-9a-f]*}} instance_method_01 +CHK_NO_REL-NEXT: types 0x{{[0-9a-f]*}} v16@0:8 +CHK_NO_REL-NEXT: imp -[MyClass instance_method_01] +CHK_NO_REL-NEXT: name 0x{{[0-9a-f]*}} instance_method_02 +CHK_NO_REL-NEXT: types 0x{{[0-9a-f]*}} v16@0:8 +CHK_NO_REL-NEXT: imp -[MyClass instance_method_02] + + +CHK_NO_REL: Meta Class +CHK_NO_REL-NEXT: _OBJC_METACLASS_$_MyClass + +CHK_NO_REL: baseMethods 0x{{[0-9a-f]*}} (struct method_list_t *) +CHK_NO_REL-NEXT: entsize 24 +CHK_NO_REL-NEXT: count 3 +CHK_NO_REL-NEXT: name 0x{{[0-9a-f]*}} class_method_00 +CHK_NO_REL-NEXT: types 0x{{[0-9a-f]*}} v16@0:8 +CHK_NO_REL-NEXT: imp +[MyClass class_method_00] +CHK_NO_REL-NEXT: name 0x{{[0-9a-f]*}} class_method_01 +CHK_NO_REL-NEXT: types 0x{{[0-9a-f]*}} v16@0:8 +CHK_NO_REL-NEXT: imp +[MyClass class_method_01] +CHK_NO_REL-NEXT: name 0x{{[0-9a-f]*}} class_method_02 +CHK_NO_REL-NEXT: types 0x{{[0-9a-f]*}} v16@0:8 +CHK_NO_REL-NEXT: imp +[MyClass class_method_02] + + +######################## Generate a64_simple_class.s ######################### +# clang -c simple_class.mm -s -o a64_simple_class.s -target arm64-apple-macos -arch arm64 -Oz + +######################## simple_class.mm ######################## +# __attribute__((objc_root_class)) +# @interface MyClass +# - (void)instance_method_00; +# - (void)instance_method_01; +# - (void)instance_method_02; +# + (void)class_method_00; +# + (void)class_method_01; +# + (void)class_method_02; +# @end +# +# @implementation MyClass +# - (void)instance_method_00 {} +# - (void)instance_method_01 {} +# - (void)instance_method_02 {} +# + (void)class_method_00 {} +# + (void)class_method_01 {} +# + (void)class_method_02 {} +# @end +# +# void *_objc_empty_cache; +# void *_objc_empty_vtable; +# + +#--- objc-macros.s +.macro .objc_selector_def name + .p2align 2 +"\name": + .cfi_startproc + ret + .cfi_endproc +.endm + +#--- a64_simple_class.s +.include "objc-macros.s" + +.section __TEXT,__text,regular,pure_instructions +.build_version macos, 11, 0 + +.objc_selector_def "-[MyClass instance_method_00]" +.objc_selector_def "-[MyClass instance_method_01]" +.objc_selector_def "-[MyClass instance_method_02]" + +.objc_selector_def "+[MyClass class_method_00]" +.objc_selector_def "+[MyClass class_method_01]" +.objc_selector_def "+[MyClass class_method_02]" + +.globl __objc_empty_vtable +.zerofill __DATA,__common,__objc_empty_vtable,8,3 +.section __DATA,__objc_data +.globl _OBJC_CLASS_$_MyClass +.p2align 3, 0x0 + +_OBJC_CLASS_$_MyClass: + .quad _OBJC_METACLASS_$_MyClass + .quad 0 + .quad __objc_empty_cache + .quad __objc_empty_vtable + .quad __OBJC_CLASS_RO_$_MyClass + .globl _OBJC_METACLASS_$_MyClass + .p2align 3, 0x0 + +_OBJC_METACLASS_$_MyClass: + .quad _OBJC_METACLASS_$_MyClass + .quad _OBJC_CLASS_$_MyClass + .quad __objc_empty_cache + .quad __objc_empty_vtable + .quad __OBJC_METACLASS_RO_$_MyClass + + .section __TEXT,__objc_classname,cstring_literals +l_OBJC_CLASS_NAME_: + .asciz "MyClass" + .section __TEXT,__objc_methname,cstring_literals +l_OBJC_METH_VAR_NAME_: + .asciz "class_method_00" + .section __TEXT,__objc_methtype,cstring_literals +l_OBJC_METH_VAR_TYPE_: + .asciz "v16@0:8" + .section __TEXT,__objc_methname,cstring_literals +l_OBJC_METH_VAR_NAME_.1: + .asciz "class_method_01" +l_OBJC_METH_VAR_NAME_.2: + .asciz "class_method_02" + .section __DATA,__objc_const + .p2align 3, 0x0 +__OBJC_$_CLASS_METHODS_MyClass: + .long 24 + .long 3 + .quad l_OBJC_METH_VAR_NAME_ + .quad l_OBJC_METH_VAR_TYPE_ + .quad "+[MyClass class_method_00]" + .quad l_OBJC_METH_VAR_NAME_.1 + .quad l_OBJC_METH_VAR_TYPE_ + .quad "+[MyClass class_method_01]" + .quad l_OBJC_METH_VAR_NAME_.2 + .quad l_OBJC_METH_VAR_TYPE_ + .quad "+[MyClass class_method_02]" + .p2align 3, 0x0 + +__OBJC_METACLASS_RO_$_MyClass: + .long 3 + .long 40 + .long 40 + .space 4 + .quad 0 + .quad l_OBJC_CLASS_NAME_ + .quad __OBJC_$_CLASS_METHODS_MyClass + .quad 0 + .quad 0 + .quad 0 + .quad 0 + + .section __TEXT,__objc_methname,cstring_literals +l_OBJC_METH_VAR_NAME_.3: + .asciz "instance_method_00" +l_OBJC_METH_VAR_NAME_.4: + .asciz "instance_method_01" +l_OBJC_METH_VAR_NAME_.5: + .asciz "instance_method_02" + + .section __DATA,__objc_const + .p2align 3, 0x0 +__OBJC_$_INSTANCE_METHODS_MyClass: + .long 24 + .long 3 + .quad l_OBJC_METH_VAR_NAME_.3 + .quad l_OBJC_METH_VAR_TYPE_ + .quad "-[MyClass instance_method_00]" + .quad l_OBJC_METH_VAR_NAME_.4 + .quad l_OBJC_METH_VAR_TYPE_ + .quad "-[MyClass instance_method_01]" + .quad l_OBJC_METH_VAR_NAME_.5 + .quad l_OBJC_METH_VAR_TYPE_ + .quad "-[MyClass instance_method_02]" + .p2align 3, 0x0 + +__OBJC_CLASS_RO_$_MyClass: + .long 2 + .long 0 + .long 0 + .space 4 + .quad 0 + .quad l_OBJC_CLASS_NAME_ + .quad __OBJC_$_INSTANCE_METHODS_MyClass + .quad 0 + .quad 0 + .quad 0 + .quad 0 + .globl __objc_empty_cache + +.zerofill __DATA,__common,__objc_empty_cache,8,3 + .section __DATA,__objc_classlist,regular,no_dead_strip + .p2align 3, 0x0 +l_OBJC_LABEL_CLASS_$: + .quad _OBJC_CLASS_$_MyClass + .section __DATA,__objc_imageinfo,regular,no_dead_strip +L_OBJC_IMAGE_INFO: + .long 0 + .long 64 +.subsections_via_symbols From d94dc5f0d63be3d786224f57c061ef16687fca9a Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Wed, 27 Mar 2024 14:32:13 -0700 Subject: [PATCH 20/54] [SLP]Fix PR86763: do not truncate reductions to the demanded bits size. Need to adjust ReductionBitWIdth after minbitwidth analysis, if the demanded bits analysis sjows tht its size is less than the size of the vectorized value. It prevents incorrect sign-zero extension transformation after. --- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 7 +++++++ .../RISCV/reduction-extension-after-bitwidth.ll | 4 ++-- .../SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll | 5 ++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index e1f26b922dbe4d..7f528848002b5b 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -14415,6 +14415,13 @@ void BoUpSLP::computeMinimumValueSizes() { unsigned MaxBitWidth = ComputeMaxBitWidth( TreeRoot, VectorizableTree[NodeIdx]->getVectorFactor(), IsTopRoot, IsProfitableToDemoteRoot, Opcode, Limit, IsTruncRoot); + if (ReductionBitWidth != 0 && (IsTopRoot || !RootDemotes.empty())) { + if (MaxBitWidth != 0 && ReductionBitWidth < MaxBitWidth) + ReductionBitWidth = bit_ceil(MaxBitWidth); + else if (MaxBitWidth == 0) + ReductionBitWidth = 0; + } + for (unsigned Idx : RootDemotes) ToDemote.append(VectorizableTree[Idx]->Scalars.begin(), VectorizableTree[Idx]->Scalars.end()); diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reduction-extension-after-bitwidth.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reduction-extension-after-bitwidth.ll index 611003a55d735b..7771e8369b6198 100644 --- a/llvm/test/Transforms/SLPVectorizer/RISCV/reduction-extension-after-bitwidth.ll +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reduction-extension-after-bitwidth.ll @@ -6,8 +6,8 @@ define i32 @test(ptr %0, ptr %1) { ; CHECK-SAME: ptr [[TMP0:%.*]], ptr [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[LOAD_5:%.*]] = load i32, ptr [[TMP1]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> ) -; CHECK-NEXT: [[TMP3:%.*]] = sext i1 [[TMP2]] to i32 +; CHECK-NEXT: [[TMP2:%.*]] = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> ) +; CHECK-NEXT: [[TMP3:%.*]] = sext i8 [[TMP2]] to i32 ; CHECK-NEXT: [[OP_RDX:%.*]] = and i32 [[TMP3]], [[LOAD_5]] ; CHECK-NEXT: ret i32 [[OP_RDX]] ; diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll index cfe3ca9f8f9e5f..7b4e2b0ce9112e 100644 --- a/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll +++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll @@ -11,9 +11,8 @@ define void @test(ptr %a, i8 %0, i16 %b.promoted.i) { ; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i128> [[TMP5]], <4 x i128> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[TMP7:%.*]] = trunc <4 x i128> [[TMP6]] to <4 x i16> ; CHECK-NEXT: [[TMP8:%.*]] = or <4 x i16> [[TMP4]], [[TMP7]] -; CHECK-NEXT: [[TMP9:%.*]] = trunc <4 x i16> [[TMP8]] to <4 x i1> -; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP9]]) -; CHECK-NEXT: [[TMP11:%.*]] = zext i1 [[TMP10]] to i64 +; CHECK-NEXT: [[TMP9:%.*]] = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> [[TMP8]]) +; CHECK-NEXT: [[TMP11:%.*]] = zext i16 [[TMP9]] to i64 ; CHECK-NEXT: [[OP_RDX:%.*]] = and i64 [[TMP11]], 1 ; CHECK-NEXT: store i64 [[OP_RDX]], ptr [[A]], align 8 ; CHECK-NEXT: ret void From 0a43ca731b1faedd885f86153ecc570dde602ca3 Mon Sep 17 00:00:00 2001 From: Shilei Tian Date: Wed, 27 Mar 2024 17:40:58 -0400 Subject: [PATCH 21/54] [AMDGPU] Fix missing `IsExact` flag when expanding vector binary operator (#86712) --- .../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 3 + .../AMDGPU/amdgpu-codegenprepare-idiv.ll | 108 ++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp index bddf3d958a1ae6..6e7d34f5adaa3f 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp @@ -1594,6 +1594,9 @@ bool AMDGPUCodeGenPrepareImpl::visitBinaryOperator(BinaryOperator &I) { } } + if (auto *NewEltI = dyn_cast(NewElt)) + NewEltI->copyIRFlags(&I); + NewDiv = Builder.CreateInsertElement(NewDiv, NewElt, N); } } else { diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll index d9001656f308e1..2ad28b8dd6ecf5 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll @@ -10668,3 +10668,111 @@ define amdgpu_kernel void @srem_v2i64_pow2_shl_denom(ptr addrspace(1) %out, <2 x store <2 x i64> %r, ptr addrspace(1) %out ret void } + +define <2 x i32> @v_sdiv_i32_exact(<2 x i32> %num) { +; CHECK-LABEL: @v_sdiv_i32_exact( +; CHECK: %1 = extractelement <2 x i32> %num, i64 0 +; CHECK-NEXT: %2 = sdiv exact i32 %1, 4096 +; CHECK-NEXT: %3 = insertelement <2 x i32> poison, i32 %2, i64 0 +; CHECK-NEXT: %4 = extractelement <2 x i32> %num, i64 1 +; CHECK-NEXT: %5 = sdiv exact i32 %4, 1024 +; CHECK-NEXT: %6 = insertelement <2 x i32> %3, i32 %5, i64 1 +; CHECK-NEXT: ret <2 x i32> %6 +; +; GFX6-LABEL: v_sdiv_i32_exact: +; GFX6: ; %bb.0: +; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX6-NEXT: v_ashrrev_i32_e32 v0, 12, v0 +; GFX6-NEXT: v_ashrrev_i32_e32 v1, 10, v1 +; GFX6-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-LABEL: v_sdiv_i32_exact: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_ashrrev_i32_e32 v0, 12, v0 +; GFX9-NEXT: v_ashrrev_i32_e32 v1, 10, v1 +; GFX9-NEXT: s_setpc_b64 s[30:31] + %result = sdiv exact <2 x i32> %num, + ret <2 x i32> %result +} + +define <2 x i64> @v_sdiv_i64_exact(<2 x i64> %num) { +; CHECK-LABEL: @v_sdiv_i64_exact( +; CHECK: %1 = extractelement <2 x i64> %num, i64 0 +; CHECK-NEXT: %2 = sdiv exact i64 %1, 4096 +; CHECK-NEXT: %3 = insertelement <2 x i64> poison, i64 %2, i64 0 +; CHECK-NEXT: %4 = extractelement <2 x i64> %num, i64 1 +; CHECK-NEXT: %5 = sdiv exact i64 %4, 1024 +; CHECK-NEXT: %6 = insertelement <2 x i64> %3, i64 %5, i64 1 +; CHECK-NEXT: ret <2 x i64> %6 +; +; GFX6-LABEL: v_sdiv_i64_exact: +; GFX6: ; %bb.0: +; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX6-NEXT: v_ashr_i64 v[0:1], v[0:1], 12 +; GFX6-NEXT: v_ashr_i64 v[2:3], v[2:3], 10 +; GFX6-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-LABEL: v_sdiv_i64_exact: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_ashrrev_i64 v[0:1], 12, v[0:1] +; GFX9-NEXT: v_ashrrev_i64 v[2:3], 10, v[2:3] +; GFX9-NEXT: s_setpc_b64 s[30:31] + %result = sdiv exact <2 x i64> %num, + ret <2 x i64> %result +} + +define <2 x i32> @v_udiv_i32_exact(<2 x i32> %num) { +; CHECK-LABEL: @v_udiv_i32_exact( +; CHECK: %1 = extractelement <2 x i32> %num, i64 0 +; CHECK-NEXT: %2 = udiv exact i32 %1, 4096 +; CHECK-NEXT: %3 = insertelement <2 x i32> poison, i32 %2, i64 0 +; CHECK-NEXT: %4 = extractelement <2 x i32> %num, i64 1 +; CHECK-NEXT: %5 = udiv exact i32 %4, 1024 +; CHECK-NEXT: %6 = insertelement <2 x i32> %3, i32 %5, i64 1 +; CHECK-NEXT: ret <2 x i32> %6 +; +; GFX6-LABEL: v_udiv_i32_exact: +; GFX6: ; %bb.0: +; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX6-NEXT: v_lshrrev_b32_e32 v0, 12, v0 +; GFX6-NEXT: v_lshrrev_b32_e32 v1, 10, v1 +; GFX6-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-LABEL: v_udiv_i32_exact: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_lshrrev_b32_e32 v0, 12, v0 +; GFX9-NEXT: v_lshrrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_setpc_b64 s[30:31] + %result = udiv exact <2 x i32> %num, + ret <2 x i32> %result +} + +define <2 x i64> @v_udiv_i64_exact(<2 x i64> %num) { +; CHECK-LABEL: @v_udiv_i64_exact( +; CHECK: %1 = extractelement <2 x i64> %num, i64 0 +; CHECK-NEXT: %2 = udiv exact i64 %1, 4096 +; CHECK-NEXT: %3 = insertelement <2 x i64> poison, i64 %2, i64 0 +; CHECK-NEXT: %4 = extractelement <2 x i64> %num, i64 1 +; CHECK-NEXT: %5 = udiv exact i64 %4, 1024 +; CHECK-NEXT: %6 = insertelement <2 x i64> %3, i64 %5, i64 1 +; CHECK-NEXT: ret <2 x i64> %6 +; +; GFX6-LABEL: v_udiv_i64_exact: +; GFX6: ; %bb.0: +; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX6-NEXT: v_lshr_b64 v[0:1], v[0:1], 12 +; GFX6-NEXT: v_lshr_b64 v[2:3], v[2:3], 10 +; GFX6-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-LABEL: v_udiv_i64_exact: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_lshrrev_b64 v[0:1], 12, v[0:1] +; GFX9-NEXT: v_lshrrev_b64 v[2:3], 10, v[2:3] +; GFX9-NEXT: s_setpc_b64 s[30:31] + %result = udiv exact <2 x i64> %num, + ret <2 x i64> %result +} From a8b90c047d5bb47702eebd4ceeb763e8537981a1 Mon Sep 17 00:00:00 2001 From: Shilei Tian Date: Wed, 27 Mar 2024 17:41:30 -0400 Subject: [PATCH 22/54] [GlobalISel] Update `MachineIRBuilder::buildAtomicRMW` interface (#86851) --- .../CodeGen/GlobalISel/MachineIRBuilder.h | 11 ++-- .../CodeGen/GlobalISel/MachineIRBuilder.cpp | 54 ++++++++++--------- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index 16a7fc446fbe1d..4c9d85fd9f5140 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -1333,9 +1333,9 @@ class MachineIRBuilder { /// /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder - buildAtomicCmpXchgWithSuccess(Register OldValRes, Register SuccessRes, - Register Addr, Register CmpVal, Register NewVal, - MachineMemOperand &MMO); + buildAtomicCmpXchgWithSuccess(const DstOp &OldValRes, const DstOp &SuccessRes, + const SrcOp &Addr, const SrcOp &CmpVal, + const SrcOp &NewVal, MachineMemOperand &MMO); /// Build and insert `OldValRes = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal, /// MMO`. @@ -1351,8 +1351,9 @@ class MachineIRBuilder { /// registers of the same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicCmpXchg(Register OldValRes, Register Addr, - Register CmpVal, Register NewVal, + MachineInstrBuilder buildAtomicCmpXchg(const DstOp &OldValRes, + const SrcOp &Addr, const SrcOp &CmpVal, + const SrcOp &NewVal, MachineMemOperand &MMO); /// Build and insert `OldValRes = G_ATOMICRMW_ Addr, Val, MMO`. diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index 07d4cb5eaa23c8..b8ba782254c370 100644 --- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -930,14 +930,14 @@ MachineIRBuilder::buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, } MachineInstrBuilder MachineIRBuilder::buildAtomicCmpXchgWithSuccess( - Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal, - Register NewVal, MachineMemOperand &MMO) { + const DstOp &OldValRes, const DstOp &SuccessRes, const SrcOp &Addr, + const SrcOp &CmpVal, const SrcOp &NewVal, MachineMemOperand &MMO) { #ifndef NDEBUG - LLT OldValResTy = getMRI()->getType(OldValRes); - LLT SuccessResTy = getMRI()->getType(SuccessRes); - LLT AddrTy = getMRI()->getType(Addr); - LLT CmpValTy = getMRI()->getType(CmpVal); - LLT NewValTy = getMRI()->getType(NewVal); + LLT OldValResTy = OldValRes.getLLTTy(*getMRI()); + LLT SuccessResTy = SuccessRes.getLLTTy(*getMRI()); + LLT AddrTy = Addr.getLLTTy(*getMRI()); + LLT CmpValTy = CmpVal.getLLTTy(*getMRI()); + LLT NewValTy = NewVal.getLLTTy(*getMRI()); assert(OldValResTy.isScalar() && "invalid operand type"); assert(SuccessResTy.isScalar() && "invalid operand type"); assert(AddrTy.isPointer() && "invalid operand type"); @@ -947,24 +947,25 @@ MachineInstrBuilder MachineIRBuilder::buildAtomicCmpXchgWithSuccess( assert(OldValResTy == NewValTy && "type mismatch"); #endif - return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS) - .addDef(OldValRes) - .addDef(SuccessRes) - .addUse(Addr) - .addUse(CmpVal) - .addUse(NewVal) - .addMemOperand(&MMO); + auto MIB = buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS); + OldValRes.addDefToMIB(*getMRI(), MIB); + SuccessRes.addDefToMIB(*getMRI(), MIB); + Addr.addSrcToMIB(MIB); + CmpVal.addSrcToMIB(MIB); + NewVal.addSrcToMIB(MIB); + MIB.addMemOperand(&MMO); + return MIB; } MachineInstrBuilder -MachineIRBuilder::buildAtomicCmpXchg(Register OldValRes, Register Addr, - Register CmpVal, Register NewVal, +MachineIRBuilder::buildAtomicCmpXchg(const DstOp &OldValRes, const SrcOp &Addr, + const SrcOp &CmpVal, const SrcOp &NewVal, MachineMemOperand &MMO) { #ifndef NDEBUG - LLT OldValResTy = getMRI()->getType(OldValRes); - LLT AddrTy = getMRI()->getType(Addr); - LLT CmpValTy = getMRI()->getType(CmpVal); - LLT NewValTy = getMRI()->getType(NewVal); + LLT OldValResTy = OldValRes.getLLTTy(*getMRI()); + LLT AddrTy = Addr.getLLTTy(*getMRI()); + LLT CmpValTy = CmpVal.getLLTTy(*getMRI()); + LLT NewValTy = NewVal.getLLTTy(*getMRI()); assert(OldValResTy.isScalar() && "invalid operand type"); assert(AddrTy.isPointer() && "invalid operand type"); assert(CmpValTy.isValid() && "invalid operand type"); @@ -973,12 +974,13 @@ MachineIRBuilder::buildAtomicCmpXchg(Register OldValRes, Register Addr, assert(OldValResTy == NewValTy && "type mismatch"); #endif - return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG) - .addDef(OldValRes) - .addUse(Addr) - .addUse(CmpVal) - .addUse(NewVal) - .addMemOperand(&MMO); + auto MIB = buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG); + OldValRes.addDefToMIB(*getMRI(), MIB); + Addr.addSrcToMIB(MIB); + CmpVal.addSrcToMIB(MIB); + NewVal.addSrcToMIB(MIB); + MIB.addMemOperand(&MMO); + return MIB; } MachineInstrBuilder MachineIRBuilder::buildAtomicRMW( From 5da39372e39f3aebf63e07faf774b6ed37cad3fb Mon Sep 17 00:00:00 2001 From: smanna12 Date: Wed, 27 Mar 2024 16:44:41 -0500 Subject: [PATCH 23/54] [clang-installapi] Remove unnecessary copy (#86808) Reported by Static Analyzer Tool: In clang::installapi::InstallAPIVisitor::VisitFunctionDecl(clang::FunctionDecl const *): Using the auto keyword without an & causes the copy of an object of type DynTypedNode. --- clang/lib/InstallAPI/Visitor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/lib/InstallAPI/Visitor.cpp b/clang/lib/InstallAPI/Visitor.cpp index f8f5d8d53d5691..6476c5107cb5cc 100644 --- a/clang/lib/InstallAPI/Visitor.cpp +++ b/clang/lib/InstallAPI/Visitor.cpp @@ -255,7 +255,7 @@ bool InstallAPIVisitor::VisitFunctionDecl(const FunctionDecl *D) { return true; // Skip methods in CXX RecordDecls. - for (auto P : D->getASTContext().getParents(*M)) { + for (const DynTypedNode &P : D->getASTContext().getParents(*M)) { if (P.get()) return true; } From b7a4ace72edf79f8250df2b08f0c14177d346770 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Wed, 27 Mar 2024 14:40:45 -0700 Subject: [PATCH 24/54] [SLP][NFC]Improve compile time by size analysis limit and reduction size limit. Used RecursionMaxDepth to limit number of lookups in BoUpSLP::getVectorElementSize and limited reduction width for bool reduced values. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 7f528848002b5b..961380ce4ad9f2 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -13928,26 +13928,29 @@ unsigned BoUpSLP::getVectorElementSize(Value *V) { // that feed it. The type of the loaded value may indicate a more suitable // width than V's type. We want to base the vector element size on the width // of memory operations where possible. - SmallVector, 16> Worklist; + SmallVector> Worklist; SmallPtrSet Visited; if (auto *I = dyn_cast(V)) { - Worklist.emplace_back(I, I->getParent()); + Worklist.emplace_back(I, I->getParent(), 0); Visited.insert(I); } // Traverse the expression tree in bottom-up order looking for loads. If we // encounter an instruction we don't yet handle, we give up. auto Width = 0u; + Value *FirstNonBool = nullptr; while (!Worklist.empty()) { - Instruction *I; - BasicBlock *Parent; - std::tie(I, Parent) = Worklist.pop_back_val(); + auto [I, Parent, Level] = Worklist.pop_back_val(); // We should only be looking at scalar instructions here. If the current // instruction has a vector type, skip. auto *Ty = I->getType(); if (isa(Ty)) continue; + if (Ty != Builder.getInt1Ty() && !FirstNonBool) + FirstNonBool = I; + if (Level > RecursionMaxDepth) + continue; // If the current instruction is a load, update MaxWidth to reflect the // width of the loaded value. @@ -13960,11 +13963,16 @@ unsigned BoUpSLP::getVectorElementSize(Value *V) { // user or the use is a PHI node, we add it to the worklist. else if (isa(I)) { - for (Use &U : I->operands()) + for (Use &U : I->operands()) { if (auto *J = dyn_cast(U.get())) if (Visited.insert(J).second && - (isa(I) || J->getParent() == Parent)) - Worklist.emplace_back(J, J->getParent()); + (isa(I) || J->getParent() == Parent)) { + Worklist.emplace_back(J, J->getParent(), Level + 1); + continue; + } + if (!FirstNonBool && U.get()->getType() != Builder.getInt1Ty()) + FirstNonBool = U.get(); + } } else { break; } @@ -13974,8 +13982,8 @@ unsigned BoUpSLP::getVectorElementSize(Value *V) { // gave up for some reason, just return the width of V. Otherwise, return the // maximum width we found. if (!Width) { - if (auto *CI = dyn_cast(V)) - V = CI->getOperand(0); + if (V->getType() == Builder.getInt1Ty() && FirstNonBool) + V = FirstNonBool; Width = DL->getTypeSizeInBits(V->getType()); } @@ -15838,7 +15846,9 @@ class HorizontalReduction { RegMaxNumber * llvm::bit_floor(MaxVecRegSize / EltSize); unsigned ReduxWidth = std::min( - llvm::bit_floor(NumReducedVals), std::max(RedValsMaxNumber, MaxElts)); + llvm::bit_floor(NumReducedVals), + std::clamp(MaxElts, RedValsMaxNumber, + RegMaxNumber * RedValsMaxNumber)); unsigned Start = 0; unsigned Pos = Start; // Restarts vectorization attempt with lower vector factor. From 88812819022ecff392dfc8d3f964899f63bdffdb Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Wed, 27 Mar 2024 14:45:20 -0700 Subject: [PATCH 25/54] [RISCV] Add test coverage for (add (shl Z, c1), Y, (shl Z, c2)) variants Basically, testing for interaction of shNadd matching with one step of reassociation in the add. --- llvm/test/CodeGen/RISCV/rv64zba.ll | 90 ++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/llvm/test/CodeGen/RISCV/rv64zba.ll b/llvm/test/CodeGen/RISCV/rv64zba.ll index f810f51f6bc07a..d9d83633a8537f 100644 --- a/llvm/test/CodeGen/RISCV/rv64zba.ll +++ b/llvm/test/CodeGen/RISCV/rv64zba.ll @@ -1282,6 +1282,96 @@ define zeroext i32 @sext_ashr_zext_i8(i8 %a) nounwind { ret i32 %1 } +define i64 @sh6_sh3_add1(i64 noundef %x, i64 noundef %y, i64 noundef %z) { +; RV64I-LABEL: sh6_sh3_add1: +; RV64I: # %bb.0: # %entry +; RV64I-NEXT: slli a2, a2, 3 +; RV64I-NEXT: slli a1, a1, 6 +; RV64I-NEXT: add a1, a1, a2 +; RV64I-NEXT: add a0, a1, a0 +; RV64I-NEXT: ret +; +; RV64ZBA-LABEL: sh6_sh3_add1: +; RV64ZBA: # %bb.0: # %entry +; RV64ZBA-NEXT: sh3add a1, a1, a2 +; RV64ZBA-NEXT: sh3add a0, a1, a0 +; RV64ZBA-NEXT: ret +entry: + %shl = shl i64 %z, 3 + %shl1 = shl i64 %y, 6 + %add = add nsw i64 %shl1, %shl + %add2 = add nsw i64 %add, %x + ret i64 %add2 +} + +define i64 @sh6_sh3_add2(i64 noundef %x, i64 noundef %y, i64 noundef %z) { +; RV64I-LABEL: sh6_sh3_add2: +; RV64I: # %bb.0: # %entry +; RV64I-NEXT: slli a2, a2, 3 +; RV64I-NEXT: slli a1, a1, 6 +; RV64I-NEXT: add a0, a1, a0 +; RV64I-NEXT: add a0, a0, a2 +; RV64I-NEXT: ret +; +; RV64ZBA-LABEL: sh6_sh3_add2: +; RV64ZBA: # %bb.0: # %entry +; RV64ZBA-NEXT: slli a1, a1, 6 +; RV64ZBA-NEXT: add a0, a1, a0 +; RV64ZBA-NEXT: sh3add a0, a2, a0 +; RV64ZBA-NEXT: ret +entry: + %shl = shl i64 %z, 3 + %shl1 = shl i64 %y, 6 + %add = add nsw i64 %shl1, %x + %add2 = add nsw i64 %add, %shl + ret i64 %add2 +} + +define i64 @sh6_sh3_add3(i64 noundef %x, i64 noundef %y, i64 noundef %z) { +; RV64I-LABEL: sh6_sh3_add3: +; RV64I: # %bb.0: # %entry +; RV64I-NEXT: slli a2, a2, 3 +; RV64I-NEXT: slli a1, a1, 6 +; RV64I-NEXT: add a1, a1, a2 +; RV64I-NEXT: add a0, a0, a1 +; RV64I-NEXT: ret +; +; RV64ZBA-LABEL: sh6_sh3_add3: +; RV64ZBA: # %bb.0: # %entry +; RV64ZBA-NEXT: sh3add a1, a1, a2 +; RV64ZBA-NEXT: sh3add a0, a1, a0 +; RV64ZBA-NEXT: ret +entry: + %shl = shl i64 %z, 3 + %shl1 = shl i64 %y, 6 + %add = add nsw i64 %shl1, %shl + %add2 = add nsw i64 %x, %add + ret i64 %add2 +} + +define i64 @sh6_sh3_add4(i64 noundef %x, i64 noundef %y, i64 noundef %z) { +; RV64I-LABEL: sh6_sh3_add4: +; RV64I: # %bb.0: # %entry +; RV64I-NEXT: slli a2, a2, 3 +; RV64I-NEXT: slli a1, a1, 6 +; RV64I-NEXT: add a0, a0, a2 +; RV64I-NEXT: add a0, a0, a1 +; RV64I-NEXT: ret +; +; RV64ZBA-LABEL: sh6_sh3_add4: +; RV64ZBA: # %bb.0: # %entry +; RV64ZBA-NEXT: slli a1, a1, 6 +; RV64ZBA-NEXT: sh3add a0, a2, a0 +; RV64ZBA-NEXT: add a0, a0, a1 +; RV64ZBA-NEXT: ret +entry: + %shl = shl i64 %z, 3 + %shl1 = shl i64 %y, 6 + %add = add nsw i64 %x, %shl + %add2 = add nsw i64 %add, %shl1 + ret i64 %add2 +} + ; Make sure we use sext.h+slli+srli for Zba+Zbb. ; FIXME: The RV64I and Zba only cases can be done with only 3 shifts. define zeroext i32 @sext_ashr_zext_i16(i16 %a) nounwind { From d5f06342a3007f414c783ba42cb49453a9cee835 Mon Sep 17 00:00:00 2001 From: Michael Jones Date: Wed, 27 Mar 2024 15:09:57 -0700 Subject: [PATCH 26/54] [libc] add flag for FP_*LOGB0/NAN values (#86723) These values vary by system, this flag allows you to toggle their value. --- libc/include/llvm-libc-macros/math-macros.h | 9 +++++++-- utils/bazel/llvm-project-overlay/libc/BUILD.bazel | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/libc/include/llvm-libc-macros/math-macros.h b/libc/include/llvm-libc-macros/math-macros.h index 03c7a823e6e96b..1497e32044e975 100644 --- a/libc/include/llvm-libc-macros/math-macros.h +++ b/libc/include/llvm-libc-macros/math-macros.h @@ -31,10 +31,15 @@ #define NAN __builtin_nanf("") #define FP_ILOGB0 (-INT_MAX - 1) -#define FP_ILOGBNAN INT_MAX - #define FP_LLOGB0 (-LONG_MAX - 1) + +#ifdef __FP_LOGBNAN_MIN +#define FP_ILOGBNAN (-INT_MAX - 1) +#define FP_LLOGBNAN (-LONG_MAX - 1) +#else +#define FP_ILOGBNAN INT_MAX #define FP_LLOGBNAN LONG_MAX +#endif #ifdef __FAST_MATH__ #define math_errhandling 0 diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index eb0afbb6dd6ffe..acdf9349fd5868 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -68,6 +68,7 @@ libc_support_library( name = "llvm_libc_macros_math_macros", hdrs = ["include/llvm-libc-macros/math-macros.h"], deps = [":llvm_libc_macros_limits_macros"], + defines = ["__FP_LOGBNAN_MIN"], ) libc_support_library( From 8a75faf4717b8258b59cf9fbb4cbd2f189114d3f Mon Sep 17 00:00:00 2001 From: smanna12 Date: Wed, 27 Mar 2024 17:12:58 -0500 Subject: [PATCH 27/54] [NFC][CLANG] Fix null pointer dereferences (#86760) This patch replaces getAs<> with castAs<> to resolve potential static analyzer bugs for 1. Dereferencing Proto1->param_type_begin(), which is known to be nullptr 2. Dereferencing Proto2->param_type_begin(), which is known to be nullptr 3. Dereferencing a pointer issue with nullptr Proto1 when calling param_type_end() 4. Dereferencing a pointer issue with nullptr Proto2 when calling param_type_end() in clang::Sema::getMoreSpecializedTemplate(). --- clang/lib/Sema/SemaTemplateDeduction.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp index 97f8445bf819c8..9a55881f644254 100644 --- a/clang/lib/Sema/SemaTemplateDeduction.cpp +++ b/clang/lib/Sema/SemaTemplateDeduction.cpp @@ -5514,9 +5514,9 @@ FunctionTemplateDecl *Sema::getMoreSpecializedTemplate( QualType Obj2Ty; if (TPOC == TPOC_Call) { const FunctionProtoType *Proto1 = - FD1->getType()->getAs(); + FD1->getType()->castAs(); const FunctionProtoType *Proto2 = - FD2->getType()->getAs(); + FD2->getType()->castAs(); // - In the context of a function call, the function parameter types are // used. From 4c2f68840e984b0f111779c46845ac00e3a7547d Mon Sep 17 00:00:00 2001 From: smanna12 Date: Wed, 27 Mar 2024 17:17:27 -0500 Subject: [PATCH 28/54] [CLANG] Fix potential integer overflow value in getRVVTypeSize() (#86810) In getRVVTypeSize(clang::ASTContext &, clang::BuiltinType const *) potential integer overflow occurs on expression VScale->first * MinElts with type unsigned int (32 bits, unsigned) is evaluated using 32-bit arithmetic, and then used in a context that expects an expression of type uint64_t (64 bits, unsigned). To avoid integer overflow, this patch changes the types of variables MinElts and EltSize to uint64_t instead of the cast. The change matches what was originally done in https://github.com/llvm/llvm-project/commit/7372c0d46d2185017c509eb30910b102b4f9cdaa. Looks like the revert happened in https://github.com/llvm/llvm-project/commit/c92ad411f2f94d8521cd18abcb37285f9a390ecb --- clang/lib/AST/ASTContext.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp index 20a5ecc99e44a7..c90fafb6f653d0 100644 --- a/clang/lib/AST/ASTContext.cpp +++ b/clang/lib/AST/ASTContext.cpp @@ -9600,11 +9600,11 @@ static uint64_t getRVVTypeSize(ASTContext &Context, const BuiltinType *Ty) { ASTContext::BuiltinVectorTypeInfo Info = Context.getBuiltinVectorTypeInfo(Ty); - unsigned EltSize = Context.getTypeSize(Info.ElementType); + uint64_t EltSize = Context.getTypeSize(Info.ElementType); if (Info.ElementType == Context.BoolTy) EltSize = 1; - unsigned MinElts = Info.EC.getKnownMinValue(); + uint64_t MinElts = Info.EC.getKnownMinValue(); return VScale->first * MinElts * EltSize; } From d8fe2e4bb05c27520a98dc14b2354901a1d57e53 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Wed, 27 Mar 2024 15:23:49 -0700 Subject: [PATCH 29/54] [BOLT] Fix enumeration of secondary entry points Make them start with 1 instead of 0 (reserved for primary entry point). Test Plan: ``` bin/llvm-lit -a tools/bolt/test/X86/yaml-secondary-entry-discriminator.s ``` Reviewers: rafaelauler, ayermolo, maksfb, dcci Reviewed By: maksfb Pull Request: https://github.com/llvm/llvm-project/pull/86848 --- bolt/lib/Core/BinaryFunction.cpp | 6 +- .../X86/yaml-secondary-entry-discriminator.s | 71 +++++++++++++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 bolt/test/X86/yaml-secondary-entry-discriminator.s diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp index fdadef9dcd3848..c9e037c225dd41 100644 --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -3547,7 +3547,7 @@ MCSymbol *BinaryFunction::getSymbolForEntryID(uint64_t EntryID) { if (!isMultiEntry()) return nullptr; - uint64_t NumEntries = 0; + uint64_t NumEntries = 1; if (hasCFG()) { for (BinaryBasicBlock *BB : BasicBlocks) { MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(*BB); @@ -3580,7 +3580,7 @@ uint64_t BinaryFunction::getEntryIDForSymbol(const MCSymbol *Symbol) const { return 0; // Check all secondary entries available as either basic blocks or lables. - uint64_t NumEntries = 0; + uint64_t NumEntries = 1; for (const BinaryBasicBlock *BB : BasicBlocks) { MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(*BB); if (!EntrySymbol) @@ -3589,7 +3589,7 @@ uint64_t BinaryFunction::getEntryIDForSymbol(const MCSymbol *Symbol) const { return NumEntries; ++NumEntries; } - NumEntries = 0; + NumEntries = 1; for (const std::pair &KV : Labels) { MCSymbol *EntrySymbol = getSecondaryEntryPointSymbol(KV.second); if (!EntrySymbol) diff --git a/bolt/test/X86/yaml-secondary-entry-discriminator.s b/bolt/test/X86/yaml-secondary-entry-discriminator.s new file mode 100644 index 00000000000000..55dc024b325e1f --- /dev/null +++ b/bolt/test/X86/yaml-secondary-entry-discriminator.s @@ -0,0 +1,71 @@ +# This reproduces a bug with BOLT setting incorrect discriminator for +# secondary entry points in YAML profile. + +# REQUIRES: system-linux +# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o +# RUN: link_fdata %s %t.o %t.fdata +# RUN: llvm-strip --strip-unneeded %t.o +# RUN: %clang %cflags %t.o -o %t.exe -Wl,-q -nostdlib +# RUN: llvm-bolt %t.exe -o %t.out --data %t.fdata -w %t.yaml --print-profile \ +# RUN: --print-only=main | FileCheck %s --check-prefix=CHECK-CFG +# RUN: FileCheck %s -input-file %t.yaml +# CHECK: - name: main +# CHECK-NEXT: fid: 2 +# CHECK-NEXT: hash: 0xADF270D550151185 +# CHECK-NEXT: exec: 0 +# CHECK-NEXT: nblocks: 4 +# CHECK-NEXT: blocks: +# CHECK: - bid: 1 +# CHECK-NEXT: insns: 1 +# CHECK-NEXT: hash: 0x36A303CBA4360014 +# CHECK-NEXT: calls: [ { off: 0x0, fid: 1, disc: 1, cnt: 1 } ] + +# Make sure that the profile is attached correctly +# RUN: llvm-bolt %t.exe -o %t.out --data %t.yaml --print-profile \ +# RUN: --print-only=main | FileCheck %s --check-prefix=CHECK-CFG + +# CHECK-CFG: Binary Function "main" after attaching profile { +# CHECK-CFG: callq secondary_entry # Offset: [[#]] # Count: 1 +# CHECK-CFG: callq *%rax # Offset: [[#]] # CallProfile: 1 (1 misses) : +# XXX: uncomment after discriminator is set correctly for indirect calls +# COM: CHECK-CFG-NEXT: { secondary_entry: 1 (1 misses) } +# CHECK-CFG: } + +.globl func +.type func, @function +func: + .cfi_startproc + pushq %rbp + movq %rsp, %rbp +.globl secondary_entry +secondary_entry: + popq %rbp + retq + nopl (%rax) + .cfi_endproc + .size func, .-func + +.globl main +.type main, @function +main: + .cfi_startproc + pushq %rbp + movq %rsp, %rbp + subq $16, %rsp + movl $0, -4(%rbp) + testq %rax, %rax + jne Lindcall +Lcall: + call secondary_entry +# FDATA: 1 main #Lcall# 1 secondary_entry 0 1 1 +Lindcall: + callq *%rax +# FDATA: 1 main #Lindcall# 1 secondary_entry 0 1 1 + xorl %eax, %eax + addq $16, %rsp + popq %rbp + retq +# For relocations against .text + call exit + .cfi_endproc + .size main, .-main From 0a17eedf7b0c1168999e7ac0492015c64c1fbef4 Mon Sep 17 00:00:00 2001 From: Marc Auberer Date: Wed, 27 Mar 2024 23:53:25 +0100 Subject: [PATCH 30/54] [CI][NFC] Fix shellcheck warnings in CI scripts (#86877) This fixes all shellcheck warnings we have in `monolithic-linux.sh` and `monolithic-windows.sh`. All of them have to do with [SC2086](https://www.shellcheck.net/wiki/SC2086) - Double quote to prevent globbing and word splitting. --- .ci/monolithic-linux.sh | 8 ++++---- .ci/monolithic-windows.sh | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh index 9e670c447fbadd..35f1aacb4985f0 100755 --- a/.ci/monolithic-linux.sh +++ b/.ci/monolithic-linux.sh @@ -18,7 +18,7 @@ set -o pipefail MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}" BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build}" -rm -rf ${BUILD_DIR} +rm -rf "${BUILD_DIR}" ccache --zero-stats @@ -37,8 +37,8 @@ projects="${1}" targets="${2}" echo "--- cmake" -pip install -q -r ${MONOREPO_ROOT}/mlir/python/requirements.txt -cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \ +pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt +cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \ -D LLVM_ENABLE_PROJECTS="${projects}" \ -G Ninja \ -D CMAKE_BUILD_TYPE=Release \ @@ -54,4 +54,4 @@ cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \ echo "--- ninja" # Targets are not escaped as they are passed as separate arguments. -ninja -C "${BUILD_DIR}" -k 0 ${targets} +ninja -C "${BUILD_DIR}" -k 0 "${targets}" diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh index 52ba13036f9159..f84c0966704e73 100755 --- a/.ci/monolithic-windows.sh +++ b/.ci/monolithic-windows.sh @@ -19,7 +19,7 @@ set -o pipefail MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}" BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build}" -rm -rf ${BUILD_DIR} +rm -rf "${BUILD_DIR}" if [[ -n "${CLEAR_CACHE:-}" ]]; then echo "clearing sccache" @@ -37,14 +37,14 @@ projects="${1}" targets="${2}" echo "--- cmake" -pip install -q -r ${MONOREPO_ROOT}/mlir/python/requirements.txt +pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt # The CMAKE_*_LINKER_FLAGS to disable the manifest come from research # on fixing a build reliability issue on the build server, please # see https://github.com/llvm/llvm-project/pull/82393 and # https://discourse.llvm.org/t/rfc-future-of-windows-pre-commit-ci/76840/40 # for further information. -cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \ +cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \ -D LLVM_ENABLE_PROJECTS="${projects}" \ -G Ninja \ -D CMAKE_BUILD_TYPE=Release \ @@ -62,4 +62,4 @@ cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \ echo "--- ninja" # Targets are not escaped as they are passed as separate arguments. -ninja -C "${BUILD_DIR}" -k 0 ${targets} +ninja -C "${BUILD_DIR}" -k 0 "${targets}" From 14ba782a87e16e9e15460a51f50e67e2744c26d9 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 27 Mar 2024 15:56:19 -0700 Subject: [PATCH 31/54] [Clang][Sema] Allow flexible arrays in unions and alone in structs (#84428) GNU and MSVC have extensions where flexible array members (or their equivalent) can be in unions or alone in structs. This is already fully supported in Clang through the 0-sized array ("fake flexible array") extension or when C99 flexible array members have been syntactically obfuscated. Clang needs to explicitly allow these extensions directly for C99 flexible arrays, since they are common code patterns in active use by the Linux kernel (and other projects). Such projects have been using either 0-sized arrays (which is considered deprecated in favor of C99 flexible array members) or via obfuscated syntax, both of which complicate their code bases. For example, these do not error by default: ``` union one { int a; int b[0]; }; union two { int a; struct { struct { } __empty; int b[]; }; }; ``` But this does: ``` union three { int a; int b[]; }; ``` Remove the default error diagnostics for this but continue to provide warnings under Microsoft or GNU extensions checks. This will allow for a seamless transition for code bases away from 0-sized arrays without losing existing code patterns. Add explicit checking for the warnings under various constructions. Additionally fixes a CodeGen bug with flexible array members in unions in C++, which was found when adding a testcase for: ``` union { char x[]; } z = {0}; ``` which only had Sema tests originally. Fixes #84565 --- clang/docs/ReleaseNotes.rst | 3 + .../clang/Basic/DiagnosticSemaKinds.td | 5 - clang/lib/Sema/SemaDecl.cpp | 8 +- clang/lib/Sema/SemaInit.cpp | 11 +- clang/test/C/drs/dr5xx.c | 2 +- clang/test/CodeGen/flexible-array-init.c | 89 ++++++++- clang/test/CodeGen/flexible-array-init.cpp | 24 +++ clang/test/Sema/flexible-array-in-union.c | 187 +++++++++++++++++- clang/test/Sema/transparent-union.c | 4 +- 9 files changed, 303 insertions(+), 30 deletions(-) create mode 100644 clang/test/CodeGen/flexible-array-init.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 0fdd9e3fb3eee2..ccc399d36dbb54 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -304,6 +304,9 @@ Improvements to Clang's diagnostics annotated with the ``clang::always_destroy`` attribute. Fixes #GH68686, #GH86486 +- ``-Wmicrosoft``, ``-Wgnu``, or ``-pedantic`` is now required to diagnose C99 + flexible array members in a union or alone in a struct. Fixes GH#84565. + Improvements to Clang's time-trace ---------------------------------- diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index fc727cef9cd835..51af81bf1f6fc5 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -6464,9 +6464,6 @@ def ext_c99_flexible_array_member : Extension< def err_flexible_array_virtual_base : Error< "flexible array member %0 not allowed in " "%select{struct|interface|union|class|enum}1 which has a virtual base class">; -def err_flexible_array_empty_aggregate : Error< - "flexible array member %0 not allowed in otherwise empty " - "%select{struct|interface|union|class|enum}1">; def err_flexible_array_has_nontrivial_dtor : Error< "flexible array member %0 of type %1 with non-trivial destruction">; def ext_flexible_array_in_struct : Extension< @@ -6481,8 +6478,6 @@ def ext_flexible_array_empty_aggregate_ms : Extension< "flexible array member %0 in otherwise empty " "%select{struct|interface|union|class|enum}1 is a Microsoft extension">, InGroup; -def err_flexible_array_union : Error< - "flexible array member %0 in a union is not allowed">; def ext_flexible_array_union_ms : Extension< "flexible array member %0 in a union is a Microsoft extension">, InGroup; diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 8b44d24f5273aa..0bd88ece2aa544 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -19429,15 +19429,11 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl, } else if (Record->isUnion()) DiagID = getLangOpts().MicrosoftExt ? diag::ext_flexible_array_union_ms - : getLangOpts().CPlusPlus - ? diag::ext_flexible_array_union_gnu - : diag::err_flexible_array_union; + : diag::ext_flexible_array_union_gnu; else if (NumNamedMembers < 1) DiagID = getLangOpts().MicrosoftExt ? diag::ext_flexible_array_empty_aggregate_ms - : getLangOpts().CPlusPlus - ? diag::ext_flexible_array_empty_aggregate_gnu - : diag::err_flexible_array_empty_aggregate; + : diag::ext_flexible_array_empty_aggregate_gnu; if (DiagID) Diag(FD->getLocation(), DiagID) diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp index 2b4805d62d07d0..dce225a7204da8 100644 --- a/clang/lib/Sema/SemaInit.cpp +++ b/clang/lib/Sema/SemaInit.cpp @@ -2329,11 +2329,11 @@ void InitListChecker::CheckStructUnionTypes( break; } - // We've already initialized a member of a union. We're done. + // We've already initialized a member of a union. We can stop entirely. if (InitializedSomething && RD->isUnion()) - break; + return; - // If we've hit the flexible array member at the end, we're done. + // Stop if we've hit a flexible array member. if (Field->getType()->isIncompleteArrayType()) break; @@ -2456,6 +2456,11 @@ void InitListChecker::CheckStructUnionTypes( else CheckImplicitInitList(MemberEntity, IList, Field->getType(), Index, StructuredList, StructuredIndex); + + if (RD->isUnion() && StructuredList) { + // Initialize the first field within the union. + StructuredList->setInitializedFieldInUnion(*Field); + } } /// Expand a field designator that refers to a member of an diff --git a/clang/test/C/drs/dr5xx.c b/clang/test/C/drs/dr5xx.c index 68bcef78baccd7..13464f78b6a654 100644 --- a/clang/test/C/drs/dr5xx.c +++ b/clang/test/C/drs/dr5xx.c @@ -29,7 +29,7 @@ void dr502(void) { */ struct t { int i; - struct { int a[]; }; /* expected-error {{flexible array member 'a' not allowed in otherwise empty struct}} + struct { int a[]; }; /* expected-warning {{flexible array member 'a' in otherwise empty struct is a GNU extension}} c89only-warning {{flexible array members are a C99 feature}} expected-warning {{'' may not be nested in a struct due to flexible array member}} */ diff --git a/clang/test/CodeGen/flexible-array-init.c b/clang/test/CodeGen/flexible-array-init.c index bae926da5feb07..15a30c15ac966e 100644 --- a/clang/test/CodeGen/flexible-array-init.c +++ b/clang/test/CodeGen/flexible-array-init.c @@ -3,9 +3,15 @@ struct { int x; int y[]; } a = { 1, 7, 11 }; // CHECK: @a ={{.*}} global { i32, [2 x i32] } { i32 1, [2 x i32] [i32 7, i32 11] } +struct { int y[]; } a1 = { 8, 12 }; +// CHECK: @a1 ={{.*}} global { [2 x i32] } { [2 x i32] [i32 8, i32 12] } + struct { int x; int y[]; } b = { 1, { 13, 15 } }; // CHECK: @b ={{.*}} global { i32, [2 x i32] } { i32 1, [2 x i32] [i32 13, i32 15] } +struct { int y[]; } b1 = { { 14, 16 } }; +// CHECK: @b1 ={{.*}} global { [2 x i32] } { [2 x i32] [i32 14, i32 16] } + // sizeof(c) == 8, so this global should be at least 8 bytes. struct { int x; char c; char y[]; } c = { 1, 2, { 13, 15 } }; // CHECK: @c ={{.*}} global { i32, i8, [2 x i8] } { i32 1, i8 2, [2 x i8] c"\0D\0F" } @@ -21,10 +27,79 @@ struct __attribute((packed, aligned(4))) { char a; int x; char z[]; } e = { 1, 2 struct { int x; char y[]; } f = { 1, { 13, 15 } }; // CHECK: @f ={{.*}} global <{ i32, [2 x i8] }> <{ i32 1, [2 x i8] c"\0D\0F" }> -union { - struct { - int a; - char b[]; - } x; -} in_union = {}; -// CHECK: @in_union ={{.*}} global %union.anon zeroinitializer +struct __attribute((packed)) { short a; char z[]; } g = { 2, { 11, 13, 15 } }; +// CHECK: @g ={{.*}} <{ i16, [3 x i8] }> <{ i16 2, [3 x i8] c"\0B\0D\0F" }>, + +// Last member is the potential flexible array, unnamed initializer skips it. +struct { int a; union { int b; short x; }; int c; int d; } h = {1, 2, {}, 3}; +// CHECK: @h = global %struct.anon{{.*}} { i32 1, %union.anon{{.*}} { i32 2 }, i32 0, i32 3 } +struct { int a; union { int b; short x[0]; }; int c; int d; } h0 = {1, 2, {}, 3}; +// CHECK: @h0 = global %struct.anon{{.*}} { i32 1, %union.anon{{.*}} { i32 2 }, i32 0, i32 3 } +struct { int a; union { int b; short x[1]; }; int c; int d; } h1 = {1, 2, {}, 3}; +// CHECK: @h1 = global %struct.anon{{.*}} { i32 1, %union.anon{{.*}} { i32 2 }, i32 0, i32 3 } +struct { + int a; + union { + int b; + struct { + struct { } __ununsed; + short x[]; + }; + }; + int c; + int d; +} hiding = {1, 2, {}, 3}; +// CHECK: @hiding = global %struct.anon{{.*}} { i32 1, %union.anon{{.*}} { i32 2 }, i32 0, i32 3 } +struct { int a; union { int b; short x[]; }; int c; int d; } hf = {1, 2, {}, 3}; +// CHECK: @hf = global %struct.anon{{.*}} { i32 1, %union.anon{{.*}} { i32 2 }, i32 0, i32 3 } + +// First member is the potential flexible array, initialization requires braces. +struct { int a; union { short x; int b; }; int c; int d; } i = {1, 2, {}, 3}; +// CHECK: @i = global { i32, { i16, [2 x i8] }, i32, i32 } { i32 1, { i16, [2 x i8] } { i16 2, [2 x i8] undef }, i32 0, i32 3 } +struct { int a; union { short x[0]; int b; }; int c; int d; } i0 = {1, {}, 2, 3}; +// CHECK: @i0 = global { i32, { [0 x i16], [4 x i8] }, i32, i32 } { i32 1, { [0 x i16], [4 x i8] } { [0 x i16] zeroinitializer, [4 x i8] undef }, i32 2, i32 3 } +struct { int a; union { short x[1]; int b; }; int c; int d; } i1 = {1, {2}, {}, 3}; +// CHECK: @i1 = global { i32, { [1 x i16], [2 x i8] }, i32, i32 } { i32 1, { [1 x i16], [2 x i8] } { [1 x i16] [i16 2], [2 x i8] undef }, i32 0, i32 3 } +struct { int a; union { short x[]; int b; }; int c; int d; } i_f = {4, {}, {}, 6}; +// CHECK: @i_f = global { i32, { [0 x i16], [4 x i8] }, i32, i32 } { i32 4, { [0 x i16], [4 x i8] } { [0 x i16] zeroinitializer, [4 x i8] undef }, i32 0, i32 6 } + +// Named initializers; order doesn't matter. +struct { int a; union { int b; short x; }; int c; int d; } hn = {.a = 1, .x = 2, .c = 3}; +// CHECK: @hn = global { i32, { i16, [2 x i8] }, i32, i32 } { i32 1, { i16, [2 x i8] } { i16 2, [2 x i8] undef }, i32 3, i32 0 } +struct { int a; union { int b; short x[0]; }; int c; int d; } hn0 = {.a = 1, .x = {2}, .c = 3}; +// CHECK: @hn0 = global { i32, { [0 x i16], [4 x i8] }, i32, i32 } { i32 1, { [0 x i16], [4 x i8] } { [0 x i16] zeroinitializer, [4 x i8] undef }, i32 3, i32 0 } +struct { int a; union { int b; short x[1]; }; int c; int d; } hn1 = {.a = 1, .x = {2}, .c = 3}; +// CHECK: @hn1 = global { i32, { [1 x i16], [2 x i8] }, i32, i32 } { i32 1, { [1 x i16], [2 x i8] } { [1 x i16] [i16 2], [2 x i8] undef }, i32 3, i32 0 } + +struct { char a[]; } empty_struct = {}; +// CHECK: @empty_struct ={{.*}} global %struct.anon{{.*}} zeroinitializer, align 1 + +struct { char a[]; } empty_struct0 = {0}; +// CHECK: @empty_struct0 = global { [1 x i8] } zeroinitializer, align 1 + +union { struct { int a; char b[]; }; } struct_in_union = {}; +// CHECK: @struct_in_union = global %union.anon{{.*}} zeroinitializer, align 4 + +union { struct { int a; char b[]; }; } struct_in_union0 = {0}; +// CHECK: @struct_in_union0 = global %union.anon{{.*}} zeroinitializer, align 4 + +union { int a; char b[]; } trailing_in_union = {}; +// CHECK: @trailing_in_union = global %union.anon{{.*}} zeroinitializer, align 4 + +union { int a; char b[]; } trailing_in_union0 = {0}; +// CHECK: @trailing_in_union0 = global %union.anon{{.*}} zeroinitializer, align 4 + +union { char a[]; } only_in_union = {}; +// CHECK: @only_in_union = global %union.anon{{.*}} zeroinitializer, align 1 + +union { char a[]; } only_in_union0 = {0}; +// CHECK: @only_in_union0 = global { [1 x i8] } zeroinitializer, align 1 + +union { char a[]; int b; } first_in_union = {}; +// CHECK: @first_in_union = global { [0 x i8], [4 x i8] } { [0 x i8] zeroinitializer, [4 x i8] undef }, align 4 + +union { char a[]; int b; } first_in_union0 = {0}; +// CHECK: @first_in_union0 = global { [1 x i8], [3 x i8] } { [1 x i8] zeroinitializer, [3 x i8] undef }, align 4 + +union { char a[]; int b; } first_in_union123 = { {1, 2, 3} }; +// CHECK: @first_in_union123 = global { [3 x i8], i8 } { [3 x i8] c"\01\02\03", i8 undef }, align 4 diff --git a/clang/test/CodeGen/flexible-array-init.cpp b/clang/test/CodeGen/flexible-array-init.cpp new file mode 100644 index 00000000000000..d067a614e1afe5 --- /dev/null +++ b/clang/test/CodeGen/flexible-array-init.cpp @@ -0,0 +1,24 @@ +// RUN: %clang_cc1 -triple i386-unknown-unknown -x c++ -emit-llvm -o - %s | FileCheck %s + +union _u { char a[]; } u = {}; +union _u0 { char a[]; } u0 = {0}; + +// CHECK: %union._u = type { [0 x i8] } + +// CHECK: @u = global %union._u zeroinitializer, align 1 +// CHECK: @u0 = global { [1 x i8] } zeroinitializer, align 1 + +union { char a[]; } z = {}; +// CHECK: @z = internal global %union.{{.*}} zeroinitializer, align 1 +union { char a[]; } z0 = {0}; +// CHECK: @z0 = internal global { [1 x i8] } zeroinitializer, align 1 + +/* C++ requires global anonymous unions have static storage, so we have to + reference them to keep them in the IR output. */ +char keep(int pick) +{ + if (pick) + return z.a[0]; + else + return z0.a[0]; +} diff --git a/clang/test/Sema/flexible-array-in-union.c b/clang/test/Sema/flexible-array-in-union.c index 5fabfbe0b1eaab..dd5e8069665fea 100644 --- a/clang/test/Sema/flexible-array-in-union.c +++ b/clang/test/Sema/flexible-array-in-union.c @@ -1,13 +1,188 @@ -// RUN: %clang_cc1 %s -verify=c -fsyntax-only -// RUN: %clang_cc1 %s -verify -fsyntax-only -x c++ -// RUN: %clang_cc1 %s -verify -fsyntax-only -fms-compatibility -// RUN: %clang_cc1 %s -verify -fsyntax-only -fms-compatibility -x c++ +// RUN: %clang_cc1 %s -verify=stock,c -fsyntax-only +// RUN: %clang_cc1 %s -verify=stock,cpp -fsyntax-only -x c++ +// RUN: %clang_cc1 %s -verify=stock,cpp -fsyntax-only -fms-compatibility -x c++ +// RUN: %clang_cc1 %s -verify=stock,c,gnu -fsyntax-only -Wgnu-flexible-array-union-member -Wgnu-empty-struct +// RUN: %clang_cc1 %s -verify=stock,c,microsoft -fsyntax-only -fms-compatibility -Wmicrosoft // The test checks that an attempt to initialize union with flexible array // member with an initializer list doesn't crash clang. -union { char x[]; } r = {0}; // c-error {{flexible array member 'x' in a union is not allowed}} +union { char x[]; } r = {0}; /* gnu-warning {{flexible array member 'x' in a union is a GNU extension}} + microsoft-warning {{flexible array member 'x' in a union is a Microsoft extension}} + */ +struct _name1 { + int a; + union { + int b; + char x[]; /* gnu-warning {{flexible array member 'x' in a union is a GNU extension}} + microsoft-warning {{flexible array member 'x' in a union is a Microsoft extension}} + */ + }; +} name1 = { + 10, + 42, /* initializes "b" */ +}; -// expected-no-diagnostics +struct _name1i { + int a; + union { + int b; + char x[]; /* gnu-warning {{flexible array member 'x' in a union is a GNU extension}} + microsoft-warning {{flexible array member 'x' in a union is a Microsoft extension}} + */ + }; +} name1i = { + .a = 10, + .b = 42, +}; + +/* Initialization of flexible array in a union is never allowed. */ +struct _name2 { + int a; + union { + int b; + char x[]; /* gnu-warning {{flexible array member 'x' in a union is a GNU extension}} + microsoft-warning {{flexible array member 'x' in a union is a Microsoft extension}} + */ + }; +} name2 = { + 12, + 13, + { 'c' }, /* c-warning {{excess elements in struct initializer}} + cpp-error {{excess elements in struct initializer}} + */ +}; + +/* Initialization of flexible array in a union is never allowed. */ +struct _name2i { + int a; + union { + int b; + char x[]; /* gnu-warning {{flexible array member 'x' in a union is a GNU extension}} + microsoft-warning {{flexible array member 'x' in a union is a Microsoft extension}} + stock-note {{initialized flexible array member 'x' is here}} + */ + }; +} name2i = { + .a = 12, + .b = 13, /* stock-note {{previous initialization is here}} */ + .x = { 'c' }, /* stock-error {{initialization of flexible array member is not allowed}} + c-warning {{initializer overrides prior initialization of this subobject}} + cpp-error {{initializer partially overrides prior initialization of this subobject}} + */ +}; + +/* Flexible array initialization always allowed when not in a union, + and when struct has another member. + */ +struct _okay { + int a; + char x[]; +} okay = { + 22, + { 'x', 'y', 'z' }, +}; + +struct _okayi { + int a; + char x[]; +} okayi = { + .a = 22, + .x = { 'x', 'y', 'z' }, +}; + +struct _okay0 { + int a; + char x[]; +} okay0 = { }; + +struct _flex_extension { + char x[]; /* gnu-warning {{flexible array member 'x' in otherwise empty struct is a GNU extension}} + microsoft-warning {{flexible array member 'x' in otherwise empty struct is a Microsoft extension}} + */ +} flex_extension = { + { 'x', 'y', 'z' }, +}; + +struct _flex_extensioni { + char x[]; /* gnu-warning {{flexible array member 'x' in otherwise empty struct is a GNU extension}} + microsoft-warning {{flexible array member 'x' in otherwise empty struct is a Microsoft extension}} + */ +} flex_extensioni = { + .x = { 'x', 'y', 'z' }, +}; + +struct already_hidden { + int a; + union { + int b; + struct { + struct { } __empty; // gnu-warning {{empty struct is a GNU extension}} + char x[]; + }; + }; +}; +struct still_zero_sized { + struct { } __unused; // gnu-warning {{empty struct is a GNU extension}} + int x[]; +}; + +struct warn1 { + int a; + union { + int b; + char x[]; /* gnu-warning {{flexible array member 'x' in a union is a GNU extension}} + microsoft-warning {{flexible array member 'x' in a union is a Microsoft extension}} + */ + }; +}; + +struct warn2 { + int x[]; /* gnu-warning {{flexible array member 'x' in otherwise empty struct is a GNU extension}} + microsoft-warning {{flexible array member 'x' in otherwise empty struct is a Microsoft extension}} + */ +}; + +union warn3 { + short x[]; /* gnu-warning {{flexible array member 'x' in a union is a GNU extension}} + microsoft-warning {{flexible array member 'x' in a union is a Microsoft extension}} + */ +}; + +struct quiet1 { + int a; + short x[]; +}; + +struct _not_at_end { + union { short x[]; }; /* stock-warning-re {{field '' with variable sized type '{{.*}}' not at the end of a struct or class is a GNU extension}} + gnu-warning {{flexible array member 'x' in a union is a GNU extension}} + microsoft-warning {{flexible array member 'x' in a union is a Microsoft extension}} + */ + int y; +} not_at_end = {{}, 3}; + +struct _not_at_end_s { + struct { int a; short x[]; }; /* stock-warning-re {{field '' with variable sized type '{{.*}}' not at the end of a struct or class is a GNU extension}} */ + int y; +} not_at_end_s = {{}, 3}; + +struct { + int a; + union { /* stock-warning-re {{field '' with variable sized type '{{.*}}' not at the end of a struct or class is a GNU extension}} */ + short x[]; /* stock-note {{initialized flexible array member 'x' is here}} + gnu-warning {{flexible array member 'x' in a union is a GNU extension}} + microsoft-warning {{flexible array member 'x' in a union is a Microsoft extension}} + */ + int b; + }; + int c; + int d; +} i_f = { 4, + {5}, /* stock-error {{initialization of flexible array member is not allowed}} */ + {}, + 6}; + +// expected-no-diagnostics diff --git a/clang/test/Sema/transparent-union.c b/clang/test/Sema/transparent-union.c index c134a7a9b1c4d0..f02c2298b51ce1 100644 --- a/clang/test/Sema/transparent-union.c +++ b/clang/test/Sema/transparent-union.c @@ -1,4 +1,4 @@ -// RUN: %clang_cc1 -fsyntax-only -verify %s +// RUN: %clang_cc1 -fsyntax-only -verify -Wgnu-flexible-array-union-member %s typedef union { int *ip; float *fp; @@ -131,7 +131,7 @@ union pr15134v2 { union pr30520v { void b; } __attribute__((transparent_union)); // expected-error {{field has incomplete type 'void'}} -union pr30520a { int b[]; } __attribute__((transparent_union)); // expected-error {{flexible array member 'b' in a union is not allowed}} +union pr30520a { int b[]; } __attribute__((transparent_union)); // expected-warning {{flexible array member 'b' in a union is a GNU extension}} // expected-note@+1 2 {{forward declaration of 'struct stb'}} union pr30520s { struct stb b; } __attribute__((transparent_union)); // expected-error {{field has incomplete type 'struct stb'}} From b76fd1eddbafaa02a533245d574048c5ad9e38fa Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Wed, 27 Mar 2024 16:37:00 -0700 Subject: [PATCH 32/54] [lldb] Avoid deadlock by unlocking before invoking callbacks (#86888) Avoid deadlocks in the Alarm class by releasing the lock before invoking callbacks. This deadlock manifested itself in the ProgressManager: 1. On the main thread, the ProgressManager acquires its lock in ProgressManager::Decrement and calls Alarm::Create. 2. On the main thread, the Alarm acquires its lock in Alarm::Create. 3. On the alarm thread, the Alarm acquires its lock after waiting on the condition variable and calls ProgressManager::Expire. 4. On the alarm thread, the ProgressManager acquires its lock in ProgressManager::Expire. Note how the two threads are acquiring the locks in different orders. Deadlocks can be avoided by always acquiring locks in the same order, but since the two mutexes here are private implementation details, belong to different classes, that's not straightforward. Luckily, we don't need to have the Alarm mutex locked when invoking the callbacks. That exactly how this patch solves the issue. --- lldb/source/Host/common/Alarm.cpp | 84 +++++++++++++++++-------------- 1 file changed, 45 insertions(+), 39 deletions(-) diff --git a/lldb/source/Host/common/Alarm.cpp b/lldb/source/Host/common/Alarm.cpp index 245cdc7ae5c2da..afc770d20d7b1e 100644 --- a/lldb/source/Host/common/Alarm.cpp +++ b/lldb/source/Host/common/Alarm.cpp @@ -154,54 +154,60 @@ lldb::thread_result_t Alarm::AlarmThread() { // // Below we only deal with the timeout expiring and fall through for dealing // with the rest. - std::unique_lock alarm_lock(m_alarm_mutex); - if (next_alarm) { - if (!m_alarm_cv.wait_until(alarm_lock, *next_alarm, predicate)) { - // The timeout for the next alarm expired. - - // Clear the next timeout to signal that we need to recompute the next - // timeout. - next_alarm.reset(); - - // Iterate over all the callbacks. Call the ones that have expired - // and remove them from the list. - const TimePoint now = std::chrono::system_clock::now(); - auto it = m_entries.begin(); - while (it != m_entries.end()) { - if (it->expiration <= now) { - it->callback(); - it = m_entries.erase(it); - } else { - it++; + llvm::SmallVector callbacks; + { + std::unique_lock alarm_lock(m_alarm_mutex); + if (next_alarm) { + if (!m_alarm_cv.wait_until(alarm_lock, *next_alarm, predicate)) { + // The timeout for the next alarm expired. + + // Clear the next timeout to signal that we need to recompute the next + // timeout. + next_alarm.reset(); + + // Iterate over all the callbacks. Call the ones that have expired + // and remove them from the list. + const TimePoint now = std::chrono::system_clock::now(); + auto it = m_entries.begin(); + while (it != m_entries.end()) { + if (it->expiration <= now) { + callbacks.emplace_back(std::move(it->callback)); + it = m_entries.erase(it); + } else { + it++; + } } } + } else { + m_alarm_cv.wait(alarm_lock, predicate); } - } else { - m_alarm_cv.wait(alarm_lock, predicate); - } - // Fall through after waiting on the condition variable. At this point - // either the predicate is true or we woke up because an alarm expired. + // Fall through after waiting on the condition variable. At this point + // either the predicate is true or we woke up because an alarm expired. - // The alarm thread is shutting down. - if (m_exit) { - exit = true; - if (m_run_callbacks_on_exit) { - for (Entry &entry : m_entries) - entry.callback(); + // The alarm thread is shutting down. + if (m_exit) { + exit = true; + if (m_run_callbacks_on_exit) { + for (Entry &entry : m_entries) + callbacks.emplace_back(std::move(entry.callback)); + } } - continue; - } - // A new alarm was added or an alarm expired. Either way we need to - // recompute when this thread should wake up for the next alarm. - if (m_recompute_next_alarm || !next_alarm) { - for (Entry &entry : m_entries) { - if (!next_alarm || entry.expiration < *next_alarm) - next_alarm = entry.expiration; + // A new alarm was added or an alarm expired. Either way we need to + // recompute when this thread should wake up for the next alarm. + if (m_recompute_next_alarm || !next_alarm) { + for (Entry &entry : m_entries) { + if (!next_alarm || entry.expiration < *next_alarm) + next_alarm = entry.expiration; + } + m_recompute_next_alarm = false; } - m_recompute_next_alarm = false; } + + // Outside the lock, call the callbacks. + for (Callback &callback : callbacks) + callback(); } return {}; } From 385e3e26c11fac9373bf7ba431b90b6f42682c84 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Wed, 27 Mar 2024 16:40:38 -0700 Subject: [PATCH 33/54] [BOLT] Set EntryDiscriminator in YAML profile for indirect calls Indirect call handling missed setting an `EntryDiscriminator` while it's set for direct calls and tail calls. Improve YAML profile accuracy by unifying the destination setting between direct and indirect calls into `setCSIDestination` method. Depends on: https://github.com/llvm/llvm-project/pull/86848 Test Plan: Updated bolt/test/X86/yaml-secondary-entry-discriminator.s Reviewers: ayermolo, maksfb, rafaelauler Reviewed By: maksfb Pull Request: https://github.com/llvm/llvm-project/pull/82128 --- bolt/lib/Profile/YAMLProfileWriter.cpp | 40 +++++++++++-------- .../X86/yaml-secondary-entry-discriminator.s | 9 +++-- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/bolt/lib/Profile/YAMLProfileWriter.cpp b/bolt/lib/Profile/YAMLProfileWriter.cpp index 6fcc4a956fa1a1..0f082086c1fc24 100644 --- a/bolt/lib/Profile/YAMLProfileWriter.cpp +++ b/bolt/lib/Profile/YAMLProfileWriter.cpp @@ -25,6 +25,25 @@ extern llvm::cl::opt ProfileUseDFS; namespace llvm { namespace bolt { +/// Set CallSiteInfo destination fields from \p Symbol and return a target +/// BinaryFunction for that symbol. +static const BinaryFunction *setCSIDestination(const BinaryContext &BC, + yaml::bolt::CallSiteInfo &CSI, + const MCSymbol *Symbol) { + CSI.DestId = 0; // designated for unknown functions + CSI.EntryDiscriminator = 0; + if (Symbol) { + uint64_t EntryID = 0; + if (const BinaryFunction *const Callee = + BC.getFunctionForSymbol(Symbol, &EntryID)) { + CSI.DestId = Callee->getFunctionNumber(); + CSI.EntryDiscriminator = EntryID; + return Callee; + } + } + return nullptr; +} + yaml::bolt::BinaryFunctionProfile YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS) { yaml::bolt::BinaryFunctionProfile YamlBF; @@ -79,31 +98,20 @@ YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS) { continue; for (const IndirectCallProfile &CSP : ICSP.get()) { StringRef TargetName = ""; - CSI.DestId = 0; // designated for unknown functions - CSI.EntryDiscriminator = 0; - if (CSP.Symbol) { - const BinaryFunction *Callee = BC.getFunctionForSymbol(CSP.Symbol); - if (Callee) { - CSI.DestId = Callee->getFunctionNumber(); - TargetName = Callee->getOneName(); - } - } + const BinaryFunction *Callee = setCSIDestination(BC, CSI, CSP.Symbol); + if (Callee) + TargetName = Callee->getOneName(); CSI.Count = CSP.Count; CSI.Mispreds = CSP.Mispreds; CSTargets.emplace_back(TargetName, CSI); } } else { // direct call or a tail call - uint64_t EntryID = 0; - CSI.DestId = 0; StringRef TargetName = ""; const MCSymbol *CalleeSymbol = BC.MIB->getTargetSymbol(Instr); const BinaryFunction *const Callee = - BC.getFunctionForSymbol(CalleeSymbol, &EntryID); - if (Callee) { - CSI.DestId = Callee->getFunctionNumber(); - CSI.EntryDiscriminator = EntryID; + setCSIDestination(BC, CSI, CalleeSymbol); + if (Callee) TargetName = Callee->getOneName(); - } auto getAnnotationWithDefault = [&](const MCInst &Inst, StringRef Ann) { return BC.MIB->getAnnotationWithDefault(Instr, Ann, 0ull); diff --git a/bolt/test/X86/yaml-secondary-entry-discriminator.s b/bolt/test/X86/yaml-secondary-entry-discriminator.s index 55dc024b325e1f..43c2e2a7f05549 100644 --- a/bolt/test/X86/yaml-secondary-entry-discriminator.s +++ b/bolt/test/X86/yaml-secondary-entry-discriminator.s @@ -19,6 +19,10 @@ # CHECK-NEXT: insns: 1 # CHECK-NEXT: hash: 0x36A303CBA4360014 # CHECK-NEXT: calls: [ { off: 0x0, fid: 1, disc: 1, cnt: 1 } ] +# CHECK: - bid: 2 +# CHECK-NEXT: insns: 5 +# CHECK-NEXT: hash: 0x8B2F5747CD0019 +# CHECK-NEXT: calls: [ { off: 0x0, fid: 1, disc: 1, cnt: 1, mis: 1 } ] # Make sure that the profile is attached correctly # RUN: llvm-bolt %t.exe -o %t.out --data %t.yaml --print-profile \ @@ -27,13 +31,12 @@ # CHECK-CFG: Binary Function "main" after attaching profile { # CHECK-CFG: callq secondary_entry # Offset: [[#]] # Count: 1 # CHECK-CFG: callq *%rax # Offset: [[#]] # CallProfile: 1 (1 misses) : -# XXX: uncomment after discriminator is set correctly for indirect calls -# COM: CHECK-CFG-NEXT: { secondary_entry: 1 (1 misses) } -# CHECK-CFG: } +# CHECK-CFG-NEXT: { secondary_entry: 1 (1 misses) } .globl func .type func, @function func: +# FDATA: 0 [unknown] 0 1 func 0 1 0 .cfi_startproc pushq %rbp movq %rsp, %rbp From 19185d5a72565fce34521cb59d6c87232a86e0d4 Mon Sep 17 00:00:00 2001 From: Ye Luo Date: Wed, 27 Mar 2024 18:40:57 -0500 Subject: [PATCH 34/54] [Libomptarget] Make dynamic loading libffi more verbose. (#86891) --- .../plugins-nextgen/host/dynamic_ffi/ffi.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/openmp/libomptarget/plugins-nextgen/host/dynamic_ffi/ffi.cpp b/openmp/libomptarget/plugins-nextgen/host/dynamic_ffi/ffi.cpp index c79daa79858171..c586ad1c1969b3 100644 --- a/openmp/libomptarget/plugins-nextgen/host/dynamic_ffi/ffi.cpp +++ b/openmp/libomptarget/plugins-nextgen/host/dynamic_ffi/ffi.cpp @@ -11,6 +11,8 @@ //===----------------------------------------------------------------------===// #include "llvm/Support/DynamicLibrary.h" + +#include "Shared/Debug.h" #include #include "DLWrap.h" @@ -37,15 +39,21 @@ uint32_t ffi_init() { std::string ErrMsg; auto DynlibHandle = std::make_unique( llvm::sys::DynamicLibrary::getPermanentLibrary(FFI_PATH, &ErrMsg)); - if (!DynlibHandle->isValid()) + + if (!DynlibHandle->isValid()) { + DP("Unable to load library '%s': %s!\n", FFI_PATH, ErrMsg.c_str()); return DYNAMIC_FFI_FAIL; + } for (size_t I = 0; I < dlwrap::size(); I++) { const char *Sym = dlwrap::symbol(I); void *P = DynlibHandle->getAddressOfSymbol(Sym); - if (P == nullptr) + if (P == nullptr) { + DP("Unable to find '%s' in '%s'!\n", Sym, FFI_PATH); return DYNAMIC_FFI_FAIL; + } + DP("Implementing %s with dlsym(%s) -> %p\n", Sym, Sym, P); *dlwrap::pointer(I) = P; } @@ -53,8 +61,10 @@ uint32_t ffi_init() { #define DYNAMIC_INIT(SYMBOL) \ { \ void *SymbolPtr = DynlibHandle->getAddressOfSymbol(#SYMBOL); \ - if (!SymbolPtr) \ + if (!SymbolPtr) { \ + DP("Unable to find '%s' in '%s'!\n", #SYMBOL, FFI_PATH); \ return DYNAMIC_FFI_FAIL; \ + } \ SYMBOL = *reinterpret_cast(SymbolPtr); \ } DYNAMIC_INIT(ffi_type_void); From e318613418e08e20d3b9e139a1a3ef0208db4844 Mon Sep 17 00:00:00 2001 From: Alex MacLean Date: Wed, 27 Mar 2024 16:49:59 -0700 Subject: [PATCH 35/54] [NFC][TLI] Move VecFuncs to statics to reduce stack usage (#86829) `TargetLibraryInfoImpl::addVectorizableFunctionsFromVecLib` has a lot of data in local stack arrays, which MSVC keeps on the stack even in release builds. To reduce stack usage, the data arrays (which are const), are moved outside the function as statics. This drops the method stack usage to be negligible. --- llvm/lib/Analysis/TargetLibraryInfo.cpp | 130 +++++++++++++----------- 1 file changed, 68 insertions(+), 62 deletions(-) diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp index c8195584ade378..9e17dcaa55925d 100644 --- a/llvm/lib/Analysis/TargetLibraryInfo.cpp +++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp @@ -1190,107 +1190,113 @@ void TargetLibraryInfoImpl::addVectorizableFunctions(ArrayRef Fns) { llvm::sort(ScalarDescs, compareByVectorFnName); } +static const VecDesc VecFuncs_Accelerate[] = { +#define TLI_DEFINE_ACCELERATE_VECFUNCS +#include "llvm/Analysis/VecFuncs.def" +}; + +static const VecDesc VecFuncs_DarwinLibSystemM[] = { +#define TLI_DEFINE_DARWIN_LIBSYSTEM_M_VECFUNCS +#include "llvm/Analysis/VecFuncs.def" +}; + +static const VecDesc VecFuncs_LIBMVEC_X86[] = { +#define TLI_DEFINE_LIBMVEC_X86_VECFUNCS +#include "llvm/Analysis/VecFuncs.def" +}; + +static const VecDesc VecFuncs_MASSV[] = { +#define TLI_DEFINE_MASSV_VECFUNCS +#include "llvm/Analysis/VecFuncs.def" +}; + +static const VecDesc VecFuncs_SVML[] = { +#define TLI_DEFINE_SVML_VECFUNCS +#include "llvm/Analysis/VecFuncs.def" +}; + +static const VecDesc VecFuncs_SLEEFGNUABI_VF2[] = { +#define TLI_DEFINE_SLEEFGNUABI_VF2_VECFUNCS +#define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, VABI_PREFIX) \ + {SCAL, VEC, VF, /* MASK = */ false, VABI_PREFIX}, +#include "llvm/Analysis/VecFuncs.def" +}; +static const VecDesc VecFuncs_SLEEFGNUABI_VF4[] = { +#define TLI_DEFINE_SLEEFGNUABI_VF4_VECFUNCS +#define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, VABI_PREFIX) \ + {SCAL, VEC, VF, /* MASK = */ false, VABI_PREFIX}, +#include "llvm/Analysis/VecFuncs.def" +}; +static const VecDesc VecFuncs_SLEEFGNUABI_VFScalable[] = { +#define TLI_DEFINE_SLEEFGNUABI_SCALABLE_VECFUNCS +#define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, MASK, VABI_PREFIX) \ + {SCAL, VEC, VF, MASK, VABI_PREFIX}, +#include "llvm/Analysis/VecFuncs.def" +}; + +static const VecDesc VecFuncs_ArmPL[] = { +#define TLI_DEFINE_ARMPL_VECFUNCS +#define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, MASK, VABI_PREFIX) \ + {SCAL, VEC, VF, MASK, VABI_PREFIX}, +#include "llvm/Analysis/VecFuncs.def" +}; + +const VecDesc VecFuncs_AMDLIBM[] = { +#define TLI_DEFINE_AMDLIBM_VECFUNCS +#define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, MASK, VABI_PREFIX) \ + {SCAL, VEC, VF, MASK, VABI_PREFIX}, +#include "llvm/Analysis/VecFuncs.def" +}; + void TargetLibraryInfoImpl::addVectorizableFunctionsFromVecLib( enum VectorLibrary VecLib, const llvm::Triple &TargetTriple) { switch (VecLib) { case Accelerate: { - const VecDesc VecFuncs[] = { - #define TLI_DEFINE_ACCELERATE_VECFUNCS - #include "llvm/Analysis/VecFuncs.def" - }; - addVectorizableFunctions(VecFuncs); + addVectorizableFunctions(VecFuncs_Accelerate); break; } case DarwinLibSystemM: { - const VecDesc VecFuncs[] = { - #define TLI_DEFINE_DARWIN_LIBSYSTEM_M_VECFUNCS - #include "llvm/Analysis/VecFuncs.def" - }; - addVectorizableFunctions(VecFuncs); + addVectorizableFunctions(VecFuncs_DarwinLibSystemM); break; } case LIBMVEC_X86: { - const VecDesc VecFuncs[] = { - #define TLI_DEFINE_LIBMVEC_X86_VECFUNCS - #include "llvm/Analysis/VecFuncs.def" - }; - addVectorizableFunctions(VecFuncs); + addVectorizableFunctions(VecFuncs_LIBMVEC_X86); break; } case MASSV: { - const VecDesc VecFuncs[] = { - #define TLI_DEFINE_MASSV_VECFUNCS - #include "llvm/Analysis/VecFuncs.def" - }; - addVectorizableFunctions(VecFuncs); + addVectorizableFunctions(VecFuncs_MASSV); break; } case SVML: { - const VecDesc VecFuncs[] = { - #define TLI_DEFINE_SVML_VECFUNCS - #include "llvm/Analysis/VecFuncs.def" - }; - addVectorizableFunctions(VecFuncs); + addVectorizableFunctions(VecFuncs_SVML); break; } case SLEEFGNUABI: { - const VecDesc VecFuncs_VF2[] = { -#define TLI_DEFINE_SLEEFGNUABI_VF2_VECFUNCS -#define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, VABI_PREFIX) \ - {SCAL, VEC, VF, /* MASK = */ false, VABI_PREFIX}, -#include "llvm/Analysis/VecFuncs.def" - }; - const VecDesc VecFuncs_VF4[] = { -#define TLI_DEFINE_SLEEFGNUABI_VF4_VECFUNCS -#define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, VABI_PREFIX) \ - {SCAL, VEC, VF, /* MASK = */ false, VABI_PREFIX}, -#include "llvm/Analysis/VecFuncs.def" - }; - const VecDesc VecFuncs_VFScalable[] = { -#define TLI_DEFINE_SLEEFGNUABI_SCALABLE_VECFUNCS -#define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, MASK, VABI_PREFIX) \ - {SCAL, VEC, VF, MASK, VABI_PREFIX}, -#include "llvm/Analysis/VecFuncs.def" - }; - switch (TargetTriple.getArch()) { default: break; case llvm::Triple::aarch64: case llvm::Triple::aarch64_be: - addVectorizableFunctions(VecFuncs_VF2); - addVectorizableFunctions(VecFuncs_VF4); - addVectorizableFunctions(VecFuncs_VFScalable); + addVectorizableFunctions(VecFuncs_SLEEFGNUABI_VF2); + addVectorizableFunctions(VecFuncs_SLEEFGNUABI_VF4); + addVectorizableFunctions(VecFuncs_SLEEFGNUABI_VFScalable); break; } break; } case ArmPL: { - const VecDesc VecFuncs[] = { -#define TLI_DEFINE_ARMPL_VECFUNCS -#define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, MASK, VABI_PREFIX) \ - {SCAL, VEC, VF, MASK, VABI_PREFIX}, -#include "llvm/Analysis/VecFuncs.def" - }; - switch (TargetTriple.getArch()) { default: break; case llvm::Triple::aarch64: case llvm::Triple::aarch64_be: - addVectorizableFunctions(VecFuncs); + addVectorizableFunctions(VecFuncs_ArmPL); break; } break; } case AMDLIBM: { - const VecDesc VecFuncs[] = { -#define TLI_DEFINE_AMDLIBM_VECFUNCS -#define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, MASK, VABI_PREFIX) \ - {SCAL, VEC, VF, MASK, VABI_PREFIX}, -#include "llvm/Analysis/VecFuncs.def" - }; - addVectorizableFunctions(VecFuncs); + addVectorizableFunctions(VecFuncs_AMDLIBM); break; } case NoLibrary: From 036e7ee9d1f1bdc194b56302f8dd27d5eb6cfdab Mon Sep 17 00:00:00 2001 From: Eli Friedman Date: Wed, 27 Mar 2024 17:06:41 -0700 Subject: [PATCH 36/54] [NFC][AArch64] Regenerate regression tests. --- .../GlobalISel/load-addressing-modes.mir | 329 ++++++++++-------- .../aarch64-split-and-bitmask-immediate.ll | 14 +- 2 files changed, 186 insertions(+), 157 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/load-addressing-modes.mir b/llvm/test/CodeGen/AArch64/GlobalISel/load-addressing-modes.mir index 0cf9602adbb094..499c08fa4966f9 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/load-addressing-modes.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/load-addressing-modes.mir @@ -40,11 +40,12 @@ body: | ; CHECK-LABEL: name: ldrxrox_breg_oreg ; CHECK: liveins: $x0, $x1 - ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 - ; CHECK: [[LDRXroX:%[0-9]+]]:gpr64 = LDRXroX [[COPY]], [[COPY1]], 0, 0 :: (load (s64) from %ir.addr) - ; CHECK: $x0 = COPY [[LDRXroX]] - ; CHECK: RET_ReallyLR implicit $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 + ; CHECK-NEXT: [[LDRXroX:%[0-9]+]]:gpr64 = LDRXroX [[COPY]], [[COPY1]], 0, 0 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: $x0 = COPY [[LDRXroX]] + ; CHECK-NEXT: RET_ReallyLR implicit $x0 %0:gpr(p0) = COPY $x0 %1:gpr(s64) = COPY $x1 %2:gpr(p0) = G_PTR_ADD %0, %1 @@ -65,11 +66,12 @@ body: | liveins: $d0, $x1 ; CHECK-LABEL: name: ldrdrox_breg_oreg ; CHECK: liveins: $d0, $x1 - ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $d0 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 - ; CHECK: [[LDRDroX:%[0-9]+]]:fpr64 = LDRDroX [[COPY]], [[COPY1]], 0, 0 :: (load (s64) from %ir.addr) - ; CHECK: $d0 = COPY [[LDRDroX]] - ; CHECK: RET_ReallyLR implicit $d0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64sp = COPY $d0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 + ; CHECK-NEXT: [[LDRDroX:%[0-9]+]]:fpr64 = LDRDroX [[COPY]], [[COPY1]], 0, 0 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: $d0 = COPY [[LDRDroX]] + ; CHECK-NEXT: RET_ReallyLR implicit $d0 %0:gpr(p0) = COPY $d0 %1:gpr(s64) = COPY $x1 %2:gpr(p0) = G_PTR_ADD %0, %1 @@ -78,6 +80,9 @@ body: | RET_ReallyLR implicit $d0 ... --- +# This shouldn't be folded, since we reuse the result of the G_PTR_ADD outside +# the G_LOAD + name: more_than_one_use alignment: 4 legalized: true @@ -87,18 +92,17 @@ machineFunctionInfo: {} body: | bb.0: liveins: $x0, $x1 - ; This shouldn't be folded, since we reuse the result of the G_PTR_ADD outside - ; the G_LOAD ; CHECK-LABEL: name: more_than_one_use ; CHECK: liveins: $x0, $x1 - ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 - ; CHECK: [[ADDXrr:%[0-9]+]]:gpr64common = ADDXrr [[COPY]], [[COPY1]] - ; CHECK: [[LDRXui:%[0-9]+]]:gpr64 = LDRXui [[ADDXrr]], 0 :: (load (s64) from %ir.addr) - ; CHECK: [[COPY2:%[0-9]+]]:gpr64 = COPY [[ADDXrr]] - ; CHECK: [[ADDXrr1:%[0-9]+]]:gpr64 = ADDXrr [[COPY2]], [[LDRXui]] - ; CHECK: $x0 = COPY [[ADDXrr1]] - ; CHECK: RET_ReallyLR implicit $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 + ; CHECK-NEXT: [[ADDXrr:%[0-9]+]]:gpr64common = ADDXrr [[COPY]], [[COPY1]] + ; CHECK-NEXT: [[LDRXui:%[0-9]+]]:gpr64 = LDRXui [[ADDXrr]], 0 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64 = COPY [[ADDXrr]] + ; CHECK-NEXT: [[ADDXrr1:%[0-9]+]]:gpr64 = ADDXrr [[COPY2]], [[LDRXui]] + ; CHECK-NEXT: $x0 = COPY [[ADDXrr1]] + ; CHECK-NEXT: RET_ReallyLR implicit $x0 %0:gpr(p0) = COPY $x0 %1:gpr(s64) = COPY $x1 %2:gpr(p0) = G_PTR_ADD %0, %1 @@ -121,11 +125,12 @@ body: | liveins: $x0, $x1, $x2 ; CHECK-LABEL: name: ldrxrox_shl ; CHECK: liveins: $x0, $x1, $x2 - ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 - ; CHECK: [[LDRXroX:%[0-9]+]]:gpr64 = LDRXroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) - ; CHECK: $x2 = COPY [[LDRXroX]] - ; CHECK: RET_ReallyLR implicit $x2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 + ; CHECK-NEXT: [[LDRXroX:%[0-9]+]]:gpr64 = LDRXroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: $x2 = COPY [[LDRXroX]] + ; CHECK-NEXT: RET_ReallyLR implicit $x2 %0:gpr(s64) = COPY $x0 %1:gpr(s64) = G_CONSTANT i64 3 %2:gpr(s64) = G_SHL %0, %1(s64) @@ -148,11 +153,12 @@ body: | liveins: $x0, $x1, $d2 ; CHECK-LABEL: name: ldrdrox_shl ; CHECK: liveins: $x0, $x1, $d2 - ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 - ; CHECK: [[LDRDroX:%[0-9]+]]:fpr64 = LDRDroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) - ; CHECK: $d2 = COPY [[LDRDroX]] - ; CHECK: RET_ReallyLR implicit $d2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 + ; CHECK-NEXT: [[LDRDroX:%[0-9]+]]:fpr64 = LDRDroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: $d2 = COPY [[LDRDroX]] + ; CHECK-NEXT: RET_ReallyLR implicit $d2 %0:gpr(s64) = COPY $x0 %1:gpr(s64) = G_CONSTANT i64 3 %2:gpr(s64) = G_SHL %0, %1(s64) @@ -175,11 +181,12 @@ body: | liveins: $x0, $x1, $x2 ; CHECK-LABEL: name: ldrxrox_mul_rhs ; CHECK: liveins: $x0, $x1, $x2 - ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 - ; CHECK: [[LDRXroX:%[0-9]+]]:gpr64 = LDRXroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) - ; CHECK: $x2 = COPY [[LDRXroX]] - ; CHECK: RET_ReallyLR implicit $x2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 + ; CHECK-NEXT: [[LDRXroX:%[0-9]+]]:gpr64 = LDRXroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: $x2 = COPY [[LDRXroX]] + ; CHECK-NEXT: RET_ReallyLR implicit $x2 %0:gpr(s64) = COPY $x0 %1:gpr(s64) = G_CONSTANT i64 8 %2:gpr(s64) = G_MUL %0, %1(s64) @@ -202,11 +209,12 @@ body: | liveins: $x0, $x1, $d2 ; CHECK-LABEL: name: ldrdrox_mul_rhs ; CHECK: liveins: $x0, $x1, $d2 - ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 - ; CHECK: [[LDRDroX:%[0-9]+]]:fpr64 = LDRDroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) - ; CHECK: $d2 = COPY [[LDRDroX]] - ; CHECK: RET_ReallyLR implicit $d2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 + ; CHECK-NEXT: [[LDRDroX:%[0-9]+]]:fpr64 = LDRDroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: $d2 = COPY [[LDRDroX]] + ; CHECK-NEXT: RET_ReallyLR implicit $d2 %0:gpr(s64) = COPY $x0 %1:gpr(s64) = G_CONSTANT i64 8 %2:gpr(s64) = G_MUL %0, %1(s64) @@ -229,11 +237,12 @@ body: | liveins: $x0, $x1, $x2 ; CHECK-LABEL: name: ldrxrox_mul_lhs ; CHECK: liveins: $x0, $x1, $x2 - ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 - ; CHECK: [[LDRXroX:%[0-9]+]]:gpr64 = LDRXroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) - ; CHECK: $x2 = COPY [[LDRXroX]] - ; CHECK: RET_ReallyLR implicit $x2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 + ; CHECK-NEXT: [[LDRXroX:%[0-9]+]]:gpr64 = LDRXroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: $x2 = COPY [[LDRXroX]] + ; CHECK-NEXT: RET_ReallyLR implicit $x2 %0:gpr(s64) = COPY $x0 %1:gpr(s64) = G_CONSTANT i64 8 %2:gpr(s64) = G_MUL %1, %0(s64) @@ -256,11 +265,12 @@ body: | liveins: $x0, $x1, $d2 ; CHECK-LABEL: name: ldrdrox_mul_lhs ; CHECK: liveins: $x0, $x1, $d2 - ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 - ; CHECK: [[LDRDroX:%[0-9]+]]:fpr64 = LDRDroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) - ; CHECK: $d2 = COPY [[LDRDroX]] - ; CHECK: RET_ReallyLR implicit $d2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 + ; CHECK-NEXT: [[LDRDroX:%[0-9]+]]:fpr64 = LDRDroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: $d2 = COPY [[LDRDroX]] + ; CHECK-NEXT: RET_ReallyLR implicit $d2 %0:gpr(s64) = COPY $x0 %1:gpr(s64) = G_CONSTANT i64 8 %2:gpr(s64) = G_MUL %1, %0(s64) @@ -272,6 +282,9 @@ body: | ... --- +# Show that we don't get a shifted load from a mul when we don't have a +# power of 2. (The bit isn't set on the load.) + name: mul_not_pow_2 alignment: 4 legalized: true @@ -280,19 +293,18 @@ tracksRegLiveness: true machineFunctionInfo: {} body: | bb.0: - ; Show that we don't get a shifted load from a mul when we don't have a - ; power of 2. (The bit isn't set on the load.) liveins: $x0, $x1, $d2 ; CHECK-LABEL: name: mul_not_pow_2 ; CHECK: liveins: $x0, $x1, $d2 - ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 - ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 7 - ; CHECK: [[SUBREG_TO_REG:%[0-9]+]]:gpr64 = SUBREG_TO_REG 0, [[MOVi32imm]], %subreg.sub_32 - ; CHECK: [[MADDXrrr:%[0-9]+]]:gpr64 = MADDXrrr [[SUBREG_TO_REG]], [[COPY]], $xzr - ; CHECK: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 - ; CHECK: [[LDRDroX:%[0-9]+]]:fpr64 = LDRDroX [[COPY1]], [[MADDXrrr]], 0, 0 :: (load (s64) from %ir.addr) - ; CHECK: $d2 = COPY [[LDRDroX]] - ; CHECK: RET_ReallyLR implicit $d2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 7 + ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gpr64 = SUBREG_TO_REG 0, [[MOVi32imm]], %subreg.sub_32 + ; CHECK-NEXT: [[MADDXrrr:%[0-9]+]]:gpr64 = MADDXrrr [[SUBREG_TO_REG]], [[COPY]], $xzr + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 + ; CHECK-NEXT: [[LDRDroX:%[0-9]+]]:fpr64 = LDRDroX [[COPY1]], [[MADDXrrr]], 0, 0 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: $d2 = COPY [[LDRDroX]] + ; CHECK-NEXT: RET_ReallyLR implicit $d2 %0:gpr(s64) = COPY $x0 %1:gpr(s64) = G_CONSTANT i64 7 %2:gpr(s64) = G_MUL %1, %0(s64) @@ -304,6 +316,9 @@ body: | ... --- +# Show that we don't get a shifted load from a mul when we don't have +# the right power of 2. (The bit isn't set on the load.) + name: mul_wrong_pow_2 alignment: 4 legalized: true @@ -312,19 +327,18 @@ tracksRegLiveness: true machineFunctionInfo: {} body: | bb.0: - ; Show that we don't get a shifted load from a mul when we don't have - ; the right power of 2. (The bit isn't set on the load.) liveins: $x0, $x1, $d2 ; CHECK-LABEL: name: mul_wrong_pow_2 ; CHECK: liveins: $x0, $x1, $d2 - ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 - ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 16 - ; CHECK: [[SUBREG_TO_REG:%[0-9]+]]:gpr64 = SUBREG_TO_REG 0, [[MOVi32imm]], %subreg.sub_32 - ; CHECK: [[MADDXrrr:%[0-9]+]]:gpr64 = MADDXrrr [[SUBREG_TO_REG]], [[COPY]], $xzr - ; CHECK: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 - ; CHECK: [[LDRDroX:%[0-9]+]]:fpr64 = LDRDroX [[COPY1]], [[MADDXrrr]], 0, 0 :: (load (s64) from %ir.addr) - ; CHECK: $d2 = COPY [[LDRDroX]] - ; CHECK: RET_ReallyLR implicit $d2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 16 + ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gpr64 = SUBREG_TO_REG 0, [[MOVi32imm]], %subreg.sub_32 + ; CHECK-NEXT: [[MADDXrrr:%[0-9]+]]:gpr64 = MADDXrrr [[SUBREG_TO_REG]], [[COPY]], $xzr + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 + ; CHECK-NEXT: [[LDRDroX:%[0-9]+]]:fpr64 = LDRDroX [[COPY1]], [[MADDXrrr]], 0, 0 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: $d2 = COPY [[LDRDroX]] + ; CHECK-NEXT: RET_ReallyLR implicit $d2 %0:gpr(s64) = COPY $x0 %1:gpr(s64) = G_CONSTANT i64 16 %2:gpr(s64) = G_MUL %1, %0(s64) @@ -336,6 +350,9 @@ body: | ... --- +# Show that we can still fall back to the register-register addressing +# mode when we fail to pull in the shift. + name: more_than_one_use_shl_1 alignment: 4 legalized: true @@ -344,19 +361,18 @@ tracksRegLiveness: true machineFunctionInfo: {} body: | bb.0: - ; Show that we can still fall back to the register-register addressing - ; mode when we fail to pull in the shift. liveins: $x0, $x1, $x2 ; CHECK-LABEL: name: more_than_one_use_shl_1 ; CHECK: liveins: $x0, $x1, $x2 - ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 - ; CHECK: [[UBFMXri:%[0-9]+]]:gpr64common = UBFMXri [[COPY]], 61, 60 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 - ; CHECK: [[LDRXroX:%[0-9]+]]:gpr64 = LDRXroX [[COPY1]], [[UBFMXri]], 0, 0 :: (load (s64) from %ir.addr) - ; CHECK: [[ADDXri:%[0-9]+]]:gpr64common = ADDXri [[UBFMXri]], 3, 0 - ; CHECK: [[ADDXrr:%[0-9]+]]:gpr64 = ADDXrr [[LDRXroX]], [[ADDXri]] - ; CHECK: $x2 = COPY [[ADDXrr]] - ; CHECK: RET_ReallyLR implicit $x2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[UBFMXri:%[0-9]+]]:gpr64common = UBFMXri [[COPY]], 61, 60 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 + ; CHECK-NEXT: [[LDRXroX:%[0-9]+]]:gpr64 = LDRXroX [[COPY1]], [[UBFMXri]], 0, 0 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: [[ADDXri:%[0-9]+]]:gpr64common = ADDXri [[UBFMXri]], 3, 0 + ; CHECK-NEXT: [[ADDXrr:%[0-9]+]]:gpr64 = ADDXrr [[LDRXroX]], [[ADDXri]] + ; CHECK-NEXT: $x2 = COPY [[ADDXrr]] + ; CHECK-NEXT: RET_ReallyLR implicit $x2 %0:gpr(s64) = COPY $x0 %1:gpr(s64) = G_CONSTANT i64 3 %2:gpr(s64) = G_SHL %0, %1(s64) @@ -370,6 +386,9 @@ body: | ... --- +# Show that when the GEP is used outside a memory op, we don't do any +# folding at all. + name: more_than_one_use_shl_2 alignment: 4 legalized: true @@ -378,22 +397,21 @@ tracksRegLiveness: true machineFunctionInfo: {} body: | bb.0: - ; Show that when the GEP is used outside a memory op, we don't do any - ; folding at all. liveins: $x0, $x1, $x2 ; CHECK-LABEL: name: more_than_one_use_shl_2 ; CHECK: liveins: $x0, $x1, $x2 - ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 - ; CHECK: [[UBFMXri:%[0-9]+]]:gpr64common = UBFMXri [[COPY]], 61, 60 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 - ; CHECK: [[ADDXrr:%[0-9]+]]:gpr64common = ADDXrr [[COPY1]], [[UBFMXri]] - ; CHECK: [[LDRXui:%[0-9]+]]:gpr64 = LDRXui [[ADDXrr]], 0 :: (load (s64) from %ir.addr) - ; CHECK: [[ADDXri:%[0-9]+]]:gpr64common = ADDXri [[UBFMXri]], 3, 0 - ; CHECK: [[ADDXrr1:%[0-9]+]]:gpr64 = ADDXrr [[LDRXui]], [[ADDXri]] - ; CHECK: [[COPY2:%[0-9]+]]:gpr64 = COPY [[ADDXrr]] - ; CHECK: [[ADDXrr2:%[0-9]+]]:gpr64 = ADDXrr [[COPY2]], [[ADDXrr1]] - ; CHECK: $x2 = COPY [[ADDXrr2]] - ; CHECK: RET_ReallyLR implicit $x2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[UBFMXri:%[0-9]+]]:gpr64common = UBFMXri [[COPY]], 61, 60 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 + ; CHECK-NEXT: [[ADDXrr:%[0-9]+]]:gpr64common = ADDXrr [[COPY1]], [[UBFMXri]] + ; CHECK-NEXT: [[LDRXui:%[0-9]+]]:gpr64 = LDRXui [[ADDXrr]], 0 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: [[ADDXri:%[0-9]+]]:gpr64common = ADDXri [[UBFMXri]], 3, 0 + ; CHECK-NEXT: [[ADDXrr1:%[0-9]+]]:gpr64 = ADDXrr [[LDRXui]], [[ADDXri]] + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64 = COPY [[ADDXrr]] + ; CHECK-NEXT: [[ADDXrr2:%[0-9]+]]:gpr64 = ADDXrr [[COPY2]], [[ADDXrr1]] + ; CHECK-NEXT: $x2 = COPY [[ADDXrr2]] + ; CHECK-NEXT: RET_ReallyLR implicit $x2 %0:gpr(s64) = COPY $x0 %1:gpr(s64) = G_CONSTANT i64 3 %2:gpr(s64) = G_SHL %0, %1(s64) @@ -409,6 +427,9 @@ body: | ... --- +# Show that when we have a fastpath for shift-left, we perform the folding +# if it has more than one use. + name: more_than_one_use_shl_lsl_fast alignment: 4 legalized: true @@ -417,18 +438,17 @@ tracksRegLiveness: true machineFunctionInfo: {} body: | bb.0: - ; Show that when we have a fastpath for shift-left, we perform the folding - ; if it has more than one use. liveins: $x0, $x1, $x2 ; CHECK-LABEL: name: more_than_one_use_shl_lsl_fast ; CHECK: liveins: $x0, $x1, $x2 - ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 - ; CHECK: [[LDRXroX:%[0-9]+]]:gpr64 = LDRXroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) - ; CHECK: [[LDRXroX1:%[0-9]+]]:gpr64 = LDRXroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) - ; CHECK: [[ADDXrr:%[0-9]+]]:gpr64 = ADDXrr [[LDRXroX]], [[LDRXroX1]] - ; CHECK: $x2 = COPY [[ADDXrr]] - ; CHECK: RET_ReallyLR implicit $x2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64sp = COPY $x1 + ; CHECK-NEXT: [[LDRXroX:%[0-9]+]]:gpr64 = LDRXroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: [[LDRXroX1:%[0-9]+]]:gpr64 = LDRXroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: [[ADDXrr:%[0-9]+]]:gpr64 = ADDXrr [[LDRXroX]], [[LDRXroX1]] + ; CHECK-NEXT: $x2 = COPY [[ADDXrr]] + ; CHECK-NEXT: RET_ReallyLR implicit $x2 %0:gpr(s64) = COPY $x0 %1:gpr(s64) = G_CONSTANT i64 3 %2:gpr(s64) = G_SHL %0, %1(s64) @@ -442,6 +462,9 @@ body: | ... --- +# Show that we don't fold into multiple memory ops when we don't have a +# fastpath for shift-left. + name: more_than_one_use_shl_lsl_slow alignment: 4 legalized: true @@ -450,19 +473,18 @@ tracksRegLiveness: true machineFunctionInfo: {} body: | bb.0: - ; Show that we don't fold into multiple memory ops when we don't have a - ; fastpath for shift-left. liveins: $x0, $x1, $x2 ; CHECK-LABEL: name: more_than_one_use_shl_lsl_slow ; CHECK: liveins: $x0, $x1, $x2 - ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 - ; CHECK: [[ADDXrs:%[0-9]+]]:gpr64common = ADDXrs [[COPY1]], [[COPY]], 3 - ; CHECK: [[LDRXui:%[0-9]+]]:gpr64 = LDRXui [[ADDXrs]], 0 :: (load (s64) from %ir.addr) - ; CHECK: [[LDRXui1:%[0-9]+]]:gpr64 = LDRXui [[ADDXrs]], 0 :: (load (s64) from %ir.addr) - ; CHECK: [[ADDXrr:%[0-9]+]]:gpr64 = ADDXrr [[LDRXui]], [[LDRXui1]] - ; CHECK: $x2 = COPY [[ADDXrr]] - ; CHECK: RET_ReallyLR implicit $x2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 + ; CHECK-NEXT: [[ADDXrs:%[0-9]+]]:gpr64common = ADDXrs [[COPY1]], [[COPY]], 3 + ; CHECK-NEXT: [[LDRXui:%[0-9]+]]:gpr64 = LDRXui [[ADDXrs]], 0 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: [[LDRXui1:%[0-9]+]]:gpr64 = LDRXui [[ADDXrs]], 0 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: [[ADDXrr:%[0-9]+]]:gpr64 = ADDXrr [[LDRXui]], [[LDRXui1]] + ; CHECK-NEXT: $x2 = COPY [[ADDXrr]] + ; CHECK-NEXT: RET_ReallyLR implicit $x2 %0:gpr(s64) = COPY $x0 %1:gpr(s64) = G_CONSTANT i64 3 %2:gpr(s64) = G_SHL %0, %1(s64) @@ -476,6 +498,9 @@ body: | ... --- +# Show that when we're optimizing for size, we'll do the folding no matter +# what. + name: more_than_one_use_shl_minsize alignment: 4 legalized: true @@ -484,22 +509,21 @@ tracksRegLiveness: true machineFunctionInfo: {} body: | bb.0: - ; Show that when we're optimizing for size, we'll do the folding no matter - ; what. liveins: $x0, $x1, $x2 ; CHECK-LABEL: name: more_than_one_use_shl_minsize ; CHECK: liveins: $x0, $x1, $x2 - ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 - ; CHECK: [[UBFMXri:%[0-9]+]]:gpr64common = UBFMXri [[COPY]], 61, 60 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1 - ; CHECK: [[COPY2:%[0-9]+]]:gpr64 = COPY [[COPY1]] - ; CHECK: [[ADDXrs:%[0-9]+]]:gpr64 = ADDXrs [[COPY2]], [[COPY]], 3 - ; CHECK: [[LDRXroX:%[0-9]+]]:gpr64 = LDRXroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) - ; CHECK: [[ADDXri:%[0-9]+]]:gpr64common = ADDXri [[UBFMXri]], 3, 0 - ; CHECK: [[ADDXrr:%[0-9]+]]:gpr64 = ADDXrr [[LDRXroX]], [[ADDXri]] - ; CHECK: [[ADDXrr1:%[0-9]+]]:gpr64 = ADDXrr [[ADDXrs]], [[ADDXrr]] - ; CHECK: $x2 = COPY [[ADDXrr1]] - ; CHECK: RET_ReallyLR implicit $x2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0 + ; CHECK-NEXT: [[UBFMXri:%[0-9]+]]:gpr64common = UBFMXri [[COPY]], 61, 60 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1 + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64 = COPY [[COPY1]] + ; CHECK-NEXT: [[ADDXrs:%[0-9]+]]:gpr64 = ADDXrs [[COPY2]], [[COPY]], 3 + ; CHECK-NEXT: [[LDRXroX:%[0-9]+]]:gpr64 = LDRXroX [[COPY1]], [[COPY]], 0, 1 :: (load (s64) from %ir.addr) + ; CHECK-NEXT: [[ADDXri:%[0-9]+]]:gpr64common = ADDXri [[UBFMXri]], 3, 0 + ; CHECK-NEXT: [[ADDXrr:%[0-9]+]]:gpr64 = ADDXrr [[LDRXroX]], [[ADDXri]] + ; CHECK-NEXT: [[ADDXrr1:%[0-9]+]]:gpr64 = ADDXrr [[ADDXrs]], [[ADDXrr]] + ; CHECK-NEXT: $x2 = COPY [[ADDXrr1]] + ; CHECK-NEXT: RET_ReallyLR implicit $x2 %0:gpr(s64) = COPY $x0 %1:gpr(s64) = G_CONSTANT i64 3 %2:gpr(s64) = G_SHL %0, %1(s64) @@ -525,11 +549,12 @@ body: | liveins: $x0, $x1 ; CHECK-LABEL: name: ldrwrox ; CHECK: liveins: $x0, $x1 - ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 - ; CHECK: [[LDRWroX:%[0-9]+]]:gpr32 = LDRWroX [[COPY]], [[COPY1]], 0, 0 :: (load (s32) from %ir.addr) - ; CHECK: $w2 = COPY [[LDRWroX]] - ; CHECK: RET_ReallyLR implicit $w2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 + ; CHECK-NEXT: [[LDRWroX:%[0-9]+]]:gpr32 = LDRWroX [[COPY]], [[COPY1]], 0, 0 :: (load (s32) from %ir.addr) + ; CHECK-NEXT: $w2 = COPY [[LDRWroX]] + ; CHECK-NEXT: RET_ReallyLR implicit $w2 %0:gpr(p0) = COPY $x0 %1:gpr(s64) = COPY $x1 %2:gpr(p0) = G_PTR_ADD %0, %1 @@ -549,11 +574,12 @@ body: | liveins: $d0, $x1 ; CHECK-LABEL: name: ldrsrox ; CHECK: liveins: $d0, $x1 - ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $d0 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 - ; CHECK: [[LDRSroX:%[0-9]+]]:fpr32 = LDRSroX [[COPY]], [[COPY1]], 0, 0 :: (load (s32) from %ir.addr) - ; CHECK: $s2 = COPY [[LDRSroX]] - ; CHECK: RET_ReallyLR implicit $h2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64sp = COPY $d0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 + ; CHECK-NEXT: [[LDRSroX:%[0-9]+]]:fpr32 = LDRSroX [[COPY]], [[COPY1]], 0, 0 :: (load (s32) from %ir.addr) + ; CHECK-NEXT: $s2 = COPY [[LDRSroX]] + ; CHECK-NEXT: RET_ReallyLR implicit $h2 %0:gpr(p0) = COPY $d0 %1:gpr(s64) = COPY $x1 %2:gpr(p0) = G_PTR_ADD %0, %1 @@ -573,11 +599,12 @@ body: | liveins: $x0, $x1 ; CHECK-LABEL: name: ldrhrox ; CHECK: liveins: $x0, $x1 - ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 - ; CHECK: [[LDRHroX:%[0-9]+]]:fpr16 = LDRHroX [[COPY]], [[COPY1]], 0, 0 :: (load (s16) from %ir.addr) - ; CHECK: $h2 = COPY [[LDRHroX]] - ; CHECK: RET_ReallyLR implicit $h2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 + ; CHECK-NEXT: [[LDRHroX:%[0-9]+]]:fpr16 = LDRHroX [[COPY]], [[COPY1]], 0, 0 :: (load (s16) from %ir.addr) + ; CHECK-NEXT: $h2 = COPY [[LDRHroX]] + ; CHECK-NEXT: RET_ReallyLR implicit $h2 %0:gpr(p0) = COPY $x0 %1:gpr(s64) = COPY $x1 %2:gpr(p0) = G_PTR_ADD %0, %1 @@ -597,11 +624,12 @@ body: | liveins: $x0, $x1 ; CHECK-LABEL: name: ldbbrox ; CHECK: liveins: $x0, $x1 - ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 - ; CHECK: [[LDRBBroX:%[0-9]+]]:gpr32 = LDRBBroX [[COPY]], [[COPY1]], 0, 0 :: (load (s8) from %ir.addr) - ; CHECK: $w2 = COPY [[LDRBBroX]] - ; CHECK: RET_ReallyLR implicit $w2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 + ; CHECK-NEXT: [[LDRBBroX:%[0-9]+]]:gpr32 = LDRBBroX [[COPY]], [[COPY1]], 0, 0 :: (load (s8) from %ir.addr) + ; CHECK-NEXT: $w2 = COPY [[LDRBBroX]] + ; CHECK-NEXT: RET_ReallyLR implicit $w2 %0:gpr(p0) = COPY $x0 %1:gpr(s64) = COPY $x1 %2:gpr(p0) = G_PTR_ADD %0, %1 @@ -621,11 +649,12 @@ body: | liveins: $d0, $x1 ; CHECK-LABEL: name: ldrqrox ; CHECK: liveins: $d0, $x1 - ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $d0 - ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 - ; CHECK: [[LDRQroX:%[0-9]+]]:fpr128 = LDRQroX [[COPY]], [[COPY1]], 0, 0 :: (load (<2 x s64>) from %ir.addr) - ; CHECK: $q0 = COPY [[LDRQroX]] - ; CHECK: RET_ReallyLR implicit $q0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64sp = COPY $d0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1 + ; CHECK-NEXT: [[LDRQroX:%[0-9]+]]:fpr128 = LDRQroX [[COPY]], [[COPY1]], 0, 0 :: (load (<2 x s64>) from %ir.addr) + ; CHECK-NEXT: $q0 = COPY [[LDRQroX]] + ; CHECK-NEXT: RET_ReallyLR implicit $q0 %0:gpr(p0) = COPY $d0 %1:gpr(s64) = COPY $x1 %2:gpr(p0) = G_PTR_ADD %0, %1 diff --git a/llvm/test/CodeGen/AArch64/aarch64-split-and-bitmask-immediate.ll b/llvm/test/CodeGen/AArch64/aarch64-split-and-bitmask-immediate.ll index cf9ed4d5f0e16a..573f921e638cf8 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-split-and-bitmask-immediate.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-split-and-bitmask-immediate.ll @@ -20,7 +20,7 @@ entry: define i8 @test2(i32 %a) { ; CHECK-LABEL: test2: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: mov w8, #135 +; CHECK-NEXT: mov w8, #135 // =0x87 ; CHECK-NEXT: and w8, w0, w8 ; CHECK-NEXT: cmp w8, #1024 ; CHECK-NEXT: cset w0, eq @@ -37,7 +37,7 @@ entry: define i8 @test3(i32 %a) { ; CHECK-LABEL: test3: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: mov w8, #1024 +; CHECK-NEXT: mov w8, #1024 // =0x400 ; CHECK-NEXT: movk w8, #33, lsl #16 ; CHECK-NEXT: and w8, w0, w8 ; CHECK-NEXT: cmp w8, #1024 @@ -84,7 +84,7 @@ entry: define i8 @test6(i64 %a) { ; CHECK-LABEL: test6: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: mov w8, #135 +; CHECK-NEXT: mov w8, #135 // =0x87 ; CHECK-NEXT: and x8, x0, x8 ; CHECK-NEXT: cmp x8, #1024 ; CHECK-NEXT: cset w0, eq @@ -101,7 +101,7 @@ entry: define i8 @test7(i64 %a) { ; CHECK-LABEL: test7: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: mov w8, #1024 +; CHECK-NEXT: mov w8, #1024 // =0x400 ; CHECK-NEXT: movk w8, #33, lsl #16 ; CHECK-NEXT: and x8, x0, x8 ; CHECK-NEXT: cmp x8, #1024 @@ -175,7 +175,7 @@ define i32 @test9(ptr nocapture %x, ptr nocapture readonly %y, i32 %n) { ; CHECK-NEXT: cmp w2, #1 ; CHECK-NEXT: b.lt .LBB8_3 ; CHECK-NEXT: // %bb.1: // %for.body.preheader -; CHECK-NEXT: mov w9, #1024 +; CHECK-NEXT: mov w9, #1024 // =0x400 ; CHECK-NEXT: mov w8, w2 ; CHECK-NEXT: movk w9, #32, lsl #16 ; CHECK-NEXT: .LBB8_2: // %for.body @@ -226,7 +226,7 @@ define void @test10(ptr nocapture %x, ptr nocapture readonly %y, ptr nocapture % ; CHECK-LABEL: test10: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ldr w8, [x1] -; CHECK-NEXT: mov w9, #1024 +; CHECK-NEXT: mov w9, #1024 // =0x400 ; CHECK-NEXT: movk w9, #32, lsl #16 ; CHECK-NEXT: and w8, w8, w9 ; CHECK-NEXT: str w8, [x0] @@ -253,7 +253,7 @@ entry: define i8 @test11(i64 %a) { ; CHECK-LABEL: test11: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: mov w8, #-1610612736 +; CHECK-NEXT: mov w8, #-1610612736 // =0xa0000000 ; CHECK-NEXT: and x8, x0, x8 ; CHECK-NEXT: cmp x8, #1024 ; CHECK-NEXT: cset w0, eq From 0fda758f26c1ec06809fdc067cd65dc146f867d0 Mon Sep 17 00:00:00 2001 From: Marc Auberer Date: Thu, 28 Mar 2024 01:19:09 +0100 Subject: [PATCH 37/54] [GISEL][NFC] Refactor OperandPredicateMatcher::isHigherPriorityThan (#86837) Fixes #86827 This will simplify code, de-duplicate some logic and fix the faulty bool compare. cc @dcb314 --- .../GlobalISel/GlobalISelMatchTable.cpp | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp index 193f95443b16ef..19d42b7688dac8 100644 --- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp @@ -1077,30 +1077,25 @@ OperandPredicateMatcher::~OperandPredicateMatcher() {} bool OperandPredicateMatcher::isHigherPriorityThan( const OperandPredicateMatcher &B) const { // Generally speaking, an instruction is more important than an Int or a - // LiteralInt because it can cover more nodes but theres an exception to + // LiteralInt because it can cover more nodes but there's an exception to // this. G_CONSTANT's are less important than either of those two because they // are more permissive. - const InstructionOperandMatcher *AOM = - dyn_cast(this); - const InstructionOperandMatcher *BOM = - dyn_cast(&B); + const auto *AOM = dyn_cast(this); + const auto *BOM = dyn_cast(&B); bool AIsConstantInsn = AOM && AOM->getInsnMatcher().isConstantInstruction(); bool BIsConstantInsn = BOM && BOM->getInsnMatcher().isConstantInstruction(); - if (AOM && BOM) { - // The relative priorities between a G_CONSTANT and any other instruction - // don't actually matter but this code is needed to ensure a strict weak - // ordering. This is particularly important on Windows where the rules will - // be incorrectly sorted without it. - if (AIsConstantInsn != BIsConstantInsn) - return AIsConstantInsn < BIsConstantInsn; - return false; - } + // The relative priorities between a G_CONSTANT and any other instruction + // don't actually matter but this code is needed to ensure a strict weak + // ordering. This is particularly important on Windows where the rules will + // be incorrectly sorted without it. + if (AOM && BOM) + return !AIsConstantInsn && BIsConstantInsn; - if (AOM && AIsConstantInsn && (B.Kind == OPM_Int || B.Kind == OPM_LiteralInt)) + if (AIsConstantInsn && (B.Kind == OPM_Int || B.Kind == OPM_LiteralInt)) return false; - if (BOM && BIsConstantInsn && (Kind == OPM_Int || Kind == OPM_LiteralInt)) + if (BIsConstantInsn && (Kind == OPM_Int || Kind == OPM_LiteralInt)) return true; return Kind < B.Kind; From bbfa50696e43f337f55f8bacf739683b181debd5 Mon Sep 17 00:00:00 2001 From: alx32 <103613512+alx32@users.noreply.github.com> Date: Wed, 27 Mar 2024 17:27:51 -0700 Subject: [PATCH 38/54] [lld-macho] Fix bug in makeSyntheticInputSection when -dead_strip flag is specified (#86878) Previously, `makeSyntheticInputSection` would create a new `ConcatInputSection` without setting `live` explicitly for it. Without `-dead_strip` this would be OK since `live` would default to `true`. However, with `-dead_strip`, `live` would default to false, and it would remain set to `false`. This hasn't resulted in any issues so far since no code paths that exposed this issue were present. However a recent change - ObjC relative method lists (https://github.com/llvm/llvm-project/pull/86231) exposes this issue by creating relocations to the `SyntheticInputSection`. When these relocations are attempted to be written, this ends up with a crash(assert), since the `SyntheticInputSection` they refer to is marked as dead (`live` = `false`). With this change, we set the correct behavior - `live` will always be `true`. We add a test case that before this change would trigger an assert in the linker. --- lld/MachO/ConcatOutputSection.cpp | 6 +----- lld/MachO/InputSection.cpp | 3 +++ lld/MachO/InputSection.h | 1 + lld/MachO/SymbolTable.cpp | 2 +- lld/MachO/SyntheticSections.cpp | 2 +- lld/MachO/Writer.cpp | 4 +--- lld/test/MachO/objc-relative-method-lists-simple.s | 4 ++++ 7 files changed, 12 insertions(+), 10 deletions(-) diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp index c5c0c8a89e2879..279423720be9d5 100644 --- a/lld/MachO/ConcatOutputSection.cpp +++ b/lld/MachO/ConcatOutputSection.cpp @@ -323,11 +323,7 @@ void TextOutputSection::finalize() { thunkInfo.isec = makeSyntheticInputSection(isec->getSegName(), isec->getName()); thunkInfo.isec->parent = this; - - // This code runs after dead code removal. Need to set the `live` bit - // on the thunk isec so that asserts that check that only live sections - // get written are happy. - thunkInfo.isec->live = true; + assert(thunkInfo.isec->live); StringRef thunkName = saver().save(funcSym->getName() + ".thunk." + std::to_string(thunkInfo.sequence++)); diff --git a/lld/MachO/InputSection.cpp b/lld/MachO/InputSection.cpp index e3d7a400e4ce92..5c1e07cd21b1fb 100644 --- a/lld/MachO/InputSection.cpp +++ b/lld/MachO/InputSection.cpp @@ -281,6 +281,9 @@ ConcatInputSection *macho::makeSyntheticInputSection(StringRef segName, Section §ion = *make
(/*file=*/nullptr, segName, sectName, flags, /*addr=*/0); auto isec = make(section, data, align); + // Since this is an explicitly created 'fake' input section, + // it should not be dead stripped. + isec->live = true; section.subsections.push_back({0, isec}); return isec; } diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h index a0e6afef92cb82..0f389e50425a32 100644 --- a/lld/MachO/InputSection.h +++ b/lld/MachO/InputSection.h @@ -149,6 +149,7 @@ class ConcatInputSection final : public InputSection { }; // Initialize a fake InputSection that does not belong to any InputFile. +// The created ConcatInputSection will always have 'live=true' ConcatInputSection *makeSyntheticInputSection(StringRef segName, StringRef sectName, uint32_t flags = 0, diff --git a/lld/MachO/SymbolTable.cpp b/lld/MachO/SymbolTable.cpp index 825242f2cc72ff..755ff270e2f7a9 100644 --- a/lld/MachO/SymbolTable.cpp +++ b/lld/MachO/SymbolTable.cpp @@ -377,7 +377,7 @@ static void handleSectionBoundarySymbol(const Undefined &sym, StringRef segSect, // live. Marking the isec live ensures an OutputSection is created that the // start/end symbol can refer to. assert(sym.isLive()); - isec->live = true; + assert(isec->live); // This runs after gatherInputSections(), so need to explicitly set parent // and add to inputSections. diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp index 808ea8eac6eb59..6f6b66118b7a94 100644 --- a/lld/MachO/SyntheticSections.cpp +++ b/lld/MachO/SyntheticSections.cpp @@ -850,7 +850,7 @@ ConcatInputSection *ObjCSelRefsHelper::makeSelRef(StringRef methname) { S_LITERAL_POINTERS | S_ATTR_NO_DEAD_STRIP, ArrayRef{selrefData, wordSize}, /*align=*/wordSize); - objcSelref->live = true; + assert(objcSelref->live); objcSelref->relocs.push_back({/*type=*/target->unsignedRelocType, /*pcrel=*/false, /*length=*/3, /*offset=*/0, diff --git a/lld/MachO/Writer.cpp b/lld/MachO/Writer.cpp index fe989de648d789..1c054912551e3e 100644 --- a/lld/MachO/Writer.cpp +++ b/lld/MachO/Writer.cpp @@ -1375,9 +1375,7 @@ void macho::createSyntheticSections() { segment_names::data, section_names::data, S_REGULAR, ArrayRef{arr, target->wordSize}, /*align=*/target->wordSize); - // References from dyld are not visible to us, so ensure this section is - // always treated as live. - in.imageLoaderCache->live = true; + assert(in.imageLoaderCache->live); } OutputSection *macho::firstTLVDataSection = nullptr; diff --git a/lld/test/MachO/objc-relative-method-lists-simple.s b/lld/test/MachO/objc-relative-method-lists-simple.s index 1ffec3c6241cf4..5a77085c7d93d8 100644 --- a/lld/test/MachO/objc-relative-method-lists-simple.s +++ b/lld/test/MachO/objc-relative-method-lists-simple.s @@ -8,6 +8,10 @@ # RUN: %no-lsystem-lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64 -objc_relative_method_lists # RUN: llvm-objdump --macho --objc-meta-data a64_rel_dylib.dylib | FileCheck %s --check-prefix=CHK_REL +## Test arm64 + relative method lists + dead-strip +# RUN: %no-lsystem-lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64 -objc_relative_method_lists -dead_strip +# RUN: llvm-objdump --macho --objc-meta-data a64_rel_dylib.dylib | FileCheck %s --check-prefix=CHK_REL + ## Test arm64 + traditional method lists (no relative offsets) # RUN: %no-lsystem-lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64 -no_objc_relative_method_lists # RUN: llvm-objdump --macho --objc-meta-data a64_rel_dylib.dylib | FileCheck %s --check-prefix=CHK_NO_REL From cb898e26f3321e0b861609f5fec7f7410e5b6778 Mon Sep 17 00:00:00 2001 From: Kai Sasaki Date: Thu, 28 Mar 2024 09:40:17 +0900 Subject: [PATCH 39/54] [mlir] Make the print function in CRunnerUtil platform agnostic (#86767) The platform running on Apple Silicon does not seem to support the negative nan. It causes the test failure where we explicitly specify the negative nan bit pattern and check the output printed by the CRunnerUtil function. We can make the print function in the utility platform agnostic by using the standard library functions (i.e. `std::isnan` and `std::signbit`) so that we can run the test across platforms that do not support the negative bit pattern. I have added two test cases that would fail in the Apple Silicon platform without print function changes. ``` $ uname -a Darwin Kernel Version 23.3.0: Wed Dec 20 21:30:44 PST 2023; root:xnu-10002.81.5~7/RELEASE_ARM64_T6000 arm64 ``` See: https://discourse.llvm.org/t/test-failure-of-sparse-sign-test-in-apple-silicon/77876/3 --- mlir/lib/ExecutionEngine/CRunnerUtils.cpp | 16 ++++++++++++++-- .../test-expand-math-approx.mlir | 18 +++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/mlir/lib/ExecutionEngine/CRunnerUtils.cpp b/mlir/lib/ExecutionEngine/CRunnerUtils.cpp index 48e4b8cd88b58e..41c619566b55df 100644 --- a/mlir/lib/ExecutionEngine/CRunnerUtils.cpp +++ b/mlir/lib/ExecutionEngine/CRunnerUtils.cpp @@ -51,8 +51,20 @@ void stdSort(uint64_t n, V *p) { // details of our vectors. Also useful for direct LLVM IR output. extern "C" void printI64(int64_t i) { fprintf(stdout, "%" PRId64, i); } extern "C" void printU64(uint64_t u) { fprintf(stdout, "%" PRIu64, u); } -extern "C" void printF32(float f) { fprintf(stdout, "%g", f); } -extern "C" void printF64(double d) { fprintf(stdout, "%lg", d); } +extern "C" void printF32(float f) { + if (std::isnan(f) && std::signbit(f)) { + fprintf(stdout, "-nan"); + } else { + fprintf(stdout, "%g", f); + } +} +extern "C" void printF64(double d) { + if (std::isnan(d) && std::signbit(d)) { + fprintf(stdout, "-nan"); + } else { + fprintf(stdout, "%lg", d); + } +} extern "C" void printString(char const *s) { fputs(s, stdout); } extern "C" void printOpen() { fputs("( ", stdout); } extern "C" void printClose() { fputs(" )", stdout); } diff --git a/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir b/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir index e2229a392bbf76..340ef30bf59c29 100644 --- a/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir +++ b/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir @@ -190,6 +190,12 @@ func.func @func_powff64(%a : f64, %b : f64) { return } +func.func @func_powff32(%a : f32, %b : f32) { + %r = math.powf %a, %b : f32 + vector.print %r : f32 + return +} + func.func @powf() { // CHECK-NEXT: 16 %a = arith.constant 4.0 : f64 @@ -230,7 +236,17 @@ func.func @powf() { %j = arith.constant 29385.0 : f64 %j_p = arith.constant 23598.0 : f64 call @func_powff64(%j, %j_p) : (f64, f64) -> () - return + + // CHECK-NEXT: -nan + %k = arith.constant 1.0 : f64 + %k_p = arith.constant 0xfff0000001000000 : f64 + call @func_powff64(%k, %k_p) : (f64, f64) -> () + + // CHECK-NEXT: -nan + %l = arith.constant 1.0 : f32 + %l_p = arith.constant 0xffffffff : f32 + call @func_powff32(%l, %l_p) : (f32, f32) -> () + return } // -------------------------------------------------------------------------- // From 17ab9e64464f989b3fe4a304a450581a618097a0 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Wed, 27 Mar 2024 17:44:49 -0700 Subject: [PATCH 40/54] [TSAN] Move test into Linux/ Linux specific test was introduced by #86537 --- compiler-rt/test/tsan/{ => Linux}/signal_in_futex_wait.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename compiler-rt/test/tsan/{ => Linux}/signal_in_futex_wait.cpp (99%) diff --git a/compiler-rt/test/tsan/signal_in_futex_wait.cpp b/compiler-rt/test/tsan/Linux/signal_in_futex_wait.cpp similarity index 99% rename from compiler-rt/test/tsan/signal_in_futex_wait.cpp rename to compiler-rt/test/tsan/Linux/signal_in_futex_wait.cpp index cf31e5467486ad..34d058edd526a1 100644 --- a/compiler-rt/test/tsan/signal_in_futex_wait.cpp +++ b/compiler-rt/test/tsan/Linux/signal_in_futex_wait.cpp @@ -1,6 +1,6 @@ // RUN: %clang_tsan %s -lstdc++ -o %t && %run %t 2>&1 | FileCheck %s -#include "test.h" +#include "../test.h" #include #include #include From 64f0410193490167aee186abf4de06b681476a86 Mon Sep 17 00:00:00 2001 From: Marc Auberer Date: Thu, 28 Mar 2024 02:03:24 +0100 Subject: [PATCH 41/54] [CI] Hotfix: CI runs failing due to target escaping (#86897) My patch #86877 contains a mistake. Should have read the comment. Recent buildkite runs fail because of this, so it is a bit urgent. --- .ci/monolithic-linux.sh | 2 +- .ci/monolithic-windows.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh index 35f1aacb4985f0..b347c443da677f 100755 --- a/.ci/monolithic-linux.sh +++ b/.ci/monolithic-linux.sh @@ -54,4 +54,4 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \ echo "--- ninja" # Targets are not escaped as they are passed as separate arguments. -ninja -C "${BUILD_DIR}" -k 0 "${targets}" +ninja -C "${BUILD_DIR}" -k 0 ${targets} diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh index f84c0966704e73..4fd88ea81c84a8 100755 --- a/.ci/monolithic-windows.sh +++ b/.ci/monolithic-windows.sh @@ -62,4 +62,4 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \ echo "--- ninja" # Targets are not escaped as they are passed as separate arguments. -ninja -C "${BUILD_DIR}" -k 0 "${targets}" +ninja -C "${BUILD_DIR}" -k 0 ${targets} From cc98ffb6dc6ca3e68fd939558eb8c4ddb1cc03de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Naz=C4=B1m=20Can=20Alt=C4=B1nova?= Date: Thu, 28 Mar 2024 02:09:20 +0100 Subject: [PATCH 42/54] [tsan][test] Remove some unneded debug comments in a tsan test (#86896) I introduced this test in #86537, let's remove some unneeded debugging comments. This PR was initially also moving the test to linux directory but looks like it's already done by 17ab9e64464f989b3fe4a304a450581a618097a0 . --- compiler-rt/test/tsan/Linux/signal_in_futex_wait.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/compiler-rt/test/tsan/Linux/signal_in_futex_wait.cpp b/compiler-rt/test/tsan/Linux/signal_in_futex_wait.cpp index 34d058edd526a1..3c8804aae3d09c 100644 --- a/compiler-rt/test/tsan/Linux/signal_in_futex_wait.cpp +++ b/compiler-rt/test/tsan/Linux/signal_in_futex_wait.cpp @@ -57,16 +57,13 @@ class Mutex { Mutex mutex; void *Thread(void *x) { - // fprintf(stderr, "canova here thread 0\n"); // Waiting for the futex. mutex.lock(); - // fprintf(stderr, "canova here thread 1\n"); // Finished waiting. return nullptr; } static void SigprofHandler(int signal, siginfo_t *info, void *context) { - // fprintf(stderr, "canova here sigprof handler\n"); // Unlock the futex. mutex.unlock(); } From f75eebab887903567906d22e790a3be20a2a6438 Mon Sep 17 00:00:00 2001 From: Akira Hatanaka Date: Wed, 27 Mar 2024 18:14:04 -0700 Subject: [PATCH 43/54] Revert "[CodeGen][arm64e] Add methods and data members to Address, which are needed to authenticate signed pointers (#86721)" (#86898) This reverts commit d9a685a9dd589486e882b722e513ee7b8c84870c. The commit broke ubsan bots. --- clang/lib/CodeGen/ABIInfoImpl.cpp | 10 +- clang/lib/CodeGen/Address.h | 195 +++-------------- clang/lib/CodeGen/CGAtomic.cpp | 53 +++-- clang/lib/CodeGen/CGBlocks.cpp | 34 ++- clang/lib/CodeGen/CGBlocks.h | 3 +- clang/lib/CodeGen/CGBuilder.h | 234 +++++++-------------- clang/lib/CodeGen/CGBuiltin.cpp | 173 ++++++++------- clang/lib/CodeGen/CGCUDANV.cpp | 19 +- clang/lib/CodeGen/CGCXXABI.cpp | 21 +- clang/lib/CodeGen/CGCXXABI.h | 14 +- clang/lib/CodeGen/CGCall.cpp | 171 +++++++-------- clang/lib/CodeGen/CGCall.h | 1 - clang/lib/CodeGen/CGClass.cpp | 76 +++---- clang/lib/CodeGen/CGCleanup.cpp | 110 ++++++---- clang/lib/CodeGen/CGCleanup.h | 2 +- clang/lib/CodeGen/CGCoroutine.cpp | 4 +- clang/lib/CodeGen/CGDecl.cpp | 28 ++- clang/lib/CodeGen/CGException.cpp | 19 +- clang/lib/CodeGen/CGExpr.cpp | 227 ++++++++++---------- clang/lib/CodeGen/CGExprAgg.cpp | 29 ++- clang/lib/CodeGen/CGExprCXX.cpp | 111 +++++----- clang/lib/CodeGen/CGExprConstant.cpp | 4 +- clang/lib/CodeGen/CGExprScalar.cpp | 23 +- clang/lib/CodeGen/CGNonTrivialStruct.cpp | 8 +- clang/lib/CodeGen/CGObjC.cpp | 43 ++-- clang/lib/CodeGen/CGObjCGNU.cpp | 42 ++-- clang/lib/CodeGen/CGObjCMac.cpp | 95 +++++---- clang/lib/CodeGen/CGObjCRuntime.cpp | 6 +- clang/lib/CodeGen/CGOpenMPRuntime.cpp | 194 ++++++++--------- clang/lib/CodeGen/CGOpenMPRuntime.h | 5 +- clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp | 76 ++++--- clang/lib/CodeGen/CGStmt.cpp | 8 +- clang/lib/CodeGen/CGStmtOpenMP.cpp | 87 ++++---- clang/lib/CodeGen/CGVTables.cpp | 9 +- clang/lib/CodeGen/CGValue.h | 250 +++++++++++----------- clang/lib/CodeGen/CodeGenFunction.cpp | 71 +++---- clang/lib/CodeGen/CodeGenFunction.h | 257 +++++++---------------- clang/lib/CodeGen/CodeGenModule.cpp | 2 +- clang/lib/CodeGen/CodeGenPGO.cpp | 10 +- clang/lib/CodeGen/CodeGenPGO.h | 6 +- clang/lib/CodeGen/ItaniumCXXABI.cpp | 52 +++-- clang/lib/CodeGen/MicrosoftCXXABI.cpp | 58 +++-- clang/lib/CodeGen/TargetInfo.h | 5 - clang/lib/CodeGen/Targets/NVPTX.cpp | 2 +- clang/lib/CodeGen/Targets/PPC.cpp | 11 +- clang/lib/CodeGen/Targets/Sparc.cpp | 2 +- clang/lib/CodeGen/Targets/SystemZ.cpp | 9 +- clang/lib/CodeGen/Targets/XCore.cpp | 2 +- clang/utils/TableGen/MveEmitter.cpp | 2 +- llvm/include/llvm/IR/IRBuilder.h | 1 - 50 files changed, 1234 insertions(+), 1640 deletions(-) diff --git a/clang/lib/CodeGen/ABIInfoImpl.cpp b/clang/lib/CodeGen/ABIInfoImpl.cpp index 3e34d82cb399ba..dd59101ecc81b8 100644 --- a/clang/lib/CodeGen/ABIInfoImpl.cpp +++ b/clang/lib/CodeGen/ABIInfoImpl.cpp @@ -187,7 +187,7 @@ CodeGen::emitVoidPtrDirectVAArg(CodeGenFunction &CGF, Address VAListAddr, CharUnits FullDirectSize = DirectSize.alignTo(SlotSize); Address NextPtr = CGF.Builder.CreateConstInBoundsByteGEP(Addr, FullDirectSize, "argp.next"); - CGF.Builder.CreateStore(NextPtr.emitRawPointer(CGF), VAListAddr); + CGF.Builder.CreateStore(NextPtr.getPointer(), VAListAddr); // If the argument is smaller than a slot, and this is a big-endian // target, the argument will be right-adjusted in its slot. @@ -239,8 +239,8 @@ Address CodeGen::emitMergePHI(CodeGenFunction &CGF, Address Addr1, const llvm::Twine &Name) { assert(Addr1.getType() == Addr2.getType()); llvm::PHINode *PHI = CGF.Builder.CreatePHI(Addr1.getType(), 2, Name); - PHI->addIncoming(Addr1.emitRawPointer(CGF), Block1); - PHI->addIncoming(Addr2.emitRawPointer(CGF), Block2); + PHI->addIncoming(Addr1.getPointer(), Block1); + PHI->addIncoming(Addr2.getPointer(), Block2); CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment()); return Address(PHI, Addr1.getElementType(), Align); } @@ -400,7 +400,7 @@ Address CodeGen::EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, llvm::Type *ElementTy = CGF.ConvertTypeForMem(Ty); llvm::Type *BaseTy = llvm::PointerType::getUnqual(ElementTy); llvm::Value *Addr = - CGF.Builder.CreateVAArg(VAListAddr.emitRawPointer(CGF), BaseTy); + CGF.Builder.CreateVAArg(VAListAddr.getPointer(), BaseTy); return Address(Addr, ElementTy, TyAlignForABI); } else { assert((AI.isDirect() || AI.isExtend()) && @@ -416,7 +416,7 @@ Address CodeGen::EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, "Unexpected CoerceToType seen in arginfo in generic VAArg emitter!"); Address Temp = CGF.CreateMemTemp(Ty, "varet"); - Val = CGF.Builder.CreateVAArg(VAListAddr.emitRawPointer(CGF), + Val = CGF.Builder.CreateVAArg(VAListAddr.getPointer(), CGF.ConvertTypeForMem(Ty)); CGF.Builder.CreateStore(Val, Temp); return Temp; diff --git a/clang/lib/CodeGen/Address.h b/clang/lib/CodeGen/Address.h index 35ec370a139c92..cf48df8f5e7367 100644 --- a/clang/lib/CodeGen/Address.h +++ b/clang/lib/CodeGen/Address.h @@ -15,7 +15,6 @@ #define LLVM_CLANG_LIB_CODEGEN_ADDRESS_H #include "clang/AST/CharUnits.h" -#include "clang/AST/Type.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/IR/Constants.h" #include "llvm/Support/MathExtras.h" @@ -23,41 +22,28 @@ namespace clang { namespace CodeGen { -class Address; -class CGBuilderTy; -class CodeGenFunction; -class CodeGenModule; - // Indicates whether a pointer is known not to be null. enum KnownNonNull_t { NotKnownNonNull, KnownNonNull }; -/// An abstract representation of an aligned address. This is designed to be an -/// IR-level abstraction, carrying just the information necessary to perform IR -/// operations on an address like loads and stores. In particular, it doesn't -/// carry C type information or allow the representation of things like -/// bit-fields; clients working at that level should generally be using -/// `LValue`. -/// The pointer contained in this class is known to be unsigned. -class RawAddress { +/// An aligned address. +class Address { llvm::PointerIntPair PointerAndKnownNonNull; llvm::Type *ElementType; CharUnits Alignment; protected: - RawAddress(std::nullptr_t) : ElementType(nullptr) {} + Address(std::nullptr_t) : ElementType(nullptr) {} public: - RawAddress(llvm::Value *Pointer, llvm::Type *ElementType, CharUnits Alignment, - KnownNonNull_t IsKnownNonNull = NotKnownNonNull) + Address(llvm::Value *Pointer, llvm::Type *ElementType, CharUnits Alignment, + KnownNonNull_t IsKnownNonNull = NotKnownNonNull) : PointerAndKnownNonNull(Pointer, IsKnownNonNull), ElementType(ElementType), Alignment(Alignment) { assert(Pointer != nullptr && "Pointer cannot be null"); assert(ElementType != nullptr && "Element type cannot be null"); } - inline RawAddress(Address Addr); - - static RawAddress invalid() { return RawAddress(nullptr); } + static Address invalid() { return Address(nullptr); } bool isValid() const { return PointerAndKnownNonNull.getPointer() != nullptr; } @@ -94,133 +80,6 @@ class RawAddress { return Alignment; } - /// Return address with different element type, but same pointer and - /// alignment. - RawAddress withElementType(llvm::Type *ElemTy) const { - return RawAddress(getPointer(), ElemTy, getAlignment(), isKnownNonNull()); - } - - KnownNonNull_t isKnownNonNull() const { - assert(isValid()); - return (KnownNonNull_t)PointerAndKnownNonNull.getInt(); - } -}; - -/// Like RawAddress, an abstract representation of an aligned address, but the -/// pointer contained in this class is possibly signed. -class Address { - friend class CGBuilderTy; - - // The boolean flag indicates whether the pointer is known to be non-null. - llvm::PointerIntPair Pointer; - - /// The expected IR type of the pointer. Carrying accurate element type - /// information in Address makes it more convenient to work with Address - /// values and allows frontend assertions to catch simple mistakes. - llvm::Type *ElementType = nullptr; - - CharUnits Alignment; - - /// Offset from the base pointer. - llvm::Value *Offset = nullptr; - - llvm::Value *emitRawPointerSlow(CodeGenFunction &CGF) const; - -protected: - Address(std::nullptr_t) : ElementType(nullptr) {} - -public: - Address(llvm::Value *pointer, llvm::Type *elementType, CharUnits alignment, - KnownNonNull_t IsKnownNonNull = NotKnownNonNull) - : Pointer(pointer, IsKnownNonNull), ElementType(elementType), - Alignment(alignment) { - assert(pointer != nullptr && "Pointer cannot be null"); - assert(elementType != nullptr && "Element type cannot be null"); - assert(!alignment.isZero() && "Alignment cannot be zero"); - } - - Address(llvm::Value *BasePtr, llvm::Type *ElementType, CharUnits Alignment, - llvm::Value *Offset, KnownNonNull_t IsKnownNonNull = NotKnownNonNull) - : Pointer(BasePtr, IsKnownNonNull), ElementType(ElementType), - Alignment(Alignment), Offset(Offset) {} - - Address(RawAddress RawAddr) - : Pointer(RawAddr.isValid() ? RawAddr.getPointer() : nullptr), - ElementType(RawAddr.isValid() ? RawAddr.getElementType() : nullptr), - Alignment(RawAddr.isValid() ? RawAddr.getAlignment() - : CharUnits::Zero()) {} - - static Address invalid() { return Address(nullptr); } - bool isValid() const { return Pointer.getPointer() != nullptr; } - - /// This function is used in situations where the caller is doing some sort of - /// opaque "laundering" of the pointer. - void replaceBasePointer(llvm::Value *P) { - assert(isValid() && "pointer isn't valid"); - assert(P->getType() == Pointer.getPointer()->getType() && - "Pointer's type changed"); - Pointer.setPointer(P); - assert(isValid() && "pointer is invalid after replacement"); - } - - CharUnits getAlignment() const { return Alignment; } - - void setAlignment(CharUnits Value) { Alignment = Value; } - - llvm::Value *getBasePointer() const { - assert(isValid() && "pointer isn't valid"); - return Pointer.getPointer(); - } - - /// Return the type of the pointer value. - llvm::PointerType *getType() const { - return llvm::PointerType::get( - ElementType, - llvm::cast(Pointer.getPointer()->getType()) - ->getAddressSpace()); - } - - /// Return the type of the values stored in this address. - llvm::Type *getElementType() const { - assert(isValid()); - return ElementType; - } - - /// Return the address space that this address resides in. - unsigned getAddressSpace() const { return getType()->getAddressSpace(); } - - /// Return the IR name of the pointer value. - llvm::StringRef getName() const { return Pointer.getPointer()->getName(); } - - // This function is called only in CGBuilderBaseTy::CreateElementBitCast. - void setElementType(llvm::Type *Ty) { - assert(hasOffset() && - "this funcion shouldn't be called when there is no offset"); - ElementType = Ty; - } - - /// Whether the pointer is known not to be null. - KnownNonNull_t isKnownNonNull() const { - assert(isValid()); - return (KnownNonNull_t)Pointer.getInt(); - } - - Address setKnownNonNull() { - assert(isValid()); - Pointer.setInt(KnownNonNull); - return *this; - } - - bool hasOffset() const { return Offset; } - - llvm::Value *getOffset() const { return Offset; } - - /// Return the pointer contained in this class after authenticating it and - /// adding offset to it if necessary. - llvm::Value *emitRawPointer(CodeGenFunction &CGF) const { - return getBasePointer(); - } - /// Return address with different pointer, but same element type and /// alignment. Address withPointer(llvm::Value *NewPointer, @@ -232,59 +91,61 @@ class Address { /// Return address with different alignment, but same pointer and element /// type. Address withAlignment(CharUnits NewAlignment) const { - return Address(Pointer.getPointer(), getElementType(), NewAlignment, + return Address(getPointer(), getElementType(), NewAlignment, isKnownNonNull()); } /// Return address with different element type, but same pointer and /// alignment. Address withElementType(llvm::Type *ElemTy) const { - if (!hasOffset()) - return Address(getBasePointer(), ElemTy, getAlignment(), nullptr, - isKnownNonNull()); - Address A(*this); - A.ElementType = ElemTy; - return A; + return Address(getPointer(), ElemTy, getAlignment(), isKnownNonNull()); } -}; -inline RawAddress::RawAddress(Address Addr) - : PointerAndKnownNonNull(Addr.isValid() ? Addr.getBasePointer() : nullptr, - Addr.isValid() ? Addr.isKnownNonNull() - : NotKnownNonNull), - ElementType(Addr.isValid() ? Addr.getElementType() : nullptr), - Alignment(Addr.isValid() ? Addr.getAlignment() : CharUnits::Zero()) {} + /// Whether the pointer is known not to be null. + KnownNonNull_t isKnownNonNull() const { + assert(isValid()); + return (KnownNonNull_t)PointerAndKnownNonNull.getInt(); + } + + /// Set the non-null bit. + Address setKnownNonNull() { + assert(isValid()); + PointerAndKnownNonNull.setInt(true); + return *this; + } +}; /// A specialization of Address that requires the address to be an /// LLVM Constant. -class ConstantAddress : public RawAddress { - ConstantAddress(std::nullptr_t) : RawAddress(nullptr) {} +class ConstantAddress : public Address { + ConstantAddress(std::nullptr_t) : Address(nullptr) {} public: ConstantAddress(llvm::Constant *pointer, llvm::Type *elementType, CharUnits alignment) - : RawAddress(pointer, elementType, alignment) {} + : Address(pointer, elementType, alignment) {} static ConstantAddress invalid() { return ConstantAddress(nullptr); } llvm::Constant *getPointer() const { - return llvm::cast(RawAddress::getPointer()); + return llvm::cast(Address::getPointer()); } ConstantAddress withElementType(llvm::Type *ElemTy) const { return ConstantAddress(getPointer(), ElemTy, getAlignment()); } - static bool isaImpl(RawAddress addr) { + static bool isaImpl(Address addr) { return llvm::isa(addr.getPointer()); } - static ConstantAddress castImpl(RawAddress addr) { + static ConstantAddress castImpl(Address addr) { return ConstantAddress(llvm::cast(addr.getPointer()), addr.getElementType(), addr.getAlignment()); } }; + } // Present a minimal LLVM-like casting interface. diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp index 56198385de9dcb..fb03d013e8afc7 100644 --- a/clang/lib/CodeGen/CGAtomic.cpp +++ b/clang/lib/CodeGen/CGAtomic.cpp @@ -80,7 +80,7 @@ namespace { AtomicSizeInBits = C.toBits( C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) .alignTo(lvalue.getAlignment())); - llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF); + llvm::Value *BitFieldPtr = lvalue.getBitFieldPointer(); auto OffsetInChars = (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * lvalue.getAlignment(); @@ -139,13 +139,13 @@ namespace { const LValue &getAtomicLValue() const { return LVal; } llvm::Value *getAtomicPointer() const { if (LVal.isSimple()) - return LVal.emitRawPointer(CGF); + return LVal.getPointer(CGF); else if (LVal.isBitField()) - return LVal.getRawBitFieldPointer(CGF); + return LVal.getBitFieldPointer(); else if (LVal.isVectorElt()) - return LVal.getRawVectorPointer(CGF); + return LVal.getVectorPointer(); assert(LVal.isExtVectorElt()); - return LVal.getRawExtVectorPointer(CGF); + return LVal.getExtVectorPointer(); } Address getAtomicAddress() const { llvm::Type *ElTy; @@ -368,7 +368,7 @@ bool AtomicInfo::emitMemSetZeroIfNecessary() const { return false; CGF.Builder.CreateMemSet( - addr.emitRawPointer(CGF), llvm::ConstantInt::get(CGF.Int8Ty, 0), + addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0), CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), LVal.getAlignment().getAsAlign()); return true; @@ -1055,8 +1055,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { return getTargetHooks().performAddrSpaceCast( *this, V, AS, LangAS::opencl_generic, DestType, false); }; - - Args.add(RValue::get(CastToGenericAddrSpace(Ptr.emitRawPointer(*this), + Args.add(RValue::get(CastToGenericAddrSpace(Ptr.getPointer(), E->getPtr()->getType())), getContext().VoidPtrTy); @@ -1087,10 +1086,10 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { LibCallName = "__atomic_compare_exchange"; RetTy = getContext().BoolTy; HaveRetTy = true; - Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), + Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(), E->getVal1()->getType())), getContext().VoidPtrTy); - Args.add(RValue::get(CastToGenericAddrSpace(Val2.emitRawPointer(*this), + Args.add(RValue::get(CastToGenericAddrSpace(Val2.getPointer(), E->getVal2()->getType())), getContext().VoidPtrTy); Args.add(RValue::get(Order), getContext().IntTy); @@ -1106,7 +1105,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__scoped_atomic_exchange: case AtomicExpr::AO__scoped_atomic_exchange_n: LibCallName = "__atomic_exchange"; - Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), + Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(), E->getVal1()->getType())), getContext().VoidPtrTy); break; @@ -1121,7 +1120,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { LibCallName = "__atomic_store"; RetTy = getContext().VoidTy; HaveRetTy = true; - Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), + Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(), E->getVal1()->getType())), getContext().VoidPtrTy); break; @@ -1200,8 +1199,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { if (!HaveRetTy) { // Value is returned through parameter before the order. RetTy = getContext().VoidTy; - Args.add(RValue::get( - CastToGenericAddrSpace(Dest.emitRawPointer(*this), RetTy)), + Args.add(RValue::get(CastToGenericAddrSpace(Dest.getPointer(), RetTy)), getContext().VoidPtrTy); } // Order is always the last parameter. @@ -1515,7 +1513,7 @@ RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, } else TempAddr = CreateTempAlloca(); - EmitAtomicLoadLibcall(TempAddr.emitRawPointer(CGF), AO, IsVolatile); + EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); // Okay, turn that back into the original value or whole atomic (for // non-simple lvalues) type. @@ -1675,9 +1673,9 @@ std::pair AtomicInfo::EmitAtomicCompareExchange( if (shouldUseLibcall()) { // Produce a source address. Address ExpectedAddr = materializeRValue(Expected); - llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); - llvm::Value *DesiredPtr = materializeRValue(Desired).emitRawPointer(CGF); - auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, + Address DesiredAddr = materializeRValue(Desired); + auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), + DesiredAddr.getPointer(), Success, Failure); return std::make_pair( convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), @@ -1759,7 +1757,7 @@ void AtomicInfo::EmitAtomicUpdateLibcall( Address ExpectedAddr = CreateTempAlloca(); - EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile); + EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); auto *ContBB = CGF.createBasicBlock("atomic_cont"); auto *ExitBB = CGF.createBasicBlock("atomic_exit"); CGF.EmitBlock(ContBB); @@ -1773,10 +1771,10 @@ void AtomicInfo::EmitAtomicUpdateLibcall( AggValueSlot::ignored(), SourceLocation(), /*AsValue=*/false); EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); - llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); - llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF); auto *Res = - EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure); + EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), + DesiredAddr.getPointer(), + AO, Failure); CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); CGF.EmitBlock(ExitBB, /*IsFinished=*/true); } @@ -1845,7 +1843,7 @@ void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, Address ExpectedAddr = CreateTempAlloca(); - EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile); + EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); auto *ContBB = CGF.createBasicBlock("atomic_cont"); auto *ExitBB = CGF.createBasicBlock("atomic_exit"); CGF.EmitBlock(ContBB); @@ -1856,10 +1854,10 @@ void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, CGF.Builder.CreateStore(OldVal, DesiredAddr); } EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); - llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); - llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF); auto *Res = - EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure); + EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), + DesiredAddr.getPointer(), + AO, Failure); CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); CGF.EmitBlock(ExitBB, /*IsFinished=*/true); } @@ -1959,8 +1957,7 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, args.add(RValue::get(atomics.getAtomicSizeValue()), getContext().getSizeType()); args.add(RValue::get(atomics.getAtomicPointer()), getContext().VoidPtrTy); - args.add(RValue::get(srcAddr.emitRawPointer(*this)), - getContext().VoidPtrTy); + args.add(RValue::get(srcAddr.getPointer()), getContext().VoidPtrTy); args.add( RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), getContext().IntTy); diff --git a/clang/lib/CodeGen/CGBlocks.cpp b/clang/lib/CodeGen/CGBlocks.cpp index a01f2c7c979840..ad0b50d799618e 100644 --- a/clang/lib/CodeGen/CGBlocks.cpp +++ b/clang/lib/CodeGen/CGBlocks.cpp @@ -36,8 +36,7 @@ CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name) : Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false), NoEscape(false), HasCXXObject(false), UsesStret(false), HasCapturedVariableLayout(false), CapturesNonExternalType(false), - LocalAddress(RawAddress::invalid()), StructureType(nullptr), - Block(block) { + LocalAddress(Address::invalid()), StructureType(nullptr), Block(block) { // Skip asm prefix, if any. 'name' is usually taken directly from // the mangled name of the enclosing function. @@ -795,7 +794,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { // Otherwise, we have to emit this as a local block. - RawAddress blockAddr = blockInfo.LocalAddress; + Address blockAddr = blockInfo.LocalAddress; assert(blockAddr.isValid() && "block has no address!"); llvm::Constant *isa; @@ -940,7 +939,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { if (CI.isNested()) byrefPointer = Builder.CreateLoad(src, "byref.capture"); else - byrefPointer = src.emitRawPointer(*this); + byrefPointer = src.getPointer(); // Write that void* into the capture field. Builder.CreateStore(byrefPointer, blockField); @@ -962,10 +961,10 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { } // If it's a reference variable, copy the reference into the block field. - } else if (auto refType = type->getAs()) { - Builder.CreateStore(src.emitRawPointer(*this), blockField); + } else if (type->isReferenceType()) { + Builder.CreateStore(src.getPointer(), blockField); - // If type is const-qualified, copy the value into the block field. + // If type is const-qualified, copy the value into the block field. } else if (type.isConstQualified() && type.getObjCLifetime() == Qualifiers::OCL_Strong && CGM.getCodeGenOpts().OptimizationLevel != 0) { @@ -1378,7 +1377,7 @@ void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D, // Allocate a stack slot like for any local variable to guarantee optimal // debug info at -O0. The mem2reg pass will eliminate it when optimizing. - RawAddress alloc = CreateMemTemp(D->getType(), D->getName() + ".addr"); + Address alloc = CreateMemTemp(D->getType(), D->getName() + ".addr"); Builder.CreateStore(arg, alloc); if (CGDebugInfo *DI = getDebugInfo()) { if (CGM.getCodeGenOpts().hasReducedDebugInfo()) { @@ -1498,7 +1497,7 @@ llvm::Function *CodeGenFunction::GenerateBlockFunction( // frame setup instruction by llvm::DwarfDebug::beginFunction(). auto NL = ApplyDebugLocation::CreateEmpty(*this); Builder.CreateStore(BlockPointer, Alloca); - BlockPointerDbgLoc = Alloca.emitRawPointer(*this); + BlockPointerDbgLoc = Alloca.getPointer(); } // If we have a C++ 'this' reference, go ahead and force it into @@ -1558,8 +1557,8 @@ llvm::Function *CodeGenFunction::GenerateBlockFunction( const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable); if (capture.isConstant()) { auto addr = LocalDeclMap.find(variable)->second; - (void)DI->EmitDeclareOfAutoVariable( - variable, addr.emitRawPointer(*this), Builder); + (void)DI->EmitDeclareOfAutoVariable(variable, addr.getPointer(), + Builder); continue; } @@ -1663,7 +1662,7 @@ struct CallBlockRelease final : EHScopeStack::Cleanup { if (LoadBlockVarAddr) { BlockVarAddr = CGF.Builder.CreateLoad(Addr); } else { - BlockVarAddr = Addr.emitRawPointer(CGF); + BlockVarAddr = Addr.getPointer(); } CGF.BuildBlockRelease(BlockVarAddr, FieldFlags, CanThrow); @@ -1963,15 +1962,13 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) { // it. It's not quite worth the annoyance to avoid creating it in the // first place. if (!needsEHCleanup(captureType.isDestructedType())) - if (auto *I = - cast_or_null(dstField.getBasePointer())) - I->eraseFromParent(); + cast(dstField.getPointer())->eraseFromParent(); } break; } case BlockCaptureEntityKind::BlockObject: { llvm::Value *srcValue = Builder.CreateLoad(srcField, "blockcopy.src"); - llvm::Value *dstAddr = dstField.emitRawPointer(*this); + llvm::Value *dstAddr = dstField.getPointer(); llvm::Value *args[] = { dstAddr, srcValue, llvm::ConstantInt::get(Int32Ty, flags.getBitMask()) }; @@ -2142,7 +2139,7 @@ class ObjectByrefHelpers final : public BlockByrefHelpers { llvm::Value *flagsVal = llvm::ConstantInt::get(CGF.Int32Ty, flags); llvm::FunctionCallee fn = CGF.CGM.getBlockObjectAssign(); - llvm::Value *args[] = {destField.emitRawPointer(CGF), srcValue, flagsVal}; + llvm::Value *args[] = { destField.getPointer(), srcValue, flagsVal }; CGF.EmitNounwindRuntimeCall(fn, args); } @@ -2699,8 +2696,7 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) { storeHeaderField(V, getPointerSize(), "byref.isa"); // Store the address of the variable into its own forwarding pointer. - storeHeaderField(addr.emitRawPointer(*this), getPointerSize(), - "byref.forwarding"); + storeHeaderField(addr.getPointer(), getPointerSize(), "byref.forwarding"); // Blocks ABI: // c) the flags field is set to either 0 if no helper functions are diff --git a/clang/lib/CodeGen/CGBlocks.h b/clang/lib/CodeGen/CGBlocks.h index 8d10c4f69b2026..4ef1ae9f33655c 100644 --- a/clang/lib/CodeGen/CGBlocks.h +++ b/clang/lib/CodeGen/CGBlocks.h @@ -271,8 +271,7 @@ class CGBlockInfo { /// The block's captures. Non-constant captures are sorted by their offsets. llvm::SmallVector SortedCaptures; - // Currently we assume that block-pointer types are never signed. - RawAddress LocalAddress; + Address LocalAddress; llvm::StructType *StructureType; const BlockDecl *Block; const BlockExpr *BlockExpression; diff --git a/clang/lib/CodeGen/CGBuilder.h b/clang/lib/CodeGen/CGBuilder.h index 6dd9da7c4cadef..bf5ab171d720d9 100644 --- a/clang/lib/CodeGen/CGBuilder.h +++ b/clang/lib/CodeGen/CGBuilder.h @@ -10,9 +10,7 @@ #define LLVM_CLANG_LIB_CODEGEN_CGBUILDER_H #include "Address.h" -#include "CGValue.h" #include "CodeGenTypeCache.h" -#include "llvm/Analysis/Utils/Local.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Type.h" @@ -20,15 +18,12 @@ namespace clang { namespace CodeGen { -class CGBuilderTy; class CodeGenFunction; /// This is an IRBuilder insertion helper that forwards to /// CodeGenFunction::InsertHelper, which adds necessary metadata to /// instructions. class CGBuilderInserter final : public llvm::IRBuilderDefaultInserter { - friend CGBuilderTy; - public: CGBuilderInserter() = default; explicit CGBuilderInserter(CodeGenFunction *CGF) : CGF(CGF) {} @@ -48,42 +43,10 @@ typedef llvm::IRBuilder CGBuilderBaseTy; class CGBuilderTy : public CGBuilderBaseTy { - friend class Address; - /// Storing a reference to the type cache here makes it a lot easier /// to build natural-feeling, target-specific IR. const CodeGenTypeCache &TypeCache; - CodeGenFunction *getCGF() const { return getInserter().CGF; } - - llvm::Value *emitRawPointerFromAddress(Address Addr) const { - return Addr.getBasePointer(); - } - - template - Address createConstGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1, - const llvm::Twine &Name) { - const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); - llvm::GetElementPtrInst *GEP; - if (IsInBounds) - GEP = cast(CreateConstInBoundsGEP2_32( - Addr.getElementType(), emitRawPointerFromAddress(Addr), Idx0, Idx1, - Name)); - else - GEP = cast(CreateConstGEP2_32( - Addr.getElementType(), emitRawPointerFromAddress(Addr), Idx0, Idx1, - Name)); - llvm::APInt Offset( - DL.getIndexSizeInBits(Addr.getType()->getPointerAddressSpace()), 0, - /*isSigned=*/true); - if (!GEP->accumulateConstantOffset(DL, Offset)) - llvm_unreachable("offset of GEP with constants is always computable"); - return Address(GEP, GEP->getResultElementType(), - Addr.getAlignment().alignmentAtOffset( - CharUnits::fromQuantity(Offset.getSExtValue())), - IsInBounds ? Addr.isKnownNonNull() : NotKnownNonNull); - } - public: CGBuilderTy(const CodeGenTypeCache &TypeCache, llvm::LLVMContext &C) : CGBuilderBaseTy(C), TypeCache(TypeCache) {} @@ -106,22 +69,20 @@ class CGBuilderTy : public CGBuilderBaseTy { // Note that we intentionally hide the CreateLoad APIs that don't // take an alignment. llvm::LoadInst *CreateLoad(Address Addr, const llvm::Twine &Name = "") { - return CreateAlignedLoad(Addr.getElementType(), - emitRawPointerFromAddress(Addr), + return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(), Addr.getAlignment().getAsAlign(), Name); } llvm::LoadInst *CreateLoad(Address Addr, const char *Name) { // This overload is required to prevent string literals from // ending up in the IsVolatile overload. - return CreateAlignedLoad(Addr.getElementType(), - emitRawPointerFromAddress(Addr), + return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(), Addr.getAlignment().getAsAlign(), Name); } llvm::LoadInst *CreateLoad(Address Addr, bool IsVolatile, const llvm::Twine &Name = "") { - return CreateAlignedLoad( - Addr.getElementType(), emitRawPointerFromAddress(Addr), - Addr.getAlignment().getAsAlign(), IsVolatile, Name); + return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(), + Addr.getAlignment().getAsAlign(), IsVolatile, + Name); } using CGBuilderBaseTy::CreateAlignedLoad; @@ -135,7 +96,7 @@ class CGBuilderTy : public CGBuilderBaseTy { // take an alignment. llvm::StoreInst *CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile = false) { - return CreateAlignedStore(Val, emitRawPointerFromAddress(Addr), + return CreateAlignedStore(Val, Addr.getPointer(), Addr.getAlignment().getAsAlign(), IsVolatile); } @@ -171,41 +132,33 @@ class CGBuilderTy : public CGBuilderBaseTy { llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID = llvm::SyncScope::System) { return CGBuilderBaseTy::CreateAtomicCmpXchg( - Addr.emitRawPointer(*getCGF()), Cmp, New, - Addr.getAlignment().getAsAlign(), SuccessOrdering, FailureOrdering, - SSID); + Addr.getPointer(), Cmp, New, Addr.getAlignment().getAsAlign(), + SuccessOrdering, FailureOrdering, SSID); } llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID = llvm::SyncScope::System) { - return CGBuilderBaseTy::CreateAtomicRMW( - Op, Addr.emitRawPointer(*getCGF()), Val, - Addr.getAlignment().getAsAlign(), Ordering, SSID); + return CGBuilderBaseTy::CreateAtomicRMW(Op, Addr.getPointer(), Val, + Addr.getAlignment().getAsAlign(), + Ordering, SSID); } using CGBuilderBaseTy::CreateAddrSpaceCast; Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, - llvm::Type *ElementTy, const llvm::Twine &Name = "") { - if (!Addr.hasOffset()) - return Address(CreateAddrSpaceCast(Addr.getBasePointer(), Ty, Name), - ElementTy, Addr.getAlignment(), nullptr, - Addr.isKnownNonNull()); - // Eagerly force a raw address if these is an offset. - return RawAddress( - CreateAddrSpaceCast(Addr.emitRawPointer(*getCGF()), Ty, Name), - ElementTy, Addr.getAlignment(), Addr.isKnownNonNull()); + return Addr.withPointer(CreateAddrSpaceCast(Addr.getPointer(), Ty, Name), + Addr.isKnownNonNull()); } using CGBuilderBaseTy::CreatePointerBitCastOrAddrSpaceCast; Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name = "") { - if (Addr.getType()->getAddressSpace() == Ty->getPointerAddressSpace()) - return Addr.withElementType(ElementTy); - return CreateAddrSpaceCast(Addr, Ty, ElementTy, Name); + llvm::Value *Ptr = + CreatePointerBitCastOrAddrSpaceCast(Addr.getPointer(), Ty, Name); + return Address(Ptr, ElementTy, Addr.getAlignment(), Addr.isKnownNonNull()); } /// Given @@ -223,11 +176,10 @@ class CGBuilderTy : public CGBuilderBaseTy { const llvm::StructLayout *Layout = DL.getStructLayout(ElTy); auto Offset = CharUnits::fromQuantity(Layout->getElementOffset(Index)); - return Address(CreateStructGEP(Addr.getElementType(), Addr.getBasePointer(), - Index, Name), - ElTy->getElementType(Index), - Addr.getAlignment().alignmentAtOffset(Offset), - Addr.isKnownNonNull()); + return Address( + CreateStructGEP(Addr.getElementType(), Addr.getPointer(), Index, Name), + ElTy->getElementType(Index), + Addr.getAlignment().alignmentAtOffset(Offset), Addr.isKnownNonNull()); } /// Given @@ -246,7 +198,7 @@ class CGBuilderTy : public CGBuilderBaseTy { CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy->getElementType())); return Address( - CreateInBoundsGEP(Addr.getElementType(), Addr.getBasePointer(), + CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), {getSize(CharUnits::Zero()), getSize(Index)}, Name), ElTy->getElementType(), Addr.getAlignment().alignmentAtOffset(Index * EltSize), @@ -264,10 +216,10 @@ class CGBuilderTy : public CGBuilderBaseTy { const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy)); - return Address( - CreateInBoundsGEP(ElTy, Addr.getBasePointer(), getSize(Index), Name), - ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize), - Addr.isKnownNonNull()); + return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), + getSize(Index), Name), + ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize), + Addr.isKnownNonNull()); } /// Given @@ -277,133 +229,110 @@ class CGBuilderTy : public CGBuilderBaseTy { /// where i64 is actually the target word size. Address CreateConstGEP(Address Addr, uint64_t Index, const llvm::Twine &Name = "") { - llvm::Type *ElTy = Addr.getElementType(); const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); - CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy)); + CharUnits EltSize = + CharUnits::fromQuantity(DL.getTypeAllocSize(Addr.getElementType())); - return Address(CreateGEP(ElTy, Addr.getBasePointer(), getSize(Index), Name), + return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(), + getSize(Index), Name), Addr.getElementType(), - Addr.getAlignment().alignmentAtOffset(Index * EltSize)); + Addr.getAlignment().alignmentAtOffset(Index * EltSize), + NotKnownNonNull); } /// Create GEP with single dynamic index. The address alignment is reduced /// according to the element size. using CGBuilderBaseTy::CreateGEP; - Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, + Address CreateGEP(Address Addr, llvm::Value *Index, const llvm::Twine &Name = "") { const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(Addr.getElementType())); return Address( - CreateGEP(Addr.getElementType(), Addr.emitRawPointer(CGF), Index, Name), + CreateGEP(Addr.getElementType(), Addr.getPointer(), Index, Name), Addr.getElementType(), - Addr.getAlignment().alignmentOfArrayElement(EltSize)); + Addr.getAlignment().alignmentOfArrayElement(EltSize), NotKnownNonNull); } /// Given a pointer to i8, adjust it by a given constant offset. Address CreateConstInBoundsByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name = "") { assert(Addr.getElementType() == TypeCache.Int8Ty); - return Address( - CreateInBoundsGEP(Addr.getElementType(), Addr.getBasePointer(), - getSize(Offset), Name), - Addr.getElementType(), Addr.getAlignment().alignmentAtOffset(Offset), - Addr.isKnownNonNull()); + return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), + getSize(Offset), Name), + Addr.getElementType(), + Addr.getAlignment().alignmentAtOffset(Offset), + Addr.isKnownNonNull()); } - Address CreateConstByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name = "") { assert(Addr.getElementType() == TypeCache.Int8Ty); - return Address(CreateGEP(Addr.getElementType(), Addr.getBasePointer(), + return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(), getSize(Offset), Name), Addr.getElementType(), - Addr.getAlignment().alignmentAtOffset(Offset)); + Addr.getAlignment().alignmentAtOffset(Offset), + NotKnownNonNull); } using CGBuilderBaseTy::CreateConstInBoundsGEP2_32; Address CreateConstInBoundsGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1, const llvm::Twine &Name = "") { - return createConstGEP2_32(Addr, Idx0, Idx1, Name); - } - - using CGBuilderBaseTy::CreateConstGEP2_32; - Address CreateConstGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1, - const llvm::Twine &Name = "") { - return createConstGEP2_32(Addr, Idx0, Idx1, Name); - } - - Address CreateGEP(Address Addr, ArrayRef IdxList, - llvm::Type *ElementType, CharUnits Align, - const Twine &Name = "") { - llvm::Value *Ptr = emitRawPointerFromAddress(Addr); - return RawAddress(CreateGEP(Addr.getElementType(), Ptr, IdxList, Name), - ElementType, Align); - } - - using CGBuilderBaseTy::CreateInBoundsGEP; - Address CreateInBoundsGEP(Address Addr, ArrayRef IdxList, - llvm::Type *ElementType, CharUnits Align, - const Twine &Name = "") { - return RawAddress(CreateInBoundsGEP(Addr.getElementType(), - emitRawPointerFromAddress(Addr), - IdxList, Name), - ElementType, Align, Addr.isKnownNonNull()); - } + const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); - using CGBuilderBaseTy::CreateIsNull; - llvm::Value *CreateIsNull(Address Addr, const Twine &Name = "") { - if (!Addr.hasOffset()) - return CreateIsNull(Addr.getBasePointer(), Name); - // The pointer isn't null if Addr has an offset since offsets can always - // be applied inbound. - return llvm::ConstantInt::getFalse(Context); + auto *GEP = cast(CreateConstInBoundsGEP2_32( + Addr.getElementType(), Addr.getPointer(), Idx0, Idx1, Name)); + llvm::APInt Offset( + DL.getIndexSizeInBits(Addr.getType()->getPointerAddressSpace()), 0, + /*isSigned=*/true); + if (!GEP->accumulateConstantOffset(DL, Offset)) + llvm_unreachable("offset of GEP with constants is always computable"); + return Address(GEP, GEP->getResultElementType(), + Addr.getAlignment().alignmentAtOffset( + CharUnits::fromQuantity(Offset.getSExtValue())), + Addr.isKnownNonNull()); } using CGBuilderBaseTy::CreateMemCpy; llvm::CallInst *CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile = false) { - llvm::Value *DestPtr = emitRawPointerFromAddress(Dest); - llvm::Value *SrcPtr = emitRawPointerFromAddress(Src); - return CreateMemCpy(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr, - Src.getAlignment().getAsAlign(), Size, IsVolatile); + return CreateMemCpy(Dest.getPointer(), Dest.getAlignment().getAsAlign(), + Src.getPointer(), Src.getAlignment().getAsAlign(), Size, + IsVolatile); } llvm::CallInst *CreateMemCpy(Address Dest, Address Src, uint64_t Size, bool IsVolatile = false) { - llvm::Value *DestPtr = emitRawPointerFromAddress(Dest); - llvm::Value *SrcPtr = emitRawPointerFromAddress(Src); - return CreateMemCpy(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr, - Src.getAlignment().getAsAlign(), Size, IsVolatile); + return CreateMemCpy(Dest.getPointer(), Dest.getAlignment().getAsAlign(), + Src.getPointer(), Src.getAlignment().getAsAlign(), Size, + IsVolatile); } using CGBuilderBaseTy::CreateMemCpyInline; llvm::CallInst *CreateMemCpyInline(Address Dest, Address Src, uint64_t Size) { - llvm::Value *DestPtr = emitRawPointerFromAddress(Dest); - llvm::Value *SrcPtr = emitRawPointerFromAddress(Src); - return CreateMemCpyInline(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr, - Src.getAlignment().getAsAlign(), getInt64(Size)); + return CreateMemCpyInline( + Dest.getPointer(), Dest.getAlignment().getAsAlign(), Src.getPointer(), + Src.getAlignment().getAsAlign(), getInt64(Size)); } using CGBuilderBaseTy::CreateMemMove; llvm::CallInst *CreateMemMove(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile = false) { - llvm::Value *DestPtr = emitRawPointerFromAddress(Dest); - llvm::Value *SrcPtr = emitRawPointerFromAddress(Src); - return CreateMemMove(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr, - Src.getAlignment().getAsAlign(), Size, IsVolatile); + return CreateMemMove(Dest.getPointer(), Dest.getAlignment().getAsAlign(), + Src.getPointer(), Src.getAlignment().getAsAlign(), + Size, IsVolatile); } using CGBuilderBaseTy::CreateMemSet; llvm::CallInst *CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile = false) { - return CreateMemSet(emitRawPointerFromAddress(Dest), Value, Size, + return CreateMemSet(Dest.getPointer(), Value, Size, Dest.getAlignment().getAsAlign(), IsVolatile); } using CGBuilderBaseTy::CreateMemSetInline; llvm::CallInst *CreateMemSetInline(Address Dest, llvm::Value *Value, uint64_t Size) { - return CreateMemSetInline(emitRawPointerFromAddress(Dest), + return CreateMemSetInline(Dest.getPointer(), Dest.getAlignment().getAsAlign(), Value, getInt64(Size)); } @@ -417,31 +346,16 @@ class CGBuilderTy : public CGBuilderBaseTy { const llvm::StructLayout *Layout = DL.getStructLayout(ElTy); auto Offset = CharUnits::fromQuantity(Layout->getElementOffset(Index)); - return Address( - CreatePreserveStructAccessIndex(ElTy, emitRawPointerFromAddress(Addr), - Index, FieldIndex, DbgInfo), - ElTy->getElementType(Index), - Addr.getAlignment().alignmentAtOffset(Offset)); - } - - using CGBuilderBaseTy::CreatePreserveUnionAccessIndex; - Address CreatePreserveUnionAccessIndex(Address Addr, unsigned FieldIndex, - llvm::MDNode *DbgInfo) { - Addr.replaceBasePointer(CreatePreserveUnionAccessIndex( - Addr.getBasePointer(), FieldIndex, DbgInfo)); - return Addr; + return Address(CreatePreserveStructAccessIndex(ElTy, Addr.getPointer(), + Index, FieldIndex, DbgInfo), + ElTy->getElementType(Index), + Addr.getAlignment().alignmentAtOffset(Offset)); } using CGBuilderBaseTy::CreateLaunderInvariantGroup; Address CreateLaunderInvariantGroup(Address Addr) { - Addr.replaceBasePointer(CreateLaunderInvariantGroup(Addr.getBasePointer())); - return Addr; - } - - using CGBuilderBaseTy::CreateStripInvariantGroup; - Address CreateStripInvariantGroup(Address Addr) { - Addr.replaceBasePointer(CreateStripInvariantGroup(Addr.getBasePointer())); - return Addr; + return Addr.withPointer(CreateLaunderInvariantGroup(Addr.getPointer()), + Addr.isKnownNonNull()); } }; diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index 5ab5917c0c8da7..fdb517eb254d3b 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -2117,9 +2117,9 @@ llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction( auto AL = ApplyDebugLocation::CreateArtificial(*this); CharUnits Offset; - Address BufAddr = makeNaturalAddressForPointer( - Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"), Ctx.VoidTy, - BufferAlignment); + Address BufAddr = + Address(Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"), Int8Ty, + BufferAlignment); Builder.CreateStore(Builder.getInt8(Layout.getSummaryByte()), Builder.CreateConstByteGEP(BufAddr, Offset++, "summary")); Builder.CreateStore(Builder.getInt8(Layout.getNumArgsByte()), @@ -2162,7 +2162,7 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) { // Ignore argument 1, the format string. It is not currently used. CallArgList Args; - Args.add(RValue::get(BufAddr.emitRawPointer(*this)), Ctx.VoidPtrTy); + Args.add(RValue::get(BufAddr.getPointer()), Ctx.VoidPtrTy); for (const auto &Item : Layout.Items) { int Size = Item.getSizeByte(); @@ -2202,8 +2202,8 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) { if (!isa(ArgVal)) { CleanupKind Cleanup = getARCCleanupKind(); QualType Ty = TheExpr->getType(); - RawAddress Alloca = RawAddress::invalid(); - RawAddress Addr = CreateMemTemp(Ty, "os.log.arg", &Alloca); + Address Alloca = Address::invalid(); + Address Addr = CreateMemTemp(Ty, "os.log.arg", &Alloca); ArgVal = EmitARCRetain(Ty, ArgVal); Builder.CreateStore(ArgVal, Addr); pushLifetimeExtendedDestroy(Cleanup, Alloca, Ty, @@ -2236,7 +2236,7 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) { llvm::Function *F = CodeGenFunction(CGM).generateBuiltinOSLogHelperFunction( Layout, BufAddr.getAlignment()); EmitCall(FI, CGCallee::forDirect(F), ReturnValueSlot(), Args); - return RValue::get(BufAddr, *this); + return RValue::get(BufAddr.getPointer()); } static bool isSpecialUnsignedMultiplySignedResult( @@ -2984,7 +2984,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, // Check NonnullAttribute/NullabilityArg and Alignment. auto EmitArgCheck = [&](TypeCheckKind Kind, Address A, const Expr *Arg, unsigned ParmNum) { - Value *Val = A.emitRawPointer(*this); + Value *Val = A.getPointer(); EmitNonNullArgCheck(RValue::get(Val), Arg->getType(), Arg->getExprLoc(), FD, ParmNum); @@ -3013,12 +3013,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_va_end: EmitVAStartEnd(BuiltinID == Builtin::BI__va_start ? EmitScalarExpr(E->getArg(0)) - : EmitVAListRef(E->getArg(0)).emitRawPointer(*this), + : EmitVAListRef(E->getArg(0)).getPointer(), BuiltinID != Builtin::BI__builtin_va_end); return RValue::get(nullptr); case Builtin::BI__builtin_va_copy: { - Value *DstPtr = EmitVAListRef(E->getArg(0)).emitRawPointer(*this); - Value *SrcPtr = EmitVAListRef(E->getArg(1)).emitRawPointer(*this); + Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer(); + Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer(); Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy, {DstPtr->getType()}), {DstPtr, SrcPtr}); return RValue::get(nullptr); @@ -3849,13 +3849,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified(); Address Src = EmitPointerWithAlignment(E->getArg(0)); - EmitNonNullArgCheck(RValue::get(Src.emitRawPointer(*this)), - E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, - 0); + EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(0)->getType(), + E->getArg(0)->getExprLoc(), FD, 0); Value *Result = MB.CreateColumnMajorLoad( - Src.getElementType(), Src.emitRawPointer(*this), + Src.getElementType(), Src.getPointer(), Align(Src.getAlignment().getQuantity()), Stride, IsVolatile, - ResultTy->getNumRows(), ResultTy->getNumColumns(), "matrix"); + ResultTy->getNumRows(), ResultTy->getNumColumns(), + "matrix"); return RValue::get(Result); } @@ -3870,13 +3870,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, assert(PtrTy && "arg1 must be of pointer type"); bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified(); - EmitNonNullArgCheck(RValue::get(Dst.emitRawPointer(*this)), - E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD, - 0); + EmitNonNullArgCheck(RValue::get(Dst.getPointer()), E->getArg(1)->getType(), + E->getArg(1)->getExprLoc(), FD, 0); Value *Result = MB.CreateColumnMajorStore( - Matrix, Dst.emitRawPointer(*this), - Align(Dst.getAlignment().getQuantity()), Stride, IsVolatile, - MatrixTy->getNumRows(), MatrixTy->getNumColumns()); + Matrix, Dst.getPointer(), Align(Dst.getAlignment().getQuantity()), + Stride, IsVolatile, MatrixTy->getNumRows(), MatrixTy->getNumColumns()); return RValue::get(Result); } @@ -4035,7 +4033,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_bzero: { Address Dest = EmitPointerWithAlignment(E->getArg(0)); Value *SizeVal = EmitScalarExpr(E->getArg(1)); - EmitNonNullArgCheck(Dest, E->getArg(0)->getType(), + EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, 0); Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false); return RValue::get(nullptr); @@ -4046,12 +4044,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Address Src = EmitPointerWithAlignment(E->getArg(0)); Address Dest = EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = EmitScalarExpr(E->getArg(2)); - EmitNonNullArgCheck(RValue::get(Src.emitRawPointer(*this)), - E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, - 0); - EmitNonNullArgCheck(RValue::get(Dest.emitRawPointer(*this)), - E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD, - 0); + EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(0)->getType(), + E->getArg(0)->getExprLoc(), FD, 0); + EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(1)->getType(), + E->getArg(1)->getExprLoc(), FD, 0); Builder.CreateMemMove(Dest, Src, SizeVal, false); return RValue::get(nullptr); } @@ -4068,10 +4064,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Builder.CreateMemCpy(Dest, Src, SizeVal, false); if (BuiltinID == Builtin::BImempcpy || BuiltinID == Builtin::BI__builtin_mempcpy) - return RValue::get(Builder.CreateInBoundsGEP( - Dest.getElementType(), Dest.emitRawPointer(*this), SizeVal)); + return RValue::get(Builder.CreateInBoundsGEP(Dest.getElementType(), + Dest.getPointer(), SizeVal)); else - return RValue::get(Dest, *this); + return RValue::get(Dest.getPointer()); } case Builtin::BI__builtin_memcpy_inline: { @@ -4103,7 +4099,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Address Src = EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); Builder.CreateMemCpy(Dest, Src, SizeVal, false); - return RValue::get(Dest, *this); + return RValue::get(Dest.getPointer()); } case Builtin::BI__builtin_objc_memmove_collectable: { @@ -4112,7 +4108,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Value *SizeVal = EmitScalarExpr(E->getArg(2)); CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestAddr, SrcAddr, SizeVal); - return RValue::get(DestAddr, *this); + return RValue::get(DestAddr.getPointer()); } case Builtin::BI__builtin___memmove_chk: { @@ -4129,7 +4125,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Address Src = EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); Builder.CreateMemMove(Dest, Src, SizeVal, false); - return RValue::get(Dest, *this); + return RValue::get(Dest.getPointer()); } case Builtin::BImemmove: @@ -4140,7 +4136,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0); EmitArgCheck(TCK_Load, Src, E->getArg(1), 1); Builder.CreateMemMove(Dest, Src, SizeVal, false); - return RValue::get(Dest, *this); + return RValue::get(Dest.getPointer()); } case Builtin::BImemset: case Builtin::BI__builtin_memset: { @@ -4148,10 +4144,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty()); Value *SizeVal = EmitScalarExpr(E->getArg(2)); - EmitNonNullArgCheck(Dest, E->getArg(0)->getType(), + EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, 0); Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); - return RValue::get(Dest, *this); + return RValue::get(Dest.getPointer()); } case Builtin::BI__builtin_memset_inline: { Address Dest = EmitPointerWithAlignment(E->getArg(0)); @@ -4159,9 +4155,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty()); uint64_t Size = E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue(); - EmitNonNullArgCheck(RValue::get(Dest.emitRawPointer(*this)), - E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, - 0); + EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), + E->getArg(0)->getExprLoc(), FD, 0); Builder.CreateMemSetInline(Dest, ByteVal, Size); return RValue::get(nullptr); } @@ -4180,7 +4175,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Builder.getInt8Ty()); Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); - return RValue::get(Dest, *this); + return RValue::get(Dest.getPointer()); } case Builtin::BI__builtin_wmemchr: { // The MSVC runtime library does not provide a definition of wmemchr, so we @@ -4402,14 +4397,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, // Store the stack pointer to the setjmp buffer. Value *StackAddr = Builder.CreateStackSave(); - assert(Buf.emitRawPointer(*this)->getType() == StackAddr->getType()); + assert(Buf.getPointer()->getType() == StackAddr->getType()); Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Buf, 2); Builder.CreateStore(StackAddr, StackSaveSlot); // Call LLVM's EH setjmp, which is lightweight. Function *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp); - return RValue::get(Builder.CreateCall(F, Buf.emitRawPointer(*this))); + return RValue::get(Builder.CreateCall(F, Buf.getPointer())); } case Builtin::BI__builtin_longjmp: { Value *Buf = EmitScalarExpr(E->getArg(0)); @@ -5582,7 +5577,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, llvm::Value *Queue = EmitScalarExpr(E->getArg(0)); llvm::Value *Flags = EmitScalarExpr(E->getArg(1)); LValue NDRangeL = EmitAggExprToLValue(E->getArg(2)); - llvm::Value *Range = NDRangeL.getAddress(*this).emitRawPointer(*this); + llvm::Value *Range = NDRangeL.getAddress(*this).getPointer(); llvm::Type *RangeTy = NDRangeL.getAddress(*this).getType(); if (NumArgs == 4) { @@ -5691,10 +5686,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, getContext(), Expr::NPC_ValueDependentIsNotNull)) { EventWaitList = llvm::ConstantPointerNull::get(PtrTy); } else { - EventWaitList = - E->getArg(4)->getType()->isArrayType() - ? EmitArrayToPointerDecay(E->getArg(4)).emitRawPointer(*this) - : EmitScalarExpr(E->getArg(4)); + EventWaitList = E->getArg(4)->getType()->isArrayType() + ? EmitArrayToPointerDecay(E->getArg(4)).getPointer() + : EmitScalarExpr(E->getArg(4)); // Convert to generic address space. EventWaitList = Builder.CreatePointerCast(EventWaitList, PtrTy); } @@ -5790,7 +5784,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, llvm::Type *GenericVoidPtrTy = Builder.getPtrTy( getContext().getTargetAddressSpace(LangAS::opencl_generic)); LValue NDRangeL = EmitAggExprToLValue(E->getArg(0)); - llvm::Value *NDRange = NDRangeL.getAddress(*this).emitRawPointer(*this); + llvm::Value *NDRange = NDRangeL.getAddress(*this).getPointer(); auto Info = CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(1)); Value *Kernel = @@ -5875,7 +5869,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, auto PTy0 = FTy->getParamType(0); if (PTy0 != Arg0Val->getType()) { if (Arg0Ty->isArrayType()) - Arg0Val = EmitArrayToPointerDecay(Arg0).emitRawPointer(*this); + Arg0Val = EmitArrayToPointerDecay(Arg0).getPointer(); else Arg0Val = Builder.CreatePointerCast(Arg0Val, PTy0); } @@ -5913,7 +5907,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, auto PTy1 = FTy->getParamType(1); if (PTy1 != Arg1Val->getType()) { if (Arg1Ty->isArrayType()) - Arg1Val = EmitArrayToPointerDecay(Arg1).emitRawPointer(*this); + Arg1Val = EmitArrayToPointerDecay(Arg1).getPointer(); else Arg1Val = Builder.CreatePointerCast(Arg1Val, PTy1); } @@ -5927,7 +5921,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_ms_va_start: case Builtin::BI__builtin_ms_va_end: return RValue::get( - EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).emitRawPointer(*this), + EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(), BuiltinID == Builtin::BI__builtin_ms_va_start)); case Builtin::BI__builtin_ms_va_copy: { @@ -5969,8 +5963,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, // If this is a predefined lib function (e.g. malloc), emit the call // using exactly the normal call path. if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID)) - return emitLibraryCall( - *this, FD, E, cast(EmitScalarExpr(E->getCallee()))); + return emitLibraryCall(*this, FD, E, + cast(EmitScalarExpr(E->getCallee()))); // Check that a call to a target specific builtin has the correct target // features. @@ -6087,7 +6081,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, return RValue::get(nullptr); return RValue::get(V); case TEK_Aggregate: - return RValue::getAggregate(ReturnValue.getAddress(), + return RValue::getAggregate(ReturnValue.getValue(), ReturnValue.isVolatile()); case TEK_Complex: llvm_unreachable("No current target builtin returns complex"); @@ -8857,7 +8851,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, // Get the alignment for the argument in addition to the value; // we'll use it later. PtrOp0 = EmitPointerWithAlignment(E->getArg(0)); - Ops.push_back(PtrOp0.emitRawPointer(*this)); + Ops.push_back(PtrOp0.getPointer()); continue; } } @@ -8884,7 +8878,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, // Get the alignment for the argument in addition to the value; // we'll use it later. PtrOp1 = EmitPointerWithAlignment(E->getArg(1)); - Ops.push_back(PtrOp1.emitRawPointer(*this)); + Ops.push_back(PtrOp1.getPointer()); continue; } } @@ -9305,7 +9299,7 @@ Value *CodeGenFunction::EmitARMMVEBuiltinExpr(unsigned BuiltinID, if (ReturnValue.isNull()) return MvecOut; else - return Builder.CreateStore(MvecOut, ReturnValue.getAddress()); + return Builder.CreateStore(MvecOut, ReturnValue.getValue()); } case CustomCodeGen::VST24: { @@ -11485,7 +11479,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, // Get the alignment for the argument in addition to the value; // we'll use it later. PtrOp0 = EmitPointerWithAlignment(E->getArg(0)); - Ops.push_back(PtrOp0.emitRawPointer(*this)); + Ops.push_back(PtrOp0.getPointer()); continue; } } @@ -13351,15 +13345,15 @@ Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID, if (!getDebugInfo()) { CGM.Error(E->getExprLoc(), "using __builtin_preserve_field_info() without -g"); - return IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*this) - : EmitLValue(Arg).emitRawPointer(*this); + return IsBitField ? EmitLValue(Arg).getBitFieldPointer() + : EmitLValue(Arg).getPointer(*this); } // Enable underlying preserve_*_access_index() generation. bool OldIsInPreservedAIRegion = IsInPreservedAIRegion; IsInPreservedAIRegion = true; - Value *FieldAddr = IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*this) - : EmitLValue(Arg).emitRawPointer(*this); + Value *FieldAddr = IsBitField ? EmitLValue(Arg).getBitFieldPointer() + : EmitLValue(Arg).getPointer(*this); IsInPreservedAIRegion = OldIsInPreservedAIRegion; ConstantInt *C = cast(EmitScalarExpr(E->getArg(1))); @@ -14351,14 +14345,14 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, } case X86::BI_mm_setcsr: case X86::BI__builtin_ia32_ldmxcsr: { - RawAddress Tmp = CreateMemTemp(E->getArg(0)->getType()); + Address Tmp = CreateMemTemp(E->getArg(0)->getType()); Builder.CreateStore(Ops[0], Tmp); return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr), Tmp.getPointer()); } case X86::BI_mm_getcsr: case X86::BI__builtin_ia32_stmxcsr: { - RawAddress Tmp = CreateMemTemp(E->getType()); + Address Tmp = CreateMemTemp(E->getType()); Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr), Tmp.getPointer()); return Builder.CreateLoad(Tmp, "stmxcsr"); @@ -17633,8 +17627,7 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, SmallVector Ops; for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) if (E->getArg(i)->getType()->isArrayType()) - Ops.push_back( - EmitArrayToPointerDecay(E->getArg(i)).emitRawPointer(*this)); + Ops.push_back(EmitArrayToPointerDecay(E->getArg(i)).getPointer()); else Ops.push_back(EmitScalarExpr(E->getArg(i))); // The first argument of these two builtins is a pointer used to store their @@ -20096,14 +20089,14 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, // Save returned values. assert(II.NumResults); if (II.NumResults == 1) { - Builder.CreateAlignedStore(Result, Dst.emitRawPointer(*this), + Builder.CreateAlignedStore(Result, Dst.getPointer(), CharUnits::fromQuantity(4)); } else { for (unsigned i = 0; i < II.NumResults; ++i) { Builder.CreateAlignedStore( Builder.CreateBitCast(Builder.CreateExtractValue(Result, i), Dst.getElementType()), - Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this), + Builder.CreateGEP(Dst.getElementType(), Dst.getPointer(), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); } @@ -20143,7 +20136,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < II.NumResults; ++i) { Value *V = Builder.CreateAlignedLoad( Src.getElementType(), - Builder.CreateGEP(Src.getElementType(), Src.emitRawPointer(*this), + Builder.CreateGEP(Src.getElementType(), Src.getPointer(), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); Values.push_back(Builder.CreateBitCast(V, ParamType)); @@ -20215,7 +20208,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < MI.NumEltsA; ++i) { Value *V = Builder.CreateAlignedLoad( SrcA.getElementType(), - Builder.CreateGEP(SrcA.getElementType(), SrcA.emitRawPointer(*this), + Builder.CreateGEP(SrcA.getElementType(), SrcA.getPointer(), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); Values.push_back(Builder.CreateBitCast(V, AType)); @@ -20225,7 +20218,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < MI.NumEltsB; ++i) { Value *V = Builder.CreateAlignedLoad( SrcB.getElementType(), - Builder.CreateGEP(SrcB.getElementType(), SrcB.emitRawPointer(*this), + Builder.CreateGEP(SrcB.getElementType(), SrcB.getPointer(), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); Values.push_back(Builder.CreateBitCast(V, BType)); @@ -20236,7 +20229,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < MI.NumEltsC; ++i) { Value *V = Builder.CreateAlignedLoad( SrcC.getElementType(), - Builder.CreateGEP(SrcC.getElementType(), SrcC.emitRawPointer(*this), + Builder.CreateGEP(SrcC.getElementType(), SrcC.getPointer(), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); Values.push_back(Builder.CreateBitCast(V, CType)); @@ -20246,7 +20239,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < MI.NumEltsD; ++i) Builder.CreateAlignedStore( Builder.CreateBitCast(Builder.CreateExtractValue(Result, i), DType), - Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this), + Builder.CreateGEP(Dst.getElementType(), Dst.getPointer(), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); return Result; @@ -20504,7 +20497,7 @@ struct BuiltinAlignArgs { BuiltinAlignArgs(const CallExpr *E, CodeGenFunction &CGF) { QualType AstType = E->getArg(0)->getType(); if (AstType->isArrayType()) - Src = CGF.EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(CGF); + Src = CGF.EmitArrayToPointerDecay(E->getArg(0)).getPointer(); else Src = CGF.EmitScalarExpr(E->getArg(0)); SrcType = Src->getType(); @@ -21122,7 +21115,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_get: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); + Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); Value *Index = EmitScalarExpr(E->getArg(1)); Function *Callee; if (E->getType().isWebAssemblyExternrefType()) @@ -21136,7 +21129,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_set: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); + Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); Value *Index = EmitScalarExpr(E->getArg(1)); Value *Val = EmitScalarExpr(E->getArg(2)); Function *Callee; @@ -21151,13 +21144,13 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_size: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Value = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); + Value *Value = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_table_size); return Builder.CreateCall(Callee, Value); } case WebAssembly::BI__builtin_wasm_table_grow: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); + Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); Value *Val = EmitScalarExpr(E->getArg(1)); Value *NElems = EmitScalarExpr(E->getArg(2)); @@ -21174,7 +21167,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_fill: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); + Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); Value *Index = EmitScalarExpr(E->getArg(1)); Value *Val = EmitScalarExpr(E->getArg(2)); Value *NElems = EmitScalarExpr(E->getArg(3)); @@ -21192,8 +21185,8 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_copy: { assert(E->getArg(0)->getType()->isArrayType()); - Value *TableX = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); - Value *TableY = EmitArrayToPointerDecay(E->getArg(1)).emitRawPointer(*this); + Value *TableX = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Value *TableY = EmitArrayToPointerDecay(E->getArg(1)).getPointer(); Value *DstIdx = EmitScalarExpr(E->getArg(2)); Value *SrcIdx = EmitScalarExpr(E->getArg(3)); Value *NElems = EmitScalarExpr(E->getArg(4)); @@ -21272,7 +21265,7 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, auto MakeCircOp = [this, E](unsigned IntID, bool IsLoad) { // The base pointer is passed by address, so it needs to be loaded. Address A = EmitPointerWithAlignment(E->getArg(0)); - Address BP = Address(A.emitRawPointer(*this), Int8PtrTy, A.getAlignment()); + Address BP = Address(A.getPointer(), Int8PtrTy, A.getAlignment()); llvm::Value *Base = Builder.CreateLoad(BP); // The treatment of both loads and stores is the same: the arguments for // the builtin are the same as the arguments for the intrinsic. @@ -21313,8 +21306,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, // EmitPointerWithAlignment and EmitScalarExpr evaluates the expression // per call. Address DestAddr = EmitPointerWithAlignment(E->getArg(1)); - DestAddr = DestAddr.withElementType(Int8Ty); - llvm::Value *DestAddress = DestAddr.emitRawPointer(*this); + DestAddr = Address(DestAddr.getPointer(), Int8Ty, DestAddr.getAlignment()); + llvm::Value *DestAddress = DestAddr.getPointer(); // Operands are Base, Dest, Modifier. // The intrinsic format in LLVM IR is defined as @@ -21365,8 +21358,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn}); llvm::Value *PredOut = Builder.CreateExtractValue(Result, 1); - Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.emitRawPointer(*this), - PredAddr.getAlignment()); + Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.getPointer(), + PredAddr.getAlignment()); return Builder.CreateExtractValue(Result, 0); } // These are identical to the builtins above, except they don't consume @@ -21384,8 +21377,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))}); llvm::Value *PredOut = Builder.CreateExtractValue(Result, 1); - Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.emitRawPointer(*this), - PredAddr.getAlignment()); + Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.getPointer(), + PredAddr.getAlignment()); return Builder.CreateExtractValue(Result, 0); } diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp index 0cb5b06a519c00..b756318c46a900 100644 --- a/clang/lib/CodeGen/CGCUDANV.cpp +++ b/clang/lib/CodeGen/CGCUDANV.cpp @@ -331,11 +331,11 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF, llvm::ConstantInt::get(SizeTy, std::max(1, Args.size()))); // Store pointers to the arguments in a locally allocated launch_args. for (unsigned i = 0; i < Args.size(); ++i) { - llvm::Value *VarPtr = CGF.GetAddrOfLocalVar(Args[i]).emitRawPointer(CGF); + llvm::Value* VarPtr = CGF.GetAddrOfLocalVar(Args[i]).getPointer(); llvm::Value *VoidVarPtr = CGF.Builder.CreatePointerCast(VarPtr, PtrTy); CGF.Builder.CreateDefaultAlignedStore( - VoidVarPtr, CGF.Builder.CreateConstGEP1_32( - PtrTy, KernelArgs.emitRawPointer(CGF), i)); + VoidVarPtr, + CGF.Builder.CreateConstGEP1_32(PtrTy, KernelArgs.getPointer(), i)); } llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end"); @@ -393,10 +393,9 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF, /*isVarArg=*/false), addUnderscoredPrefixToName("PopCallConfiguration")); - CGF.EmitRuntimeCallOrInvoke(cudaPopConfigFn, {GridDim.emitRawPointer(CGF), - BlockDim.emitRawPointer(CGF), - ShmemSize.emitRawPointer(CGF), - Stream.emitRawPointer(CGF)}); + CGF.EmitRuntimeCallOrInvoke(cudaPopConfigFn, + {GridDim.getPointer(), BlockDim.getPointer(), + ShmemSize.getPointer(), Stream.getPointer()}); // Emit the call to cudaLaunch llvm::Value *Kernel = @@ -406,7 +405,7 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF, cudaLaunchKernelFD->getParamDecl(0)->getType()); LaunchKernelArgs.add(RValue::getAggregate(GridDim), Dim3Ty); LaunchKernelArgs.add(RValue::getAggregate(BlockDim), Dim3Ty); - LaunchKernelArgs.add(RValue::get(KernelArgs, CGF), + LaunchKernelArgs.add(RValue::get(KernelArgs.getPointer()), cudaLaunchKernelFD->getParamDecl(3)->getType()); LaunchKernelArgs.add(RValue::get(CGF.Builder.CreateLoad(ShmemSize)), cudaLaunchKernelFD->getParamDecl(4)->getType()); @@ -439,8 +438,8 @@ void CGNVCUDARuntime::emitDeviceStubBodyLegacy(CodeGenFunction &CGF, auto TInfo = CGM.getContext().getTypeInfoInChars(A->getType()); Offset = Offset.alignTo(TInfo.Align); llvm::Value *Args[] = { - CGF.Builder.CreatePointerCast( - CGF.GetAddrOfLocalVar(A).emitRawPointer(CGF), PtrTy), + CGF.Builder.CreatePointerCast(CGF.GetAddrOfLocalVar(A).getPointer(), + PtrTy), llvm::ConstantInt::get(SizeTy, TInfo.Width.getQuantity()), llvm::ConstantInt::get(SizeTy, Offset.getQuantity()), }; diff --git a/clang/lib/CodeGen/CGCXXABI.cpp b/clang/lib/CodeGen/CGCXXABI.cpp index 7c6dfc3e59d8c0..a8bf57a277e909 100644 --- a/clang/lib/CodeGen/CGCXXABI.cpp +++ b/clang/lib/CodeGen/CGCXXABI.cpp @@ -20,12 +20,6 @@ using namespace CodeGen; CGCXXABI::~CGCXXABI() { } -Address CGCXXABI::getThisAddress(CodeGenFunction &CGF) { - return CGF.makeNaturalAddressForPointer( - CGF.CXXABIThisValue, CGF.CXXABIThisDecl->getType()->getPointeeType(), - CGF.CXXABIThisAlignment); -} - void CGCXXABI::ErrorUnsupportedABI(CodeGenFunction &CGF, StringRef S) { DiagnosticsEngine &Diags = CGF.CGM.getDiags(); unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error, @@ -50,12 +44,8 @@ CGCallee CGCXXABI::EmitLoadOfMemberFunctionPointer( llvm::Value *MemPtr, const MemberPointerType *MPT) { ErrorUnsupportedABI(CGF, "calls through member pointers"); - const auto *RD = - cast(MPT->getClass()->castAs()->getDecl()); - ThisPtrForCall = - CGF.getAsNaturalPointerTo(This, CGF.getContext().getRecordType(RD)); - const FunctionProtoType *FPT = - MPT->getPointeeType()->getAs(); + ThisPtrForCall = This.getPointer(); + const auto *FPT = MPT->getPointeeType()->castAs(); llvm::Constant *FnPtr = llvm::Constant::getNullValue( llvm::PointerType::getUnqual(CGM.getLLVMContext())); return CGCallee::forDirect(FnPtr, FPT); @@ -261,15 +251,16 @@ void CGCXXABI::ReadArrayCookie(CodeGenFunction &CGF, Address ptr, // If we don't need an array cookie, bail out early. if (!requiresArrayCookie(expr, eltTy)) { - allocPtr = ptr.emitRawPointer(CGF); + allocPtr = ptr.getPointer(); numElements = nullptr; cookieSize = CharUnits::Zero(); return; } cookieSize = getArrayCookieSizeImpl(eltTy); - Address allocAddr = CGF.Builder.CreateConstInBoundsByteGEP(ptr, -cookieSize); - allocPtr = allocAddr.emitRawPointer(CGF); + Address allocAddr = + CGF.Builder.CreateConstInBoundsByteGEP(ptr, -cookieSize); + allocPtr = allocAddr.getPointer(); numElements = readArrayCookieImpl(CGF, allocAddr, cookieSize); } diff --git a/clang/lib/CodeGen/CGCXXABI.h b/clang/lib/CodeGen/CGCXXABI.h index c7eccbd0095a94..ad1ad08d085688 100644 --- a/clang/lib/CodeGen/CGCXXABI.h +++ b/clang/lib/CodeGen/CGCXXABI.h @@ -57,8 +57,12 @@ class CGCXXABI { llvm::Value *getThisValue(CodeGenFunction &CGF) { return CGF.CXXABIThisValue; } - - Address getThisAddress(CodeGenFunction &CGF); + Address getThisAddress(CodeGenFunction &CGF) { + return Address( + CGF.CXXABIThisValue, + CGF.ConvertTypeForMem(CGF.CXXABIThisDecl->getType()->getPointeeType()), + CGF.CXXABIThisAlignment); + } /// Issue a diagnostic about unsupported features in the ABI. void ErrorUnsupportedABI(CodeGenFunction &CGF, StringRef S); @@ -471,6 +475,12 @@ class CGCXXABI { BaseSubobject Base, const CXXRecordDecl *NearestVBase) = 0; + /// Get the address point of the vtable for the given base subobject while + /// building a constexpr. + virtual llvm::Constant * + getVTableAddressPointForConstExpr(BaseSubobject Base, + const CXXRecordDecl *VTableClass) = 0; + /// Get the address of the vtable for the given record decl which should be /// used for the vptr at the given offset in RD. virtual llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD, diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index fb0078214b07ff..b8adf5c26b3a35 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -1037,9 +1037,15 @@ static void forConstantArrayExpansion(CodeGenFunction &CGF, ConstantArrayExpansion *CAE, Address BaseAddr, llvm::function_ref Fn) { + CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy); + CharUnits EltAlign = + BaseAddr.getAlignment().alignmentOfArrayElement(EltSize); + llvm::Type *EltTy = CGF.ConvertTypeForMem(CAE->EltTy); + for (int i = 0, n = CAE->NumElts; i < n; i++) { - Address EltAddr = CGF.Builder.CreateConstGEP2_32(BaseAddr, 0, i); - Fn(EltAddr); + llvm::Value *EltAddr = CGF.Builder.CreateConstGEP2_32( + BaseAddr.getElementType(), BaseAddr.getPointer(), 0, i); + Fn(Address(EltAddr, EltTy, EltAlign)); } } @@ -1154,10 +1160,9 @@ void CodeGenFunction::ExpandTypeToArgs( } /// Create a temporary allocation for the purposes of coercion. -static RawAddress CreateTempAllocaForCoercion(CodeGenFunction &CGF, - llvm::Type *Ty, - CharUnits MinAlign, - const Twine &Name = "tmp") { +static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty, + CharUnits MinAlign, + const Twine &Name = "tmp") { // Don't use an alignment that's worse than what LLVM would prefer. auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty); CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign)); @@ -1327,11 +1332,11 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, } // Otherwise do coercion through memory. This is stupid, but simple. - RawAddress Tmp = + Address Tmp = CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName()); CGF.Builder.CreateMemCpy( - Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), - Src.emitRawPointer(CGF), Src.getAlignment().getAsAlign(), + Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), Src.getPointer(), + Src.getAlignment().getAsAlign(), llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinValue())); return CGF.Builder.CreateLoad(Tmp); } @@ -1415,12 +1420,11 @@ static void CreateCoercedStore(llvm::Value *Src, // // FIXME: Assert that we aren't truncating non-padding bits when have access // to that information. - RawAddress Tmp = - CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment()); + Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment()); CGF.Builder.CreateStore(Src, Tmp); CGF.Builder.CreateMemCpy( - Dst.emitRawPointer(CGF), Dst.getAlignment().getAsAlign(), - Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), + Dst.getPointer(), Dst.getAlignment().getAsAlign(), Tmp.getPointer(), + Tmp.getAlignment().getAsAlign(), llvm::ConstantInt::get(CGF.IntPtrTy, DstSize.getFixedValue())); } } @@ -3020,17 +3024,17 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, case ABIArgInfo::Indirect: case ABIArgInfo::IndirectAliased: { assert(NumIRArgs == 1); - Address ParamAddr = makeNaturalAddressForPointer( - Fn->getArg(FirstIRArg), Ty, ArgI.getIndirectAlign(), false, nullptr, - nullptr, KnownNonNull); + Address ParamAddr = Address(Fn->getArg(FirstIRArg), ConvertTypeForMem(Ty), + ArgI.getIndirectAlign(), KnownNonNull); if (!hasScalarEvaluationKind(Ty)) { // Aggregates and complex variables are accessed by reference. All we // need to do is realign the value, if requested. Also, if the address // may be aliased, copy it to ensure that the parameter variable is // mutable and has a unique adress, as C requires. + Address V = ParamAddr; if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) { - RawAddress AlignedTemp = CreateMemTemp(Ty, "coerce"); + Address AlignedTemp = CreateMemTemp(Ty, "coerce"); // Copy from the incoming argument pointer to the temporary with the // appropriate alignment. @@ -3040,12 +3044,11 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, CharUnits Size = getContext().getTypeSizeInChars(Ty); Builder.CreateMemCpy( AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(), - ParamAddr.emitRawPointer(*this), - ParamAddr.getAlignment().getAsAlign(), + ParamAddr.getPointer(), ParamAddr.getAlignment().getAsAlign(), llvm::ConstantInt::get(IntPtrTy, Size.getQuantity())); - ParamAddr = AlignedTemp; + V = AlignedTemp; } - ArgVals.push_back(ParamValue::forIndirect(ParamAddr)); + ArgVals.push_back(ParamValue::forIndirect(V)); } else { // Load scalar value from indirect argument. llvm::Value *V = @@ -3159,10 +3162,10 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, == ParameterABI::SwiftErrorResult) { QualType pointeeTy = Ty->getPointeeType(); assert(pointeeTy->isPointerType()); - RawAddress temp = - CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp"); - Address arg = makeNaturalAddressForPointer( - V, pointeeTy, getContext().getTypeAlignInChars(pointeeTy)); + Address temp = + CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp"); + Address arg(V, ConvertTypeForMem(pointeeTy), + getContext().getTypeAlignInChars(pointeeTy)); llvm::Value *incomingErrorValue = Builder.CreateLoad(arg); Builder.CreateStore(incomingErrorValue, temp); V = temp.getPointer(); @@ -3499,7 +3502,7 @@ static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF, llvm::LoadInst *load = dyn_cast(retainedValue->stripPointerCasts()); if (!load || load->isAtomic() || load->isVolatile() || - load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getBasePointer()) + load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer()) return nullptr; // Okay! Burn it all down. This relies for correctness on the @@ -3536,15 +3539,12 @@ static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF, /// Heuristically search for a dominating store to the return-value slot. static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { - llvm::Value *ReturnValuePtr = CGF.ReturnValue.getBasePointer(); - // Check if a User is a store which pointerOperand is the ReturnValue. // We are looking for stores to the ReturnValue, not for stores of the // ReturnValue to some other location. - auto GetStoreIfValid = [&CGF, - ReturnValuePtr](llvm::User *U) -> llvm::StoreInst * { + auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * { auto *SI = dyn_cast(U); - if (!SI || SI->getPointerOperand() != ReturnValuePtr || + if (!SI || SI->getPointerOperand() != CGF.ReturnValue.getPointer() || SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType()) return nullptr; // These aren't actually possible for non-coerced returns, and we @@ -3558,7 +3558,7 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { // for something immediately preceding the IP. Sometimes this can // happen with how we generate implicit-returns; it can also happen // with noreturn cleanups. - if (!ReturnValuePtr->hasOneUse()) { + if (!CGF.ReturnValue.getPointer()->hasOneUse()) { llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock(); if (IP->empty()) return nullptr; @@ -3576,7 +3576,8 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { return nullptr; } - llvm::StoreInst *store = GetStoreIfValid(ReturnValuePtr->user_back()); + llvm::StoreInst *store = + GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back()); if (!store) return nullptr; // Now do a first-and-dirty dominance check: just walk up the @@ -4120,11 +4121,7 @@ void CodeGenFunction::EmitDelegateCallArg(CallArgList &args, } static bool isProvablyNull(llvm::Value *addr) { - return llvm::isa_and_nonnull(addr); -} - -static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) { - return llvm::isKnownNonZero(Addr.getBasePointer(), CGF.CGM.getDataLayout()); + return isa(addr); } /// Emit the actual writing-back of a writeback. @@ -4132,20 +4129,21 @@ static void emitWriteback(CodeGenFunction &CGF, const CallArgList::Writeback &writeback) { const LValue &srcLV = writeback.Source; Address srcAddr = srcLV.getAddress(CGF); - assert(!isProvablyNull(srcAddr.getBasePointer()) && + assert(!isProvablyNull(srcAddr.getPointer()) && "shouldn't have writeback for provably null argument"); llvm::BasicBlock *contBB = nullptr; // If the argument wasn't provably non-null, we need to null check // before doing the store. - bool provablyNonNull = isProvablyNonNull(srcAddr, CGF); - + bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(), + CGF.CGM.getDataLayout()); if (!provablyNonNull) { llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback"); contBB = CGF.createBasicBlock("icr.done"); - llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull"); + llvm::Value *isNull = + CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull"); CGF.Builder.CreateCondBr(isNull, contBB, writebackBB); CGF.EmitBlock(writebackBB); } @@ -4249,7 +4247,7 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, CGF.ConvertTypeForMem(CRE->getType()->getPointeeType()); // If the address is a constant null, just pass the appropriate null. - if (isProvablyNull(srcAddr.getBasePointer())) { + if (isProvablyNull(srcAddr.getPointer())) { args.add(RValue::get(llvm::ConstantPointerNull::get(destType)), CRE->getType()); return; @@ -4278,16 +4276,17 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, // If the address is *not* known to be non-null, we need to switch. llvm::Value *finalArgument; - bool provablyNonNull = isProvablyNonNull(srcAddr, CGF); - + bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(), + CGF.CGM.getDataLayout()); if (provablyNonNull) { - finalArgument = temp.emitRawPointer(CGF); + finalArgument = temp.getPointer(); } else { - llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull"); + llvm::Value *isNull = + CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull"); - finalArgument = CGF.Builder.CreateSelect( - isNull, llvm::ConstantPointerNull::get(destType), - temp.emitRawPointer(CGF), "icr.argument"); + finalArgument = CGF.Builder.CreateSelect(isNull, + llvm::ConstantPointerNull::get(destType), + temp.getPointer(), "icr.argument"); // If we need to copy, then the load has to be conditional, which // means we need control flow. @@ -4411,16 +4410,6 @@ void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType, EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, std::nullopt); } -void CodeGenFunction::EmitNonNullArgCheck(Address Addr, QualType ArgType, - SourceLocation ArgLoc, - AbstractCallee AC, unsigned ParmNum) { - if (!AC.getDecl() || !(SanOpts.has(SanitizerKind::NonnullAttribute) || - SanOpts.has(SanitizerKind::NullabilityArg))) - return; - - EmitNonNullArgCheck(RValue::get(Addr, *this), ArgType, ArgLoc, AC, ParmNum); -} - // Check if the call is going to use the inalloca convention. This needs to // agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged // later, so we can't check it directly. @@ -4761,20 +4750,10 @@ CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) { llvm::CallInst * CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const llvm::Twine &name) { - return EmitNounwindRuntimeCall(callee, ArrayRef(), name); + return EmitNounwindRuntimeCall(callee, std::nullopt, name); } /// Emits a call to the given nounwind runtime function. -llvm::CallInst * -CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee, - ArrayRef
args, - const llvm::Twine &name) { - SmallVector values; - for (auto arg : args) - values.push_back(arg.emitRawPointer(*this)); - return EmitNounwindRuntimeCall(callee, values, name); -} - llvm::CallInst * CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee, ArrayRef args, @@ -5053,7 +5032,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // If we're using inalloca, insert the allocation after the stack save. // FIXME: Do this earlier rather than hacking it in here! - RawAddress ArgMemory = RawAddress::invalid(); + Address ArgMemory = Address::invalid(); if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) { const llvm::DataLayout &DL = CGM.getDataLayout(); llvm::Instruction *IP = CallArgs.getStackBase(); @@ -5069,7 +5048,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, AI->setAlignment(Align.getAsAlign()); AI->setUsedWithInAlloca(true); assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca()); - ArgMemory = RawAddress(AI, ArgStruct, Align); + ArgMemory = Address(AI, ArgStruct, Align); } ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo); @@ -5078,11 +5057,11 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // If the call returns a temporary with struct return, create a temporary // alloca to hold the result, unless one is given to us. Address SRetPtr = Address::invalid(); - RawAddress SRetAlloca = RawAddress::invalid(); + Address SRetAlloca = Address::invalid(); llvm::Value *UnusedReturnSizePtr = nullptr; if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) { if (!ReturnValue.isNull()) { - SRetPtr = ReturnValue.getAddress(); + SRetPtr = ReturnValue.getValue(); } else { SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca); if (HaveInsertPoint() && ReturnValue.isUnused()) { @@ -5092,16 +5071,15 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, } } if (IRFunctionArgs.hasSRetArg()) { - IRCallArgs[IRFunctionArgs.getSRetArgNo()] = - getAsNaturalPointerTo(SRetPtr, RetTy); + IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer(); } else if (RetAI.isInAlloca()) { Address Addr = Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex()); - Builder.CreateStore(getAsNaturalPointerTo(SRetPtr, RetTy), Addr); + Builder.CreateStore(SRetPtr.getPointer(), Addr); } } - RawAddress swiftErrorTemp = RawAddress::invalid(); + Address swiftErrorTemp = Address::invalid(); Address swiftErrorArg = Address::invalid(); // When passing arguments using temporary allocas, we need to add the @@ -5134,9 +5112,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, assert(NumIRArgs == 0); assert(getTarget().getTriple().getArch() == llvm::Triple::x86); if (I->isAggregate()) { - RawAddress Addr = I->hasLValue() - ? I->getKnownLValue().getAddress(*this) - : I->getKnownRValue().getAggregateAddress(); + Address Addr = I->hasLValue() + ? I->getKnownLValue().getAddress(*this) + : I->getKnownRValue().getAggregateAddress(); llvm::Instruction *Placeholder = cast(Addr.getPointer()); @@ -5160,7 +5138,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, } else if (ArgInfo.getInAllocaIndirect()) { // Make a temporary alloca and store the address of it into the argument // struct. - RawAddress Addr = CreateMemTempWithoutCast( + Address Addr = CreateMemTempWithoutCast( I->Ty, getContext().getTypeAlignInChars(I->Ty), "indirect-arg-temp"); I->copyInto(*this, Addr); @@ -5182,12 +5160,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, assert(NumIRArgs == 1); if (!I->isAggregate()) { // Make a temporary alloca to pass the argument. - RawAddress Addr = CreateMemTempWithoutCast( + Address Addr = CreateMemTempWithoutCast( I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp"); - llvm::Value *Val = getAsNaturalPointerTo(Addr, I->Ty); + llvm::Value *Val = Addr.getPointer(); if (ArgHasMaybeUndefAttr) - Val = Builder.CreateFreeze(Val); + Val = Builder.CreateFreeze(Addr.getPointer()); IRCallArgs[FirstIRArg] = Val; I->copyInto(*this, Addr); @@ -5203,6 +5181,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, Address Addr = I->hasLValue() ? I->getKnownLValue().getAddress(*this) : I->getKnownRValue().getAggregateAddress(); + llvm::Value *V = Addr.getPointer(); CharUnits Align = ArgInfo.getIndirectAlign(); const llvm::DataLayout *TD = &CGM.getDataLayout(); @@ -5213,9 +5192,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, bool NeedCopy = false; if (Addr.getAlignment() < Align && - llvm::getOrEnforceKnownAlignment(Addr.emitRawPointer(*this), - Align.getAsAlign(), - *TD) < Align.getAsAlign()) { + llvm::getOrEnforceKnownAlignment(V, Align.getAsAlign(), *TD) < + Align.getAsAlign()) { NeedCopy = true; } else if (I->hasLValue()) { auto LV = I->getKnownLValue(); @@ -5246,11 +5224,11 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, if (NeedCopy) { // Create an aligned temporary, and copy to it. - RawAddress AI = CreateMemTempWithoutCast( + Address AI = CreateMemTempWithoutCast( I->Ty, ArgInfo.getIndirectAlign(), "byval-temp"); - llvm::Value *Val = getAsNaturalPointerTo(AI, I->Ty); + llvm::Value *Val = AI.getPointer(); if (ArgHasMaybeUndefAttr) - Val = Builder.CreateFreeze(Val); + Val = Builder.CreateFreeze(AI.getPointer()); IRCallArgs[FirstIRArg] = Val; // Emit lifetime markers for the temporary alloca. @@ -5267,7 +5245,6 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, I->copyInto(*this, AI); } else { // Skip the extra memcpy call. - llvm::Value *V = getAsNaturalPointerTo(Addr, I->Ty); auto *T = llvm::PointerType::get( CGM.getLLVMContext(), CGM.getDataLayout().getAllocaAddrSpace()); @@ -5307,8 +5284,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, assert(!swiftErrorTemp.isValid() && "multiple swifterror args"); QualType pointeeTy = I->Ty->getPointeeType(); - swiftErrorArg = makeNaturalAddressForPointer( - V, pointeeTy, getContext().getTypeAlignInChars(pointeeTy)); + swiftErrorArg = Address(V, ConvertTypeForMem(pointeeTy), + getContext().getTypeAlignInChars(pointeeTy)); swiftErrorTemp = CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp"); @@ -5445,7 +5422,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, llvm::Value *tempSize = nullptr; Address addr = Address::invalid(); - RawAddress AllocaAddr = RawAddress::invalid(); + Address AllocaAddr = Address::invalid(); if (I->isAggregate()) { addr = I->hasLValue() ? I->getKnownLValue().getAddress(*this) : I->getKnownRValue().getAggregateAddress(); @@ -5879,7 +5856,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, return RValue::getComplex(std::make_pair(Real, Imag)); } case TEK_Aggregate: { - Address DestPtr = ReturnValue.getAddress(); + Address DestPtr = ReturnValue.getValue(); bool DestIsVolatile = ReturnValue.isVolatile(); if (!DestPtr.isValid()) { diff --git a/clang/lib/CodeGen/CGCall.h b/clang/lib/CodeGen/CGCall.h index 6b676ac196db2a..1bd48a07259307 100644 --- a/clang/lib/CodeGen/CGCall.h +++ b/clang/lib/CodeGen/CGCall.h @@ -377,7 +377,6 @@ class ReturnValueSlot { Address getValue() const { return Addr; } bool isUnused() const { return IsUnused; } bool isExternallyDestructed() const { return IsExternallyDestructed; } - Address getAddress() const { return Addr; } }; /// Adds attributes to \p F according to our \p CodeGenOpts and \p LangOpts, as diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp index 8c1c8ee455d2e6..34319381901af6 100644 --- a/clang/lib/CodeGen/CGClass.cpp +++ b/clang/lib/CodeGen/CGClass.cpp @@ -139,9 +139,8 @@ Address CodeGenFunction::LoadCXXThisAddress() { CXXThisAlignment = CGM.getClassPointerAlignment(MD->getParent()); } - return makeNaturalAddressForPointer( - LoadCXXThis(), MD->getFunctionObjectParameterType(), CXXThisAlignment, - false, nullptr, nullptr, KnownNonNull); + llvm::Type *Ty = ConvertType(MD->getFunctionObjectParameterType()); + return Address(LoadCXXThis(), Ty, CXXThisAlignment, KnownNonNull); } /// Emit the address of a field using a member data pointer. @@ -271,7 +270,7 @@ ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr, } // Apply the base offset. - llvm::Value *ptr = addr.emitRawPointer(CGF); + llvm::Value *ptr = addr.getPointer(); ptr = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ptr, baseOffset, "add.ptr"); // If we have a virtual component, the alignment of the result will @@ -339,8 +338,8 @@ Address CodeGenFunction::GetAddressOfBaseClass( if (sanitizePerformTypeCheck()) { SanitizerSet SkippedChecks; SkippedChecks.set(SanitizerKind::Null, !NullCheckValue); - EmitTypeCheck(TCK_Upcast, Loc, Value.emitRawPointer(*this), DerivedTy, - DerivedAlign, SkippedChecks); + EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(), + DerivedTy, DerivedAlign, SkippedChecks); } return Value.withElementType(BaseValueTy); } @@ -355,7 +354,7 @@ Address CodeGenFunction::GetAddressOfBaseClass( llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull"); endBB = createBasicBlock("cast.end"); - llvm::Value *isNull = Builder.CreateIsNull(Value); + llvm::Value *isNull = Builder.CreateIsNull(Value.getPointer()); Builder.CreateCondBr(isNull, endBB, notNullBB); EmitBlock(notNullBB); } @@ -364,15 +363,14 @@ Address CodeGenFunction::GetAddressOfBaseClass( SanitizerSet SkippedChecks; SkippedChecks.set(SanitizerKind::Null, true); EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc, - Value.emitRawPointer(*this), DerivedTy, DerivedAlign, - SkippedChecks); + Value.getPointer(), DerivedTy, DerivedAlign, SkippedChecks); } // Compute the virtual offset. llvm::Value *VirtualOffset = nullptr; if (VBase) { VirtualOffset = - CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase); + CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase); } // Apply both offsets. @@ -389,7 +387,7 @@ Address CodeGenFunction::GetAddressOfBaseClass( EmitBlock(endBB); llvm::PHINode *PHI = Builder.CreatePHI(PtrTy, 2, "cast.result"); - PHI->addIncoming(Value.emitRawPointer(*this), notNullBB); + PHI->addIncoming(Value.getPointer(), notNullBB); PHI->addIncoming(llvm::Constant::getNullValue(PtrTy), origBB); Value = Value.withPointer(PHI, NotKnownNonNull); } @@ -426,19 +424,15 @@ CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr, CastNotNull = createBasicBlock("cast.notnull"); CastEnd = createBasicBlock("cast.end"); - llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr); + llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr.getPointer()); Builder.CreateCondBr(IsNull, CastNull, CastNotNull); EmitBlock(CastNotNull); } // Apply the offset. - Address Addr = BaseAddr.withElementType(Int8Ty); - Addr = Builder.CreateInBoundsGEP( - Addr, Builder.CreateNeg(NonVirtualOffset), Int8Ty, - CGM.getClassPointerAlignment(Derived), "sub.ptr"); - - // Just cast. - Addr = Addr.withElementType(DerivedValueTy); + llvm::Value *Value = BaseAddr.getPointer(); + Value = Builder.CreateInBoundsGEP( + Int8Ty, Value, Builder.CreateNeg(NonVirtualOffset), "sub.ptr"); // Produce a PHI if we had a null-check. if (NullCheckValue) { @@ -447,15 +441,13 @@ CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr, Builder.CreateBr(CastEnd); EmitBlock(CastEnd); - llvm::Value *Value = Addr.emitRawPointer(*this); llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2); PHI->addIncoming(Value, CastNotNull); PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull); - return Address(PHI, Addr.getElementType(), - CGM.getClassPointerAlignment(Derived)); + Value = PHI; } - return Addr; + return Address(Value, DerivedValueTy, CGM.getClassPointerAlignment(Derived)); } llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD, @@ -1727,7 +1719,7 @@ namespace { // Use the base class declaration location as inline DebugLocation. All // fields of the class are destroyed. DeclAsInlineDebugLocation InlineHere(CGF, *BaseClass); - EmitSanitizerDtorFieldsCallback(CGF, Addr.emitRawPointer(CGF), + EmitSanitizerDtorFieldsCallback(CGF, Addr.getPointer(), BaseSize.getQuantity()); // Prevent the current stack frame from disappearing from the stack trace. @@ -2030,7 +2022,7 @@ void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor, // Find the end of the array. llvm::Type *elementType = arrayBase.getElementType(); - llvm::Value *arrayBegin = arrayBase.emitRawPointer(*this); + llvm::Value *arrayBegin = arrayBase.getPointer(); llvm::Value *arrayEnd = Builder.CreateInBoundsGEP( elementType, arrayBegin, numElements, "arrayctor.end"); @@ -2126,15 +2118,14 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, Address This = ThisAVS.getAddress(); LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace(); LangAS ThisAS = D->getFunctionObjectParameterType().getAddressSpace(); - llvm::Value *ThisPtr = - getAsNaturalPointerTo(This, D->getThisType()->getPointeeType()); + llvm::Value *ThisPtr = This.getPointer(); if (SlotAS != ThisAS) { unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS); llvm::Type *NewType = llvm::PointerType::get(getLLVMContext(), TargetThisAS); - ThisPtr = getTargetHooks().performAddrSpaceCast(*this, ThisPtr, ThisAS, - SlotAS, NewType); + ThisPtr = getTargetHooks().performAddrSpaceCast(*this, This.getPointer(), + ThisAS, SlotAS, NewType); } // Push the this ptr. @@ -2203,7 +2194,7 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, const CXXRecordDecl *ClassDecl = D->getParent(); if (!NewPointerIsChecked) - EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This, + EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This.getPointer(), getContext().getRecordType(ClassDecl), CharUnits::Zero()); if (D->isTrivial() && D->isDefaultConstructor()) { @@ -2216,9 +2207,10 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, // model that copy. if (isMemcpyEquivalentSpecialMember(D)) { assert(Args.size() == 2 && "unexpected argcount for trivial ctor"); + QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType(); - Address Src = makeNaturalAddressForPointer( - Args[1].getRValue(*this).getScalarVal(), SrcTy); + Address Src = Address(Args[1].getRValue(*this).getScalarVal(), ConvertTypeForMem(SrcTy), + CGM.getNaturalTypeAlignment(SrcTy)); LValue SrcLVal = MakeAddrLValue(Src, SrcTy); QualType DestTy = getContext().getTypeDeclType(ClassDecl); LValue DestLVal = MakeAddrLValue(This, DestTy); @@ -2271,9 +2263,7 @@ void CodeGenFunction::EmitInheritedCXXConstructorCall( const CXXConstructorDecl *D, bool ForVirtualBase, Address This, bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) { CallArgList Args; - CallArg ThisArg(RValue::get(getAsNaturalPointerTo( - This, D->getThisType()->getPointeeType())), - D->getThisType()); + CallArg ThisArg(RValue::get(This.getPointer()), D->getThisType()); // Forward the parameters. if (InheritedFromVBase && @@ -2398,14 +2388,12 @@ CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, CallArgList Args; // Push the this ptr. - Args.add(RValue::get(getAsNaturalPointerTo(This, D->getThisType())), - D->getThisType()); + Args.add(RValue::get(This.getPointer()), D->getThisType()); // Push the src ptr. QualType QT = *(FPT->param_type_begin()); llvm::Type *t = CGM.getTypes().ConvertType(QT); - llvm::Value *Val = getAsNaturalPointerTo(Src, D->getThisType()); - llvm::Value *SrcVal = Builder.CreateBitCast(Val, t); + llvm::Value *SrcVal = Builder.CreateBitCast(Src.getPointer(), t); Args.add(RValue::get(SrcVal), QT); // Skip over first argument (Src). @@ -2430,9 +2418,7 @@ CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor, // this Address This = LoadCXXThisAddress(); - DelegateArgs.add(RValue::get(getAsNaturalPointerTo( - This, (*I)->getType()->getPointeeType())), - (*I)->getType()); + DelegateArgs.add(RValue::get(This.getPointer()), (*I)->getType()); ++I; // FIXME: The location of the VTT parameter in the parameter list is @@ -2789,7 +2775,7 @@ void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, Address Derived, if (MayBeNull) { llvm::Value *DerivedNotNull = - Builder.CreateIsNotNull(Derived.emitRawPointer(*this), "cast.nonnull"); + Builder.CreateIsNotNull(Derived.getPointer(), "cast.nonnull"); llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check"); ContBlock = createBasicBlock("cast.cont"); @@ -2990,7 +2976,7 @@ void CodeGenFunction::EmitLambdaBlockInvokeBody() { QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda)); Address ThisPtr = GetAddrOfBlockDecl(variable); - CallArgs.add(RValue::get(getAsNaturalPointerTo(ThisPtr, ThisType)), ThisType); + CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType); // Add the rest of the parameters. for (auto *param : BD->parameters()) @@ -3018,7 +3004,7 @@ void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) { QualType LambdaType = getContext().getRecordType(Lambda); QualType ThisType = getContext().getPointerType(LambdaType); Address ThisPtr = CreateMemTemp(LambdaType, "unused.capture"); - CallArgs.add(RValue::get(ThisPtr.emitRawPointer(*this)), ThisType); + CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType); EmitLambdaDelegatingInvokeBody(MD, CallArgs); } diff --git a/clang/lib/CodeGen/CGCleanup.cpp b/clang/lib/CodeGen/CGCleanup.cpp index e6f8e6873004f2..f87caf050eeaa7 100644 --- a/clang/lib/CodeGen/CGCleanup.cpp +++ b/clang/lib/CodeGen/CGCleanup.cpp @@ -27,7 +27,7 @@ bool DominatingValue::saved_type::needsSaving(RValue rv) { if (rv.isScalar()) return DominatingLLVMValue::needsSaving(rv.getScalarVal()); if (rv.isAggregate()) - return DominatingValue
::needsSaving(rv.getAggregateAddress()); + return DominatingLLVMValue::needsSaving(rv.getAggregatePointer()); return true; } @@ -35,40 +35,69 @@ DominatingValue::saved_type DominatingValue::saved_type::save(CodeGenFunction &CGF, RValue rv) { if (rv.isScalar()) { llvm::Value *V = rv.getScalarVal(); - return saved_type(DominatingLLVMValue::save(CGF, V), - DominatingLLVMValue::needsSaving(V) ? ScalarAddress - : ScalarLiteral); + + // These automatically dominate and don't need to be saved. + if (!DominatingLLVMValue::needsSaving(V)) + return saved_type(V, nullptr, ScalarLiteral); + + // Everything else needs an alloca. + Address addr = + CGF.CreateDefaultAlignTempAlloca(V->getType(), "saved-rvalue"); + CGF.Builder.CreateStore(V, addr); + return saved_type(addr.getPointer(), nullptr, ScalarAddress); } if (rv.isComplex()) { CodeGenFunction::ComplexPairTy V = rv.getComplexVal(); - return saved_type(DominatingLLVMValue::save(CGF, V.first), - DominatingLLVMValue::save(CGF, V.second)); + llvm::Type *ComplexTy = + llvm::StructType::get(V.first->getType(), V.second->getType()); + Address addr = CGF.CreateDefaultAlignTempAlloca(ComplexTy, "saved-complex"); + CGF.Builder.CreateStore(V.first, CGF.Builder.CreateStructGEP(addr, 0)); + CGF.Builder.CreateStore(V.second, CGF.Builder.CreateStructGEP(addr, 1)); + return saved_type(addr.getPointer(), nullptr, ComplexAddress); } assert(rv.isAggregate()); - Address V = rv.getAggregateAddress(); - return saved_type( - DominatingValue
::save(CGF, V), rv.isVolatileQualified(), - DominatingValue
::needsSaving(V) ? AggregateAddress - : AggregateLiteral); + Address V = rv.getAggregateAddress(); // TODO: volatile? + if (!DominatingLLVMValue::needsSaving(V.getPointer())) + return saved_type(V.getPointer(), V.getElementType(), AggregateLiteral, + V.getAlignment().getQuantity()); + + Address addr = + CGF.CreateTempAlloca(V.getType(), CGF.getPointerAlign(), "saved-rvalue"); + CGF.Builder.CreateStore(V.getPointer(), addr); + return saved_type(addr.getPointer(), V.getElementType(), AggregateAddress, + V.getAlignment().getQuantity()); } /// Given a saved r-value produced by SaveRValue, perform the code /// necessary to restore it to usability at the current insertion /// point. RValue DominatingValue::saved_type::restore(CodeGenFunction &CGF) { + auto getSavingAddress = [&](llvm::Value *value) { + auto *AI = cast(value); + return Address(value, AI->getAllocatedType(), + CharUnits::fromQuantity(AI->getAlign().value())); + }; switch (K) { case ScalarLiteral: + return RValue::get(Value); case ScalarAddress: - return RValue::get(DominatingLLVMValue::restore(CGF, Vals.first)); + return RValue::get(CGF.Builder.CreateLoad(getSavingAddress(Value))); case AggregateLiteral: - case AggregateAddress: return RValue::getAggregate( - DominatingValue
::restore(CGF, AggregateAddr), IsVolatile); + Address(Value, ElementType, CharUnits::fromQuantity(Align))); + case AggregateAddress: { + auto addr = CGF.Builder.CreateLoad(getSavingAddress(Value)); + return RValue::getAggregate( + Address(addr, ElementType, CharUnits::fromQuantity(Align))); + } case ComplexAddress: { - llvm::Value *real = DominatingLLVMValue::restore(CGF, Vals.first); - llvm::Value *imag = DominatingLLVMValue::restore(CGF, Vals.second); + Address address = getSavingAddress(Value); + llvm::Value *real = + CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(address, 0)); + llvm::Value *imag = + CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(address, 1)); return RValue::getComplex(real, imag); } } @@ -265,14 +294,14 @@ void EHScopeStack::popNullFixups() { BranchFixups.pop_back(); } -RawAddress CodeGenFunction::createCleanupActiveFlag() { +Address CodeGenFunction::createCleanupActiveFlag() { // Create a variable to decide whether the cleanup needs to be run. - RawAddress active = CreateTempAllocaWithoutCast( + Address active = CreateTempAllocaWithoutCast( Builder.getInt1Ty(), CharUnits::One(), "cleanup.cond"); // Initialize it to false at a site that's guaranteed to be run // before each evaluation. - setBeforeOutermostConditional(Builder.getFalse(), active, *this); + setBeforeOutermostConditional(Builder.getFalse(), active); // Initialize it to true at the current location. Builder.CreateStore(Builder.getTrue(), active); @@ -280,7 +309,7 @@ RawAddress CodeGenFunction::createCleanupActiveFlag() { return active; } -void CodeGenFunction::initFullExprCleanupWithFlag(RawAddress ActiveFlag) { +void CodeGenFunction::initFullExprCleanupWithFlag(Address ActiveFlag) { // Set that as the active flag in the cleanup. EHCleanupScope &cleanup = cast(*EHStack.begin()); assert(!cleanup.hasActiveFlag() && "cleanup already has active flag?"); @@ -293,17 +322,15 @@ void CodeGenFunction::initFullExprCleanupWithFlag(RawAddress ActiveFlag) { void EHScopeStack::Cleanup::anchor() {} static void createStoreInstBefore(llvm::Value *value, Address addr, - llvm::Instruction *beforeInst, - CodeGenFunction &CGF) { - auto store = new llvm::StoreInst(value, addr.emitRawPointer(CGF), beforeInst); + llvm::Instruction *beforeInst) { + auto store = new llvm::StoreInst(value, addr.getPointer(), beforeInst); store->setAlignment(addr.getAlignment().getAsAlign()); } static llvm::LoadInst *createLoadInstBefore(Address addr, const Twine &name, - llvm::Instruction *beforeInst, - CodeGenFunction &CGF) { - return new llvm::LoadInst(addr.getElementType(), addr.emitRawPointer(CGF), - name, false, addr.getAlignment().getAsAlign(), + llvm::Instruction *beforeInst) { + return new llvm::LoadInst(addr.getElementType(), addr.getPointer(), name, + false, addr.getAlignment().getAsAlign(), beforeInst); } @@ -330,8 +357,8 @@ static void ResolveAllBranchFixups(CodeGenFunction &CGF, // entry which we're currently popping. if (Fixup.OptimisticBranchBlock == nullptr) { createStoreInstBefore(CGF.Builder.getInt32(Fixup.DestinationIndex), - CGF.getNormalCleanupDestSlot(), Fixup.InitialBranch, - CGF); + CGF.getNormalCleanupDestSlot(), + Fixup.InitialBranch); Fixup.InitialBranch->setSuccessor(0, CleanupEntry); } @@ -358,7 +385,7 @@ static llvm::SwitchInst *TransitionToCleanupSwitch(CodeGenFunction &CGF, if (llvm::BranchInst *Br = dyn_cast(Term)) { assert(Br->isUnconditional()); auto Load = createLoadInstBefore(CGF.getNormalCleanupDestSlot(), - "cleanup.dest", Term, CGF); + "cleanup.dest", Term); llvm::SwitchInst *Switch = llvm::SwitchInst::Create(Load, Br->getSuccessor(0), 4, Block); Br->eraseFromParent(); @@ -486,8 +513,8 @@ void CodeGenFunction::PopCleanupBlocks( I += Header.getSize(); if (Header.isConditional()) { - RawAddress ActiveFlag = - reinterpret_cast(LifetimeExtendedCleanupStack[I]); + Address ActiveFlag = + reinterpret_cast
(LifetimeExtendedCleanupStack[I]); initFullExprCleanupWithFlag(ActiveFlag); I += sizeof(ActiveFlag); } @@ -861,7 +888,7 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { if (NormalCleanupDestSlot->hasOneUse()) { NormalCleanupDestSlot->user_back()->eraseFromParent(); NormalCleanupDestSlot->eraseFromParent(); - NormalCleanupDest = RawAddress::invalid(); + NormalCleanupDest = Address::invalid(); } llvm::BasicBlock *BranchAfter = Scope.getBranchAfterBlock(0); @@ -885,8 +912,9 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { // pass the abnormal exit flag to Fn (SEH cleanup) cleanupFlags.setHasExitSwitch(); - llvm::LoadInst *Load = createLoadInstBefore( - getNormalCleanupDestSlot(), "cleanup.dest", nullptr, *this); + llvm::LoadInst *Load = + createLoadInstBefore(getNormalCleanupDestSlot(), "cleanup.dest", + nullptr); llvm::SwitchInst *Switch = llvm::SwitchInst::Create(Load, Default, SwitchCapacity); @@ -933,8 +961,8 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { if (!Fixup.Destination) continue; if (!Fixup.OptimisticBranchBlock) { createStoreInstBefore(Builder.getInt32(Fixup.DestinationIndex), - getNormalCleanupDestSlot(), Fixup.InitialBranch, - *this); + getNormalCleanupDestSlot(), + Fixup.InitialBranch); Fixup.InitialBranch->setSuccessor(0, NormalEntry); } Fixup.OptimisticBranchBlock = NormalExit; @@ -1107,7 +1135,7 @@ void CodeGenFunction::EmitBranchThroughCleanup(JumpDest Dest) { // Store the index at the start. llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex()); - createStoreInstBefore(Index, getNormalCleanupDestSlot(), BI, *this); + createStoreInstBefore(Index, getNormalCleanupDestSlot(), BI); // Adjust BI to point to the first cleanup block. { @@ -1241,9 +1269,9 @@ static void SetupCleanupBlockActivation(CodeGenFunction &CGF, // If we're in a conditional block, ignore the dominating IP and // use the outermost conditional branch. if (CGF.isInConditionalBranch()) { - CGF.setBeforeOutermostConditional(value, var, CGF); + CGF.setBeforeOutermostConditional(value, var); } else { - createStoreInstBefore(value, var, dominatingIP, CGF); + createStoreInstBefore(value, var, dominatingIP); } } @@ -1293,7 +1321,7 @@ void CodeGenFunction::DeactivateCleanupBlock(EHScopeStack::stable_iterator C, Scope.setActive(false); } -RawAddress CodeGenFunction::getNormalCleanupDestSlot() { +Address CodeGenFunction::getNormalCleanupDestSlot() { if (!NormalCleanupDest.isValid()) NormalCleanupDest = CreateDefaultAlignTempAlloca(Builder.getInt32Ty(), "cleanup.dest.slot"); diff --git a/clang/lib/CodeGen/CGCleanup.h b/clang/lib/CodeGen/CGCleanup.h index 03e4a29d7b3dbf..7a7344c07160db 100644 --- a/clang/lib/CodeGen/CGCleanup.h +++ b/clang/lib/CodeGen/CGCleanup.h @@ -333,7 +333,7 @@ class alignas(8) EHCleanupScope : public EHScope { Address getActiveFlag() const { return ActiveFlag; } - void setActiveFlag(RawAddress Var) { + void setActiveFlag(Address Var) { assert(Var.getAlignment().isOne()); ActiveFlag = Var; } diff --git a/clang/lib/CodeGen/CGCoroutine.cpp b/clang/lib/CodeGen/CGCoroutine.cpp index 93ca711f716fce..b7142ec08af986 100644 --- a/clang/lib/CodeGen/CGCoroutine.cpp +++ b/clang/lib/CodeGen/CGCoroutine.cpp @@ -867,8 +867,8 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) { EmitStmt(S.getPromiseDeclStmt()); Address PromiseAddr = GetAddrOfLocalVar(S.getPromiseDecl()); - auto *PromiseAddrVoidPtr = new llvm::BitCastInst( - PromiseAddr.emitRawPointer(*this), VoidPtrTy, "", CoroId); + auto *PromiseAddrVoidPtr = + new llvm::BitCastInst(PromiseAddr.getPointer(), VoidPtrTy, "", CoroId); // Update CoroId to refer to the promise. We could not do it earlier because // promise local variable was not emitted yet. CoroId->setArgOperand(1, PromiseAddrVoidPtr); diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp index 267f2e40a7bbaa..2ef5ed04af30b6 100644 --- a/clang/lib/CodeGen/CGDecl.cpp +++ b/clang/lib/CodeGen/CGDecl.cpp @@ -1461,7 +1461,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { bool EmitDebugInfo = DI && CGM.getCodeGenOpts().hasReducedDebugInfo(); Address address = Address::invalid(); - RawAddress AllocaAddr = RawAddress::invalid(); + Address AllocaAddr = Address::invalid(); Address OpenMPLocalAddr = Address::invalid(); if (CGM.getLangOpts().OpenMPIRBuilder) OpenMPLocalAddr = OMPBuilderCBHelpers::getAddressOfLocalVariable(*this, &D); @@ -1524,10 +1524,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { // return slot, so that we can elide the copy when returning this // variable (C++0x [class.copy]p34). address = ReturnValue; - AllocaAddr = - RawAddress(ReturnValue.emitRawPointer(*this), - ReturnValue.getElementType(), ReturnValue.getAlignment()); - ; + AllocaAddr = ReturnValue; if (const RecordType *RecordTy = Ty->getAs()) { const auto *RD = RecordTy->getDecl(); @@ -1538,7 +1535,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { // to this variable. Set it to zero to indicate that NRVO was not // applied. llvm::Value *Zero = Builder.getFalse(); - RawAddress NRVOFlag = + Address NRVOFlag = CreateTempAlloca(Zero->getType(), CharUnits::One(), "nrvo"); EnsureInsertPoint(); Builder.CreateStore(Zero, NRVOFlag); @@ -1681,7 +1678,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { } if (D.hasAttr() && HaveInsertPoint()) - EmitVarAnnotations(&D, address.emitRawPointer(*this)); + EmitVarAnnotations(&D, address.getPointer()); // Make sure we call @llvm.lifetime.end. if (emission.useLifetimeMarkers()) @@ -1854,13 +1851,12 @@ void CodeGenFunction::emitZeroOrPatternForAutoVarInit(QualType type, llvm::Value *BaseSizeInChars = llvm::ConstantInt::get(IntPtrTy, EltSize.getQuantity()); Address Begin = Loc.withElementType(Int8Ty); - llvm::Value *End = Builder.CreateInBoundsGEP(Begin.getElementType(), - Begin.emitRawPointer(*this), - SizeVal, "vla.end"); + llvm::Value *End = Builder.CreateInBoundsGEP( + Begin.getElementType(), Begin.getPointer(), SizeVal, "vla.end"); llvm::BasicBlock *OriginBB = Builder.GetInsertBlock(); EmitBlock(LoopBB); llvm::PHINode *Cur = Builder.CreatePHI(Begin.getType(), 2, "vla.cur"); - Cur->addIncoming(Begin.emitRawPointer(*this), OriginBB); + Cur->addIncoming(Begin.getPointer(), OriginBB); CharUnits CurAlign = Loc.getAlignment().alignmentOfArrayElement(EltSize); auto *I = Builder.CreateMemCpy(Address(Cur, Int8Ty, CurAlign), @@ -2287,7 +2283,7 @@ void CodeGenFunction::emitDestroy(Address addr, QualType type, checkZeroLength = false; } - llvm::Value *begin = addr.emitRawPointer(*this); + llvm::Value *begin = addr.getPointer(); llvm::Value *end = Builder.CreateInBoundsGEP(addr.getElementType(), begin, length); emitArrayDestroy(begin, end, type, elementAlign, destroyer, @@ -2547,7 +2543,7 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg, } Address DeclPtr = Address::invalid(); - RawAddress AllocaPtr = Address::invalid(); + Address AllocaPtr = Address::invalid(); bool DoStore = false; bool IsScalar = hasScalarEvaluationKind(Ty); bool UseIndirectDebugAddress = false; @@ -2559,8 +2555,8 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg, // Indirect argument is in alloca address space, which may be different // from the default address space. auto AllocaAS = CGM.getASTAllocaAddressSpace(); - auto *V = DeclPtr.emitRawPointer(*this); - AllocaPtr = RawAddress(V, DeclPtr.getElementType(), DeclPtr.getAlignment()); + auto *V = DeclPtr.getPointer(); + AllocaPtr = DeclPtr; // For truly ABI indirect arguments -- those that are not `byval` -- store // the address of the argument on the stack to preserve debug information. @@ -2699,7 +2695,7 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg, } if (D.hasAttr()) - EmitVarAnnotations(&D, DeclPtr.emitRawPointer(*this)); + EmitVarAnnotations(&D, DeclPtr.getPointer()); // We can only check return value nullability if all arguments to the // function satisfy their nullability preconditions. This makes it necessary diff --git a/clang/lib/CodeGen/CGException.cpp b/clang/lib/CodeGen/CGException.cpp index 34f289334a7df9..5a9d06da12de57 100644 --- a/clang/lib/CodeGen/CGException.cpp +++ b/clang/lib/CodeGen/CGException.cpp @@ -397,7 +397,7 @@ namespace { void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) { // Make sure the exception object is cleaned up if there's an // exception during initialization. - pushFullExprCleanup(EHCleanup, addr.emitRawPointer(*this)); + pushFullExprCleanup(EHCleanup, addr.getPointer()); EHScopeStack::stable_iterator cleanup = EHStack.stable_begin(); // __cxa_allocate_exception returns a void*; we need to cast this @@ -416,8 +416,8 @@ void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) { /*IsInit*/ true); // Deactivate the cleanup block. - DeactivateCleanupBlock( - cleanup, cast(typedAddr.emitRawPointer(*this))); + DeactivateCleanupBlock(cleanup, + cast(typedAddr.getPointer())); } Address CodeGenFunction::getExceptionSlot() { @@ -1834,8 +1834,7 @@ Address CodeGenFunction::recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF, llvm::Value *ParentFP) { llvm::CallInst *RecoverCall = nullptr; CGBuilderTy Builder(*this, AllocaInsertPt); - if (auto *ParentAlloca = - dyn_cast_or_null(ParentVar.getBasePointer())) { + if (auto *ParentAlloca = dyn_cast(ParentVar.getPointer())) { // Mark the variable escaped if nobody else referenced it and compute the // localescape index. auto InsertPair = ParentCGF.EscapedLocals.insert( @@ -1852,8 +1851,8 @@ Address CodeGenFunction::recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF, // If the parent didn't have an alloca, we're doing some nested outlining. // Just clone the existing localrecover call, but tweak the FP argument to // use our FP value. All other arguments are constants. - auto *ParentRecover = cast( - ParentVar.emitRawPointer(*this)->stripPointerCasts()); + auto *ParentRecover = + cast(ParentVar.getPointer()->stripPointerCasts()); assert(ParentRecover->getIntrinsicID() == llvm::Intrinsic::localrecover && "expected alloca or localrecover in parent LocalDeclMap"); RecoverCall = cast(ParentRecover->clone()); @@ -1926,8 +1925,7 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF, if (isa(D) && D->getType() == getContext().VoidPtrTy) { assert(D->getName().starts_with("frame_pointer")); - FramePtrAddrAlloca = - cast(I.second.getBasePointer()); + FramePtrAddrAlloca = cast(I.second.getPointer()); break; } } @@ -1988,8 +1986,7 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF, LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField); if (!LambdaThisCaptureField->getType()->isPointerType()) { - CXXThisValue = - ThisFieldLValue.getAddress(*this).emitRawPointer(*this); + CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer(); } else { CXXThisValue = EmitLoadOfLValue(ThisFieldLValue, SourceLocation()) .getScalarVal(); diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp index 36872c0fedb76e..6491835cf8ef17 100644 --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -65,21 +65,21 @@ static llvm::cl::opt ClSanitizeDebugDeoptimization( /// CreateTempAlloca - This creates a alloca and inserts it into the entry /// block. -RawAddress -CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits Align, - const Twine &Name, - llvm::Value *ArraySize) { +Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, + CharUnits Align, + const Twine &Name, + llvm::Value *ArraySize) { auto Alloca = CreateTempAlloca(Ty, Name, ArraySize); Alloca->setAlignment(Align.getAsAlign()); - return RawAddress(Alloca, Ty, Align, KnownNonNull); + return Address(Alloca, Ty, Align, KnownNonNull); } /// CreateTempAlloca - This creates a alloca and inserts it into the entry /// block. The alloca is casted to default address space if necessary. -RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, - const Twine &Name, - llvm::Value *ArraySize, - RawAddress *AllocaAddr) { +Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, + const Twine &Name, + llvm::Value *ArraySize, + Address *AllocaAddr) { auto Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize); if (AllocaAddr) *AllocaAddr = Alloca; @@ -101,7 +101,7 @@ RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, Ty->getPointerTo(DestAddrSpace), /*non-null*/ true); } - return RawAddress(V, Ty, Align, KnownNonNull); + return Address(V, Ty, Align, KnownNonNull); } /// CreateTempAlloca - This creates an alloca and inserts it into the entry @@ -120,29 +120,28 @@ llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, /// default alignment of the corresponding LLVM type, which is *not* /// guaranteed to be related in any way to the expected alignment of /// an AST type that might have been lowered to Ty. -RawAddress CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty, - const Twine &Name) { +Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty, + const Twine &Name) { CharUnits Align = CharUnits::fromQuantity(CGM.getDataLayout().getPrefTypeAlign(Ty)); return CreateTempAlloca(Ty, Align, Name); } -RawAddress CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) { +Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) { CharUnits Align = getContext().getTypeAlignInChars(Ty); return CreateTempAlloca(ConvertType(Ty), Align, Name); } -RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name, - RawAddress *Alloca) { +Address CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name, + Address *Alloca) { // FIXME: Should we prefer the preferred type alignment here? return CreateMemTemp(Ty, getContext().getTypeAlignInChars(Ty), Name, Alloca); } -RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align, - const Twine &Name, - RawAddress *Alloca) { - RawAddress Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name, - /*ArraySize=*/nullptr, Alloca); +Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align, + const Twine &Name, Address *Alloca) { + Address Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name, + /*ArraySize=*/nullptr, Alloca); if (Ty->isConstantMatrixType()) { auto *ArrayTy = cast(Result.getElementType()); @@ -155,14 +154,13 @@ RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align, return Result; } -RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, - CharUnits Align, - const Twine &Name) { +Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, CharUnits Align, + const Twine &Name) { return CreateTempAllocaWithoutCast(ConvertTypeForMem(Ty), Align, Name); } -RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, - const Twine &Name) { +Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, + const Twine &Name) { return CreateMemTempWithoutCast(Ty, getContext().getTypeAlignInChars(Ty), Name); } @@ -361,7 +359,7 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, } else { CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor( GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete)); - CleanupArg = cast(ReferenceTemporary.emitRawPointer(CGF)); + CleanupArg = cast(ReferenceTemporary.getPointer()); } CGF.CGM.getCXXABI().registerGlobalDtor( CGF, *cast(M->getExtendingDecl()), CleanupFn, CleanupArg); @@ -386,10 +384,10 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, } } -static RawAddress createReferenceTemporary(CodeGenFunction &CGF, - const MaterializeTemporaryExpr *M, - const Expr *Inner, - RawAddress *Alloca = nullptr) { +static Address createReferenceTemporary(CodeGenFunction &CGF, + const MaterializeTemporaryExpr *M, + const Expr *Inner, + Address *Alloca = nullptr) { auto &TCG = CGF.getTargetHooks(); switch (M->getStorageDuration()) { case SD_FullExpression: @@ -418,7 +416,7 @@ static RawAddress createReferenceTemporary(CodeGenFunction &CGF, GV->getValueType()->getPointerTo( CGF.getContext().getTargetAddressSpace(LangAS::Default))); // FIXME: Should we put the new global into a COMDAT? - return RawAddress(C, GV->getValueType(), alignment); + return Address(C, GV->getValueType(), alignment); } return CGF.CreateMemTemp(Ty, "ref.tmp", Alloca); } @@ -450,7 +448,7 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { auto ownership = M->getType().getObjCLifetime(); if (ownership != Qualifiers::OCL_None && ownership != Qualifiers::OCL_ExplicitNone) { - RawAddress Object = createReferenceTemporary(*this, M, E); + Address Object = createReferenceTemporary(*this, M, E); if (auto *Var = dyn_cast(Object.getPointer())) { llvm::Type *Ty = ConvertTypeForMem(E->getType()); Object = Object.withElementType(Ty); @@ -504,8 +502,8 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { } // Create and initialize the reference temporary. - RawAddress Alloca = Address::invalid(); - RawAddress Object = createReferenceTemporary(*this, M, E, &Alloca); + Address Alloca = Address::invalid(); + Address Object = createReferenceTemporary(*this, M, E, &Alloca); if (auto *Var = dyn_cast( Object.getPointer()->stripPointerCasts())) { llvm::Type *TemporaryType = ConvertTypeForMem(E->getType()); @@ -1113,12 +1111,12 @@ llvm::Value *CodeGenFunction::EmitCountedByFieldExpr( } else if (const MemberExpr *ME = dyn_cast(StructBase)) { LValue LV = EmitMemberExpr(ME); Address Addr = LV.getAddress(*this); - Res = Addr.emitRawPointer(*this); + Res = Addr.getPointer(); } else if (StructBase->getType()->isPointerType()) { LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; Address Addr = EmitPointerWithAlignment(StructBase, &BaseInfo, &TBAAInfo); - Res = Addr.emitRawPointer(*this); + Res = Addr.getPointer(); } else { return nullptr; } @@ -1284,7 +1282,8 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) { if (BaseInfo) BaseInfo->mergeForCast(TargetTypeBaseInfo); - Addr.setAlignment(Align); + Addr = Address(Addr.getPointer(), Addr.getElementType(), Align, + IsKnownNonNull); } } @@ -1301,8 +1300,8 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, CGF.ConvertTypeForMem(E->getType()->getPointeeType()); Addr = Addr.withElementType(ElemTy); if (CE->getCastKind() == CK_AddressSpaceConversion) - Addr = CGF.Builder.CreateAddrSpaceCast( - Addr, CGF.ConvertType(E->getType()), ElemTy); + Addr = CGF.Builder.CreateAddrSpaceCast(Addr, + CGF.ConvertType(E->getType())); return Addr; } break; @@ -1365,9 +1364,10 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, // TODO: conditional operators, comma. // Otherwise, use the alignment of the type. - return CGF.makeNaturalAddressForPointer( - CGF.EmitScalarExpr(E), E->getType()->getPointeeType(), CharUnits(), - /*ForPointeeType=*/true, BaseInfo, TBAAInfo, IsKnownNonNull); + CharUnits Align = + CGF.CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo); + llvm::Type *ElemTy = CGF.ConvertTypeForMem(E->getType()->getPointeeType()); + return Address(CGF.EmitScalarExpr(E), ElemTy, Align, IsKnownNonNull); } /// EmitPointerWithAlignment - Given an expression of pointer type, try to @@ -1468,7 +1468,8 @@ LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) { if (IsBaseCXXThis || isa(ME->getBase())) SkippedChecks.set(SanitizerKind::Null, true); } - EmitTypeCheck(TCK, E->getExprLoc(), LV, E->getType(), SkippedChecks); + EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(*this), E->getType(), + LV.getAlignment(), SkippedChecks); } return LV; } @@ -1580,11 +1581,11 @@ LValue CodeGenFunction::EmitLValueHelper(const Expr *E, // Defend against branches out of gnu statement expressions surrounded by // cleanups. Address Addr = LV.getAddress(*this); - llvm::Value *V = Addr.getBasePointer(); + llvm::Value *V = Addr.getPointer(); Scope.ForceCleanup({&V}); - Addr.replaceBasePointer(V); - return LValue::MakeAddr(Addr, LV.getType(), getContext(), - LV.getBaseInfo(), LV.getTBAAInfo()); + return LValue::MakeAddr(Addr.withPointer(V, Addr.isKnownNonNull()), + LV.getType(), getContext(), LV.getBaseInfo(), + LV.getTBAAInfo()); } // FIXME: Is it possible to create an ExprWithCleanups that produces a // bitfield lvalue or some other non-simple lvalue? @@ -1928,7 +1929,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo, bool isNontemporal) { - if (auto *GV = dyn_cast(Addr.getBasePointer())) + if (auto *GV = dyn_cast(Addr.getPointer())) if (GV->isThreadLocal()) Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV), NotKnownNonNull); @@ -2038,9 +2039,8 @@ llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) { // Convert the pointer of \p Addr to a pointer to a vector (the value type of // MatrixType), if it points to a array (the memory type of MatrixType). -static RawAddress MaybeConvertMatrixAddress(RawAddress Addr, - CodeGenFunction &CGF, - bool IsVector = true) { +static Address MaybeConvertMatrixAddress(Address Addr, CodeGenFunction &CGF, + bool IsVector = true) { auto *ArrayTy = dyn_cast(Addr.getElementType()); if (ArrayTy && IsVector) { auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(), @@ -2077,7 +2077,7 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo, bool isInit, bool isNontemporal) { - if (auto *GV = dyn_cast(Addr.getBasePointer())) + if (auto *GV = dyn_cast(Addr.getPointer())) if (GV->isThreadLocal()) Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV), NotKnownNonNull); @@ -2432,12 +2432,14 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL"); llvm::Type *ResultType = IntPtrTy; Address dst = EmitPointerWithAlignment(Dst.getBaseIvarExp()); - llvm::Value *RHS = dst.emitRawPointer(*this); + llvm::Value *RHS = dst.getPointer(); RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast"); - llvm::Value *LHS = Builder.CreatePtrToInt(LvalueDst.emitRawPointer(*this), - ResultType, "sub.ptr.lhs.cast"); + llvm::Value *LHS = + Builder.CreatePtrToInt(LvalueDst.getPointer(), ResultType, + "sub.ptr.lhs.cast"); llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset"); - CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst, BytesBetween); + CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst, + BytesBetween); } else if (Dst.isGlobalObjCRef()) { CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst, Dst.isThreadLocalRef()); @@ -2768,9 +2770,12 @@ CodeGenFunction::EmitLoadOfReference(LValue RefLVal, llvm::LoadInst *Load = Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile()); CGM.DecorateInstructionWithTBAA(Load, RefLVal.getTBAAInfo()); - return makeNaturalAddressForPointer(Load, RefLVal.getType()->getPointeeType(), - CharUnits(), /*ForPointeeType=*/true, - PointeeBaseInfo, PointeeTBAAInfo); + + QualType PointeeType = RefLVal.getType()->getPointeeType(); + CharUnits Align = CGM.getNaturalTypeAlignment( + PointeeType, PointeeBaseInfo, PointeeTBAAInfo, + /* forPointeeType= */ true); + return Address(Load, ConvertTypeForMem(PointeeType), Align); } LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) { @@ -2787,9 +2792,10 @@ Address CodeGenFunction::EmitLoadOfPointer(Address Ptr, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo) { llvm::Value *Addr = Builder.CreateLoad(Ptr); - return makeNaturalAddressForPointer(Addr, PtrTy->getPointeeType(), - CharUnits(), /*ForPointeeType=*/true, - BaseInfo, TBAAInfo); + return Address(Addr, ConvertTypeForMem(PtrTy->getPointeeType()), + CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(), BaseInfo, + TBAAInfo, + /*forPointeeType=*/true)); } LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr, @@ -2985,7 +2991,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { /* BaseInfo= */ nullptr, /* TBAAInfo= */ nullptr, /* forPointeeType= */ true); - Addr = makeNaturalAddressForPointer(Val, T, Alignment); + Addr = Address(Val, ConvertTypeForMem(E->getType()), Alignment); } return MakeAddrLValue(Addr, T, AlignmentSource::Decl); } @@ -3017,12 +3023,11 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD), CapturedStmtInfo->getContextValue()); Address LValueAddress = CapLVal.getAddress(*this); - CapLVal = MakeAddrLValue(Address(LValueAddress.emitRawPointer(*this), - LValueAddress.getElementType(), - getContext().getDeclAlign(VD)), - CapLVal.getType(), - LValueBaseInfo(AlignmentSource::Decl), - CapLVal.getTBAAInfo()); + CapLVal = MakeAddrLValue( + Address(LValueAddress.getPointer(), LValueAddress.getElementType(), + getContext().getDeclAlign(VD)), + CapLVal.getType(), LValueBaseInfo(AlignmentSource::Decl), + CapLVal.getTBAAInfo()); // Mark lvalue as nontemporal if the variable is marked as nontemporal // in simd context. if (getLangOpts().OpenMP && @@ -3078,8 +3083,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { // Handle threadlocal function locals. if (VD->getTLSKind() != VarDecl::TLS_None) addr = addr.withPointer( - Builder.CreateThreadLocalAddress(addr.getBasePointer()), - NotKnownNonNull); + Builder.CreateThreadLocalAddress(addr.getPointer()), NotKnownNonNull); // Check for OpenMP threadprivate variables. if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd && @@ -3347,7 +3351,7 @@ llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) { // Pointers are passed directly, everything else is passed by address. if (!V->getType()->isPointerTy()) { - RawAddress Ptr = CreateDefaultAlignTempAlloca(V->getType()); + Address Ptr = CreateDefaultAlignTempAlloca(V->getType()); Builder.CreateStore(V, Ptr); V = Ptr.getPointer(); } @@ -3920,21 +3924,6 @@ static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF, } } -static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, - ArrayRef indices, - llvm::Type *elementType, bool inbounds, - bool signedIndices, SourceLocation loc, - CharUnits align, - const llvm::Twine &name = "arrayidx") { - if (inbounds) { - return CGF.EmitCheckedInBoundsGEP(addr, indices, elementType, signedIndices, - CodeGenFunction::NotSubtraction, loc, - align, name); - } else { - return CGF.Builder.CreateGEP(addr, indices, elementType, align, name); - } -} - static CharUnits getArrayElementAlign(CharUnits arrayAlign, llvm::Value *idx, CharUnits eltSize) { @@ -3982,7 +3971,7 @@ static Address wrapWithBPFPreserveStaticOffset(CodeGenFunction &CGF, llvm::Function *Fn = CGF.CGM.getIntrinsic(llvm::Intrinsic::preserve_static_offset); - llvm::CallInst *Call = CGF.Builder.CreateCall(Fn, {Addr.emitRawPointer(CGF)}); + llvm::CallInst *Call = CGF.Builder.CreateCall(Fn, {Addr.getPointer()}); return Address(Call, Addr.getElementType(), Addr.getAlignment()); } @@ -4045,7 +4034,7 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, // We can use that to compute the best alignment of the element. CharUnits eltSize = CGF.getContext().getTypeSizeInChars(eltType); CharUnits eltAlign = - getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize); + getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize); if (hasBPFPreserveStaticOffset(Base)) addr = wrapWithBPFPreserveStaticOffset(CGF, addr); @@ -4054,19 +4043,19 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, auto LastIndex = dyn_cast(indices.back()); if (!LastIndex || (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, Base))) { - addr = emitArraySubscriptGEP(CGF, addr, indices, - CGF.ConvertTypeForMem(eltType), inbounds, - signedIndices, loc, eltAlign, name); - return addr; + eltPtr = emitArraySubscriptGEP( + CGF, addr.getElementType(), addr.getPointer(), indices, inbounds, + signedIndices, loc, name); } else { // Remember the original array subscript for bpf target unsigned idx = LastIndex->getZExtValue(); llvm::DIType *DbgInfo = nullptr; if (arrayType) DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(*arrayType, loc); - eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex( - addr.getElementType(), addr.emitRawPointer(CGF), indices.size() - 1, - idx, DbgInfo); + eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(addr.getElementType(), + addr.getPointer(), + indices.size() - 1, + idx, DbgInfo); } return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign); @@ -4235,8 +4224,8 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, CharUnits EltAlign = getArrayElementAlign(Addr.getAlignment(), Idx, InterfaceSize); llvm::Value *EltPtr = - emitArraySubscriptGEP(*this, Int8Ty, Addr.emitRawPointer(*this), - ScaledIdx, false, SignedIndices, E->getExprLoc()); + emitArraySubscriptGEP(*this, Int8Ty, Addr.getPointer(), ScaledIdx, + false, SignedIndices, E->getExprLoc()); Addr = Address(EltPtr, OrigBaseElemTy, EltAlign); } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) { // If this is A[i] where A is an array, the frontend will have decayed the @@ -4282,7 +4271,7 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, llvm::Type *CountTy = ConvertType(CountFD->getType()); llvm::Value *Res = Builder.CreateInBoundsGEP( - Int8Ty, Addr.emitRawPointer(*this), + Int8Ty, Addr.getPointer(), Builder.getInt32(OffsetDiff.getQuantity()), ".counted_by.gep"); Res = Builder.CreateAlignedLoad(CountTy, Res, getIntAlign(), ".counted_by.load"); @@ -4528,9 +4517,9 @@ LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, BaseInfo = ArrayLV.getBaseInfo(); TBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, ResultExprTy); } else { - Address Base = - emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, TBAAInfo, BaseTy, - ResultExprTy, IsLowerBound); + Address Base = emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, + TBAAInfo, BaseTy, ResultExprTy, + IsLowerBound); EltPtr = emitArraySubscriptGEP(*this, Base, Idx, ResultExprTy, !getLangOpts().isSignedOverflowDefined(), /*signedIndices=*/false, E->getExprLoc()); @@ -4617,7 +4606,7 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) { SkippedChecks.set(SanitizerKind::Alignment, true); if (IsBaseCXXThis || isa(BaseExpr)) SkippedChecks.set(SanitizerKind::Null, true); - EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr, PtrTy, + EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr.getPointer(), PtrTy, /*Alignment=*/CharUnits::Zero(), SkippedChecks); BaseLV = MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo); } else @@ -4666,8 +4655,8 @@ LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field, LambdaLV = EmitLoadOfReferenceLValue(AddrOfExplicitObject, D->getType(), AlignmentSource::Decl); else - LambdaLV = MakeAddrLValue(AddrOfExplicitObject, - D->getType().getNonReferenceType()); + LambdaLV = MakeNaturalAlignAddrLValue(AddrOfExplicitObject.getPointer(), + D->getType().getNonReferenceType()); } else { QualType LambdaTagType = getContext().getTagDeclType(Field->getParent()); LambdaLV = MakeNaturalAlignAddrLValue(ThisValue, LambdaTagType); @@ -4857,8 +4846,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, // information provided by invariant.group. This is because accessing // fields may leak the real address of dynamic object, which could result // in miscompilation when leaked pointer would be compared. - auto *stripped = - Builder.CreateStripInvariantGroup(addr.emitRawPointer(*this)); + auto *stripped = Builder.CreateStripInvariantGroup(addr.getPointer()); addr = Address(stripped, addr.getElementType(), addr.getAlignment()); } } @@ -4877,11 +4865,10 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, // Remember the original union field index llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(base.getType(), rec->getLocation()); - addr = - Address(Builder.CreatePreserveUnionAccessIndex( - addr.emitRawPointer(*this), - getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo), - addr.getElementType(), addr.getAlignment()); + addr = Address( + Builder.CreatePreserveUnionAccessIndex( + addr.getPointer(), getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo), + addr.getElementType(), addr.getAlignment()); } if (FieldType->isReferenceType()) @@ -5118,9 +5105,11 @@ LValue CodeGenFunction::EmitConditionalOperatorLValue( if (Info.LHS && Info.RHS) { Address lhsAddr = Info.LHS->getAddress(*this); Address rhsAddr = Info.RHS->getAddress(*this); - Address result = mergeAddressesInConditionalExpr( - lhsAddr, rhsAddr, Info.lhsBlock, Info.rhsBlock, - Builder.GetInsertBlock(), expr->getType()); + llvm::PHINode *phi = Builder.CreatePHI(lhsAddr.getType(), 2, "cond-lvalue"); + phi->addIncoming(lhsAddr.getPointer(), Info.lhsBlock); + phi->addIncoming(rhsAddr.getPointer(), Info.rhsBlock); + Address result(phi, lhsAddr.getElementType(), + std::min(lhsAddr.getAlignment(), rhsAddr.getAlignment())); AlignmentSource alignSource = std::max(Info.LHS->getBaseInfo().getAlignmentSource(), Info.RHS->getBaseInfo().getAlignmentSource()); @@ -5207,7 +5196,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { LValue LV = EmitLValue(E->getSubExpr()); Address V = LV.getAddress(*this); const auto *DCE = cast(E); - return MakeNaturalAlignRawAddrLValue(EmitDynamicCast(V, DCE), E->getType()); + return MakeNaturalAlignAddrLValue(EmitDynamicCast(V, DCE), E->getType()); } case CK_ConstructorConversion: @@ -5272,8 +5261,8 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is // performed and the object is not of the derived type. if (sanitizePerformTypeCheck()) - EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(), Derived, - E->getType()); + EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(), + Derived.getPointer(), E->getType()); if (SanOpts.has(SanitizerKind::CFIDerivedCast)) EmitVTablePtrCheckForCast(E->getType(), Derived, @@ -5629,7 +5618,7 @@ LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) { LValue CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) { - return MakeNaturalAlignRawAddrLValue(EmitCXXTypeidExpr(E), E->getType()); + return MakeNaturalAlignAddrLValue(EmitCXXTypeidExpr(E), E->getType()); } Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) { diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp index 143855aa84ca3f..5190b22bcc1622 100644 --- a/clang/lib/CodeGen/CGExprAgg.cpp +++ b/clang/lib/CodeGen/CGExprAgg.cpp @@ -294,10 +294,10 @@ void AggExprEmitter::withReturnValueSlot( // Otherwise, EmitCall will emit its own, notice that it's "unused", and end // its lifetime before we have the chance to emit a proper destructor call. bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() || - (RequiresDestruction && Dest.isIgnored()); + (RequiresDestruction && !Dest.getAddress().isValid()); Address RetAddr = Address::invalid(); - RawAddress RetAllocaAddr = RawAddress::invalid(); + Address RetAllocaAddr = Address::invalid(); EHScopeStack::stable_iterator LifetimeEndBlock; llvm::Value *LifetimeSizePtr = nullptr; @@ -329,8 +329,7 @@ void AggExprEmitter::withReturnValueSlot( if (!UseTemp) return; - assert(Dest.isIgnored() || Dest.emitRawPointer(CGF) != - Src.getAggregatePointer(E->getType(), CGF)); + assert(Dest.isIgnored() || Dest.getPointer() != Src.getAggregatePointer()); EmitFinalDestCopy(E->getType(), Src); if (!RequiresDestruction && LifetimeStartInst) { @@ -449,8 +448,7 @@ AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) { llvm::Value *Zero = llvm::ConstantInt::get(CGF.PtrDiffTy, 0); llvm::Value *IdxStart[] = { Zero, Zero }; llvm::Value *ArrayStart = Builder.CreateInBoundsGEP( - ArrayPtr.getElementType(), ArrayPtr.emitRawPointer(CGF), IdxStart, - "arraystart"); + ArrayPtr.getElementType(), ArrayPtr.getPointer(), IdxStart, "arraystart"); CGF.EmitStoreThroughLValue(RValue::get(ArrayStart), Start); ++Field; @@ -467,8 +465,7 @@ AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) { // End pointer. llvm::Value *IdxEnd[] = { Zero, Size }; llvm::Value *ArrayEnd = Builder.CreateInBoundsGEP( - ArrayPtr.getElementType(), ArrayPtr.emitRawPointer(CGF), IdxEnd, - "arrayend"); + ArrayPtr.getElementType(), ArrayPtr.getPointer(), IdxEnd, "arrayend"); CGF.EmitStoreThroughLValue(RValue::get(ArrayEnd), EndOrLength); } else if (Ctx.hasSameType(Field->getType(), Ctx.getSizeType())) { // Length. @@ -519,9 +516,9 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, // down a level. llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0); llvm::Value *indices[] = { zero, zero }; - llvm::Value *begin = Builder.CreateInBoundsGEP(DestPtr.getElementType(), - DestPtr.emitRawPointer(CGF), - indices, "arrayinit.begin"); + llvm::Value *begin = Builder.CreateInBoundsGEP( + DestPtr.getElementType(), DestPtr.getPointer(), indices, + "arrayinit.begin"); CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType); CharUnits elementAlign = @@ -1062,7 +1059,7 @@ void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) { if (RV.isScalar()) return {RV.getScalarVal(), nullptr}; if (RV.isAggregate()) - return {RV.getAggregatePointer(E->getType(), CGF), nullptr}; + return {RV.getAggregatePointer(), nullptr}; assert(RV.isComplex()); return RV.getComplexVal(); }; @@ -1821,7 +1818,7 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr( // else, clean it up for -O0 builds and general tidiness. if (!pushedCleanup && LV.isSimple()) if (llvm::GetElementPtrInst *GEP = - dyn_cast(LV.emitRawPointer(CGF))) + dyn_cast(LV.getPointer(CGF))) if (GEP->use_empty()) GEP->eraseFromParent(); } @@ -1852,9 +1849,9 @@ void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E, // destPtr is an array*. Construct an elementType* by drilling down a level. llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0); llvm::Value *indices[] = {zero, zero}; - llvm::Value *begin = Builder.CreateInBoundsGEP(destPtr.getElementType(), - destPtr.emitRawPointer(CGF), - indices, "arrayinit.begin"); + llvm::Value *begin = Builder.CreateInBoundsGEP( + destPtr.getElementType(), destPtr.getPointer(), indices, + "arrayinit.begin"); // Prepare to special-case multidimensional array initialization: we avoid // emitting multiple destructor loops in that case. diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp index 5c16a48d3ee6d6..35da0f1a89bc3f 100644 --- a/clang/lib/CodeGen/CGExprCXX.cpp +++ b/clang/lib/CodeGen/CGExprCXX.cpp @@ -280,8 +280,7 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; Address ThisValue = EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo); - This = MakeAddrLValue(ThisValue, Base->getType()->getPointeeType(), - BaseInfo, TBAAInfo); + This = MakeAddrLValue(ThisValue, Base->getType(), BaseInfo, TBAAInfo); } else { This = EmitLValue(Base); } @@ -354,8 +353,10 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( if (IsImplicitObjectCXXThis || isa(IOA)) SkippedChecks.set(SanitizerKind::Null, true); } - EmitTypeCheck(CodeGenFunction::TCK_MemberCall, CallLoc, This, - C.getRecordType(CalleeDecl->getParent()), SkippedChecks); + EmitTypeCheck(CodeGenFunction::TCK_MemberCall, CallLoc, + This.getPointer(*this), + C.getRecordType(CalleeDecl->getParent()), + /*Alignment=*/CharUnits::Zero(), SkippedChecks); // C++ [class.virtual]p12: // Explicit qualification with the scope operator (5.1) suppresses the @@ -454,7 +455,7 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E, else This = EmitLValue(BaseExpr, KnownNonNull).getAddress(*this); - EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.emitRawPointer(*this), + EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.getPointer(), QualType(MPT->getClass(), 0)); // Get the member function pointer. @@ -1108,10 +1109,9 @@ void CodeGenFunction::EmitNewArrayInitializer( // alloca. EndOfInit = CreateTempAlloca(BeginPtr.getType(), getPointerAlign(), "array.init.end"); - CleanupDominator = - Builder.CreateStore(BeginPtr.emitRawPointer(*this), EndOfInit); - pushIrregularPartialArrayCleanup(BeginPtr.emitRawPointer(*this), - EndOfInit, ElementType, ElementAlign, + CleanupDominator = Builder.CreateStore(BeginPtr.getPointer(), EndOfInit); + pushIrregularPartialArrayCleanup(BeginPtr.getPointer(), EndOfInit, + ElementType, ElementAlign, getDestroyer(DtorKind)); Cleanup = EHStack.stable_begin(); } @@ -1123,17 +1123,16 @@ void CodeGenFunction::EmitNewArrayInitializer( // element. TODO: some of these stores can be trivially // observed to be unnecessary. if (EndOfInit.isValid()) { - Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit); + Builder.CreateStore(CurPtr.getPointer(), EndOfInit); } // FIXME: If the last initializer is an incomplete initializer list for // an array, and we have an array filler, we can fold together the two // initialization loops. StoreAnyExprIntoOneUnit(*this, IE, IE->getType(), CurPtr, AggValueSlot::DoesNotOverlap); - CurPtr = Address(Builder.CreateInBoundsGEP(CurPtr.getElementType(), - CurPtr.emitRawPointer(*this), - Builder.getSize(1), - "array.exp.next"), + CurPtr = Address(Builder.CreateInBoundsGEP( + CurPtr.getElementType(), CurPtr.getPointer(), + Builder.getSize(1), "array.exp.next"), CurPtr.getElementType(), StartAlign.alignmentAtOffset((++i) * ElementSize)); } @@ -1187,7 +1186,7 @@ void CodeGenFunction::EmitNewArrayInitializer( // FIXME: Share this cleanup with the constructor call emission rather than // having it create a cleanup of its own. if (EndOfInit.isValid()) - Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit); + Builder.CreateStore(CurPtr.getPointer(), EndOfInit); // Emit a constructor call loop to initialize the remaining elements. if (InitListElements) @@ -1250,15 +1249,15 @@ void CodeGenFunction::EmitNewArrayInitializer( llvm::BasicBlock *ContBB = createBasicBlock("new.loop.end"); // Find the end of the array, hoisted out of the loop. - llvm::Value *EndPtr = Builder.CreateInBoundsGEP( - BeginPtr.getElementType(), BeginPtr.emitRawPointer(*this), NumElements, - "array.end"); + llvm::Value *EndPtr = + Builder.CreateInBoundsGEP(BeginPtr.getElementType(), BeginPtr.getPointer(), + NumElements, "array.end"); // If the number of elements isn't constant, we have to now check if there is // anything left to initialize. if (!ConstNum) { - llvm::Value *IsEmpty = Builder.CreateICmpEQ(CurPtr.emitRawPointer(*this), - EndPtr, "array.isempty"); + llvm::Value *IsEmpty = + Builder.CreateICmpEQ(CurPtr.getPointer(), EndPtr, "array.isempty"); Builder.CreateCondBr(IsEmpty, ContBB, LoopBB); } @@ -1268,20 +1267,19 @@ void CodeGenFunction::EmitNewArrayInitializer( // Set up the current-element phi. llvm::PHINode *CurPtrPhi = Builder.CreatePHI(CurPtr.getType(), 2, "array.cur"); - CurPtrPhi->addIncoming(CurPtr.emitRawPointer(*this), EntryBB); + CurPtrPhi->addIncoming(CurPtr.getPointer(), EntryBB); CurPtr = Address(CurPtrPhi, CurPtr.getElementType(), ElementAlign); // Store the new Cleanup position for irregular Cleanups. if (EndOfInit.isValid()) - Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit); + Builder.CreateStore(CurPtr.getPointer(), EndOfInit); // Enter a partial-destruction Cleanup if necessary. if (!CleanupDominator && needsEHCleanup(DtorKind)) { - llvm::Value *BeginPtrRaw = BeginPtr.emitRawPointer(*this); - llvm::Value *CurPtrRaw = CurPtr.emitRawPointer(*this); - pushRegularPartialArrayCleanup(BeginPtrRaw, CurPtrRaw, ElementType, - ElementAlign, getDestroyer(DtorKind)); + pushRegularPartialArrayCleanup(BeginPtr.getPointer(), CurPtr.getPointer(), + ElementType, ElementAlign, + getDestroyer(DtorKind)); Cleanup = EHStack.stable_begin(); CleanupDominator = Builder.CreateUnreachable(); } @@ -1297,8 +1295,9 @@ void CodeGenFunction::EmitNewArrayInitializer( } // Advance to the next element by adjusting the pointer type as necessary. - llvm::Value *NextPtr = Builder.CreateConstInBoundsGEP1_32( - ElementTy, CurPtr.emitRawPointer(*this), 1, "array.next"); + llvm::Value *NextPtr = + Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr.getPointer(), 1, + "array.next"); // Check whether we've gotten to the end of the array and, if so, // exit the loop. @@ -1524,9 +1523,14 @@ static void EnterNewDeleteCleanup(CodeGenFunction &CGF, typedef CallDeleteDuringNew DirectCleanup; - DirectCleanup *Cleanup = CGF.EHStack.pushCleanupWithExtra( - EHCleanup, E->getNumPlacementArgs(), E->getOperatorDelete(), - NewPtr.emitRawPointer(CGF), AllocSize, E->passAlignment(), AllocAlign); + DirectCleanup *Cleanup = CGF.EHStack + .pushCleanupWithExtra(EHCleanup, + E->getNumPlacementArgs(), + E->getOperatorDelete(), + NewPtr.getPointer(), + AllocSize, + E->passAlignment(), + AllocAlign); for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) { auto &Arg = NewArgs[I + NumNonPlacementArgs]; Cleanup->setPlacementArg(I, Arg.getRValue(CGF), Arg.Ty); @@ -1537,7 +1541,7 @@ static void EnterNewDeleteCleanup(CodeGenFunction &CGF, // Otherwise, we need to save all this stuff. DominatingValue::saved_type SavedNewPtr = - DominatingValue::save(CGF, RValue::get(NewPtr, CGF)); + DominatingValue::save(CGF, RValue::get(NewPtr.getPointer())); DominatingValue::saved_type SavedAllocSize = DominatingValue::save(CGF, RValue::get(AllocSize)); @@ -1614,14 +1618,14 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { // In these cases, discard the computed alignment and use the // formal alignment of the allocated type. if (BaseInfo.getAlignmentSource() != AlignmentSource::Decl) - allocation.setAlignment(allocAlign); + allocation = allocation.withAlignment(allocAlign); // Set up allocatorArgs for the call to operator delete if it's not // the reserved global operator. if (E->getOperatorDelete() && !E->getOperatorDelete()->isReservedGlobalPlacementOperator()) { allocatorArgs.add(RValue::get(allocSize), getContext().getSizeType()); - allocatorArgs.add(RValue::get(allocation, *this), arg->getType()); + allocatorArgs.add(RValue::get(allocation.getPointer()), arg->getType()); } } else { @@ -1709,7 +1713,8 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { llvm::BasicBlock *notNullBB = createBasicBlock("new.notnull"); contBB = createBasicBlock("new.cont"); - llvm::Value *isNull = Builder.CreateIsNull(allocation, "new.isnull"); + llvm::Value *isNull = + Builder.CreateIsNull(allocation.getPointer(), "new.isnull"); Builder.CreateCondBr(isNull, contBB, notNullBB); EmitBlock(notNullBB); } @@ -1755,12 +1760,12 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { SkippedChecks.set(SanitizerKind::Null, nullCheck); EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, E->getAllocatedTypeSourceInfo()->getTypeLoc().getBeginLoc(), - result, allocType, result.getAlignment(), SkippedChecks, - numElements); + result.getPointer(), allocType, result.getAlignment(), + SkippedChecks, numElements); EmitNewInitializer(*this, E, allocType, elementTy, result, numElements, allocSizeWithoutCookie); - llvm::Value *resultPtr = result.emitRawPointer(*this); + llvm::Value *resultPtr = result.getPointer(); if (E->isArray()) { // NewPtr is a pointer to the base element type. If we're // allocating an array of arrays, we'll need to cast back to the @@ -1904,8 +1909,7 @@ static void EmitDestroyingObjectDelete(CodeGenFunction &CGF, CGF.CGM.getCXXABI().emitVirtualObjectDelete(CGF, DE, Ptr, ElementType, Dtor); else - CGF.EmitDeleteCall(DE->getOperatorDelete(), Ptr.emitRawPointer(CGF), - ElementType); + CGF.EmitDeleteCall(DE->getOperatorDelete(), Ptr.getPointer(), ElementType); } /// Emit the code for deleting a single object. @@ -1921,7 +1925,8 @@ static bool EmitObjectDelete(CodeGenFunction &CGF, // dynamic type, the static type shall be a base class of the dynamic type // of the object to be deleted and the static type shall have a virtual // destructor or the behavior is undefined. - CGF.EmitTypeCheck(CodeGenFunction::TCK_MemberCall, DE->getExprLoc(), Ptr, + CGF.EmitTypeCheck(CodeGenFunction::TCK_MemberCall, + DE->getExprLoc(), Ptr.getPointer(), ElementType); const FunctionDecl *OperatorDelete = DE->getOperatorDelete(); @@ -1970,8 +1975,9 @@ static bool EmitObjectDelete(CodeGenFunction &CGF, // Make sure that we call delete even if the dtor throws. // This doesn't have to a conditional cleanup because we're going // to pop it off in a second. - CGF.EHStack.pushCleanup( - NormalAndEHCleanup, Ptr.emitRawPointer(CGF), OperatorDelete, ElementType); + CGF.EHStack.pushCleanup(NormalAndEHCleanup, + Ptr.getPointer(), + OperatorDelete, ElementType); if (Dtor) CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, @@ -2058,7 +2064,7 @@ static void EmitArrayDelete(CodeGenFunction &CGF, CharUnits elementAlign = deletedPtr.getAlignment().alignmentOfArrayElement(elementSize); - llvm::Value *arrayBegin = deletedPtr.emitRawPointer(CGF); + llvm::Value *arrayBegin = deletedPtr.getPointer(); llvm::Value *arrayEnd = CGF.Builder.CreateInBoundsGEP( deletedPtr.getElementType(), arrayBegin, numElements, "delete.end"); @@ -2089,7 +2095,7 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) { llvm::BasicBlock *DeleteNotNull = createBasicBlock("delete.notnull"); llvm::BasicBlock *DeleteEnd = createBasicBlock("delete.end"); - llvm::Value *IsNull = Builder.CreateIsNull(Ptr, "isnull"); + llvm::Value *IsNull = Builder.CreateIsNull(Ptr.getPointer(), "isnull"); Builder.CreateCondBr(IsNull, DeleteEnd, DeleteNotNull); EmitBlock(DeleteNotNull); @@ -2124,8 +2130,10 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) { GEP.push_back(Zero); } - Ptr = Builder.CreateInBoundsGEP(Ptr, GEP, ConvertTypeForMem(DeleteTy), - Ptr.getAlignment(), "del.first"); + Ptr = Address(Builder.CreateInBoundsGEP(Ptr.getElementType(), + Ptr.getPointer(), GEP, "del.first"), + ConvertTypeForMem(DeleteTy), Ptr.getAlignment(), + Ptr.isKnownNonNull()); } assert(ConvertTypeForMem(DeleteTy) == Ptr.getElementType()); @@ -2183,7 +2191,7 @@ static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E, // destruction and the static type of the operand is neither the constructor // or destructor’s class nor one of its bases, the behavior is undefined. CGF.EmitTypeCheck(CodeGenFunction::TCK_DynamicOperation, E->getExprLoc(), - ThisPtr, SrcRecordTy); + ThisPtr.getPointer(), SrcRecordTy); // C++ [expr.typeid]p2: // If the glvalue expression is obtained by applying the unary * operator to @@ -2199,7 +2207,7 @@ static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E, CGF.createBasicBlock("typeid.bad_typeid"); llvm::BasicBlock *EndBlock = CGF.createBasicBlock("typeid.end"); - llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr); + llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr.getPointer()); CGF.Builder.CreateCondBr(IsNull, BadTypeidBlock, EndBlock); CGF.EmitBlock(BadTypeidBlock); @@ -2285,7 +2293,8 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr, // construction or destruction and the static type of the operand is not a // pointer to or object of the constructor or destructor’s own class or one // of its bases, the dynamic_cast results in undefined behavior. - EmitTypeCheck(TCK_DynamicOperation, DCE->getExprLoc(), ThisAddr, SrcRecordTy); + EmitTypeCheck(TCK_DynamicOperation, DCE->getExprLoc(), ThisAddr.getPointer(), + SrcRecordTy); if (DCE->isAlwaysNull()) { if (llvm::Value *T = EmitDynamicCastToNull(*this, DestTy)) { @@ -2320,7 +2329,7 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr, CastNull = createBasicBlock("dynamic_cast.null"); CastNotNull = createBasicBlock("dynamic_cast.notnull"); - llvm::Value *IsNull = Builder.CreateIsNull(ThisAddr); + llvm::Value *IsNull = Builder.CreateIsNull(ThisAddr.getPointer()); Builder.CreateCondBr(IsNull, CastNull, CastNotNull); EmitBlock(CastNotNull); } diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp index 36d7493d9a6baf..67a3cdc770426e 100644 --- a/clang/lib/CodeGen/CGExprConstant.cpp +++ b/clang/lib/CodeGen/CGExprConstant.cpp @@ -800,8 +800,8 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD, // Add a vtable pointer, if we need one and it hasn't already been added. if (Layout.hasOwnVFPtr()) { llvm::Constant *VTableAddressPoint = - CGM.getCXXABI().getVTableAddressPoint(BaseSubobject(CD, Offset), - VTableClass); + CGM.getCXXABI().getVTableAddressPointForConstExpr( + BaseSubobject(CD, Offset), VTableClass); if (!AppendBytes(Offset, VTableAddressPoint)) return false; } diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp index 83247aa48f8609..8536570087ad0f 100644 --- a/clang/lib/CodeGen/CGExprScalar.cpp +++ b/clang/lib/CodeGen/CGExprScalar.cpp @@ -2250,7 +2250,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { // performed and the object is not of the derived type. if (CGF.sanitizePerformTypeCheck()) CGF.EmitTypeCheck(CodeGenFunction::TCK_DowncastPointer, CE->getExprLoc(), - Derived, DestTy->getPointeeType()); + Derived.getPointer(), DestTy->getPointeeType()); if (CGF.SanOpts.has(SanitizerKind::CFIDerivedCast)) CGF.EmitVTablePtrCheckForCast(DestTy->getPointeeType(), Derived, @@ -2258,14 +2258,13 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { CodeGenFunction::CFITCK_DerivedCast, CE->getBeginLoc()); - return CGF.getAsNaturalPointerTo(Derived, CE->getType()->getPointeeType()); + return Derived.getPointer(); } case CK_UncheckedDerivedToBase: case CK_DerivedToBase: { // The EmitPointerWithAlignment path does this fine; just discard // the alignment. - return CGF.getAsNaturalPointerTo(CGF.EmitPointerWithAlignment(CE), - CE->getType()->getPointeeType()); + return CGF.EmitPointerWithAlignment(CE).getPointer(); } case CK_Dynamic: { @@ -2275,8 +2274,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { } case CK_ArrayToPointerDecay: - return CGF.getAsNaturalPointerTo(CGF.EmitArrayToPointerDecay(E), - CE->getType()->getPointeeType()); + return CGF.EmitArrayToPointerDecay(E).getPointer(); case CK_FunctionToPointerDecay: return EmitLValue(E).getPointer(CGF); @@ -5590,16 +5588,3 @@ CodeGenFunction::EmitCheckedInBoundsGEP(llvm::Type *ElemTy, Value *Ptr, return GEPVal; } - -Address CodeGenFunction::EmitCheckedInBoundsGEP( - Address Addr, ArrayRef IdxList, llvm::Type *elementType, - bool SignedIndices, bool IsSubtraction, SourceLocation Loc, CharUnits Align, - const Twine &Name) { - if (!SanOpts.has(SanitizerKind::PointerOverflow)) - return Builder.CreateInBoundsGEP(Addr, IdxList, elementType, Align, Name); - - return RawAddress( - EmitCheckedInBoundsGEP(Addr.getElementType(), Addr.emitRawPointer(*this), - IdxList, SignedIndices, IsSubtraction, Loc, Name), - elementType, Align); -} diff --git a/clang/lib/CodeGen/CGNonTrivialStruct.cpp b/clang/lib/CodeGen/CGNonTrivialStruct.cpp index 8fade0fac21e93..75c1d7fbea8406 100644 --- a/clang/lib/CodeGen/CGNonTrivialStruct.cpp +++ b/clang/lib/CodeGen/CGNonTrivialStruct.cpp @@ -366,7 +366,7 @@ template struct GenFuncBase { llvm::Value *SizeInBytes = CGF.Builder.CreateNUWMul(BaseEltSizeVal, NumElts); llvm::Value *DstArrayEnd = CGF.Builder.CreateInBoundsGEP( - CGF.Int8Ty, DstAddr.emitRawPointer(CGF), SizeInBytes); + CGF.Int8Ty, DstAddr.getPointer(), SizeInBytes); llvm::BasicBlock *PreheaderBB = CGF.Builder.GetInsertBlock(); // Create the header block and insert the phi instructions. @@ -376,7 +376,7 @@ template struct GenFuncBase { for (unsigned I = 0; I < N; ++I) { PHIs[I] = CGF.Builder.CreatePHI(CGF.CGM.Int8PtrPtrTy, 2, "addr.cur"); - PHIs[I]->addIncoming(StartAddrs[I].emitRawPointer(CGF), PreheaderBB); + PHIs[I]->addIncoming(StartAddrs[I].getPointer(), PreheaderBB); } // Create the exit and loop body blocks. @@ -410,7 +410,7 @@ template struct GenFuncBase { // Instrs to update the destination and source addresses. // Update phi instructions. NewAddrs[I] = getAddrWithOffset(NewAddrs[I], EltSize); - PHIs[I]->addIncoming(NewAddrs[I].emitRawPointer(CGF), LoopBB); + PHIs[I]->addIncoming(NewAddrs[I].getPointer(), LoopBB); } // Insert an unconditional branch to the header block. @@ -488,7 +488,7 @@ template struct GenFuncBase { for (unsigned I = 0; I < N; ++I) { Alignments[I] = Addrs[I].getAlignment(); - Ptrs[I] = Addrs[I].emitRawPointer(CallerCGF); + Ptrs[I] = Addrs[I].getPointer(); } if (llvm::Function *F = diff --git a/clang/lib/CodeGen/CGObjC.cpp b/clang/lib/CodeGen/CGObjC.cpp index c7f497a7c8451b..f3a948cf13f9c9 100644 --- a/clang/lib/CodeGen/CGObjC.cpp +++ b/clang/lib/CodeGen/CGObjC.cpp @@ -94,8 +94,8 @@ CodeGenFunction::EmitObjCBoxedExpr(const ObjCBoxedExpr *E) { // and cast value to correct type Address Temporary = CreateMemTemp(SubExpr->getType()); EmitAnyExprToMem(SubExpr, Temporary, Qualifiers(), /*isInit*/ true); - llvm::Value *BitCast = Builder.CreateBitCast( - Temporary.emitRawPointer(*this), ConvertType(ArgQT)); + llvm::Value *BitCast = + Builder.CreateBitCast(Temporary.getPointer(), ConvertType(ArgQT)); Args.add(RValue::get(BitCast), ArgQT); // Create char array to store type encoding @@ -204,11 +204,11 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E, ObjCMethodDecl::param_const_iterator PI = MethodWithObjects->param_begin(); const ParmVarDecl *argDecl = *PI++; QualType ArgQT = argDecl->getType().getUnqualifiedType(); - Args.add(RValue::get(Objects, *this), ArgQT); + Args.add(RValue::get(Objects.getPointer()), ArgQT); if (DLE) { argDecl = *PI++; ArgQT = argDecl->getType().getUnqualifiedType(); - Args.add(RValue::get(Keys, *this), ArgQT); + Args.add(RValue::get(Keys.getPointer()), ArgQT); } argDecl = *PI; ArgQT = argDecl->getType().getUnqualifiedType(); @@ -827,7 +827,7 @@ static void emitStructGetterCall(CodeGenFunction &CGF, ObjCIvarDecl *ivar, // sizeof (Type of Ivar), isAtomic, false); CallArgList args; - llvm::Value *dest = CGF.ReturnValue.emitRawPointer(CGF); + llvm::Value *dest = CGF.ReturnValue.getPointer(); args.add(RValue::get(dest), Context.VoidPtrTy); args.add(RValue::get(src), Context.VoidPtrTy); @@ -1147,8 +1147,8 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, callCStructCopyConstructor(Dst, Src); } else { ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl(); - emitCPPObjectAtomicGetterCall(*this, ReturnValue.emitRawPointer(*this), - ivar, AtomicHelperFn); + emitCPPObjectAtomicGetterCall(*this, ReturnValue.getPointer(), ivar, + AtomicHelperFn); } return; } @@ -1163,7 +1163,7 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, } else { ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl(); - emitCPPObjectAtomicGetterCall(*this, ReturnValue.emitRawPointer(*this), + emitCPPObjectAtomicGetterCall(*this, ReturnValue.getPointer(), ivar, AtomicHelperFn); } return; @@ -1287,7 +1287,7 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, case TEK_Scalar: { llvm::Value *value; if (propType->isReferenceType()) { - value = LV.getAddress(*this).emitRawPointer(*this); + value = LV.getAddress(*this).getPointer(); } else { // We want to load and autoreleaseReturnValue ARC __weak ivars. if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) { @@ -1821,14 +1821,16 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ CallArgList Args; // The first argument is a temporary of the enumeration-state type. - Args.add(RValue::get(StatePtr, *this), getContext().getPointerType(StateTy)); + Args.add(RValue::get(StatePtr.getPointer()), + getContext().getPointerType(StateTy)); // The second argument is a temporary array with space for NumItems // pointers. We'll actually be loading elements from the array // pointer written into the control state; this buffer is so that // collections that *aren't* backed by arrays can still queue up // batches of elements. - Args.add(RValue::get(ItemsPtr, *this), getContext().getPointerType(ItemsTy)); + Args.add(RValue::get(ItemsPtr.getPointer()), + getContext().getPointerType(ItemsTy)); // The third argument is the capacity of that temporary array. llvm::Type *NSUIntegerTy = ConvertType(getContext().getNSUIntegerType()); @@ -2196,7 +2198,7 @@ static llvm::Value *emitARCLoadOperation(CodeGenFunction &CGF, Address addr, if (!fn) fn = getARCIntrinsic(IntID, CGF.CGM); - return CGF.EmitNounwindRuntimeCall(fn, addr.emitRawPointer(CGF)); + return CGF.EmitNounwindRuntimeCall(fn, addr.getPointer()); } /// Perform an operation having the following signature: @@ -2214,8 +2216,9 @@ static llvm::Value *emitARCStoreOperation(CodeGenFunction &CGF, Address addr, llvm::Type *origType = value->getType(); llvm::Value *args[] = { - CGF.Builder.CreateBitCast(addr.emitRawPointer(CGF), CGF.Int8PtrPtrTy), - CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy)}; + CGF.Builder.CreateBitCast(addr.getPointer(), CGF.Int8PtrPtrTy), + CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy) + }; llvm::CallInst *result = CGF.EmitNounwindRuntimeCall(fn, args); if (ignored) return nullptr; @@ -2234,8 +2237,9 @@ static void emitARCCopyOperation(CodeGenFunction &CGF, Address dst, Address src, fn = getARCIntrinsic(IntID, CGF.CGM); llvm::Value *args[] = { - CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), CGF.Int8PtrPtrTy), - CGF.Builder.CreateBitCast(src.emitRawPointer(CGF), CGF.Int8PtrPtrTy)}; + CGF.Builder.CreateBitCast(dst.getPointer(), CGF.Int8PtrPtrTy), + CGF.Builder.CreateBitCast(src.getPointer(), CGF.Int8PtrPtrTy) + }; CGF.EmitNounwindRuntimeCall(fn, args); } @@ -2486,8 +2490,9 @@ llvm::Value *CodeGenFunction::EmitARCStoreStrongCall(Address addr, fn = getARCIntrinsic(llvm::Intrinsic::objc_storeStrong, CGM); llvm::Value *args[] = { - Builder.CreateBitCast(addr.emitRawPointer(*this), Int8PtrPtrTy), - Builder.CreateBitCast(value, Int8PtrTy)}; + Builder.CreateBitCast(addr.getPointer(), Int8PtrPtrTy), + Builder.CreateBitCast(value, Int8PtrTy) + }; EmitNounwindRuntimeCall(fn, args); if (ignored) return nullptr; @@ -2638,7 +2643,7 @@ void CodeGenFunction::EmitARCDestroyWeak(Address addr) { if (!fn) fn = getARCIntrinsic(llvm::Intrinsic::objc_destroyWeak, CGM); - EmitNounwindRuntimeCall(fn, addr.emitRawPointer(*this)); + EmitNounwindRuntimeCall(fn, addr.getPointer()); } /// void \@objc_moveWeak(i8** %dest, i8** %src) diff --git a/clang/lib/CodeGen/CGObjCGNU.cpp b/clang/lib/CodeGen/CGObjCGNU.cpp index 4e7f777ba1d916..a36b0cdddaf0af 100644 --- a/clang/lib/CodeGen/CGObjCGNU.cpp +++ b/clang/lib/CodeGen/CGObjCGNU.cpp @@ -706,8 +706,7 @@ class CGObjCGCC : public CGObjCGNU { llvm::Value *cmd, MessageSendInfo &MSI) override { CGBuilderTy &Builder = CGF.Builder; llvm::Value *lookupArgs[] = { - EnforceType(Builder, ObjCSuper.emitRawPointer(CGF), PtrToObjCSuperTy), - cmd}; + EnforceType(Builder, ObjCSuper.getPointer(), PtrToObjCSuperTy), cmd}; return CGF.EmitNounwindRuntimeCall(MsgLookupSuperFn, lookupArgs); } @@ -762,8 +761,8 @@ class CGObjCGNUstep : public CGObjCGNU { llvm::FunctionCallee LookupFn = SlotLookupFn; // Store the receiver on the stack so that we can reload it later - RawAddress ReceiverPtr = - CGF.CreateTempAlloca(Receiver->getType(), CGF.getPointerAlign()); + Address ReceiverPtr = + CGF.CreateTempAlloca(Receiver->getType(), CGF.getPointerAlign()); Builder.CreateStore(Receiver, ReceiverPtr); llvm::Value *self; @@ -779,9 +778,9 @@ class CGObjCGNUstep : public CGObjCGNU { LookupFn2->addParamAttr(0, llvm::Attribute::NoCapture); llvm::Value *args[] = { - EnforceType(Builder, ReceiverPtr.getPointer(), PtrToIdTy), - EnforceType(Builder, cmd, SelectorTy), - EnforceType(Builder, self, IdTy)}; + EnforceType(Builder, ReceiverPtr.getPointer(), PtrToIdTy), + EnforceType(Builder, cmd, SelectorTy), + EnforceType(Builder, self, IdTy) }; llvm::CallBase *slot = CGF.EmitRuntimeCallOrInvoke(LookupFn, args); slot->setOnlyReadsMemory(); slot->setMetadata(msgSendMDKind, node); @@ -801,7 +800,7 @@ class CGObjCGNUstep : public CGObjCGNU { llvm::Value *cmd, MessageSendInfo &MSI) override { CGBuilderTy &Builder = CGF.Builder; - llvm::Value *lookupArgs[] = {ObjCSuper.emitRawPointer(CGF), cmd}; + llvm::Value *lookupArgs[] = {ObjCSuper.getPointer(), cmd}; llvm::CallInst *slot = CGF.EmitNounwindRuntimeCall(SlotLookupSuperFn, lookupArgs); @@ -1222,10 +1221,10 @@ class CGObjCGNUstep2 : public CGObjCGNUstep { llvm::Value *cmd, MessageSendInfo &MSI) override { // Don't access the slot unless we're trying to cache the result. CGBuilderTy &Builder = CGF.Builder; - llvm::Value *lookupArgs[] = { - CGObjCGNU::EnforceType(Builder, ObjCSuper.emitRawPointer(CGF), - PtrToObjCSuperTy), - cmd}; + llvm::Value *lookupArgs[] = {CGObjCGNU::EnforceType(Builder, + ObjCSuper.getPointer(), + PtrToObjCSuperTy), + cmd}; return CGF.EmitNounwindRuntimeCall(MsgLookupSuperFn, lookupArgs); } @@ -2187,8 +2186,7 @@ class CGObjCObjFW: public CGObjCGNU { llvm::Value *cmd, MessageSendInfo &MSI) override { CGBuilderTy &Builder = CGF.Builder; llvm::Value *lookupArgs[] = { - EnforceType(Builder, ObjCSuper.emitRawPointer(CGF), PtrToObjCSuperTy), - cmd, + EnforceType(Builder, ObjCSuper.getPointer(), PtrToObjCSuperTy), cmd, }; if (CGM.ReturnTypeUsesSRet(MSI.CallInfo)) @@ -4203,15 +4201,15 @@ void CGObjCGNU::EmitThrowStmt(CodeGenFunction &CGF, llvm::Value * CGObjCGNU::EmitObjCWeakRead(CodeGenFunction &CGF, Address AddrWeakObj) { CGBuilderTy &B = CGF.Builder; - return B.CreateCall( - WeakReadFn, EnforceType(B, AddrWeakObj.emitRawPointer(CGF), PtrToIdTy)); + return B.CreateCall(WeakReadFn, + EnforceType(B, AddrWeakObj.getPointer(), PtrToIdTy)); } void CGObjCGNU::EmitObjCWeakAssign(CodeGenFunction &CGF, llvm::Value *src, Address dst) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); - llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy); + llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy); B.CreateCall(WeakAssignFn, {src, dstVal}); } @@ -4220,7 +4218,7 @@ void CGObjCGNU::EmitObjCGlobalAssign(CodeGenFunction &CGF, bool threadlocal) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); - llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy); + llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy); // FIXME. Add threadloca assign API assert(!threadlocal && "EmitObjCGlobalAssign - Threal Local API NYI"); B.CreateCall(GlobalAssignFn, {src, dstVal}); @@ -4231,7 +4229,7 @@ void CGObjCGNU::EmitObjCIvarAssign(CodeGenFunction &CGF, llvm::Value *ivarOffset) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); - llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), IdTy); + llvm::Value *dstVal = EnforceType(B, dst.getPointer(), IdTy); B.CreateCall(IvarAssignFn, {src, dstVal, ivarOffset}); } @@ -4239,7 +4237,7 @@ void CGObjCGNU::EmitObjCStrongCastAssign(CodeGenFunction &CGF, llvm::Value *src, Address dst) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); - llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy); + llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy); B.CreateCall(StrongCastAssignFn, {src, dstVal}); } @@ -4248,8 +4246,8 @@ void CGObjCGNU::EmitGCMemmoveCollectable(CodeGenFunction &CGF, Address SrcPtr, llvm::Value *Size) { CGBuilderTy &B = CGF.Builder; - llvm::Value *DestPtrVal = EnforceType(B, DestPtr.emitRawPointer(CGF), PtrTy); - llvm::Value *SrcPtrVal = EnforceType(B, SrcPtr.emitRawPointer(CGF), PtrTy); + llvm::Value *DestPtrVal = EnforceType(B, DestPtr.getPointer(), PtrTy); + llvm::Value *SrcPtrVal = EnforceType(B, SrcPtr.getPointer(), PtrTy); B.CreateCall(MemMoveFn, {DestPtrVal, SrcPtrVal, Size}); } diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp index 8a599c10e1caf1..ed8d7b9a065d7b 100644 --- a/clang/lib/CodeGen/CGObjCMac.cpp +++ b/clang/lib/CodeGen/CGObjCMac.cpp @@ -1310,7 +1310,7 @@ class CGObjCMac : public CGObjCCommonMac { /// EmitSelector - Return a Value*, of type ObjCTypes.SelectorPtrTy, /// for the given selector. llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel); - ConstantAddress EmitSelectorAddr(Selector Sel); + Address EmitSelectorAddr(Selector Sel); public: CGObjCMac(CodeGen::CodeGenModule &cgm); @@ -1538,7 +1538,7 @@ class CGObjCNonFragileABIMac : public CGObjCCommonMac { /// EmitSelector - Return a Value*, of type ObjCTypes.SelectorPtrTy, /// for the given selector. llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel); - ConstantAddress EmitSelectorAddr(Selector Sel); + Address EmitSelectorAddr(Selector Sel); /// GetInterfaceEHType - Get the cached ehtype for the given Objective-C /// interface. The return value has type EHTypePtrTy. @@ -2064,8 +2064,9 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, const ObjCMethodDecl *Method) { // Create and init a super structure; this is a (receiver, class) // pair we will pass to objc_msgSendSuper. - RawAddress ObjCSuper = CGF.CreateTempAlloca( - ObjCTypes.SuperTy, CGF.getPointerAlign(), "objc_super"); + Address ObjCSuper = + CGF.CreateTempAlloca(ObjCTypes.SuperTy, CGF.getPointerAlign(), + "objc_super"); llvm::Value *ReceiverAsObject = CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy); CGF.Builder.CreateStore(ReceiverAsObject, @@ -4258,7 +4259,7 @@ namespace { CGF.EmitBlock(FinallyCallExit); CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryExitFn(), - ExceptionData.emitRawPointer(CGF)); + ExceptionData.getPointer()); CGF.EmitBlock(FinallyNoCallExit); @@ -4424,9 +4425,7 @@ void FragileHazards::emitHazardsInNewBlocks() { } static void addIfPresent(llvm::DenseSet &S, Address V) { - if (V.isValid()) - if (llvm::Value *Ptr = V.getBasePointer()) - S.insert(Ptr); + if (V.isValid()) S.insert(V.getPointer()); } void FragileHazards::collectLocals() { @@ -4629,13 +4628,13 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // - Call objc_exception_try_enter to push ExceptionData on top of // the EH stack. CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(), - ExceptionData.emitRawPointer(CGF)); + ExceptionData.getPointer()); // - Call setjmp on the exception data buffer. llvm::Constant *Zero = llvm::ConstantInt::get(CGF.Builder.getInt32Ty(), 0); llvm::Value *GEPIndexes[] = { Zero, Zero, Zero }; llvm::Value *SetJmpBuffer = CGF.Builder.CreateGEP( - ObjCTypes.ExceptionDataTy, ExceptionData.emitRawPointer(CGF), GEPIndexes, + ObjCTypes.ExceptionDataTy, ExceptionData.getPointer(), GEPIndexes, "setjmp_buffer"); llvm::CallInst *SetJmpResult = CGF.EmitNounwindRuntimeCall( ObjCTypes.getSetJmpFn(), SetJmpBuffer, "setjmp_result"); @@ -4674,9 +4673,9 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, } else { // Retrieve the exception object. We may emit multiple blocks but // nothing can cross this so the value is already in SSA form. - llvm::CallInst *Caught = CGF.EmitNounwindRuntimeCall( - ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF), - "caught"); + llvm::CallInst *Caught = + CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(), + ExceptionData.getPointer(), "caught"); // Push the exception to rethrow onto the EH value stack for the // benefit of any @throws in the handlers. @@ -4699,7 +4698,7 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // Enter a new exception try block (in case a @catch block // throws an exception). CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(), - ExceptionData.emitRawPointer(CGF)); + ExceptionData.getPointer()); llvm::CallInst *SetJmpResult = CGF.EmitNounwindRuntimeCall(ObjCTypes.getSetJmpFn(), @@ -4830,9 +4829,9 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // Extract the new exception and save it to the // propagating-exception slot. assert(PropagatingExnVar.isValid()); - llvm::CallInst *NewCaught = CGF.EmitNounwindRuntimeCall( - ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF), - "caught"); + llvm::CallInst *NewCaught = + CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(), + ExceptionData.getPointer(), "caught"); CGF.Builder.CreateStore(NewCaught, PropagatingExnVar); // Don't pop the catch handler; the throw already did. @@ -4862,8 +4861,9 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // Otherwise, just look in the buffer for the exception to throw. } else { - llvm::CallInst *Caught = CGF.EmitNounwindRuntimeCall( - ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF)); + llvm::CallInst *Caught = + CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(), + ExceptionData.getPointer()); PropagatingExn = Caught; } @@ -4906,7 +4906,7 @@ llvm::Value * CGObjCMac::EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF, Address AddrWeakObj) { llvm::Type* DestTy = AddrWeakObj.getElementType(); llvm::Value *AddrWeakObjVal = CGF.Builder.CreateBitCast( - AddrWeakObj.emitRawPointer(CGF), ObjCTypes.PtrObjectPtrTy); + AddrWeakObj.getPointer(), ObjCTypes.PtrObjectPtrTy); llvm::Value *read_weak = CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(), AddrWeakObjVal, "weakread"); @@ -4928,8 +4928,8 @@ void CGObjCMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), - ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = + CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = { src, dstVal }; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(), args, "weakassign"); @@ -4950,8 +4950,8 @@ void CGObjCMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), - ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = + CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; if (!threadlocal) CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(), @@ -4977,8 +4977,8 @@ void CGObjCMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), - ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = + CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal, ivarOffset}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args); } @@ -4997,8 +4997,8 @@ void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), - ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = + CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(), args, "strongassign"); @@ -5007,8 +5007,7 @@ void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF, void CGObjCMac::EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *size) { - llvm::Value *args[] = {DestPtr.emitRawPointer(CGF), - SrcPtr.emitRawPointer(CGF), size}; + llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), size }; CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args); } @@ -5244,7 +5243,7 @@ llvm::Value *CGObjCMac::EmitSelector(CodeGenFunction &CGF, Selector Sel) { return CGF.Builder.CreateLoad(EmitSelectorAddr(Sel)); } -ConstantAddress CGObjCMac::EmitSelectorAddr(Selector Sel) { +Address CGObjCMac::EmitSelectorAddr(Selector Sel) { CharUnits Align = CGM.getPointerAlign(); llvm::GlobalVariable *&Entry = SelectorReferences[Sel]; @@ -5255,7 +5254,7 @@ ConstantAddress CGObjCMac::EmitSelectorAddr(Selector Sel) { Entry->setExternallyInitialized(true); } - return ConstantAddress(Entry, ObjCTypes.SelectorPtrTy, Align); + return Address(Entry, ObjCTypes.SelectorPtrTy, Align); } llvm::Constant *CGObjCCommonMac::GetClassName(StringRef RuntimeName) { @@ -7324,7 +7323,7 @@ CGObjCNonFragileABIMac::EmitVTableMessageSend(CodeGenFunction &CGF, ObjCTypes.MessageRefTy, CGF.getPointerAlign()); // Update the message ref argument. - args[1].setRValue(RValue::get(mref, CGF)); + args[1].setRValue(RValue::get(mref.getPointer())); // Load the function to call from the message ref table. Address calleeAddr = CGF.Builder.CreateStructGEP(mref, 0); @@ -7553,8 +7552,9 @@ CGObjCNonFragileABIMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, // ... // Create and init a super structure; this is a (receiver, class) // pair we will pass to objc_msgSendSuper. - RawAddress ObjCSuper = CGF.CreateTempAlloca( - ObjCTypes.SuperTy, CGF.getPointerAlign(), "objc_super"); + Address ObjCSuper = + CGF.CreateTempAlloca(ObjCTypes.SuperTy, CGF.getPointerAlign(), + "objc_super"); llvm::Value *ReceiverAsObject = CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy); @@ -7594,7 +7594,7 @@ llvm::Value *CGObjCNonFragileABIMac::EmitSelector(CodeGenFunction &CGF, return LI; } -ConstantAddress CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) { +Address CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) { llvm::GlobalVariable *&Entry = SelectorReferences[Sel]; CharUnits Align = CGM.getPointerAlign(); if (!Entry) { @@ -7610,7 +7610,7 @@ ConstantAddress CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) { CGM.addCompilerUsedGlobal(Entry); } - return ConstantAddress(Entry, ObjCTypes.SelectorPtrTy, Align); + return Address(Entry, ObjCTypes.SelectorPtrTy, Align); } /// EmitObjCIvarAssign - Code gen for assigning to a __strong object. @@ -7629,8 +7629,8 @@ void CGObjCNonFragileABIMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), - ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = + CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal, ivarOffset}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args); } @@ -7650,8 +7650,8 @@ void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign( src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), - ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = + CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(), args, "weakassign"); @@ -7660,8 +7660,7 @@ void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign( void CGObjCNonFragileABIMac::EmitGCMemmoveCollectable( CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *Size) { - llvm::Value *args[] = {DestPtr.emitRawPointer(CGF), - SrcPtr.emitRawPointer(CGF), Size}; + llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), Size }; CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args); } @@ -7673,7 +7672,7 @@ llvm::Value * CGObjCNonFragileABIMac::EmitObjCWeakRead( Address AddrWeakObj) { llvm::Type *DestTy = AddrWeakObj.getElementType(); llvm::Value *AddrWeakObjVal = CGF.Builder.CreateBitCast( - AddrWeakObj.emitRawPointer(CGF), ObjCTypes.PtrObjectPtrTy); + AddrWeakObj.getPointer(), ObjCTypes.PtrObjectPtrTy); llvm::Value *read_weak = CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(), AddrWeakObjVal, "weakread"); @@ -7695,8 +7694,8 @@ void CGObjCNonFragileABIMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), - ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = + CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(), args, "weakassign"); @@ -7717,8 +7716,8 @@ void CGObjCNonFragileABIMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), - ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = + CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; if (!threadlocal) CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(), diff --git a/clang/lib/CodeGen/CGObjCRuntime.cpp b/clang/lib/CodeGen/CGObjCRuntime.cpp index 01d0f35da19643..424564f9759995 100644 --- a/clang/lib/CodeGen/CGObjCRuntime.cpp +++ b/clang/lib/CodeGen/CGObjCRuntime.cpp @@ -67,7 +67,7 @@ LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF, V = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, V, Offset, "add.ptr"); if (!Ivar->isBitField()) { - LValue LV = CGF.MakeNaturalAlignRawAddrLValue(V, IvarTy); + LValue LV = CGF.MakeNaturalAlignAddrLValue(V, IvarTy); return LV; } @@ -233,7 +233,7 @@ void CGObjCRuntime::EmitTryCatchStmt(CodeGenFunction &CGF, llvm::Instruction *CPICandidate = Handler.Block->getFirstNonPHI(); if (auto *CPI = dyn_cast_or_null(CPICandidate)) { CGF.CurrentFuncletPad = CPI; - CPI->setOperand(2, CGF.getExceptionSlot().emitRawPointer(CGF)); + CPI->setOperand(2, CGF.getExceptionSlot().getPointer()); CGF.EHStack.pushCleanup(NormalCleanup, CPI); } } @@ -405,7 +405,7 @@ bool CGObjCRuntime::canMessageReceiverBeNull(CodeGenFunction &CGF, auto self = curMethod->getSelfDecl(); if (self->getType().isConstQualified()) { if (auto LI = dyn_cast(receiver->stripPointerCasts())) { - llvm::Value *selfAddr = CGF.GetAddrOfLocalVar(self).emitRawPointer(CGF); + llvm::Value *selfAddr = CGF.GetAddrOfLocalVar(self).getPointer(); if (selfAddr == LI->getPointerOperand()) { return false; } diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp index bc363313dec6f8..00e395c2a207f9 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -622,7 +622,7 @@ static void emitInitWithReductionInitializer(CodeGenFunction &CGF, auto *GV = new llvm::GlobalVariable( CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, Init, Name); - LValue LV = CGF.MakeNaturalAlignRawAddrLValue(GV, Ty); + LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty); RValue InitRVal; switch (CGF.getEvaluationKind(Ty)) { case TEK_Scalar: @@ -668,8 +668,8 @@ static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, llvm::Value *SrcBegin = nullptr; if (DRD) - SrcBegin = SrcAddr.emitRawPointer(CGF); - llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF); + SrcBegin = SrcAddr.getPointer(); + llvm::Value *DestBegin = DestAddr.getPointer(); // Cast from pointer to array type to pointer to single element. llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestAddr.getElementType(), DestBegin, NumElements); @@ -912,7 +912,7 @@ static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, Address OriginalBaseAddress, llvm::Value *Addr) { - RawAddress Tmp = RawAddress::invalid(); + Address Tmp = Address::invalid(); Address TopTmp = Address::invalid(); Address MostTopTmp = Address::invalid(); BaseTy = BaseTy.getNonReferenceType(); @@ -971,10 +971,10 @@ Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address SharedAddr = SharedAddresses[N].first.getAddress(CGF); llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff( SharedAddr.getElementType(), BaseLValue.getPointer(CGF), - SharedAddr.emitRawPointer(CGF)); + SharedAddr.getPointer()); llvm::Value *PrivatePointer = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - PrivateAddr.emitRawPointer(CGF), SharedAddr.getType()); + PrivateAddr.getPointer(), SharedAddr.getType()); llvm::Value *Ptr = CGF.Builder.CreateGEP( SharedAddr.getElementType(), PrivatePointer, Adjustment); return castToBase(CGF, OrigVD->getType(), @@ -1557,7 +1557,7 @@ static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc( return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack, ParentName); } -ConstantAddress CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) { +Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) { auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(VD); }; auto LinkageForVariable = [&VD, this]() { @@ -1579,8 +1579,8 @@ ConstantAddress CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) { LinkageForVariable); if (!addr) - return ConstantAddress::invalid(); - return ConstantAddress(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(VD)); + return Address::invalid(); + return Address(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(VD)); } llvm::Constant * @@ -1604,7 +1604,7 @@ Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, llvm::Type *VarTy = VDAddr.getElementType(); llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), - CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy), + CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy), CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)), getOrCreateThreadPrivateCache(VD)}; return Address( @@ -1627,8 +1627,7 @@ void CGOpenMPRuntime::emitThreadPrivateVarInit( // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor) // to register constructor/destructor for variable. llvm::Value *Args[] = { - OMPLoc, - CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.VoidPtrTy), + OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy), Ctor, CopyCtor, Dtor}; CGF.EmitRuntimeCall( OMPBuilder.getOrCreateRuntimeFunction( @@ -1901,13 +1900,13 @@ void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, // OutlinedFn(>id, &zero_bound, CapturedStruct); Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc); - RawAddress ZeroAddrBound = + Address ZeroAddrBound = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, /*Name=*/".bound.zero.addr"); CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddrBound); llvm::SmallVector OutlinedFnArgs; // ThreadId for serialized parallels is 0. - OutlinedFnArgs.push_back(ThreadIDAddr.emitRawPointer(CGF)); + OutlinedFnArgs.push_back(ThreadIDAddr.getPointer()); OutlinedFnArgs.push_back(ZeroAddrBound.getPointer()); OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end()); @@ -2273,7 +2272,7 @@ void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, emitUpdateLocation(CGF, Loc), // ident_t * getThreadID(CGF, Loc), // i32 BufSize, // size_t - CL.emitRawPointer(CGF), // void * + CL.getPointer(), // void * CpyFn, // void (*) (void *, void *) DidItVal // i32 did_it }; @@ -2592,10 +2591,10 @@ static void emitForStaticInitCall( ThreadId, CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1, M2)), // Schedule type - Values.IL.emitRawPointer(CGF), // &isLastIter - Values.LB.emitRawPointer(CGF), // &LB - Values.UB.emitRawPointer(CGF), // &UB - Values.ST.emitRawPointer(CGF), // &Stride + Values.IL.getPointer(), // &isLastIter + Values.LB.getPointer(), // &LB + Values.UB.getPointer(), // &UB + Values.ST.getPointer(), // &Stride CGF.Builder.getIntN(Values.IVSize, 1), // Incr Chunk // Chunk }; @@ -2698,11 +2697,12 @@ llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF, // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper, // kmp_int[32|64] *p_stride); llvm::Value *Args[] = { - emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), - IL.emitRawPointer(CGF), // &isLastIter - LB.emitRawPointer(CGF), // &Lower - UB.emitRawPointer(CGF), // &Upper - ST.emitRawPointer(CGF) // &Stride + emitUpdateLocation(CGF, Loc), + getThreadID(CGF, Loc), + IL.getPointer(), // &isLastIter + LB.getPointer(), // &Lower + UB.getPointer(), // &Upper + ST.getPointer() // &Stride }; llvm::Value *Call = CGF.EmitRuntimeCall( OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args); @@ -3047,7 +3047,7 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, CGF.Builder .CreatePointerBitCastOrAddrSpaceCast(TDBase.getAddress(CGF), CGF.VoidPtrTy, CGF.Int8Ty) - .emitRawPointer(CGF)}; + .getPointer()}; SmallVector CallArgs(std::begin(CommonArgs), std::end(CommonArgs)); if (isOpenMPTaskLoopDirective(Kind)) { @@ -3574,8 +3574,7 @@ getPointerAndSize(CodeGenFunction &CGF, const Expr *E) { CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false); Address UpAddrAddress = UpAddrLVal.getAddress(CGF); llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32( - UpAddrAddress.getElementType(), UpAddrAddress.emitRawPointer(CGF), - /*Idx0=*/1); + UpAddrAddress.getElementType(), UpAddrAddress.getPointer(), /*Idx0=*/1); llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy); llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy); SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr); @@ -3889,9 +3888,8 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *Size; std::tie(Addr, Size) = getPointerAndSize(CGF, E); llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); - LValue Base = - CGF.MakeAddrLValue(CGF.Builder.CreateGEP(CGF, AffinitiesArray, Idx), - KmpTaskAffinityInfoTy); + LValue Base = CGF.MakeAddrLValue( + CGF.Builder.CreateGEP(AffinitiesArray, Idx), KmpTaskAffinityInfoTy); // affs[i].base_addr = &; LValue BaseAddrLVal = CGF.EmitLValueForField( Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr)); @@ -3912,7 +3910,7 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *LocRef = emitUpdateLocation(CGF, Loc); llvm::Value *GTid = getThreadID(CGF, Loc); llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - AffinitiesArray.emitRawPointer(CGF), CGM.VoidPtrTy); + AffinitiesArray.getPointer(), CGM.VoidPtrTy); // FIXME: Emit the function and ignore its result for now unless the // runtime function is properly implemented. (void)CGF.EmitRuntimeCall( @@ -3923,8 +3921,8 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *NewTaskNewTaskTTy = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( NewTask, KmpTaskTWithPrivatesPtrTy); - LValue Base = CGF.MakeNaturalAlignRawAddrLValue(NewTaskNewTaskTTy, - KmpTaskTWithPrivatesQTy); + LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy, + KmpTaskTWithPrivatesQTy); LValue TDBase = CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin()); // Fill the data in the resulting kmp_task_t record. @@ -4049,7 +4047,7 @@ CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, CGF.ConvertTypeForMem(KmpDependInfoPtrTy)), KmpDependInfoPtrTy->castAs()); Address DepObjAddr = CGF.Builder.CreateGEP( - CGF, Base.getAddress(CGF), + Base.getAddress(CGF), llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); LValue NumDepsBase = CGF.MakeAddrLValue( DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo()); @@ -4099,7 +4097,7 @@ static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, LValue &PosLVal = *Pos.get(); llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); Base = CGF.MakeAddrLValue( - CGF.Builder.CreateGEP(CGF, DependenciesArray, Idx), KmpDependInfoTy); + CGF.Builder.CreateGEP(DependenciesArray, Idx), KmpDependInfoTy); } // deps[i].base_addr = &; LValue BaseAddrLVal = CGF.EmitLValueForField( @@ -4197,7 +4195,7 @@ void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF, ElSize, CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false)); llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); - Address DepAddr = CGF.Builder.CreateGEP(CGF, DependenciesArray, Pos); + Address DepAddr = CGF.Builder.CreateGEP(DependenciesArray, Pos); CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(CGF), Size); // Increase pos. @@ -4432,7 +4430,7 @@ void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy), CGF.ConvertTypeForMem(KmpDependInfoTy)); llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( - Addr.getElementType(), Addr.emitRawPointer(CGF), + Addr.getElementType(), Addr.getPointer(), llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr, CGF.VoidPtrTy); @@ -4462,8 +4460,8 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, Address Begin = Base.getAddress(CGF); // Cast from pointer to array type to pointer to single element. - llvm::Value *End = CGF.Builder.CreateGEP(Begin.getElementType(), - Begin.emitRawPointer(CGF), NumDeps); + llvm::Value *End = CGF.Builder.CreateGEP( + Begin.getElementType(), Begin.getPointer(), NumDeps); // The basic structure here is a while-do loop. llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body"); llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done"); @@ -4471,7 +4469,7 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, CGF.EmitBlock(BodyBB); llvm::PHINode *ElementPHI = CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast"); - ElementPHI->addIncoming(Begin.emitRawPointer(CGF), EntryBB); + ElementPHI->addIncoming(Begin.getPointer(), EntryBB); Begin = Begin.withPointer(ElementPHI, KnownNonNull); Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo()); @@ -4485,12 +4483,12 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, FlagsLVal); // Shift the address forward by one element. - llvm::Value *ElementNext = - CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext") - .emitRawPointer(CGF); - ElementPHI->addIncoming(ElementNext, CGF.Builder.GetInsertBlock()); + Address ElementNext = + CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext"); + ElementPHI->addIncoming(ElementNext.getPointer(), + CGF.Builder.GetInsertBlock()); llvm::Value *IsEmpty = - CGF.Builder.CreateICmpEQ(ElementNext, End, "omp.isempty"); + CGF.Builder.CreateICmpEQ(ElementNext.getPointer(), End, "omp.isempty"); CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); // Done. CGF.EmitBlock(DoneBB, /*IsFinished=*/true); @@ -4533,7 +4531,7 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, DepTaskArgs[1] = ThreadID; DepTaskArgs[2] = NewTask; DepTaskArgs[3] = NumOfElements; - DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF); + DepTaskArgs[4] = DependenciesArray.getPointer(); DepTaskArgs[5] = CGF.Builder.getInt32(0); DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); } @@ -4565,7 +4563,7 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, DepWaitTaskArgs[0] = UpLoc; DepWaitTaskArgs[1] = ThreadID; DepWaitTaskArgs[2] = NumOfElements; - DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF); + DepWaitTaskArgs[3] = DependenciesArray.getPointer(); DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); DepWaitTaskArgs[6] = @@ -4727,8 +4725,8 @@ static void EmitOMPAggregateReduction( const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe(); llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr); - llvm::Value *RHSBegin = RHSAddr.emitRawPointer(CGF); - llvm::Value *LHSBegin = LHSAddr.emitRawPointer(CGF); + llvm::Value *RHSBegin = RHSAddr.getPointer(); + llvm::Value *LHSBegin = LHSAddr.getPointer(); // Cast from pointer to array type to pointer to single element. llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSAddr.getElementType(), LHSBegin, NumElements); @@ -4992,7 +4990,7 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, QualType ReductionArrayTy = C.getConstantArrayType( C.VoidPtrTy, ArraySize, nullptr, ArraySizeModifier::Normal, /*IndexTypeQuals=*/0); - RawAddress ReductionList = + Address ReductionList = CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); const auto *IPriv = Privates.begin(); unsigned Idx = 0; @@ -5464,7 +5462,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit( C.getConstantArrayType(RDType, ArraySize, nullptr, ArraySizeModifier::Normal, /*IndexTypeQuals=*/0); // kmp_task_red_input_t .rd_input.[Size]; - RawAddress TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input."); + Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input."); ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionOrigs, Data.ReductionCopies, Data.ReductionOps); for (unsigned Cnt = 0; Cnt < Size; ++Cnt) { @@ -5475,7 +5473,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit( TaskRedInput.getElementType(), TaskRedInput.getPointer(), Idxs, /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc, ".rd_input.gep."); - LValue ElemLVal = CGF.MakeNaturalAlignRawAddrLValue(GEP, RDType); + LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType); // ElemLVal.reduce_shar = &Shareds[Cnt]; LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD); RCG.emitSharedOrigLValue(CGF, Cnt); @@ -5631,7 +5629,7 @@ void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, DepWaitTaskArgs[0] = UpLoc; DepWaitTaskArgs[1] = ThreadID; DepWaitTaskArgs[2] = NumOfElements; - DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF); + DepWaitTaskArgs[3] = DependenciesArray.getPointer(); DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); DepWaitTaskArgs[6] = @@ -5854,7 +5852,7 @@ void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF, AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, CGF.getContext().VoidPtrTy, AllocatorTraitsLVal.getBaseInfo(), AllocatorTraitsLVal.getTBAAInfo()); - llvm::Value *Traits = Addr.emitRawPointer(CGF); + llvm::Value *Traits = Addr.getPointer(); llvm::Value *AllocatorVal = CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( @@ -7314,19 +7312,17 @@ class MappableExprsHandler { CGF.EmitOMPSharedLValue(MC.getAssociatedExpression()) .getAddress(CGF); } - llvm::Value *ComponentLBPtr = ComponentLB.emitRawPointer(CGF); - llvm::Value *LBPtr = LB.emitRawPointer(CGF); - Size = CGF.Builder.CreatePtrDiff(CGF.Int8Ty, ComponentLBPtr, - LBPtr); + Size = CGF.Builder.CreatePtrDiff( + CGF.Int8Ty, ComponentLB.getPointer(), LB.getPointer()); break; } } assert(Size && "Failed to determine structure size"); CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF)); + CombinedInfo.BasePointers.push_back(BP.getPointer()); CombinedInfo.DevicePtrDecls.push_back(nullptr); CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); + CombinedInfo.Pointers.push_back(LB.getPointer()); CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( Size, CGF.Int64Ty, /*isSigned=*/true)); CombinedInfo.Types.push_back(Flags); @@ -7336,14 +7332,13 @@ class MappableExprsHandler { LB = CGF.Builder.CreateConstGEP(ComponentLB, 1); } CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF)); + CombinedInfo.BasePointers.push_back(BP.getPointer()); CombinedInfo.DevicePtrDecls.push_back(nullptr); CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); - llvm::Value *LBPtr = LB.emitRawPointer(CGF); + CombinedInfo.Pointers.push_back(LB.getPointer()); Size = CGF.Builder.CreatePtrDiff( - CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).emitRawPointer(CGF), - LBPtr); + CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).getPointer(), + LB.getPointer()); CombinedInfo.Sizes.push_back( CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true)); CombinedInfo.Types.push_back(Flags); @@ -7361,21 +7356,20 @@ class MappableExprsHandler { (Next == CE && MapType != OMPC_MAP_unknown)) { if (!IsMappingWholeStruct) { CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF)); + CombinedInfo.BasePointers.push_back(BP.getPointer()); CombinedInfo.DevicePtrDecls.push_back(nullptr); CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); + CombinedInfo.Pointers.push_back(LB.getPointer()); CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( Size, CGF.Int64Ty, /*isSigned=*/true)); CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize : 1); } else { StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - StructBaseCombinedInfo.BasePointers.push_back( - BP.emitRawPointer(CGF)); + StructBaseCombinedInfo.BasePointers.push_back(BP.getPointer()); StructBaseCombinedInfo.DevicePtrDecls.push_back(nullptr); StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - StructBaseCombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); + StructBaseCombinedInfo.Pointers.push_back(LB.getPointer()); StructBaseCombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( Size, CGF.Int64Ty, /*isSigned=*/true)); StructBaseCombinedInfo.NonContigInfo.Dims.push_back( @@ -8217,11 +8211,11 @@ class MappableExprsHandler { } CombinedInfo.Exprs.push_back(VD); // Base is the base of the struct - CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF)); + CombinedInfo.BasePointers.push_back(PartialStruct.Base.getPointer()); CombinedInfo.DevicePtrDecls.push_back(nullptr); CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); // Pointer is the address of the lowest element - llvm::Value *LB = LBAddr.emitRawPointer(CGF); + llvm::Value *LB = LBAddr.getPointer(); const CXXMethodDecl *MD = CGF.CurFuncDecl ? dyn_cast(CGF.CurFuncDecl) : nullptr; const CXXRecordDecl *RD = MD ? MD->getParent() : nullptr; @@ -8235,7 +8229,7 @@ class MappableExprsHandler { // if the this[:1] expression had appeared in a map clause with a map-type // of tofrom. // Emit this[:1] - CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF)); + CombinedInfo.Pointers.push_back(PartialStruct.Base.getPointer()); QualType Ty = MD->getFunctionObjectParameterType(); llvm::Value *Size = CGF.Builder.CreateIntCast(CGF.getTypeSize(Ty), CGF.Int64Ty, @@ -8244,7 +8238,7 @@ class MappableExprsHandler { } else { CombinedInfo.Pointers.push_back(LB); // Size is (addr of {highest+1} element) - (addr of lowest element) - llvm::Value *HB = HBAddr.emitRawPointer(CGF); + llvm::Value *HB = HBAddr.getPointer(); llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32( HBAddr.getElementType(), HB, /*Idx0=*/1); llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy); @@ -8753,7 +8747,7 @@ class MappableExprsHandler { Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue( CV, ElementType, CGF.getContext().getDeclAlign(VD), AlignmentSource::Decl)); - CombinedInfo.Pointers.push_back(PtrAddr.emitRawPointer(CGF)); + CombinedInfo.Pointers.push_back(PtrAddr.getPointer()); } else { CombinedInfo.Pointers.push_back(CV); } @@ -9564,11 +9558,10 @@ static void emitTargetCallKernelLaunch( bool HasNoWait = D.hasClausesOfKind(); unsigned NumTargetItems = InputInfo.NumberOfTargetItems; - llvm::Value *BasePointersArray = - InputInfo.BasePointersArray.emitRawPointer(CGF); - llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF); - llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF); - llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF); + llvm::Value *BasePointersArray = InputInfo.BasePointersArray.getPointer(); + llvm::Value *PointersArray = InputInfo.PointersArray.getPointer(); + llvm::Value *SizesArray = InputInfo.SizesArray.getPointer(); + llvm::Value *MappersArray = InputInfo.MappersArray.getPointer(); auto &&EmitTargetCallFallbackCB = [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS, @@ -10316,16 +10309,15 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall( // Source location for the ident struct llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); - llvm::Value *OffloadingArgs[] = { - RTLoc, - DeviceID, - PointerNum, - InputInfo.BasePointersArray.emitRawPointer(CGF), - InputInfo.PointersArray.emitRawPointer(CGF), - InputInfo.SizesArray.emitRawPointer(CGF), - MapTypesArray, - MapNamesArray, - InputInfo.MappersArray.emitRawPointer(CGF)}; + llvm::Value *OffloadingArgs[] = {RTLoc, + DeviceID, + PointerNum, + InputInfo.BasePointersArray.getPointer(), + InputInfo.PointersArray.getPointer(), + InputInfo.SizesArray.getPointer(), + MapTypesArray, + MapNamesArray, + InputInfo.MappersArray.getPointer()}; // Select the right runtime function call for each standalone // directive. @@ -11136,7 +11128,7 @@ void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF, getThreadID(CGF, D.getBeginLoc()), llvm::ConstantInt::getSigned(CGM.Int32Ty, NumIterations.size()), CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).emitRawPointer(CGF), + CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).getPointer(), CGM.VoidPtrTy)}; llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction( @@ -11170,8 +11162,7 @@ static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM, /*Volatile=*/false, Int64Ty); } llvm::Value *Args[] = { - ULoc, ThreadID, - CGF.Builder.CreateConstArrayGEP(CntAddr, 0).emitRawPointer(CGF)}; + ULoc, ThreadID, CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()}; llvm::FunctionCallee RTLFn; llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder(); OMPDoacrossKind ODK; @@ -11341,7 +11332,7 @@ Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF, Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID( CGF, SourceLocation::getFromRawEncoding(LocEncoding)); Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - Addr.emitRawPointer(CGF), CGF.VoidPtrTy); + Addr.getPointer(), CGF.VoidPtrTy); llvm::Value *AllocVal = getAllocatorVal(CGF, AllocExpr); Args[2] = AllocVal; CGF.EmitRuntimeCall(RTLFn, Args); @@ -11699,17 +11690,15 @@ void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LLIVTy, getName({UniqueDeclName, "iv"})); cast(LastIV)->setAlignment( IVLVal.getAlignment().getAsAlign()); - LValue LastIVLVal = - CGF.MakeNaturalAlignRawAddrLValue(LastIV, IVLVal.getType()); + LValue LastIVLVal = CGF.MakeNaturalAlignAddrLValue(LastIV, IVLVal.getType()); // Last value of the lastprivate conditional. // decltype(priv_a) last_a; llvm::GlobalVariable *Last = OMPBuilder.getOrCreateInternalVariable( CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName); - cast(Last)->setAlignment( - LVal.getAlignment().getAsAlign()); - LValue LastLVal = - CGF.MakeRawAddrLValue(Last, LVal.getType(), LVal.getAlignment()); + Last->setAlignment(LVal.getAlignment().getAsAlign()); + LValue LastLVal = CGF.MakeAddrLValue( + Address(Last, Last->getValueType(), LVal.getAlignment()), LVal.getType()); // Global loop counter. Required to handle inner parallel-for regions. // iv @@ -11882,8 +11871,9 @@ void CGOpenMPRuntime::emitLastprivateConditionalFinalUpdate( // The variable was not updated in the region - exit. if (!GV) return; - LValue LPLVal = CGF.MakeRawAddrLValue( - GV, PrivLVal.getType().getNonReferenceType(), PrivLVal.getAlignment()); + LValue LPLVal = CGF.MakeAddrLValue( + Address(GV, GV->getValueType(), PrivLVal.getAlignment()), + PrivLVal.getType().getNonReferenceType()); llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc); CGF.EmitStoreOfScalar(Res, PrivLVal); } diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h index 522ae3d35d22d7..c3206427b143e1 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.h +++ b/clang/lib/CodeGen/CGOpenMPRuntime.h @@ -1068,12 +1068,13 @@ class CGOpenMPRuntime { /// \param Loc Location of the reference to threadprivate var. /// \return Address of the threadprivate variable for the current thread. virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, - const VarDecl *VD, Address VDAddr, + const VarDecl *VD, + Address VDAddr, SourceLocation Loc); /// Returns the address of the variable marked as declare target with link /// clause OR as declare target with to clause and unified memory. - virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD); + virtual Address getAddrOfDeclareTargetVar(const VarDecl *VD); /// Emit a code for initialization of threadprivate variable. It emits /// a call to runtime library which adds initial value to the newly created diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp index 5baac8f0e3e268..299ee1460b3db0 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp @@ -1096,8 +1096,7 @@ void CGOpenMPRuntimeGPU::emitGenericVarsProlog(CodeGenFunction &CGF, llvm::PointerType *VarPtrTy = CGF.ConvertTypeForMem(VarTy)->getPointerTo(); llvm::Value *CastedVoidPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( VoidPtr, VarPtrTy, VD->getName() + "_on_stack"); - LValue VarAddr = - CGF.MakeNaturalAlignPointeeRawAddrLValue(CastedVoidPtr, VarTy); + LValue VarAddr = CGF.MakeNaturalAlignAddrLValue(CastedVoidPtr, VarTy); Rec.second.PrivateAddr = VarAddr.getAddress(CGF); Rec.second.GlobalizedVal = VoidPtr; @@ -1207,8 +1206,8 @@ void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF, bool IsBareKernel = D.getSingleClause(); - RawAddress ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, - /*Name=*/".zero.addr"); + Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, + /*Name=*/".zero.addr"); CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddr); llvm::SmallVector OutlinedFnArgs; // We don't emit any thread id function call in bare kernel, but because the @@ -1216,7 +1215,7 @@ void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF, if (IsBareKernel) OutlinedFnArgs.push_back(llvm::ConstantPointerNull::get(CGM.VoidPtrTy)); else - OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).emitRawPointer(CGF)); + OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).getPointer()); OutlinedFnArgs.push_back(ZeroAddr.getPointer()); OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end()); emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); @@ -1290,7 +1289,7 @@ void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF, llvm::ConstantInt::get(CGF.Int32Ty, -1), FnPtr, ID, - Bld.CreateBitOrPointerCast(CapturedVarsAddrs.emitRawPointer(CGF), + Bld.CreateBitOrPointerCast(CapturedVarsAddrs.getPointer(), CGF.VoidPtrPtrTy), llvm::ConstantInt::get(CGM.SizeTy, CapturedVars.size())}; CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( @@ -1504,18 +1503,17 @@ static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr, CGF.EmitBlock(PreCondBB); llvm::PHINode *PhiSrc = Bld.CreatePHI(Ptr.getType(), /*NumReservedValues=*/2); - PhiSrc->addIncoming(Ptr.emitRawPointer(CGF), CurrentBB); + PhiSrc->addIncoming(Ptr.getPointer(), CurrentBB); llvm::PHINode *PhiDest = Bld.CreatePHI(ElemPtr.getType(), /*NumReservedValues=*/2); - PhiDest->addIncoming(ElemPtr.emitRawPointer(CGF), CurrentBB); + PhiDest->addIncoming(ElemPtr.getPointer(), CurrentBB); Ptr = Address(PhiSrc, Ptr.getElementType(), Ptr.getAlignment()); ElemPtr = Address(PhiDest, ElemPtr.getElementType(), ElemPtr.getAlignment()); - llvm::Value *PtrEndRaw = PtrEnd.emitRawPointer(CGF); - llvm::Value *PtrRaw = Ptr.emitRawPointer(CGF); llvm::Value *PtrDiff = Bld.CreatePtrDiff( - CGF.Int8Ty, PtrEndRaw, - Bld.CreatePointerBitCastOrAddrSpaceCast(PtrRaw, CGF.VoidPtrTy)); + CGF.Int8Ty, PtrEnd.getPointer(), + Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr.getPointer(), + CGF.VoidPtrTy)); Bld.CreateCondBr(Bld.CreateICmpSGT(PtrDiff, Bld.getInt64(IntSize - 1)), ThenBB, ExitBB); CGF.EmitBlock(ThenBB); @@ -1530,8 +1528,8 @@ static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr, TBAAAccessInfo()); Address LocalPtr = Bld.CreateConstGEP(Ptr, 1); Address LocalElemPtr = Bld.CreateConstGEP(ElemPtr, 1); - PhiSrc->addIncoming(LocalPtr.emitRawPointer(CGF), ThenBB); - PhiDest->addIncoming(LocalElemPtr.emitRawPointer(CGF), ThenBB); + PhiSrc->addIncoming(LocalPtr.getPointer(), ThenBB); + PhiDest->addIncoming(LocalElemPtr.getPointer(), ThenBB); CGF.EmitBranch(PreCondBB); CGF.EmitBlock(ExitBB); } else { @@ -1678,10 +1676,10 @@ static void emitReductionListCopy( // scope and that of functions it invokes (i.e., reduce_function). // RemoteReduceData[i] = (void*)&RemoteElem if (UpdateDestListPtr) { - CGF.EmitStoreOfScalar( - Bld.CreatePointerBitCastOrAddrSpaceCast( - DestElementAddr.emitRawPointer(CGF), CGF.VoidPtrTy), - DestElementPtrAddr, /*Volatile=*/false, C.VoidPtrTy); + CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast( + DestElementAddr.getPointer(), CGF.VoidPtrTy), + DestElementPtrAddr, /*Volatile=*/false, + C.VoidPtrTy); } ++Idx; @@ -1832,7 +1830,7 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM, // elemptr = ((CopyType*)(elemptrptr)) + I Address ElemPtr(ElemPtrPtr, CopyType, Align); if (NumIters > 1) - ElemPtr = Bld.CreateGEP(CGF, ElemPtr, Cnt); + ElemPtr = Bld.CreateGEP(ElemPtr, Cnt); // Get pointer to location in transfer medium. // MediumPtr = &medium[warp_id] @@ -1896,7 +1894,7 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM, TargetElemPtrPtr, /*Volatile=*/false, C.VoidPtrTy, Loc); Address TargetElemPtr(TargetElemPtrVal, CopyType, Align); if (NumIters > 1) - TargetElemPtr = Bld.CreateGEP(CGF, TargetElemPtr, Cnt); + TargetElemPtr = Bld.CreateGEP(TargetElemPtr, Cnt); // *TargetElemPtr = SrcMediumVal; llvm::Value *SrcMediumValue = @@ -2107,9 +2105,9 @@ static llvm::Function *emitShuffleAndReduceFunction( CGF.EmitBlock(ThenBB); // reduce_function(LocalReduceList, RemoteReduceList) llvm::Value *LocalReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( - LocalReduceList.emitRawPointer(CGF), CGF.VoidPtrTy); + LocalReduceList.getPointer(), CGF.VoidPtrTy); llvm::Value *RemoteReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( - RemoteReduceList.emitRawPointer(CGF), CGF.VoidPtrTy); + RemoteReduceList.getPointer(), CGF.VoidPtrTy); CGM.getOpenMPRuntime().emitOutlinedFunctionCall( CGF, Loc, ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr}); Bld.CreateBr(MergeBB); @@ -2220,9 +2218,9 @@ static llvm::Value *emitListToGlobalCopyFunction( llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); LValue GlobLVal = CGF.EmitLValueForField( - CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD); + CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); Address GlobAddr = GlobLVal.getAddress(CGF); - GlobLVal.setAddress(Address(GlobAddr.emitRawPointer(CGF), + GlobLVal.setAddress(Address(GlobAddr.getPointer(), CGF.ConvertTypeForMem(Private->getType()), GlobAddr.getAlignment())); switch (CGF.getEvaluationKind(Private->getType())) { @@ -2306,7 +2304,7 @@ static llvm::Value *emitListToGlobalReduceFunction( // 1. Build a list of reduction variables. // void *RedList[] = {[0], ..., [-1]}; - RawAddress ReductionList = + Address ReductionList = CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); auto IPriv = Privates.begin(); llvm::Value *Idxs[] = {CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), @@ -2321,10 +2319,10 @@ static llvm::Value *emitListToGlobalReduceFunction( llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); LValue GlobLVal = CGF.EmitLValueForField( - CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD); + CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); Address GlobAddr = GlobLVal.getAddress(CGF); - CGF.EmitStoreOfScalar(GlobAddr.emitRawPointer(CGF), Elem, - /*Volatile=*/false, C.VoidPtrTy); + CGF.EmitStoreOfScalar(GlobAddr.getPointer(), Elem, /*Volatile=*/false, + C.VoidPtrTy); if ((*IPriv)->getType()->isVariablyModifiedType()) { // Store array size. ++Idx; @@ -2427,9 +2425,9 @@ static llvm::Value *emitGlobalToListCopyFunction( llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); LValue GlobLVal = CGF.EmitLValueForField( - CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD); + CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); Address GlobAddr = GlobLVal.getAddress(CGF); - GlobLVal.setAddress(Address(GlobAddr.emitRawPointer(CGF), + GlobLVal.setAddress(Address(GlobAddr.getPointer(), CGF.ConvertTypeForMem(Private->getType()), GlobAddr.getAlignment())); switch (CGF.getEvaluationKind(Private->getType())) { @@ -2528,10 +2526,10 @@ static llvm::Value *emitGlobalToListReduceFunction( llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); LValue GlobLVal = CGF.EmitLValueForField( - CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD); + CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); Address GlobAddr = GlobLVal.getAddress(CGF); - CGF.EmitStoreOfScalar(GlobAddr.emitRawPointer(CGF), Elem, - /*Volatile=*/false, C.VoidPtrTy); + CGF.EmitStoreOfScalar(GlobAddr.getPointer(), Elem, /*Volatile=*/false, + C.VoidPtrTy); if ((*IPriv)->getType()->isVariablyModifiedType()) { // Store array size. ++Idx; @@ -2547,7 +2545,7 @@ static llvm::Value *emitGlobalToListReduceFunction( } // Call reduce_function(ReduceList, GlobalReduceList) - llvm::Value *GlobalReduceList = ReductionList.emitRawPointer(CGF); + llvm::Value *GlobalReduceList = ReductionList.getPointer(); Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar( AddrReduceListArg, /*Volatile=*/false, C.VoidPtrTy, Loc); @@ -2878,7 +2876,7 @@ void CGOpenMPRuntimeGPU::emitReduction( } llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - ReductionList.emitRawPointer(CGF), CGF.VoidPtrTy); + ReductionList.getPointer(), CGF.VoidPtrTy); llvm::Function *ReductionFn = emitReductionFunction( CGF.CurFn->getName(), Loc, CGF.ConvertTypeForMem(ReductionArrayTy), Privates, LHSExprs, RHSExprs, ReductionOps); @@ -3108,15 +3106,15 @@ llvm::Function *CGOpenMPRuntimeGPU::createParallelDataSharingWrapper( // Get the array of arguments. SmallVector Args; - Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).emitRawPointer(CGF)); - Args.emplace_back(ZeroAddr.emitRawPointer(CGF)); + Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).getPointer()); + Args.emplace_back(ZeroAddr.getPointer()); CGBuilderTy &Bld = CGF.Builder; auto CI = CS.capture_begin(); // Use global memory for data sharing. // Handle passing of global args to workers. - RawAddress GlobalArgs = + Address GlobalArgs = CGF.CreateDefaultAlignTempAlloca(CGF.VoidPtrPtrTy, "global_args"); llvm::Value *GlobalArgsPtr = GlobalArgs.getPointer(); llvm::Value *DataSharingArgs[] = {GlobalArgsPtr}; @@ -3402,7 +3400,7 @@ void CGOpenMPRuntimeGPU::adjustTargetSpecificDataForLambdas( VDAddr = CGF.EmitLoadOfReferenceLValue(VDAddr, VD->getType().getCanonicalType()) .getAddress(CGF); - CGF.EmitStoreOfScalar(VDAddr.emitRawPointer(CGF), VarLVal); + CGF.EmitStoreOfScalar(VDAddr.getPointer(), VarLVal); } } } diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp index 576fe2f7a2d46f..cb5a004e4f4a6c 100644 --- a/clang/lib/CodeGen/CGStmt.cpp +++ b/clang/lib/CodeGen/CGStmt.cpp @@ -2294,7 +2294,7 @@ std::pair CodeGenFunction::EmitAsmInputLValue( Address Addr = InputValue.getAddress(*this); ConstraintStr += '*'; - return {InputValue.getPointer(*this), Addr.getElementType()}; + return {Addr.getPointer(), Addr.getElementType()}; } std::pair @@ -2701,7 +2701,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { ArgTypes.push_back(DestAddr.getType()); ArgElemTypes.push_back(DestAddr.getElementType()); - Args.push_back(DestAddr.emitRawPointer(*this)); + Args.push_back(DestAddr.getPointer()); Constraints += "=*"; Constraints += OutputConstraint; ReadOnly = ReadNone = false; @@ -3076,8 +3076,8 @@ CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) { CapturedStmtInfo->setContextValue(Builder.CreateLoad(DeclPtr)); // Initialize variable-length arrays. - LValue Base = MakeNaturalAlignRawAddrLValue( - CapturedStmtInfo->getContextValue(), Ctx.getTagDeclType(RD)); + LValue Base = MakeNaturalAlignAddrLValue(CapturedStmtInfo->getContextValue(), + Ctx.getTagDeclType(RD)); for (auto *FD : RD->fields()) { if (FD->hasCapturedVLAType()) { auto *ExprArg = diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp index e6d504bcdeca5b..f37ac549d10a82 100644 --- a/clang/lib/CodeGen/CGStmtOpenMP.cpp +++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp @@ -350,8 +350,7 @@ void CodeGenFunction::GenerateOpenMPCapturedVars( LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType()); llvm::Value *SrcAddrVal = EmitScalarConversion( - DstAddr.emitRawPointer(*this), - Ctx.getPointerType(Ctx.getUIntPtrType()), + DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()), Ctx.getPointerType(CurField->getType()), CurCap->getLocation()); LValue SrcLV = MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType()); @@ -365,8 +364,7 @@ void CodeGenFunction::GenerateOpenMPCapturedVars( CapturedVars.push_back(CV); } else { assert(CurCap->capturesVariable() && "Expected capture by reference."); - CapturedVars.push_back( - EmitLValue(*I).getAddress(*this).emitRawPointer(*this)); + CapturedVars.push_back(EmitLValue(*I).getAddress(*this).getPointer()); } } } @@ -377,9 +375,8 @@ static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, ASTContext &Ctx = CGF.getContext(); llvm::Value *CastedPtr = CGF.EmitScalarConversion( - AddrLV.getAddress(CGF).emitRawPointer(CGF), Ctx.getUIntPtrType(), + AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(), Ctx.getPointerType(DstType), Loc); - // FIXME: should the pointee type (DstType) be passed? Address TmpAddr = CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress(CGF); return TmpAddr; @@ -705,8 +702,8 @@ void CodeGenFunction::EmitOMPAggregateAssign( llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr); SrcAddr = SrcAddr.withElementType(DestAddr.getElementType()); - llvm::Value *SrcBegin = SrcAddr.emitRawPointer(*this); - llvm::Value *DestBegin = DestAddr.emitRawPointer(*this); + llvm::Value *SrcBegin = SrcAddr.getPointer(); + llvm::Value *DestBegin = DestAddr.getPointer(); // Cast from pointer to array type to pointer to single element. llvm::Value *DestEnd = Builder.CreateInBoundsGEP(DestAddr.getElementType(), DestBegin, NumElements); @@ -1010,10 +1007,10 @@ bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) { CopyBegin = createBasicBlock("copyin.not.master"); CopyEnd = createBasicBlock("copyin.not.master.end"); // TODO: Avoid ptrtoint conversion. - auto *MasterAddrInt = Builder.CreatePtrToInt( - MasterAddr.emitRawPointer(*this), CGM.IntPtrTy); - auto *PrivateAddrInt = Builder.CreatePtrToInt( - PrivateAddr.emitRawPointer(*this), CGM.IntPtrTy); + auto *MasterAddrInt = + Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy); + auto *PrivateAddrInt = + Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy); Builder.CreateCondBr( Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin, CopyEnd); @@ -1669,7 +1666,7 @@ Address CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate( llvm::Type *VarTy = VDAddr.getElementType(); llvm::Value *Data = - CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy); + CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy); llvm::ConstantInt *Size = CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)); std::string Suffix = getNameWithSeparators({"cache", ""}); llvm::Twine CacheName = Twine(CGM.getMangledName(VD)).concat(Suffix); @@ -2048,7 +2045,7 @@ void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) { ->getParam(0) ->getType() .getNonReferenceType(); - RawAddress CountAddr = CreateMemTemp(LogicalTy, ".count.addr"); + Address CountAddr = CreateMemTemp(LogicalTy, ".count.addr"); emitCapturedStmtCall(*this, DistanceClosure, {CountAddr.getPointer()}); llvm::Value *DistVal = Builder.CreateLoad(CountAddr, ".count"); @@ -2064,7 +2061,7 @@ void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) { LValue LCVal = EmitLValue(LoopVarRef); Address LoopVarAddress = LCVal.getAddress(*this); emitCapturedStmtCall(*this, LoopVarClosure, - {LoopVarAddress.emitRawPointer(*this), IndVar}); + {LoopVarAddress.getPointer(), IndVar}); RunCleanupsScope BodyScope(*this); EmitStmt(BodyStmt); @@ -4798,7 +4795,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( ParamTypes.push_back(PrivatesPtr->getType()); for (const Expr *E : Data.PrivateVars) { const auto *VD = cast(cast(E)->getDecl()); - RawAddress PrivatePtr = CGF.CreateMemTemp( + Address PrivatePtr = CGF.CreateMemTemp( CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); CallArgs.push_back(PrivatePtr.getPointer()); @@ -4806,7 +4803,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( } for (const Expr *E : Data.FirstprivateVars) { const auto *VD = cast(cast(E)->getDecl()); - RawAddress PrivatePtr = + Address PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".firstpriv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); @@ -4816,7 +4813,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( } for (const Expr *E : Data.LastprivateVars) { const auto *VD = cast(cast(E)->getDecl()); - RawAddress PrivatePtr = + Address PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".lastpriv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); @@ -4829,7 +4826,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( Ty = CGF.getContext().getPointerType(Ty); if (isAllocatableDecl(VD)) Ty = CGF.getContext().getPointerType(Ty); - RawAddress PrivatePtr = CGF.CreateMemTemp( + Address PrivatePtr = CGF.CreateMemTemp( CGF.getContext().getPointerType(Ty), ".local.ptr.addr"); auto Result = UntiedLocalVars.insert( std::make_pair(VD, std::make_pair(PrivatePtr, Address::invalid()))); @@ -4862,7 +4859,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( if (auto *DI = CGF.getDebugInfo()) if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo()) (void)DI->EmitDeclareOfAutoVariable( - Pair.first, Pair.second.getBasePointer(), CGF.Builder, + Pair.first, Pair.second.getPointer(), CGF.Builder, /*UsePointerValue*/ true); } // Adjust mapping for internal locals by mapping actual memory instead of @@ -4915,14 +4912,14 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( RedCG, Cnt); Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); - Replacement = Address( - CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF), - CGF.getContext().VoidPtrTy, - CGF.getContext().getPointerType( - Data.ReductionCopies[Cnt]->getType()), - Data.ReductionCopies[Cnt]->getExprLoc()), - CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), - Replacement.getAlignment()); + Replacement = + Address(CGF.EmitScalarConversion( + Replacement.getPointer(), CGF.getContext().VoidPtrTy, + CGF.getContext().getPointerType( + Data.ReductionCopies[Cnt]->getType()), + Data.ReductionCopies[Cnt]->getExprLoc()), + CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), + Replacement.getAlignment()); Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement); } @@ -4973,7 +4970,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); Replacement = Address( CGF.EmitScalarConversion( - Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy, + Replacement.getPointer(), CGF.getContext().VoidPtrTy, CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), InRedPrivs[Cnt]->getExprLoc()), CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()), @@ -5092,7 +5089,7 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective( // If there is no user-defined mapper, the mapper array will be nullptr. In // this case, we don't need to privatize it. if (!isa_and_nonnull( - InputInfo.MappersArray.emitRawPointer(*this))) { + InputInfo.MappersArray.getPointer())) { MVD = createImplicitFirstprivateForType( getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc()); TargetScope.addPrivate(MVD, InputInfo.MappersArray); @@ -5118,7 +5115,7 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective( ParamTypes.push_back(PrivatesPtr->getType()); for (const Expr *E : Data.FirstprivateVars) { const auto *VD = cast(cast(E)->getDecl()); - RawAddress PrivatePtr = + Address PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".firstpriv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); @@ -5197,14 +5194,14 @@ void CodeGenFunction::processInReduction(const OMPExecutableDirective &S, RedCG, Cnt); Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); - Replacement = Address( - CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF), - CGF.getContext().VoidPtrTy, - CGF.getContext().getPointerType( - Data.ReductionCopies[Cnt]->getType()), - Data.ReductionCopies[Cnt]->getExprLoc()), - CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), - Replacement.getAlignment()); + Replacement = + Address(CGF.EmitScalarConversion( + Replacement.getPointer(), CGF.getContext().VoidPtrTy, + CGF.getContext().getPointerType( + Data.ReductionCopies[Cnt]->getType()), + Data.ReductionCopies[Cnt]->getExprLoc()), + CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), + Replacement.getAlignment()); Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement); } @@ -5250,7 +5247,7 @@ void CodeGenFunction::processInReduction(const OMPExecutableDirective &S, CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); Replacement = Address( CGF.EmitScalarConversion( - Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy, + Replacement.getPointer(), CGF.getContext().VoidPtrTy, CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), InRedPrivs[Cnt]->getExprLoc()), CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()), @@ -5397,7 +5394,7 @@ void CodeGenFunction::EmitOMPDepobjDirective(const OMPDepobjDirective &S) { Dependencies.DepExprs.append(DC->varlist_begin(), DC->varlist_end()); Address DepAddr = CGM.getOpenMPRuntime().emitDepobjDependClause( *this, Dependencies, DC->getBeginLoc()); - EmitStoreOfScalar(DepAddr.emitRawPointer(*this), DOLVal); + EmitStoreOfScalar(DepAddr.getPointer(), DOLVal); return; } if (const auto *DC = S.getSingleClause()) { @@ -6474,21 +6471,21 @@ static void emitOMPAtomicCompareExpr( D->getType()->hasSignedIntegerRepresentation()); llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{ - XAddr.emitRawPointer(CGF), XAddr.getElementType(), + XAddr.getPointer(), XAddr.getElementType(), X->getType()->hasSignedIntegerRepresentation(), X->getType().isVolatileQualified()}; llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal; if (V) { LValue LV = CGF.EmitLValue(V); Address Addr = LV.getAddress(CGF); - VOpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(), + VOpVal = {Addr.getPointer(), Addr.getElementType(), V->getType()->hasSignedIntegerRepresentation(), V->getType().isVolatileQualified()}; } if (R) { LValue LV = CGF.EmitLValue(R); Address Addr = LV.getAddress(CGF); - ROpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(), + ROpVal = {Addr.getPointer(), Addr.getElementType(), R->getType()->hasSignedIntegerRepresentation(), R->getType().isVolatileQualified()}; } @@ -7032,7 +7029,7 @@ void CodeGenFunction::EmitOMPInteropDirective(const OMPInteropDirective &S) { std::tie(NumDependences, DependenciesArray) = CGM.getOpenMPRuntime().emitDependClause(*this, Data.Dependences, S.getBeginLoc()); - DependenceList = DependenciesArray.emitRawPointer(*this); + DependenceList = DependenciesArray.getPointer(); } Data.HasNowaitClause = S.hasClausesOfKind(); diff --git a/clang/lib/CodeGen/CGVTables.cpp b/clang/lib/CodeGen/CGVTables.cpp index 862369ae009f48..8dee3f74b44b4e 100644 --- a/clang/lib/CodeGen/CGVTables.cpp +++ b/clang/lib/CodeGen/CGVTables.cpp @@ -201,13 +201,14 @@ CodeGenFunction::GenerateVarArgsThunk(llvm::Function *Fn, // Find the first store of "this", which will be to the alloca associated // with "this". - Address ThisPtr = makeNaturalAddressForPointer( - &*AI, MD->getFunctionObjectParameterType(), - CGM.getClassPointerAlignment(MD->getParent())); + Address ThisPtr = + Address(&*AI, ConvertTypeForMem(MD->getFunctionObjectParameterType()), + CGM.getClassPointerAlignment(MD->getParent())); llvm::BasicBlock *EntryBB = &Fn->front(); llvm::BasicBlock::iterator ThisStore = llvm::find_if(*EntryBB, [&](llvm::Instruction &I) { - return isa(I) && I.getOperand(0) == &*AI; + return isa(I) && + I.getOperand(0) == ThisPtr.getPointer(); }); assert(ThisStore != EntryBB->end() && "Store of this should be in entry block?"); diff --git a/clang/lib/CodeGen/CGValue.h b/clang/lib/CodeGen/CGValue.h index cc9ad10ae5969d..1e6f67250583d6 100644 --- a/clang/lib/CodeGen/CGValue.h +++ b/clang/lib/CodeGen/CGValue.h @@ -14,13 +14,12 @@ #ifndef LLVM_CLANG_LIB_CODEGEN_CGVALUE_H #define LLVM_CLANG_LIB_CODEGEN_CGVALUE_H -#include "Address.h" -#include "CodeGenTBAA.h" -#include "EHScopeStack.h" #include "clang/AST/ASTContext.h" #include "clang/AST/Type.h" -#include "llvm/IR/Type.h" #include "llvm/IR/Value.h" +#include "llvm/IR/Type.h" +#include "Address.h" +#include "CodeGenTBAA.h" namespace llvm { class Constant; @@ -29,64 +28,57 @@ namespace llvm { namespace clang { namespace CodeGen { -class AggValueSlot; -class CGBuilderTy; -class CodeGenFunction; -struct CGBitFieldInfo; + class AggValueSlot; + class CodeGenFunction; + struct CGBitFieldInfo; /// RValue - This trivial value class is used to represent the result of an /// expression that is evaluated. It can be one of three things: either a /// simple LLVM SSA value, a pair of SSA values for complex numbers, or the /// address of an aggregate value in memory. class RValue { - friend struct DominatingValue; + enum Flavor { Scalar, Complex, Aggregate }; - enum FlavorEnum { Scalar, Complex, Aggregate }; + // The shift to make to an aggregate's alignment to make it look + // like a pointer. + enum { AggAlignShift = 4 }; - union { - // Stores first and second value. - struct { - llvm::Value *first; - llvm::Value *second; - } Vals; - - // Stores aggregate address. - Address AggregateAddr; - }; - - unsigned IsVolatile : 1; - unsigned Flavor : 2; + // Stores first value and flavor. + llvm::PointerIntPair V1; + // Stores second value and volatility. + llvm::PointerIntPair V2; + // Stores element type for aggregate values. + llvm::Type *ElementType; public: - RValue() : Vals{nullptr, nullptr}, Flavor(Scalar) {} - - bool isScalar() const { return Flavor == Scalar; } - bool isComplex() const { return Flavor == Complex; } - bool isAggregate() const { return Flavor == Aggregate; } + bool isScalar() const { return V1.getInt() == Scalar; } + bool isComplex() const { return V1.getInt() == Complex; } + bool isAggregate() const { return V1.getInt() == Aggregate; } - bool isVolatileQualified() const { return IsVolatile; } + bool isVolatileQualified() const { return V2.getInt(); } /// getScalarVal() - Return the Value* of this scalar value. llvm::Value *getScalarVal() const { assert(isScalar() && "Not a scalar!"); - return Vals.first; + return V1.getPointer(); } /// getComplexVal - Return the real/imag components of this complex value. /// std::pair getComplexVal() const { - return std::make_pair(Vals.first, Vals.second); + return std::make_pair(V1.getPointer(), V2.getPointer()); } /// getAggregateAddr() - Return the Value* of the address of the aggregate. Address getAggregateAddress() const { assert(isAggregate() && "Not an aggregate!"); - return AggregateAddr; + auto align = reinterpret_cast(V2.getPointer()) >> AggAlignShift; + return Address( + V1.getPointer(), ElementType, CharUnits::fromQuantity(align)); } - - llvm::Value *getAggregatePointer(QualType PointeeType, - CodeGenFunction &CGF) const { - return getAggregateAddress().getBasePointer(); + llvm::Value *getAggregatePointer() const { + assert(isAggregate() && "Not an aggregate!"); + return V1.getPointer(); } static RValue getIgnored() { @@ -96,19 +88,17 @@ class RValue { static RValue get(llvm::Value *V) { RValue ER; - ER.Vals.first = V; - ER.Flavor = Scalar; - ER.IsVolatile = false; + ER.V1.setPointer(V); + ER.V1.setInt(Scalar); + ER.V2.setInt(false); return ER; } - static RValue get(Address Addr, CodeGenFunction &CGF) { - return RValue::get(Addr.emitRawPointer(CGF)); - } static RValue getComplex(llvm::Value *V1, llvm::Value *V2) { RValue ER; - ER.Vals = {V1, V2}; - ER.Flavor = Complex; - ER.IsVolatile = false; + ER.V1.setPointer(V1); + ER.V2.setPointer(V2); + ER.V1.setInt(Complex); + ER.V2.setInt(false); return ER; } static RValue getComplex(const std::pair &C) { @@ -117,15 +107,15 @@ class RValue { // FIXME: Aggregate rvalues need to retain information about whether they are // volatile or not. Remove default to find all places that probably get this // wrong. - - /// Convert an Address to an RValue. If the Address is not - /// signed, create an RValue using the unsigned address. Otherwise, resign the - /// address using the provided type. static RValue getAggregate(Address addr, bool isVolatile = false) { RValue ER; - ER.AggregateAddr = addr; - ER.Flavor = Aggregate; - ER.IsVolatile = isVolatile; + ER.V1.setPointer(addr.getPointer()); + ER.V1.setInt(Aggregate); + ER.ElementType = addr.getElementType(); + + auto align = static_cast(addr.getAlignment().getQuantity()); + ER.V2.setPointer(reinterpret_cast(align << AggAlignShift)); + ER.V2.setInt(isVolatile); return ER; } }; @@ -188,10 +178,8 @@ class LValue { MatrixElt // This is a matrix element, use getVector* } LVType; - union { - Address Addr = Address::invalid(); - llvm::Value *V; - }; + llvm::Value *V; + llvm::Type *ElementType; union { // Index into a vector subscript: V[i] @@ -209,6 +197,10 @@ class LValue { // 'const' is unused here Qualifiers Quals; + // The alignment to use when accessing this lvalue. (For vector elements, + // this is the alignment of the whole vector.) + unsigned Alignment; + // objective-c's ivar bool Ivar:1; @@ -242,19 +234,23 @@ class LValue { Expr *BaseIvarExp; private: - void Initialize(QualType Type, Qualifiers Quals, Address Addr, + void Initialize(QualType Type, Qualifiers Quals, CharUnits Alignment, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) { + assert((!Alignment.isZero() || Type->isIncompleteType()) && + "initializing l-value with zero alignment!"); + if (isGlobalReg()) + assert(ElementType == nullptr && "Global reg does not store elem type"); + else + assert(ElementType != nullptr && "Must have elem type"); + this->Type = Type; this->Quals = Quals; const unsigned MaxAlign = 1U << 31; - CharUnits Alignment = Addr.getAlignment(); - assert((isGlobalReg() || !Alignment.isZero() || Type->isIncompleteType()) && - "initializing l-value with zero alignment!"); - if (Alignment.getQuantity() > MaxAlign) { - assert(false && "Alignment exceeds allowed max!"); - Alignment = CharUnits::fromQuantity(MaxAlign); - } - this->Addr = Addr; + this->Alignment = Alignment.getQuantity() <= MaxAlign + ? Alignment.getQuantity() + : MaxAlign; + assert(this->Alignment == Alignment.getQuantity() && + "Alignment exceeds allowed max!"); this->BaseInfo = BaseInfo; this->TBAAInfo = TBAAInfo; @@ -263,20 +259,9 @@ class LValue { this->ImpreciseLifetime = false; this->Nontemporal = false; this->ThreadLocalRef = false; - this->IsKnownNonNull = false; this->BaseIvarExp = nullptr; } - void initializeSimpleLValue(Address Addr, QualType Type, - LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo, - ASTContext &Context) { - Qualifiers QS = Type.getQualifiers(); - QS.setObjCGCAttr(Context.getObjCGCAttrKind(Type)); - LVType = Simple; - Initialize(Type, QS, Addr, BaseInfo, TBAAInfo); - assert(Addr.getBasePointer()->getType()->isPointerTy()); - } - public: bool isSimple() const { return LVType == Simple; } bool isVectorElt() const { return LVType == VectorElt; } @@ -343,8 +328,8 @@ class LValue { LangAS getAddressSpace() const { return Quals.getAddressSpace(); } - CharUnits getAlignment() const { return Addr.getAlignment(); } - void setAlignment(CharUnits A) { Addr.setAlignment(A); } + CharUnits getAlignment() const { return CharUnits::fromQuantity(Alignment); } + void setAlignment(CharUnits A) { Alignment = A.getQuantity(); } LValueBaseInfo getBaseInfo() const { return BaseInfo; } void setBaseInfo(LValueBaseInfo Info) { BaseInfo = Info; } @@ -360,32 +345,28 @@ class LValue { // simple lvalue llvm::Value *getPointer(CodeGenFunction &CGF) const { assert(isSimple()); - return Addr.getBasePointer(); + return V; } - llvm::Value *emitRawPointer(CodeGenFunction &CGF) const { - assert(isSimple()); - return Addr.isValid() ? Addr.emitRawPointer(CGF) : nullptr; - } - Address getAddress(CodeGenFunction &CGF) const { - // FIXME: remove parameter. - return Addr; + return Address(getPointer(CGF), ElementType, getAlignment(), + isKnownNonNull()); + } + void setAddress(Address address) { + assert(isSimple()); + V = address.getPointer(); + ElementType = address.getElementType(); + Alignment = address.getAlignment().getQuantity(); + IsKnownNonNull = address.isKnownNonNull(); } - - void setAddress(Address address) { Addr = address; } // vector elt lvalue Address getVectorAddress() const { - assert(isVectorElt()); - return Addr; - } - llvm::Value *getRawVectorPointer(CodeGenFunction &CGF) const { - assert(isVectorElt()); - return Addr.emitRawPointer(CGF); + return Address(getVectorPointer(), ElementType, getAlignment(), + (KnownNonNull_t)isKnownNonNull()); } llvm::Value *getVectorPointer() const { assert(isVectorElt()); - return Addr.getBasePointer(); + return V; } llvm::Value *getVectorIdx() const { assert(isVectorElt()); @@ -393,12 +374,12 @@ class LValue { } Address getMatrixAddress() const { - assert(isMatrixElt()); - return Addr; + return Address(getMatrixPointer(), ElementType, getAlignment(), + (KnownNonNull_t)isKnownNonNull()); } llvm::Value *getMatrixPointer() const { assert(isMatrixElt()); - return Addr.getBasePointer(); + return V; } llvm::Value *getMatrixIdx() const { assert(isMatrixElt()); @@ -407,12 +388,12 @@ class LValue { // extended vector elements. Address getExtVectorAddress() const { - assert(isExtVectorElt()); - return Addr; + return Address(getExtVectorPointer(), ElementType, getAlignment(), + (KnownNonNull_t)isKnownNonNull()); } - llvm::Value *getRawExtVectorPointer(CodeGenFunction &CGF) const { + llvm::Value *getExtVectorPointer() const { assert(isExtVectorElt()); - return Addr.emitRawPointer(CGF); + return V; } llvm::Constant *getExtVectorElts() const { assert(isExtVectorElt()); @@ -421,14 +402,10 @@ class LValue { // bitfield lvalue Address getBitFieldAddress() const { - assert(isBitField()); - return Addr; - } - llvm::Value *getRawBitFieldPointer(CodeGenFunction &CGF) const { - assert(isBitField()); - return Addr.emitRawPointer(CGF); + return Address(getBitFieldPointer(), ElementType, getAlignment(), + (KnownNonNull_t)isKnownNonNull()); } - + llvm::Value *getBitFieldPointer() const { assert(isBitField()); return V; } const CGBitFieldInfo &getBitFieldInfo() const { assert(isBitField()); return *BitFieldInfo; @@ -437,13 +414,18 @@ class LValue { // global register lvalue llvm::Value *getGlobalReg() const { assert(isGlobalReg()); return V; } - static LValue MakeAddr(Address Addr, QualType type, ASTContext &Context, + static LValue MakeAddr(Address address, QualType type, ASTContext &Context, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) { + Qualifiers qs = type.getQualifiers(); + qs.setObjCGCAttr(Context.getObjCGCAttrKind(type)); + LValue R; R.LVType = Simple; - R.initializeSimpleLValue(Addr, type, BaseInfo, TBAAInfo, Context); - R.Addr = Addr; - assert(Addr.getType()->isPointerTy()); + assert(address.getPointer()->getType()->isPointerTy()); + R.V = address.getPointer(); + R.ElementType = address.getElementType(); + R.IsKnownNonNull = address.isKnownNonNull(); + R.Initialize(type, qs, address.getAlignment(), BaseInfo, TBAAInfo); return R; } @@ -452,18 +434,26 @@ class LValue { TBAAAccessInfo TBAAInfo) { LValue R; R.LVType = VectorElt; + R.V = vecAddress.getPointer(); + R.ElementType = vecAddress.getElementType(); R.VectorIdx = Idx; - R.Initialize(type, type.getQualifiers(), vecAddress, BaseInfo, TBAAInfo); + R.IsKnownNonNull = vecAddress.isKnownNonNull(); + R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(), + BaseInfo, TBAAInfo); return R; } - static LValue MakeExtVectorElt(Address Addr, llvm::Constant *Elts, + static LValue MakeExtVectorElt(Address vecAddress, llvm::Constant *Elts, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) { LValue R; R.LVType = ExtVectorElt; + R.V = vecAddress.getPointer(); + R.ElementType = vecAddress.getElementType(); R.VectorElts = Elts; - R.Initialize(type, type.getQualifiers(), Addr, BaseInfo, TBAAInfo); + R.IsKnownNonNull = vecAddress.isKnownNonNull(); + R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(), + BaseInfo, TBAAInfo); return R; } @@ -478,8 +468,12 @@ class LValue { TBAAAccessInfo TBAAInfo) { LValue R; R.LVType = BitField; + R.V = Addr.getPointer(); + R.ElementType = Addr.getElementType(); R.BitFieldInfo = &Info; - R.Initialize(type, type.getQualifiers(), Addr, BaseInfo, TBAAInfo); + R.IsKnownNonNull = Addr.isKnownNonNull(); + R.Initialize(type, type.getQualifiers(), Addr.getAlignment(), BaseInfo, + TBAAInfo); return R; } @@ -487,9 +481,11 @@ class LValue { QualType type) { LValue R; R.LVType = GlobalReg; - R.Initialize(type, type.getQualifiers(), Address::invalid(), - LValueBaseInfo(AlignmentSource::Decl), TBAAAccessInfo()); R.V = V; + R.ElementType = nullptr; + R.IsKnownNonNull = true; + R.Initialize(type, type.getQualifiers(), alignment, + LValueBaseInfo(AlignmentSource::Decl), TBAAAccessInfo()); return R; } @@ -498,8 +494,12 @@ class LValue { TBAAAccessInfo TBAAInfo) { LValue R; R.LVType = MatrixElt; + R.V = matAddress.getPointer(); + R.ElementType = matAddress.getElementType(); R.VectorIdx = Idx; - R.Initialize(type, type.getQualifiers(), matAddress, BaseInfo, TBAAInfo); + R.IsKnownNonNull = matAddress.isKnownNonNull(); + R.Initialize(type, type.getQualifiers(), matAddress.getAlignment(), + BaseInfo, TBAAInfo); return R; } @@ -643,17 +643,17 @@ class AggValueSlot { return NeedsGCBarriers_t(ObjCGCFlag); } - llvm::Value *getPointer(QualType PointeeTy, CodeGenFunction &CGF) const; - - llvm::Value *emitRawPointer(CodeGenFunction &CGF) const { - return Addr.isValid() ? Addr.emitRawPointer(CGF) : nullptr; + llvm::Value *getPointer() const { + return Addr.getPointer(); } Address getAddress() const { return Addr; } - bool isIgnored() const { return !Addr.isValid(); } + bool isIgnored() const { + return !Addr.isValid(); + } CharUnits getAlignment() const { return Addr.getAlignment(); diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp index 44103884940fd9..f2ebaf7674523c 100644 --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -193,35 +193,26 @@ CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() { CGF.Builder.setDefaultConstrainedRounding(OldRounding); } -static LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T, - bool ForPointeeType, - CodeGenFunction &CGF) { +LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) { LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; - CharUnits Alignment = - CGF.CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo, ForPointeeType); - Address Addr = Address(V, CGF.ConvertTypeForMem(T), Alignment); - return CGF.MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo); -} - -LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) { - return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false, *this); + CharUnits Alignment = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo); + Address Addr(V, ConvertTypeForMem(T), Alignment); + return LValue::MakeAddr(Addr, T, getContext(), BaseInfo, TBAAInfo); } +/// Given a value of type T* that may not be to a complete object, +/// construct an l-value with the natural pointee alignment of T. LValue CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) { - return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true, *this); -} - -LValue CodeGenFunction::MakeNaturalAlignRawAddrLValue(llvm::Value *V, - QualType T) { - return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false, *this); + LValueBaseInfo BaseInfo; + TBAAAccessInfo TBAAInfo; + CharUnits Align = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo, + /* forPointeeType= */ true); + Address Addr(V, ConvertTypeForMem(T), Align); + return MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo); } -LValue CodeGenFunction::MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, - QualType T) { - return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true, *this); -} llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) { return CGM.getTypes().ConvertTypeForMem(T); @@ -534,8 +525,7 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) { ReturnBlock.getBlock()->eraseFromParent(); } if (ReturnValue.isValid()) { - auto *RetAlloca = - dyn_cast(ReturnValue.emitRawPointer(*this)); + auto *RetAlloca = dyn_cast(ReturnValue.getPointer()); if (RetAlloca && RetAlloca->use_empty()) { RetAlloca->eraseFromParent(); ReturnValue = Address::invalid(); @@ -1132,14 +1122,13 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, auto AI = CurFn->arg_begin(); if (CurFnInfo->getReturnInfo().isSRetAfterThis()) ++AI; - ReturnValue = makeNaturalAddressForPointer( - &*AI, RetTy, CurFnInfo->getReturnInfo().getIndirectAlign(), false, - nullptr, nullptr, KnownNonNull); + ReturnValue = + Address(&*AI, ConvertType(RetTy), + CurFnInfo->getReturnInfo().getIndirectAlign(), KnownNonNull); if (!CurFnInfo->getReturnInfo().getIndirectByVal()) { - ReturnValuePointer = - CreateDefaultAlignTempAlloca(ReturnValue.getType(), "result.ptr"); - Builder.CreateStore(ReturnValue.emitRawPointer(*this), - ReturnValuePointer); + ReturnValuePointer = CreateDefaultAlignTempAlloca( + ReturnValue.getPointer()->getType(), "result.ptr"); + Builder.CreateStore(ReturnValue.getPointer(), ReturnValuePointer); } } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca && !hasScalarEvaluationKind(CurFnInfo->getReturnType())) { @@ -1200,9 +1189,8 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, // or contains the address of the enclosing object). LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField); if (!LambdaThisCaptureField->getType()->isPointerType()) { - // If the enclosing object was captured by value, just use its - // address. Sign this pointer. - CXXThisValue = ThisFieldLValue.getPointer(*this); + // If the enclosing object was captured by value, just use its address. + CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer(); } else { // Load the lvalue pointed to by the field, since '*this' was captured // by reference. @@ -2024,9 +2012,8 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity()); Address begin = dest.withElementType(CGF.Int8Ty); - llvm::Value *end = Builder.CreateInBoundsGEP(begin.getElementType(), - begin.emitRawPointer(CGF), - sizeInChars, "vla.end"); + llvm::Value *end = Builder.CreateInBoundsGEP( + begin.getElementType(), begin.getPointer(), sizeInChars, "vla.end"); llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock(); llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop"); @@ -2037,7 +2024,7 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, CGF.EmitBlock(loopBB); llvm::PHINode *cur = Builder.CreatePHI(begin.getType(), 2, "vla.cur"); - cur->addIncoming(begin.emitRawPointer(CGF), originBB); + cur->addIncoming(begin.getPointer(), originBB); CharUnits curAlign = dest.getAlignment().alignmentOfArrayElement(baseSize); @@ -2230,10 +2217,10 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType, addr = addr.withElementType(baseType); } else { // Create the actual GEP. - addr = Address(Builder.CreateInBoundsGEP(addr.getElementType(), - addr.emitRawPointer(*this), - gepIndices, "array.begin"), - ConvertTypeForMem(eltType), addr.getAlignment()); + addr = Address(Builder.CreateInBoundsGEP( + addr.getElementType(), addr.getPointer(), gepIndices, "array.begin"), + ConvertTypeForMem(eltType), + addr.getAlignment()); } baseType = eltType; @@ -2574,7 +2561,7 @@ void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) { Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D, Address Addr) { assert(D->hasAttr() && "no annotate attribute"); - llvm::Value *V = Addr.emitRawPointer(*this); + llvm::Value *V = Addr.getPointer(); llvm::Type *VTy = V->getType(); auto *PTy = dyn_cast(VTy); unsigned AS = PTy ? PTy->getAddressSpace() : 0; diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h index 8dd6da5f85f11d..e8f8aa601ed017 100644 --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -151,9 +151,6 @@ struct DominatingLLVMValue { /// Answer whether the given value needs extra work to be saved. static bool needsSaving(llvm::Value *value) { - if (!value) - return false; - // If it's not an instruction, we don't need to save. if (!isa(value)) return false; @@ -180,28 +177,21 @@ template <> struct DominatingValue
{ typedef Address type; struct saved_type { - DominatingLLVMValue::saved_type BasePtr; + DominatingLLVMValue::saved_type SavedValue; llvm::Type *ElementType; CharUnits Alignment; - DominatingLLVMValue::saved_type Offset; - llvm::PointerType *EffectiveType; }; static bool needsSaving(type value) { - if (DominatingLLVMValue::needsSaving(value.getBasePointer()) || - DominatingLLVMValue::needsSaving(value.getOffset())) - return true; - return false; + return DominatingLLVMValue::needsSaving(value.getPointer()); } static saved_type save(CodeGenFunction &CGF, type value) { - return {DominatingLLVMValue::save(CGF, value.getBasePointer()), - value.getElementType(), value.getAlignment(), - DominatingLLVMValue::save(CGF, value.getOffset()), value.getType()}; + return { DominatingLLVMValue::save(CGF, value.getPointer()), + value.getElementType(), value.getAlignment() }; } static type restore(CodeGenFunction &CGF, saved_type value) { - return Address(DominatingLLVMValue::restore(CGF, value.BasePtr), - value.ElementType, value.Alignment, - DominatingLLVMValue::restore(CGF, value.Offset)); + return Address(DominatingLLVMValue::restore(CGF, value.SavedValue), + value.ElementType, value.Alignment); } }; @@ -211,26 +201,14 @@ template <> struct DominatingValue { class saved_type { enum Kind { ScalarLiteral, ScalarAddress, AggregateLiteral, AggregateAddress, ComplexAddress }; - union { - struct { - DominatingLLVMValue::saved_type first, second; - } Vals; - DominatingValue
::saved_type AggregateAddr; - }; + + llvm::Value *Value; + llvm::Type *ElementType; LLVM_PREFERRED_TYPE(Kind) unsigned K : 3; - unsigned IsVolatile : 1; - - saved_type(DominatingLLVMValue::saved_type Val1, unsigned K) - : Vals{Val1, DominatingLLVMValue::saved_type()}, K(K) {} - - saved_type(DominatingLLVMValue::saved_type Val1, - DominatingLLVMValue::saved_type Val2) - : Vals{Val1, Val2}, K(ComplexAddress) {} - - saved_type(DominatingValue
::saved_type AggregateAddr, - bool IsVolatile, unsigned K) - : AggregateAddr(AggregateAddr), K(K) {} + unsigned Align : 29; + saved_type(llvm::Value *v, llvm::Type *e, Kind k, unsigned a = 0) + : Value(v), ElementType(e), K(k), Align(a) {} public: static bool needsSaving(RValue value); @@ -681,7 +659,7 @@ class CodeGenFunction : public CodeGenTypeCache { llvm::Value *Size; public: - CallLifetimeEnd(RawAddress addr, llvm::Value *size) + CallLifetimeEnd(Address addr, llvm::Value *size) : Addr(addr.getPointer()), Size(size) {} void Emit(CodeGenFunction &CGF, Flags flags) override { @@ -706,7 +684,7 @@ class CodeGenFunction : public CodeGenTypeCache { }; /// i32s containing the indexes of the cleanup destinations. - RawAddress NormalCleanupDest = RawAddress::invalid(); + Address NormalCleanupDest = Address::invalid(); unsigned NextCleanupDestIndex = 1; @@ -841,10 +819,10 @@ class CodeGenFunction : public CodeGenTypeCache { template void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) { if (!isInConditionalBranch()) - return pushCleanupAfterFullExprWithActiveFlag( - Kind, RawAddress::invalid(), A...); + return pushCleanupAfterFullExprWithActiveFlag(Kind, Address::invalid(), + A...); - RawAddress ActiveFlag = createCleanupActiveFlag(); + Address ActiveFlag = createCleanupActiveFlag(); assert(!DominatingValue
::needsSaving(ActiveFlag) && "cleanup active flag should never need saving"); @@ -857,7 +835,7 @@ class CodeGenFunction : public CodeGenTypeCache { template void pushCleanupAfterFullExprWithActiveFlag(CleanupKind Kind, - RawAddress ActiveFlag, As... A) { + Address ActiveFlag, As... A) { LifetimeExtendedCleanupHeader Header = {sizeof(T), Kind, ActiveFlag.isValid()}; @@ -872,7 +850,7 @@ class CodeGenFunction : public CodeGenTypeCache { new (Buffer) LifetimeExtendedCleanupHeader(Header); new (Buffer + sizeof(Header)) T(A...); if (Header.IsConditional) - new (Buffer + sizeof(Header) + sizeof(T)) RawAddress(ActiveFlag); + new (Buffer + sizeof(Header) + sizeof(T)) Address(ActiveFlag); } /// Set up the last cleanup that was pushed as a conditional @@ -881,8 +859,8 @@ class CodeGenFunction : public CodeGenTypeCache { initFullExprCleanupWithFlag(createCleanupActiveFlag()); } - void initFullExprCleanupWithFlag(RawAddress ActiveFlag); - RawAddress createCleanupActiveFlag(); + void initFullExprCleanupWithFlag(Address ActiveFlag); + Address createCleanupActiveFlag(); /// PushDestructorCleanup - Push a cleanup to call the /// complete-object destructor of an object of the given type at the @@ -1070,7 +1048,7 @@ class CodeGenFunction : public CodeGenTypeCache { QualType VarTy = LocalVD->getType(); if (VarTy->isReferenceType()) { Address Temp = CGF.CreateMemTemp(VarTy); - CGF.Builder.CreateStore(TempAddr.emitRawPointer(CGF), Temp); + CGF.Builder.CreateStore(TempAddr.getPointer(), Temp); TempAddr = Temp; } SavedTempAddresses.try_emplace(LocalVD, TempAddr); @@ -1265,12 +1243,10 @@ class CodeGenFunction : public CodeGenTypeCache { /// one branch or the other of a conditional expression. bool isInConditionalBranch() const { return OutermostConditional != nullptr; } - void setBeforeOutermostConditional(llvm::Value *value, Address addr, - CodeGenFunction &CGF) { + void setBeforeOutermostConditional(llvm::Value *value, Address addr) { assert(isInConditionalBranch()); llvm::BasicBlock *block = OutermostConditional->getStartingBlock(); - auto store = - new llvm::StoreInst(value, addr.emitRawPointer(CGF), &block->back()); + auto store = new llvm::StoreInst(value, addr.getPointer(), &block->back()); store->setAlignment(addr.getAlignment().getAsAlign()); } @@ -1625,7 +1601,7 @@ class CodeGenFunction : public CodeGenTypeCache { /// If \p StepV is null, the default increment is 1. void maybeUpdateMCDCTestVectorBitmap(const Expr *E) { if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) { - PGO.emitMCDCTestVectorBitmapUpdate(Builder, E, MCDCCondBitmapAddr, *this); + PGO.emitMCDCTestVectorBitmapUpdate(Builder, E, MCDCCondBitmapAddr); PGO.setCurrentStmt(E); } } @@ -1633,7 +1609,7 @@ class CodeGenFunction : public CodeGenTypeCache { /// Update the MCDC temp value with the condition's evaluated result. void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val) { if (isMCDCCoverageEnabled()) { - PGO.emitMCDCCondBitmapUpdate(Builder, E, MCDCCondBitmapAddr, Val, *this); + PGO.emitMCDCCondBitmapUpdate(Builder, E, MCDCCondBitmapAddr, Val); PGO.setCurrentStmt(E); } } @@ -1728,7 +1704,7 @@ class CodeGenFunction : public CodeGenTypeCache { : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue), OldCXXThisAlignment(CGF.CXXThisAlignment), SourceLocScope(E, CGF.CurSourceLocExprScope) { - CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getBasePointer(); + CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getPointer(); CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment(); } ~CXXDefaultInitExprScope() { @@ -2114,7 +2090,7 @@ class CodeGenFunction : public CodeGenTypeCache { llvm::Value *getExceptionFromSlot(); llvm::Value *getSelectorFromSlot(); - RawAddress getNormalCleanupDestSlot(); + Address getNormalCleanupDestSlot(); llvm::BasicBlock *getUnreachableBlock() { if (!UnreachableBlock) { @@ -2603,40 +2579,10 @@ class CodeGenFunction : public CodeGenTypeCache { // Helpers //===--------------------------------------------------------------------===// - Address mergeAddressesInConditionalExpr(Address LHS, Address RHS, - llvm::BasicBlock *LHSBlock, - llvm::BasicBlock *RHSBlock, - llvm::BasicBlock *MergeBlock, - QualType MergedType) { - Builder.SetInsertPoint(MergeBlock); - llvm::PHINode *PtrPhi = Builder.CreatePHI(LHS.getType(), 2, "cond"); - PtrPhi->addIncoming(LHS.getBasePointer(), LHSBlock); - PtrPhi->addIncoming(RHS.getBasePointer(), RHSBlock); - LHS.replaceBasePointer(PtrPhi); - LHS.setAlignment(std::min(LHS.getAlignment(), RHS.getAlignment())); - return LHS; - } - - /// Construct an address with the natural alignment of T. If a pointer to T - /// is expected to be signed, the pointer passed to this function must have - /// been signed, and the returned Address will have the pointer authentication - /// information needed to authenticate the signed pointer. - Address makeNaturalAddressForPointer( - llvm::Value *Ptr, QualType T, CharUnits Alignment = CharUnits::Zero(), - bool ForPointeeType = false, LValueBaseInfo *BaseInfo = nullptr, - TBAAAccessInfo *TBAAInfo = nullptr, - KnownNonNull_t IsKnownNonNull = NotKnownNonNull) { - if (Alignment.isZero()) - Alignment = - CGM.getNaturalTypeAlignment(T, BaseInfo, TBAAInfo, ForPointeeType); - return Address(Ptr, ConvertTypeForMem(T), Alignment, nullptr, - IsKnownNonNull); - } - LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source = AlignmentSource::Type) { - return MakeAddrLValue(Addr, T, LValueBaseInfo(Source), - CGM.getTBAAAccessInfo(T)); + return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source), + CGM.getTBAAAccessInfo(T)); } LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo, @@ -2646,14 +2592,6 @@ class CodeGenFunction : public CodeGenTypeCache { LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, AlignmentSource Source = AlignmentSource::Type) { - return MakeAddrLValue(makeNaturalAddressForPointer(V, T, Alignment), T, - LValueBaseInfo(Source), CGM.getTBAAAccessInfo(T)); - } - - /// Same as MakeAddrLValue above except that the pointer is known to be - /// unsigned. - LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, - AlignmentSource Source = AlignmentSource::Type) { Address Addr(V, ConvertTypeForMem(T), Alignment); return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source), CGM.getTBAAAccessInfo(T)); @@ -2666,18 +2604,9 @@ class CodeGenFunction : public CodeGenTypeCache { TBAAAccessInfo()); } - /// Given a value of type T* that may not be to a complete object, construct - /// an l-value with the natural pointee alignment of T. LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T); - LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T); - /// Same as MakeNaturalAlignPointeeAddrLValue except that the pointer is known - /// to be unsigned. - LValue MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, QualType T); - - LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T); - Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo = nullptr, TBAAAccessInfo *PointeeTBAAInfo = nullptr); @@ -2726,13 +2655,13 @@ class CodeGenFunction : public CodeGenTypeCache { /// more efficient if the caller knows that the address will not be exposed. llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp", llvm::Value *ArraySize = nullptr); - RawAddress CreateTempAlloca(llvm::Type *Ty, CharUnits align, - const Twine &Name = "tmp", - llvm::Value *ArraySize = nullptr, - RawAddress *Alloca = nullptr); - RawAddress CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align, - const Twine &Name = "tmp", - llvm::Value *ArraySize = nullptr); + Address CreateTempAlloca(llvm::Type *Ty, CharUnits align, + const Twine &Name = "tmp", + llvm::Value *ArraySize = nullptr, + Address *Alloca = nullptr); + Address CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align, + const Twine &Name = "tmp", + llvm::Value *ArraySize = nullptr); /// CreateDefaultAlignedTempAlloca - This creates an alloca with the /// default ABI alignment of the given LLVM type. @@ -2744,8 +2673,8 @@ class CodeGenFunction : public CodeGenTypeCache { /// not hand this address off to arbitrary IRGen routines, and especially /// do not pass it as an argument to a function that might expect a /// properly ABI-aligned value. - RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty, - const Twine &Name = "tmp"); + Address CreateDefaultAlignTempAlloca(llvm::Type *Ty, + const Twine &Name = "tmp"); /// CreateIRTemp - Create a temporary IR object of the given type, with /// appropriate alignment. This routine should only be used when an temporary @@ -2755,31 +2684,32 @@ class CodeGenFunction : public CodeGenTypeCache { /// /// That is, this is exactly equivalent to CreateMemTemp, but calling /// ConvertType instead of ConvertTypeForMem. - RawAddress CreateIRTemp(QualType T, const Twine &Name = "tmp"); + Address CreateIRTemp(QualType T, const Twine &Name = "tmp"); /// CreateMemTemp - Create a temporary memory object of the given type, with /// appropriate alignmen and cast it to the default address space. Returns /// the original alloca instruction by \p Alloca if it is not nullptr. - RawAddress CreateMemTemp(QualType T, const Twine &Name = "tmp", - RawAddress *Alloca = nullptr); - RawAddress CreateMemTemp(QualType T, CharUnits Align, - const Twine &Name = "tmp", - RawAddress *Alloca = nullptr); + Address CreateMemTemp(QualType T, const Twine &Name = "tmp", + Address *Alloca = nullptr); + Address CreateMemTemp(QualType T, CharUnits Align, const Twine &Name = "tmp", + Address *Alloca = nullptr); /// CreateMemTemp - Create a temporary memory object of the given type, with /// appropriate alignmen without casting it to the default address space. - RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp"); - RawAddress CreateMemTempWithoutCast(QualType T, CharUnits Align, - const Twine &Name = "tmp"); + Address CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp"); + Address CreateMemTempWithoutCast(QualType T, CharUnits Align, + const Twine &Name = "tmp"); /// CreateAggTemp - Create a temporary memory object for the given /// aggregate type. AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp", - RawAddress *Alloca = nullptr) { - return AggValueSlot::forAddr( - CreateMemTemp(T, Name, Alloca), T.getQualifiers(), - AggValueSlot::IsNotDestructed, AggValueSlot::DoesNotNeedGCBarriers, - AggValueSlot::IsNotAliased, AggValueSlot::DoesNotOverlap); + Address *Alloca = nullptr) { + return AggValueSlot::forAddr(CreateMemTemp(T, Name, Alloca), + T.getQualifiers(), + AggValueSlot::IsNotDestructed, + AggValueSlot::DoesNotNeedGCBarriers, + AggValueSlot::IsNotAliased, + AggValueSlot::DoesNotOverlap); } /// EvaluateExprAsBool - Perform the usual unary conversions on the specified @@ -3153,25 +3083,6 @@ class CodeGenFunction : public CodeGenTypeCache { /// calls to EmitTypeCheck can be skipped. bool sanitizePerformTypeCheck() const; - void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV, - QualType Type, SanitizerSet SkippedChecks = SanitizerSet(), - llvm::Value *ArraySize = nullptr) { - if (!sanitizePerformTypeCheck()) - return; - EmitTypeCheck(TCK, Loc, LV.emitRawPointer(*this), Type, LV.getAlignment(), - SkippedChecks, ArraySize); - } - - void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, Address Addr, - QualType Type, CharUnits Alignment = CharUnits::Zero(), - SanitizerSet SkippedChecks = SanitizerSet(), - llvm::Value *ArraySize = nullptr) { - if (!sanitizePerformTypeCheck()) - return; - EmitTypeCheck(TCK, Loc, Addr.emitRawPointer(*this), Type, Alignment, - SkippedChecks, ArraySize); - } - /// Emit a check that \p V is the address of storage of the /// appropriate size and alignment for an object of type \p Type /// (or if ArraySize is provided, for an array of that bound). @@ -3272,17 +3183,17 @@ class CodeGenFunction : public CodeGenTypeCache { /// Address with original alloca instruction. Invalid if the variable was /// emitted as a global constant. - RawAddress AllocaAddr; + Address AllocaAddr; struct Invalid {}; AutoVarEmission(Invalid) : Variable(nullptr), Addr(Address::invalid()), - AllocaAddr(RawAddress::invalid()) {} + AllocaAddr(Address::invalid()) {} AutoVarEmission(const VarDecl &variable) : Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr), IsEscapingByRef(false), IsConstantAggregate(false), - SizeForLifetimeMarkers(nullptr), AllocaAddr(RawAddress::invalid()) {} + SizeForLifetimeMarkers(nullptr), AllocaAddr(Address::invalid()) {} bool wasEmittedAsGlobal() const { return !Addr.isValid(); } @@ -3305,7 +3216,7 @@ class CodeGenFunction : public CodeGenTypeCache { } /// Returns the address for the original alloca instruction. - RawAddress getOriginalAllocatedAddress() const { return AllocaAddr; } + Address getOriginalAllocatedAddress() const { return AllocaAddr; } /// Returns the address of the object within this declaration. /// Note that this does not chase the forwarding pointer for @@ -3335,32 +3246,23 @@ class CodeGenFunction : public CodeGenTypeCache { llvm::GlobalValue::LinkageTypes Linkage); class ParamValue { - union { - Address Addr; - llvm::Value *Value; - }; - - bool IsIndirect; - - ParamValue(llvm::Value *V) : Value(V), IsIndirect(false) {} - ParamValue(Address A) : Addr(A), IsIndirect(true) {} - + llvm::Value *Value; + llvm::Type *ElementType; + unsigned Alignment; + ParamValue(llvm::Value *V, llvm::Type *T, unsigned A) + : Value(V), ElementType(T), Alignment(A) {} public: static ParamValue forDirect(llvm::Value *value) { - return ParamValue(value); + return ParamValue(value, nullptr, 0); } static ParamValue forIndirect(Address addr) { assert(!addr.getAlignment().isZero()); - return ParamValue(addr); + return ParamValue(addr.getPointer(), addr.getElementType(), + addr.getAlignment().getQuantity()); } - bool isIndirect() const { return IsIndirect; } - llvm::Value *getAnyValue() const { - if (!isIndirect()) - return Value; - assert(!Addr.hasOffset() && "unexpected offset"); - return Addr.getBasePointer(); - } + bool isIndirect() const { return Alignment != 0; } + llvm::Value *getAnyValue() const { return Value; } llvm::Value *getDirectValue() const { assert(!isIndirect()); @@ -3369,7 +3271,8 @@ class CodeGenFunction : public CodeGenTypeCache { Address getIndirectAddress() const { assert(isIndirect()); - return Addr; + return Address(Value, ElementType, CharUnits::fromQuantity(Alignment), + KnownNonNull); } }; @@ -4279,9 +4182,6 @@ class CodeGenFunction : public CodeGenTypeCache { const Twine &name = ""); llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name = ""); - llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee, - ArrayRef
args, - const Twine &name = ""); llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee, ArrayRef args, const Twine &name = ""); @@ -4308,12 +4208,6 @@ class CodeGenFunction : public CodeGenTypeCache { CXXDtorType Type, const CXXRecordDecl *RD); - llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) { - return Addr.getBasePointer(); - } - - bool isPointerKnownNonNull(const Expr *E); - // Return the copy constructor name with the prefix "__copy_constructor_" // removed. static std::string getNonTrivialCopyConstructorStr(QualType QT, @@ -4886,11 +4780,6 @@ class CodeGenFunction : public CodeGenTypeCache { SourceLocation Loc, const Twine &Name = ""); - Address EmitCheckedInBoundsGEP(Address Addr, ArrayRef IdxList, - llvm::Type *elementType, bool SignedIndices, - bool IsSubtraction, SourceLocation Loc, - CharUnits Align, const Twine &Name = ""); - /// Specifies which type of sanitizer check to apply when handling a /// particular builtin. enum BuiltinCheckKind { @@ -4953,10 +4842,6 @@ class CodeGenFunction : public CodeGenTypeCache { void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc, AbstractCallee AC, unsigned ParmNum); - void EmitNonNullArgCheck(Address Addr, QualType ArgType, - SourceLocation ArgLoc, AbstractCallee AC, - unsigned ParmNum); - /// EmitCallArg - Emit a single call argument. void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType); @@ -5165,7 +5050,7 @@ DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) { CGF.CreateTempAlloca(value->getType(), align, "cond-cleanup.save"); CGF.Builder.CreateStore(value, alloca); - return saved_type(alloca.emitRawPointer(CGF), true); + return saved_type(alloca.getPointer(), true); } inline llvm::Value *DominatingLLVMValue::restore(CodeGenFunction &CGF, diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index 00b3bfcaa0bc25..e3ed5e90f2d36b 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -7267,7 +7267,7 @@ void CodeGenFunction::EmitDeclMetadata() { for (auto &I : LocalDeclMap) { const Decl *D = I.first; - llvm::Value *Addr = I.second.emitRawPointer(*this); + llvm::Value *Addr = I.second.getPointer(); if (auto *Alloca = dyn_cast(Addr)) { llvm::Value *DAddr = GetPointerConstant(getLLVMContext(), D); Alloca->setMetadata( diff --git a/clang/lib/CodeGen/CodeGenPGO.cpp b/clang/lib/CodeGen/CodeGenPGO.cpp index 76704c4d7be4a4..2619edfeb7dc76 100644 --- a/clang/lib/CodeGen/CodeGenPGO.cpp +++ b/clang/lib/CodeGen/CodeGenPGO.cpp @@ -1239,8 +1239,7 @@ void CodeGenPGO::emitMCDCParameters(CGBuilderTy &Builder) { void CodeGenPGO::emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, const Expr *S, - Address MCDCCondBitmapAddr, - CodeGenFunction &CGF) { + Address MCDCCondBitmapAddr) { if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState) return; @@ -1263,7 +1262,7 @@ void CodeGenPGO::emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, Builder.getInt64(FunctionHash), Builder.getInt32(RegionMCDCState->BitmapBytes), Builder.getInt32(MCDCTestVectorBitmapOffset), - MCDCCondBitmapAddr.emitRawPointer(CGF)}; + MCDCCondBitmapAddr.getPointer()}; Builder.CreateCall( CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_tvbitmap_update), Args); } @@ -1284,8 +1283,7 @@ void CodeGenPGO::emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S, void CodeGenPGO::emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S, Address MCDCCondBitmapAddr, - llvm::Value *Val, - CodeGenFunction &CGF) { + llvm::Value *Val) { if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState) return; @@ -1314,7 +1312,7 @@ void CodeGenPGO::emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S, llvm::Value *Args[5] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), Builder.getInt64(FunctionHash), Builder.getInt32(Branch.ID), - MCDCCondBitmapAddr.emitRawPointer(CGF), Val}; + MCDCCondBitmapAddr.getPointer(), Val}; Builder.CreateCall( CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_condbitmap_update), Args); diff --git a/clang/lib/CodeGen/CodeGenPGO.h b/clang/lib/CodeGen/CodeGenPGO.h index 9d66ffad6f4350..036fbf6815a49d 100644 --- a/clang/lib/CodeGen/CodeGenPGO.h +++ b/clang/lib/CodeGen/CodeGenPGO.h @@ -113,14 +113,12 @@ class CodeGenPGO { void emitCounterSetOrIncrement(CGBuilderTy &Builder, const Stmt *S, llvm::Value *StepV); void emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, const Expr *S, - Address MCDCCondBitmapAddr, - CodeGenFunction &CGF); + Address MCDCCondBitmapAddr); void emitMCDCParameters(CGBuilderTy &Builder); void emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S, Address MCDCCondBitmapAddr); void emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S, - Address MCDCCondBitmapAddr, llvm::Value *Val, - CodeGenFunction &CGF); + Address MCDCCondBitmapAddr, llvm::Value *Val); /// Return the region count for the counter at the given index. uint64_t getRegionCount(const Stmt *S) { diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp index fd71317572f0c9..bdd53a192f828a 100644 --- a/clang/lib/CodeGen/ItaniumCXXABI.cpp +++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -307,6 +307,10 @@ class ItaniumCXXABI : public CodeGen::CGCXXABI { CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base, const CXXRecordDecl *NearestVBase); + llvm::Constant * + getVTableAddressPointForConstExpr(BaseSubobject Base, + const CXXRecordDecl *VTableClass) override; + llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) override; @@ -642,7 +646,7 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer( // Apply the adjustment and cast back to the original struct type // for consistency. - llvm::Value *This = ThisAddr.emitRawPointer(CGF); + llvm::Value *This = ThisAddr.getPointer(); This = Builder.CreateInBoundsGEP(Builder.getInt8Ty(), This, Adj); ThisPtrForCall = This; @@ -846,7 +850,7 @@ llvm::Value *ItaniumCXXABI::EmitMemberDataPointerAddress( CGBuilderTy &Builder = CGF.Builder; // Apply the offset, which we assume is non-null. - return Builder.CreateInBoundsGEP(CGF.Int8Ty, Base.emitRawPointer(CGF), MemPtr, + return Builder.CreateInBoundsGEP(CGF.Int8Ty, Base.getPointer(), MemPtr, "memptr.offset"); } @@ -1241,7 +1245,7 @@ void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF, CGF.getPointerAlign()); // Apply the offset. - llvm::Value *CompletePtr = Ptr.emitRawPointer(CGF); + llvm::Value *CompletePtr = Ptr.getPointer(); CompletePtr = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, CompletePtr, Offset); @@ -1478,8 +1482,7 @@ llvm::Value *ItaniumCXXABI::emitDynamicCastCall( computeOffsetHint(CGF.getContext(), SrcDecl, DestDecl).getQuantity()); // Emit the call to __dynamic_cast. - llvm::Value *Args[] = {ThisAddr.emitRawPointer(CGF), SrcRTTI, DestRTTI, - OffsetHint}; + llvm::Value *Args[] = {ThisAddr.getPointer(), SrcRTTI, DestRTTI, OffsetHint}; llvm::Value *Value = CGF.EmitNounwindRuntimeCall(getItaniumDynamicCastFn(CGF), Args); @@ -1568,7 +1571,7 @@ llvm::Value *ItaniumCXXABI::emitExactDynamicCast( VPtr, CGM.getTBAAVTablePtrAccessInfo(CGF.VoidPtrPtrTy)); llvm::Value *Success = CGF.Builder.CreateICmpEQ( VPtr, getVTableAddressPoint(BaseSubobject(SrcDecl, *Offset), DestDecl)); - llvm::Value *Result = ThisAddr.emitRawPointer(CGF); + llvm::Value *Result = ThisAddr.getPointer(); if (!Offset->isZero()) Result = CGF.Builder.CreateInBoundsGEP( CGF.CharTy, Result, @@ -1608,7 +1611,7 @@ llvm::Value *ItaniumCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF, PtrDiffLTy, OffsetToTop, CGF.getPointerAlign(), "offset.to.top"); } // Finally, add the offset to the pointer. - return CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ThisAddr.emitRawPointer(CGF), + return CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ThisAddr.getPointer(), OffsetToTop); } @@ -1789,8 +1792,8 @@ void ItaniumCXXABI::EmitDestructorCall(CodeGenFunction &CGF, else Callee = CGCallee::forDirect(CGM.getAddrOfCXXStructor(GD), GD); - CGF.EmitCXXDestructorCall(GD, Callee, CGF.getAsNaturalPointerTo(This, ThisTy), - ThisTy, VTT, VTTTy, nullptr); + CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, VTT, VTTTy, + nullptr); } void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, @@ -1949,6 +1952,11 @@ llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructorWithVTT( CGF.getPointerAlign()); } +llvm::Constant *ItaniumCXXABI::getVTableAddressPointForConstExpr( + BaseSubobject Base, const CXXRecordDecl *VTableClass) { + return getVTableAddressPoint(Base, VTableClass); +} + llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) { assert(VPtrOffset.isZero() && "Itanium ABI only supports zero vptr offsets"); @@ -2080,8 +2088,8 @@ llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall( ThisTy = D->getDestroyedType(); } - CGF.EmitCXXDestructorCall(GD, Callee, This.emitRawPointer(CGF), ThisTy, - nullptr, QualType(), nullptr); + CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, nullptr, + QualType(), nullptr); return nullptr; } @@ -2154,7 +2162,7 @@ static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF, int64_t VirtualAdjustment, bool IsReturnAdjustment) { if (!NonVirtualAdjustment && !VirtualAdjustment) - return InitialPtr.emitRawPointer(CGF); + return InitialPtr.getPointer(); Address V = InitialPtr.withElementType(CGF.Int8Ty); @@ -2187,10 +2195,10 @@ static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF, CGF.getPointerAlign()); } // Adjust our pointer. - ResultPtr = CGF.Builder.CreateInBoundsGEP(V.getElementType(), - V.emitRawPointer(CGF), Offset); + ResultPtr = CGF.Builder.CreateInBoundsGEP( + V.getElementType(), V.getPointer(), Offset); } else { - ResultPtr = V.emitRawPointer(CGF); + ResultPtr = V.getPointer(); } // In a derived-to-base conversion, the non-virtual adjustment is @@ -2276,7 +2284,7 @@ Address ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF, llvm::FunctionType::get(CGM.VoidTy, NumElementsPtr.getType(), false); llvm::FunctionCallee F = CGM.CreateRuntimeFunction(FTy, "__asan_poison_cxx_array_cookie"); - CGF.Builder.CreateCall(F, NumElementsPtr.emitRawPointer(CGF)); + CGF.Builder.CreateCall(F, NumElementsPtr.getPointer()); } // Finally, compute a pointer to the actual data buffer by skipping @@ -2307,7 +2315,7 @@ llvm::Value *ItaniumCXXABI::readArrayCookieImpl(CodeGenFunction &CGF, llvm::FunctionType::get(CGF.SizeTy, CGF.UnqualPtrTy, false); llvm::FunctionCallee F = CGM.CreateRuntimeFunction(FTy, "__asan_load_cxx_array_cookie"); - return CGF.Builder.CreateCall(F, numElementsPtr.emitRawPointer(CGF)); + return CGF.Builder.CreateCall(F, numElementsPtr.getPointer()); } CharUnits ARMCXXABI::getArrayCookieSizeImpl(QualType elementType) { @@ -2619,7 +2627,7 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF, // Call __cxa_guard_release. This cannot throw. CGF.EmitNounwindRuntimeCall(getGuardReleaseFn(CGM, guardPtrTy), - guardAddr.emitRawPointer(CGF)); + guardAddr.getPointer()); } else if (D.isLocalVarDecl()) { // For local variables, store 1 into the first byte of the guard variable // after the object initialization completes so that initialization is @@ -3112,10 +3120,10 @@ LValue ItaniumCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, LValue LV; if (VD->getType()->isReferenceType()) - LV = CGF.MakeNaturalAlignRawAddrLValue(CallVal, LValType); + LV = CGF.MakeNaturalAlignAddrLValue(CallVal, LValType); else - LV = CGF.MakeRawAddrLValue(CallVal, LValType, - CGF.getContext().getDeclAlign(VD)); + LV = CGF.MakeAddrLValue(CallVal, LValType, + CGF.getContext().getDeclAlign(VD)); // FIXME: need setObjCGCLValueClass? return LV; } @@ -4596,7 +4604,7 @@ static void InitCatchParam(CodeGenFunction &CGF, CGF.Builder.CreateStore(Casted, ExnPtrTmp); // Bind the reference to the temporary. - AdjustedExn = ExnPtrTmp.emitRawPointer(CGF); + AdjustedExn = ExnPtrTmp.getPointer(); } } diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp index d38a26940a3cb6..172c4c937b9728 100644 --- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp +++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp @@ -327,6 +327,10 @@ class MicrosoftCXXABI : public CGCXXABI { CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base, const CXXRecordDecl *NearestVBase) override; + llvm::Constant * + getVTableAddressPointForConstExpr(BaseSubobject Base, + const CXXRecordDecl *VTableClass) override; + llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) override; @@ -933,7 +937,7 @@ void MicrosoftCXXABI::emitBeginCatch(CodeGenFunction &CGF, } CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(*CatchParam); - CPI->setArgOperand(2, var.getObjectAddress(CGF).emitRawPointer(CGF)); + CPI->setArgOperand(2, var.getObjectAddress(CGF).getPointer()); CGF.EHStack.pushCleanup(NormalCleanup, CPI); CGF.EmitAutoVarCleanups(var); } @@ -970,7 +974,7 @@ MicrosoftCXXABI::performBaseAdjustment(CodeGenFunction &CGF, Address Value, llvm::Value *Offset = GetVirtualBaseClassOffset(CGF, Value, SrcDecl, PolymorphicBase); llvm::Value *Ptr = CGF.Builder.CreateInBoundsGEP( - Value.getElementType(), Value.emitRawPointer(CGF), Offset); + Value.getElementType(), Value.getPointer(), Offset); CharUnits VBaseAlign = CGF.CGM.getVBaseAlignment(Value.getAlignment(), SrcDecl, PolymorphicBase); return std::make_tuple(Address(Ptr, CGF.Int8Ty, VBaseAlign), Offset, @@ -1007,7 +1011,7 @@ llvm::Value *MicrosoftCXXABI::EmitTypeid(CodeGenFunction &CGF, llvm::Type *StdTypeInfoPtrTy) { std::tie(ThisPtr, std::ignore, std::ignore) = performBaseAdjustment(CGF, ThisPtr, SrcRecordTy); - llvm::CallBase *Typeid = emitRTtypeidCall(CGF, ThisPtr.emitRawPointer(CGF)); + llvm::CallBase *Typeid = emitRTtypeidCall(CGF, ThisPtr.getPointer()); return CGF.Builder.CreateBitCast(Typeid, StdTypeInfoPtrTy); } @@ -1029,7 +1033,7 @@ llvm::Value *MicrosoftCXXABI::emitDynamicCastCall( llvm::Value *Offset; std::tie(This, Offset, std::ignore) = performBaseAdjustment(CGF, This, SrcRecordTy); - llvm::Value *ThisPtr = This.emitRawPointer(CGF); + llvm::Value *ThisPtr = This.getPointer(); Offset = CGF.Builder.CreateTrunc(Offset, CGF.Int32Ty); // PVOID __RTDynamicCast( @@ -1061,7 +1065,7 @@ llvm::Value *MicrosoftCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF, llvm::FunctionCallee Function = CGF.CGM.CreateRuntimeFunction( llvm::FunctionType::get(CGF.Int8PtrTy, ArgTypes, false), "__RTCastToVoid"); - llvm::Value *Args[] = {Value.emitRawPointer(CGF)}; + llvm::Value *Args[] = {Value.getPointer()}; return CGF.EmitRuntimeCall(Function, Args); } @@ -1489,7 +1493,7 @@ Address MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall( llvm::Value *VBaseOffset = GetVirtualBaseClassOffset(CGF, Result, Derived, VBase); llvm::Value *VBasePtr = CGF.Builder.CreateInBoundsGEP( - Result.getElementType(), Result.emitRawPointer(CGF), VBaseOffset); + Result.getElementType(), Result.getPointer(), VBaseOffset); CharUnits VBaseAlign = CGF.CGM.getVBaseAlignment(Result.getAlignment(), Derived, VBase); Result = Address(VBasePtr, CGF.Int8Ty, VBaseAlign); @@ -1656,8 +1660,7 @@ void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF, llvm::Value *Implicit = getCXXDestructorImplicitParam(CGF, DD, Type, ForVirtualBase, Delegating); // = nullptr - CGF.EmitCXXDestructorCall(GD, Callee, CGF.getAsNaturalPointerTo(This, ThisTy), - ThisTy, + CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, /*ImplicitParam=*/Implicit, /*ImplicitParamTy=*/QualType(), nullptr); if (BaseDtorEndBB) { @@ -1788,6 +1791,13 @@ MicrosoftCXXABI::getVTableAddressPoint(BaseSubobject Base, return VFTablesMap[ID]; } +llvm::Constant *MicrosoftCXXABI::getVTableAddressPointForConstExpr( + BaseSubobject Base, const CXXRecordDecl *VTableClass) { + llvm::Constant *VFTable = getVTableAddressPoint(Base, VTableClass); + assert(VFTable && "Couldn't find a vftable for the given base?"); + return VFTable; +} + llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) { // getAddrOfVTable may return 0 if asked to get an address of a vtable which @@ -2003,9 +2013,8 @@ llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall( } This = adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, true); - RValue RV = - CGF.EmitCXXDestructorCall(GD, Callee, This.emitRawPointer(CGF), ThisTy, - ImplicitParam, Context.IntTy, CE); + RValue RV = CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, + ImplicitParam, Context.IntTy, CE); return RV.getScalarVal(); } @@ -2203,13 +2212,13 @@ llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF, Address This, const ThisAdjustment &TA) { if (TA.isEmpty()) - return This.emitRawPointer(CGF); + return This.getPointer(); This = This.withElementType(CGF.Int8Ty); llvm::Value *V; if (TA.Virtual.isEmpty()) { - V = This.emitRawPointer(CGF); + V = This.getPointer(); } else { assert(TA.Virtual.Microsoft.VtordispOffset < 0); // Adjust the this argument based on the vtordisp value. @@ -2218,7 +2227,7 @@ llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF, CharUnits::fromQuantity(TA.Virtual.Microsoft.VtordispOffset)); VtorDispPtr = VtorDispPtr.withElementType(CGF.Int32Ty); llvm::Value *VtorDisp = CGF.Builder.CreateLoad(VtorDispPtr, "vtordisp"); - V = CGF.Builder.CreateGEP(This.getElementType(), This.emitRawPointer(CGF), + V = CGF.Builder.CreateGEP(This.getElementType(), This.getPointer(), CGF.Builder.CreateNeg(VtorDisp)); // Unfortunately, having applied the vtordisp means that we no @@ -2255,11 +2264,11 @@ llvm::Value * MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret, const ReturnAdjustment &RA) { if (RA.isEmpty()) - return Ret.emitRawPointer(CGF); + return Ret.getPointer(); Ret = Ret.withElementType(CGF.Int8Ty); - llvm::Value *V = Ret.emitRawPointer(CGF); + llvm::Value *V = Ret.getPointer(); if (RA.Virtual.Microsoft.VBIndex) { assert(RA.Virtual.Microsoft.VBIndex > 0); int32_t IntSize = CGF.getIntSize().getQuantity(); @@ -2574,7 +2583,7 @@ struct ResetGuardBit final : EHScopeStack::Cleanup { struct CallInitThreadAbort final : EHScopeStack::Cleanup { llvm::Value *Guard; - CallInitThreadAbort(RawAddress Guard) : Guard(Guard.getPointer()) {} + CallInitThreadAbort(Address Guard) : Guard(Guard.getPointer()) {} void Emit(CodeGenFunction &CGF, Flags flags) override { // Calling _Init_thread_abort will reset the guard's state. @@ -3114,8 +3123,8 @@ MicrosoftCXXABI::GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF, llvm::Value **VBPtrOut) { CGBuilderTy &Builder = CGF.Builder; // Load the vbtable pointer from the vbptr in the instance. - llvm::Value *VBPtr = Builder.CreateInBoundsGEP( - CGM.Int8Ty, This.emitRawPointer(CGF), VBPtrOffset, "vbptr"); + llvm::Value *VBPtr = Builder.CreateInBoundsGEP(CGM.Int8Ty, This.getPointer(), + VBPtrOffset, "vbptr"); if (VBPtrOut) *VBPtrOut = VBPtr; @@ -3194,7 +3203,7 @@ llvm::Value *MicrosoftCXXABI::AdjustVirtualBase( Builder.CreateBr(SkipAdjustBB); CGF.EmitBlock(SkipAdjustBB); llvm::PHINode *Phi = Builder.CreatePHI(CGM.Int8PtrTy, 2, "memptr.base"); - Phi->addIncoming(Base.emitRawPointer(CGF), OriginalBB); + Phi->addIncoming(Base.getPointer(), OriginalBB); Phi->addIncoming(AdjustedBase, VBaseAdjustBB); return Phi; } @@ -3229,7 +3238,7 @@ llvm::Value *MicrosoftCXXABI::EmitMemberDataPointerAddress( Addr = AdjustVirtualBase(CGF, E, RD, Base, VirtualBaseAdjustmentOffset, VBPtrOffset); } else { - Addr = Base.emitRawPointer(CGF); + Addr = Base.getPointer(); } // Apply the offset, which we assume is non-null. @@ -3517,7 +3526,7 @@ CGCallee MicrosoftCXXABI::EmitLoadOfMemberFunctionPointer( ThisPtrForCall = AdjustVirtualBase(CGF, E, RD, This, VirtualBaseAdjustmentOffset, VBPtrOffset); } else { - ThisPtrForCall = This.emitRawPointer(CGF); + ThisPtrForCall = This.getPointer(); } if (NonVirtualBaseAdjustment) @@ -4436,7 +4445,10 @@ void MicrosoftCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) { llvm::GlobalVariable *TI = getThrowInfo(ThrowType); // Call into the runtime to throw the exception. - llvm::Value *Args[] = {AI.emitRawPointer(CGF), TI}; + llvm::Value *Args[] = { + AI.getPointer(), + TI + }; CGF.EmitNoreturnRuntimeCallOrInvoke(getThrowFn(), Args); } diff --git a/clang/lib/CodeGen/TargetInfo.h b/clang/lib/CodeGen/TargetInfo.h index b1dfe5bf8f274d..6893b50a3cfe90 100644 --- a/clang/lib/CodeGen/TargetInfo.h +++ b/clang/lib/CodeGen/TargetInfo.h @@ -295,11 +295,6 @@ class TargetCodeGenInfo { /// Get the AST address space for alloca. virtual LangAS getASTAllocaAddressSpace() const { return LangAS::Default; } - Address performAddrSpaceCast(CodeGen::CodeGenFunction &CGF, Address Addr, - LangAS SrcAddr, LangAS DestAddr, - llvm::Type *DestTy, - bool IsNonNull = false) const; - /// Perform address space cast of an expression of pointer type. /// \param V is the LLVM value to be casted to another address space. /// \param SrcAddr is the language address space of \p V. diff --git a/clang/lib/CodeGen/Targets/NVPTX.cpp b/clang/lib/CodeGen/Targets/NVPTX.cpp index 7dce5042c3dc20..8718f1ecf3a7e0 100644 --- a/clang/lib/CodeGen/Targets/NVPTX.cpp +++ b/clang/lib/CodeGen/Targets/NVPTX.cpp @@ -85,7 +85,7 @@ class NVPTXTargetCodeGenInfo : public TargetCodeGenInfo { LValue Src) { llvm::Value *Handle = nullptr; llvm::Constant *C = - llvm::dyn_cast(Src.getAddress(CGF).emitRawPointer(CGF)); + llvm::dyn_cast(Src.getAddress(CGF).getPointer()); // Lookup `addrspacecast` through the constant pointer if any. if (auto *ASC = llvm::dyn_cast_or_null(C)) C = llvm::cast(ASC->getPointerOperand()); diff --git a/clang/lib/CodeGen/Targets/PPC.cpp b/clang/lib/CodeGen/Targets/PPC.cpp index 362add30b43500..00b04723f17dd2 100644 --- a/clang/lib/CodeGen/Targets/PPC.cpp +++ b/clang/lib/CodeGen/Targets/PPC.cpp @@ -513,10 +513,9 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, CharUnits RegSize = CharUnits::fromQuantity((isInt || IsSoftFloatABI) ? 4 : 8); llvm::Value *RegOffset = Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity())); - RegAddr = Address(Builder.CreateInBoundsGEP( - CGF.Int8Ty, RegAddr.emitRawPointer(CGF), RegOffset), - DirectTy, - RegAddr.getAlignment().alignmentOfArrayElement(RegSize)); + RegAddr = Address( + Builder.CreateInBoundsGEP(CGF.Int8Ty, RegAddr.getPointer(), RegOffset), + DirectTy, RegAddr.getAlignment().alignmentOfArrayElement(RegSize)); // Increase the used-register count. NumRegs = @@ -552,7 +551,7 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, // Round up address of argument to alignment CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty); if (Align > OverflowAreaAlign) { - llvm::Value *Ptr = OverflowArea.emitRawPointer(CGF); + llvm::Value *Ptr = OverflowArea.getPointer(); OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align), OverflowArea.getElementType(), Align); } @@ -561,7 +560,7 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, // Increase the overflow area. OverflowArea = Builder.CreateConstInBoundsByteGEP(OverflowArea, Size); - Builder.CreateStore(OverflowArea.emitRawPointer(CGF), OverflowAreaAddr); + Builder.CreateStore(OverflowArea.getPointer(), OverflowAreaAddr); CGF.EmitBranch(Cont); } diff --git a/clang/lib/CodeGen/Targets/Sparc.cpp b/clang/lib/CodeGen/Targets/Sparc.cpp index 9025a633f328e2..a337a52a94eca9 100644 --- a/clang/lib/CodeGen/Targets/Sparc.cpp +++ b/clang/lib/CodeGen/Targets/Sparc.cpp @@ -326,7 +326,7 @@ Address SparcV9ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, // Update VAList. Address NextPtr = Builder.CreateConstInBoundsByteGEP(Addr, Stride, "ap.next"); - Builder.CreateStore(NextPtr.emitRawPointer(CGF), VAListAddr); + Builder.CreateStore(NextPtr.getPointer(), VAListAddr); return ArgAddr.withElementType(ArgTy); } diff --git a/clang/lib/CodeGen/Targets/SystemZ.cpp b/clang/lib/CodeGen/Targets/SystemZ.cpp index deaafc85a31576..6eb0c6ef2f7d63 100644 --- a/clang/lib/CodeGen/Targets/SystemZ.cpp +++ b/clang/lib/CodeGen/Targets/SystemZ.cpp @@ -306,7 +306,7 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, // Update overflow_arg_area_ptr pointer llvm::Value *NewOverflowArgArea = CGF.Builder.CreateGEP( - OverflowArgArea.getElementType(), OverflowArgArea.emitRawPointer(CGF), + OverflowArgArea.getElementType(), OverflowArgArea.getPointer(), PaddedSizeV, "overflow_arg_area"); CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr); @@ -382,9 +382,10 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, Address MemAddr = RawMemAddr.withElementType(DirectTy); // Update overflow_arg_area_ptr pointer - llvm::Value *NewOverflowArgArea = CGF.Builder.CreateGEP( - OverflowArgArea.getElementType(), OverflowArgArea.emitRawPointer(CGF), - PaddedSizeV, "overflow_arg_area"); + llvm::Value *NewOverflowArgArea = + CGF.Builder.CreateGEP(OverflowArgArea.getElementType(), + OverflowArgArea.getPointer(), PaddedSizeV, + "overflow_arg_area"); CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr); CGF.EmitBranch(ContBlock); diff --git a/clang/lib/CodeGen/Targets/XCore.cpp b/clang/lib/CodeGen/Targets/XCore.cpp index 88edb781a947b1..aeb48f851e1693 100644 --- a/clang/lib/CodeGen/Targets/XCore.cpp +++ b/clang/lib/CodeGen/Targets/XCore.cpp @@ -180,7 +180,7 @@ Address XCoreABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, // Increment the VAList. if (!ArgSize.isZero()) { Address APN = Builder.CreateConstInBoundsByteGEP(AP, ArgSize); - Builder.CreateStore(APN.emitRawPointer(CGF), VAListAddr); + Builder.CreateStore(APN.getPointer(), VAListAddr); } return Val; diff --git a/clang/utils/TableGen/MveEmitter.cpp b/clang/utils/TableGen/MveEmitter.cpp index aa20c758d84a60..3a90eee5f1c921 100644 --- a/clang/utils/TableGen/MveEmitter.cpp +++ b/clang/utils/TableGen/MveEmitter.cpp @@ -575,7 +575,7 @@ class BuiltinArgResult : public Result { // Emit code to generate this result as a Value *. std::string asValue() override { if (AddressType) - return "(" + varname() + ".emitRawPointer(*this))"; + return "(" + varname() + ".getPointer())"; return Result::asValue(); } bool hasIntegerValue() const override { return Immediate; } diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h index 2e2ec9a1c83026..2a0c1e9e8c446b 100644 --- a/llvm/include/llvm/IR/IRBuilder.h +++ b/llvm/include/llvm/IR/IRBuilder.h @@ -2708,7 +2708,6 @@ class IRBuilder : public IRBuilderBase { IRBuilder(const IRBuilder &) = delete; InserterTy &getInserter() { return Inserter; } - const InserterTy &getInserter() const { return Inserter; } }; template From 1095f71bdfe25778c169954f249819bc5b553c91 Mon Sep 17 00:00:00 2001 From: smanna12 Date: Wed, 27 Mar 2024 20:20:22 -0500 Subject: [PATCH 44/54] [NFC][Clang] Fix potential dereferencing of nullptr (#86759) This patch replaces dyn_cast<> with cast<> to resolve potential static analyzer bugs for 1. Dereferencing a pointer issue with nullptr GVar when calling addAttribute() in AIXTargetCodeGenInfo::setTargetAttributes(clang::Decl const *, llvm::GlobalValue *, clang::CodeGen::CodeGenModule &). 2. Dereferencing a pointer issue with nullptr GG when calling getCorrespondingConstructor() in DeclareImplicitDeductionGuidesForTypeAlias(clang::Sema &, clang::TypeAliasTemplateDecl *, clang::SourceLocation). 3. Dereferencing a pointer issue with nullptr CurrentBT when calling getKind() in ComplexExprEmitter::GetHigherPrecisionFPType(clang::QualType). --- clang/lib/CodeGen/CGExprComplex.cpp | 2 +- clang/lib/CodeGen/Targets/PPC.cpp | 2 +- clang/lib/Sema/SemaTemplate.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp index b873bc6737bb0a..c3774d0cb75edc 100644 --- a/clang/lib/CodeGen/CGExprComplex.cpp +++ b/clang/lib/CodeGen/CGExprComplex.cpp @@ -289,7 +289,7 @@ class ComplexExprEmitter const BinOpInfo &Op); QualType GetHigherPrecisionFPType(QualType ElementType) { - const auto *CurrentBT = dyn_cast(ElementType); + const auto *CurrentBT = cast(ElementType); switch (CurrentBT->getKind()) { case BuiltinType::Kind::Float16: return CGF.getContext().FloatTy; diff --git a/clang/lib/CodeGen/Targets/PPC.cpp b/clang/lib/CodeGen/Targets/PPC.cpp index 00b04723f17dd2..3eadb19bd2058f 100644 --- a/clang/lib/CodeGen/Targets/PPC.cpp +++ b/clang/lib/CodeGen/Targets/PPC.cpp @@ -274,7 +274,7 @@ void AIXTargetCodeGenInfo::setTargetAttributes( if (!isa(GV)) return; - auto *GVar = dyn_cast(GV); + auto *GVar = cast(GV); auto GVId = GV->getName(); // Is this a global variable specified by the user as toc-data? diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index 005529a53270c3..aab72dbaf48c46 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -2974,7 +2974,7 @@ void DeclareImplicitDeductionGuidesForTypeAlias( if (auto *FPrime = SemaRef.InstantiateFunctionDeclaration( F, TemplateArgListForBuildingFPrime, AliasTemplate->getLocation(), Sema::CodeSynthesisContext::BuildingDeductionGuides)) { - auto *GG = dyn_cast(FPrime); + auto *GG = cast(FPrime); buildDeductionGuide(SemaRef, AliasTemplate, FPrimeTemplateParamList, GG->getCorrespondingConstructor(), GG->getExplicitSpecifier(), GG->getTypeSourceInfo(), From 0c1c0d53931636331b59a03ed08f70936835399c Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Thu, 28 Mar 2024 01:32:27 +0000 Subject: [PATCH 45/54] [MLIR] Add patterns to bubble-up pack and push-down unpack through collapse/expand shape ops (#85297) Add DataLayoutPropagation patterns to bubble-up pack and push-down unpack through collapse/expand shape ops. --------- Co-authored-by: Quinn Dawkins --- .../Transforms/DataLayoutPropagation.cpp | 303 +++++++++++++++++- .../Linalg/data-layout-propagation.mlir | 160 +++++++++ 2 files changed, 462 insertions(+), 1 deletion(-) diff --git a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp index 5ceb85e7d9903b..7fd88dec71d491 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp @@ -17,6 +17,7 @@ #include "mlir/Dialect/Utils/IndexingUtils.h" #include "mlir/IR/Dominance.h" #include "mlir/Transforms/GreedyPatternRewriteDriver.h" +#include "llvm/ADT/TypeSwitch.h" #include "llvm/Support/Debug.h" #include @@ -552,6 +553,305 @@ class BubbleUpPackThroughPadOp final : public OpRewritePattern { ControlPropagationFn controlFn; }; +/// Project dimsPos to the inner-most non-unit dim pos with reassocIndices. +/// +/// For example, given dimsPos [0, 2], reassocIndices [[0, 1], [2, 3]], and +/// targetShape [16, 16, 32, 1], it returns [1, 2]. Because for pos 0, the +/// inner-most projected dim in pos [0, 1] is 1. And for pos 2, the inner-most +/// non-unit projected dims in pos [2, 3] is 2. +/// +/// If all candidates in a reassociation are unit dims, it chooses the +/// inner-most dim pos. +static SmallVector +projectToInnerMostNonUnitDimsPos(ArrayRef dimsPos, + ArrayRef reassocIndices, + ArrayRef targetShape) { + SmallVector projectedDimsPos; + for (auto pos : dimsPos) { + // In the case all dims are unit, this will return the inner-most one. + int64_t projectedPos = reassocIndices[pos].back(); + for (auto i : llvm::reverse(reassocIndices[pos])) { + int64_t dim = targetShape[i]; + if (dim > 1 || ShapedType::isDynamic(dim)) { + projectedPos = i; + break; + } + } + projectedDimsPos.push_back(projectedPos); + } + return projectedDimsPos; +} + +/// Check if all dims in dimsPos are divisible by the corresponding tile sizes. +static bool isDimsDivisibleByTileSizes(ArrayRef dimsPos, + ArrayRef shape, + ArrayRef tileSizes) { + for (auto [pos, tileSize] : llvm::zip_equal(dimsPos, tileSizes)) { + int64_t dim = shape[pos]; + if (ShapedType::isDynamic(dim) || (dim % tileSize) != 0) + return false; + } + return true; +} + +/// Permutate the reassociation indices and reindex them in the sequence order. +/// Returns the next dim pos in the sequence. +/// +/// For example, given reassocIndices [[0, 1], [2]] and permutation [1, 0], it +/// applies the permutation to get [[2], [0, 1]] and reindexes the indices into +/// [[0], [1, 2]]. +static int64_t applyPermutationAndReindexReassoc( + SmallVector &reassocIndices, + ArrayRef permutation) { + applyPermutationToVector(reassocIndices, permutation); + int64_t nextPos = 0; + for (ReassociationIndices &indices : reassocIndices) { + for (auto &index : indices) { + index = nextPos; + nextPos += 1; + } + } + return nextPos; +} + +/// Bubble up pack op through collapse shape op when the packed dims can be +/// projected to the dims before collapsing. This is possible when the inner +/// tile sizes can divide the projected dims. +/// +/// For example: +/// +/// %collapsed = tensor.collapse_shape %in [[0, 1], 2] +/// : tensor into tensor +/// %pack = tensor.pack %collapsed outer_dims_perm = [0, 1] +/// inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %empty +/// : tensor -> tensor +/// +/// can be transformed into: +/// +/// %pack = tensor.pack %in outer_dims_perm = [1, 2] +/// inner_dims_pos = [1, 2] inner_tiles = [8, 1] into %empty +/// : tensor -> tensor +/// %collapsed = tensor.collapse_shape %pack [[0, 1], 2, 3, 4] +/// : tensor into tensor +static LogicalResult +bubbleUpPackOpThroughCollapseShape(tensor::CollapseShapeOp collapseOp, + tensor::PackOp packOp, + PatternRewriter &rewriter) { + SmallVector innerTileSizes = packOp.getStaticTiles(); + ArrayRef innerDimsPos = packOp.getInnerDimsPos(); + ArrayRef outerDimsPerm = packOp.getOuterDimsPerm(); + + ArrayRef srcShape = collapseOp.getSrcType().getShape(); + SmallVector reassocIndices = + collapseOp.getReassociationIndices(); + // Project inner tile pos to the dim pos before collapsing. For example, if + // dims [x, y] is collapsed into [z], packing on dim z can be projected back + // to pack on dim y. + // + // Project to inner-most non-unit dims to increase the chance that they can be + // divided by the inner tile sizes. This is correct because for [..., x, 1], + // packing on dim 1 is equivalent to packing on dim x. + SmallVector projectedInnerDimsPos = + projectToInnerMostNonUnitDimsPos(innerDimsPos, reassocIndices, srcShape); + + if (!isDimsDivisibleByTileSizes(projectedInnerDimsPos, srcShape, + innerTileSizes)) { + return failure(); + } + // Expand the outer dims permutation with the associated source dims for the + // new permutation after bubbling. This is because moving a collapsed dim is + // equivalent to moving the associated source dims together. + SmallVector newOuterDimsPerm; + for (auto outerPos : outerDimsPerm) { + newOuterDimsPerm.insert(newOuterDimsPerm.end(), + reassocIndices[outerPos].begin(), + reassocIndices[outerPos].end()); + } + + auto emptyOp = tensor::PackOp::createDestinationTensor( + rewriter, packOp.getLoc(), collapseOp.getSrc(), packOp.getMixedTiles(), + projectedInnerDimsPos, newOuterDimsPerm); + auto newPackOp = rewriter.create( + packOp.getLoc(), collapseOp.getSrc(), emptyOp, projectedInnerDimsPos, + packOp.getMixedTiles(), packOp.getPaddingValue(), newOuterDimsPerm); + + SmallVector newReassocIndices = reassocIndices; + // First apply the permutation on the reassociations of the outer dims. + // For example given the permutation [1, 0], the reassociations [[0, 1], [2]] + // -> [[0], [1, 2]] + int64_t nextPos = + applyPermutationAndReindexReassoc(newReassocIndices, outerDimsPerm); + // Then add direct mapping for the inner tile dims. + for (size_t i = 0; i < innerDimsPos.size(); ++i) { + newReassocIndices.push_back({nextPos}); + nextPos += 1; + } + + auto newCollapseOp = rewriter.create( + collapseOp.getLoc(), packOp.getType(), newPackOp, newReassocIndices); + rewriter.replaceOp(packOp, newCollapseOp); + + return success(); +} + +class BubbleUpPackOpThroughReshapeOp final + : public OpRewritePattern { +public: + BubbleUpPackOpThroughReshapeOp(MLIRContext *context, ControlPropagationFn fun) + : OpRewritePattern(context), controlFn(std::move(fun)) {} + + LogicalResult matchAndRewrite(tensor::PackOp packOp, + PatternRewriter &rewriter) const override { + Operation *srcOp = packOp.getSource().getDefiningOp(); + // Currently only support when the pack op is the only user. + if (!srcOp || !(srcOp->getNumResults() == 1) || + !srcOp->getResult(0).hasOneUse()) { + return failure(); + } + // Currently only support static inner tile sizes. + if (llvm::any_of(packOp.getStaticTiles(), [](int64_t size) { + return ShapedType::isDynamic(size); + })) { + return failure(); + } + + // User controlled propagation function. + if (!controlFn(srcOp)) + return failure(); + + return TypeSwitch(srcOp) + .Case([&](tensor::CollapseShapeOp op) { + return bubbleUpPackOpThroughCollapseShape(op, packOp, rewriter); + }) + .Default([](Operation *) { return failure(); }); + } + +private: + ControlPropagationFn controlFn; +}; + +/// Push down unpack op through expand shape op when the packed dims can be +/// projected to the dims after expanding. This is possible when the inner tile +/// sizes can divide the projected dims. +/// +/// For example: +/// +/// %unpack = tensor.unpack %in outer_dims_perm = [0, 1] +/// inner_dims_pos = [0, 1] inner_tiles = [8, 8] into %empty +/// : tensor -> tensor +/// %expanded = tensor.expand_shape %unpack [[0, 1], [2]] +/// : tensor into tensor +/// +/// can be transformed into: +/// +/// %expanded = tensor.expand_shape %ain [[0, 1], [2], [3], [4]] +/// : tensor into tensor +/// %unpack = tensor.unpack %expanded outer_dims_perm = [0, 1, 2] +/// inner_dims_pos = [1, 2] inner_tiles = [8, 8] into %empty +/// : tensor -> tensor +static LogicalResult +pushDownUnPackOpThroughExpandShape(tensor::UnPackOp unPackOp, + tensor::ExpandShapeOp expandOp, + PatternRewriter &rewriter) { + SmallVector innerTileSizes = unPackOp.getStaticTiles(); + ArrayRef innerDimsPos = unPackOp.getInnerDimsPos(); + ArrayRef outerDimsPerm = unPackOp.getOuterDimsPerm(); + + ArrayRef dstShape = expandOp.getType().getShape(); + SmallVector reassocIndices = + expandOp.getReassociationIndices(); + // Project inner tile pos to the dim pos after expanding. For example, if dims + // [z] is expanded into [x, y], unpacking on dim z can be projected to unpack + // on dim y. + // + // Project to inner-most non-unit dims to increase the chance that they can be + // divided by the inner tile sizes. This is correct because for [..., x, 1], + // unpacking on dim 1 is equivalent to unpacking on dim x. + SmallVector projectedInnerDimsPos = + projectToInnerMostNonUnitDimsPos(innerDimsPos, reassocIndices, dstShape); + + if (!isDimsDivisibleByTileSizes(projectedInnerDimsPos, dstShape, + innerTileSizes)) { + return failure(); + } + // Expand the outer dims permutation with the associated expanded dims for the + // new permutation after pushing. This is because moving a source dim is + // equivalent to moving the associated expanded dims together. + SmallVector newOuterDimsPerm; + for (auto outerPos : outerDimsPerm) { + newOuterDimsPerm.insert(newOuterDimsPerm.end(), + reassocIndices[outerPos].begin(), + reassocIndices[outerPos].end()); + } + + SmallVector newReassocIndices = reassocIndices; + // First apply the permutation on the reassociations of the outer dims. + // For example given the permutation [1, 0], the reassociations [[0, 1], [2]] + // -> [[0], [1, 2]] + int64_t nextPos = + applyPermutationAndReindexReassoc(newReassocIndices, outerDimsPerm); + // Then add direct mapping for the inner tile dims. + for (size_t i = 0; i < innerDimsPos.size(); ++i) { + newReassocIndices.push_back({nextPos}); + nextPos += 1; + } + + RankedTensorType newExpandType = + tensor::PackOp::inferPackedType(expandOp.getType(), innerTileSizes, + projectedInnerDimsPos, newOuterDimsPerm); + auto newExpandOp = rewriter.create( + expandOp.getLoc(), newExpandType, unPackOp.getSource(), + newReassocIndices); + + auto emptyOp = tensor::UnPackOp::createDestinationTensor( + rewriter, unPackOp.getLoc(), newExpandOp, unPackOp.getMixedTiles(), + projectedInnerDimsPos, newOuterDimsPerm); + auto newUnPackOp = rewriter.create( + unPackOp.getLoc(), newExpandOp.getResult(), emptyOp, + projectedInnerDimsPos, unPackOp.getMixedTiles(), newOuterDimsPerm); + rewriter.replaceOp(expandOp, newUnPackOp); + + return success(); +} + +class PushDownUnPackOpThroughReshapeOp final + : public OpRewritePattern { +public: + PushDownUnPackOpThroughReshapeOp(MLIRContext *context, + ControlPropagationFn fun) + : OpRewritePattern(context), controlFn(std::move(fun)) { + } + + LogicalResult matchAndRewrite(tensor::UnPackOp unPackOp, + PatternRewriter &rewriter) const override { + Value result = unPackOp.getResult(); + // Currently only support unpack op with the single user. + if (!result.hasOneUse()) { + return failure(); + } + // Currently only support static inner tile sizes. + if (llvm::any_of(unPackOp.getStaticTiles(), [](int64_t size) { + return ShapedType::isDynamic(size); + })) { + return failure(); + } + + Operation *consumerOp = *result.user_begin(); + // User controlled propagation function. + if (!controlFn(consumerOp)) + return failure(); + + return TypeSwitch(consumerOp) + .Case([&](tensor::ExpandShapeOp op) { + return pushDownUnPackOpThroughExpandShape(unPackOp, op, rewriter); + }) + .Default([](Operation *) { return failure(); }); + } + +private: + ControlPropagationFn controlFn; +}; + // TODO: Relax this restriction. We should unpack a generic op also // in the presence of multiple unpack ops as producers. /// Return the unpacked operand, if present, for the current generic op. @@ -774,6 +1074,7 @@ void mlir::linalg::populateDataLayoutPropagationPatterns( const ControlPropagationFn &controlPackUnPackPropagation) { patterns .insert( + BubbleUpPackOpThroughReshapeOp, PushDownUnPackOpThroughGenericOp, + PushDownUnPackThroughPadOp, PushDownUnPackOpThroughReshapeOp>( patterns.getContext(), controlPackUnPackPropagation); } diff --git a/mlir/test/Dialect/Linalg/data-layout-propagation.mlir b/mlir/test/Dialect/Linalg/data-layout-propagation.mlir index e036695a2ac9fd..79d61ab757e327 100644 --- a/mlir/test/Dialect/Linalg/data-layout-propagation.mlir +++ b/mlir/test/Dialect/Linalg/data-layout-propagation.mlir @@ -905,3 +905,163 @@ func.func @unpack_different_destination_shape(%arg0: tensor<1x1x1080x1920x16xi32 // CHECK-SAME: inner_dims_pos = [0] inner_tiles = [16] // CHECK-SAME: into %[[UNPACK_NEW_DEST]] // CHECK: return %[[UNPACK]] : tensor<16x540x960xi32> + +// ----- + +func.func @bubble_up_pack_through_collapse(%1: tensor, %dim : index) -> tensor { + %collapsed = tensor.collapse_shape %1 [[0, 1], [2]] : tensor into tensor + %2 = tensor.empty(%dim) : tensor + %pack = tensor.pack %collapsed outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor -> tensor + func.return %pack : tensor +} +// CHECK-LABEL: func.func @bubble_up_pack_through_collapse +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]] +// CHECK-SAME: %[[ARG1:[a-zA-Z0-9]+]] +// CHECK: %[[C0:.+]] = arith.constant 0 : index +// CHECK: %[[DIM:.+]] = tensor.dim %[[ARG0]], %[[C0]] : tensor +// CHECK: %[[EMPTY:.+]] = tensor.empty(%[[DIM]]) : tensor +// CHECK: %[[PACK:.+]] = tensor.pack %[[ARG0]] outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [8, 1] into %[[EMPTY]] : tensor -> tensor +// CHECK: %[[COLLAPSED:.+]] = tensor.collapse_shape %[[PACK]] {{\[}}[0, 1], [2], [3], [4]] : tensor into tensor +// CHECK: return %[[COLLAPSED]] : tensor + +// ----- + +func.func @bubble_up_permuted_pack_through_collapse(%1: tensor<4x192x16x256xf32>) -> tensor<4x32x3072x8x1xf32> { + %collapsed = tensor.collapse_shape %1 [[0], [1, 2], [3]] : tensor<4x192x16x256xf32> into tensor<4x3072x256xf32> + %2 = tensor.empty() : tensor<4x32x3072x8x1xf32> + %pack = tensor.pack %collapsed outer_dims_perm = [0, 2, 1] inner_dims_pos = [2, 1] inner_tiles = [8, 1] into %2 : tensor<4x3072x256xf32> -> tensor<4x32x3072x8x1xf32> + func.return %pack : tensor<4x32x3072x8x1xf32> +} +// CHECK-LABEL: func.func @bubble_up_permuted_pack_through_collapse +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]] +// CHECK: %[[EMPTY:.+]] = tensor.empty() : tensor<4x32x192x16x8x1xf32> +// CHECK: %[[PACK:.+]] = tensor.pack %[[ARG0]] outer_dims_perm = [0, 3, 1, 2] inner_dims_pos = [3, 2] inner_tiles = [8, 1] into %[[EMPTY]] : tensor<4x192x16x256xf32> -> tensor<4x32x192x16x8x1xf32> +// CHECK: %[[COLLAPSED:.+]] = tensor.collapse_shape %pack {{\[}}[0], [1], [2, 3], [4], [5]] : tensor<4x32x192x16x8x1xf32> into tensor<4x32x3072x8x1xf32> +// CHECK: return %[[COLLAPSED]] : tensor<4x32x3072x8x1xf32> + +// ----- + +func.func @bubble_up_pack_through_unit_collapse(%1: tensor<1x64x1x4xf32>) -> tensor<8x4x8x1xf32> { + %collapsed = tensor.collapse_shape %1 [[0, 1, 2], [3]] : tensor<1x64x1x4xf32> into tensor<64x4xf32> + %2 = tensor.empty() : tensor<8x4x8x1xf32> + %pack = tensor.pack %collapsed outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x4xf32> -> tensor<8x4x8x1xf32> + func.return %pack : tensor<8x4x8x1xf32> +} +// CHECK-LABEL: func.func @bubble_up_pack_through_unit_collapse +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]] +// CHECK: %[[EMPTY:.+]] = tensor.empty() : tensor<1x8x1x4x8x1xf32> +// CHECK: %[[PACK:.+]] = tensor.pack %[[ARG0]] outer_dims_perm = [0, 1, 2, 3] inner_dims_pos = [1, 3] inner_tiles = [8, 1] into %[[EMPTY]] : tensor<1x64x1x4xf32> -> tensor<1x8x1x4x8x1xf32> +// CHECK: %[[COLLAPSED:.+]] = tensor.collapse_shape %[[PACK]] {{\[}}[0, 1, 2], [3], [4], [5]] : tensor<1x8x1x4x8x1xf32> into tensor<8x4x8x1xf32> +// CHECK: return %[[COLLAPSED]] : tensor<8x4x8x1xf32> + +// ----- + +func.func @bubble_up_pack_through_collapse_on_outer_dims(%1: tensor, %dim : index) -> tensor { + %collapsed = tensor.collapse_shape %1 [[0, 1], [2]] : tensor into tensor + %2 = tensor.empty(%dim) : tensor + %pack = tensor.pack %collapsed outer_dims_perm = [0, 1] inner_dims_pos = [1] inner_tiles = [4] into %2 : tensor -> tensor + func.return %pack : tensor +} +// CHECK-LABEL: func.func @bubble_up_pack_through_collapse_on_outer_dims +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]] +// CHECK-SAME: %[[ARG1:[a-zA-Z0-9]+]] +// CHECK: %[[C0:.+]] = arith.constant 0 : index +// CHECK: %[[DIM:.+]] = tensor.dim %[[ARG0]], %[[C0]] : tensor +// CHECK: %[[EMPTY:.+]] = tensor.empty(%[[DIM]]) : tensor +// CHECK: %[[PACK:.+]] = tensor.pack %[[ARG0]] outer_dims_perm = [0, 1, 2] inner_dims_pos = [2] inner_tiles = [4] into %[[EMPTY]] : tensor -> tensor +// CHECK: %[[COLLAPSED:.+]] = tensor.collapse_shape %[[PACK]] {{\[}}[0, 1], [2], [3]] : tensor into tensor +// CHECK: return %[[COLLAPSED]] : tensor + +// ----- + +func.func @no_bubble_up_pack_through_non_divisible_collapse(%1: tensor<3072x64x4xf32>) -> tensor<384x32x8x8xf32> { + %collapsed = tensor.collapse_shape %1 [[0], [1, 2]] : tensor<3072x64x4xf32> into tensor<3072x256xf32> + %2 = tensor.empty() : tensor<384x32x8x8xf32> + %pack = tensor.pack %collapsed outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 8] into %2 : tensor<3072x256xf32> -> tensor<384x32x8x8xf32> + func.return %pack : tensor<384x32x8x8xf32> +} +// CHECK-LABEL: func.func @no_bubble_up_pack_through_non_divisible_collapse +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]] +// CHECK: %[[COLLAPSED:.+]] = tensor.collapse_shape %[[ARG0]] {{\[}}[0], [1, 2]] : tensor<3072x64x4xf32> into tensor<3072x256xf32> +// CHECK: %[[PACK:.+]] = tensor.pack %[[COLLAPSED]] +// CHECK: return %[[PACK]] : tensor<384x32x8x8xf32> + +// ----- + +func.func @push_down_unpack_through_expand(%5: tensor, %dim: index) -> tensor { + %6 = tensor.empty(%dim) : tensor + %unpack = tensor.unpack %5 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 8] into %6 : tensor -> tensor + %expanded = tensor.expand_shape %unpack [[0, 1], [2]] : tensor into tensor + func.return %expanded : tensor +} +// CHECK-LABEL: func.func @push_down_unpack_through_expand +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]] +// CHECK-SAME: %[[ARG1:[a-zA-Z0-9]+]] +// CHECK: %[[C0:.+]] = arith.constant 0 : index +// CHECK: %[[EXPANDED:.+]] = tensor.expand_shape %[[ARG0]] {{\[}}[0, 1], [2], [3], [4]] : tensor into tensor +// CHECK: %[[DIM:.+]] = tensor.dim %[[EXPANDED]], %[[C0]] : tensor +// CHECK: %[[EMPTY:.+]] = tensor.empty(%[[DIM]]) : tensor +// CHECK: %[[UNPACK:.+]] = tensor.unpack %[[EXPANDED:.+]] outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [8, 8] into %[[EMPTY]] : tensor -> tensor +// CHECK: return %[[UNPACK]] : tensor + +// ----- + +func.func @push_down_permuted_unpack_through_expand(%5: tensor<4x32x384x8x8xf32>) -> tensor<4x12x256x256xf32> { + %6 = tensor.empty() : tensor<4x3072x256xf32> + %unpack = tensor.unpack %5 outer_dims_perm = [0, 2, 1] inner_dims_pos = [2, 1] inner_tiles = [8, 8] into %6 : tensor<4x32x384x8x8xf32> -> tensor<4x3072x256xf32> + %expanded = tensor.expand_shape %unpack [[0], [1, 2], [3]] : tensor<4x3072x256xf32> into tensor<4x12x256x256xf32> + func.return %expanded : tensor<4x12x256x256xf32> +} +// CHECK-LABEL: @push_down_permuted_unpack_through_expand +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]] +// CHECK: %[[EXPANDED:.+]] = tensor.expand_shape %[[ARG0]] {{\[}}[0], [1], [2, 3], [4], [5]] : tensor<4x32x384x8x8xf32> into tensor<4x32x12x32x8x8xf32> +// CHECK: %[[EMPTY:.+]] = tensor.empty() : tensor<4x12x256x256xf32> +// CHECK: %[[UNPACK:.+]] = tensor.unpack %[[EXPANDED]] outer_dims_perm = [0, 3, 1, 2] inner_dims_pos = [3, 2] inner_tiles = [8, 8] into %[[EMPTY]] : tensor<4x32x12x32x8x8xf32> -> tensor<4x12x256x256xf32> +// CHECK: return %[[UNPACK]] : tensor<4x12x256x256xf32> + +// ----- + +func.func @push_down_unpack_through_unit_expand(%5: tensor<6x32x8x8xf32>) -> tensor<3x16x1x256xf32> { + %6 = tensor.empty() : tensor<48x256xf32> + %unpack = tensor.unpack %5 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 8] into %6 : tensor<6x32x8x8xf32> -> tensor<48x256xf32> + %expanded = tensor.expand_shape %unpack [[0, 1, 2], [3]] : tensor<48x256xf32> into tensor<3x16x1x256xf32> + func.return %expanded : tensor<3x16x1x256xf32> +} +// CHECK-LABEL: func.func @push_down_unpack_through_unit_expand +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]] +// CHECK: %[[EXPANDED:.+]] = tensor.expand_shape %[[ARG0]] {{\[}}[0, 1, 2], [3], [4], [5]] : tensor<6x32x8x8xf32> into tensor<3x2x1x32x8x8xf32> +// CHECK: %[[EMPTY:.+]] = tensor.empty() : tensor<3x16x1x256xf32> +// CHECK: %[[UNPACK:.+]] = tensor.unpack %[[EXPANDED]] outer_dims_perm = [0, 1, 2, 3] inner_dims_pos = [1, 3] inner_tiles = [8, 8] into %[[EMPTY]] : tensor<3x2x1x32x8x8xf32> -> tensor<3x16x1x256xf32> +// CHECK: return %[[UNPACK]] : tensor<3x16x1x256xf32> + +// ----- + +func.func @push_down_unpack_through_expand_on_outer_dims(%5: tensor, %dim: index) -> tensor { + %6 = tensor.empty(%dim) : tensor + %unpack = tensor.unpack %5 outer_dims_perm = [0, 1] inner_dims_pos = [1] inner_tiles = [8] into %6 : tensor -> tensor + %expanded = tensor.expand_shape %unpack [[0, 1], [2]] : tensor into tensor + func.return %expanded : tensor +} +// CHECK-LABEL: func.func @push_down_unpack_through_expand_on_outer_dims +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]] +// CHECK-SAME: %[[ARG1:[a-zA-Z0-9]+]] +// CHECK: %[[C0:.+]] = arith.constant 0 : index +// CHECK: %[[EXPANDED:.+]] = tensor.expand_shape %[[ARG0]] {{\[}}[0, 1], [2], [3]] : tensor into tensor +// CHECK: %[[DIM:.+]] = tensor.dim %[[EXPANDED]], %[[C0]] : tensor +// CHECK: %[[EMPTY:.+]] = tensor.empty(%[[DIM]]) : tensor +// CHECK: %[[UNPACK:.+]] = tensor.unpack %[[EXPANDED:.+]] outer_dims_perm = [0, 1, 2] inner_dims_pos = [2] inner_tiles = [8] into %[[EMPTY]] : tensor -> tensor +// CHECK: return %[[UNPACK]] : tensor + +// ----- + +func.func @no_push_down_unpack_through_non_divisible_expand(%5: tensor<384x32x8x8xf32>) -> tensor<256x12x256xf32> { + %6 = tensor.empty() : tensor<3072x256xf32> + %unpack = tensor.unpack %5 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 8] into %6 : tensor<384x32x8x8xf32> -> tensor<3072x256xf32> + %expanded = tensor.expand_shape %unpack [[0, 1], [2]] : tensor<3072x256xf32> into tensor<256x12x256xf32> + func.return %expanded : tensor<256x12x256xf32> +} +// CHECK-LABEL: func.func @no_push_down_unpack_through_non_divisible_expand +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]] +// CHECK: %[[UNPACK:.+]] = tensor.unpack %[[ARG0]] +// CHECK: %[[EXPANDED:.+]] = tensor.expand_shape %[[UNPACK]] {{\[}}[0, 1], [2]] : tensor<3072x256xf32> into tensor<256x12x256xf32> +// CHECK: return %[[EXPANDED]] : tensor<256x12x256xf32> From 443baed56c770aca050d27581d5d6f0c5c168285 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 27 Mar 2024 20:01:30 -0700 Subject: [PATCH 46/54] [ELF,test] Update tests that depend on --export-dynamic creating dynamic sections The CloudABI change from https://reviews.llvm.org/D30175 does not make sense. Update tests not to rely on the --export-dynamic behavior. --- lld/test/ELF/common-gc2.s | 8 +++++--- lld/test/ELF/executable-undefined-ignoreall.s | 2 -- .../ELF/relro-non-contiguous-script-data.s | 6 ++++-- lld/test/ELF/riscv-undefined-weak.s | 18 ++++++++---------- lld/test/ELF/x86-64-dyn-rel-error.s | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/lld/test/ELF/common-gc2.s b/lld/test/ELF/common-gc2.s index fec1c4be86b5ee..1ecaef7d9af5aa 100644 --- a/lld/test/ELF/common-gc2.s +++ b/lld/test/ELF/common-gc2.s @@ -1,7 +1,9 @@ # REQUIRES: x86 -# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t -# RUN: ld.lld -gc-sections -export-dynamic %t -o %t1 -# RUN: llvm-readobj --dyn-symbols %t1 | FileCheck %s +# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o +# RUN: llvm-mc -filetype=obj -triple=x86_64 /dev/null -o %t2.o +# RUN: ld.lld -shared -soname=t2 %t2.o -o %t2.so +# RUN: ld.lld -gc-sections -export-dynamic %t.o %t2.so -o %t +# RUN: llvm-readobj --dyn-symbols %t | FileCheck %s # CHECK: Name: bar # CHECK: Name: foo diff --git a/lld/test/ELF/executable-undefined-ignoreall.s b/lld/test/ELF/executable-undefined-ignoreall.s index cc38e17cdf619b..073b22bd84543a 100644 --- a/lld/test/ELF/executable-undefined-ignoreall.s +++ b/lld/test/ELF/executable-undefined-ignoreall.s @@ -7,8 +7,6 @@ # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o # RUN: ld.lld %t.o -o %t --unresolved-symbols=ignore-all -pie # RUN: llvm-readobj -r %t | FileCheck %s -# RUN: ld.lld %t.o -o %t --unresolved-symbols=ignore-all --export-dynamic -# RUN: llvm-readobj -r %t | FileCheck %s # CHECK: Relocations [ # CHECK-NEXT: Section ({{.*}}) .rela.plt { diff --git a/lld/test/ELF/relro-non-contiguous-script-data.s b/lld/test/ELF/relro-non-contiguous-script-data.s index fd485e89167fcc..530fc7c84eb91e 100644 --- a/lld/test/ELF/relro-non-contiguous-script-data.s +++ b/lld/test/ELF/relro-non-contiguous-script-data.s @@ -1,19 +1,21 @@ // REQUIRES: x86 +// RUN: llvm-mc -filetype=obj -triple=x86_64 /dev/null -o %t2.o +// RUN: ld.lld -shared -soname=t2 %t2.o -o %t2.so // RUN: echo "SECTIONS { \ // RUN: .dynamic : { *(.dynamic) } \ // RUN: .non_ro : { . += 1; } \ // RUN: .jcr : { *(.jcr) } \ // RUN: } " > %t.script // RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o -// RUN: not ld.lld --export-dynamic %t.o -o /dev/null --script=%t.script 2>&1 | FileCheck %s +// RUN: not ld.lld %t.o %t2.so -o /dev/null --script=%t.script 2>&1 | FileCheck %s // RUN: echo "SECTIONS { \ // RUN: .dynamic : { *(.dynamic) } \ // RUN: .non_ro : { BYTE(1); } \ // RUN: .jcr : { *(.jcr) } \ // RUN: } " > %t2.script -// RUN: not ld.lld --export-dynamic %t.o -o /dev/null --script=%t2.script 2>&1 | FileCheck %s +// RUN: not ld.lld %t.o %t2.so -o /dev/null --script=%t2.script 2>&1 | FileCheck %s // CHECK: error: section: .jcr is not contiguous with other relro sections diff --git a/lld/test/ELF/riscv-undefined-weak.s b/lld/test/ELF/riscv-undefined-weak.s index 303a27f920c57c..8a78e1f8383386 100644 --- a/lld/test/ELF/riscv-undefined-weak.s +++ b/lld/test/ELF/riscv-undefined-weak.s @@ -1,4 +1,6 @@ # REQUIRES: riscv +# RUN: llvm-mc -filetype=obj -triple=riscv64 /dev/null -o %t2.o +# RUN: ld.lld -shared -soname=t2 %t2.o -o %t2.so # RUN: llvm-mc -filetype=obj -triple=riscv64 -riscv-asm-relax-branches=0 %s -o %t.o # RUN: llvm-readobj -r %t.o | FileCheck --check-prefix=RELOC %s @@ -6,7 +8,7 @@ # RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck --check-prefixes=CHECK,PC %s # RUN: llvm-readelf -x .data %t | FileCheck --check-prefixes=HEX,HEX-WITHOUT-PLT %s -# RUN: ld.lld -e absolute %t.o -o %t --export-dynamic +# RUN: ld.lld -e absolute %t.o -o %t %t2.so # RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck --check-prefixes=CHECK,PLT %s # RUN: llvm-readelf -x .data %t | FileCheck --check-prefixes=HEX,HEX-WITH-PLT %s @@ -34,11 +36,11 @@ absolute: # CHECK-LABEL: : # CHECK-NEXT: 11{{...}}: auipc a1, 0xfffef # PC-NEXT: addi a1, a1, -0x160 -# PLT-NEXT: addi a1, a1, -0x318 +# PLT-NEXT: addi a1, a1, -0x290 # CHECK-LABEL: <.Lpcrel_hi1>: # CHECK-NEXT: 11{{...}}: auipc t1, 0xfffef # PC-NEXT: sd a2, -0x166(t1) -# PLT-NEXT: sd a2, -0x31e(t1) +# PLT-NEXT: sd a2, -0x296(t1) relative: la a1, target sd a2, target+2, t1 @@ -62,7 +64,7 @@ relative: ## We create a PLT entry and redirect the reference to it. # PLT-LABEL: : # PLT-NEXT: auipc ra, 0x0 -# PLT-NEXT: jalr 0x38(ra) +# PLT-NEXT: jalr 0x30(ra) # PLT-NEXT: [[#%x,ADDR:]]: # PLT-SAME: j 0x[[#ADDR]] # PLT-NEXT: [[#%x,ADDR:]]: @@ -84,12 +86,8 @@ branch: ## A plt entry is created for target, so this is the offset between the ## plt entry and this address. ## -## S = 0x11360 (the address of the plt entry for target) -## A = 0 -## P = 0x1343c (the address of `.`) -## -## S - A + P = -0x0x20dc = 0xffffdf24 -# HEX-WITH-PLT-SAME: 24dfffff +## S - A + P = -0x0x20ec = 0xffffdf14 +# HEX-WITH-PLT-SAME: 14dfffff .data .p2align 3 diff --git a/lld/test/ELF/x86-64-dyn-rel-error.s b/lld/test/ELF/x86-64-dyn-rel-error.s index a03adf89072f31..1590045312d4a3 100644 --- a/lld/test/ELF/x86-64-dyn-rel-error.s +++ b/lld/test/ELF/x86-64-dyn-rel-error.s @@ -19,7 +19,7 @@ # CHECK-NOT: error: # RUN: ld.lld --noinhibit-exec %t.o %t2.so -o /dev/null 2>&1 | FileCheck --check-prefix=WARN %s -# RUN: not ld.lld --export-dynamic --unresolved-symbols=ignore-all %t.o -o /dev/null 2>&1 | FileCheck --check-prefix=WARN %s +# RUN: not ld.lld --export-dynamic --unresolved-symbols=ignore-all %t.o %t2.so -o /dev/null 2>&1 | FileCheck --check-prefix=WARN %s # WARN: relocation R_X86_64_32 cannot be used against symbol 'zed'; recompile with -fPIC # WARN: relocation R_X86_64_PC32 cannot be used against symbol 'zed'; recompile with -fPIC From 070d7af0c56b993806fa47f77b607b1849a2172f Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 27 Mar 2024 20:04:59 -0700 Subject: [PATCH 47/54] [ELF] --export-dynamic: don't create dynamic sections for non-PIC static links The CloudABI (removed from Clang Driver) change from https://reviews.llvm.org/D29982 does not make sense. GNU ld and gold don't create dynamic sections for a non-PIC static link when --export-dynamic is specified. Creating dynamic sections is harmful in this scenario because we would consider undefined weak symbols preemptible and generate GLOB_DAT relocations, breaking the expectation that non-PIC static links only contain IRELATIVE relocations. In addition, there are other options that export symbols (--export-dynamic-symbol, --dynamic-list, etc). It does not make sense to special case --export-dynamic. --- lld/ELF/Driver.cpp | 10 ++----- lld/test/ELF/static-with-export-dynamic.s | 32 ----------------------- lld/test/ELF/weak-undef.s | 9 ++++--- 3 files changed, 7 insertions(+), 44 deletions(-) delete mode 100644 lld/test/ELF/static-with-export-dynamic.s diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp index f14a2376601b08..b43da7727e22ae 100644 --- a/lld/ELF/Driver.cpp +++ b/lld/ELF/Driver.cpp @@ -2724,14 +2724,8 @@ template void LinkerDriver::link(opt::InputArgList &args) { parseFiles(files, armCmseImpLib); - // Now that we have every file, we can decide if we will need a - // dynamic symbol table. - // We need one if we were asked to export dynamic symbols or if we are - // producing a shared library. - // We also need one if any shared libraries are used and for pie executables - // (probably because the dynamic linker needs it). - config->hasDynSymTab = - !ctx.sharedFiles.empty() || config->isPic || config->exportDynamic; + // Create dynamic sections for dynamic linking and static PIE. + config->hasDynSymTab = !ctx.sharedFiles.empty() || config->isPic; script->addScriptReferencedSymbolsToSymTable(); diff --git a/lld/test/ELF/static-with-export-dynamic.s b/lld/test/ELF/static-with-export-dynamic.s deleted file mode 100644 index b0349b85e30343..00000000000000 --- a/lld/test/ELF/static-with-export-dynamic.s +++ /dev/null @@ -1,32 +0,0 @@ -// REQUIRES: x86 -// RUN: llvm-mc -filetype=obj -triple=i686-unknown-cloudabi %s -o %t.o -// RUN: ld.lld --export-dynamic %t.o -o %t -// RUN: llvm-readobj --dyn-syms %t | FileCheck %s - -// Ensure that a dynamic symbol table is present when --export-dynamic -// is passed in, even when creating statically linked executables. -// -// CHECK: DynamicSymbols [ -// CHECK-NEXT: Symbol { -// CHECK-NEXT: Name: -// CHECK-NEXT: Value: 0x0 -// CHECK-NEXT: Size: 0 -// CHECK-NEXT: Binding: Local -// CHECK-NEXT: Type: None -// CHECK-NEXT: Other: 0 -// CHECK-NEXT: Section: Undefined -// CHECK-NEXT: } -// CHECK-NEXT: Symbol { -// CHECK-NEXT: Name: _start -// CHECK-NEXT: Value: -// CHECK-NEXT: Size: 0 -// CHECK-NEXT: Binding: Global -// CHECK-NEXT: Type: None -// CHECK-NEXT: Other: 0 -// CHECK-NEXT: Section: .text -// CHECK-NEXT: } -// CHECK-NEXT: ] - -.global _start -_start: - ret diff --git a/lld/test/ELF/weak-undef.s b/lld/test/ELF/weak-undef.s index 3a9d5f462c21b6..21488023a79e10 100644 --- a/lld/test/ELF/weak-undef.s +++ b/lld/test/ELF/weak-undef.s @@ -16,10 +16,11 @@ # RELOC-NEXT: Offset Info Type Symbol's Value Symbol's Name + Addend # RELOC-NEXT: {{.*}} 0000000100000001 R_X86_64_64 0000000000000000 foo + 0 -# COMMON: Symbol table '.dynsym' contains 2 entries: -# COMMON-NEXT: Num: Value Size Type Bind Vis Ndx Name -# COMMON-NEXT: 0: 0000000000000000 0 NOTYPE LOCAL DEFAULT UND -# COMMON-NEXT: 1: 0000000000000000 0 NOTYPE WEAK DEFAULT UND foo +# NORELOC-NOT: Symbol table '.dynsym' +# RELOC: Symbol table '.dynsym' contains 2 entries: +# RELOC-NEXT: Num: Value Size Type Bind Vis Ndx Name +# RELOC-NEXT: 0: 0000000000000000 0 NOTYPE LOCAL DEFAULT UND +# RELOC-NEXT: 1: 0000000000000000 0 NOTYPE WEAK DEFAULT UND foo # COMMON: Hex dump of section '.data': # COMMON-NEXT: {{.*}} 00000000 00000000 # COMMON-EMPTY: From 2c7610cc43cd70192a0ed5eac58471c50045c6de Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Wed, 27 Mar 2024 20:40:01 -0700 Subject: [PATCH 48/54] [nfc]Make InstrProfSymtab non-copyable and non-movable (#86882) - The direct use case (in [1]) is to add `llvm::IntervalMap` [2] and the allocator required by IntervalMap ctor [3] to class `InstrProfSymtab` as owned members. The allocator class doesn't have a move-assignment operator; and it's going to take much effort to implement move-assignment operator for the allocator class such that the enclosing class is movable. - There is only one use of compiler-generated move-assignment operator in the repo, which is in CoverageMappingReader.cpp. Luckily it's possible to use std::unique_ptr instead, so did the change. [1] https://github.com/llvm/llvm-project/pull/66825 [2] https://github.com/llvm/llvm-project/blob/4c2f68840e984b0f111779c46845ac00e3a7547d/llvm/include/llvm/ADT/IntervalMap.h#L936 [3] https://github.com/llvm/llvm-project/blob/4c2f68840e984b0f111779c46845ac00e3a7547d/llvm/include/llvm/ADT/IntervalMap.h#L1041 --- .../Coverage/CoverageMappingReader.h | 17 +++++---- llvm/include/llvm/ProfileData/InstrProf.h | 7 ++++ .../Coverage/CoverageMappingReader.cpp | 35 ++++++++++--------- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h index 346ca4ad2eb314..f05b90114d75a6 100644 --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h @@ -184,7 +184,7 @@ class BinaryCoverageReader : public CoverageMappingReader { private: std::vector Filenames; std::vector MappingRecords; - InstrProfSymtab ProfileNames; + std::unique_ptr ProfileNames; size_t CurrentRecord = 0; std::vector FunctionsFilenames; std::vector Expressions; @@ -195,8 +195,9 @@ class BinaryCoverageReader : public CoverageMappingReader { // D69471, which can split up function records into multiple sections on ELF. FuncRecordsStorage FuncRecords; - BinaryCoverageReader(FuncRecordsStorage &&FuncRecords) - : FuncRecords(std::move(FuncRecords)) {} + BinaryCoverageReader(std::unique_ptr Symtab, + FuncRecordsStorage &&FuncRecords) + : ProfileNames(std::move(Symtab)), FuncRecords(std::move(FuncRecords)) {} public: BinaryCoverageReader(const BinaryCoverageReader &) = delete; @@ -209,12 +210,10 @@ class BinaryCoverageReader : public CoverageMappingReader { SmallVectorImpl *BinaryIDs = nullptr); static Expected> - createCoverageReaderFromBuffer(StringRef Coverage, - FuncRecordsStorage &&FuncRecords, - InstrProfSymtab &&ProfileNames, - uint8_t BytesInAddress, - llvm::endianness Endian, - StringRef CompilationDir = ""); + createCoverageReaderFromBuffer( + StringRef Coverage, FuncRecordsStorage &&FuncRecords, + std::unique_ptr ProfileNamesPtr, uint8_t BytesInAddress, + llvm::endianness Endian, StringRef CompilationDir = ""); Error readNextRecord(CoverageMappingRecord &Record) override; }; diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 25ec06a7392027..612c444faec648 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -471,6 +471,13 @@ class InstrProfSymtab { public: InstrProfSymtab() = default; + // Not copyable or movable. + // Consider std::unique_ptr for move. + InstrProfSymtab(const InstrProfSymtab &) = delete; + InstrProfSymtab &operator=(const InstrProfSymtab &) = delete; + InstrProfSymtab(InstrProfSymtab &&) = delete; + InstrProfSymtab &operator=(InstrProfSymtab &&) = delete; + /// Create InstrProfSymtab from an object file section which /// contains function PGO names. When section may contain raw /// string data or string data in compressed form. This method diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp index d328460510830a..445b48067a9755 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -894,31 +894,34 @@ static Error readCoverageMappingData( Expected> BinaryCoverageReader::createCoverageReaderFromBuffer( StringRef Coverage, FuncRecordsStorage &&FuncRecords, - InstrProfSymtab &&ProfileNames, uint8_t BytesInAddress, + std::unique_ptr ProfileNamesPtr, uint8_t BytesInAddress, llvm::endianness Endian, StringRef CompilationDir) { - std::unique_ptr Reader( - new BinaryCoverageReader(std::move(FuncRecords))); - Reader->ProfileNames = std::move(ProfileNames); + if (ProfileNamesPtr == nullptr) + return make_error(coveragemap_error::malformed, + "Caller must provide ProfileNames"); + std::unique_ptr Reader(new BinaryCoverageReader( + std::move(ProfileNamesPtr), std::move(FuncRecords))); + InstrProfSymtab &ProfileNames = *Reader->ProfileNames; StringRef FuncRecordsRef = Reader->FuncRecords->getBuffer(); if (BytesInAddress == 4 && Endian == llvm::endianness::little) { if (Error E = readCoverageMappingData( - Reader->ProfileNames, Coverage, FuncRecordsRef, - Reader->MappingRecords, CompilationDir, Reader->Filenames)) + ProfileNames, Coverage, FuncRecordsRef, Reader->MappingRecords, + CompilationDir, Reader->Filenames)) return std::move(E); } else if (BytesInAddress == 4 && Endian == llvm::endianness::big) { if (Error E = readCoverageMappingData( - Reader->ProfileNames, Coverage, FuncRecordsRef, - Reader->MappingRecords, CompilationDir, Reader->Filenames)) + ProfileNames, Coverage, FuncRecordsRef, Reader->MappingRecords, + CompilationDir, Reader->Filenames)) return std::move(E); } else if (BytesInAddress == 8 && Endian == llvm::endianness::little) { if (Error E = readCoverageMappingData( - Reader->ProfileNames, Coverage, FuncRecordsRef, - Reader->MappingRecords, CompilationDir, Reader->Filenames)) + ProfileNames, Coverage, FuncRecordsRef, Reader->MappingRecords, + CompilationDir, Reader->Filenames)) return std::move(E); } else if (BytesInAddress == 8 && Endian == llvm::endianness::big) { if (Error E = readCoverageMappingData( - Reader->ProfileNames, Coverage, FuncRecordsRef, - Reader->MappingRecords, CompilationDir, Reader->Filenames)) + ProfileNames, Coverage, FuncRecordsRef, Reader->MappingRecords, + CompilationDir, Reader->Filenames)) return std::move(E); } else return make_error( @@ -963,8 +966,8 @@ loadTestingFormat(StringRef Data, StringRef CompilationDir) { if (Data.size() < ProfileNamesSize) return make_error(coveragemap_error::malformed, "the size of ProfileNames is too big"); - InstrProfSymtab ProfileNames; - if (Error E = ProfileNames.create(Data.substr(0, ProfileNamesSize), Address)) + auto ProfileNames = std::make_unique(); + if (Error E = ProfileNames->create(Data.substr(0, ProfileNamesSize), Address)) return std::move(E); Data = Data.substr(ProfileNamesSize); @@ -1099,7 +1102,7 @@ loadBinaryFormat(std::unique_ptr Bin, StringRef Arch, OF->isLittleEndian() ? llvm::endianness::little : llvm::endianness::big; // Look for the sections that we are interested in. - InstrProfSymtab ProfileNames; + auto ProfileNames = std::make_unique(); std::vector NamesSectionRefs; // If IPSK_name is not found, fallback to search for IPK_covname, which is // used when binary correlation is enabled. @@ -1116,7 +1119,7 @@ loadBinaryFormat(std::unique_ptr Bin, StringRef Arch, return make_error( coveragemap_error::malformed, "the size of coverage mapping section is not one"); - if (Error E = ProfileNames.create(NamesSectionRefs.back())) + if (Error E = ProfileNames->create(NamesSectionRefs.back())) return std::move(E); auto CoverageSection = lookupSections(*OF, IPSK_covmap); From 056b4043543cbc9e4ecad183db185bd26324b5b1 Mon Sep 17 00:00:00 2001 From: Job Henandez Lara Date: Wed, 27 Mar 2024 20:55:12 -0700 Subject: [PATCH 49/54] [libc][NFC] refactor fmin and fmax (#86718) Hello, So, I worked on the fmaximum and fminimum functions recently and the reviewers suggested the structure: ``` if (bitsx ...) return ...; if (bitsy ..) return ... return ...; ``` So I went ahead and did the same for fmin and fmax. I hope this isnt an issue for you all. thanks. --------- Co-authored-by: Job Hernandez --- libc/src/__support/FPUtil/BasicOperations.h | 24 +++++++++------------ 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/libc/src/__support/FPUtil/BasicOperations.h b/libc/src/__support/FPUtil/BasicOperations.h index f746d7ac6ad41f..a47931bb33900a 100644 --- a/libc/src/__support/FPUtil/BasicOperations.h +++ b/libc/src/__support/FPUtil/BasicOperations.h @@ -30,36 +30,32 @@ template , int> = 0> LIBC_INLINE T fmin(T x, T y) { const FPBits bitx(x), bity(y); - if (bitx.is_nan()) { + if (bitx.is_nan()) return y; - } else if (bity.is_nan()) { + if (bity.is_nan()) return x; - } else if (bitx.sign() != bity.sign()) { + if (bitx.sign() != bity.sign()) // To make sure that fmin(+0, -0) == -0 == fmin(-0, +0), whenever x and // y has different signs and both are not NaNs, we return the number // with negative sign. - return (bitx.is_neg()) ? x : y; - } else { - return (x < y ? x : y); - } + return bitx.is_neg() ? x : y; + return x < y ? x : y; } template , int> = 0> LIBC_INLINE T fmax(T x, T y) { FPBits bitx(x), bity(y); - if (bitx.is_nan()) { + if (bitx.is_nan()) return y; - } else if (bity.is_nan()) { + if (bity.is_nan()) return x; - } else if (bitx.sign() != bity.sign()) { + if (bitx.sign() != bity.sign()) // To make sure that fmax(+0, -0) == +0 == fmax(-0, +0), whenever x and // y has different signs and both are not NaNs, we return the number // with positive sign. - return (bitx.is_neg() ? y : x); - } else { - return (x > y ? x : y); - } + return bitx.is_neg() ? y : x; + return x > y ? x : y; } template , int> = 0> From e766f87b922933d6b1aefcfd24e5111162369e2e Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Wed, 27 Mar 2024 21:22:57 -0700 Subject: [PATCH 50/54] [clang-format] Handle C++ Core Guidelines suppression tags (#86458) Fixes #86451. --- clang/lib/Format/TokenAnnotator.cpp | 4 ++++ clang/unittests/Format/FormatTest.cpp | 1 + 2 files changed, 5 insertions(+) diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index 4c83a7a3a323be..b9144cf55452e2 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -4827,6 +4827,10 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line, Right.is(TT_TemplateOpener)) { return true; } + if (Left.is(tok::identifier) && Right.is(tok::numeric_constant) && + Right.TokenText[0] == '.') { + return false; + } } else if (Style.isProto()) { if (Right.is(tok::period) && Left.isOneOf(Keywords.kw_optional, Keywords.kw_required, diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index d1e977dfa66af5..33dec7dae319f0 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -12075,6 +12075,7 @@ TEST_F(FormatTest, UnderstandsSquareAttributes) { verifyFormat("SomeType s [[gnu::unused]] (InitValue);"); verifyFormat("SomeType s [[using gnu: unused]] (InitValue);"); verifyFormat("[[gsl::suppress(\"clang-tidy-check-name\")]] void f() {}"); + verifyFormat("[[suppress(type.5)]] int uninitialized_on_purpose;"); verifyFormat("void f() [[deprecated(\"so sorry\")]];"); verifyFormat("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n" " [[unused]] aaaaaaaaaaaaaaaaaaaaaaa(int i);"); From d9e3e11ae57612ec61f6fcab4afc27d8d0ff5841 Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Wed, 27 Mar 2024 21:23:37 -0700 Subject: [PATCH 51/54] [clang-format] Exit clang-format-diff only after all diffs are printed (#86776) See https://github.com/llvm/llvm-project/pull/70883#issuecomment-2020811077. --- clang/tools/clang-format/clang-format-diff.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/clang/tools/clang-format/clang-format-diff.py b/clang/tools/clang-format/clang-format-diff.py index 0a2c24743678d0..3a74b90e731578 100755 --- a/clang/tools/clang-format/clang-format-diff.py +++ b/clang/tools/clang-format/clang-format-diff.py @@ -138,6 +138,7 @@ def main(): ) # Reformat files containing changes in place. + has_diff = False for filename, lines in lines_by_file.items(): if args.i and args.verbose: print("Formatting {}".format(filename)) @@ -169,7 +170,7 @@ def main(): stdout, stderr = p.communicate() if p.returncode != 0: - sys.exit(p.returncode) + return p.returncode if not args.i: with open(filename) as f: @@ -185,9 +186,12 @@ def main(): ) diff_string = "".join(diff) if len(diff_string) > 0: + has_diff = True sys.stdout.write(diff_string) - sys.exit(1) + + if has_diff: + return 1 if __name__ == "__main__": - main() + sys.exit(main()) From 6b7ecc7979134c152ee5f8286f904bba18f41185 Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Thu, 28 Mar 2024 04:41:29 +0000 Subject: [PATCH 52/54] Revert "[WebAssembly] Remove threwValue comparison after __wasm_setjmp_test (#86633)" This reverts commit 52431fdb1ab8d29be078edd55250e06381e4b6b0. The PR assumed `__threwValue` couldn't be 0, but it could be when the thrown thing is not a longjmp but an exception, so that `if` check was actually necessary. --- .../WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp | 8 +++++--- llvm/test/CodeGen/WebAssembly/lower-em-ehsjlj.ll | 7 ++++--- llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll | 4 +++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp index 0788d0c3a72136..027ee1086bf4e0 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp @@ -153,7 +153,7 @@ /// %__THREW__.val = __THREW__; /// __THREW__ = 0; /// %__threwValue.val = __threwValue; -/// if (%__THREW__.val != 0) { +/// if (%__THREW__.val != 0 & %__threwValue.val != 0) { /// %label = __wasm_setjmp_test(%__THREW__.val, functionInvocationId); /// if (%label == 0) /// emscripten_longjmp(%__THREW__.val, %__threwValue.val); @@ -712,10 +712,12 @@ void WebAssemblyLowerEmscriptenEHSjLj::wrapTestSetjmp( BasicBlock *ThenBB1 = BasicBlock::Create(C, "if.then1", F); BasicBlock *ElseBB1 = BasicBlock::Create(C, "if.else1", F); BasicBlock *EndBB1 = BasicBlock::Create(C, "if.end", F); + Value *ThrewCmp = IRB.CreateICmpNE(Threw, getAddrSizeInt(M, 0)); Value *ThrewValue = IRB.CreateLoad(IRB.getInt32Ty(), ThrewValueGV, ThrewValueGV->getName() + ".val"); - Value *ThrewCmp = IRB.CreateICmpNE(Threw, getAddrSizeInt(M, 0)); - IRB.CreateCondBr(ThrewCmp, ThenBB1, ElseBB1); + Value *ThrewValueCmp = IRB.CreateICmpNE(ThrewValue, IRB.getInt32(0)); + Value *Cmp1 = IRB.CreateAnd(ThrewCmp, ThrewValueCmp, "cmp1"); + IRB.CreateCondBr(Cmp1, ThenBB1, ElseBB1); // Generate call.em.longjmp BB once and share it within the function if (!CallEmLongjmpBB) { diff --git a/llvm/test/CodeGen/WebAssembly/lower-em-ehsjlj.ll b/llvm/test/CodeGen/WebAssembly/lower-em-ehsjlj.ll index d88f42a4dc5847..32942cd92e684f 100644 --- a/llvm/test/CodeGen/WebAssembly/lower-em-ehsjlj.ll +++ b/llvm/test/CodeGen/WebAssembly/lower-em-ehsjlj.ll @@ -22,8 +22,10 @@ entry: to label %try.cont unwind label %lpad ; CHECK: entry.split.split: -; CHECK: %__threwValue.val = load i32, ptr @__threwValue -; CHECK-NEXT: %[[CMP:.*]] = icmp ne i32 %__THREW__.val, 0 +; CHECK: %[[CMP0:.*]] = icmp ne i32 %__THREW__.val, 0 +; CHECK-NEXT: %__threwValue.val = load i32, ptr @__threwValue +; CHECK-NEXT: %[[CMP1:.*]] = icmp ne i32 %__threwValue.val, 0 +; CHECK-NEXT: %[[CMP:.*]] = and i1 %[[CMP0]], %[[CMP1]] ; CHECK-NEXT: br i1 %[[CMP]], label %if.then1, label %if.else1 ; This is exception checking part. %if.else1 leads here @@ -119,7 +121,6 @@ if.end: ; preds = %entry ; CHECK-NEXT: unreachable ; CHECK: normal: -; CHECK-NEXT: %__threwValue.val = load i32, ptr @__threwValue, align 4 ; CHECK-NEXT: icmp ne i32 %__THREW__.val, 0 return: ; preds = %if.end, %entry diff --git a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll index dca4c59d7c8740..27ec95a2c462ab 100644 --- a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll +++ b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll @@ -37,8 +37,10 @@ entry: ; CHECK-NEXT: call cc{{.*}} void @__invoke_void_[[PTR]]_i32(ptr @emscripten_longjmp, [[PTR]] %[[JMPBUF]], i32 1) ; CHECK-NEXT: %[[__THREW__VAL:.*]] = load [[PTR]], ptr @__THREW__ ; CHECK-NEXT: store [[PTR]] 0, ptr @__THREW__ +; CHECK-NEXT: %[[CMP0:.*]] = icmp ne [[PTR]] %__THREW__.val, 0 ; CHECK-NEXT: %[[THREWVALUE_VAL:.*]] = load i32, ptr @__threwValue -; CHECK-NEXT: %[[CMP:.*]] = icmp ne [[PTR]] %__THREW__.val, 0 +; CHECK-NEXT: %[[CMP1:.*]] = icmp ne i32 %[[THREWVALUE_VAL]], 0 +; CHECK-NEXT: %[[CMP:.*]] = and i1 %[[CMP0]], %[[CMP1]] ; CHECK-NEXT: br i1 %[[CMP]], label %if.then1, label %if.else1 ; CHECK: entry.split.split.split: From a41bfea5c049737e9c51e8d9a5769f72fbc55f59 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 27 Mar 2024 22:10:11 -0700 Subject: [PATCH 53/54] [MC] Simplify ELFObjectWriter. NFC And fix `if (hasRelocationAddend())` to `usesRela` to properly treat SHT_LLVM_CALL_GRAPH_PROFILE as SHT_REL. The incorrect does not cause a problem because the synthesized SHT_LLVM_CALL_GRAPH_PROFILE has zero addends. --- llvm/lib/MC/ELFObjectWriter.cpp | 36 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp index f4c6cbc8dd4442..005521bad6e014 100644 --- a/llvm/lib/MC/ELFObjectWriter.cpp +++ b/llvm/lib/MC/ELFObjectWriter.cpp @@ -141,7 +141,6 @@ struct ELFWriter { // TargetObjectWriter wrappers. bool is64Bit() const; - bool usesRela(const MCSectionELF &Sec) const; uint64_t align(Align Alignment); @@ -260,6 +259,7 @@ class ELFObjectWriter : public MCObjectWriter { void recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout, const MCFragment *Fragment, const MCFixup &Fixup, MCValue Target, uint64_t &FixedValue) override; + bool usesRela(const MCSectionELF &Sec) const; void executePostLayoutBinding(MCAssembler &Asm, const MCAsmLayout &Layout) override; @@ -394,11 +394,6 @@ bool ELFWriter::is64Bit() const { return OWriter.TargetObjectWriter->is64Bit(); } -bool ELFWriter::usesRela(const MCSectionELF &Sec) const { - return OWriter.hasRelocationAddend() && - Sec.getType() != ELF::SHT_LLVM_CALL_GRAPH_PROFILE; -} - // Emit the ELF header. void ELFWriter::writeHeader(const MCAssembler &Asm) { // ELF Header @@ -825,24 +820,22 @@ MCSectionELF *ELFWriter::createRelocationSection(MCContext &Ctx, if (OWriter.Relocations[&Sec].empty()) return nullptr; - const StringRef SectionName = Sec.getName(); - bool Rela = usesRela(Sec); - std::string RelaSectionName = Rela ? ".rela" : ".rel"; - RelaSectionName += SectionName; + unsigned Flags = ELF::SHF_INFO_LINK; + if (Sec.getFlags() & ELF::SHF_GROUP) + Flags = ELF::SHF_GROUP; + const StringRef SectionName = Sec.getName(); + const bool Rela = OWriter.usesRela(Sec); unsigned EntrySize; if (Rela) EntrySize = is64Bit() ? sizeof(ELF::Elf64_Rela) : sizeof(ELF::Elf32_Rela); else EntrySize = is64Bit() ? sizeof(ELF::Elf64_Rel) : sizeof(ELF::Elf32_Rel); - unsigned Flags = ELF::SHF_INFO_LINK; - if (Sec.getFlags() & ELF::SHF_GROUP) - Flags = ELF::SHF_GROUP; - - MCSectionELF *RelaSection = Ctx.createELFRelSection( - RelaSectionName, Rela ? ELF::SHT_RELA : ELF::SHT_REL, Flags, EntrySize, - Sec.getGroup(), &Sec); + MCSectionELF *RelaSection = + Ctx.createELFRelSection(((Rela ? ".rela" : ".rel") + SectionName), + Rela ? ELF::SHT_RELA : ELF::SHT_REL, Flags, + EntrySize, Sec.getGroup(), &Sec); RelaSection->setAlignment(is64Bit() ? Align(8) : Align(4)); return RelaSection; } @@ -938,11 +931,11 @@ void ELFWriter::WriteSecHdrEntry(uint32_t Name, uint32_t Type, uint64_t Flags, void ELFWriter::writeRelocations(const MCAssembler &Asm, const MCSectionELF &Sec) { std::vector &Relocs = OWriter.Relocations[&Sec]; + const bool Rela = OWriter.usesRela(Sec); // Sort the relocation entries. MIPS needs this. OWriter.TargetObjectWriter->sortRelocs(Asm, Relocs); - const bool Rela = usesRela(Sec); if (OWriter.TargetObjectWriter->getEMachine() == ELF::EM_MIPS) { for (const ELFRelocationEntry &Entry : Relocs) { uint32_t Symidx = Entry.Symbol ? Entry.Symbol->getIndex() : 0; @@ -1499,7 +1492,7 @@ void ELFObjectWriter::recordRelocation(MCAssembler &Asm, FixedValue = !RelocateWithSymbol && SymA && !SymA->isUndefined() ? C + Layout.getSymbolOffset(*SymA) : C; - if (hasRelocationAddend()) { + if (usesRela(FixupSection)) { Addend = FixedValue; FixedValue = 0; } @@ -1528,6 +1521,11 @@ void ELFObjectWriter::recordRelocation(MCAssembler &Asm, Relocations[&FixupSection].push_back(Rec); } +bool ELFObjectWriter::usesRela(const MCSectionELF &Sec) const { + return hasRelocationAddend() && + Sec.getType() != ELF::SHT_LLVM_CALL_GRAPH_PROFILE; +} + bool ELFObjectWriter::isSymbolRefDifferenceFullyResolvedImpl( const MCAssembler &Asm, const MCSymbol &SA, const MCFragment &FB, bool InSet, bool IsPCRel) const { From f8bab38b6dd02f2cd3acc28521d0ccb3186ef616 Mon Sep 17 00:00:00 2001 From: Lei Wang Date: Wed, 27 Mar 2024 22:27:22 -0700 Subject: [PATCH 54/54] [CSSPGO] Fix the issue of missing callee profile matches (#85715) Two fixes related to the callee/inlinee profile: 1. Fix the bug that the matching results are missing to distribute to the callee profiles (should be pass-by-reference). 2. Narrow imported function matching to checksum mismatched functions. More context: before we run matchings for all imported functions even checksums are matched, however, after we fix 1), we got a regression, it's likely due to the matching is not no-op for checksum matched function, so we want to make it consistent to only run matching for checksum mismatched (imported)functions. Since the metadata(pseudo_probe_desc) are dropped for imported function, we leverage the function attribute mechanism and add a new function attribute(`profile-checksum-mismatch`) to transfer the info from pre-link to post-link. --- .../Utils/SampleProfileLoaderBaseImpl.h | 25 +++---- llvm/lib/Transforms/IPO/SampleProfile.cpp | 46 +++++++++---- .../pseudo-probe-callee-profile-mismatch.prof | 16 +++++ .../csspgo-profile-checksum-mismatch-attr.ll | 67 +++++++++++++++++++ .../pseudo-probe-callee-profile-mismatch.ll | 63 +++++++++++++++++ ...pseudo-probe-stale-profile-matching-lto.ll | 2 +- .../pseudo-probe-stale-profile-matching.ll | 2 + 7 files changed, 194 insertions(+), 27 deletions(-) create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-callee-profile-mismatch.prof create mode 100644 llvm/test/Transforms/SampleProfile/csspgo-profile-checksum-mismatch-attr.ll create mode 100644 llvm/test/Transforms/SampleProfile/pseudo-probe-callee-profile-mismatch.ll diff --git a/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h b/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h index 66814d39527301..bd7496a799c579 100644 --- a/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h +++ b/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h @@ -86,9 +86,12 @@ template <> struct IRTraits { // SampleProfileProber. class PseudoProbeManager { DenseMap GUIDToProbeDescMap; + const ThinOrFullLTOPhase LTOPhase; public: - PseudoProbeManager(const Module &M) { + PseudoProbeManager(const Module &M, + ThinOrFullLTOPhase LTOPhase = ThinOrFullLTOPhase::None) + : LTOPhase(LTOPhase) { if (NamedMDNode *FuncInfo = M.getNamedMetadata(PseudoProbeDescMetadataName)) { for (const auto *Operand : FuncInfo->operands()) { @@ -126,17 +129,15 @@ class PseudoProbeManager { bool profileIsValid(const Function &F, const FunctionSamples &Samples) const { const auto *Desc = getDesc(F); - if (!Desc) { - LLVM_DEBUG(dbgs() << "Probe descriptor missing for Function " - << F.getName() << "\n"); - return false; - } - if (Desc->getFunctionHash() != Samples.getFunctionHash()) { - LLVM_DEBUG(dbgs() << "Hash mismatch for Function " << F.getName() - << "\n"); - return false; - } - return true; + assert((LTOPhase != ThinOrFullLTOPhase::ThinLTOPostLink || !Desc || + profileIsHashMismatched(*Desc, Samples) == + F.hasFnAttribute("profile-checksum-mismatch")) && + "In post-link, profile checksum matching state doesn't match " + "function 'profile-checksum-mismatch' attribute."); + // The desc for import function is unavailable. Check the function attribute + // for mismatch. + return (!Desc && !F.hasFnAttribute("profile-checksum-mismatch")) || + (Desc && !profileIsHashMismatched(*Desc, Samples)); } }; diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp index 2cbef8a7ae611d..7545a92c114ef2 100644 --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -453,6 +453,7 @@ class SampleProfileMatcher { Module &M; SampleProfileReader &Reader; const PseudoProbeManager *ProbeManager; + const ThinOrFullLTOPhase LTOPhase; SampleProfileMap FlattenedProfiles; // For each function, the matcher generates a map, of which each entry is a // mapping from the source location of current build to the source location in @@ -504,8 +505,9 @@ class SampleProfileMatcher { public: SampleProfileMatcher(Module &M, SampleProfileReader &Reader, - const PseudoProbeManager *ProbeManager) - : M(M), Reader(Reader), ProbeManager(ProbeManager){}; + const PseudoProbeManager *ProbeManager, + ThinOrFullLTOPhase LTOPhase) + : M(M), Reader(Reader), ProbeManager(ProbeManager), LTOPhase(LTOPhase){}; void runOnModule(); void clearMatchingData() { // Do not clear FuncMappings, it stores IRLoc to ProfLoc remappings which @@ -521,7 +523,7 @@ class SampleProfileMatcher { return &It->second; return nullptr; } - void runOnFunction(const Function &F); + void runOnFunction(Function &F); void findIRAnchors(const Function &F, std::map &IRAnchors); void findProfileAnchors( @@ -1911,15 +1913,22 @@ bool SampleProfileLoader::emitAnnotations(Function &F) { bool Changed = false; if (FunctionSamples::ProfileIsProbeBased) { - if (!ProbeManager->profileIsValid(F, *Samples)) { + LLVM_DEBUG({ + if (!ProbeManager->getDesc(F)) + dbgs() << "Probe descriptor missing for Function " << F.getName() + << "\n"; + }); + + if (ProbeManager->profileIsValid(F, *Samples)) { + ++NumMatchedProfile; + } else { + ++NumMismatchedProfile; LLVM_DEBUG( dbgs() << "Profile is invalid due to CFG mismatch for Function " << F.getName() << "\n"); - ++NumMismatchedProfile; if (!SalvageStaleProfile) return false; } - ++NumMatchedProfile; } else { if (getFunctionLoc(F) == 0) return false; @@ -2185,7 +2194,7 @@ bool SampleProfileLoader::doInitialization(Module &M, // Load pseudo probe descriptors for probe-based function samples. if (Reader->profileIsProbeBased()) { - ProbeManager = std::make_unique(M); + ProbeManager = std::make_unique(M, LTOPhase); if (!ProbeManager->moduleIsProbed(M)) { const char *Msg = "Pseudo-probe-based profile requires SampleProfileProbePass"; @@ -2197,8 +2206,8 @@ bool SampleProfileLoader::doInitialization(Module &M, if (ReportProfileStaleness || PersistProfileStaleness || SalvageStaleProfile) { - MatchingManager = - std::make_unique(M, *Reader, ProbeManager.get()); + MatchingManager = std::make_unique( + M, *Reader, ProbeManager.get(), LTOPhase); } return true; @@ -2452,7 +2461,7 @@ void SampleProfileMatcher::runStaleProfileMatching( } } -void SampleProfileMatcher::runOnFunction(const Function &F) { +void SampleProfileMatcher::runOnFunction(Function &F) { // We need to use flattened function samples for matching. // Unlike IR, which includes all callsites from the source code, the callsites // in profile only show up when they are hit by samples, i,e. the profile @@ -2481,8 +2490,16 @@ void SampleProfileMatcher::runOnFunction(const Function &F) { // support for pseudo-probe. if (SalvageStaleProfile && FunctionSamples::ProfileIsProbeBased && !ProbeManager->profileIsValid(F, *FSFlattened)) { - // The matching result will be saved to IRToProfileLocationMap, create a new - // map for each function. + // For imported functions, the checksum metadata(pseudo_probe_desc) are + // dropped, so we leverage function attribute(profile-checksum-mismatch) to + // transfer the info: add the attribute during pre-link phase and check it + // during post-link phase(see "profileIsValid"). + if (FunctionSamples::ProfileIsProbeBased && + LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) + F.addFnAttr("profile-checksum-mismatch"); + + // The matching result will be saved to IRToProfileLocationMap, create a + // new map for each function. auto &IRToProfileLocationMap = getIRToProfileLocationMap(F); runStaleProfileMatching(F, IRAnchors, ProfileAnchors, IRToProfileLocationMap); @@ -2758,8 +2775,9 @@ void SampleProfileMatcher::distributeIRToProfileLocationMap( FS.setIRToProfileLocationMap(&(ProfileMappings->second)); } - for (auto &Inlinees : FS.getCallsiteSamples()) { - for (auto FS : Inlinees.second) { + for (auto &Callees : + const_cast(FS.getCallsiteSamples())) { + for (auto &FS : Callees.second) { distributeIRToProfileLocationMap(FS.second); } } diff --git a/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-callee-profile-mismatch.prof b/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-callee-profile-mismatch.prof new file mode 100644 index 00000000000000..76a8fc9d19a85d --- /dev/null +++ b/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-callee-profile-mismatch.prof @@ -0,0 +1,16 @@ +main:252:0 + 1: 0 + 2: 50 + 5: 50 + 7: bar:102 + 1: 51 + 2: baz:51 + 1: 51 + !CFGChecksum: 4294967295 + !Attributes: 3 + !CFGChecksum: 281479271677951 + !Attributes: 2 + !CFGChecksum: 281582081721716 +bar:1:1 + 1: 1 + !CFGChecksum: 281479271677951 diff --git a/llvm/test/Transforms/SampleProfile/csspgo-profile-checksum-mismatch-attr.ll b/llvm/test/Transforms/SampleProfile/csspgo-profile-checksum-mismatch-attr.ll new file mode 100644 index 00000000000000..df56b55dcdf3c0 --- /dev/null +++ b/llvm/test/Transforms/SampleProfile/csspgo-profile-checksum-mismatch-attr.ll @@ -0,0 +1,67 @@ +; REQUIRES: x86_64-linux +; REQUIRES: asserts +; RUN: opt < %s -passes='thinlto-pre-link' -pgo-kind=pgo-sample-use-pipeline -sample-profile-file=%S/Inputs/pseudo-probe-callee-profile-mismatch.prof -pass-remarks=inline -S -o %t 2>&1 | FileCheck %s --check-prefix=INLINE +; RUN: FileCheck %s < %t +; RUN: FileCheck %s < %t --check-prefix=MERGE + + +; Make sure bar is inlined into main for attr merging verification. +; INLINE: 'bar' inlined into 'main' + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define i32 @baz() #0 { +entry: + ret i32 0 +} + +define i32 @bar() #0 !dbg !11 { +; CHECK: define {{.*}} @bar() {{.*}} #[[#BAR_ATTR:]] ! +entry: + %call = call i32 @baz() + ret i32 0 +} + +define i32 @main() #0 { +; MERGE: define {{.*}} @main() {{.*}} #[[#MAIN_ATTR:]] ! +entry: + br label %for.cond + +for.cond: ; preds = %for.cond, %entry + %call = call i32 @bar(), !dbg !14 + br label %for.cond +} + +; CHECK: attributes #[[#BAR_ATTR]] = {{{.*}} "profile-checksum-mismatch" {{.*}}} + +; Verify the attribute is not merged into the caller. +; MERGE-NOT: attributes #[[#MAIN_ATTR]] = {{{.*}} "profile-checksum-mismatch" {{.*}}} + +attributes #0 = { "use-sample-profile" } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!7} +!llvm.pseudo_probe_desc = !{!8, !9, !10} + +!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 19.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !2, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "test.c", directory: "/home", checksumkind: CSK_MD5, checksum: "0df0c950a93a603a7d13f0a9d4623642") +!2 = !{!3} +!3 = !DIGlobalVariableExpression(var: !4, expr: !DIExpression()) +!4 = distinct !DIGlobalVariable(name: "x", scope: !0, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true) +!5 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !6) +!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!7 = !{i32 2, !"Debug Info Version", i32 3} +!8 = !{i64 7546896869197086323, i64 4294967295, !"baz"} +!9 = !{i64 -2012135647395072713, i64 281530612780802, !"bar"} +!10 = !{i64 -2624081020897602054, i64 281582081721716, !"main"} +!11 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 5, type: !12, scopeLine: 5, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !13) +!12 = distinct !DISubroutineType(types: !13) +!13 = !{} +!14 = !DILocation(line: 15, column: 10, scope: !15) +!15 = !DILexicalBlockFile(scope: !16, file: !1, discriminator: 186646591) +!16 = distinct !DILexicalBlock(scope: !17, file: !1, line: 14, column: 40) +!17 = distinct !DILexicalBlock(scope: !18, file: !1, line: 14, column: 3) +!18 = distinct !DILexicalBlock(scope: !19, file: !1, line: 14, column: 3) +!19 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 12, type: !20, scopeLine: 13, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !13) +!20 = !DISubroutineType(types: !13) diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-callee-profile-mismatch.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-callee-profile-mismatch.ll new file mode 100644 index 00000000000000..e00b737cae4e85 --- /dev/null +++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-callee-profile-mismatch.ll @@ -0,0 +1,63 @@ +; REQUIRES: x86_64-linux +; REQUIRES: asserts +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-callee-profile-mismatch.prof --salvage-stale-profile -S --debug-only=sample-profile,sample-profile-impl -pass-remarks=inline 2>&1 | FileCheck %s + + +; CHECK: Run stale profile matching for bar +; CHECK: Callsite with callee:baz is matched from 4 to 2 +; CHECK: 'baz' inlined into 'main' to match profiling context with (cost=always): preinliner at callsite bar:3:8.4 @ main:3:10.7 + +; CHECK: Probe descriptor missing for Function bar +; CHECK: Profile is invalid due to CFG mismatch for Function bar + + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define i32 @main() #0 { + %1 = call i32 @bar(), !dbg !13 + ret i32 0 +} + +define available_externally i32 @bar() #1 !dbg !21 { + %1 = call i32 @baz(), !dbg !23 + ret i32 0 +} + +define available_externally i32 @baz() #0 !dbg !25 { + ret i32 0 +} + +attributes #0 = { "use-sample-profile" } +attributes #1 = { "profile-checksum-mismatch" "use-sample-profile" } + +!llvm.dbg.cu = !{!0, !7, !9} +!llvm.module.flags = !{!11} +!llvm.pseudo_probe_desc = !{!12} + +!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 19.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !2, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "test.c", directory: "/home/test", checksumkind: CSK_MD5, checksum: "7220f1a2d70ff869f1a6ab7958e3c393") +!2 = !{!3} +!3 = !DIGlobalVariableExpression(var: !4, expr: !DIExpression()) +!4 = distinct !DIGlobalVariable(name: "x", scope: !0, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true) +!5 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !6) +!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!7 = distinct !DICompileUnit(language: DW_LANG_C11, file: !8, producer: "clang version 19.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) +!8 = !DIFile(filename: "test1.v1.c", directory: "/home/test", checksumkind: CSK_MD5, checksum: "76696bd6bfe16a9f227fe03cfdb6a82c") +!9 = distinct !DICompileUnit(language: DW_LANG_C11, file: !10, producer: "clang version 19.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) +!10 = !DIFile(filename: "test2.c", directory: "/home/test", checksumkind: CSK_MD5, checksum: "553093afc026f9c73562eb3b0c5b7532") +!11 = !{i32 2, !"Debug Info Version", i32 3} +!12 = !{i64 -2624081020897602054, i64 281582081721716, !"main"} +!13 = !DILocation(line: 8, column: 10, scope: !14) +!14 = !DILexicalBlockFile(scope: !15, file: !1, discriminator: 186646591) +!15 = distinct !DILexicalBlock(scope: !16, file: !1, line: 7, column: 40) +!16 = distinct !DILexicalBlock(scope: !17, file: !1, line: 7, column: 3) +!17 = distinct !DILexicalBlock(scope: !18, file: !1, line: 7, column: 3) +!18 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 5, type: !19, scopeLine: 6, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !20) +!19 = distinct !DISubroutineType(types: !20) +!20 = !{} +!21 = distinct !DISubprogram(name: "bar", scope: !8, file: !8, line: 3, type: !22, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !7, retainedNodes: !20) +!22 = !DISubroutineType(types: !20) +!23 = !DILocation(line: 6, column: 8, scope: !24) +!24 = !DILexicalBlockFile(scope: !21, file: !8, discriminator: 186646567) +!25 = distinct !DISubprogram(name: "baz", scope: !10, file: !10, line: 1, type: !22, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !9, retainedNodes: !20) diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-lto.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-lto.ll index 55225b415d4abc..270beee4ebc2bd 100644 --- a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-lto.ll +++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-lto.ll @@ -106,7 +106,7 @@ define available_externally dso_local i32 @bar(i32 noundef %0) local_unnamed_add ret i32 %2, !dbg !132 } -attributes #0 = { nounwind uwtable "disable-tail-calls"="true" "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" "use-sample-profile" } +attributes #0 = { nounwind uwtable "disable-tail-calls"="true" "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" "use-sample-profile" "profile-checksum-mismatch"} attributes #1 = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite) } attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } attributes #3 = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) uwtable "disable-tail-calls"="true" "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" "use-sample-profile" } diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll index 89477ea5fecf1e..29877fb22a2c2e 100644 --- a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll +++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll @@ -48,6 +48,8 @@ ; } ; } +; Verify not running profile matching for checksum matched function. +; CHECK-NOT: Run stale profile matching for bar ; CHECK: Run stale profile matching for main