Merge "merge main into amd-staging" into amd-staging

clang/lib/Analysis/FlowSensitive/ControlFlowContext.cpp

@@ -39,8 +39,35 @@ buildStmtToBasicBlockMap(const CFG &Cfg) {
 
  StmtToBlock[Stmt->getStmt()] = Block;
  }
- if (const Stmt *TerminatorStmt = Block->getTerminatorStmt())
- StmtToBlock[TerminatorStmt] = Block;
+ }
+ // Some terminator conditions don't appear as a `CFGElement` anywhere else -
+ // for example, this is true if the terminator condition is a `&&` or `||`
+ // operator.
+ // We associate these conditions with the block the terminator appears in,
+ // but only if the condition has not already appeared as a regular
+ // `CFGElement`. (The `insert()` below does nothing if the key already exists
+ // in the map.)
+ for (const CFGBlock *Block : Cfg) {
+ if (Block != nullptr)
+ if (const Stmt *TerminatorCond = Block->getTerminatorCondition())
+ StmtToBlock.insert({TerminatorCond, Block});
+ }
+ // Terminator statements typically don't appear as a `CFGElement` anywhere
+ // else, so we want to associate them with the block that they terminate.
+ // However, there are some important special cases:
+ // - The conditional operator is a type of terminator, but it also appears
+ // as a regular `CFGElement`, and we want to associate it with the block
+ // in which it appears as a `CFGElement`.
+ // - The `&&` and `||` operators are types of terminators, but like the
+ // conditional operator, they can appear as a regular `CFGElement` or
+ // as a terminator condition (see above).
+ // We process terminators last to make sure that we only associate them with
+ // the block they terminate if they haven't previously occurred as a regular
+ // `CFGElement` or as a terminator condition.
+ for (const CFGBlock *Block : Cfg) {
+ if (Block != nullptr)
+ if (const Stmt *TerminatorStmt = Block->getTerminatorStmt())
+ StmtToBlock.insert({TerminatorStmt, Block});
  }
  return StmtToBlock;
 }

clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp

@@ -77,17 +77,33 @@ class DataflowAnalysisTest : public Test {
  return runDataflowAnalysis(*CFCtx, Analysis, Env);
  }
 
+ /// Returns the `CFGBlock` containing `S` (and asserts that it exists).
+ const CFGBlock *blockForStmt(const Stmt &S) {
+ const CFGBlock *Block = CFCtx->getStmtToBlock().lookup(&S);
+ assert(Block != nullptr);
+ return Block;
+ }
+
  template <typename StateT>
  const StateT &
  blockStateForStmt(const std::vector<std::optional<StateT>> &BlockStates,
- const Stmt *S) {
- const CFGBlock *Block = CFCtx->getStmtToBlock().lookup(S);
- assert(Block != nullptr);
- const std::optional<StateT> &MaybeState = BlockStates[Block->getBlockID()];
+ const Stmt &S) {
+ const std::optional<StateT> &MaybeState =
+ BlockStates[blockForStmt(S)->getBlockID()];
  assert(MaybeState.has_value());
  return *MaybeState;
  }
 
+ /// Returns the first node that matches `Matcher` (and asserts that the match
+ /// was successful, i.e. the returned node is not null).
+ template <typename NodeT, typename MatcherT>
+ const NodeT &matchNode(MatcherT Matcher) {
+ const auto *Node = selectFirst<NodeT>(
+ "node", match(Matcher.bind("node"), AST->getASTContext()));
+ assert(Node != nullptr);
+ return *Node;
+ }
+
  std::unique_ptr<ASTUnit> AST;
  std::unique_ptr<ControlFlowContext> CFCtx;
  std::unique_ptr<DataflowAnalysisContext> DACtx;
@@ -130,6 +146,79 @@ TEST_F(DataflowAnalysisTest, DiagnoseFunctionDiagnoserCalledOnEachElement) {
  " (Lifetime ends)\n")));
 }
 
+// Tests for the statement-to-block map.
+using StmtToBlockTest = DataflowAnalysisTest;
+
+TEST_F(StmtToBlockTest, ConditionalOperator) {
+ std::string Code = R"(
+ void target(bool b) {
+ int i = b ? 1 : 0;
+ }
+ )";
+ ASSERT_THAT_ERROR(runAnalysis<NoopAnalysis>(
+ Code, [](ASTContext &C) { return NoopAnalysis(C); })
+ .takeError(),
+ llvm::Succeeded());
+
+ const auto &IDecl = matchNode<DeclStmt>(declStmt(has(varDecl(hasName("i")))));
+ const auto &ConditionalOp =
+ matchNode<ConditionalOperator>(conditionalOperator());
+
+ // The conditional operator should be associated with the same block as the
+ // `DeclStmt` for `i`. (Specifically, the conditional operator should not be
+ // associated with the block for which it is the terminator.)
+ EXPECT_EQ(blockForStmt(IDecl), blockForStmt(ConditionalOp));
+}
+
+TEST_F(StmtToBlockTest, LogicalAnd) {
+ std::string Code = R"(
+ void target(bool b1, bool b2) {
+ bool b = b1 && b2;
+ }
+ )";
+ ASSERT_THAT_ERROR(runAnalysis<NoopAnalysis>(
+ Code, [](ASTContext &C) { return NoopAnalysis(C); })
+ .takeError(),
+ llvm::Succeeded());
+
+ const auto &BDecl = matchNode<DeclStmt>(declStmt(has(varDecl(hasName("b")))));
+ const auto &AndOp =
+ matchNode<BinaryOperator>(binaryOperator(hasOperatorName("&&")));
+
+ // The `&&` operator should be associated with the same block as the
+ // `DeclStmt` for `b`. (Specifically, the `&&` operator should not be
+ // associated with the block for which it is the terminator.)
+ EXPECT_EQ(blockForStmt(BDecl), blockForStmt(AndOp));
+}
+
+TEST_F(StmtToBlockTest, IfStatementWithLogicalAnd) {
+ std::string Code = R"(
+ void target(bool b1, bool b2) {
+ if (b1 && b2)
+ ;
+ }
+ )";
+ ASSERT_THAT_ERROR(runAnalysis<NoopAnalysis>(
+ Code, [](ASTContext &C) { return NoopAnalysis(C); })
+ .takeError(),
+ llvm::Succeeded());
+
+ const auto &If = matchNode<IfStmt>(ifStmt());
+ const auto &B2 =
+ matchNode<DeclRefExpr>(declRefExpr(to(varDecl(hasName("b2")))));
+ const auto &AndOp =
+ matchNode<BinaryOperator>(binaryOperator(hasOperatorName("&&")));
+
+ // The if statement is the terminator for the block that contains both `b2`
+ // and the `&&` operator (which appears only as a terminator condition, not
+ // as a regular `CFGElement`).
+ const CFGBlock *IfBlock = blockForStmt(If);
+ const CFGBlock *B2Block = blockForStmt(B2);
+ const CFGBlock *AndOpBlock = blockForStmt(AndOp);
+ EXPECT_EQ(IfBlock, B2Block);
+ EXPECT_EQ(IfBlock, AndOpBlock);
+}
+
 // Tests that check we discard state for expressions correctly.
 using DiscardExprStateTest = DataflowAnalysisTest;
 
@@ -144,25 +233,20 @@ TEST_F(DiscardExprStateTest, WhileStatement) {
  auto BlockStates = llvm::cantFail(runAnalysis<NoopAnalysis>(
  Code, [](ASTContext &C) { return NoopAnalysis(C); }));
 
- auto *NotEqOp = selectFirst<BinaryOperator>(
- "op", match(binaryOperator(hasOperatorName("!=")).bind("op"),
- AST->getASTContext()));
- ASSERT_NE(NotEqOp, nullptr);
-
- auto *CallFoo = selectFirst<CallExpr>(
- "call", match(callExpr(callee(functionDecl(hasName("foo")))).bind("call"),
- AST->getASTContext()));
- ASSERT_NE(CallFoo, nullptr);
+ const auto &NotEqOp =
+ matchNode<BinaryOperator>(binaryOperator(hasOperatorName("!=")));
+ const auto &CallFoo =
+ matchNode<CallExpr>(callExpr(callee(functionDecl(hasName("foo")))));
 
  // In the block that evaluates the expression `p != nullptr`, this expression
  // is associated with a value.
  const auto &NotEqOpState = blockStateForStmt(BlockStates, NotEqOp);
- EXPECT_NE(NotEqOpState.Env.getValue(*NotEqOp), nullptr);
+ EXPECT_NE(NotEqOpState.Env.getValue(NotEqOp), nullptr);
 
  // In the block that calls `foo(p)`, the value for `p != nullptr` is discarded
  // because it is not consumed by this block.
  const auto &CallFooState = blockStateForStmt(BlockStates, CallFoo);
- EXPECT_EQ(CallFooState.Env.getValue(*NotEqOp), nullptr);
+ EXPECT_EQ(CallFooState.Env.getValue(NotEqOp), nullptr);
 }
 
 TEST_F(DiscardExprStateTest, BooleanOperator) {
@@ -174,29 +258,24 @@ TEST_F(DiscardExprStateTest, BooleanOperator) {
  auto BlockStates = llvm::cantFail(runAnalysis<NoopAnalysis>(
  Code, [](ASTContext &C) { return NoopAnalysis(C); }));
 
- auto *AndOp = selectFirst<BinaryOperator>(
- "op", match(binaryOperator(hasOperatorName("&&")).bind("op"),
- AST->getASTContext()));
- ASSERT_NE(AndOp, nullptr);
-
- auto *Return = selectFirst<ReturnStmt>(
- "return", match(returnStmt().bind("return"), AST->getASTContext()));
- ASSERT_NE(Return, nullptr);
+ const auto &AndOp =
+ matchNode<BinaryOperator>(binaryOperator(hasOperatorName("&&")));
+ const auto &Return = matchNode<ReturnStmt>(returnStmt());
 
  // In the block that evaluates the LHS of the `&&` operator, the LHS is
  // associated with a value, while the right-hand side is not (unsurprisingly,
  // as it hasn't been evaluated yet).
- const auto &LHSState = blockStateForStmt(BlockStates, AndOp->getLHS());
- auto *LHSValue = cast<BoolValue>(LHSState.Env.getValue(*AndOp->getLHS()));
+ const auto &LHSState = blockStateForStmt(BlockStates, *AndOp.getLHS());
+ auto *LHSValue = cast<BoolValue>(LHSState.Env.getValue(*AndOp.getLHS()));
  ASSERT_NE(LHSValue, nullptr);
- EXPECT_EQ(LHSState.Env.getValue(*AndOp->getRHS()), nullptr);
+ EXPECT_EQ(LHSState.Env.getValue(*AndOp.getRHS()), nullptr);
 
  // In the block that evaluates the RHS, the RHS is associated with a
  // value. The value for the LHS has been discarded as it is not consumed by
  // this block.
- const auto &RHSState = blockStateForStmt(BlockStates, AndOp->getRHS());
- EXPECT_EQ(RHSState.Env.getValue(*AndOp->getLHS()), nullptr);
- auto *RHSValue = cast<BoolValue>(RHSState.Env.getValue(*AndOp->getRHS()));
+ const auto &RHSState = blockStateForStmt(BlockStates, *AndOp.getRHS());
+ EXPECT_EQ(RHSState.Env.getValue(*AndOp.getLHS()), nullptr);
+ auto *RHSValue = cast<BoolValue>(RHSState.Env.getValue(*AndOp.getRHS()));
  ASSERT_NE(RHSValue, nullptr);
 
  // In the block that evaluates the return statement, the expression `b1 && b2`
@@ -217,9 +296,9 @@ TEST_F(DiscardExprStateTest, BooleanOperator) {
  // operands, rather than from the environment for the block that contains the
  // `&&`.
  const auto &ReturnState = blockStateForStmt(BlockStates, Return);
- EXPECT_EQ(ReturnState.Env.getValue(*AndOp->getLHS()), nullptr);
- EXPECT_EQ(ReturnState.Env.getValue(*AndOp->getRHS()), nullptr);
- EXPECT_EQ(ReturnState.Env.getValue(*AndOp),
+ EXPECT_EQ(ReturnState.Env.getValue(*AndOp.getLHS()), nullptr);
+ EXPECT_EQ(ReturnState.Env.getValue(*AndOp.getRHS()), nullptr);
+ EXPECT_EQ(ReturnState.Env.getValue(AndOp),
  &ReturnState.Env.makeAnd(*LHSValue, *RHSValue));
 }
 

llvm/include/llvm/Config/llvm-config.h.cmake

@@ -17,7 +17,7 @@
 /* The number of commits in the linear history from the
  * start of the universe up to the latest llvm main commit
  * that has been merged */
-#define LLVM_MAIN_REVISION 1
+#define LLVM_MAIN_REVISION 490468
 
 /* Define if LLVM_ENABLE_DUMP is enabled */
 #cmakedefine LLVM_ENABLE_DUMP

llvm/lib/CodeGen/CodeGenPrepare.cpp

@@ -2686,8 +2686,9 @@ bool CodeGenPrepare::dupRetToEnableTailCallOpts(BasicBlock *BB,
  attributesPermitTailCall(F, CI, RetI, *TLI)) {
  // Either we return void or the return value must be the first
  // argument of a known intrinsic or library function.
- if (!V || (isIntrinsicOrLFToBeTailCalled(TLInfo, CI) &&
- V == CI->getArgOperand(0))) {
+ if (!V || isa<UndefValue>(V) ||
+ (isIntrinsicOrLFToBeTailCalled(TLInfo, CI) &&
+ V == CI->getArgOperand(0))) {
  TailCallBBs.push_back(Pred);
  }
  }

llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp

@@ -712,7 +712,7 @@ bool AArch64Arm64ECCallLowering::processFunction(
  // name (emitting the definition) can grab it from the metadata.
  //
  // FIXME: Handle functions with weak linkage?
- if (F.hasExternalLinkage() || F.hasWeakLinkage() || F.hasLinkOnceLinkage()) {
+ if (!F.hasLocalLinkage() || F.hasAddressTaken()) {
  if (std::optional<std::string> MangledName =
  getArm64ECMangledFunctionName(F.getName().str())) {
  F.setMetadata("arm64ec_unmangled_name",

llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp

@@ -1121,7 +1121,8 @@ void AArch64AsmPrinter::emitFunctionEntryLabel() {
  TS->emitDirectiveVariantPCS(CurrentFnSym);
  }
 
- if (TM.getTargetTriple().isWindowsArm64EC()) {
+ if (TM.getTargetTriple().isWindowsArm64EC() &&
+ !MF->getFunction().hasLocalLinkage()) {
  // For ARM64EC targets, a function definition's name is mangled differently
  // from the normal symbol. We emit the alias from the unmangled symbol to
  // mangled symbol name here.

llvm/lib/Target/AArch64/AArch64FrameLowering.cpp

@@ -3195,11 +3195,6 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
  return MIB->getIterator();
  };
 
- // SVE objects are always restored in reverse order.
- for (const RegPairInfo &RPI : reverse(RegPairs))
- if (RPI.isScalable())
- EmitMI(RPI);
-
  if (homogeneousPrologEpilog(MF, &MBB)) {
  auto MIB = BuildMI(MBB, MBBI, DL, TII.get(AArch64::HOM_Epilog))
  .setMIFlag(MachineInstr::FrameDestroy);
@@ -3210,11 +3205,19 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
  return true;
  }
 
+ // For performance reasons restore SVE register in increasing order
+ auto IsPPR = [](const RegPairInfo &c) { return c.Type == RegPairInfo::PPR; };
+ auto PPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsPPR);
+ auto PPREnd = std::find_if_not(PPRBegin, RegPairs.end(), IsPPR);
+ std::reverse(PPRBegin, PPREnd);
+ auto IsZPR = [](const RegPairInfo &c) { return c.Type == RegPairInfo::ZPR; };
+ auto ZPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsZPR);
+ auto ZPREnd = std::find_if_not(ZPRBegin, RegPairs.end(), IsZPR);
+ std::reverse(ZPRBegin, ZPREnd);
+
  if (ReverseCSRRestoreSeq) {
  MachineBasicBlock::iterator First = MBB.end();
  for (const RegPairInfo &RPI : reverse(RegPairs)) {
- if (RPI.isScalable())
- continue;
  MachineBasicBlock::iterator It = EmitMI(RPI);
  if (First == MBB.end())
  First = It;
@@ -3223,8 +3226,6 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
  MBB.splice(MBBI, &MBB, First);
  } else {
  for (const RegPairInfo &RPI : RegPairs) {
- if (RPI.isScalable())
- continue;
  (void)EmitMI(RPI);
  }
  }

llvm/lib/Target/AArch64/AArch64ISelLowering.h

@@ -1308,6 +1308,8 @@ class AArch64TargetLowering : public TargetLowering {
  bool preferScalarizeSplat(SDNode *N) const override;
 
  unsigned getMinimumJumpTableEntries() const override;
+
+ bool softPromoteHalfType() const override { return true; }
 };
 
 namespace AArch64 {

llvm/lib/Target/AArch64/AArch64SchedTSV110.td

@@ -419,10 +419,10 @@ def : InstRW<[TSV110Wr_12cyc_1MDU], (instregex "^(S|U)DIVWr$")>;
 def : InstRW<[TSV110Wr_20cyc_1MDU], (instregex "^(S|U)DIVXr$")>;
 
 def TSV110ReadMAW : SchedReadAdvance<2, [TSV110Wr_3cyc_1MDU]>;
-def : InstRW<[TSV110Wr_3cyc_1MDU, TSV110ReadMAW], (instrs MADDWrrr, MSUBWrrr)>;
+def : InstRW<[TSV110Wr_3cyc_1MDU, ReadIM, ReadIM, TSV110ReadMAW], (instrs MADDWrrr, MSUBWrrr)>;
 def TSV110ReadMAQ : SchedReadAdvance<3, [TSV110Wr_4cyc_1MDU]>;
-def : InstRW<[TSV110Wr_4cyc_1MDU, TSV110ReadMAQ], (instrs MADDXrrr, MSUBXrrr)>;
-def : InstRW<[TSV110Wr_3cyc_1MDU, TSV110ReadMAW], (instregex "(S|U)(MADDL|MSUBL)rrr")>;
+def : InstRW<[TSV110Wr_4cyc_1MDU, ReadIM, ReadIM, TSV110ReadMAQ], (instrs MADDXrrr, MSUBXrrr)>;
+def : InstRW<[TSV110Wr_3cyc_1MDU, ReadIM, ReadIM, TSV110ReadMAW], (instregex "(S|U)(MADDL|MSUBL)rrr")>;
 def : InstRW<[TSV110Wr_4cyc_1MDU], (instregex "^(S|U)MULHrr$")>;
 
 

llvm/lib/Target/AMDGPU/AMDGPUCombine.td

@@ -33,6 +33,12 @@ def rcp_sqrt_to_rsq : GICombineRule<
  [{ return matchRcpSqrtToRsq(*${rcp}, ${matchinfo}); }]),
  (apply [{ Helper.applyBuildFn(*${rcp}, ${matchinfo}); }])>;
 
+def fdiv_by_sqrt_to_rsq_f16 : GICombineRule<
+ (defs root:$root),
+ (match (G_FSQRT f16:$sqrt, $x, (MIFlags FmContract)),
+ (G_FDIV f16:$dst, $y, $sqrt, (MIFlags FmContract)):$root,
+ [{ return matchFDivSqrtToRsqF16(*${root}); }]),
+ (apply [{ applyFDivSqrtToRsqF16(*${root}, ${x}.getReg()); }])>;
 
 def cvt_f32_ubyteN_matchdata : GIDefMatchData<"CvtF32UByteMatchInfo">;
 
@@ -156,7 +162,7 @@ def AMDGPUPostLegalizerCombiner: GICombiner<
  "AMDGPUPostLegalizerCombinerImpl",
  [all_combines, gfx6gfx7_combines, gfx8_combines,
  uchar_to_float, cvt_f32_ubyteN, remove_fcanonicalize, foldable_fneg,
- rcp_sqrt_to_rsq, sign_extension_in_reg, smulu64]> {
+ rcp_sqrt_to_rsq, fdiv_by_sqrt_to_rsq_f16, sign_extension_in_reg, smulu64]> {
  let CombineAllMethodName = "tryCombineAllImpl";
 }