diff --git a/.github/workflows/commit-access-review.py b/.github/workflows/commit-access-review.py
index 97589c138c0b70..8ea9b1fcc2fb08 100644
--- a/.github/workflows/commit-access-review.py
+++ b/.github/workflows/commit-access-review.py
@@ -358,11 +358,10 @@ def main():
     gh = github.Github(login_or_token=token)
     org = gh.get_organization("llvm")
     repo = org.get_repo("llvm-project")
-    team = org.get_team_by_slug("llvm-committers")
     one_year_ago = datetime.datetime.now() - datetime.timedelta(days=365)
     triage_list = {}
-    for member in team.get_members():
-        triage_list[member.login] = User(member.login, triage_list)
+    for collaborator in repo.get_collaborators(permission="push"):
+        triage_list[collaborator.login] = User(collaborator.login, triage_list)
 
     print("Start:", len(triage_list), "triagers")
     # Step 0 Check if users have requested commit access in the last year.
diff --git a/bolt/test/lit.local.cfg b/bolt/test/lit.local.cfg
index 8aa5f15d5ccfb4..e2fa0a4a2210f6 100644
--- a/bolt/test/lit.local.cfg
+++ b/bolt/test/lit.local.cfg
@@ -1,6 +1,6 @@
 host_linux_triple = config.target_triple.split("-")[0] + "-unknown-linux-gnu"
-common_linker_flags = "-fuse-ld=lld -Wl,--unresolved-symbols=ignore-all"
-flags = f"--target={host_linux_triple} {common_linker_flags}"
+common_linker_flags = "-fuse-ld=lld -Wl,--unresolved-symbols=ignore-all -pie"
+flags = f"--target={host_linux_triple} -fPIE {common_linker_flags}"
 
 config.substitutions.insert(0, ("%cflags", f"%cflags {flags}"))
 config.substitutions.insert(0, ("%cxxflags", f"%cxxflags {flags}"))
diff --git a/bolt/test/perf2bolt/lit.local.cfg b/bolt/test/perf2bolt/lit.local.cfg
index 4ee9ad08cc78a0..0fecf913aa98b8 100644
--- a/bolt/test/perf2bolt/lit.local.cfg
+++ b/bolt/test/perf2bolt/lit.local.cfg
@@ -1,4 +1,5 @@
 import shutil
+import subprocess
 
-if shutil.which("perf") is not None:
-    config.available_features.add("perf")
\ No newline at end of file
+if shutil.which("perf") is not None and subprocess.run(["perf", "record", "-e", "cycles:u", "-o", "/dev/null", "--", "perf", "--version"], capture_output=True).returncode == 0:
+    config.available_features.add("perf")
diff --git a/clang-tools-extra/clang-tidy/bugprone/ForwardingReferenceOverloadCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/ForwardingReferenceOverloadCheck.cpp
index c87b3ea7e26163..00e8f7e514368b 100644
--- a/clang-tools-extra/clang-tidy/bugprone/ForwardingReferenceOverloadCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/ForwardingReferenceOverloadCheck.cpp
@@ -9,7 +9,6 @@
 #include "ForwardingReferenceOverloadCheck.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
-#include <algorithm>
 
 using namespace clang::ast_matchers;
 
@@ -19,14 +18,14 @@ namespace {
 // Check if the given type is related to std::enable_if.
 AST_MATCHER(QualType, isEnableIf) {
   auto CheckTemplate = [](const TemplateSpecializationType *Spec) {
-    if (!Spec || !Spec->getTemplateName().getAsTemplateDecl()) {
+    if (!Spec)
       return false;
-    }
-    const NamedDecl *TypeDecl =
-        Spec->getTemplateName().getAsTemplateDecl()->getTemplatedDecl();
-    return TypeDecl->isInStdNamespace() &&
-           (TypeDecl->getName() == "enable_if" ||
-            TypeDecl->getName() == "enable_if_t");
+
+    const TemplateDecl *TDecl = Spec->getTemplateName().getAsTemplateDecl();
+
+    return TDecl && TDecl->isInStdNamespace() &&
+           (TDecl->getName() == "enable_if" ||
+            TDecl->getName() == "enable_if_t");
   };
   const Type *BaseType = Node.getTypePtr();
   // Case: pointer or reference to enable_if.
diff --git a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp
index d517e8473d94ae..a30e63f9b0fd6a 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp
@@ -48,6 +48,8 @@ AST_MATCHER_P2(Expr, hasSizeOfDescendant, int, Depth,
   return false;
 }
 
+AST_MATCHER(Expr, offsetOfExpr) { return isa<OffsetOfExpr>(Node); }
+
 CharUnits getSizeOfType(const ASTContext &Ctx, const Type *Ty) {
   if (!Ty || Ty->isIncompleteType() || Ty->isDependentType() ||
       isa<DependentSizedArrayType>(Ty) || !Ty->isConstantSizeType())
@@ -221,17 +223,15 @@ void SizeofExpressionCheck::registerMatchers(MatchFinder *Finder) {
   const auto ElemType =
       arrayType(hasElementType(recordType().bind("elem-type")));
   const auto ElemPtrType = pointerType(pointee(type().bind("elem-ptr-type")));
+  const auto SizeofDivideExpr = binaryOperator(
+      hasOperatorName("/"),
+      hasLHS(
+          ignoringParenImpCasts(sizeOfExpr(hasArgumentOfType(hasCanonicalType(
+              type(anyOf(ElemType, ElemPtrType, type())).bind("num-type")))))),
+      hasRHS(ignoringParenImpCasts(sizeOfExpr(
+          hasArgumentOfType(hasCanonicalType(type().bind("denom-type")))))));
 
-  Finder->addMatcher(
-      binaryOperator(
-          hasOperatorName("/"),
-          hasLHS(ignoringParenImpCasts(sizeOfExpr(hasArgumentOfType(
-              hasCanonicalType(type(anyOf(ElemType, ElemPtrType, type()))
-                                   .bind("num-type")))))),
-          hasRHS(ignoringParenImpCasts(sizeOfExpr(
-              hasArgumentOfType(hasCanonicalType(type().bind("denom-type")))))))
-          .bind("sizeof-divide-expr"),
-      this);
+  Finder->addMatcher(SizeofDivideExpr.bind("sizeof-divide-expr"), this);
 
   // Detect expression like: sizeof(...) * sizeof(...)); most likely an error.
   Finder->addMatcher(binaryOperator(hasOperatorName("*"),
@@ -257,8 +257,9 @@ void SizeofExpressionCheck::registerMatchers(MatchFinder *Finder) {
                          .bind("sizeof-sizeof-expr"),
                      this);
 
-  // Detect sizeof in pointer arithmetic like: N * sizeof(S) == P1 - P2 or
-  // (P1 - P2) / sizeof(S) where P1 and P2 are pointers to type S.
+  // Detect sizeof usage in comparisons involving pointer arithmetics, such as
+  // N * sizeof(T) == P1 - P2 or (P1 - P2) / sizeof(T), where P1 and P2 are
+  // pointers to a type T.
   const auto PtrDiffExpr = binaryOperator(
       hasOperatorName("-"),
       hasLHS(hasType(hasUnqualifiedDesugaredType(pointerType(pointee(
@@ -285,6 +286,47 @@ void SizeofExpressionCheck::registerMatchers(MatchFinder *Finder) {
           hasRHS(ignoringParenImpCasts(SizeOfExpr.bind("sizeof-ptr-div-expr"))))
           .bind("sizeof-in-ptr-arithmetic-div"),
       this);
+
+  // SEI CERT ARR39-C. Do not add or subtract a scaled integer to a pointer.
+  // Detect sizeof, alignof and offsetof usage in pointer arithmetics where
+  // they are used to scale the numeric distance, which is scaled again by
+  // the pointer arithmetic operator. This can result in forming invalid
+  // offsets.
+  //
+  // Examples, where P is a pointer, N is some integer (both compile-time and
+  // run-time): P + sizeof(T), P + sizeof(*P), P + N * sizeof(*P).
+  //
+  // This check does not warn on cases where the pointee type is "1 byte",
+  // as those cases can often come from generics and also do not constitute a
+  // problem because the size does not affect the scale used.
+  const auto InterestingPtrTyForPtrArithmetic =
+      pointerType(pointee(qualType().bind("pointee-type")));
+  const auto SizeofLikeScaleExpr =
+      expr(anyOf(unaryExprOrTypeTraitExpr(ofKind(UETT_SizeOf)),
+                 unaryExprOrTypeTraitExpr(ofKind(UETT_AlignOf)),
+                 offsetOfExpr()))
+          .bind("sizeof-in-ptr-arithmetic-scale-expr");
+  const auto PtrArithmeticIntegerScaleExpr = binaryOperator(
+      hasAnyOperatorName("*", "/"),
+      // sizeof(...) * sizeof(...) and sizeof(...) / sizeof(...) is handled
+      // by this check on another path.
+      hasOperands(expr(hasType(isInteger()), unless(SizeofLikeScaleExpr)),
+                  SizeofLikeScaleExpr));
+  const auto PtrArithmeticScaledIntegerExpr =
+      expr(anyOf(SizeofLikeScaleExpr, PtrArithmeticIntegerScaleExpr),
+           unless(SizeofDivideExpr));
+
+  Finder->addMatcher(
+      expr(anyOf(
+          binaryOperator(hasAnyOperatorName("+", "-"),
+                         hasOperands(hasType(InterestingPtrTyForPtrArithmetic),
+                                     PtrArithmeticScaledIntegerExpr))
+              .bind("sizeof-in-ptr-arithmetic-plusminus"),
+          binaryOperator(hasAnyOperatorName("+=", "-="),
+                         hasLHS(hasType(InterestingPtrTyForPtrArithmetic)),
+                         hasRHS(PtrArithmeticScaledIntegerExpr))
+              .bind("sizeof-in-ptr-arithmetic-plusminus"))),
+      this);
 }
 
 void SizeofExpressionCheck::check(const MatchFinder::MatchResult &Result) {
@@ -409,6 +451,43 @@ void SizeofExpressionCheck::check(const MatchFinder::MatchResult &Result) {
           << SizeOfExpr->getSourceRange() << E->getOperatorLoc()
           << E->getLHS()->getSourceRange() << E->getRHS()->getSourceRange();
     }
+  } else if (const auto *E = Result.Nodes.getNodeAs<BinaryOperator>(
+                 "sizeof-in-ptr-arithmetic-plusminus")) {
+    const auto *PointeeTy = Result.Nodes.getNodeAs<QualType>("pointee-type");
+    const auto *ScaleExpr =
+        Result.Nodes.getNodeAs<Expr>("sizeof-in-ptr-arithmetic-scale-expr");
+    const CharUnits PointeeSize = getSizeOfType(Ctx, PointeeTy->getTypePtr());
+    const int ScaleKind = [ScaleExpr]() {
+      if (const auto *UTTE = dyn_cast<UnaryExprOrTypeTraitExpr>(ScaleExpr))
+        switch (UTTE->getKind()) {
+        case UETT_SizeOf:
+          return 0;
+        case UETT_AlignOf:
+          return 1;
+        default:
+          return -1;
+        }
+
+      if (isa<OffsetOfExpr>(ScaleExpr))
+        return 2;
+
+      return -1;
+    }();
+
+    if (ScaleKind != -1 && PointeeSize > CharUnits::One()) {
+      diag(E->getExprLoc(),
+           "suspicious usage of '%select{sizeof|alignof|offsetof}0(...)' in "
+           "pointer arithmetic; this scaled value will be scaled again by the "
+           "'%1' operator")
+          << ScaleKind << E->getOpcodeStr() << ScaleExpr->getSourceRange();
+      diag(E->getExprLoc(),
+           "'%0' in pointer arithmetic internally scales with 'sizeof(%1)' == "
+           "%2",
+           DiagnosticIDs::Note)
+          << E->getOpcodeStr()
+          << PointeeTy->getAsString(Ctx.getPrintingPolicy())
+          << PointeeSize.getQuantity();
+    }
   }
 }
 
diff --git a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.h b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.h
index 9ca17bc9e6f124..66d7c34cc9e940 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.h
+++ b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.h
@@ -13,7 +13,7 @@
 
 namespace clang::tidy::bugprone {
 
-/// Find suspicious usages of sizeof expression.
+/// Find suspicious usages of sizeof expressions.
 ///
 /// For the user-facing documentation see:
 /// http://clang.llvm.org/extra/clang-tidy/checks/bugprone/sizeof-expression.html
diff --git a/clang-tools-extra/clang-tidy/cert/CERTTidyModule.cpp b/clang-tools-extra/clang-tidy/cert/CERTTidyModule.cpp
index 8b5be9cd95f767..26befe0de59ae4 100644
--- a/clang-tools-extra/clang-tidy/cert/CERTTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/cert/CERTTidyModule.cpp
@@ -14,6 +14,7 @@
 #include "../bugprone/ReservedIdentifierCheck.h"
 #include "../bugprone/SignalHandlerCheck.h"
 #include "../bugprone/SignedCharMisuseCheck.h"
+#include "../bugprone/SizeofExpressionCheck.h"
 #include "../bugprone/SpuriouslyWakeUpFunctionsCheck.h"
 #include "../bugprone/SuspiciousMemoryComparisonCheck.h"
 #include "../bugprone/UnhandledSelfAssignmentCheck.h"
@@ -281,6 +282,9 @@ class CERTModule : public ClangTidyModule {
         "cert-oop58-cpp");
 
     // C checkers
+    // ARR
+    CheckFactories.registerCheck<bugprone::SizeofExpressionCheck>(
+        "cert-arr39-c");
     // CON
     CheckFactories.registerCheck<bugprone::SpuriouslyWakeUpFunctionsCheck>(
         "cert-con36-c");
@@ -332,6 +336,12 @@ class CERTModule : public ClangTidyModule {
   ClangTidyOptions getModuleOptions() override {
     ClangTidyOptions Options;
     ClangTidyOptions::OptionMap &Opts = Options.CheckOptions;
+    Opts["cert-arr39-c.WarnOnSizeOfConstant"] = "false";
+    Opts["cert-arr39-c.WarnOnSizeOfIntegerExpression"] = "false";
+    Opts["cert-arr39-c.WarnOnSizeOfThis"] = "false";
+    Opts["cert-arr39-c.WarnOnSizeOfCompareToConstant"] = "false";
+    Opts["cert-arr39-c.WarnOnSizeOfPointer"] = "false";
+    Opts["cert-arr39-c.WarnOnSizeOfPointerToAggregate"] = "false";
     Opts["cert-dcl16-c.NewSuffixes"] = "L;LL;LU;LLU";
     Opts["cert-err33-c.CheckedFunctions"] = CertErr33CCheckedFunctions;
     Opts["cert-err33-c.AllowCastToVoid"] = "true";
diff --git a/clang-tools-extra/clang-tidy/misc/DefinitionsInHeadersCheck.cpp b/clang-tools-extra/clang-tidy/misc/DefinitionsInHeadersCheck.cpp
index 21008bc144b91a..ee869256898989 100644
--- a/clang-tools-extra/clang-tidy/misc/DefinitionsInHeadersCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/DefinitionsInHeadersCheck.cpp
@@ -102,7 +102,7 @@ void DefinitionsInHeadersCheck::check(const MatchFinder::MatchResult &Result) {
     // inline is not allowed for main function.
     if (FD->isMain())
       return;
-    diag(FD->getLocation(), /*Description=*/"make as 'inline'",
+    diag(FD->getLocation(), "mark the definition as 'inline'",
          DiagnosticIDs::Note)
         << FixItHint::CreateInsertion(FD->getInnerLocStart(), "inline ");
   } else if (const auto *VD = dyn_cast<VarDecl>(ND)) {
diff --git a/clang-tools-extra/clang-tidy/modernize/AvoidCArraysCheck.cpp b/clang-tools-extra/clang-tidy/modernize/AvoidCArraysCheck.cpp
index 89790ea70cf229..98778192dbd3ae 100644
--- a/clang-tools-extra/clang-tidy/modernize/AvoidCArraysCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/AvoidCArraysCheck.cpp
@@ -9,6 +9,7 @@
 #include "AvoidCArraysCheck.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
 
 using namespace clang::ast_matchers;
 
@@ -60,6 +61,7 @@ void AvoidCArraysCheck::registerMatchers(MatchFinder *Finder) {
 
   Finder->addMatcher(
       typeLoc(hasValidBeginLoc(), hasType(arrayType()),
+              optionally(hasParent(parmVarDecl().bind("param_decl"))),
               unless(anyOf(hasParent(parmVarDecl(isArgvOfMain())),
                            hasParent(varDecl(isExternC())),
                            hasParent(fieldDecl(
@@ -72,11 +74,28 @@ void AvoidCArraysCheck::registerMatchers(MatchFinder *Finder) {
 
 void AvoidCArraysCheck::check(const MatchFinder::MatchResult &Result) {
   const auto *ArrayType = Result.Nodes.getNodeAs<TypeLoc>("typeloc");
-
+  const bool IsInParam =
+      Result.Nodes.getNodeAs<ParmVarDecl>("param_decl") != nullptr;
+  const bool IsVLA = ArrayType->getTypePtr()->isVariableArrayType();
+  enum class RecommendType { Array, Vector, Span };
+  llvm::SmallVector<const char *> RecommendTypes{};
+  if (IsVLA) {
+    RecommendTypes.push_back("std::vector<>");
+  } else if (ArrayType->getTypePtr()->isIncompleteArrayType() && IsInParam) {
+    // in function parameter, we also don't know the size of
+    // IncompleteArrayType.
+    if (Result.Context->getLangOpts().CPlusPlus20)
+      RecommendTypes.push_back("std::span<>");
+    else {
+      RecommendTypes.push_back("std::array<>");
+      RecommendTypes.push_back("std::vector<>");
+    }
+  } else {
+    RecommendTypes.push_back("std::array<>");
+  }
   diag(ArrayType->getBeginLoc(),
-       "do not declare %select{C-style|C VLA}0 arrays, use "
-       "%select{std::array<>|std::vector<>}0 instead")
-      << ArrayType->getTypePtr()->isVariableArrayType();
+       "do not declare %select{C-style|C VLA}0 arrays, use %1 instead")
+      << IsVLA << llvm::join(RecommendTypes, " or ");
 }
 
 } // namespace clang::tidy::modernize
diff --git a/clang-tools-extra/clang-tidy/modernize/MinMaxUseInitializerListCheck.cpp b/clang-tools-extra/clang-tidy/modernize/MinMaxUseInitializerListCheck.cpp
index 418699ffbc4d1a..9861f4681db1b4 100644
--- a/clang-tools-extra/clang-tidy/modernize/MinMaxUseInitializerListCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/MinMaxUseInitializerListCheck.cpp
@@ -72,7 +72,11 @@ static FindArgsResult findArgs(const CallExpr *Call) {
   return Result;
 }
 
-static SmallVector<FixItHint>
+// Returns `true` as `first` only if a nested call to `std::min` or
+// `std::max` was found. Checking if `FixItHint`s were generated is not enough,
+// as the explicit casts that the check introduces may be generated without a
+// nested `std::min` or `std::max` call.
+static std::pair<bool, SmallVector<FixItHint>>
 generateReplacements(const MatchFinder::MatchResult &Match,
                      const CallExpr *TopCall, const FindArgsResult &Result,
                      const bool IgnoreNonTrivialTypes,
@@ -91,13 +95,15 @@ generateReplacements(const MatchFinder::MatchResult &Match,
   const bool IsResultTypeTrivial = ResultType.isTrivialType(*Match.Context);
 
   if ((!IsResultTypeTrivial && IgnoreNonTrivialTypes))
-    return FixItHints;
+    return {false, FixItHints};
 
   if (IsResultTypeTrivial &&
       static_cast<std::uint64_t>(
           Match.Context->getTypeSizeInChars(ResultType).getQuantity()) >
           IgnoreTrivialTypesOfSizeAbove)
-    return FixItHints;
+    return {false, FixItHints};
+
+  bool FoundNestedCall = false;
 
   for (const Expr *Arg : Result.Args) {
     const auto *InnerCall = dyn_cast<CallExpr>(Arg->IgnoreParenImpCasts());
@@ -146,6 +152,9 @@ generateReplacements(const MatchFinder::MatchResult &Match,
                                        *Match.Context))
       continue;
 
+    // We have found a nested call
+    FoundNestedCall = true;
+
     // remove the function call
     FixItHints.push_back(
         FixItHint::CreateRemoval(InnerCall->getCallee()->getSourceRange()));
@@ -168,7 +177,7 @@ generateReplacements(const MatchFinder::MatchResult &Match,
           CharSourceRange::getTokenRange(InnerResult.First->getEndLoc())));
     }
 
-    const SmallVector<FixItHint> InnerReplacements = generateReplacements(
+    const auto [_, InnerReplacements] = generateReplacements(
         Match, InnerCall, InnerResult, IgnoreNonTrivialTypes,
         IgnoreTrivialTypesOfSizeAbove);
 
@@ -189,7 +198,7 @@ generateReplacements(const MatchFinder::MatchResult &Match,
     }
   }
 
-  return FixItHints;
+  return {FoundNestedCall, FixItHints};
 }
 
 MinMaxUseInitializerListCheck::MinMaxUseInitializerListCheck(
@@ -238,11 +247,11 @@ void MinMaxUseInitializerListCheck::check(
   const auto *TopCall = Match.Nodes.getNodeAs<CallExpr>("topCall");
 
   const FindArgsResult Result = findArgs(TopCall);
-  const SmallVector<FixItHint> Replacements =
+  const auto [FoundNestedCall, Replacements] =
       generateReplacements(Match, TopCall, Result, IgnoreNonTrivialTypes,
                            IgnoreTrivialTypesOfSizeAbove);
 
-  if (Replacements.empty())
+  if (!FoundNestedCall)
     return;
 
   const DiagnosticBuilder Diagnostic =
diff --git a/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.cpp b/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.cpp
index 8f2841c32259a2..1cb95c2b2347b7 100644
--- a/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.cpp
@@ -141,16 +141,18 @@ void EnumInitialValueCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
 }
 
 void EnumInitialValueCheck::registerMatchers(MatchFinder *Finder) {
-  Finder->addMatcher(
-      enumDecl(unless(isMacro()), unless(hasConsistentInitialValues()))
-          .bind("inconsistent"),
-      this);
+  Finder->addMatcher(enumDecl(isDefinition(), unless(isMacro()),
+                              unless(hasConsistentInitialValues()))
+                         .bind("inconsistent"),
+                     this);
   if (!AllowExplicitZeroFirstInitialValue)
     Finder->addMatcher(
-        enumDecl(hasZeroInitialValueForFirstEnumerator()).bind("zero_first"),
+        enumDecl(isDefinition(), hasZeroInitialValueForFirstEnumerator())
+            .bind("zero_first"),
         this);
   if (!AllowExplicitSequentialInitialValues)
-    Finder->addMatcher(enumDecl(unless(isMacro()), hasSequentialInitialValues())
+    Finder->addMatcher(enumDecl(isDefinition(), unless(isMacro()),
+                                hasSequentialInitialValues())
                            .bind("sequential"),
                        this);
 }
@@ -159,7 +161,7 @@ void EnumInitialValueCheck::check(const MatchFinder::MatchResult &Result) {
   if (const auto *Enum = Result.Nodes.getNodeAs<EnumDecl>("inconsistent")) {
     DiagnosticBuilder Diag =
         diag(Enum->getBeginLoc(),
-             "inital values in enum %0 are not consistent, consider explicit "
+             "initial values in enum %0 are not consistent, consider explicit "
              "initialization of all, none or only the first enumerator")
         << Enum;
     for (const EnumConstantDecl *ECD : Enum->enumerators())
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 8d0c093b312dd5..2370b594d22269 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -104,6 +104,10 @@ New checks
 New check aliases
 ^^^^^^^^^^^^^^^^^
 
+- New alias :doc:`cert-arr39-c <clang-tidy/checks/cert/arr39-c>` to
+  :doc:`bugprone-sizeof-expression
+  <clang-tidy/checks/bugprone/sizeof-expression>` was added.
+
 Changes in existing checks
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -111,14 +115,32 @@ Changes in existing checks
   <clang-tidy/checks/bugprone/casting-through-void>` check to suggest replacing
   the offending code with ``reinterpret_cast``, to more clearly express intent.
 
-- Improved :doc:`cert-flp30-c<clang-tidy/checks/cert/flp30-c>` check to 
+- Improved :doc:`bugprone-forwarding-reference-overload
+  <clang-tidy/checks/bugprone/forwarding-reference-overload>` check by fixing
+  a crash when determining if an ``enable_if[_t]`` was found.
+
+- Improved :doc:`bugprone-sizeof-expression
+  <clang-tidy/checks/bugprone/sizeof-expression>` check to find suspicious
+  usages of ``sizeof()``, ``alignof()``, and ``offsetof()`` when adding or
+  subtracting from a pointer.
+
+- Improved :doc:`cert-flp30-c <clang-tidy/checks/cert/flp30-c>` check to
   fix false positive that floating point variable is only used in increment
   expression.
 
 - Improved :doc:`cppcoreguidelines-prefer-member-initializer
-  <clang-tidy/checks/cppcoreguidelines/prefer-member-initializer>` check to avoid
-  false positive when member initialization depends on a structured binging
-  variable.
+  <clang-tidy/checks/cppcoreguidelines/prefer-member-initializer>` check to
+  avoid false positive when member initialization depends on a structured
+  binding variable.
+
+- Improved :doc:`misc-definitions-in-headers
+  <clang-tidy/checks/misc/definitions-in-headers>` check by rewording the
+  diagnostic note that suggests adding ``inline``.
+
+- Improved :doc:`modernize-avoid-c-arrays
+  <clang-tidy/checks/modernize/avoid-c-arrays>` check to suggest using ``std::span``
+  as a replacement for parameters of incomplete C array type in C++20 and 
+  ``std::array`` or ``std::vector`` before C++20.
 
 - Improved :doc:`modernize-use-std-format
   <clang-tidy/checks/modernize/use-std-format>` check to support replacing
@@ -128,10 +150,20 @@ Changes in existing checks
   <clang-tidy/checks/misc/unconventional-assign-operator>` check to avoid
   false positive for C++23 deducing this.
 
+- Improved :doc:`modernize-min-max-use-initializer-list
+  <clang-tidy/checks/modernize/min-max-use-initializer-list>` check by fixing
+  a false positive when only an implicit conversion happened inside an
+  initializer list.
+
 - Improved :doc:`modernize-use-std-print
   <clang-tidy/checks/modernize/use-std-print>` check to support replacing
   member function calls too.
 
+- Improved :doc:`readability-enum-initial-value
+  <clang-tidy/checks/readability/enum-initial-value>` check by only issuing
+  diagnostics for the definition of an ``enum``, and by fixing a typo in the
+  diagnostic.
+
 - Improved :doc:`performance-avoid-endl
   <clang-tidy/checks/performance/avoid-endl>` check to use ``std::endl`` as
   placeholder when lexer cannot get source text.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/sizeof-expression.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/sizeof-expression.rst
index ed5bb4fbb89baf..89c88d2740dba2 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/sizeof-expression.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/sizeof-expression.rst
@@ -164,6 +164,103 @@ hidden through macros.
     memcpy(dst, buf, sizeof(INT_SZ));  // sizeof(sizeof(int)) is suspicious.
   }
 
+Suspicious usages of 'sizeof(...)' in pointer arithmetic
+--------------------------------------------------------
+
+Arithmetic operators on pointers automatically scale the result with the size
+of the pointed typed.
+Further use of ``sizeof`` around pointer arithmetic will typically result in an
+unintended result.
+
+Scaling the result of pointer difference
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Subtracting two pointers results in an integer expression (of type
+``ptrdiff_t``) which expresses the distance between the two pointed objects in
+"number of objects between".
+A common mistake is to think that the result is "number of bytes between", and
+scale the difference with ``sizeof``, such as ``P1 - P2 == N * sizeof(T)``
+(instead of ``P1 - P2 == N``) or ``(P1 - P2) / sizeof(T)`` instead of
+``P1 - P2``.
+
+.. code-block:: c++
+
+  void splitFour(const Obj* Objs, size_t N, Obj Delimiter) {
+    const Obj *P = Objs;
+    while (P < Objs + N) {
+      if (*P == Delimiter) {
+        break;
+      }
+    }
+
+    if (P - Objs != 4 * sizeof(Obj)) { // Expecting a distance multiplied by sizeof is suspicious.
+      error();
+    }
+  }
+
+.. code-block:: c++
+
+  void iterateIfEvenLength(int *Begin, int *End) {
+    auto N = (Begin - End) / sizeof(int); // Dividing by sizeof() is suspicious.
+    if (N % 2)
+      return;
+
+    // ...
+  }
+
+Stepping a pointer with a scaled integer
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Conversely, when performing pointer arithmetics to add or subtract from a
+pointer, the arithmetic operator implicitly scales the value actually added to
+the pointer with the size of the pointee, as ``Ptr + N`` expects ``N`` to be
+"number of objects to step", and not "number of bytes to step".
+
+Seeing the calculation of a pointer where ``sizeof`` appears is suspicious,
+and the result is typically unintended, often out of bounds.
+``Ptr + sizeof(T)`` will offset the pointer by ``sizeof(T)`` elements,
+effectively exponentiating the scaling factor to the power of 2.
+
+This case also checks suspicious ``alignof`` and ``offsetof`` usages in
+pointer arithmetic, as both return the "size" in bytes and not elements,
+potentially resulting in doubly-scaled offsets.
+
+.. code-block:: c++
+
+  void printEveryEvenIndexElement(int *Array, size_t N) {
+    int *P = Array;
+    while (P <= Array + N * sizeof(int)) { // Suspicious pointer arithmetic using sizeof()!
+      printf("%d ", *P);
+
+      P += 2 * sizeof(int); // Suspicious pointer arithmetic using sizeof()!
+    }
+  }
+
+.. code-block:: c++
+
+  struct Message { /* ... */; char Flags[8]; };
+  void clearFlags(Message *Array, size_t N) {
+    const Message *End = Array + N;
+    while (Array < End) {
+      memset(Array + offsetof(Message, Flags), // Suspicious pointer arithmetic using offsetof()!
+             0, sizeof(Message::Flags));
+      ++Array;
+    }
+  }
+
+For this checked bogus pattern, `cert-arr39-c` redirects here as an alias of
+this check.
+
+This check corresponds to the CERT C Coding Standard rule
+`ARR39-C. Do not add or subtract a scaled integer to a pointer
+<http://wiki.sei.cmu.edu/confluence/display/c/ARR39-C.+Do+not+add+or+subtract+a+scaled+integer+to+a+pointer>`_.
+
+Limitations
+"""""""""""
+
+Cases where the pointee type has a size of `1` byte (such as, and most
+importantly, ``char``) are excluded.
+
 Options
 -------
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/cert/arr39-c.rst b/clang-tools-extra/docs/clang-tidy/checks/cert/arr39-c.rst
new file mode 100644
index 00000000000000..21353c61eede86
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/cert/arr39-c.rst
@@ -0,0 +1,10 @@
+.. title:: clang-tidy - cert-arr39-c
+.. meta::
+   :http-equiv=refresh: 5;URL=../bugprone/sizeof-expression.html
+
+cert-arr39-c
+============
+
+The `cert-arr39-c` check is an alias, please see
+:doc:`bugprone-sizeof-expression <../bugprone/sizeof-expression>`
+for more information.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst
index a931ebf025a10e..1909d7b8d8e246 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/list.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst
@@ -407,6 +407,7 @@ Check aliases
    :header: "Name", "Redirect", "Offers fixes"
 
    :doc:`bugprone-narrowing-conversions <bugprone/narrowing-conversions>`, :doc:`cppcoreguidelines-narrowing-conversions <cppcoreguidelines/narrowing-conversions>`,
+   :doc:`cert-arr39-c <cert/arr39-c>`, :doc:`bugprone-sizeof-expression <bugprone/sizeof-expression>`,
    :doc:`cert-con36-c <cert/con36-c>`, :doc:`bugprone-spuriously-wake-up-functions <bugprone/spuriously-wake-up-functions>`,
    :doc:`cert-con54-cpp <cert/con54-cpp>`, :doc:`bugprone-spuriously-wake-up-functions <bugprone/spuriously-wake-up-functions>`,
    :doc:`cert-ctr56-cpp <cert/ctr56-cpp>`, :doc:`bugprone-pointer-arithmetic-on-polymorphic-object <bugprone/pointer-arithmetic-on-polymorphic-object>`,
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize/avoid-c-arrays.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize/avoid-c-arrays.rst
index 8f13ca4466a310..2d72352989ab94 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize/avoid-c-arrays.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize/avoid-c-arrays.rst
@@ -10,6 +10,9 @@ modernize-avoid-c-arrays
 Finds C-style array types and recommend to use ``std::array<>`` /
 ``std::vector<>``. All types of C arrays are diagnosed.
 
+For incomplete C-style array types appeared in parameters, It would be better to
+use ``std::span`` / ``gsl::span`` as replacement.
+
 However, fix-it are potentially dangerous in header files and are therefore not
 emitted right now.
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-format.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-format.rst
index 1ec753ef090de1..b88fde5162e28c 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-format.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-format.rst
@@ -43,7 +43,7 @@ Options
     extern std::string strprintf(const char *format, ...);
     int i = -42;
     unsigned int u = 0xffffffff;
-    return strprintf("%d %u\n", i, u);
+    return strprintf("%u %d\n", i, u);
 
   would be converted to
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-print.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-print.rst
index 59bb722e2c24fc..e70402ad8b3341 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-print.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-print.rst
@@ -103,7 +103,7 @@ Options
 
     int i = -42;
     unsigned int u = 0xffffffff;
-    printf("%d %u\n", i, u);
+    printf("%u %d\n", i, u);
 
   would be converted to:
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/forwarding-reference-overload.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/forwarding-reference-overload.cpp
index 92dfb718bb51b7..27315199c7ebae 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/forwarding-reference-overload.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/forwarding-reference-overload.cpp
@@ -261,3 +261,9 @@ class Test11 {
   Test11(const Test11 &) = default;
   Test11(Test11 &&) = default;
 };
+
+template <template <class> typename T, typename U>
+struct gh106333
+{
+    gh106333(U && arg1, T<int> arg2) {}
+};
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression-pointer-arithmetics-c11.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression-pointer-arithmetics-c11.c
new file mode 100644
index 00000000000000..944328e61c3404
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression-pointer-arithmetics-c11.c
@@ -0,0 +1,22 @@
+// RUN: %check_clang_tidy -std=c11-or-later %s bugprone-sizeof-expression %t
+
+#define alignof(type_name) _Alignof(type_name)
+extern void sink(const void *P);
+
+enum { BufferSize = 1024 };
+
+struct S {
+  long A, B, C;
+};
+
+void bad4d(void) {
+  struct S Buffer[BufferSize];
+
+  struct S *P = &Buffer[0];
+  struct S *Q = P;
+  while (Q < P + alignof(Buffer)) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: suspicious usage of 'alignof(...)' in pointer arithmetic; this scaled value will be scaled again by the '+' operator [bugprone-sizeof-expression]
+    // CHECK-MESSAGES: :[[@LINE-2]]:16: note: '+' in pointer arithmetic internally scales with 'sizeof(struct S)' == {{[0-9]+}}
+    sink(Q++);
+  }
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression-pointer-arithmetics.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression-pointer-arithmetics.c
new file mode 100644
index 00000000000000..360ce00a6889d7
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression-pointer-arithmetics.c
@@ -0,0 +1,372 @@
+// RUN: %check_clang_tidy %s bugprone-sizeof-expression %t
+
+#define offsetof(type, member) __builtin_offsetof(type, member)
+
+typedef __SIZE_TYPE__ size_t;
+typedef __WCHAR_TYPE__ wchar_t;
+
+extern void *memset(void *Dest, int Ch, size_t Count);
+extern size_t strlen(const char *Str);
+extern size_t wcslen(const wchar_t *Str);
+extern char *strcpy(char *Dest, const char *Src);
+extern wchar_t *wcscpy(wchar_t *Dest, const wchar_t *Src);
+extern int scanf(const char *Format, ...);
+extern int wscanf(const wchar_t *Format, ...);
+
+extern void sink(const void *P);
+
+enum { BufferSize = 1024 };
+
+void bad1a(void) {
+  int Buffer[BufferSize];
+
+  int *P = &Buffer[0];
+  int *Q = P;
+  while (Q < P + sizeof(Buffer)) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: suspicious usage of 'sizeof(...)' in pointer arithmetic; this scaled value will be scaled again by the '+' operator [bugprone-sizeof-expression]
+    // CHECK-MESSAGES: :[[@LINE-2]]:16: note: '+' in pointer arithmetic internally scales with 'sizeof(int)' == {{[0-9]+}}
+    *Q++ = 0;
+  }
+}
+
+void bad1b(void) {
+  typedef int Integer;
+  Integer Buffer[BufferSize];
+
+  Integer *P = &Buffer[0];
+  Integer *Q = P;
+  while (Q < P + sizeof(Buffer)) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: suspicious usage of 'sizeof(...)' in pointer arithmetic; this scaled value will be scaled again by the '+' operator
+    // CHECK-MESSAGES: :[[@LINE-2]]:16: note: '+' in pointer arithmetic internally scales with 'sizeof(Integer)' == {{[0-9]+}}
+    *Q++ = 0;
+  }
+}
+
+void good1(void) {
+  int Buffer[BufferSize];
+
+  int *P = &Buffer[0];
+  int *Q = P;
+  while (Q < P + BufferSize) {
+    *Q++ = 0;
+  }
+}
+
+void bad2(void) {
+  int Buffer[BufferSize];
+  int *P = Buffer;
+
+  while (P < Buffer + sizeof(Buffer)) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:21: warning: suspicious usage of 'sizeof(...)' in pointer arithmetic; this scaled value will be scaled again by the '+' operator
+    // CHECK-MESSAGES: :[[@LINE-2]]:21: note: '+' in pointer arithmetic internally scales with 'sizeof(int)' == {{[0-9]+}}
+    *P++ = 0;
+  }
+}
+
+void good2(void) {
+  int Buffer[BufferSize];
+  int *P = Buffer;
+
+  while (P < Buffer + BufferSize) {
+    *P++ = 0;
+  }
+}
+
+struct S {
+  long A, B, C;
+};
+
+void bad3a(struct S *S) {
+  const size_t Offset = offsetof(struct S, B);
+  struct S *P = S;
+
+  // This is not captureable by Tidy because the size/offset expression is
+  // not a direct child of the pointer arithmetics.
+  memset(P + Offset, 0, sizeof(struct S) - Offset);
+}
+
+void good3a(struct S *S) {
+  const size_t Offset = offsetof(struct S, B);
+  char *P = (char*)S;
+
+  // This is not captureable by Tidy because the size/offset expression is
+  // not a direct child of the pointer arithmetics.
+  memset(P + Offset, 0, sizeof(struct S) - Offset);
+}
+
+void bad3b(struct S *S) {
+  memset(S + offsetof(struct S, B), 0,
+         sizeof(struct S) - offsetof(struct S, B));
+  // CHECK-MESSAGES: :[[@LINE-2]]:12: warning: suspicious usage of 'offsetof(...)' in pointer arithmetic; this scaled value will be scaled again by the '+' operator
+  // CHECK-MESSAGES: :[[@LINE-3]]:12: note: '+' in pointer arithmetic internally scales with 'sizeof(struct S)' == {{[0-9]+}}
+}
+
+void good3b(struct S *S) {
+  char *P = (char*)S;
+  memset(P + offsetof(struct S, B), 0,
+         sizeof(struct S) - offsetof(struct S, B));
+}
+
+void bad3c(void) {
+  struct S Buffer[BufferSize];
+
+  struct S *P = &Buffer[0];
+  struct S *Q = P;
+  while (Q < P + sizeof(Buffer)) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: suspicious usage of 'sizeof(...)' in pointer arithmetic; this scaled value will be scaled again by the '+' operator
+    // CHECK-MESSAGES: :[[@LINE-2]]:16: note: '+' in pointer arithmetic internally scales with 'sizeof(struct S)' == {{[0-9]+}}
+    sink(Q++);
+  }
+}
+
+void bad4(void) {
+  int Buffer[BufferSize];
+
+  int *P = &Buffer[0];
+  int *Q = P;
+  while (Q < P + BufferSize) {
+    *Q = 0;
+    Q += sizeof(*Q);
+    // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: suspicious usage of 'sizeof(...)' in pointer arithmetic; this scaled value will be scaled again by the '+=' operator
+    // CHECK-MESSAGES: :[[@LINE-2]]:7: note: '+=' in pointer arithmetic internally scales with 'sizeof(int)' == {{[0-9]+}}
+  }
+}
+
+void silenced4(void) {
+  char Buffer[BufferSize];
+
+  char *P = &Buffer[0];
+  char *Q = P;
+  while (Q < P + BufferSize) {
+    *Q = 0;
+    Q += sizeof(*Q);
+  }
+}
+
+void good4(void) {
+  char Buffer[BufferSize];
+
+  char *P = &Buffer[0];
+  char *Q = P;
+  while (Q < P + BufferSize) {
+    *Q = 0;
+    Q += 1;
+  }
+}
+
+void good5aa(void) {
+  int Buffer[BufferSize];
+
+  int *P = &Buffer[0];
+  int *Q = P;
+  while (Q < P + BufferSize) {
+    *Q = 0;
+    Q += ( sizeof(Buffer) / sizeof(Buffer[0]) );
+  }
+}
+
+void good5ab(void) {
+  int Buffer[BufferSize];
+
+  int *P = &Buffer[0];
+  int *Q = P;
+  while (Q < P + BufferSize) {
+    *Q = 0;
+    Q = Q + ( sizeof(Buffer) / sizeof(Buffer[0]) );
+  }
+}
+
+void good5ba(void) {
+  int Buffer[BufferSize];
+
+  int *P = &Buffer[0];
+  int *Q = P;
+  while (Q < P + BufferSize) {
+    *Q = 0;
+    Q -= ( sizeof(Buffer) / sizeof(Buffer[0]) );
+  }
+}
+
+void good5bb(void) {
+  int Buffer[BufferSize];
+
+  int *P = &Buffer[0];
+  int *Q = P;
+  while (Q < P + BufferSize) {
+    *Q = 0;
+    Q = Q - ( sizeof(Buffer) / sizeof(Buffer[0]) );
+  }
+}
+
+void bad6(void) {
+  int Buffer[BufferSize];
+
+  int *P = &Buffer[0];
+  int *Q = P;
+  while (Q < P + BufferSize) {
+    *Q = 0;
+    Q = Q + sizeof(*Q);
+    // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: suspicious usage of 'sizeof(...)' in pointer arithmetic; this scaled value will be scaled again by the '+' operator
+    // CHECK-MESSAGES: :[[@LINE-2]]:11: note: '+' in pointer arithmetic internally scales with 'sizeof(int)' == {{[0-9]+}}
+  }
+}
+
+void silenced6(void) {
+  char Buffer[BufferSize];
+
+  char *P = &Buffer[0];
+  char *Q = P;
+  while (Q < P + BufferSize) {
+    *Q = 0;
+    Q = Q + sizeof(*Q);
+  }
+}
+
+void good6(void) {
+  char Buffer[BufferSize];
+
+  char *P = &Buffer[0];
+  char *Q = P;
+  while (Q < P + BufferSize) {
+    *Q = 0;
+    Q = Q + 1;
+  }
+}
+
+void silenced7(void) {
+  char Buffer[BufferSize];
+
+  char *P = &Buffer[0];
+  const char *Q = P;
+  while (Q < P + BufferSize) {
+    sink(Q);
+    Q = Q + sizeof(*Q);
+  }
+}
+
+void good7(void) {
+  char Buffer[BufferSize];
+
+  char *P = &Buffer[0];
+  const char *Q = P;
+  while (Q < P + BufferSize) {
+    sink(Q);
+    Q = Q + 1;
+  }
+}
+
+void bad8(void) {
+  int Buffer[BufferSize];
+
+  int *P = &Buffer[0];
+  int *Q = P;
+  while (Q >= P) {
+    *Q = 0;
+    Q = Q - sizeof(*Q);
+    // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: suspicious usage of 'sizeof(...)' in pointer arithmetic; this scaled value will be scaled again by the '-' operator
+    // CHECK-MESSAGES: :[[@LINE-2]]:11: note: '-' in pointer arithmetic internally scales with 'sizeof(int)' == {{[0-9]+}}
+  }
+}
+
+void silenced8(void) {
+  char Buffer[BufferSize];
+
+  char *P = &Buffer[0];
+  char *Q = P;
+  while (Q >= P) {
+    *Q = 0;
+    Q = Q - sizeof(*Q);
+  }
+}
+
+void good8(void) {
+  char Buffer[BufferSize];
+
+  char *P = &Buffer[0];
+  char *Q = P;
+  while (Q >= P) {
+    *Q = 0;
+    Q = Q - 1;
+  }
+}
+
+void good9(void) {
+  int Buffer[BufferSize];
+
+  int *P = &Buffer[0];
+  int *Q = P + BufferSize;
+  int N = Q - P;
+  while (N >= 0) {
+    Q[N] = 0;
+    N = N - 1;
+  }
+}
+
+void good10(void) {
+  int Buffer[BufferSize];
+
+  int *P = &Buffer[0];
+  int *Q = Buffer + BufferSize;
+  int I = sizeof(*P) - sizeof(*Q);
+
+  sink(&I);
+}
+
+void good11(void) {
+  int Buffer[BufferSize];
+
+  int *P = &Buffer[0];
+  int *Q = Buffer + BufferSize;
+  int I = sizeof(Q) - sizeof(*P);
+
+  sink(&I);
+}
+
+void bad12(void) {
+  wchar_t Message[BufferSize];
+  wcscpy(Message, L"Message: ");
+  wscanf(L"%s", Message + wcslen(Message) * sizeof(wchar_t));
+  // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: suspicious usage of 'sizeof(...)' in pointer arithmetic; this scaled value will be scaled again by the '+' operator
+  // CHECK-MESSAGES: :[[@LINE-2]]:25: note: '+' in pointer arithmetic internally scales with 'sizeof(wchar_t)' == {{[0-9]+}}
+}
+
+void silenced12(void) {
+  char Message[BufferSize];
+  strcpy(Message, "Message: ");
+  scanf("%s", Message + strlen(Message) * sizeof(char));
+}
+
+void nomatch12(void) {
+  char Message[BufferSize];
+  strcpy(Message, "Message: ");
+  scanf("%s", Message + strlen(Message));
+}
+
+void good12(void) {
+  wchar_t Message[BufferSize];
+  wcscpy(Message, L"Message: ");
+  wscanf(L"%s", Message + wcslen(Message));
+}
+
+void good13(void) {
+  int Buffer[BufferSize];
+
+  int *P = &Buffer[0];
+  while (P < (Buffer + sizeof(Buffer) / sizeof(int))) {
+    // NO-WARNING: Calculating the element count of the buffer here, which is
+    // safe with this idiom (as long as the types don't change).
+    ++P;
+  }
+
+  while (P < (Buffer + sizeof(Buffer) / sizeof(Buffer[0]))) {
+    // NO-WARNING: Calculating the element count of the buffer here, which is
+    // safe with this idiom.
+    ++P;
+  }
+
+  while (P < (Buffer + sizeof(Buffer) / sizeof(*P))) {
+    // NO-WARNING: Calculating the element count of the buffer here, which is
+    // safe with this idiom.
+    ++P;
+  }
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression-2.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression.c
similarity index 100%
rename from clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression-2.c
rename to clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression.c
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression.cpp
index 671fd8370894ff..81efd87345c97e 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression.cpp
@@ -289,6 +289,35 @@ int Test6() {
   return sum;
 }
 
+static constexpr inline int BufferSize = 1024;
+
+template <typename T>
+T next(const T *&Read) {
+  T value = *Read;
+  Read += sizeof(T);
+  return value;
+}
+
+void Test7() {
+  int Buffer[BufferSize];
+  int *P = &Buffer[0];
+
+  const int *P2 = P;
+  int V1 = next(P2);
+  // CHECK-MESSAGES: :[[@LINE-10]]:8: warning: suspicious usage of 'sizeof(...)' in pointer arithmetic; this scaled value will be scaled again by the '+=' operator
+  // CHECK-MESSAGES: :[[@LINE-11]]:8: note: '+=' in pointer arithmetic internally scales with 'sizeof(const int)' == {{[0-9]+}}
+  int V2 = next(P2);
+  (void)V1;
+  (void)V2;
+
+  int *Q = P;
+  while (Q < P + sizeof(Buffer)) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: suspicious usage of 'sizeof(...)' in pointer arithmetic; this scaled value will be scaled again by the '+' operator
+    // CHECK-MESSAGES: :[[@LINE-2]]:16: note: '+' in pointer arithmetic internally scales with 'sizeof(int)' == {{[0-9]+}}
+    *Q++ = 0;
+  }
+}
+
 #ifdef __SIZEOF_INT128__
 template <__int128_t N>
 #else
@@ -296,7 +325,7 @@ template <long N> // Fallback for platforms which do not define `__int128_t`
 #endif
 bool Baz() { return sizeof(A) < N; }
 // CHECK-MESSAGES: :[[@LINE-1]]:31: warning: suspicious comparison of 'sizeof(expr)' to a constant
-bool Test7() { return Baz<-1>(); }
+bool Test8() { return Baz<-1>(); }
 
 void some_generic_function(const void *arg, int argsize);
 int *IntP, **IntPP;
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/prefer-member-initializer.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/prefer-member-initializer.cpp
index fa4307dec02308..3432b1c84a9a50 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/prefer-member-initializer.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/prefer-member-initializer.cpp
@@ -641,9 +641,9 @@ struct S3 {
 }
 
 namespace GH82970 {
-struct InitFromBingingDecl {
+struct InitFromBindingDecl {
   int m;
-  InitFromBingingDecl() {
+  InitFromBindingDecl() {
     struct { int i; } a;
     auto [n] = a;
     m = n;
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/definitions-in-headers.hpp b/clang-tools-extra/test/clang-tidy/checkers/misc/definitions-in-headers.hpp
index 4cf07077a230a5..9c91cb7033087d 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/definitions-in-headers.hpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/definitions-in-headers.hpp
@@ -2,7 +2,7 @@
 
 int f() {
 // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: function 'f' defined in a header file; function definitions in header files can lead to ODR violations [misc-definitions-in-headers]
-// CHECK-MESSAGES: :[[@LINE-2]]:5: note: make as 'inline'
+// CHECK-MESSAGES: :[[@LINE-2]]:5: note: mark the definition as 'inline'
 // CHECK-FIXES: inline int f() {
   return 1;
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays-c++20.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays-c++20.cpp
new file mode 100644
index 00000000000000..e53cfeba0e4601
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays-c++20.cpp
@@ -0,0 +1,11 @@
+// RUN: %check_clang_tidy -std=c++20 %s modernize-avoid-c-arrays %t
+
+int f1(int data[], int size) {
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: do not declare C-style arrays, use std::span<> instead
+  int f4[] = {1, 2};
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: do not declare C-style arrays, use std::array<> instead
+}
+
+int f2(int data[100]) {
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: do not declare C-style arrays, use std::array<> instead
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays-ignores-main.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays-ignores-main.cpp
index 6549422f393aaa..ad12b3d6f95b91 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays-ignores-main.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays-ignores-main.cpp
@@ -1,7 +1,7 @@
-// RUN: %check_clang_tidy %s modernize-avoid-c-arrays %t
+// RUN: %check_clang_tidy -std=c++17 %s modernize-avoid-c-arrays %t
 
 int not_main(int argc, char *argv[]) {
-  // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: do not declare C-style arrays, use std::array<> instead
+  // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: do not declare C-style arrays, use std::array<> or std::vector<> instead
   int f4[] = {1, 2};
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: do not declare C-style arrays, use std::array<> instead
 }
@@ -11,7 +11,7 @@ int main(int argc, char *argv[]) {
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: do not declare C-style arrays, use std::array<> instead
 
   auto not_main = [](int argc, char *argv[]) {
-    // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: do not declare C-style arrays, use std::array<> instead
+    // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: do not declare C-style arrays, use std::array<> or std::vector<> instead
     int f6[] = {1, 2};
     // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: do not declare C-style arrays, use std::array<> instead
   };
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays-ignores-strings.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays-ignores-strings.cpp
index f6d64848f9e3a0..b607068f5b7c9c 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays-ignores-strings.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays-ignores-strings.cpp
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy %s modernize-avoid-c-arrays %t -- \
+// RUN: %check_clang_tidy -std=c++17 %s modernize-avoid-c-arrays %t -- \
 // RUN:  -config='{CheckOptions: { modernize-avoid-c-arrays.AllowStringArrays: true }}'
 
 const char name[] = "name";
@@ -6,4 +6,4 @@ const char array[] = {'n', 'a', 'm', 'e', '\0'};
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: do not declare C-style arrays, use std::array<> instead [modernize-avoid-c-arrays]
 
 void takeCharArray(const char name[]);
-// CHECK-MESSAGES: :[[@LINE-1]]:26: warning: do not declare C-style arrays, use std::array<> instead [modernize-avoid-c-arrays]
+// CHECK-MESSAGES: :[[@LINE-1]]:26: warning: do not declare C-style arrays, use std::array<> or std::vector<> instead [modernize-avoid-c-arrays]
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays-ignores-three-arg-main.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays-ignores-three-arg-main.cpp
index 22a4016f79f4da..c04edf2b5aea08 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays-ignores-three-arg-main.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays-ignores-three-arg-main.cpp
@@ -1,8 +1,8 @@
-// RUN: %check_clang_tidy %s modernize-avoid-c-arrays %t
+// RUN: %check_clang_tidy -std=c++17 %s modernize-avoid-c-arrays %t
 
 int not_main(int argc, char *argv[], char *argw[]) {
-  // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: do not declare C-style arrays, use std::array<> instead
-  // CHECK-MESSAGES: :[[@LINE-2]]:38: warning: do not declare C-style arrays, use std::array<> instead
+  // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: do not declare C-style arrays, use std::array<> or std::vector<> instead
+  // CHECK-MESSAGES: :[[@LINE-2]]:38: warning: do not declare C-style arrays, use std::array<> or std::vector<> instead
   int f4[] = {1, 2};
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: do not declare C-style arrays, use std::array<> instead
 }
@@ -12,8 +12,8 @@ int main(int argc, char *argv[], char *argw[]) {
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: do not declare C-style arrays, use std::array<> instead
 
   auto not_main = [](int argc, char *argv[], char *argw[]) {
-    // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: do not declare C-style arrays, use std::array<> instead
-    // CHECK-MESSAGES: :[[@LINE-2]]:46: warning: do not declare C-style arrays, use std::array<> instead
+    // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: do not declare C-style arrays, use std::array<> or std::vector<> instead
+    // CHECK-MESSAGES: :[[@LINE-2]]:46: warning: do not declare C-style arrays, use std::array<> or std::vector<> instead
     int f6[] = {1, 2};
     // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: do not declare C-style arrays, use std::array<> instead
   };
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays.cpp
index ce99f0821b2230..b0aaa4962a8351 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-c-arrays.cpp
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy %s modernize-avoid-c-arrays %t
+// RUN: %check_clang_tidy -std=c++17 %s modernize-avoid-c-arrays %t
 
 int a[] = {1, 2};
 // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: do not declare C-style arrays, use std::array<> instead
@@ -91,4 +91,4 @@ const char name[] = "Some string";
 // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: do not declare C-style arrays, use std::array<> instead [modernize-avoid-c-arrays]
 
 void takeCharArray(const char name[]);
-// CHECK-MESSAGES: :[[@LINE-1]]:26: warning: do not declare C-style arrays, use std::array<> instead [modernize-avoid-c-arrays]
+// CHECK-MESSAGES: :[[@LINE-1]]:26: warning: do not declare C-style arrays, use std::array<> or std::vector<> instead [modernize-avoid-c-arrays]
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/min-max-use-initializer-list.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/min-max-use-initializer-list.cpp
index c7632fe007a4f4..f4e21316718045 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/min-max-use-initializer-list.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/min-max-use-initializer-list.cpp
@@ -323,5 +323,11 @@ struct GH91982 {
   }
 };
 
+struct GH107594 {
+    int foo(int a, int b, char c) {
+        return std::max<int>({a, b, c});
+    }
+};
+
 } // namespace
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.c b/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.c
index c66288cbe3e957..b9a34d0683d7f3 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.c
@@ -6,8 +6,8 @@
 // RUN:     }}'
 
 enum EError {
-  // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: inital values in enum 'EError' are not consistent
-  // CHECK-MESSAGES-ENABLE: :[[@LINE-2]]:1: warning: inital values in enum 'EError' are not consistent
+  // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: initial values in enum 'EError' are not consistent
+  // CHECK-MESSAGES-ENABLE: :[[@LINE-2]]:1: warning: initial values in enum 'EError' are not consistent
   EError_a = 1,
   EError_b,
   // CHECK-FIXES: EError_b = 2,
@@ -34,8 +34,8 @@ enum EAll {
 
 #define ENUMERATOR_1 EMacro1_b
 enum EMacro1 {
-  // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: inital values in enum 'EMacro1' are not consistent
-  // CHECK-MESSAGES-ENABLE: :[[@LINE-2]]:1: warning: inital values in enum 'EMacro1' are not consistent
+  // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: initial values in enum 'EMacro1' are not consistent
+  // CHECK-MESSAGES-ENABLE: :[[@LINE-2]]:1: warning: initial values in enum 'EMacro1' are not consistent
   EMacro1_a = 1,
   ENUMERATOR_1,
   // CHECK-FIXES: ENUMERATOR_1 = 2,
@@ -45,8 +45,8 @@ enum EMacro1 {
 
 #define ENUMERATOR_2 EMacro2_b = 2
 enum EMacro2 {
-  // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: inital values in enum 'EMacro2' are not consistent
-  // CHECK-MESSAGES-ENABLE: :[[@LINE-2]]:1: warning: inital values in enum 'EMacro2' are not consistent
+  // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: initial values in enum 'EMacro2' are not consistent
+  // CHECK-MESSAGES-ENABLE: :[[@LINE-2]]:1: warning: initial values in enum 'EMacro2' are not consistent
   EMacro2_a = 1,
   ENUMERATOR_2,
   EMacro2_c,
@@ -78,3 +78,40 @@ enum EnumSequentialInitialValue {
   EnumSequentialInitialValue_2 = 4,
   // CHECK-FIXES-ENABLE: EnumSequentialInitialValue_2 ,
 };
+
+// gh107590
+enum WithFwdDeclInconsistent : int;
+
+enum WithFwdDeclInconsistent : int {
+  // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: initial values in enum 'WithFwdDeclInconsistent' are not consistent
+  // CHECK-MESSAGES-ENABLE: :[[@LINE-2]]:1: warning: initial values in enum 'WithFwdDeclInconsistent' are not consistent
+  EFI0,
+  // CHECK-FIXES: EFI0 = 0,
+  EFI1 = 1,
+  EFI2,
+  // CHECK-FIXES: EFI2 = 2,
+};
+
+enum WithFwdDeclZeroFirst : int;
+
+enum WithFwdDeclZeroFirst : int {
+  // CHECK-MESSAGES-ENABLE: :[[@LINE+1]]:3: warning: zero initial value for the first enumerator in 'WithFwdDeclZeroFirst' can be disregarded
+  EFZ0 = 0,
+  // CHECK-FIXES-ENABLE: EFZ0 ,
+  EFZ1,
+  EFZ2,
+};
+
+
+enum WithFwdDeclSequential : int;
+
+enum WithFwdDeclSequential : int {
+  // CHECK-MESSAGES-ENABLE: :[[@LINE-1]]:1: warning: sequential initial value in 'WithFwdDeclSequential' can be ignored
+  EFS0 = 2,
+  // CHECK-FIXES-ENABLE: EFS0 = 2,
+  EFS1 = 3,
+  // CHECK-FIXES-ENABLE: EFS1 ,
+  EFS2 = 4,
+  // CHECK-FIXES-ENABLE: EFS2 ,
+};
+
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.cpp
index 3c4ba970372a07..9d324a39ee6a36 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/enum-initial-value.cpp
@@ -1,7 +1,7 @@
 // RUN: %check_clang_tidy %s readability-enum-initial-value %t
 
 enum class EError {
-  // CHECK-MESSAGES: :[[@LINE-1]]:1:  warning: inital values in enum 'EError' are not consistent
+  // CHECK-MESSAGES: :[[@LINE-1]]:1:  warning: initial values in enum 'EError' are not consistent
   EError_a = 1,
   EError_b,
   // CHECK-FIXES: EError_b = 2,
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 69b2aea52aa9d3..d92b59334f8f32 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -170,6 +170,9 @@ Resolutions to C++ Defect Reports
   in constant expressions. These comparisons always worked in non-constant expressions.
   (`CWG2749: Treatment of "pointer to void" for relational comparisons <https://cplusplus.github.io/CWG/issues/2749.html>`_).
 
+- Reject explicit object parameters with type ``void`` (``this void``).
+  (`CWG2915: Explicit object parameters of type void <https://cplusplus.github.io/CWG/issues/2915.html>`_).
+
 C Language Changes
 ------------------
 
@@ -506,6 +509,8 @@ AST Matchers
 - Fixed an ordering issue with the `hasOperands` matcher occurring when setting a
   binding in the first matcher and using it in the second matcher.
 
+- Fixed a crash when traverse lambda expr with invalid captures. (#GH106444)
+
 clang-format
 ------------
 
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index 168bdca3c880b2..b65a1f7dff5bc1 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -782,6 +782,23 @@ class ASTContext : public RefCountedBase<ASTContext> {
   const TargetInfo &getTargetInfo() const { return *Target; }
   const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
 
+  const QualType GetHigherPrecisionFPType(QualType ElementType) const {
+    const auto *CurrentBT = cast<BuiltinType>(ElementType);
+    switch (CurrentBT->getKind()) {
+    case BuiltinType::Kind::Half:
+    case BuiltinType::Kind::Float16:
+      return FloatTy;
+    case BuiltinType::Kind::Float:
+    case BuiltinType::Kind::BFloat16:
+      return DoubleTy;
+    case BuiltinType::Kind::Double:
+      return LongDoubleTy;
+    default:
+      return ElementType;
+    }
+    return ElementType;
+  }
+
   /// getIntTypeForBitwidth -
   /// sets integer QualTy according to specified details:
   /// bitwidth, signed/unsigned.
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index ef36a73716454f..dc87b84153e74a 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -6168,10 +6168,18 @@ class HLSLAttributedResourceType : public Type, public llvm::FoldingSetNode {
   struct Attributes {
     // Data gathered from HLSL resource attributes
     llvm::dxil::ResourceClass ResourceClass;
+
+    LLVM_PREFERRED_TYPE(bool)
     uint8_t IsROV : 1;
-    Attributes(llvm::dxil::ResourceClass ResourceClass, bool IsROV)
-        : ResourceClass(ResourceClass), IsROV(IsROV) {}
-    Attributes() : ResourceClass(llvm::dxil::ResourceClass::UAV), IsROV(0) {}
+
+    LLVM_PREFERRED_TYPE(bool)
+    uint8_t RawBuffer : 1;
+
+    Attributes(llvm::dxil::ResourceClass ResourceClass, bool IsROV,
+               bool RawBuffer)
+        : ResourceClass(ResourceClass), IsROV(IsROV), RawBuffer(RawBuffer) {}
+
+    Attributes() : Attributes(llvm::dxil::ResourceClass::UAV, false, false) {}
   };
 
 private:
@@ -6204,6 +6212,7 @@ class HLSLAttributedResourceType : public Type, public llvm::FoldingSetNode {
     ID.AddPointer(Contained.getAsOpaquePtr());
     ID.AddInteger(static_cast<uint32_t>(Attrs.ResourceClass));
     ID.AddBoolean(Attrs.IsROV);
+    ID.AddBoolean(Attrs.RawBuffer);
   }
 
   static bool classof(const Type *T) {
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index 03fbdcf60140df..62ca52e508ba20 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -942,6 +942,7 @@ class BTFTagAttributedTypeLoc
 
 struct HLSLAttributedResourceLocInfo {
   SourceRange Range;
+  TypeSourceInfo *ContainedTyInfo;
 };
 
 /// Type source information for HLSL attributed resource type.
@@ -952,8 +953,11 @@ class HLSLAttributedResourceTypeLoc
 public:
   TypeLoc getWrappedLoc() const { return getInnerTypeLoc(); }
 
-  TypeLoc getContainedLoc() const {
-    return TypeLoc(getTypePtr()->getContainedType(), getNonLocalData());
+  TypeSourceInfo *getContainedTypeSourceInfo() const {
+    return getLocalData()->ContainedTyInfo;
+  }
+  void setContainedTypeSourceInfo(TypeSourceInfo *TSI) const {
+    getLocalData()->ContainedTyInfo = TSI;
   }
 
   void setSourceRange(const SourceRange &R) { getLocalData()->Range = R; }
diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index 539a344cb0b690..bb7bfa8cd0b76e 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -697,6 +697,9 @@ let Class = HLSLAttributedResourceType in {
   def : Property<"isROV", Bool> {
     let Read = [{ node->getAttrs().IsROV }];
   }
+  def : Property<"rawBuffer", Bool> {
+    let Read = [{ node->getAttrs().RawBuffer }];
+  }
   def : Property<"wrappedTy", QualType> {
     let Read = [{ node->getWrappedType() }];
   }
@@ -704,7 +707,7 @@ let Class = HLSLAttributedResourceType in {
     let Read = [{ node->getContainedType() }];
   }
   def : Creator<[{
-    HLSLAttributedResourceType::Attributes attrs(static_cast<llvm::dxil::ResourceClass>(resClass), isROV);
+    HLSLAttributedResourceType::Attributes attrs(static_cast<llvm::dxil::ResourceClass>(resClass), isROV, rawBuffer);
     return ctx.getHLSLAttributedResourceType(wrappedTy, containedTy, attrs);
   }]>;
 }
diff --git a/clang/include/clang/Basic/AMDGPUTypes.def b/clang/include/clang/Basic/AMDGPUTypes.def
index e0d7be470a3259..7454d61f5dd516 100644
--- a/clang/include/clang/Basic/AMDGPUTypes.def
+++ b/clang/include/clang/Basic/AMDGPUTypes.def
@@ -11,11 +11,11 @@
 //===----------------------------------------------------------------------===//
 
 #ifndef AMDGPU_OPAQUE_PTR_TYPE
-#define AMDGPU_OPAQUE_PTR_TYPE(Name, MangledName, AS, Width, Align, Id, SingletonId) \
+#define AMDGPU_OPAQUE_PTR_TYPE(Name, AS, Width, Align, Id, SingletonId) \
   AMDGPU_TYPE(Name, Id, SingletonId)
 #endif
 
-AMDGPU_OPAQUE_PTR_TYPE("__amdgpu_buffer_rsrc_t", "__amdgpu_buffer_rsrc_t", 8, 128, 128, AMDGPUBufferRsrc, AMDGPUBufferRsrcTy)
+AMDGPU_OPAQUE_PTR_TYPE("__amdgpu_buffer_rsrc_t", 8, 128, 128, AMDGPUBufferRsrc, AMDGPUBufferRsrcTy)
 
 #undef AMDGPU_TYPE
 #undef AMDGPU_OPAQUE_PTR_TYPE
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 70fad60d4edbb5..35b9716e13ff21 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -4667,6 +4667,19 @@ def HLSLResourceClass : TypeAttr {
   let Documentation = [InternalOnly];
 }
 
+def HLSLContainedType : TypeAttr {
+  let Spellings = [CXX11<"hlsl", "contained_type">];
+  let LangOpts = [HLSL];
+  let Args = [TypeArgument<"Type", /*opt=*/0>];
+  let Documentation = [InternalOnly];
+}
+
+def HLSLRawBuffer : TypeAttr {
+  let Spellings = [CXX11<"hlsl", "raw_buffer">];
+  let LangOpts = [HLSL]; 
+  let Documentation = [InternalOnly];
+}
+
 def HLSLGroupSharedAddressSpace : TypeAttr {
   let Spellings = [CustomKeyword<"groupshared">];
   let Subjects = SubjectList<[Var]>;
diff --git a/clang/include/clang/Basic/DiagnosticCommonKinds.td b/clang/include/clang/Basic/DiagnosticCommonKinds.td
index 33b1d58bb5b099..ae709e45a700a1 100644
--- a/clang/include/clang/Basic/DiagnosticCommonKinds.td
+++ b/clang/include/clang/Basic/DiagnosticCommonKinds.td
@@ -45,11 +45,6 @@ def note_using : Note<"using">;
 def note_possibility : Note<"one possibility">;
 def note_also_found : Note<"also found">;
 
-def warn_next_larger_fp_type_same_size_than_fp : Warning<
-  "higher precision floating-point type size has the same size than "
-  "floating-point type size">,
-  InGroup<DiagGroup<"higher-precision-fp">>;
-
 // Parse && Lex
 
 let CategoryName = "Lexical or Preprocessor Issue" in {
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index d42558d2223aae..bfda5b521c8fd2 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -4687,6 +4687,8 @@ def err_void_only_param : Error<
   "'void' must be the first and only parameter if specified">;
 def err_void_param_qualified : Error<
   "'void' as parameter must not have type qualifiers">;
+def err_void_explicit_object_param : Error<
+  "explicit object parameter cannot have 'void' type">;
 def err_ident_list_in_fn_declaration : Error<
   "a parameter list without types is only allowed in a function definition">;
 def ext_param_not_declared : ExtWarn<
@@ -6800,6 +6802,12 @@ def warn_arc_lifetime_result_type : Warning<
   "ARC %select{unused|__unsafe_unretained|__strong|__weak|__autoreleasing}0 "
   "lifetime qualifier on return type is ignored">,
   InGroup<IgnoredQualifiers>;
+def warn_excess_precision_not_supported : Warning<
+  "excess precision is requested but the target does not support excess "
+  "precision which may result in observable differences in complex division "
+  "behavior%select{|, additional uses where the requested higher precision "
+  "cannot be honored were found but not diagnosed}0">,
+  InGroup<DiagGroup<"higher-precision-for-complex-division">>;
 
 let CategoryName = "ARC Retain Cycle" in {
 
@@ -12373,6 +12381,7 @@ def warn_hlsl_deprecated_register_type_b: Warning<"binding type 'b' only applies
 def warn_hlsl_deprecated_register_type_i: Warning<"binding type 'i' ignored. The 'integer constant' binding type is no longer supported">, InGroup<LegacyConstantRegisterBinding>, DefaultError;
 def err_hlsl_unsupported_register_number : Error<"register number should be an integer">;
 def err_hlsl_expected_space : Error<"invalid space specifier '%0' used; expected 'space' followed by an integer, like space1">;
+def err_hlsl_space_on_global_constant : Error<"register space cannot be specified on global constants">;
 def warn_hlsl_packoffset_mix : Warning<"cannot mix packoffset elements with nonpackoffset elements in a cbuffer">,
     InGroup<HLSLMixPackOffset>;
 def err_hlsl_packoffset_overlap : Error<"packoffset overlap between %0, %1">;
diff --git a/clang/include/clang/Basic/LangStandards.def b/clang/include/clang/Basic/LangStandards.def
index 41e756d9365dcf..982a79873adb59 100644
--- a/clang/include/clang/Basic/LangStandards.def
+++ b/clang/include/clang/Basic/LangStandards.def
@@ -230,27 +230,27 @@ LANGSTANDARD_ALIAS_DEPR(openclcpp2021, "CLC++2021")
 // HLSL
 LANGSTANDARD(hlsl, "hlsl",
              HLSL, "High Level Shader Language",
-             LineComment | HLSL | CPlusPlus )
+             LineComment | HLSL | CPlusPlus | CPlusPlus11)
 
 LANGSTANDARD(hlsl2015, "hlsl2015",
              HLSL, "High Level Shader Language 2015",
-             LineComment | HLSL | CPlusPlus )
+             LineComment | HLSL | CPlusPlus | CPlusPlus11)
 
 LANGSTANDARD(hlsl2016, "hlsl2016",
              HLSL, "High Level Shader Language 2016",
-             LineComment | HLSL | CPlusPlus )
+             LineComment | HLSL | CPlusPlus | CPlusPlus11)
 
 LANGSTANDARD(hlsl2017, "hlsl2017",
              HLSL, "High Level Shader Language 2017",
-             LineComment | HLSL | CPlusPlus )
+             LineComment | HLSL | CPlusPlus | CPlusPlus11)
 
 LANGSTANDARD(hlsl2018, "hlsl2018",
              HLSL, "High Level Shader Language 2018",
-             LineComment | HLSL | CPlusPlus )
+             LineComment | HLSL | CPlusPlus | CPlusPlus11)
 
 LANGSTANDARD(hlsl2021, "hlsl2021",
              HLSL, "High Level Shader Language 2021",
-             LineComment | HLSL | CPlusPlus )
+             LineComment | HLSL | CPlusPlus | CPlusPlus11)
 
 LANGSTANDARD(hlsl202x, "hlsl202x",
              HLSL, "High Level Shader Language 202x",
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 7f123335ce8cfa..aa3ae92fb6ae78 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5572,7 +5572,8 @@ def : Flag<["-"], "nocudalib">, Alias<nogpulib>;
 def gpulibc : Flag<["-"], "gpulibc">, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Link the LLVM C Library for GPUs">;
 def nogpulibc : Flag<["-"], "nogpulibc">, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>;
-def nodefaultlibs : Flag<["-"], "nodefaultlibs">;
+def nodefaultlibs : Flag<["-"], "nodefaultlibs">,
+  Visibility<[ClangOption, FlangOption, CLOption, DXCOption]>;
 def nodriverkitlib : Flag<["-"], "nodriverkitlib">;
 def nofixprebinding : Flag<["-"], "nofixprebinding">;
 def nolibc : Flag<["-"], "nolibc">;
@@ -5592,7 +5593,9 @@ def nostdincxx : Flag<["-"], "nostdinc++">, Visibility<[ClangOption, CC1Option]>
   Group<IncludePath_Group>,
   HelpText<"Disable standard #include directories for the C++ standard library">,
   MarshallingInfoNegativeFlag<HeaderSearchOpts<"UseStandardCXXIncludes">>;
-def nostdlib : Flag<["-"], "nostdlib">, Group<Link_Group>;
+def nostdlib : Flag<["-"], "nostdlib">,
+  Visibility<[ClangOption, CLOption, FlangOption, DXCOption]>,
+  Group<Link_Group>;
 def nostdlibxx : Flag<["-"], "nostdlib++">;
 def object : Flag<["-"], "object">;
 def o : JoinedOrSeparate<["-"], "o">,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 99eef472223a00..0809ac1b144ef6 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -7957,6 +7957,10 @@ class Sema final : public SemaBase {
   /// Do an explicit extend of the given block pointer if we're in ARC.
   void maybeExtendBlockObject(ExprResult &E);
 
+  std::vector<std::pair<QualType, unsigned>> ExcessPrecisionNotSatisfied;
+  SourceLocation LocationOfExcessPrecisionNotSatisfied;
+  void DiagnosePrecisionLossInComplexDivision();
+
 private:
   static BinaryOperatorKind ConvertTokenKindToBinaryOpcode(tok::TokenKind Kind);
 
diff --git a/clang/include/clang/Sema/SemaHLSL.h b/clang/include/clang/Sema/SemaHLSL.h
index 64b39ca7712eeb..e088254c566d3e 100644
--- a/clang/include/clang/Sema/SemaHLSL.h
+++ b/clang/include/clang/Sema/SemaHLSL.h
@@ -16,6 +16,7 @@
 #include "clang/AST/ASTFwd.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/Type.h"
+#include "clang/AST/TypeLoc.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Sema/SemaBase.h"
 #include "llvm/ADT/SmallVector.h"
@@ -30,9 +31,9 @@ class Scope;
 
 // FIXME: This can be hidden (as static function in SemaHLSL.cpp) once we no
 // longer need to create builtin buffer types in HLSLExternalSemaSource.
-bool CreateHLSLAttributedResourceType(Sema &S, QualType Wrapped,
-                                      ArrayRef<const Attr *> AttrList,
-                                      QualType &ResType);
+bool CreateHLSLAttributedResourceType(
+    Sema &S, QualType Wrapped, ArrayRef<const Attr *> AttrList,
+    QualType &ResType, HLSLAttributedResourceLocInfo *LocInfo = nullptr);
 
 class SemaHLSL : public SemaBase {
 public:
@@ -73,7 +74,8 @@ class SemaHLSL : public SemaBase {
 
   bool CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall);
   QualType ProcessResourceTypeAttributes(QualType Wrapped);
-  SourceLocation TakeLocForHLSLAttribute(const HLSLAttributedResourceType *RT);
+  HLSLAttributedResourceLocInfo
+  TakeLocForHLSLAttribute(const HLSLAttributedResourceType *RT);
 
   // HLSL Type trait implementations
   bool IsScalarizedLayoutCompatible(QualType T1, QualType T2) const;
@@ -90,9 +92,10 @@ class SemaHLSL : public SemaBase {
   // This is a list to collect them.
   llvm::SmallVector<const Attr *> HLSLResourcesTypeAttrs;
 
-  /// SourceLocation corresponding to HLSLAttributedResourceTypeLocs that we
+  /// TypeLoc data for HLSLAttributedResourceType instances that we
   /// have not yet populated.
-  llvm::DenseMap<const HLSLAttributedResourceType *, SourceLocation>
+  llvm::DenseMap<const HLSLAttributedResourceType *,
+                 HLSLAttributedResourceLocInfo>
       LocsForHLSLAttributedResources;
 };
 
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 7cc69ca4a8a814..67841a30a571f3 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -2243,8 +2243,7 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
     Align = 8;                                                                 \
     break;
 #include "clang/Basic/WebAssemblyReferenceTypes.def"
-#define AMDGPU_OPAQUE_PTR_TYPE(NAME, MANGLEDNAME, AS, WIDTH, ALIGN, ID,        \
-                               SINGLETONID)                                    \
+#define AMDGPU_OPAQUE_PTR_TYPE(NAME, AS, WIDTH, ALIGN, ID, SINGLETONID)        \
   case BuiltinType::ID:                                                        \
     Width = WIDTH;                                                             \
     Align = ALIGN;                                                             \
diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp
index f13ca2d08d769f..21f0562f9d72ae 100644
--- a/clang/lib/AST/ASTStructuralEquivalence.cpp
+++ b/clang/lib/AST/ASTStructuralEquivalence.cpp
@@ -808,8 +808,8 @@ static bool
 IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                          const HLSLAttributedResourceType::Attributes &Attrs1,
                          const HLSLAttributedResourceType::Attributes &Attrs2) {
-  return std::tie(Attrs1.ResourceClass, Attrs1.IsROV) ==
-         std::tie(Attrs2.ResourceClass, Attrs2.IsROV);
+  return std::tie(Attrs1.ResourceClass, Attrs1.IsROV, Attrs1.RawBuffer) ==
+         std::tie(Attrs2.ResourceClass, Attrs2.IsROV, Attrs2.RawBuffer);
 }
 
 /// Determine structural equivalence of two types.
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index 3d507e2e2ba764..52ccefee88642a 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -159,8 +159,10 @@ bool CallBI(InterpState &S, CodePtr OpPC, const Function *Func,
 bool CallPtr(InterpState &S, CodePtr OpPC, uint32_t ArgSize,
              const CallExpr *CE);
 
+enum class ShiftDir { Left, Right };
+
 /// Checks if the shift operation is legal.
-template <typename LT, typename RT>
+template <ShiftDir Dir, typename LT, typename RT>
 bool CheckShift(InterpState &S, CodePtr OpPC, const LT &LHS, const RT &RHS,
                 unsigned Bits) {
   if (RHS.isNegative()) {
@@ -181,19 +183,21 @@ bool CheckShift(InterpState &S, CodePtr OpPC, const LT &LHS, const RT &RHS,
       return false;
   }
 
-  if (LHS.isSigned() && !S.getLangOpts().CPlusPlus20) {
-    const Expr *E = S.Current->getExpr(OpPC);
-    // C++11 [expr.shift]p2: A signed left shift must have a non-negative
-    // operand, and must not overflow the corresponding unsigned type.
-    if (LHS.isNegative()) {
-      S.CCEDiag(E, diag::note_constexpr_lshift_of_negative) << LHS.toAPSInt();
-      if (!S.noteUndefinedBehavior())
-        return false;
-    } else if (LHS.toUnsigned().countLeadingZeros() <
-               static_cast<unsigned>(RHS)) {
-      S.CCEDiag(E, diag::note_constexpr_lshift_discards);
-      if (!S.noteUndefinedBehavior())
-        return false;
+  if constexpr (Dir == ShiftDir::Left) {
+    if (LHS.isSigned() && !S.getLangOpts().CPlusPlus20) {
+      const Expr *E = S.Current->getExpr(OpPC);
+      // C++11 [expr.shift]p2: A signed left shift must have a non-negative
+      // operand, and must not overflow the corresponding unsigned type.
+      if (LHS.isNegative()) {
+        S.CCEDiag(E, diag::note_constexpr_lshift_of_negative) << LHS.toAPSInt();
+        if (!S.noteUndefinedBehavior())
+          return false;
+      } else if (LHS.toUnsigned().countLeadingZeros() <
+                 static_cast<unsigned>(RHS)) {
+        S.CCEDiag(E, diag::note_constexpr_lshift_discards);
+        if (!S.noteUndefinedBehavior())
+          return false;
+      }
     }
   }
 
@@ -2394,7 +2398,6 @@ inline bool RVOPtr(InterpState &S, CodePtr OpPC) {
 //===----------------------------------------------------------------------===//
 // Shr, Shl
 //===----------------------------------------------------------------------===//
-enum class ShiftDir { Left, Right };
 
 template <class LT, class RT, ShiftDir Dir>
 inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS) {
@@ -2431,7 +2434,7 @@ inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS) {
     }
   }
 
-  if (!CheckShift(S, OpPC, LHS, RHS, Bits))
+  if (!CheckShift<Dir>(S, OpPC, LHS, RHS, Bits))
     return false;
 
   // Limit the shift amount to Bits - 1. If this happened,
diff --git a/clang/lib/AST/ByteCode/Program.cpp b/clang/lib/AST/ByteCode/Program.cpp
index bd5860beabaecd..79c645257306e0 100644
--- a/clang/lib/AST/ByteCode/Program.cpp
+++ b/clang/lib/AST/ByteCode/Program.cpp
@@ -204,9 +204,18 @@ std::optional<unsigned> Program::createGlobal(const ValueDecl *VD,
     IsStatic = false;
     IsExtern = true;
   }
+
+  // Register all previous declarations as well. For extern blocks, just replace
+  // the index with the new variable.
   if (auto Idx = createGlobal(VD, VD->getType(), IsStatic, IsExtern, Init)) {
-    for (const Decl *P = VD; P; P = P->getPreviousDecl())
+    for (const Decl *P = VD; P; P = P->getPreviousDecl()) {
+      if (P != VD) {
+        unsigned PIdx = GlobalIndices[P];
+        if (Globals[PIdx]->block()->isExtern())
+          Globals[PIdx] = Globals[*Idx];
+      }
       GlobalIndices[P] = *Idx;
+    }
     return *Idx;
   }
   return std::nullopt;
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 4af7752d3b238b..6387e375dda79c 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -2146,7 +2146,7 @@ static bool IsLiteralLValue(const LValue &Value) {
   if (Value.getLValueCallIndex())
     return false;
   const Expr *E = Value.Base.dyn_cast<const Expr*>();
-  return E && !isa<MaterializeTemporaryExpr, StringLiteral>(E);
+  return E && !isa<MaterializeTemporaryExpr>(E);
 }
 
 static bool IsWeakLValue(const LValue &Value) {
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index be627a6242eb40..ca75bb97c158e1 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -1945,6 +1945,8 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
 
   case attr::HLSLResourceClass:
   case attr::HLSLROV:
+  case attr::HLSLRawBuffer:
+  case attr::HLSLContainedType:
     llvm_unreachable("HLSL resource type attributes handled separately");
 
   case attr::OpenCLPrivateAddressSpace:
@@ -2078,6 +2080,16 @@ void TypePrinter::printHLSLAttributedResourceAfter(
      << ")]]";
   if (Attrs.IsROV)
     OS << " [[hlsl::is_rov]]";
+  if (Attrs.RawBuffer)
+    OS << " [[hlsl::raw_buffer]]";
+
+  QualType ContainedTy = T->getContainedType();
+  if (!ContainedTy.isNull()) {
+    OS << " [[hlsl::contained_type(";
+    printBefore(ContainedTy, OS);
+    printAfter(ContainedTy, OS);
+    OS << ")]]";
+  }
 }
 
 void TypePrinter::printObjCInterfaceBefore(const ObjCInterfaceType *T,
diff --git a/clang/lib/ASTMatchers/ASTMatchFinder.cpp b/clang/lib/ASTMatchers/ASTMatchFinder.cpp
index 0bac2ed63a927e..3d01a70395a9bb 100644
--- a/clang/lib/ASTMatchers/ASTMatchFinder.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchFinder.cpp
@@ -285,12 +285,13 @@ class MatchChildASTVisitor
     ScopedIncrement ScopedDepth(&CurrentDepth);
 
     for (unsigned I = 0, N = Node->capture_size(); I != N; ++I) {
-      const auto *C = Node->capture_begin() + I;
+      const LambdaCapture *C = Node->capture_begin() + I;
       if (!C->isExplicit())
         continue;
       if (Node->isInitCapture(C) && !match(*C->getCapturedVar()))
         return false;
-      if (!match(*Node->capture_init_begin()[I]))
+      const Expr *CIE = Node->capture_init_begin()[I];
+      if (CIE != nullptr && !match(*CIE))
         return false;
     }
 
diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp
index a00b166a8b4f93..a16762244b1766 100644
--- a/clang/lib/Analysis/UnsafeBufferUsage.cpp
+++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp
@@ -560,13 +560,22 @@ static bool hasUnsafeFormatOrSArg(const CallExpr *Call, const Expr *&UnsafeArg,
   const Expr *Fmt = Call->getArg(FmtArgIdx);
 
   if (auto *SL = dyn_cast<StringLiteral>(Fmt->IgnoreParenImpCasts())) {
-    StringRef FmtStr = SL->getString();
+    StringRef FmtStr;
+
+    if (SL->getCharByteWidth() == 1)
+      FmtStr = SL->getString();
+    else if (auto EvaledFmtStr = SL->tryEvaluateString(Ctx))
+      FmtStr = *EvaledFmtStr;
+    else
+      goto CHECK_UNSAFE_PTR;
+
     StringFormatStringHandler Handler(Call, FmtArgIdx, UnsafeArg);
 
     return analyze_format_string::ParsePrintfString(
         Handler, FmtStr.begin(), FmtStr.end(), Ctx.getLangOpts(),
         Ctx.getTargetInfo(), isKprintf);
   }
+CHECK_UNSAFE_PTR:
   // If format is not a string literal, we cannot analyze the format string.
   // In this case, this call is considered unsafe if at least one argument
   // (including the format argument) is unsafe pointer.
diff --git a/clang/lib/Basic/LangStandards.cpp b/clang/lib/Basic/LangStandards.cpp
index b9b914b0adc772..214567a53efe95 100644
--- a/clang/lib/Basic/LangStandards.cpp
+++ b/clang/lib/Basic/LangStandards.cpp
@@ -117,7 +117,7 @@ LangStandard::Kind clang::getDefaultLanguageStandard(clang::Language Lang,
   case Language::RenderScript:
     return LangStandard::lang_c99;
   case Language::HLSL:
-    return LangStandard::lang_hlsl2021;
+    return LangStandard::lang_hlsl202x;
   }
   llvm_unreachable("unhandled Language kind!");
 }
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 63fc15f916c558..5f5dfcb722f9d4 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -399,7 +399,13 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
   Builder.defineMacro("__AARCH64_CMODEL_" + CodeModel + "__");
 
   // ACLE predefines. Many can only have one possible value on v8 AArch64.
-  Builder.defineMacro("__ARM_ACLE", "200");
+  Builder.defineMacro("__ARM_ACLE_VERSION(year, quarter, patch)",
+                      "(100 * (year) + 10 * (quarter) + (patch))");
+#define ARM_ACLE_VERSION(Y, Q, P) (100 * (Y) + 10 * (Q) + (P))
+  Builder.defineMacro("__ARM_ACLE", Twine(ARM_ACLE_VERSION(2024, 2, 0)));
+  Builder.defineMacro("__FUNCTION_MULTI_VERSIONING_SUPPORT_LEVEL",
+                      Twine(ARM_ACLE_VERSION(2024, 2, 0)));
+#undef ARM_ACLE_VERSION
   Builder.defineMacro("__ARM_ARCH",
                       std::to_string(ArchInfo->Version.getMajor()));
   Builder.defineMacro("__ARM_ARCH_PROFILE",
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 6c433cf6a85d5e..2d2c280941bd64 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -900,13 +900,12 @@ llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) {
     return SingletonId;                                                        \
   }
 #include "clang/Basic/WebAssemblyReferenceTypes.def"
-#define AMDGPU_OPAQUE_PTR_TYPE(Name, MangledName, AS, Width, Align, Id,        \
-                               SingletonId)                                    \
+#define AMDGPU_OPAQUE_PTR_TYPE(Name, AS, Width, Align, Id, SingletonId)        \
   case BuiltinType::Id: {                                                      \
     if (!SingletonId)                                                          \
       SingletonId =                                                            \
-          DBuilder.createForwardDecl(llvm::dwarf::DW_TAG_structure_type,       \
-                                     MangledName, TheCU, TheCU->getFile(), 0); \
+          DBuilder.createForwardDecl(llvm::dwarf::DW_TAG_structure_type, Name, \
+                                     TheCU, TheCU->getFile(), 0);              \
     return SingletonId;                                                        \
   }
 #include "clang/Basic/AMDGPUTypes.def"
diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp
index 828a09856099ac..fef26e7b4ccdbd 100644
--- a/clang/lib/CodeGen/CGExprComplex.cpp
+++ b/clang/lib/CodeGen/CGExprComplex.cpp
@@ -288,28 +288,15 @@ class ComplexExprEmitter
   ComplexPairTy EmitComplexBinOpLibCall(StringRef LibCallName,
                                         const BinOpInfo &Op);
 
-  QualType GetHigherPrecisionFPType(QualType ElementType) {
-    const auto *CurrentBT = cast<BuiltinType>(ElementType);
-    switch (CurrentBT->getKind()) {
-    case BuiltinType::Kind::Float16:
-      return CGF.getContext().FloatTy;
-    case BuiltinType::Kind::Float:
-    case BuiltinType::Kind::BFloat16:
-      return CGF.getContext().DoubleTy;
-    case BuiltinType::Kind::Double:
-      return CGF.getContext().LongDoubleTy;
-    default:
-      return ElementType;
-    }
-  }
-
   QualType HigherPrecisionTypeForComplexArithmetic(QualType ElementType,
                                                    bool IsDivOpCode) {
-    QualType HigherElementType = GetHigherPrecisionFPType(ElementType);
+    ASTContext &Ctx = CGF.getContext();
+    const QualType HigherElementType =
+        Ctx.GetHigherPrecisionFPType(ElementType);
     const llvm::fltSemantics &ElementTypeSemantics =
-        CGF.getContext().getFloatTypeSemantics(ElementType);
+        Ctx.getFloatTypeSemantics(ElementType);
     const llvm::fltSemantics &HigherElementTypeSemantics =
-        CGF.getContext().getFloatTypeSemantics(HigherElementType);
+        Ctx.getFloatTypeSemantics(HigherElementType);
     // Check that the promoted type can handle the intermediate values without
     // overflowing. This can be interpreted as:
     // (SmallerType.LargestFiniteVal * SmallerType.LargestFiniteVal) * 2 <=
@@ -320,10 +307,10 @@ class ComplexExprEmitter
     if (llvm::APFloat::semanticsMaxExponent(ElementTypeSemantics) * 2 + 1 <=
         llvm::APFloat::semanticsMaxExponent(HigherElementTypeSemantics)) {
       FPHasBeenPromoted = true;
-      return CGF.getContext().getComplexType(HigherElementType);
+      return Ctx.getComplexType(HigherElementType);
     } else {
-      DiagnosticsEngine &Diags = CGF.CGM.getDiags();
-      Diags.Report(diag::warn_next_larger_fp_type_same_size_than_fp);
+      // The intermediate values can't be represented in the promoted type
+      // without overflowing.
       return QualType();
     }
   }
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp
index b6e6555e63fca1..59d8fc830dcc8f 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.cpp
+++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp
@@ -338,6 +338,14 @@ void clang::CodeGen::CGHLSLRuntime::setHLSLEntryAttributes(
                 NumThreadsAttr->getZ());
     Fn->addFnAttr(NumThreadsKindStr, NumThreadsStr);
   }
+  if (HLSLWaveSizeAttr *WaveSizeAttr = FD->getAttr<HLSLWaveSizeAttr>()) {
+    const StringRef WaveSizeKindStr = "hlsl.wavesize";
+    std::string WaveSizeStr =
+        formatv("{0},{1},{2}", WaveSizeAttr->getMin(), WaveSizeAttr->getMax(),
+                WaveSizeAttr->getPreferred());
+    Fn->addFnAttr(WaveSizeKindStr, WaveSizeStr);
+  }
+  Fn->addFnAttr(llvm::Attribute::NoInline);
 }
 
 static Value *buildVectorInput(IRBuilder<> &B, Function *F, llvm::Type *Ty) {
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index ba2d6588900a11..17b82b205063d4 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -2473,11 +2473,14 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
     B.addAttribute(llvm::Attribute::StackProtectReq);
 
   if (!D) {
+    // Non-entry HLSL functions must always be inlined.
+    if (getLangOpts().HLSL && !F->hasFnAttribute(llvm::Attribute::NoInline))
+      B.addAttribute(llvm::Attribute::AlwaysInline);
     // If we don't have a declaration to control inlining, the function isn't
     // explicitly marked as alwaysinline for semantic reasons, and inlining is
     // disabled, mark the function as noinline.
-    if (!F->hasFnAttribute(llvm::Attribute::AlwaysInline) &&
-        CodeGenOpts.getInlining() == CodeGenOptions::OnlyAlwaysInlining)
+    else if (!F->hasFnAttribute(llvm::Attribute::AlwaysInline) &&
+             CodeGenOpts.getInlining() == CodeGenOptions::OnlyAlwaysInlining)
       B.addAttribute(llvm::Attribute::NoInline);
 
     F->addFnAttrs(B);
@@ -2504,9 +2507,13 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
   ShouldAddOptNone &= !D->hasAttr<MinSizeAttr>();
   ShouldAddOptNone &= !D->hasAttr<AlwaysInlineAttr>();
 
-  // Add optnone, but do so only if the function isn't always_inline.
-  if ((ShouldAddOptNone || D->hasAttr<OptimizeNoneAttr>()) &&
-      !F->hasFnAttribute(llvm::Attribute::AlwaysInline)) {
+  // Non-entry HLSL functions must always be inlined.
+  if (getLangOpts().HLSL && !F->hasFnAttribute(llvm::Attribute::NoInline) &&
+      !D->hasAttr<NoInlineAttr>()) {
+    B.addAttribute(llvm::Attribute::AlwaysInline);
+  } else if ((ShouldAddOptNone || D->hasAttr<OptimizeNoneAttr>()) &&
+             !F->hasFnAttribute(llvm::Attribute::AlwaysInline)) {
+    // Add optnone, but do so only if the function isn't always_inline.
     B.addAttribute(llvm::Attribute::OptimizeNone);
 
     // OptimizeNone implies noinline; we should not be inlining such functions.
@@ -2526,7 +2533,8 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
     B.addAttribute(llvm::Attribute::NoInline);
   } else if (D->hasAttr<NoDuplicateAttr>()) {
     B.addAttribute(llvm::Attribute::NoDuplicate);
-  } else if (D->hasAttr<NoInlineAttr>() && !F->hasFnAttribute(llvm::Attribute::AlwaysInline)) {
+  } else if (D->hasAttr<NoInlineAttr>() &&
+             !F->hasFnAttribute(llvm::Attribute::AlwaysInline)) {
     // Add noinline if the function isn't always_inline.
     B.addAttribute(llvm::Attribute::NoInline);
   } else if (D->hasAttr<AlwaysInlineAttr>() &&
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index d087e714882b33..77c1b27cebf401 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -558,8 +558,7 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
       llvm_unreachable("Unexpected wasm reference builtin type!");             \
   } break;
 #include "clang/Basic/WebAssemblyReferenceTypes.def"
-#define AMDGPU_OPAQUE_PTR_TYPE(Name, MangledName, AS, Width, Align, Id,        \
-                               SingletonId)                                    \
+#define AMDGPU_OPAQUE_PTR_TYPE(Name, AS, Width, Align, Id, SingletonId)        \
   case BuiltinType::Id:                                                        \
     return llvm::PointerType::get(getLLVMContext(), AS);
 #include "clang/Basic/AMDGPUTypes.def"
diff --git a/clang/lib/Driver/ToolChains/AIX.cpp b/clang/lib/Driver/ToolChains/AIX.cpp
index c2de7328c25c5d..09a8dc2f4fa5dd 100644
--- a/clang/lib/Driver/ToolChains/AIX.cpp
+++ b/clang/lib/Driver/ToolChains/AIX.cpp
@@ -330,7 +330,8 @@ void aix::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     }
   }
 
-  if (D.IsFlangMode()) {
+  if (D.IsFlangMode() &&
+      !Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
     addFortranRuntimeLibraryPath(ToolChain, Args, CmdArgs);
     addFortranRuntimeLibs(ToolChain, Args, CmdArgs);
     CmdArgs.push_back("-lm");
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 3fe4ce5d893b8d..494883500342e4 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -6004,7 +6004,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   if (Args.hasFlag(options::OPT_fms_volatile, options::OPT_fno_ms_volatile,
-                   Triple.isX86() && D.IsCLMode()))
+                   Triple.isX86() && IsWindowsMSVC))
     CmdArgs.push_back("-fms-volatile");
 
   // Non-PIC code defaults to -fdirect-access-external-data while PIC code
diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp
index ebc9ed1aadb0ab..87380869f6fdab 100644
--- a/clang/lib/Driver/ToolChains/Darwin.cpp
+++ b/clang/lib/Driver/ToolChains/Darwin.cpp
@@ -713,7 +713,8 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   // Additional linker set-up and flags for Fortran. This is required in order
   // to generate executables.
-  if (getToolChain().getDriver().IsFlangMode()) {
+  if (getToolChain().getDriver().IsFlangMode() &&
+      !Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
     addFortranRuntimeLibraryPath(getToolChain(), Args, CmdArgs);
     addFortranRuntimeLibs(getToolChain(), Args, CmdArgs);
   }
diff --git a/clang/lib/Driver/ToolChains/DragonFly.cpp b/clang/lib/Driver/ToolChains/DragonFly.cpp
index 1dbc46763c1156..1e0a4159bf4ad7 100644
--- a/clang/lib/Driver/ToolChains/DragonFly.cpp
+++ b/clang/lib/Driver/ToolChains/DragonFly.cpp
@@ -151,7 +151,8 @@ void dragonfly::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     // to generate executables. As Fortran runtime depends on the C runtime,
     // these dependencies need to be listed before the C runtime below (i.e.
     // AddRunTimeLibs).
-    if (D.IsFlangMode()) {
+    if (D.IsFlangMode() &&
+        !Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
       addFortranRuntimeLibraryPath(ToolChain, Args, CmdArgs);
       addFortranRuntimeLibs(ToolChain, Args, CmdArgs);
       CmdArgs.push_back("-lm");
diff --git a/clang/lib/Driver/ToolChains/FreeBSD.cpp b/clang/lib/Driver/ToolChains/FreeBSD.cpp
index a8ee6540001ee4..3d744bc087f467 100644
--- a/clang/lib/Driver/ToolChains/FreeBSD.cpp
+++ b/clang/lib/Driver/ToolChains/FreeBSD.cpp
@@ -313,7 +313,8 @@ void freebsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     // to generate executables. As Fortran runtime depends on the C runtime,
     // these dependencies need to be listed before the C runtime below (i.e.
     // AddRunTimeLibs).
-    if (D.IsFlangMode()) {
+    if (D.IsFlangMode() &&
+        !Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
       addFortranRuntimeLibraryPath(ToolChain, Args, CmdArgs);
       addFortranRuntimeLibs(ToolChain, Args, CmdArgs);
       if (Profiling)
diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp
index b7ae0deeb1219b..603d0468dd3f3c 100644
--- a/clang/lib/Driver/ToolChains/Gnu.cpp
+++ b/clang/lib/Driver/ToolChains/Gnu.cpp
@@ -572,7 +572,8 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   // to generate executables. As Fortran runtime depends on the C runtime,
   // these dependencies need to be listed before the C runtime below (i.e.
   // AddRunTimeLibs).
-  if (D.IsFlangMode()) {
+  if (D.IsFlangMode() &&
+      !Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
     addFortranRuntimeLibraryPath(ToolChain, Args, CmdArgs);
     addFortranRuntimeLibs(ToolChain, Args, CmdArgs);
     CmdArgs.push_back("-lm");
diff --git a/clang/lib/Driver/ToolChains/Haiku.cpp b/clang/lib/Driver/ToolChains/Haiku.cpp
index 346652a7e4bd8e..af74f43e48364c 100644
--- a/clang/lib/Driver/ToolChains/Haiku.cpp
+++ b/clang/lib/Driver/ToolChains/Haiku.cpp
@@ -119,7 +119,8 @@ void haiku::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     // to generate executables. As Fortran runtime depends on the C runtime,
     // these dependencies need to be listed before the C runtime below (i.e.
     // AddRunTimeLibs).
-    if (D.IsFlangMode()) {
+    if (D.IsFlangMode() &&
+        !Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
       addFortranRuntimeLibraryPath(ToolChain, Args, CmdArgs);
       addFortranRuntimeLibs(ToolChain, Args, CmdArgs);
     }
diff --git a/clang/lib/Driver/ToolChains/MSVC.cpp b/clang/lib/Driver/ToolChains/MSVC.cpp
index 7eb38098bed25b..80799d1e715f07 100644
--- a/clang/lib/Driver/ToolChains/MSVC.cpp
+++ b/clang/lib/Driver/ToolChains/MSVC.cpp
@@ -138,7 +138,8 @@ void visualstudio::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     for (const auto &LibPath : Args.getAllArgValues(options::OPT_L))
       CmdArgs.push_back(Args.MakeArgString("-libpath:" + LibPath));
 
-  if (C.getDriver().IsFlangMode()) {
+  if (C.getDriver().IsFlangMode() &&
+      !Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
     addFortranRuntimeLibraryPath(TC, Args, CmdArgs);
     addFortranRuntimeLibs(TC, Args, CmdArgs);
 
diff --git a/clang/lib/Driver/ToolChains/MinGW.cpp b/clang/lib/Driver/ToolChains/MinGW.cpp
index 85f40893e54247..e51daca5025a80 100644
--- a/clang/lib/Driver/ToolChains/MinGW.cpp
+++ b/clang/lib/Driver/ToolChains/MinGW.cpp
@@ -251,7 +251,8 @@ void tools::MinGW::Linker::ConstructJob(Compilation &C, const JobAction &JA,
                   D.getLTOMode() == LTOK_Thin);
   }
 
-  if (C.getDriver().IsFlangMode()) {
+  if (C.getDriver().IsFlangMode() &&
+      !Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
     addFortranRuntimeLibraryPath(TC, Args, CmdArgs);
     addFortranRuntimeLibs(TC, Args, CmdArgs);
   }
diff --git a/clang/lib/Driver/ToolChains/NetBSD.cpp b/clang/lib/Driver/ToolChains/NetBSD.cpp
index d54f2288294949..abd5e1aa003b38 100644
--- a/clang/lib/Driver/ToolChains/NetBSD.cpp
+++ b/clang/lib/Driver/ToolChains/NetBSD.cpp
@@ -326,7 +326,8 @@ void netbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     // to generate executables. As Fortran runtime depends on the C runtime,
     // these dependencies need to be listed before the C runtime below (i.e.
     // AddRunTimeLibs).
-    if (D.IsFlangMode()) {
+    if (D.IsFlangMode() &&
+        !Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
       addFortranRuntimeLibraryPath(ToolChain, Args, CmdArgs);
       addFortranRuntimeLibs(ToolChain, Args, CmdArgs);
       CmdArgs.push_back("-lm");
diff --git a/clang/lib/Driver/ToolChains/OpenBSD.cpp b/clang/lib/Driver/ToolChains/OpenBSD.cpp
index 3770471bae7c0d..f668a11e78f81d 100644
--- a/clang/lib/Driver/ToolChains/OpenBSD.cpp
+++ b/clang/lib/Driver/ToolChains/OpenBSD.cpp
@@ -239,7 +239,8 @@ void openbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     // to generate executables. As Fortran runtime depends on the C runtime,
     // these dependencies need to be listed before the C runtime below (i.e.
     // AddRunTimeLibs).
-    if (D.IsFlangMode()) {
+    if (D.IsFlangMode() &&
+        !Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
       addFortranRuntimeLibraryPath(ToolChain, Args, CmdArgs);
       addFortranRuntimeLibs(ToolChain, Args, CmdArgs);
       if (Profiling)
diff --git a/clang/lib/Driver/ToolChains/PS4CPU.cpp b/clang/lib/Driver/ToolChains/PS4CPU.cpp
index 48d824171303cc..647580e4e235dc 100644
--- a/clang/lib/Driver/ToolChains/PS4CPU.cpp
+++ b/clang/lib/Driver/ToolChains/PS4CPU.cpp
@@ -135,8 +135,8 @@ void tools::PS4cpu::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   // handled somewhere else.
   Args.ClaimAllArgs(options::OPT_w);
 
-  if (!D.SysRoot.empty())
-    CmdArgs.push_back(Args.MakeArgString("--sysroot=" + D.SysRoot));
+  CmdArgs.push_back(
+      Args.MakeArgString("--sysroot=" + TC.getSDKLibraryRootDir()));
 
   if (Args.hasArg(options::OPT_pie))
     CmdArgs.push_back("-pie");
@@ -234,8 +234,8 @@ void tools::PS5cpu::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   // handled somewhere else.
   Args.ClaimAllArgs(options::OPT_w);
 
-  if (!D.SysRoot.empty())
-    CmdArgs.push_back(Args.MakeArgString("--sysroot=" + D.SysRoot));
+  CmdArgs.push_back(
+      Args.MakeArgString("--sysroot=" + TC.getSDKLibraryRootDir()));
 
   // Default to PIE for non-static executables.
   const bool PIE =
@@ -325,46 +325,63 @@ toolchains::PS4PS5Base::PS4PS5Base(const Driver &D, const llvm::Triple &Triple,
                                    const ArgList &Args, StringRef Platform,
                                    const char *EnvVar)
     : Generic_ELF(D, Triple, Args) {
-  // Determine where to find the PS4/PS5 libraries.
-  // If -isysroot was passed, use that as the SDK base path.
-  // If not, we use the EnvVar if it exists; otherwise use the driver's
-  // installation path, which should be <SDK_DIR>/host_tools/bin.
+  // Determine the baseline SDK directory from the environment, else
+  // the driver's location, which should be <SDK_DIR>/host_tools/bin.
+  SmallString<128> SDKRootDir;
   SmallString<80> Whence;
-  if (const Arg *A = Args.getLastArg(options::OPT_isysroot)) {
-    SDKRootDir = A->getValue();
-    if (!llvm::sys::fs::exists(SDKRootDir))
-      D.Diag(clang::diag::warn_missing_sysroot) << SDKRootDir;
-    Whence = A->getSpelling();
-  } else if (const char *EnvValue = getenv(EnvVar)) {
+  if (const char *EnvValue = getenv(EnvVar)) {
     SDKRootDir = EnvValue;
-    Whence = { "environment variable '", EnvVar, "'" };
+    Whence = {"environment variable '", EnvVar, "'"};
   } else {
     SDKRootDir = D.Dir + "/../../";
     Whence = "compiler's location";
   }
 
-  SmallString<512> SDKIncludeDir(SDKRootDir);
-  llvm::sys::path::append(SDKIncludeDir, "target/include");
-  if (!Args.hasArg(options::OPT_nostdinc) &&
-      !Args.hasArg(options::OPT_nostdlibinc) &&
-      !Args.hasArg(options::OPT_isysroot) &&
-      !Args.hasArg(options::OPT__sysroot_EQ) &&
-      !llvm::sys::fs::exists(SDKIncludeDir)) {
-    D.Diag(clang::diag::warn_drv_unable_to_find_directory_expected)
-        << Twine(Platform, " system headers").str() << SDKIncludeDir << Whence;
-  }
+  // Allow --sysroot= to override the root directory for header and library
+  // search, and -sysroot to override header search. If both are specified,
+  // -isysroot overrides --sysroot for header search.
+  auto OverrideRoot = [&](const options::ID &Opt, std::string &Root,
+                          StringRef Default) {
+    if (const Arg *A = Args.getLastArg(Opt)) {
+      Root = A->getValue();
+      if (!llvm::sys::fs::exists(Root))
+        D.Diag(clang::diag::warn_missing_sysroot) << Root;
+      return true;
+    }
+    Root = Default.str();
+    return false;
+  };
 
-  SmallString<512> SDKLibDir(SDKRootDir);
-  llvm::sys::path::append(SDKLibDir, "target/lib");
-  if (!Args.hasArg(options::OPT__sysroot_EQ) && !Args.hasArg(options::OPT_E) &&
-      !Args.hasArg(options::OPT_c) && !Args.hasArg(options::OPT_S) &&
-      !Args.hasArg(options::OPT_emit_ast) &&
-      !llvm::sys::fs::exists(SDKLibDir)) {
+  bool CustomSysroot =
+      OverrideRoot(options::OPT__sysroot_EQ, SDKLibraryRootDir, SDKRootDir);
+  bool CustomISysroot =
+      OverrideRoot(options::OPT_isysroot, SDKHeaderRootDir, SDKLibraryRootDir);
+
+  // Emit warnings if parts of the SDK are missing, unless the user has taken
+  // control of header or library search. If we're not linking, don't check
+  // for missing libraries.
+  auto CheckSDKPartExists = [&](StringRef Dir, StringRef Desc) {
+    if (llvm::sys::fs::exists(Dir))
+      return true;
     D.Diag(clang::diag::warn_drv_unable_to_find_directory_expected)
-        << Twine(Platform, " system libraries").str() << SDKLibDir << Whence;
-    return;
+        << (Twine(Platform) + " " + Desc).str() << Dir << Whence;
+    return false;
+  };
+
+  bool Linking = !Args.hasArg(options::OPT_E, options::OPT_c, options::OPT_S,
+                              options::OPT_emit_ast);
+  if (!CustomSysroot && Linking) {
+    SmallString<128> Dir(SDKLibraryRootDir);
+    llvm::sys::path::append(Dir, "target/lib");
+    if (CheckSDKPartExists(Dir, "system libraries"))
+      getFilePaths().push_back(std::string(Dir));
+  }
+  if (!CustomSysroot && !CustomISysroot &&
+      !Args.hasArg(options::OPT_nostdinc, options::OPT_nostdlibinc)) {
+    SmallString<128> Dir(SDKHeaderRootDir);
+    llvm::sys::path::append(Dir, "target/include");
+    CheckSDKPartExists(Dir, "system headers");
   }
-  getFilePaths().push_back(std::string(SDKLibDir));
 }
 
 void toolchains::PS4PS5Base::AddClangSystemIncludeArgs(
@@ -385,9 +402,9 @@ void toolchains::PS4PS5Base::AddClangSystemIncludeArgs(
     return;
 
   addExternCSystemInclude(DriverArgs, CC1Args,
-                          SDKRootDir + "/target/include");
+                          SDKHeaderRootDir + "/target/include");
   addExternCSystemInclude(DriverArgs, CC1Args,
-                          SDKRootDir + "/target/include_common");
+                          SDKHeaderRootDir + "/target/include_common");
 }
 
 Tool *toolchains::PS4CPU::buildAssembler() const {
diff --git a/clang/lib/Driver/ToolChains/PS4CPU.h b/clang/lib/Driver/ToolChains/PS4CPU.h
index a33728bce5186a..456e966c2d400a 100644
--- a/clang/lib/Driver/ToolChains/PS4CPU.h
+++ b/clang/lib/Driver/ToolChains/PS4CPU.h
@@ -128,9 +128,12 @@ class LLVM_LIBRARY_VISIBILITY PS4PS5Base : public Generic_ELF {
                                 const char *Suffix) const = 0;
   virtual const char *getProfileRTLibName() const = 0;
 
+  StringRef getSDKLibraryRootDir() const { return SDKLibraryRootDir; }
+
 private:
-  // We compute the SDK root dir in the ctor, and use it later.
-  std::string SDKRootDir;
+  // We compute the SDK locations in the ctor, and use them later.
+  std::string SDKHeaderRootDir;
+  std::string SDKLibraryRootDir;
 };
 
 // PS4-specific Toolchain class.
diff --git a/clang/lib/Driver/ToolChains/Solaris.cpp b/clang/lib/Driver/ToolChains/Solaris.cpp
index e82ed2ca79ffd6..cf39038dcac37c 100644
--- a/clang/lib/Driver/ToolChains/Solaris.cpp
+++ b/clang/lib/Driver/ToolChains/Solaris.cpp
@@ -223,7 +223,8 @@ void solaris::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     // Additional linker set-up and flags for Fortran. This is required in order
     // to generate executables. As Fortran runtime depends on the C runtime,
     // these dependencies need to be listed before the C runtime below.
-    if (D.IsFlangMode()) {
+    if (D.IsFlangMode() &&
+        !Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
       addFortranRuntimeLibraryPath(getToolChain(), Args, CmdArgs);
       addFortranRuntimeLibs(getToolChain(), Args, CmdArgs);
       CmdArgs.push_back("-lm");
diff --git a/clang/lib/Sema/HLSLExternalSemaSource.cpp b/clang/lib/Sema/HLSLExternalSemaSource.cpp
index da7bbf8baa74df..d19f79b6ddefcd 100644
--- a/clang/lib/Sema/HLSLExternalSemaSource.cpp
+++ b/clang/lib/Sema/HLSLExternalSemaSource.cpp
@@ -16,6 +16,7 @@
 #include "clang/AST/Type.h"
 #include "clang/Basic/AttrKinds.h"
 #include "clang/Basic/HLSLRuntime.h"
+#include "clang/Basic/SourceLocation.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Sema.h"
 #include "clang/Sema/SemaHLSL.h"
@@ -111,22 +112,35 @@ struct BuiltinTypeDeclBuilder {
 
   BuiltinTypeDeclBuilder &
   addHandleMember(Sema &S, ResourceClass RC, ResourceKind RK, bool IsROV,
+                  bool RawBuffer,
                   AccessSpecifier Access = AccessSpecifier::AS_private) {
     if (Record->isCompleteDefinition())
       return *this;
+
+    TypeSourceInfo *ElementTypeInfo = nullptr;
+
     QualType Ty = Record->getASTContext().VoidPtrTy;
     if (Template) {
       if (const auto *TTD = dyn_cast<TemplateTypeParmDecl>(
-              Template->getTemplateParameters()->getParam(0)))
+              Template->getTemplateParameters()->getParam(0))) {
         Ty = Record->getASTContext().getPointerType(
             QualType(TTD->getTypeForDecl(), 0));
+        QualType ElemType = QualType(TTD->getTypeForDecl(), 0);
+        ElementTypeInfo = S.getASTContext().getTrivialTypeSourceInfo(
+            ElemType, SourceLocation());
+      }
     }
 
     // add handle member with resource type attributes
     QualType AttributedResTy = QualType();
     SmallVector<const Attr *> Attrs = {
         HLSLResourceClassAttr::CreateImplicit(Record->getASTContext(), RC),
-        IsROV ? HLSLROVAttr::CreateImplicit(Record->getASTContext()) : nullptr};
+        IsROV ? HLSLROVAttr::CreateImplicit(Record->getASTContext()) : nullptr,
+        RawBuffer ? HLSLRawBufferAttr::CreateImplicit(Record->getASTContext())
+                  : nullptr,
+        ElementTypeInfo ? HLSLContainedTypeAttr::CreateImplicit(
+                              Record->getASTContext(), ElementTypeInfo)
+                        : nullptr};
     Attr *ResourceAttr =
         HLSLResourceAttr::CreateImplicit(Record->getASTContext(), RK);
     if (CreateHLSLAttributedResourceType(S, Ty, Attrs, AttributedResTy))
@@ -495,9 +509,9 @@ void HLSLExternalSemaSource::defineTrivialHLSLTypes() {
 /// Set up common members and attributes for buffer types
 static BuiltinTypeDeclBuilder setupBufferType(CXXRecordDecl *Decl, Sema &S,
                                               ResourceClass RC, ResourceKind RK,
-                                              bool IsROV) {
+                                              bool IsROV, bool RawBuffer) {
   return BuiltinTypeDeclBuilder(Decl)
-      .addHandleMember(S, RC, RK, IsROV)
+      .addHandleMember(S, RC, RK, IsROV, RawBuffer)
       .addDefaultHandleConstructor(S, RC);
 }
 
@@ -510,7 +524,7 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() {
   onCompletion(Decl, [this](CXXRecordDecl *Decl) {
     setupBufferType(Decl, *SemaPtr, ResourceClass::UAV,
                     ResourceKind::TypedBuffer,
-                    /*IsROV=*/false)
+                    /*IsROV=*/false, /*RawBuffer=*/false)
         .addArraySubscriptOperators()
         .completeDefinition();
   });
@@ -521,7 +535,8 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() {
           .Record;
   onCompletion(Decl, [this](CXXRecordDecl *Decl) {
     setupBufferType(Decl, *SemaPtr, ResourceClass::UAV,
-                    ResourceKind::TypedBuffer, /*IsROV=*/true)
+                    ResourceKind::TypedBuffer, /*IsROV=*/true,
+                    /*RawBuffer=*/false)
         .addArraySubscriptOperators()
         .completeDefinition();
   });
@@ -531,7 +546,8 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() {
              .Record;
   onCompletion(Decl, [this](CXXRecordDecl *Decl) {
     setupBufferType(Decl, *SemaPtr, ResourceClass::UAV,
-                    ResourceKind::TypedBuffer, /*IsROV=*/false)
+                    ResourceKind::TypedBuffer, /*IsROV=*/false,
+                    /*RawBuffer=*/true)
         .addArraySubscriptOperators()
         .completeDefinition();
   });
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index d567de703903c9..85cbbe7750c2b3 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -1213,6 +1213,7 @@ void Sema::ActOnEndOfTranslationUnit() {
   DiagnoseUnterminatedPragmaAlignPack();
   DiagnoseUnterminatedPragmaAttribute();
   OpenMP().DiagnoseUnterminatedOpenMPDeclareTarget();
+  DiagnosePrecisionLossInComplexDivision();
 
   // All delayed member exception specs should be checked or we end up accepting
   // incompatible declarations.
diff --git a/clang/lib/Sema/SemaAttr.cpp b/clang/lib/Sema/SemaAttr.cpp
index fb594e6f13c0b3..cf2a5a622a3a4d 100644
--- a/clang/lib/Sema/SemaAttr.cpp
+++ b/clang/lib/Sema/SemaAttr.cpp
@@ -1180,6 +1180,16 @@ void Sema::PrintPragmaAttributeInstantiationPoint() {
                diag::note_pragma_attribute_applied_decl_here);
 }
 
+void Sema::DiagnosePrecisionLossInComplexDivision() {
+  for (auto &[Type, Num] : ExcessPrecisionNotSatisfied) {
+    assert(LocationOfExcessPrecisionNotSatisfied.isValid() &&
+           "expected a valid source location");
+    Diag(LocationOfExcessPrecisionNotSatisfied,
+         diag::warn_excess_precision_not_supported)
+        << static_cast<bool>(Num);
+  }
+}
+
 void Sema::DiagnoseUnterminatedPragmaAttribute() {
   if (PragmaAttributeStack.empty())
     return;
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 8557c25b93a8da..31bf50a32a83c3 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -9768,7 +9768,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
     bool ImplicitInlineCXX20 = !getLangOpts().CPlusPlusModules ||
                                NewFD->isConstexpr() || NewFD->isConsteval() ||
                                !NewFD->getOwningModule() ||
-                               NewFD->isFromExplicitGlobalModule() ||
+                               NewFD->isFromGlobalModule() ||
                                NewFD->getOwningModule()->isHeaderLikeModule();
     bool isInline = D.getDeclSpec().isInlineSpecified();
     bool isVirtual = D.getDeclSpec().isVirtualSpecified();
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 8f3e15cc9a9bb7..80c252c79e4d7a 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -15115,6 +15115,37 @@ static void DiagnoseBinOpPrecedence(Sema &Self, BinaryOperatorKind Opc,
     DiagnoseShiftCompare(Self, OpLoc, LHSExpr, RHSExpr);
 }
 
+static void DetectPrecisionLossInComplexDivision(Sema &S, SourceLocation OpLoc,
+                                                 Expr *Operand) {
+  if (auto *CT = Operand->getType()->getAs<ComplexType>()) {
+    QualType ElementType = CT->getElementType();
+    bool IsComplexRangePromoted = S.getLangOpts().getComplexRange() ==
+                                  LangOptions::ComplexRangeKind::CX_Promoted;
+    if (ElementType->isFloatingType() && IsComplexRangePromoted) {
+      ASTContext &Ctx = S.getASTContext();
+      QualType HigherElementType = Ctx.GetHigherPrecisionFPType(ElementType);
+      const llvm::fltSemantics &ElementTypeSemantics =
+          Ctx.getFloatTypeSemantics(ElementType);
+      const llvm::fltSemantics &HigherElementTypeSemantics =
+          Ctx.getFloatTypeSemantics(HigherElementType);
+      if (llvm::APFloat::semanticsMaxExponent(ElementTypeSemantics) * 2 + 1 >
+          llvm::APFloat::semanticsMaxExponent(HigherElementTypeSemantics)) {
+        // Retain the location of the first use of higher precision type.
+        if (!S.LocationOfExcessPrecisionNotSatisfied.isValid())
+          S.LocationOfExcessPrecisionNotSatisfied = OpLoc;
+        for (auto &[Type, Num] : S.ExcessPrecisionNotSatisfied) {
+          if (Type == HigherElementType) {
+            Num++;
+            return;
+          }
+        }
+        S.ExcessPrecisionNotSatisfied.push_back(std::make_pair(
+            HigherElementType, S.ExcessPrecisionNotSatisfied.size()));
+      }
+    }
+  }
+}
+
 ExprResult Sema::ActOnBinOp(Scope *S, SourceLocation TokLoc,
                             tok::TokenKind Kind,
                             Expr *LHSExpr, Expr *RHSExpr) {
@@ -15125,6 +15156,11 @@ ExprResult Sema::ActOnBinOp(Scope *S, SourceLocation TokLoc,
   // Emit warnings for tricky precedence issues, e.g. "bitfield & 0x4 == 0"
   DiagnoseBinOpPrecedence(*this, Opc, TokLoc, LHSExpr, RHSExpr);
 
+  // Emit warnings if the requested higher precision type equal to the current
+  // type precision.
+  if (Kind == tok::TokenKind::slash)
+    DetectPrecisionLossInComplexDivision(*this, TokLoc, LHSExpr);
+
   return BuildBinOp(S, TokLoc, Opc, LHSExpr, RHSExpr);
 }
 
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 527718c8e7e324..a303f211501348 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -10,12 +10,15 @@
 
 #include "clang/Sema/SemaHLSL.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Attr.h"
+#include "clang/AST/Attrs.inc"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclBase.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/Type.h"
+#include "clang/AST/TypeLoc.h"
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/LLVM.h"
@@ -564,18 +567,23 @@ void SemaHLSL::handleShaderAttr(Decl *D, const ParsedAttr &AL) {
     D->addAttr(NewAttr);
 }
 
-bool clang::CreateHLSLAttributedResourceType(Sema &S, QualType Wrapped,
-                                             ArrayRef<const Attr *> AttrList,
-                                             QualType &ResType) {
+bool clang::CreateHLSLAttributedResourceType(
+    Sema &S, QualType Wrapped, ArrayRef<const Attr *> AttrList,
+    QualType &ResType, HLSLAttributedResourceLocInfo *LocInfo) {
   assert(AttrList.size() && "expected list of resource attributes");
 
-  QualType Contained = QualType();
-  HLSLAttributedResourceType::Attributes ResAttrs = {};
+  QualType ContainedTy = QualType();
+  TypeSourceInfo *ContainedTyInfo = nullptr;
+  SourceLocation LocBegin = AttrList[0]->getRange().getBegin();
+  SourceLocation LocEnd = AttrList[0]->getRange().getEnd();
+
+  HLSLAttributedResourceType::Attributes ResAttrs;
 
   bool HasResourceClass = false;
   for (const Attr *A : AttrList) {
     if (!A)
       continue;
+    LocEnd = A->getRange().getEnd();
     switch (A->getKind()) {
     case attr::HLSLResourceClass: {
       llvm::dxil::ResourceClass RC =
@@ -598,6 +606,27 @@ bool clang::CreateHLSLAttributedResourceType(Sema &S, QualType Wrapped,
       }
       ResAttrs.IsROV = true;
       break;
+    case attr::HLSLRawBuffer:
+      if (ResAttrs.RawBuffer) {
+        S.Diag(A->getLocation(), diag::warn_duplicate_attribute_exact) << A;
+        return false;
+      }
+      ResAttrs.RawBuffer = true;
+      break;
+    case attr::HLSLContainedType: {
+      const HLSLContainedTypeAttr *CTAttr = cast<HLSLContainedTypeAttr>(A);
+      QualType Ty = CTAttr->getType();
+      if (!ContainedTy.isNull()) {
+        S.Diag(A->getLocation(), ContainedTy == Ty
+                                     ? diag::warn_duplicate_attribute_exact
+                                     : diag::warn_duplicate_attribute)
+            << A;
+        return false;
+      }
+      ContainedTy = Ty;
+      ContainedTyInfo = CTAttr->getTypeLoc();
+      break;
+    }
     default:
       llvm_unreachable("unhandled resource attribute type");
     }
@@ -609,8 +638,13 @@ bool clang::CreateHLSLAttributedResourceType(Sema &S, QualType Wrapped,
     return false;
   }
 
-  ResType = S.getASTContext().getHLSLAttributedResourceType(Wrapped, Contained,
-                                                            ResAttrs);
+  ResType = S.getASTContext().getHLSLAttributedResourceType(
+      Wrapped, ContainedTy, ResAttrs);
+
+  if (LocInfo && ContainedTyInfo) {
+    LocInfo->Range = SourceRange(LocBegin, LocEnd);
+    LocInfo->ContainedTyInfo = ContainedTyInfo;
+  }
   return true;
 }
 
@@ -647,9 +681,31 @@ bool SemaHLSL::handleResourceTypeAttr(const ParsedAttr &AL) {
     A = HLSLResourceClassAttr::Create(getASTContext(), RC, AL.getLoc());
     break;
   }
+
   case ParsedAttr::AT_HLSLROV:
     A = HLSLROVAttr::Create(getASTContext(), AL.getLoc());
     break;
+
+  case ParsedAttr::AT_HLSLRawBuffer:
+    A = HLSLRawBufferAttr::Create(getASTContext(), AL.getLoc());
+    break;
+
+  case ParsedAttr::AT_HLSLContainedType: {
+    if (AL.getNumArgs() != 1 && !AL.hasParsedType()) {
+      Diag(AL.getLoc(), diag::err_attribute_wrong_number_arguments) << AL << 1;
+      return false;
+    }
+
+    TypeSourceInfo *TSI = nullptr;
+    QualType QT = SemaRef.GetTypeFromParser(AL.getTypeArg(), &TSI);
+    assert(TSI && "no type source info for attribute argument");
+    if (SemaRef.RequireCompleteType(TSI->getTypeLoc().getBeginLoc(), QT,
+                                    diag::err_incomplete_type))
+      return false;
+    A = HLSLContainedTypeAttr::Create(getASTContext(), TSI, AL.getLoc());
+    break;
+  }
+
   default:
     llvm_unreachable("unhandled HLSL attribute");
   }
@@ -664,30 +720,34 @@ QualType SemaHLSL::ProcessResourceTypeAttributes(QualType CurrentType) {
     return CurrentType;
 
   QualType QT = CurrentType;
+  HLSLAttributedResourceLocInfo LocInfo;
   if (CreateHLSLAttributedResourceType(SemaRef, CurrentType,
-                                       HLSLResourcesTypeAttrs, QT)) {
+                                       HLSLResourcesTypeAttrs, QT, &LocInfo)) {
     const HLSLAttributedResourceType *RT =
-        dyn_cast<HLSLAttributedResourceType>(QT.getTypePtr());
-    // Use the location of the first attribute as the location of the aggregated
-    // type. The attributes are stored in HLSLResourceTypeAttrs in the same
-    // order as they are parsed.
-    SourceLocation Loc = HLSLResourcesTypeAttrs[0]->getLoc();
-    LocsForHLSLAttributedResources.insert(std::pair(RT, Loc));
+        cast<HLSLAttributedResourceType>(QT.getTypePtr());
+
+    // Temporarily store TypeLoc information for the new type.
+    // It will be transferred to HLSLAttributesResourceTypeLoc
+    // shortly after the type is created by TypeSpecLocFiller which
+    // will call the TakeLocForHLSLAttribute method below.
+    LocsForHLSLAttributedResources.insert(std::pair(RT, LocInfo));
   }
   HLSLResourcesTypeAttrs.clear();
   return QT;
 }
 
 // Returns source location for the HLSLAttributedResourceType
-SourceLocation
+HLSLAttributedResourceLocInfo
 SemaHLSL::TakeLocForHLSLAttribute(const HLSLAttributedResourceType *RT) {
+  HLSLAttributedResourceLocInfo LocInfo = {};
   auto I = LocsForHLSLAttributedResources.find(RT);
   if (I != LocsForHLSLAttributedResources.end()) {
-    SourceLocation Loc = I->second;
+    LocInfo = I->second;
     LocsForHLSLAttributedResources.erase(I);
-    return Loc;
+    return LocInfo;
   }
-  return SourceLocation();
+  LocInfo.Range = SourceRange();
+  return LocInfo;
 }
 
 struct RegisterBindingFlags {
@@ -909,7 +969,8 @@ static void ValidateMultipleRegisterAnnotations(Sema &S, Decl *TheDecl,
 }
 
 static void DiagnoseHLSLRegisterAttribute(Sema &S, SourceLocation &ArgLoc,
-                                          Decl *TheDecl, RegisterType regType) {
+                                          Decl *TheDecl, RegisterType RegType,
+                                          const bool SpecifiedSpace) {
 
   // exactly one of these two types should be set
   assert(((isa<VarDecl>(TheDecl) && !isa<HLSLBufferDecl>(TheDecl)) ||
@@ -922,42 +983,46 @@ static void DiagnoseHLSLRegisterAttribute(Sema &S, SourceLocation &ArgLoc,
              1 &&
          "only one resource analysis result should be expected");
 
-  int regTypeNum = static_cast<int>(regType);
+  int RegTypeNum = static_cast<int>(RegType);
 
   // first, if "other" is set, emit an error
   if (Flags.Other) {
-    S.Diag(ArgLoc, diag::err_hlsl_binding_type_mismatch) << regTypeNum;
+    S.Diag(ArgLoc, diag::err_hlsl_binding_type_mismatch) << RegTypeNum;
     return;
   }
 
   // next, if multiple register annotations exist, check that none conflict.
-  ValidateMultipleRegisterAnnotations(S, TheDecl, regType);
+  ValidateMultipleRegisterAnnotations(S, TheDecl, RegType);
 
   // next, if resource is set, make sure the register type in the register
   // annotation is compatible with the variable's resource type.
   if (Flags.Resource) {
-    RegisterType expRegType = getRegisterType(Flags.ResourceClass.value());
-    if (regType != expRegType) {
+    RegisterType ExpRegType = getRegisterType(Flags.ResourceClass.value());
+    if (RegType != ExpRegType) {
       S.Diag(TheDecl->getLocation(), diag::err_hlsl_binding_type_mismatch)
-          << regTypeNum;
+          << RegTypeNum;
     }
+
     return;
   }
 
   // next, handle diagnostics for when the "basic" flag is set
   if (Flags.Basic) {
+    if (SpecifiedSpace && !isDeclaredWithinCOrTBuffer(TheDecl))
+      S.Diag(ArgLoc, diag::err_hlsl_space_on_global_constant);
+
     if (Flags.DefaultGlobals) {
-      if (regType == RegisterType::CBuffer)
+      if (RegType == RegisterType::CBuffer)
         S.Diag(ArgLoc, diag::warn_hlsl_deprecated_register_type_b);
-      else if (regType != RegisterType::C)
-        S.Diag(ArgLoc, diag::err_hlsl_binding_type_mismatch) << regTypeNum;
+      else if (RegType != RegisterType::C)
+        S.Diag(ArgLoc, diag::err_hlsl_binding_type_mismatch) << RegTypeNum;
       return;
     }
 
-    if (regType == RegisterType::C)
+    if (RegType == RegisterType::C)
       S.Diag(ArgLoc, diag::warn_hlsl_register_type_c_packoffset);
     else
-      S.Diag(ArgLoc, diag::err_hlsl_binding_type_mismatch) << regTypeNum;
+      S.Diag(ArgLoc, diag::err_hlsl_binding_type_mismatch) << RegTypeNum;
 
     return;
   }
@@ -966,15 +1031,13 @@ static void DiagnoseHLSLRegisterAttribute(Sema &S, SourceLocation &ArgLoc,
   if (Flags.UDT) {
     const bool ExpectedRegisterTypesForUDT[] = {
         Flags.SRV, Flags.UAV, Flags.CBV, Flags.Sampler, Flags.ContainsNumeric};
-    assert((size_t)regTypeNum < std::size(ExpectedRegisterTypesForUDT) &&
+    assert((size_t)RegTypeNum < std::size(ExpectedRegisterTypesForUDT) &&
            "regType has unexpected value");
 
-    if (!ExpectedRegisterTypesForUDT[regTypeNum])
+    if (!ExpectedRegisterTypesForUDT[RegTypeNum])
       S.Diag(TheDecl->getLocation(),
              diag::warn_hlsl_user_defined_type_missing_member)
-          << regTypeNum;
-
-    return;
+          << RegTypeNum;
   }
 }
 
@@ -999,7 +1062,9 @@ void SemaHLSL::handleResourceBindingAttr(Decl *TheDecl, const ParsedAttr &AL) {
   SourceLocation ArgLoc = Loc->Loc;
 
   SourceLocation SpaceArgLoc;
+  bool SpecifiedSpace = false;
   if (AL.getNumArgs() == 2) {
+    SpecifiedSpace = true;
     Slot = Str;
     if (!AL.isArgIdent(1)) {
       Diag(AL.getLoc(), diag::err_attribute_argument_type)
@@ -1047,7 +1112,8 @@ void SemaHLSL::handleResourceBindingAttr(Decl *TheDecl, const ParsedAttr &AL) {
     return;
   }
 
-  DiagnoseHLSLRegisterAttribute(SemaRef, ArgLoc, TheDecl, regType);
+  DiagnoseHLSLRegisterAttribute(SemaRef, ArgLoc, TheDecl, regType,
+                                SpecifiedSpace);
 
   HLSLResourceBindingAttr *NewAttr =
       HLSLResourceBindingAttr::Create(getASTContext(), Slot, Space, AL);
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index b952ffbd69f5d5..9afb8cea26fe78 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -2861,113 +2861,120 @@ void SemaOpenMP::EndOpenMPDSABlock(Stmt *CurDirective) {
   //  clause requires an accessible, unambiguous default constructor for the
   //  class type, unless the list item is also specified in a firstprivate
   //  clause.
-  if (const auto *D = dyn_cast_or_null<OMPExecutableDirective>(CurDirective)) {
-    for (OMPClause *C : D->clauses()) {
-      if (auto *Clause = dyn_cast<OMPLastprivateClause>(C)) {
-        SmallVector<Expr *, 8> PrivateCopies;
-        for (Expr *DE : Clause->varlist()) {
-          if (DE->isValueDependent() || DE->isTypeDependent()) {
-            PrivateCopies.push_back(nullptr);
-            continue;
-          }
-          auto *DRE = cast<DeclRefExpr>(DE->IgnoreParens());
-          auto *VD = cast<VarDecl>(DRE->getDecl());
-          QualType Type = VD->getType().getNonReferenceType();
-          const DSAStackTy::DSAVarData DVar =
-              DSAStack->getTopDSA(VD, /*FromParent=*/false);
-          if (DVar.CKind == OMPC_lastprivate) {
-            // Generate helper private variable and initialize it with the
-            // default value. The address of the original variable is replaced
-            // by the address of the new private variable in CodeGen. This new
-            // variable is not added to IdResolver, so the code in the OpenMP
-            // region uses original variable for proper diagnostics.
-            VarDecl *VDPrivate = buildVarDecl(
-                SemaRef, DE->getExprLoc(), Type.getUnqualifiedType(),
-                VD->getName(), VD->hasAttrs() ? &VD->getAttrs() : nullptr, DRE);
-            SemaRef.ActOnUninitializedDecl(VDPrivate);
-            if (VDPrivate->isInvalidDecl()) {
-              PrivateCopies.push_back(nullptr);
-              continue;
-            }
-            PrivateCopies.push_back(buildDeclRefExpr(
-                SemaRef, VDPrivate, DE->getType(), DE->getExprLoc()));
-          } else {
-            // The variable is also a firstprivate, so initialization sequence
-            // for private copy is generated already.
-            PrivateCopies.push_back(nullptr);
-          }
-        }
-        Clause->setPrivateCopies(PrivateCopies);
+
+  auto FinalizeLastprivate = [&](OMPLastprivateClause *Clause) {
+    SmallVector<Expr *, 8> PrivateCopies;
+    for (Expr *DE : Clause->varlist()) {
+      if (DE->isValueDependent() || DE->isTypeDependent()) {
+        PrivateCopies.push_back(nullptr);
         continue;
       }
-      // Finalize nontemporal clause by handling private copies, if any.
-      if (auto *Clause = dyn_cast<OMPNontemporalClause>(C)) {
-        SmallVector<Expr *, 8> PrivateRefs;
-        for (Expr *RefExpr : Clause->varlist()) {
-          assert(RefExpr && "NULL expr in OpenMP nontemporal clause.");
-          SourceLocation ELoc;
-          SourceRange ERange;
-          Expr *SimpleRefExpr = RefExpr;
-          auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange);
-          if (Res.second)
-            // It will be analyzed later.
-            PrivateRefs.push_back(RefExpr);
-          ValueDecl *D = Res.first;
-          if (!D)
-            continue;
-
-          const DSAStackTy::DSAVarData DVar =
-              DSAStack->getTopDSA(D, /*FromParent=*/false);
-          PrivateRefs.push_back(DVar.PrivateCopy ? DVar.PrivateCopy
-                                                 : SimpleRefExpr);
-        }
-        Clause->setPrivateRefs(PrivateRefs);
+      auto *DRE = cast<DeclRefExpr>(DE->IgnoreParens());
+      auto *VD = cast<VarDecl>(DRE->getDecl());
+      QualType Type = VD->getType().getNonReferenceType();
+      const DSAStackTy::DSAVarData DVar =
+          DSAStack->getTopDSA(VD, /*FromParent=*/false);
+      if (DVar.CKind != OMPC_lastprivate) {
+        // The variable is also a firstprivate, so initialization sequence
+        // for private copy is generated already.
+        PrivateCopies.push_back(nullptr);
         continue;
       }
-      if (auto *Clause = dyn_cast<OMPUsesAllocatorsClause>(C)) {
-        for (unsigned I = 0, E = Clause->getNumberOfAllocators(); I < E; ++I) {
-          OMPUsesAllocatorsClause::Data D = Clause->getAllocatorData(I);
-          auto *DRE = dyn_cast<DeclRefExpr>(D.Allocator->IgnoreParenImpCasts());
-          if (!DRE)
-            continue;
-          ValueDecl *VD = DRE->getDecl();
-          if (!VD || !isa<VarDecl>(VD))
-            continue;
-          DSAStackTy::DSAVarData DVar =
-              DSAStack->getTopDSA(VD, /*FromParent=*/false);
-          // OpenMP [2.12.5, target Construct]
-          // Memory allocators that appear in a uses_allocators clause cannot
-          // appear in other data-sharing attribute clauses or data-mapping
-          // attribute clauses in the same construct.
-          Expr *MapExpr = nullptr;
-          if (DVar.RefExpr ||
-              DSAStack->checkMappableExprComponentListsForDecl(
-                  VD, /*CurrentRegionOnly=*/true,
-                  [VD, &MapExpr](
-                      OMPClauseMappableExprCommon::MappableExprComponentListRef
-                          MapExprComponents,
-                      OpenMPClauseKind C) {
-                    auto MI = MapExprComponents.rbegin();
-                    auto ME = MapExprComponents.rend();
-                    if (MI != ME &&
-                        MI->getAssociatedDeclaration()->getCanonicalDecl() ==
-                            VD->getCanonicalDecl()) {
-                      MapExpr = MI->getAssociatedExpression();
-                      return true;
-                    }
-                    return false;
-                  })) {
-            Diag(D.Allocator->getExprLoc(),
-                 diag::err_omp_allocator_used_in_clauses)
-                << D.Allocator->getSourceRange();
-            if (DVar.RefExpr)
-              reportOriginalDsa(SemaRef, DSAStack, VD, DVar);
-            else
-              Diag(MapExpr->getExprLoc(), diag::note_used_here)
-                  << MapExpr->getSourceRange();
-          }
-        }
+      // Generate helper private variable and initialize it with the
+      // default value. The address of the original variable is replaced
+      // by the address of the new private variable in CodeGen. This new
+      // variable is not added to IdResolver, so the code in the OpenMP
+      // region uses original variable for proper diagnostics.
+      VarDecl *VDPrivate = buildVarDecl(
+          SemaRef, DE->getExprLoc(), Type.getUnqualifiedType(), VD->getName(),
+          VD->hasAttrs() ? &VD->getAttrs() : nullptr, DRE);
+      SemaRef.ActOnUninitializedDecl(VDPrivate);
+      if (VDPrivate->isInvalidDecl()) {
+        PrivateCopies.push_back(nullptr);
+        continue;
+      }
+      PrivateCopies.push_back(buildDeclRefExpr(
+          SemaRef, VDPrivate, DE->getType(), DE->getExprLoc()));
+    }
+    Clause->setPrivateCopies(PrivateCopies);
+  };
+
+  auto FinalizeNontemporal = [&](OMPNontemporalClause *Clause) {
+    // Finalize nontemporal clause by handling private copies, if any.
+    SmallVector<Expr *, 8> PrivateRefs;
+    for (Expr *RefExpr : Clause->varlist()) {
+      assert(RefExpr && "NULL expr in OpenMP nontemporal clause.");
+      SourceLocation ELoc;
+      SourceRange ERange;
+      Expr *SimpleRefExpr = RefExpr;
+      auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange);
+      if (Res.second)
+        // It will be analyzed later.
+        PrivateRefs.push_back(RefExpr);
+      ValueDecl *D = Res.first;
+      if (!D)
         continue;
+
+      const DSAStackTy::DSAVarData DVar =
+          DSAStack->getTopDSA(D, /*FromParent=*/false);
+      PrivateRefs.push_back(DVar.PrivateCopy ? DVar.PrivateCopy
+                                             : SimpleRefExpr);
+    }
+    Clause->setPrivateRefs(PrivateRefs);
+  };
+
+  auto FinalizeAllocators = [&](OMPUsesAllocatorsClause *Clause) {
+    for (unsigned I = 0, E = Clause->getNumberOfAllocators(); I < E; ++I) {
+      OMPUsesAllocatorsClause::Data D = Clause->getAllocatorData(I);
+      auto *DRE = dyn_cast<DeclRefExpr>(D.Allocator->IgnoreParenImpCasts());
+      if (!DRE)
+        continue;
+      ValueDecl *VD = DRE->getDecl();
+      if (!VD || !isa<VarDecl>(VD))
+        continue;
+      DSAStackTy::DSAVarData DVar =
+          DSAStack->getTopDSA(VD, /*FromParent=*/false);
+      // OpenMP [2.12.5, target Construct]
+      // Memory allocators that appear in a uses_allocators clause cannot
+      // appear in other data-sharing attribute clauses or data-mapping
+      // attribute clauses in the same construct.
+      Expr *MapExpr = nullptr;
+      if (DVar.RefExpr ||
+          DSAStack->checkMappableExprComponentListsForDecl(
+              VD, /*CurrentRegionOnly=*/true,
+              [VD, &MapExpr](
+                  OMPClauseMappableExprCommon::MappableExprComponentListRef
+                      MapExprComponents,
+                  OpenMPClauseKind C) {
+                auto MI = MapExprComponents.rbegin();
+                auto ME = MapExprComponents.rend();
+                if (MI != ME &&
+                    MI->getAssociatedDeclaration()->getCanonicalDecl() ==
+                        VD->getCanonicalDecl()) {
+                  MapExpr = MI->getAssociatedExpression();
+                  return true;
+                }
+                return false;
+              })) {
+        Diag(D.Allocator->getExprLoc(), diag::err_omp_allocator_used_in_clauses)
+            << D.Allocator->getSourceRange();
+        if (DVar.RefExpr)
+          reportOriginalDsa(SemaRef, DSAStack, VD, DVar);
+        else
+          Diag(MapExpr->getExprLoc(), diag::note_used_here)
+              << MapExpr->getSourceRange();
+      }
+    }
+  };
+
+  if (const auto *D = dyn_cast_or_null<OMPExecutableDirective>(CurDirective)) {
+    for (OMPClause *C : D->clauses()) {
+      if (auto *Clause = dyn_cast<OMPLastprivateClause>(C)) {
+        FinalizeLastprivate(Clause);
+      } else if (auto *Clause = dyn_cast<OMPNontemporalClause>(C)) {
+        FinalizeNontemporal(Clause);
+      } else if (auto *Clause = dyn_cast<OMPUsesAllocatorsClause>(C)) {
+        FinalizeAllocators(Clause);
       }
     }
     // Check allocate clauses.
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index e627fee51b66b8..950bd6db0359d1 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -5166,6 +5166,14 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
               if (ParamTy.hasQualifiers())
                 S.Diag(DeclType.Loc, diag::err_void_param_qualified);
 
+              // Reject, but continue to parse 'float(this void)' as
+              // 'float(void)'.
+              if (Param->isExplicitObjectParameter()) {
+                S.Diag(Param->getLocation(),
+                       diag::err_void_explicit_object_param);
+                Param->setExplicitObjectParameterLoc(SourceLocation());
+              }
+
               // Do not add 'void' to the list.
               break;
             }
@@ -5801,8 +5809,10 @@ static void fillAttributedTypeLoc(AttributedTypeLoc TL,
 
 static void fillHLSLAttributedResourceTypeLoc(HLSLAttributedResourceTypeLoc TL,
                                               TypeProcessingState &State) {
-  TL.setSourceRange(
-      State.getSema().HLSL().TakeLocForHLSLAttribute(TL.getTypePtr()));
+  HLSLAttributedResourceLocInfo LocInfo =
+      State.getSema().HLSL().TakeLocForHLSLAttribute(TL.getTypePtr());
+  TL.setSourceRange(LocInfo.Range);
+  TL.setContainedTypeSourceInfo(LocInfo.ContainedTyInfo);
 }
 
 static void fillMatrixTypeLoc(MatrixTypeLoc MTL,
@@ -8843,7 +8853,9 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
       break;
     }
     case ParsedAttr::AT_HLSLResourceClass:
-    case ParsedAttr::AT_HLSLROV: {
+    case ParsedAttr::AT_HLSLROV:
+    case ParsedAttr::AT_HLSLRawBuffer:
+    case ParsedAttr::AT_HLSLContainedType: {
       // Only collect HLSL resource type attributes that are in
       // decl-specifier-seq; do not collect attributes on declarations or those
       // that get to slide after declaration name.
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index ff745b3385fcd9..c37fa2fcc94815 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -7477,8 +7477,17 @@ QualType TreeTransform<Derived>::TransformHLSLAttributedResourceType(
     return QualType();
 
   QualType ContainedTy = QualType();
-  if (!oldType->getContainedType().isNull())
-    ContainedTy = getDerived().TransformType(TLB, TL.getContainedLoc());
+  QualType OldContainedTy = oldType->getContainedType();
+  if (!OldContainedTy.isNull()) {
+    TypeSourceInfo *oldContainedTSI = TL.getContainedTypeSourceInfo();
+    if (!oldContainedTSI)
+      oldContainedTSI = getSema().getASTContext().getTrivialTypeSourceInfo(
+          OldContainedTy, SourceLocation());
+    TypeSourceInfo *ContainedTSI = getDerived().TransformType(oldContainedTSI);
+    if (!ContainedTSI)
+      return QualType();
+    ContainedTy = ContainedTSI->getType();
+  }
 
   QualType Result = TL.getType();
   if (getDerived().AlwaysRebuild() || WrappedTy != oldType->getWrappedType() ||
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
index be07cf51eefb3d..394cb26f03cf99 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
@@ -12,6 +12,7 @@
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/ExprCXX.h"
+#include "clang/AST/ExprObjC.h"
 #include <optional>
 
 namespace clang {
@@ -35,6 +36,12 @@ bool tryToFindPtrOrigin(
         break;
       }
     }
+    if (auto *POE = dyn_cast<PseudoObjectExpr>(E)) {
+      if (auto *RF = POE->getResultExpr()) {
+        E = RF;
+        continue;
+      }
+    }
     if (auto *tempExpr = dyn_cast<ParenExpr>(E)) {
       E = tempExpr->getSubExpr();
       continue;
@@ -88,7 +95,7 @@ bool tryToFindPtrOrigin(
           continue;
         }
 
-        if (isReturnValueRefCounted(callee))
+        if (isRefType(callee->getReturnType()))
           return callback(E, true);
 
         if (isSingleton(callee))
@@ -100,6 +107,12 @@ bool tryToFindPtrOrigin(
         }
       }
     }
+    if (auto *ObjCMsgExpr = dyn_cast<ObjCMessageExpr>(E)) {
+      if (auto *Method = ObjCMsgExpr->getMethodDecl()) {
+        if (isRefType(Method->getReturnType()))
+          return callback(E, true);
+      }
+    }
     if (auto *unaryOp = dyn_cast<UnaryOperator>(E)) {
       // FIXME: Currently accepts ANY unary operator. Is it OK?
       E = unaryOp->getSubExpr();
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
index f48b2fd9dca71b..9da3e54e454317 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
@@ -123,9 +123,8 @@ bool isCtorOfRefCounted(const clang::FunctionDecl *F) {
          || FunctionName == "Identifier";
 }
 
-bool isReturnValueRefCounted(const clang::FunctionDecl *F) {
-  assert(F);
-  QualType type = F->getReturnType();
+bool isRefType(const clang::QualType T) {
+  QualType type = T;
   while (!type.isNull()) {
     if (auto *elaboratedT = type->getAs<ElaboratedType>()) {
       type = elaboratedT->desugar();
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h
index 2932e62ad06e4b..e2d0342bebd52c 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h
@@ -62,8 +62,8 @@ bool isRefType(const std::string &Name);
 /// false if not.
 bool isCtorOfRefCounted(const clang::FunctionDecl *F);
 
-/// \returns true if \p F returns a ref-counted object, false if not.
-bool isReturnValueRefCounted(const clang::FunctionDecl *F);
+/// \returns true if \p T is RefPtr, Ref, or its variant, false if not.
+bool isRefType(const clang::QualType T);
 
 /// \returns true if \p M is getter of a ref-counted class, false if not.
 std::optional<bool> isGetterOfRefCounted(const clang::CXXMethodDecl* Method);
diff --git a/clang/test/AST/ByteCode/cxx20.cpp b/clang/test/AST/ByteCode/cxx20.cpp
index b47a4b47d09644..9bbc3dbe0073d3 100644
--- a/clang/test/AST/ByteCode/cxx20.cpp
+++ b/clang/test/AST/ByteCode/cxx20.cpp
@@ -108,16 +108,22 @@ constexpr auto p2 = "test2";
 constexpr bool b1 = foo(p1) == foo(p1);
 static_assert(b1);
 
-constexpr bool b2 = foo(p1) == foo(p2);
-static_assert(!b2);
+constexpr bool b2 = foo(p1) == foo(p2); // ref-error {{must be initialized by a constant expression}} \
+                                        // ref-note {{comparison of addresses of literals}} \
+                                        // ref-note {{declared here}}
+static_assert(!b2); // ref-error {{not an integral constant expression}} \
+                    // ref-note {{not a constant expression}}
 
 constexpr auto name1() { return "name1"; }
 constexpr auto name2() { return "name2"; }
 
 constexpr auto b3 = name1() == name1();
 static_assert(b3);
-constexpr auto b4 = name1() == name2();
-static_assert(!b4);
+constexpr auto b4 = name1() == name2(); // ref-error {{must be initialized by a constant expression}} \
+                                        // ref-note {{has unspecified value}} \
+                                        // ref-note {{declared here}}
+static_assert(!b4); // ref-error {{not an integral constant expression}} \
+                    // ref-note {{not a constant expression}}
 
 namespace UninitializedFields {
   class A {
diff --git a/clang/test/AST/ByteCode/extern.cpp b/clang/test/AST/ByteCode/extern.cpp
new file mode 100644
index 00000000000000..a616269911a7ec
--- /dev/null
+++ b/clang/test/AST/ByteCode/extern.cpp
@@ -0,0 +1,13 @@
+// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -verify=both,expected %s
+// RUN: %clang_cc1 -verify=both,ref %s
+
+
+// both-no-diagnostics
+
+extern const int E;
+constexpr int getE() {
+  return E;
+}
+const int E = 10;
+static_assert(getE() == 10);
+
diff --git a/clang/test/AST/ByteCode/shifts.cpp b/clang/test/AST/ByteCode/shifts.cpp
index 360b87b7ee04f8..0b3383731c6774 100644
--- a/clang/test/AST/ByteCode/shifts.cpp
+++ b/clang/test/AST/ByteCode/shifts.cpp
@@ -5,6 +5,7 @@
 
 #define INT_MIN (~__INT_MAX__)
 
+constexpr int a = -1 >> 3;
 
 namespace shifts {
   constexpr void test() { // ref-error {{constexpr function never produces a constant expression}} \
diff --git a/clang/test/AST/HLSL/RWBuffer-AST.hlsl b/clang/test/AST/HLSL/RWBuffer-AST.hlsl
index 0e7803ce50a890..bad9961c10b02d 100644
--- a/clang/test/AST/HLSL/RWBuffer-AST.hlsl
+++ b/clang/test/AST/HLSL/RWBuffer-AST.hlsl
@@ -1,7 +1,6 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -DEMPTY %s | FileCheck -check-prefix=EMPTY %s
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump %s | FileCheck %s
 
-
 // This test tests two different AST generations. The "EMPTY" test mode verifies
 // the AST generated by forward declaration of the HLSL types which happens on
 // initializing the HLSL external AST with an AST Context.
@@ -30,7 +29,7 @@ RWBuffer<float> Buffer;
 // CHECK-NEXT: CXXRecordDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit class RWBuffer definition
 
 // CHECK: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
-// CHECK-NEXT: implicit h 'element_type * {{\[\[}}hlsl::resource_class(UAV)]]':'element_type *'
+// CHECK-NEXT: implicit h 'element_type * {{\[\[}}hlsl::resource_class(UAV)]] {{\[\[}}hlsl::contained_type(element_type)]]':'element_type *'
 // CHECK-NEXT: HLSLResourceAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit TypedBuffer
 
 // CHECK: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'element_type &const (unsigned int) const'
@@ -38,7 +37,7 @@ RWBuffer<float> Buffer;
 // CHECK-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
 // CHECK-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
 // CHECK-NEXT: ArraySubscriptExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' lvalue
-// CHECK-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type * {{\[\[}}hlsl::resource_class(UAV)]]':'element_type *' lvalue .h 0x{{[0-9A-Fa-f]+}}
+// CHECK-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type * {{\[\[}}hlsl::resource_class(UAV)]]  {{\[\[}}hlsl::contained_type(element_type)]]':'element_type *' lvalue .h 0x{{[0-9A-Fa-f]+}}
 // CHECK-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'const RWBuffer<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int' ParmVar 0x{{[0-9A-Fa-f]+}} 'Idx' 'unsigned int'
 // CHECK-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
@@ -48,7 +47,7 @@ RWBuffer<float> Buffer;
 // CHECK-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
 // CHECK-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
 // CHECK-NEXT: ArraySubscriptExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' lvalue
-// CHECK-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type * {{\[\[}}hlsl::resource_class(UAV)]]':'element_type *' lvalue .h 0x{{[0-9A-Fa-f]+}}
+// CHECK-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type * {{\[\[}}hlsl::resource_class(UAV)]]  {{\[\[}}hlsl::contained_type(element_type)]]':'element_type *' lvalue .h 0x{{[0-9A-Fa-f]+}}
 // CHECK-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'RWBuffer<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int' ParmVar 0x{{[0-9A-Fa-f]+}} 'Idx' 'unsigned int'
 // CHECK-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
@@ -58,5 +57,5 @@ RWBuffer<float> Buffer;
 // CHECK: TemplateArgument type 'float'
 // CHECK-NEXT: BuiltinType 0x{{[0-9A-Fa-f]+}} 'float'
 // CHECK-NEXT: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
-// CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit referenced h 'float * {{\[\[}}hlsl::resource_class(UAV)]]':'float *'
+// CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit referenced h 'float * {{\[\[}}hlsl::resource_class(UAV)]]  {{\[\[}}hlsl::contained_type(float)]]':'float *'
 // CHECK-NEXT: HLSLResourceAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit TypedBuffer
diff --git a/clang/test/AST/HLSL/StructuredBuffer-AST.hlsl b/clang/test/AST/HLSL/StructuredBuffer-AST.hlsl
index 11d84ac7b85db2..e50ba6a4b5b346 100644
--- a/clang/test/AST/HLSL/StructuredBuffer-AST.hlsl
+++ b/clang/test/AST/HLSL/StructuredBuffer-AST.hlsl
@@ -30,7 +30,7 @@ StructuredBuffer<float> Buffer;
 // CHECK-NEXT: CXXRecordDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit class StructuredBuffer definition
 
 // CHECK: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
-// CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit h 'element_type * {{\[\[}}hlsl::resource_class(UAV)]]':'element_type *'
+// CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit h 'element_type * {{\[\[}}hlsl::resource_class(UAV)]] {{\[\[}}hlsl::raw_buffer]] {{\[\[}}hlsl::contained_type(element_type)]]':'element_type *'
 // CHECK-NEXT: HLSLResourceAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit TypedBuffer
 
 // CHECK: CXXMethodDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> operator[] 'element_type &const (unsigned int) const'
@@ -38,7 +38,7 @@ StructuredBuffer<float> Buffer;
 // CHECK-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
 // CHECK-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
 // CHECK-NEXT: ArraySubscriptExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' lvalue
-// CHECK-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type * {{\[\[}}hlsl::resource_class(UAV)]]':'element_type *' lvalue .h 0x{{[0-9A-Fa-f]+}}
+// CHECK-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type * {{\[\[}}hlsl::resource_class(UAV)]] {{\[\[}}hlsl::raw_buffer]] {{\[\[}}hlsl::contained_type(element_type)]]':'element_type *' lvalue .h 0x{{[0-9A-Fa-f]+}}
 // CHECK-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'const StructuredBuffer<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int' ParmVar 0x{{[0-9A-Fa-f]+}} 'Idx' 'unsigned int'
 // CHECK-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
@@ -48,7 +48,7 @@ StructuredBuffer<float> Buffer;
 // CHECK-NEXT: CompoundStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
 // CHECK-NEXT: ReturnStmt 0x{{[0-9A-Fa-f]+}} <<invalid sloc>>
 // CHECK-NEXT: ArraySubscriptExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type' lvalue
-// CHECK-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type * {{\[\[}}hlsl::resource_class(UAV)]]':'element_type *' lvalue .h 0x{{[0-9A-Fa-f]+}}
+// CHECK-NEXT: MemberExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'element_type * {{\[\[}}hlsl::resource_class(UAV)]] {{\[\[}}hlsl::raw_buffer]] {{\[\[}}hlsl::contained_type(element_type)]]':'element_type *' lvalue .h 0x{{[0-9A-Fa-f]+}}
 // CHECK-NEXT: CXXThisExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'StructuredBuffer<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> 'unsigned int' ParmVar 0x{{[0-9A-Fa-f]+}} 'Idx' 'unsigned int'
 // CHECK-NEXT: AlwaysInlineAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit always_inline
@@ -58,5 +58,5 @@ StructuredBuffer<float> Buffer;
 // CHECK: TemplateArgument type 'float'
 // CHECK-NEXT: BuiltinType 0x{{[0-9A-Fa-f]+}} 'float'
 // CHECK-NEXT: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
-// CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit referenced h 'float * {{\[\[}}hlsl::resource_class(UAV)]]':'float *'
+// CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit referenced h 'float * {{\[\[}}hlsl::resource_class(UAV)]] {{\[\[}}hlsl::raw_buffer]] {{\[\[}}hlsl::contained_type(float)]]':'float *'
 // CHECK-NEXT: HLSLResourceAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit TypedBuffer
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.mm b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.mm
index db0c5b19eec5bb..9ad1880e9d1188 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.mm
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.mm
@@ -24,3 +24,20 @@ - (void)execute {
 }
 
 @end
+
+class RefCountedObject {
+public:
+  void ref() const;
+  void deref() const;
+  Ref<RefCountedObject> copy() const;
+};
+
+@interface WrapperObj : NSObject
+
+- (Ref<RefCountedObject>)_protectedWebExtensionControllerConfiguration;
+
+@end
+
+static void foo(WrapperObj *configuration) {
+  configuration._protectedWebExtensionControllerConfiguration->copy();
+}
diff --git a/clang/test/CXX/drs/cwg29xx.cpp b/clang/test/CXX/drs/cwg29xx.cpp
index ca598bb39a6c31..e55e8e35e86f28 100644
--- a/clang/test/CXX/drs/cwg29xx.cpp
+++ b/clang/test/CXX/drs/cwg29xx.cpp
@@ -6,6 +6,14 @@
 // RUN: %clang_cc1 -std=c++23 -pedantic-errors -verify=expected %s
 // RUN: %clang_cc1 -std=c++2c -pedantic-errors -verify=expected %s
 
+namespace cwg2915 { // cwg2915: 20 tentatively ready 2024-08-16
+#if __cplusplus >= 202302L
+struct A {
+  void f(this void); // expected-error {{explicit object parameter cannot have 'void' type}}
+};
+#endif
+}
+
 namespace cwg2917 { // cwg2917: 20 review 2024-07-30
 template <typename>
 class Foo;
diff --git a/clang/test/CodeGenHLSL/GlobalConstructorFunction.hlsl b/clang/test/CodeGenHLSL/GlobalConstructorFunction.hlsl
index f954c9d2f029f2..b39311ad67cd62 100644
--- a/clang/test/CodeGenHLSL/GlobalConstructorFunction.hlsl
+++ b/clang/test/CodeGenHLSL/GlobalConstructorFunction.hlsl
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s --check-prefixes=CHECK,NOINLINE
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -emit-llvm -O0 %s -o - | FileCheck %s --check-prefixes=CHECK,INLINE
 
 int i;
 
@@ -7,7 +8,7 @@ __attribute__((constructor)) void call_me_first(void) {
 }
 
 __attribute__((constructor)) void then_call_me(void) {
-  i = 12;
+  i = 13;
 }
 
 __attribute__((destructor)) void call_me_last(void) {
@@ -21,11 +22,21 @@ void main(unsigned GI : SV_GroupIndex) {}
 // CHECK-NOT:@llvm.global_ctors
 // CHECK-NOT:@llvm.global_dtors
 
-//CHECK: define void @main()
-//CHECK-NEXT: entry:
-//CHECK-NEXT:   call void @"?call_me_first@@YAXXZ"()
-//CHECK-NEXT:   call void @"?then_call_me@@YAXXZ"()
-//CHECK-NEXT:   %0 = call i32 @llvm.dx.flattened.thread.id.in.group()
-//CHECK-NEXT:   call void @"?main@@YAXI@Z"(i32 %0)
-//CHECK-NEXT:   call void @"?call_me_last@@YAXXZ"(
-//CHECK-NEXT:   ret void
+// CHECK: define void @main()
+// CHECK-NEXT: entry:
+// Verify function constructors are emitted
+// NOINLINE-NEXT:   call void @"?call_me_first@@YAXXZ"()
+// NOINLINE-NEXT:   call void @"?then_call_me@@YAXXZ"()
+// NOINLINE-NEXT:   %0 = call i32 @llvm.dx.flattened.thread.id.in.group()
+// NOINLINE-NEXT:   call void @"?main@@YAXI@Z"(i32 %0)
+// NOINLINE-NEXT:   call void @"?call_me_last@@YAXXZ"(
+// NOINLINE-NEXT:   ret void
+
+// Verify constructor calls are inlined when AlwaysInline is run
+// INLINE-NEXT:   alloca
+// INLINE-NEXT:   store i32 12
+// INLINE-NEXT:   store i32 13
+// INLINE-NEXT:   %0 = call i32 @llvm.dx.flattened.thread.id.in.group()
+// INLINE-NEXT:   store i32 %
+// INLINE-NEXT:   store i32 0
+// INLINE:   ret void
diff --git a/clang/test/CodeGenHLSL/GlobalConstructorLib.hlsl b/clang/test/CodeGenHLSL/GlobalConstructorLib.hlsl
index 2c5c4e19c3296d..78f6475462bc47 100644
--- a/clang/test/CodeGenHLSL/GlobalConstructorLib.hlsl
+++ b/clang/test/CodeGenHLSL/GlobalConstructorLib.hlsl
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s --check-prefixes=CHECK,NOINLINE
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -emit-llvm -O0 %s -o - | FileCheck %s --check-prefixes=CHECK,INLINE
 
 // Make sure global variable for ctors exist for lib profile.
 // CHECK:@llvm.global_ctors
@@ -11,7 +12,11 @@ void FirstEntry() {}
 
 // CHECK: define void @FirstEntry()
 // CHECK-NEXT: entry:
-// CHECK-NEXT:   call void @_GLOBAL__sub_I_GlobalConstructorLib.hlsl()
+// NOINLINE-NEXT:   call void @_GLOBAL__sub_I_GlobalConstructorLib.hlsl()
+// NOINLINE-NEXT:   call void @"?FirstEntry@@YAXXZ"()
+// Verify inlining leaves only calls to "llvm." intrinsics
+// INLINE-NOT:   call {{[^@]*}} @{{[^l][^l][^v][^m][^\.]}}
+// CHECK: ret void
 
 [shader("compute")]
 [numthreads(1,1,1)]
@@ -19,5 +24,15 @@ void SecondEntry() {}
 
 // CHECK: define void @SecondEntry()
 // CHECK-NEXT: entry:
-// CHECK-NEXT:   call void @_GLOBAL__sub_I_GlobalConstructorLib.hlsl()
-// CHECK-NEXT:   call void @"?SecondEntry@@YAXXZ"()
+// NOINLINE-NEXT:   call void @_GLOBAL__sub_I_GlobalConstructorLib.hlsl()
+// NOINLINE-NEXT:   call void @"?SecondEntry@@YAXXZ"()
+// Verify inlining leaves only calls to "llvm." intrinsics
+// INLINE-NOT:   call {{[^@]*}} @{{[^l][^l][^v][^m][^\.]}}
+// CHECK: ret void
+
+
+// Verify the constructor is alwaysinline
+// NOINLINE: ; Function Attrs: {{.*}}alwaysinline
+// NOINLINE-NEXT: define internal void @_GLOBAL__sub_I_GlobalConstructorLib.hlsl() [[IntAttr:\#[0-9]+]]
+
+// NOINLINE: attributes [[IntAttr]] = {{.*}} alwaysinline
diff --git a/clang/test/CodeGenHLSL/GlobalDestructors.hlsl b/clang/test/CodeGenHLSL/GlobalDestructors.hlsl
index 24c3c039fc6192..ea28354222f885 100644
--- a/clang/test/CodeGenHLSL/GlobalDestructors.hlsl
+++ b/clang/test/CodeGenHLSL/GlobalDestructors.hlsl
@@ -1,10 +1,18 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -std=hlsl202x -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s --check-prefixes=CS,CHECK
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -std=hlsl202x -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s --check-prefixes=LIB,CHECK
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -std=hlsl202x -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s --check-prefixes=CS,NOINLINE,CHECK
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -std=hlsl202x -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s --check-prefixes=LIB,NOINLINE,CHECK
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -std=hlsl202x -emit-llvm -O0 %s -o - | FileCheck %s --check-prefixes=INLINE,CHECK
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -std=hlsl202x -emit-llvm -O0 %s -o - | FileCheck %s --check-prefixes=INLINE,CHECK
 
-// Make sure global variable for dtors exist for lib profile.
+// Tests that constructors and destructors are appropriately generated for globals
+// and that their calls are inlined when AlwaysInline is run
+// but global variables are retained for the library profiles
+
+// Make sure global variable for ctors/dtors exist for lib profile.
+// LIB:@llvm.global_ctors
 // LIB:@llvm.global_dtors
-// Make sure global variable for dtors removed for compute profile.
-// CS-NOT:llvm.global_dtors
+// Make sure global variable for ctors/dtors removed for compute profile.
+// CS-NOT:@llvm.global_ctors
+// CS-NOT:@llvm.global_dtors
 
 struct Tail {
   Tail() {
@@ -46,22 +54,25 @@ void main(unsigned GI : SV_GroupIndex) {
   Wag();
 }
 
-// Make sure global variable for ctors/dtors removed.
-// CHECK-NOT:@llvm.global_ctors
-// CHECK-NOT:@llvm.global_dtors
-//CHECK:      define void @main()
-//CHECK-NEXT: entry:
-//CHECK-NEXT:   call void @_GLOBAL__sub_I_GlobalDestructors.hlsl()
-//CHECK-NEXT:   %0 = call i32 @llvm.dx.flattened.thread.id.in.group()
-//CHECK-NEXT:   call void @"?main@@YAXI@Z"(i32 %0)
-//CHECK-NEXT:   call void @_GLOBAL__D_a()
-//CHECK-NEXT:   ret void
+// CHECK:      define void @main()
+// CHECK-NEXT: entry:
+// Verify destructor is emitted
+// NOINLINE-NEXT:   call void @_GLOBAL__sub_I_GlobalDestructors.hlsl()
+// NOINLINE-NEXT:   %0 = call i32 @llvm.dx.flattened.thread.id.in.group()
+// NOINLINE-NEXT:   call void @"?main@@YAXI@Z"(i32 %0)
+// NOINLINE-NEXT:   call void @_GLOBAL__D_a()
+// NOINLINE-NEXT:   ret void
+// Verify inlining leaves only calls to "llvm." intrinsics
+// INLINE-NOT:   call {{[^@]*}} @{{[^l][^l][^v][^m][^\.]}}
+// INLINE:   ret void
 
 // This is really just a sanity check I needed for myself to verify that
 // function scope static variables also get destroyed properly.
 
-//CHECK: define internal void @_GLOBAL__D_a()
-//CHECK-NEXT: entry:
-//CHECK-NEXT:   call void @"??1Tail@@QAA@XZ"(ptr @"?T@?1??Wag@@YAXXZ@4UTail@@A")
-//CHECK-NEXT:   call void @"??1Pupper@@QAA@XZ"(ptr @"?GlobalPup@@3UPupper@@A")
-//CHECK-NEXT:   ret void
+// NOINLINE: define internal void @_GLOBAL__D_a() [[IntAttr:\#[0-9]+]]
+// NOINLINE-NEXT: entry:
+// NOINLINE-NEXT:   call void @"??1Tail@@QAA@XZ"(ptr @"?T@?1??Wag@@YAXXZ@4UTail@@A")
+// NOINLINE-NEXT:   call void @"??1Pupper@@QAA@XZ"(ptr @"?GlobalPup@@3UPupper@@A")
+// NOINLINE-NEXT:   ret void
+
+// NOINLINE: attributes [[IntAttr]] = {{.*}} alwaysinline
diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-constructor.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-constructor.hlsl
index baddfcf2cf1d52..174f4c3eaaad26 100644
--- a/clang/test/CodeGenHLSL/builtins/RWBuffer-constructor.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-constructor.hlsl
@@ -1,3 +1,4 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple spirv-vulkan-library -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s --check-prefix=CHECK-SPIRV
 
 RWBuffer<float> Buf;
diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl
index da8a1e538ec5e7..2a350c1619bd6e 100644
--- a/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl
@@ -11,6 +11,7 @@ void main(unsigned GI : SV_GroupIndex) {
 // Even at -O0 the subscript operators get inlined. The -O0 IR is a bit messy
 // and confusing to follow so the match here is pretty weak.
 
-// CHECK: define internal void @"?main@@YAXI@Z"
-// CHECK-NOT: call
+// CHECK: define void @main()
+// Verify inlining leaves only calls to "llvm." intrinsics
+// CHECK-NOT:   call {{[^@]*}} @{{[^l][^l][^v][^m][^\.]}}
 // CHECK: ret void
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-subscript.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-subscript.hlsl
index 9bd885d94d7e75..155749ec4f94a9 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-subscript.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-subscript.hlsl
@@ -11,6 +11,7 @@ void main(unsigned GI : SV_GroupIndex) {
 // Even at -O0 the subscript operators get inlined. The -O0 IR is a bit messy
 // and confusing to follow so the match here is pretty weak.
 
-// CHECK: define internal void @"?main@@YAXI@Z"
-// CHECK-NOT: call
+// CHECK: define void @main()
+// Verify inlining leaves only calls to "llvm." intrinsics
+// CHECK-NOT:   call {{[^@]*}} @{{[^l][^l][^v][^m][^\.]}}
 // CHECK: ret void
diff --git a/clang/test/CodeGenHLSL/inline-constructors.hlsl b/clang/test/CodeGenHLSL/inline-constructors.hlsl
new file mode 100644
index 00000000000000..995878a9c0f798
--- /dev/null
+++ b/clang/test/CodeGenHLSL/inline-constructors.hlsl
@@ -0,0 +1,76 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -std=hlsl202x -emit-llvm -o - -disable-llvm-passes %s | FileCheck %s --check-prefixes=CHECK,NOINLINE
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -std=hlsl202x -emit-llvm -o - -disable-llvm-passes %s | FileCheck %s --check-prefixes=CHECK,NOINLINE
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -std=hlsl202x -emit-llvm -o - -O0 %s | FileCheck %s --check-prefixes=CHECK,INLINE
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -std=hlsl202x -emit-llvm -o - -O0 %s | FileCheck %s --check-prefixes=CHECK,INLINE
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -std=hlsl202x -emit-llvm -o - -O1 %s | FileCheck %s --check-prefixes=CHECK,INLINE
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -std=hlsl202x -emit-llvm -o - -O1 %s | FileCheck %s --check-prefixes=CHECK,INLINE
+
+// Tests that implicit constructor calls for user classes will always be inlined.
+
+struct Weed {
+  Weed() {Count += 1;}
+  [[maybe_unused]] void pull() {Count--;}
+  static int weedCount() { return Count; }
+private:
+  static int Count;
+
+} YardWeeds;
+
+int Weed::Count = 1; // It begins. . .
+
+struct Kitty {
+  unsigned burrsInFur;
+
+  Kitty() {
+    burrsInFur = 0;
+  }
+
+  void wanderInYard(int hours) {
+    burrsInFur = hours*Weed::weedCount()/8;
+  }
+
+  void lick() {
+    if(burrsInFur) {
+      burrsInFur--;
+      Weed w;
+    }
+  }
+
+} Nion;
+
+void NionsDay(int hours) {
+  static Kitty Nion;
+  Nion.wanderInYard(hours);
+  while(Nion.burrsInFur) Nion.lick();
+}
+
+// CHECK:      define void @main()
+// CHECK-NEXT: entry:
+// Verify constructor is emitted
+// NOINLINE-NEXT: call void @_GLOBAL__sub_I_inline_constructors.hlsl()
+// NOINLINE-NEXT: %0 = call i32 @llvm.dx.flattened.thread.id.in.group()
+// NOINLINE-NEXT: call void @"?main@@YAXI@Z"(i32 %0)
+// Verify inlining leaves only calls to "llvm." intrinsics
+// INLINE-NOT:    call {{[^@]*}} @{{[^l][^l][^v][^m][^\.]}}
+// CHECK:         ret void
+[shader("compute")]
+[numthreads(1,1,1)]
+void main(unsigned GI : SV_GroupIndex) {
+  NionsDay(10);
+}
+
+
+// CHECK:      define void @rainyMain()
+// CHECK-NEXT: entry:
+// Verify constructor is emitted
+// NOINLINE-NEXT:   call void @_GLOBAL__sub_I_inline_constructors.hlsl()
+// NOINLINE-NEXT:   call void @"?rainyMain@@YAXXZ"()
+// Verify inlining leaves only calls to "llvm." intrinsics
+// INLINE-NOT:      call {{[^@]*}} @{{[^l][^l][^v][^m][^\.]}}
+// CHECK:           ret void
+[shader("compute")]
+[numthreads(1,1,1)]
+void rainyMain() {
+  NionsDay(1);
+}
+
diff --git a/clang/test/CodeGenHLSL/inline-functions.hlsl b/clang/test/CodeGenHLSL/inline-functions.hlsl
new file mode 100644
index 00000000000000..7dd905e966e069
--- /dev/null
+++ b/clang/test/CodeGenHLSL/inline-functions.hlsl
@@ -0,0 +1,116 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s --check-prefixes=CHECK,NOINLINE
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library %s -emit-llvm -O0 -o - | FileCheck %s --check-prefixes=CHECK,INLINE
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library %s -emit-llvm -O1 -o - | FileCheck %s --check-prefixes=CHECK,INLINE
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s --check-prefixes=CHECK,NOINLINE
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute %s -emit-llvm -O0 -o - | FileCheck %s --check-prefixes=CHECK,INLINE
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute %s -emit-llvm -O1 -o - | FileCheck %s --check-prefixes=CHECK,INLINE
+
+// Tests that user functions will always be inlined.
+// This includes exported functions and mangled entry point implementation functions.
+// The unmangled entry functions must not be alwaysinlined.
+
+#define MAX 100
+
+float nums[MAX];
+
+// Verify that all functions have the alwaysinline attribute
+// NOINLINE: Function Attrs: alwaysinline
+// NOINLINE: define void @"?swap@@YAXY0GE@III@Z"(ptr noundef byval([100 x i32]) align 4 %Buf, i32 noundef %ix1, i32 noundef %ix2) [[IntAttr:\#[0-9]+]]
+// NOINLINE: ret void
+// Swap the values of Buf at indices ix1 and ix2
+void swap(unsigned Buf[MAX], unsigned ix1, unsigned ix2) {
+  float tmp = Buf[ix1];
+  Buf[ix1] = Buf[ix2];
+  Buf[ix2] = tmp;
+}
+
+// NOINLINE: Function Attrs: alwaysinline
+// NOINLINE: define void @"?BubbleSort@@YAXY0GE@II@Z"(ptr noundef byval([100 x i32]) align 4 %Buf, i32 noundef %size) [[IntAttr]]
+// NOINLINE: ret void
+// Inefficiently sort Buf in place
+void BubbleSort(unsigned Buf[MAX], unsigned size) {
+  bool swapped = true;
+  while (swapped) {
+    swapped = false;
+    for (unsigned i = 1; i < size; i++) {
+      if (Buf[i] < Buf[i-1]) {
+	swap(Buf, i, i-1);
+	swapped = true;
+      }
+    }
+  }
+}
+
+// Note ExtAttr is the inlined export set of attribs
+// CHECK: Function Attrs: alwaysinline
+// CHECK: define noundef i32 @"?RemoveDupes@@YAIY0GE@II@Z"(ptr {{[a-z_ ]*}}noundef byval([100 x i32]) align 4 %Buf, i32 noundef %size) {{[a-z_ ]*}}[[ExtAttr:\#[0-9]+]]
+// CHECK: ret i32
+// Sort Buf and remove any duplicate values
+// returns the number of values left
+export
+unsigned RemoveDupes(unsigned Buf[MAX], unsigned size) {
+  BubbleSort(Buf, size);
+  unsigned insertPt = 0;
+  for (unsigned i = 1; i < size; i++) {
+    if (Buf[i] == Buf[i-1])
+      insertPt++;
+    else
+      Buf[insertPt] = Buf[i];
+  }
+  return insertPt;
+}
+
+
+RWBuffer<unsigned> Indices;
+
+// The mangled version of main only remains without inlining
+// because it has internal linkage from the start
+// Note main functions get the norecurse attrib, which IntAttr reflects
+// NOINLINE: Function Attrs: alwaysinline
+// NOINLINE: define internal void @"?main@@YAXI@Z"(i32 noundef %GI) [[IntAttr]]
+// NOINLINE: ret void
+
+// The unmangled version is not inlined, EntryAttr reflects that
+// CHECK: Function Attrs: {{.*}}noinline
+// CHECK: define void @main() {{[a-z_ ]*}}[[EntryAttr:\#[0-9]+]]
+// Make sure function calls are inlined when AlwaysInline is run
+// This only leaves calls to llvm. intrinsics
+// INLINE-NOT:   call {{[^@]*}} @{{[^l][^l][^v][^m][^\.]}}
+// CHECK: ret void
+
+[numthreads(1,1,1)]
+[shader("compute")]
+void main(unsigned int GI : SV_GroupIndex) {
+  unsigned tmpIndices[MAX];
+  if (GI > MAX) return;
+  for (unsigned i = 1; i < GI; i++)
+    tmpIndices[i] = Indices[i];
+  RemoveDupes(tmpIndices, GI);
+  for (unsigned i = 1; i < GI; i++)
+    tmpIndices[i] = Indices[i];
+}
+
+// The mangled version of main only remains without inlining
+// because it has internal linkage from the start
+// Note main functions get the norecurse attrib, which IntAttr reflects
+// NOINLINE: Function Attrs: alwaysinline
+// NOINLINE: define internal void @"?main10@@YAXXZ"() [[IntAttr]]
+// NOINLINE: ret void
+
+// The unmangled version is not inlined, EntryAttr reflects that
+// CHECK: Function Attrs: {{.*}}noinline
+// CHECK: define void @main10() {{[a-z_ ]*}}[[EntryAttr]]
+// Make sure function calls are inlined when AlwaysInline is run
+// This only leaves calls to llvm. intrinsics
+// INLINE-NOT:   call {{[^@]*}} @{{[^l][^l][^v][^m][^\.]}}
+// CHECK: ret void
+
+[numthreads(1,1,1)]
+[shader("compute")]
+void main10() {
+  main(10);
+}
+
+// NOINLINE: attributes [[IntAttr]] = {{.*}} alwaysinline
+// CHECK: attributes [[ExtAttr]] = {{.*}} alwaysinline
+// CHECK: attributes [[EntryAttr]] = {{.*}} noinline
diff --git a/clang/test/CodeGenHLSL/wavesize.hlsl b/clang/test/CodeGenHLSL/wavesize.hlsl
new file mode 100644
index 00000000000000..fcb817ce0d0629
--- /dev/null
+++ b/clang/test/CodeGenHLSL/wavesize.hlsl
@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -x hlsl -triple \
+// RUN:   dxil-pc-shadermodel6.6-compute %s -DSM66 -hlsl-entry foo \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s
+
+// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -x hlsl -triple \
+// RUN:   dxil-pc-shadermodel6.8-compute %s -DNO_PREFERR -hlsl-entry foo \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s --check-prefix=NO_PREFERR
+
+// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -x hlsl -triple \
+// RUN:   dxil-pc-shadermodel6.8-compute %s -hlsl-entry foo \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s --check-prefix=CHECK-SM68
+
+
+// Make sure wavesize attribute get correct value for sm66 and sm68.
+// CHECK:define void @foo()
+// CHECK:"hlsl.wavesize"="8,0,0"
+
+// NO_PREFERR:define void @foo()
+// NO_PREFERR:"hlsl.wavesize"="8,128,0"
+
+// CHECK-SM68:define void @foo()
+// CHECK-SM68:"hlsl.wavesize"="8,128,64"
+
+[numthreads(16,8,1)]
+#ifdef SM66
+[WaveSize(8)]
+#elif NO_PREFERR
+[WaveSize(8, 128)]
+#else
+[WaveSize(8, 128, 64)]
+#endif
+void foo() {
+
+}
diff --git a/clang/test/Driver/clang_f_opts.c b/clang/test/Driver/clang_f_opts.c
index 335fa546a13887..adb6f075b6c15d 100644
--- a/clang/test/Driver/clang_f_opts.c
+++ b/clang/test/Driver/clang_f_opts.c
@@ -623,6 +623,10 @@
 // RUN: %clang -### --target=aarch64-windows-msvc %s 2>&1 | FileCheck -check-prefix=CHECK-NO-MS-VOLATILE %s
 // RUN: %clang -### --target=aarch64-windows-msvc -fms-volatile %s 2>&1 | FileCheck -check-prefix=CHECK-MS-VOLATILE %s
 // RUN: %clang -### --target=aarch64-windows-msvc -fno-ms-volatile %s 2>&1 | FileCheck -check-prefix=CHECK-NO-MS-VOLATILE %s
+// RUN: %clang -### --target=x86_64-windows-msvc %s 2>&1 | FileCheck -check-prefix=CHECK-MS-VOLATILE %s
+// RUN: %clang -### --target=x86_64-windows-msvc -fno-ms-volatile %s 2>&1 | FileCheck -check-prefix=CHECK-NO-MS-VOLATILE %s
+// RUN: %clang -### --target=i686-windows-msvc %s 2>&1 | FileCheck -check-prefix=CHECK-MS-VOLATILE %s
+// RUN: %clang -### --target=i686-windows-msvc -fno-ms-volatile %s 2>&1 | FileCheck -check-prefix=CHECK-NO-MS-VOLATILE %s
 // CHECK-MS-VOLATILE: -fms-volatile
 // CHECK-NO-MS-VOLATILE-NOT: -fms-volatile
 
diff --git a/clang/test/Driver/ps4-linker.c b/clang/test/Driver/ps4-linker.c
index a1bc8f92537545..f3a304a956fd8a 100644
--- a/clang/test/Driver/ps4-linker.c
+++ b/clang/test/Driver/ps4-linker.c
@@ -28,3 +28,12 @@
 
 // RUN: %clang --target=x86_64-scei-ps4 %s -### 2>&1 | FileCheck --check-prefixes=CHECK-NO-LTO %s
 // CHECK-NO-LTO-NOT: -lto-debug-options
+
+// Test the driver passes a sysroot to the linker. Without --sysroot, its value
+// is sourced from the SDK environment variable.
+
+// RUN: env SCE_ORBIS_SDK_DIR=mysdk %clang --target=x64_64-scei-ps4 %s -### 2>&1 | FileCheck --check-prefixes=CHECK-SYSROOT %s
+// RUN: env SCE_ORBIS_SDK_DIR=other %clang --target=x64_64-scei-ps4 %s -### --sysroot=mysdk 2>&1 | FileCheck --check-prefixes=CHECK-SYSROOT %s
+
+// CHECK-SYSROOT: {{ld(\.exe)?}}"
+// CHECK-SYSROOT-SAME: "--sysroot=mysdk"
diff --git a/clang/test/Driver/ps4-ps5-header-search.c b/clang/test/Driver/ps4-ps5-header-search.c
index 066d284d08065e..7c24523df7a039 100644
--- a/clang/test/Driver/ps4-ps5-header-search.c
+++ b/clang/test/Driver/ps4-ps5-header-search.c
@@ -1,6 +1,6 @@
 /// PS4 and PS5 use the same SDK layout, so use the same tree for both.
-// RUN: env SCE_ORBIS_SDK_DIR=%S/Inputs/scei-ps4_tree %clang -target x86_64-scei-ps4 --sysroot="" -E -v %s 2>&1 | FileCheck %s --check-prefix=ENVPS4
-// RUN: env SCE_PROSPERO_SDK_DIR=%S/Inputs/scei-ps4_tree %clang -target x86_64-sie-ps5 --sysroot="" -E -v %s 2>&1 | FileCheck %s --check-prefix=ENVPS4
+// RUN: env SCE_ORBIS_SDK_DIR=%S/Inputs/scei-ps4_tree %clang -target x86_64-scei-ps4 -E -v %s 2>&1 | FileCheck %s --check-prefix=ENVPS4
+// RUN: env SCE_PROSPERO_SDK_DIR=%S/Inputs/scei-ps4_tree %clang -target x86_64-sie-ps5 -E -v %s 2>&1 | FileCheck %s --check-prefix=ENVPS4
 // ENVPS4: Inputs/scei-ps4_tree/target/include
 // ENVPS4: Inputs/scei-ps4_tree/target/include_common
 // ENVPS4-NOT: /usr/include
diff --git a/clang/test/Driver/ps4-sdk-root.c b/clang/test/Driver/ps4-sdk-root.c
index 3e02fa9fc3bc29..78eb1ce2ba6302 100644
--- a/clang/test/Driver/ps4-sdk-root.c
+++ b/clang/test/Driver/ps4-sdk-root.c
@@ -1,42 +1,67 @@
-// Check that PS4 clang doesn't report a warning message when locating
-// system header files (either by looking at the value of SCE_ORBIS_SDK_DIR
-// or relative to the location of the compiler driver), if "-nostdinc",
-// "--sysroot" or "-isysroot" option is specified on the command line.
-// Otherwise, check that PS4 clang reports a warning.
-
-// Check that PS4 clang doesn't report a warning message when locating
-// system libraries (either by looking at the value of SCE_ORBIS_SDK_DIR
-// or relative to the location of the compiler driver), if "-c", "-S", "-E"
-// or "--sysroot" option is specified on the command line.
-// Otherwise, check that PS4 clang reports a warning.
-
-// Setting up SCE_ORBIS_SDK_DIR to existing location, which is not a PS4 SDK.
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=WARN-SYS-HEADERS -check-prefix=WARN-SYS-LIBS -check-prefix=NO-WARN %s
-
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -c -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=WARN-SYS-HEADERS -check-prefix=NO-WARN %s
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -S -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=WARN-SYS-HEADERS -check-prefix=NO-WARN %s
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -E -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=WARN-SYS-HEADERS -check-prefix=NO-WARN %s
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -emit-ast -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=WARN-SYS-HEADERS -check-prefix=NO-WARN %s
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -isysroot foo -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=WARN-ISYSROOT -check-prefix=WARN-SYS-LIBS -check-prefix=NO-WARN %s
-
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -c -nostdinc -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -S -nostdinc -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -E -nostdinc -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -emit-ast -nostdinc -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
-
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -c --sysroot=foo/ -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -S --sysroot=foo/ -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -E --sysroot=foo/ -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -emit-ast --sysroot=foo/ -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
-
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -c -isysroot foo -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=WARN-ISYSROOT -check-prefix=NO-WARN %s
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -S -isysroot foo -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=WARN-ISYSROOT -check-prefix=NO-WARN %s
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -E -isysroot foo -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=WARN-ISYSROOT -check-prefix=NO-WARN %s
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -emit-ast -isysroot foo -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=WARN-ISYSROOT -check-prefix=NO-WARN %s
-// RUN: env SCE_ORBIS_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### --sysroot=foo/ -isysroot foo -target x86_64-scei-ps4 %s 2>&1 | FileCheck -check-prefix=WARN-ISYSROOT -check-prefix=NO-WARN %s
+/// PS4 clang emits warnings when SDK headers (<SDKROOT>/target/include/) or
+/// libraries (<SDKROOT>/target/lib/) are missing, unless the user takes control
+/// of search paths, when corresponding existence checks are skipped.
+///
+/// User control of header search is assumed if `--sysroot`, `-isysroot`,
+/// `-nostdinc` or `-nostdlibinc` is supplied. User control of library search
+/// is assumed if `--sysroot` is supplied.
+///
+/// Warnings are emitted if a specified `-isysroot` or `--sysroot` does not
+/// exist.
+///
+/// The default <SDKROOT> for both headers and libraries is taken from the
+/// SCE_ORBIS_SDK_DIR environment variable.
+
+// RUN: echo "-### -Winvalid-or-nonexistent-directory -target x86_64-scei-ps4" > %t.rsp
+
+/// If SDK headers and/or libraries are found, associated warnings are absent.
+// RUN: rm -rf %t.inconly && mkdir -p %t.inconly/target/include
+// RUN: env SCE_ORBIS_SDK_DIR=%t.inconly %clang @%t.rsp %s 2>&1 | FileCheck -check-prefixes=WARN-SYS-LIBS,NO-WARN %s
+
+// RUN: rm -rf %t.libonly && mkdir -p %t.libonly/target/lib
+// RUN: env SCE_ORBIS_SDK_DIR=%t.libonly %clang @%t.rsp %s 2>&1 | FileCheck -check-prefixes=WARN-SYS-HEADERS,NO-WARN %s
+
+// RUN: rm -rf %t.both && mkdir -p %t.both/target/lib && mkdir %t.both/target/include
+// RUN: env SCE_ORBIS_SDK_DIR=%t.both %clang @%t.rsp %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
+
+/// In the following invocations, SCE_ORBIS_SDK_DIR is set to an existing
+/// location where SDK headers and libraries are absent.
+
+/// When compiling and linking, we should see a warnings about both missing
+/// headers and libraries.
+// RUN: env SCE_ORBIS_SDK_DIR=.. %clang @%t.rsp %s 2>&1 | FileCheck -check-prefixes=WARN-SYS-HEADERS,WARN-SYS-LIBS,NO-WARN %s
+
+/// If `-c`, `-S`, `-E` or `-emit-ast` is supplied, the existence check for SDK
+/// libraries is skipped because no linking will be performed. We only expect
+/// warnings about missing headers.
+// RUN: env SCE_ORBIS_SDK_DIR=.. %clang @%t.rsp %s -c 2>&1 | FileCheck -check-prefixes=WARN-SYS-HEADERS,NO-WARN %s
+// RUN: env SCE_ORBIS_SDK_DIR=.. %clang @%t.rsp %s -S 2>&1 | FileCheck -check-prefixes=WARN-SYS-HEADERS,NO-WARN %s
+// RUN: env SCE_ORBIS_SDK_DIR=.. %clang @%t.rsp %s -E 2>&1 | FileCheck -check-prefixes=WARN-SYS-HEADERS,NO-WARN %s
+// RUN: env SCE_ORBIS_SDK_DIR=.. %clang @%t.rsp %s -emit-ast 2>&1 | FileCheck -check-prefixes=WARN-SYS-HEADERS,NO-WARN %s
+
+/// If the user takes control of include paths, the existence check for headers
+/// is not performed.
+// RUN: env SCE_ORBIS_SDK_DIR=.. %clang @%t.rsp %s -c -nostdinc 2>&1 | FileCheck -check-prefix=NO-WARN %s
+// RUN: env SCE_ORBIS_SDK_DIR=.. %clang @%t.rsp %s -c -nostdlibinc 2>&1 | FileCheck -check-prefix=NO-WARN %s
+// RUN: env SCE_ORBIS_SDK_DIR=.. %clang @%t.rsp %s -c -isysroot . 2>&1 | FileCheck -check-prefixes=NO-WARN %s
+// RUN: env SCE_ORBIS_SDK_DIR=.. %clang @%t.rsp %s -c --sysroot=. 2>&1 | FileCheck -check-prefixes=NO-WARN %s
+
+/// --sysroot disables the existence check for libraries and headers.
+// RUN: env SCE_ORBIS_SDK_DIR=.. %clang @%t.rsp %s --sysroot=. 2>&1 | FileCheck -check-prefix=NO-WARN %s
+
+/// -isysroot overrides --sysroot for header search, but not library search.
+// RUN: env SCE_ORBIS_SDK_DIR=.. %clang @%t.rsp %s -isysroot . --sysroot=.. 2>&1 | FileCheck -check-prefixes=ISYSTEM,NO-WARN %s
+// RUN: env SCE_ORBIS_SDK_DIR=.. %clang @%t.rsp %s --sysroot=.. -isysroot . 2>&1 | FileCheck -check-prefixes=ISYSTEM,NO-WARN %s
+
+/// Warnings are emitted if non-existent --sysroot/-isysroot are supplied.
+// RUN: env SCE_ORBIS_SDK_DIR=.. %clang @%t.rsp %s --sysroot=foo -isysroot . 2>&1 | FileCheck -check-prefixes=WARN-SYSROOT,NO-WARN %s
+// RUN: env SCE_ORBIS_SDK_DIR=.. %clang @%t.rsp %s -isysroot foo --sysroot=. 2>&1 | FileCheck -check-prefixes=WARN-SYSROOT,NO-WARN %s
+// RUN: env SCE_ORBIS_SDK_DIR=.. %clang @%t.rsp %s --sysroot=foo -isysroot bar 2>&1 | FileCheck -check-prefixes=WARN-SYSROOT,WARN-SYSROOT2,NO-WARN %s
 
 // NO-WARN-NOT: {{warning:|error:}}
-// WARN-SYS-HEADERS: warning: unable to find PS4 system headers directory
-// WARN-ISYSROOT: warning: no such sysroot directory: 'foo'
 // WARN-SYS-LIBS: warning: unable to find PS4 system libraries directory
+// WARN-SYS-HEADERS: warning: unable to find PS4 system headers directory
+// WARN-SYSROOT: warning: no such sysroot directory: 'foo'
+// WARN-SYSROOT2: warning: no such sysroot directory: 'bar'
 // NO-WARN-NOT: {{warning:|error:}}
+// ISYSTEM: "-cc1"{{.*}}"-internal-externc-isystem" "./target/include"
diff --git a/clang/test/Driver/ps5-linker.c b/clang/test/Driver/ps5-linker.c
index 84363deb0337f7..c0cf0b864028c8 100644
--- a/clang/test/Driver/ps5-linker.c
+++ b/clang/test/Driver/ps5-linker.c
@@ -37,3 +37,12 @@
 // RUN: %clang --target=x86_64-sie-ps5 -flto -fcrash-diagnostics-dir=mydumps %s -### 2>&1 | FileCheck --check-prefixes=CHECK-DIAG %s
 
 // CHECK-DIAG: -plugin-opt=-crash-diagnostics-dir=mydumps
+
+// Test the driver passes a sysroot to the linker. Without --sysroot, its value
+// is sourced from the SDK environment variable.
+
+// RUN: env SCE_PROSPERO_SDK_DIR=mysdk %clang --target=x64_64-sie-ps5 %s -### 2>&1 | FileCheck --check-prefixes=CHECK-SYSROOT %s
+// RUN: env SCE_PROSPERO_SDK_DIR=other %clang --target=x64_64-sie-ps5 %s -### --sysroot=mysdk 2>&1 | FileCheck --check-prefixes=CHECK-SYSROOT %s
+
+// CHECK-SYSROOT: {{ld(\.exe)?}}"
+// CHECK-SYSROOT-SAME: "--sysroot=mysdk"
diff --git a/clang/test/Driver/ps5-sdk-root.c b/clang/test/Driver/ps5-sdk-root.c
index 2a82d8e72283b7..a12e0dfffeb584 100644
--- a/clang/test/Driver/ps5-sdk-root.c
+++ b/clang/test/Driver/ps5-sdk-root.c
@@ -1,44 +1,69 @@
 /// (Essentially identical to ps4-sdk-root.c except for the target.)
 
-/// Check that PS5 clang doesn't report a warning message when locating
-/// system header files (either by looking at the value of SCE_PROSPERO_SDK_DIR
-/// or relative to the location of the compiler driver), if "-nostdinc",
-/// "--sysroot" or "-isysroot" option is specified on the command line.
-/// Otherwise, check that PS5 clang reports a warning.
-
-// Check that PS5 clang doesn't report a warning message when locating
-// system libraries (either by looking at the value of SCE_PROSPERO_SDK_DIR
-// or relative to the location of the compiler driver), if "-c", "-S", "-E"
-// or "--sysroot" option is specified on the command line.
-// Otherwise, check that PS5 clang reports a warning.
-
-// Setting up SCE_PROSPERO_SDK_DIR to existing location, which is not a PS5 SDK.
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=WARN-SYS-HEADERS -check-prefix=WARN-SYS-LIBS -check-prefix=NO-WARN %s
-
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -c -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=WARN-SYS-HEADERS -check-prefix=NO-WARN %s
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -S -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=WARN-SYS-HEADERS -check-prefix=NO-WARN %s
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -E -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=WARN-SYS-HEADERS -check-prefix=NO-WARN %s
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -emit-ast -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=WARN-SYS-HEADERS -check-prefix=NO-WARN %s
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -isysroot foo -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=WARN-ISYSROOT -check-prefix=WARN-SYS-LIBS -check-prefix=NO-WARN %s
-
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -c -nostdinc -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -S -nostdinc -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -E -nostdinc -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -emit-ast -nostdinc -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
-
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -c --sysroot=foo/ -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -S --sysroot=foo/ -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -E --sysroot=foo/ -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -emit-ast --sysroot=foo/ -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
-
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -c -isysroot foo -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=WARN-ISYSROOT -check-prefix=NO-WARN %s
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -S -isysroot foo -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=WARN-ISYSROOT -check-prefix=NO-WARN %s
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -E -isysroot foo -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=WARN-ISYSROOT -check-prefix=NO-WARN %s
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### -emit-ast -isysroot foo -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=WARN-ISYSROOT -check-prefix=NO-WARN %s
-// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang -Winvalid-or-nonexistent-directory -### --sysroot=foo/ -isysroot foo -target x86_64-sie-ps5 %s 2>&1 | FileCheck -check-prefix=WARN-ISYSROOT -check-prefix=NO-WARN %s
+/// PS5 clang emits warnings when SDK headers (<SDKROOT>/target/include/) or
+/// libraries (<SDKROOT>/target/lib/) are missing, unless the user takes control
+/// of search paths, when corresponding existence checks are skipped.
+///
+/// User control of header search is assumed if `--sysroot`, `-isysroot`,
+/// `-nostdinc` or `-nostdlibinc` is supplied. User control of library search
+/// is assumed if `--sysroot` is supplied.
+///
+/// Warnings are emitted if a specified `-isysroot` or `--sysroot` does not
+/// exist.
+///
+/// The default <SDKROOT> for both headers and libraries is taken from the
+/// SCE_PROSPERO_SDK_DIR environment variable.
+
+// RUN: echo "-### -Winvalid-or-nonexistent-directory -target x86_64-sie-ps5" > %t.rsp
+
+/// If SDK headers and/or libraries are found, associated warnings are absent.
+// RUN: rm -rf %t.inconly && mkdir -p %t.inconly/target/include
+// RUN: env SCE_PROSPERO_SDK_DIR=%t.inconly %clang @%t.rsp %s 2>&1 | FileCheck -check-prefixes=WARN-SYS-LIBS,NO-WARN %s
+
+// RUN: rm -rf %t.libonly && mkdir -p %t.libonly/target/lib
+// RUN: env SCE_PROSPERO_SDK_DIR=%t.libonly %clang @%t.rsp %s 2>&1 | FileCheck -check-prefixes=WARN-SYS-HEADERS,NO-WARN %s
+
+// RUN: rm -rf %t.both && mkdir -p %t.both/target/lib && mkdir %t.both/target/include
+// RUN: env SCE_PROSPERO_SDK_DIR=%t.both %clang @%t.rsp %s 2>&1 | FileCheck -check-prefix=NO-WARN %s
+
+/// In the following invocations, SCE_PROSPERO_SDK_DIR is set to an existing
+/// location where SDK headers and libraries are absent.
+
+/// When compiling and linking, we should see a warnings about both missing
+/// headers and libraries.
+// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang @%t.rsp %s 2>&1 | FileCheck -check-prefixes=WARN-SYS-HEADERS,WARN-SYS-LIBS,NO-WARN %s
+
+/// If `-c`, `-S`, `-E` or `-emit-ast` is supplied, the existence check for SDK
+/// libraries is skipped because no linking will be performed. We only expect
+/// warnings about missing headers.
+// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang @%t.rsp %s -c 2>&1 | FileCheck -check-prefixes=WARN-SYS-HEADERS,NO-WARN %s
+// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang @%t.rsp %s -S 2>&1 | FileCheck -check-prefixes=WARN-SYS-HEADERS,NO-WARN %s
+// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang @%t.rsp %s -E 2>&1 | FileCheck -check-prefixes=WARN-SYS-HEADERS,NO-WARN %s
+// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang @%t.rsp %s -emit-ast 2>&1 | FileCheck -check-prefixes=WARN-SYS-HEADERS,NO-WARN %s
+
+/// If the user takes control of include paths, the existence check for headers
+/// is not performed.
+// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang @%t.rsp %s -c -nostdinc 2>&1 | FileCheck -check-prefix=NO-WARN %s
+// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang @%t.rsp %s -c -nostdlibinc 2>&1 | FileCheck -check-prefix=NO-WARN %s
+// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang @%t.rsp %s -c -isysroot . 2>&1 | FileCheck -check-prefixes=NO-WARN %s
+// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang @%t.rsp %s -c --sysroot=. 2>&1 | FileCheck -check-prefixes=NO-WARN %s
+
+/// --sysroot disables the existence check for libraries and headers.
+// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang @%t.rsp %s --sysroot=. 2>&1 | FileCheck -check-prefix=NO-WARN %s
+
+/// -isysroot overrides --sysroot for header search, but not library search.
+// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang @%t.rsp %s -isysroot . --sysroot=.. 2>&1 | FileCheck -check-prefixes=ISYSTEM,NO-WARN %s
+// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang @%t.rsp %s --sysroot=.. -isysroot . 2>&1 | FileCheck -check-prefixes=ISYSTEM,NO-WARN %s
+
+/// Warnings are emitted if non-existent --sysroot/-isysroot are supplied.
+// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang @%t.rsp %s --sysroot=foo -isysroot . 2>&1 | FileCheck -check-prefixes=WARN-SYSROOT,NO-WARN %s
+// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang @%t.rsp %s -isysroot foo --sysroot=. 2>&1 | FileCheck -check-prefixes=WARN-SYSROOT,NO-WARN %s
+// RUN: env SCE_PROSPERO_SDK_DIR=.. %clang @%t.rsp %s --sysroot=foo -isysroot bar 2>&1 | FileCheck -check-prefixes=WARN-SYSROOT,WARN-SYSROOT2,NO-WARN %s
 
 // NO-WARN-NOT: {{warning:|error:}}
-// WARN-SYS-HEADERS: warning: unable to find PS5 system headers directory
-// WARN-ISYSROOT: warning: no such sysroot directory: 'foo'
 // WARN-SYS-LIBS: warning: unable to find PS5 system libraries directory
+// WARN-SYS-HEADERS: warning: unable to find PS5 system headers directory
+// WARN-SYSROOT: warning: no such sysroot directory: 'foo'
+// WARN-SYSROOT2: warning: no such sysroot directory: 'bar'
 // NO-WARN-NOT: {{warning:|error:}}
+// ISYSTEM: "-cc1"{{.*}}"-internal-externc-isystem" "./target/include"
diff --git a/clang/test/Modules/pr108732.cppm b/clang/test/Modules/pr108732.cppm
new file mode 100644
index 00000000000000..f3b495aa826ce3
--- /dev/null
+++ b/clang/test/Modules/pr108732.cppm
@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 -std=c++20 %s -ast-dump | FileCheck %s
+export module mod;
+
+extern "C++" {
+class C
+{
+public:
+bool foo() const {
+    return true;
+}
+};
+}
+
+// CHECK: foo {{.*}}implicit-inline
diff --git a/clang/test/ParserHLSL/hlsl_contained_type_attr.hlsl b/clang/test/ParserHLSL/hlsl_contained_type_attr.hlsl
new file mode 100644
index 00000000000000..ff90208acd0978
--- /dev/null
+++ b/clang/test/ParserHLSL/hlsl_contained_type_attr.hlsl
@@ -0,0 +1,19 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -std=hlsl202x -x hlsl -ast-dump -o - %s | FileCheck %s
+
+typedef vector<float, 4> float4;
+
+// CHECK: -TypeAliasDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 2]]:1, col:83>
+// CHECK: -HLSLAttributedResourceType 0x{{[0-9a-f]+}} '__hlsl_resource_t {{\[\[}}hlsl::resource_class(UAV)]] {{\[\[}}hlsl::contained_type(int)]]' sugar
+using ResourceIntAliasT = __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(int)]];
+ResourceIntAliasT h1;
+
+// CHECK: -VarDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 1]]:1, col:82> col:82 h2 '__hlsl_resource_t {{\[\[}}hlsl::resource_class(UAV)]] {{\[\[}}hlsl::contained_type(float4)]]':'__hlsl_resource_t'
+__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(float4)]] h2;
+
+// ClassTemplateDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 4]]:1, line:16:1> line:14:30 S
+// TemplateTypeParmDecl 0x{{[0-9a-f]+}} <col:[[# @LINE + 3]], col:20> col:20 referenced typename depth 0 index 0 T
+// CXXRecordDecl 0x{{[0-9a-f]+}} <col:23, line:[[# @LINE + 2]]:1> line:14:30 struct S definition
+// FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 1]]:3, col:79> col:79 h '__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(T)]]':'__hlsl_resource_t'
+template <typename T> struct S {
+  __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(T)]] h;
+};
diff --git a/clang/test/ParserHLSL/hlsl_contained_type_attr_error.hlsl b/clang/test/ParserHLSL/hlsl_contained_type_attr_error.hlsl
new file mode 100644
index 00000000000000..1c37d72de86148
--- /dev/null
+++ b/clang/test/ParserHLSL/hlsl_contained_type_attr_error.hlsl
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -std=hlsl202x -x hlsl -o - %s -verify
+
+typedef vector<float, 4> float4;
+
+// expected-error@+1{{'contained_type' attribute cannot be applied to a declaration}}
+[[hlsl::contained_type(float4)]] __hlsl_resource_t h1;
+
+// expected-error@+1{{'contained_type' attribute takes one argument}}
+__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type()]] h3;
+
+// expected-error@+1{{expected a type}}
+__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(0)]] h4;
+
+// expected-error@+1{{unknown type name 'a'}}
+__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(a)]] h5;
+
+// expected-error@+1{{expected a type}}
+__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type("b", c)]] h6;
+
+// expected-warning@+1{{attribute 'contained_type' is already applied}}
+__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(float)]] [[hlsl::contained_type(float)]] h7;
+
+// expected-warning@+1{{attribute 'contained_type' is already applied with different arguments}}
+__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(float)]] [[hlsl::contained_type(int)]] h8;
diff --git a/clang/test/ParserHLSL/hlsl_raw_buffer_attr.hlsl b/clang/test/ParserHLSL/hlsl_raw_buffer_attr.hlsl
new file mode 100644
index 00000000000000..1955be63cf7607
--- /dev/null
+++ b/clang/test/ParserHLSL/hlsl_raw_buffer_attr.hlsl
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -ast-dump -o - %s | FileCheck %s
+
+// CHECK: CXXRecordDecl 0x{{[0-9a-f]+}} {{.*}} struct MyBuffer definition
+// CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:6:3, col:72> col:72 h1 '__hlsl_resource_t {{\[\[}}hlsl::resource_class(UAV)]] {{\[\[}}hlsl::raw_buffer]]':'__hlsl_resource_t'
+struct MyBuffer {
+  __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer]] h1;
+};
+
+// CHECK: VarDecl 0x{{[0-9a-f]+}} <line:10:1, col:70> col:70 h2 '__hlsl_resource_t {{\[\[}}hlsl::resource_class(SRV)]] {{\[\[}}hlsl::raw_buffer]]':'__hlsl_resource_t'
+__hlsl_resource_t [[hlsl::raw_buffer]] [[hlsl::resource_class(SRV)]] h2;
+
+// CHECK: FunctionDecl 0x{{[0-9a-f]+}} <line:14:1, line:16:1> line:14:6 f 'void ()
+// CHECK: VarDecl 0x{{[0-9a-f]+}} <col:3, col:72> col:72 h3 '__hlsl_resource_t {{\[\[}}hlsl::resource_class(UAV)]] {{\[\[}}hlsl::raw_buffer]]':'__hlsl_resource_t'
+void f() {
+  __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer]] h3;
+}
diff --git a/clang/test/ParserHLSL/hlsl_raw_buffer_attr_error.hlsl b/clang/test/ParserHLSL/hlsl_raw_buffer_attr_error.hlsl
new file mode 100644
index 00000000000000..83273426017ed0
--- /dev/null
+++ b/clang/test/ParserHLSL/hlsl_raw_buffer_attr_error.hlsl
@@ -0,0 +1,13 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -o - %s -verify
+
+// expected-error@+1{{'raw_buffer' attribute cannot be applied to a declaration}}
+[[hlsl::raw_buffer]] __hlsl_resource_t res0;
+
+// expected-error@+1{{'raw_buffer' attribute takes no arguments}}
+__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer(3)]] res2;
+  
+// expected-error@+1{{use of undeclared identifier 'gibberish'}}
+__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer(gibberish)]] res3;
+
+// expected-warning@+1{{attribute 'raw_buffer' is already applied}}
+__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer]] [[hlsl::raw_buffer]] res4;
diff --git a/clang/test/ParserHLSL/hlsl_resource_handle_attrs.hlsl b/clang/test/ParserHLSL/hlsl_resource_handle_attrs.hlsl
index 7c3830a291970c..24e3ae26d8e800 100644
--- a/clang/test/ParserHLSL/hlsl_resource_handle_attrs.hlsl
+++ b/clang/test/ParserHLSL/hlsl_resource_handle_attrs.hlsl
@@ -3,7 +3,7 @@
 // CHECK: -ClassTemplateSpecializationDecl 0x{{[0-9a-f]+}} <<invalid sloc>> <invalid sloc> class RWBuffer definition implicit_instantiation
 // CHECK: -TemplateArgument type 'float'
 // CHECK: `-BuiltinType 0x{{[0-9a-f]+}} 'float'
-// CHECK: -FieldDecl 0x{{[0-9a-f]+}} <<invalid sloc>> <invalid sloc> implicit referenced h 'float * {{\[\[}}hlsl::resource_class(UAV)]]':'float *'
+// CHECK: -FieldDecl 0x{{[0-9a-f]+}} <<invalid sloc>> <invalid sloc> implicit referenced h 'float * {{\[\[}}hlsl::resource_class(UAV)]] {{\[\[}}hlsl::contained_type(float)]]':'float *'
 // CHECK: -HLSLResourceAttr 0x{{[0-9a-f]+}} <<invalid sloc>> Implicit TypedBuffer
 RWBuffer<float> Buffer1;
 
@@ -11,6 +11,6 @@ RWBuffer<float> Buffer1;
 // CHECK: -TemplateArgument type 'vector<float, 4>'
 // CHECK: `-ExtVectorType 0x{{[0-9a-f]+}} 'vector<float, 4>' 4
 // CHECK: `-BuiltinType 0x{{[0-9a-f]+}} 'float'
-// CHECK: -FieldDecl 0x{{[0-9a-f]+}} <<invalid sloc>> <invalid sloc> implicit referenced h 'vector<float *, 4> {{\[\[}}hlsl::resource_class(UAV)]] {{\[\[}}hlsl::is_rov]]':'vector<float *, 4>'
+// CHECK: -FieldDecl 0x{{[0-9a-f]+}} <<invalid sloc>> <invalid sloc> implicit referenced h 'vector<float *, 4> {{\[\[}}hlsl::resource_class(UAV)]] {{\[\[}}hlsl::is_rov]] {{\[\[}}hlsl::contained_type(vector<float, 4>)]]':'vector<float *, 4>'
 // CHECK: -HLSLResourceAttr 0x{{[0-9a-f]+}} <<invalid sloc>> Implicit TypedBuffer
 RasterizerOrderedBuffer<vector<float, 4> > BufferArray3[4];
diff --git a/clang/test/Preprocessor/aarch64-target-features.c b/clang/test/Preprocessor/aarch64-target-features.c
index ae2bdda6f536c5..bcc0640c55e35a 100644
--- a/clang/test/Preprocessor/aarch64-target-features.c
+++ b/clang/test/Preprocessor/aarch64-target-features.c
@@ -4,7 +4,6 @@
 // CHECK: __AARCH64EL__ 1
 // CHECK: __ARM_64BIT_STATE 1
 // CHECK-NOT: __ARM_32BIT_STATE
-// CHECK: __ARM_ACLE 200
 // CHECK: __ARM_ALIGN_MAX_STACK_PWR 4
 // CHECK: __ARM_ARCH 8
 // CHECK: __ARM_ARCH_ISA_A64 1
diff --git a/clang/test/Preprocessor/init-aarch64.c b/clang/test/Preprocessor/init-aarch64.c
index 81c9a66f137512..c52c49a94e016e 100644
--- a/clang/test/Preprocessor/init-aarch64.c
+++ b/clang/test/Preprocessor/init-aarch64.c
@@ -12,7 +12,8 @@
 // AARCH64_LE-NEXT: #define __AARCH64EL__ 1
 // AARCH64_LE-NEXT: #define __AARCH64_CMODEL_SMALL__ 1
 // AARCH64-NEXT: #define __ARM_64BIT_STATE 1
-// AARCH64-NEXT: #define __ARM_ACLE 200
+// AARCH64-NEXT: #define __ARM_ACLE 202420
+// AARCH64-NEXT: #define __ARM_ACLE_VERSION(year,quarter,patch) (100 * (year) + 10 * (quarter) + (patch))
 // AARCH64-NEXT: #define __ARM_ALIGN_MAX_STACK_PWR 4
 // AARCH64-NEXT: #define __ARM_ARCH 8
 // AARCH64-NEXT: #define __ARM_ARCH_ISA_A64 1
@@ -121,6 +122,7 @@
 // AARCH64-NEXT: #define __FPCLASS_SNAN 0x0001
 // AARCH64-NEXT: #define __FP_FAST_FMA 1
 // AARCH64-NEXT: #define __FP_FAST_FMAF 1
+// AARCH64-NEXT: #define __FUNCTION_MULTI_VERSIONING_SUPPORT_LEVEL 202420
 // AARCH64-NEXT: #define __GCC_ASM_FLAG_OUTPUTS__ 1
 // AARCH64-NEXT: #define __GCC_CONSTRUCTIVE_SIZE {{.+}}
 // AARCH64-NEXT: #define __GCC_DESTRUCTIVE_SIZE {{.+}}
@@ -393,6 +395,8 @@
 // AARCH64-DARWIN: #define __AARCH64EL__ 1
 // AARCH64-DARWIN-NOT: #define __AARCH_BIG_ENDIAN 1
 // AARCH64-DARWIN: #define __ARM_64BIT_STATE 1
+// AARCH64-DARWIN: #define __ARM_ACLE 202420
+// AARCH64-DARWIN: #define __ARM_ACLE_VERSION(year,quarter,patch) (100 * (year) + 10 * (quarter) + (patch))
 // AARCH64-DARWIN: #define __ARM_ARCH 8
 // AARCH64-DARWIN: #define __ARM_ARCH_ISA_A64 1
 // AARCH64-DARWIN-NOT: #define __ARM_BIG_ENDIAN 1
@@ -429,6 +433,7 @@
 // AARCH64-DARWIN: #define __FLT_MIN_EXP__ (-125)
 // AARCH64-DARWIN: #define __FLT_MIN__ 1.17549435e-38F
 // AARCH64-DARWIN: #define __FLT_RADIX__ 2
+// AARCH64-DARWIN: #define __FUNCTION_MULTI_VERSIONING_SUPPORT_LEVEL 202420
 // AARCH64-DARWIN: #define __INT16_C_SUFFIX__
 // AARCH64-DARWIN: #define __INT16_FMTd__ "hd"
 // AARCH64-DARWIN: #define __INT16_FMTi__ "hi"
@@ -588,7 +593,8 @@
 // AARCH64-MSVC: #define _WIN64 1
 // AARCH64-MSVC: #define __AARCH64EL__ 1
 // AARCH64-MSVC: #define __ARM_64BIT_STATE 1
-// AARCH64-MSVC: #define __ARM_ACLE 200
+// AARCH64-MSVC: #define __ARM_ACLE 202420
+// AARCH64-MSVC: #define __ARM_ACLE_VERSION(year,quarter,patch) (100 * (year) + 10 * (quarter) + (patch))
 // AARCH64-MSVC: #define __ARM_ALIGN_MAX_STACK_PWR 4
 // AARCH64-MSVC: #define __ARM_ARCH 8
 // AARCH64-MSVC: #define __ARM_ARCH_ISA_A64 1
@@ -644,6 +650,7 @@
 // AARCH64-MSVC: #define __FLT_MIN_EXP__ (-125)
 // AARCH64-MSVC: #define __FLT_MIN__ 1.17549435e-38F
 // AARCH64-MSVC: #define __FLT_RADIX__ 2
+// AARCH64-MSVC: #define __FUNCTION_MULTI_VERSIONING_SUPPORT_LEVEL 202420
 // AARCH64-MSVC: #define __INT_MAX__ 2147483647
 // AARCH64-MSVC: #define __LDBL_DECIMAL_DIG__ 17
 // AARCH64-MSVC: #define __LDBL_DENORM_MIN__ 4.9406564584124654e-324L
@@ -750,7 +757,8 @@
 // ARM64EC-MSVC: #define __AARCH64EL__ 1
 // ARM64EC-MSVC: #define __AARCH64_CMODEL_SMALL__ 1
 // ARM64EC-MSVC: #define __ARM_64BIT_STATE 1
-// ARM64EC-MSVC: #define __ARM_ACLE 200
+// ARM64EC-MSVC: #define __ARM_ACLE 202420
+// ARM64EC-MSVC: #define __ARM_ACLE_VERSION(year,quarter,patch) (100 * (year) + 10 * (quarter) + (patch))
 // ARM64EC-MSVC: #define __ARM_ALIGN_MAX_STACK_PWR 4
 // ARM64EC-MSVC: #define __ARM_ARCH 8
 // ARM64EC-MSVC: #define __ARM_ARCH_ISA_A64 1
@@ -850,6 +858,7 @@
 // ARM64EC-MSVC: #define __FPCLASS_SNAN 0x0001
 // ARM64EC-MSVC: #define __FP_FAST_FMA 1
 // ARM64EC-MSVC: #define __FP_FAST_FMAF 1
+// ARM64EC-MSVC: #define __FUNCTION_MULTI_VERSIONING_SUPPORT_LEVEL 202420
 // ARM64EC-MSVC: #define __GCC_ASM_FLAG_OUTPUTS__ 1
 // ARM64EC-MSVC: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_1 1
 // ARM64EC-MSVC: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 1
diff --git a/clang/test/Preprocessor/predefined-macros-hlsl.hlsl b/clang/test/Preprocessor/predefined-macros-hlsl.hlsl
index bc3779e4129f0d..93a8455fd673b2 100644
--- a/clang/test/Preprocessor/predefined-macros-hlsl.hlsl
+++ b/clang/test/Preprocessor/predefined-macros-hlsl.hlsl
@@ -12,7 +12,7 @@
 // HALF: #define __HLSL_ENABLE_16_BIT 1
 // NOHALF-NOT: __HLSL_ENABLE_16_BIT
 
-// CHECK: #define __HLSL_VERSION 2021
+// CHECK: #define __HLSL_VERSION 2028
 
 // CHECK: #define __SHADER_STAGE_AMPLIFICATION 14
 // CHECK: #define __SHADER_STAGE_COMPUTE 5
diff --git a/clang/test/Sema/complex-arithmetic.c b/clang/test/Sema/complex-arithmetic.c
index c9e84da6daa9dc..6cafbfb47ab74d 100644
--- a/clang/test/Sema/complex-arithmetic.c
+++ b/clang/test/Sema/complex-arithmetic.c
@@ -1,5 +1,9 @@
-// RUN: %clang_cc1 -verify %s
-// expected-no-diagnostics
+// RUN: %clang_cc1 -verify=no-diag %s
+// RUN: %clang_cc1 -complex-range=promoted -triple x86_64-unknown-linux \
+// RUN: -verify=no-diag %s
+// RUN: %clang_cc1 -complex-range=promoted -triple x86_64-unknown-windows \
+// RUN: -verify=div-precision %s
+// no-diag-no-diagnostics
 
 // This tests evaluation of _Complex arithmetic at compile time.
 
@@ -21,6 +25,7 @@ void a() {
   EVALF((2. + 3i) + (4. + 5i), 6. + 8i);
   EVALF((2. + 3i) - (4. + 5i), -2. - 2i);
   EVALF((2. + 3i) * (4. + 5i), -7. + 22i);
+  // div-precision-warning@+1 {{excess precision is requested but the target does not support excess precision which may result in observable differences in complex division behavior, additional uses where the requested higher precision cannot be honored were found but not diagnosed}}
   EVALF((2. + 3i) / (4. + 5i), .5609 + .0487i);
 }
 
diff --git a/clang/test/SemaCXX/constant-expression-cxx11.cpp b/clang/test/SemaCXX/constant-expression-cxx11.cpp
index d2d2f1a127eaeb..44ef540f41fa8c 100644
--- a/clang/test/SemaCXX/constant-expression-cxx11.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx11.cpp
@@ -358,9 +358,11 @@ struct Str {
 
 extern char externalvar[];
 constexpr bool constaddress = (void *)externalvar == (void *)0x4000UL; // expected-error {{must be initialized by a constant expression}} expected-note {{reinterpret_cast}}
-constexpr bool litaddress = "foo" == "foo"; // cxx20_23-warning {{comparison between two arrays is deprecated}}
+constexpr bool litaddress = "foo" == "foo"; // expected-error {{must be initialized by a constant expression}}
+// expected-note@-1 {{comparison of addresses of literals has unspecified value}}
+// cxx20_23-warning@-2 {{comparison between two arrays is deprecated}}
 static_assert(0 != "foo", "");
-static_assert("foo" != "foo", "");// cxx20_23-warning {{comparison between two arrays is deprecated}}
+
 }
 
 namespace MaterializeTemporary {
diff --git a/clang/test/SemaCXX/warn-unsafe-buffer-usage-libc-functions.cpp b/clang/test/SemaCXX/warn-unsafe-buffer-usage-libc-functions.cpp
index 8e777a7a51c994..25b6d8e9e22dc2 100644
--- a/clang/test/SemaCXX/warn-unsafe-buffer-usage-libc-functions.cpp
+++ b/clang/test/SemaCXX/warn-unsafe-buffer-usage-libc-functions.cpp
@@ -20,6 +20,7 @@ int snwprintf_s( char* buffer, unsigned buf_size, const char* format, ... );
 int vsnprintf( char* buffer, unsigned buf_size, const char* format, ... );
 int sscanf_s(const char * buffer, const char * format, ...);
 int sscanf(const char * buffer, const char * format, ... );
+int wprintf(const wchar_t* format, ... );
 int __asan_printf();
 
 namespace std {
@@ -83,9 +84,11 @@ void f(char * p, char * q, std::span<char> s, std::span<char> s2) {
   sscanf(p, "%s%d", "hello", *p);    // expected-warning{{function 'sscanf' is unsafe}}
   sscanf_s(p, "%s%d", "hello", *p);  // expected-warning{{function 'sscanf_s' is unsafe}}
   fprintf((FILE*)p, "%P%d%p%i hello world %32s", *p, *p, p, *p, p); // expected-warning{{function 'fprintf' is unsafe}} expected-note{{string argument is not guaranteed to be null-terminated}}
+  wprintf(L"hello %s", p); // expected-warning{{function 'wprintf' is unsafe}} expected-note{{string argument is not guaranteed to be null-terminated}}
 
   char a[10], b[11];
   int c[10];
+  std::wstring WS;
 
   snprintf(a, sizeof(b), "%s", __PRETTY_FUNCTION__);         // expected-warning{{function 'snprintf' is unsafe}} expected-note{{buffer pointer and size may not match}}
   snprintf((char*)c, sizeof(c), "%s", __PRETTY_FUNCTION__);  // expected-warning{{function 'snprintf' is unsafe}} expected-note{{buffer pointer and size may not match}}
@@ -95,6 +98,8 @@ void f(char * p, char * q, std::span<char> s, std::span<char> s2) {
   snprintf(s.data(), s.size_bytes(), "%s%d", __PRETTY_FUNCTION__, *p); // no warn
   snwprintf(s.data(), s.size_bytes(), "%s%d", __PRETTY_FUNCTION__, *p); // no warn
   snwprintf_s(s.data(), s.size_bytes(), "%s%d", __PRETTY_FUNCTION__, *p); // no warn
+  wprintf(L"hello %ls", L"world"); // no warn
+  wprintf(L"hello %ls", WS.c_str()); // no warn
   strlen("hello");// no warn
   __asan_printf();// a printf but no argument, so no warn
 }
diff --git a/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl b/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl
index 774309c714f657..76b5d01b8036eb 100644
--- a/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl
@@ -14,6 +14,6 @@ RWBuffer<> BufferErr2;
 
 [numthreads(1,1,1)]
 void main() {
-  (void)Buffer.h; // expected-error {{'h' is a private member of 'hlsl::RWBuffer<vector<float, 3> >'}}
+  (void)Buffer.h; // expected-error {{'h' is a private member of 'hlsl::RWBuffer<vector<float, 3>>'}}
   // expected-note@* {{implicitly declared private here}}
 }
diff --git a/clang/test/SemaHLSL/BuiltIns/StructuredBuffers.hlsl b/clang/test/SemaHLSL/BuiltIns/StructuredBuffers.hlsl
index 2450941f5d9b46..a472d5519dc51f 100644
--- a/clang/test/SemaHLSL/BuiltIns/StructuredBuffers.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/StructuredBuffers.hlsl
@@ -14,6 +14,6 @@ StructuredBuffer<> BufferErr2;
 
 [numthreads(1,1,1)]
 void main() {
-  (void)Buffer.h; // expected-error {{'h' is a private member of 'hlsl::StructuredBuffer<vector<float, 3> >'}}
+  (void)Buffer.h; // expected-error {{'h' is a private member of 'hlsl::StructuredBuffer<vector<float, 3>>'}}
   // expected-note@* {{implicitly declared private here}}
 }
diff --git a/clang/test/SemaHLSL/Types/Arithmetic/literal_suffixes_202x.hlsl b/clang/test/SemaHLSL/Types/Arithmetic/literal_suffixes_202x.hlsl
index 2aeb4047565d6a..95f08463f55efe 100644
--- a/clang/test/SemaHLSL/Types/Arithmetic/literal_suffixes_202x.hlsl
+++ b/clang/test/SemaHLSL/Types/Arithmetic/literal_suffixes_202x.hlsl
@@ -37,7 +37,7 @@ _Static_assert(is_same<__decltype(4294967296), int64_t>::value, "4294967296 is i
 // Clang emits a warning that it is interpreting it as unsigned because that is
 // not conforming to the C standard.
 
-// expected-warning@+1{{integer literal is too large to be represented in type 'long' and is subject to undefined behavior under C++98, interpreting as 'unsigned long'; this literal will be ill-formed in C++11 onwards}}
+// expected-warning@+1{{integer literal is too large to be represented in a signed integer type, interpreting as unsigned}}
 static const uint64_t V = 9223372036854775808;
 
 _Static_assert(is_same<__decltype(0x0), int>::value, "0x0 is int");
diff --git a/clang/test/SemaHLSL/Types/Traits/IsIntangibleTypeErrors.hlsl b/clang/test/SemaHLSL/Types/Traits/IsIntangibleTypeErrors.hlsl
index 0803086749bd7d..de9ac90b895fc6 100644
--- a/clang/test/SemaHLSL/Types/Traits/IsIntangibleTypeErrors.hlsl
+++ b/clang/test/SemaHLSL/Types/Traits/IsIntangibleTypeErrors.hlsl
@@ -3,9 +3,10 @@
 struct Undefined; // expected-note {{forward declaration of 'Undefined'}}
 _Static_assert(!__builtin_hlsl_is_intangible(Undefined), ""); // expected-error{{incomplete type 'Undefined' used in type trait expression}}
 
-void fn(int X) {
+void fn(int X) { // expected-note {{declared here}}
   // expected-error@#vla {{variable length arrays are not supported for the current target}}
   // expected-error@#vla {{variable length arrays are not supported in '__builtin_hlsl_is_intangible'}}
   // expected-warning@#vla {{variable length arrays in C++ are a Clang extension}}
+  // expected-note@#vla {{function parameter 'X' with unknown value cannot be used in a constant expression}}
   _Static_assert(!__builtin_hlsl_is_intangible(int[X]), ""); // #vla
 }
diff --git a/clang/test/SemaHLSL/Types/Traits/ScalarizedLayoutCompatibleErrors.hlsl b/clang/test/SemaHLSL/Types/Traits/ScalarizedLayoutCompatibleErrors.hlsl
index 4c96795da7fd0c..85c6580d052de9 100644
--- a/clang/test/SemaHLSL/Types/Traits/ScalarizedLayoutCompatibleErrors.hlsl
+++ b/clang/test/SemaHLSL/Types/Traits/ScalarizedLayoutCompatibleErrors.hlsl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.6-library  -finclude-default-header -verify %s
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.6-library -finclude-default-header -verify %s
 
 // Some things that don't work!
 
@@ -14,11 +14,12 @@ _Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(Undefined, Defined
 
 // Case 2: No variable length arrays!
 
-void fn(int X) {
+void fn(int X) { // expected-note{{declared here}}
   // expected-error@#vla {{variable length arrays are not supported for the current target}}
   // expected-error@#vla {{variable length arrays are not supported in '__builtin_hlsl_is_scalarized_layout_compatible'}}
   // expected-error@#vla {{static assertion failed due to requirement '__builtin_hlsl_is_scalarized_layout_compatible(int[4], int[X])'}}
   // expected-warning@#vla {{variable length arrays in C++ are a Clang extension}}
+  // expected-note@#vla{{function parameter 'X' with unknown value cannot be used in a constant expression}}
   _Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(int[4], int[X]), ""); // #vla
 }
 
diff --git a/clang/test/SemaHLSL/group_shared.hlsl b/clang/test/SemaHLSL/group_shared.hlsl
index b51114700b041d..0293ddd15eeb94 100644
--- a/clang/test/SemaHLSL/group_shared.hlsl
+++ b/clang/test/SemaHLSL/group_shared.hlsl
@@ -58,7 +58,6 @@ template<typename T>
   T tfoo(T t) {
      return t;
   }
-  // expected-warning@+1 {{alias declarations are a C++11 extension}}
  using GSF = groupshared float;
  GSF gs;
  // expected-error@+1 {{no matching function for call to 'tfoo'}}
@@ -73,8 +72,7 @@ groupshared void (*fp)();
 void (*fp2)(groupshared float);
 // NOTE: HLSL not support trailing return types.
 // expected-warning@#func{{'auto' type specifier is a HLSL 202y extension}}
-// expected-warning@#func {{'auto' type specifier is a C++11 extension}}
-// expected-error@#func {{expected function body after function declarator}}
+// expected-error@#func{{return type cannot be qualified with address space}}
 auto func() -> groupshared void; // #func
 // expected-warning@+2 {{'groupshared' attribute only applies to variables}}
 // expected-error@+1 {{return type cannot be qualified with address space}}
diff --git a/clang/test/SemaHLSL/prohibit_reference.hlsl b/clang/test/SemaHLSL/prohibit_reference.hlsl
index 6e76cfa1422381..11ee9b20cf5544 100644
--- a/clang/test/SemaHLSL/prohibit_reference.hlsl
+++ b/clang/test/SemaHLSL/prohibit_reference.hlsl
@@ -3,7 +3,6 @@
 int& bark(int); // expected-error {{references are unsupported in HLSL}}
 void meow(int&); // expected-error {{references are unsupported in HLSL}}
 void chirp(int &&); // expected-error {{references are unsupported in HLSL}}
-// expected-warning@-1 {{rvalue references are a C++11 extension}}
 
 struct Foo {
   int X;
diff --git a/clang/test/SemaHLSL/resource_binding_attr_error.hlsl b/clang/test/SemaHLSL/resource_binding_attr_error.hlsl
index cb728dca838c3b..74aff79f0e37fb 100644
--- a/clang/test/SemaHLSL/resource_binding_attr_error.hlsl
+++ b/clang/test/SemaHLSL/resource_binding_attr_error.hlsl
@@ -13,16 +13,9 @@ float a : register(c0);
 cbuffer b : register(i0) {
 
 }
-// expected-error@+1 {{invalid space specifier 's2' used; expected 'space' followed by an integer, like space1}}
-cbuffer c : register(b0, s2) {
 
-}
 // expected-error@+1 {{register number should be an integer}}
-cbuffer d : register(bf, s2) {
-
-}
-// expected-error@+1 {{invalid space specifier 'spaces' used; expected 'space' followed by an integer, like space1}}
-cbuffer e : register(b2, spaces) {
+cbuffer c : register(bf, s2) {
 
 }
 
@@ -35,9 +28,8 @@ cbuffer B : register(space1) {}
 // expected-error@+1 {{wrong argument format for hlsl attribute, use b2 instead}}
 cbuffer C : register(b 2) {}
 
-// expected-error@+2 {{wrong argument format for hlsl attribute, use b2 instead}}
-// expected-error@+1 {{wrong argument format for hlsl attribute, use space3 instead}}
-cbuffer D : register(b 2, space 3) {}
+// expected-error@+1 {{wrong argument format for hlsl attribute, use b2 instead}}
+cbuffer D : register(b 2, space3) {}
 
 // expected-error@+1 {{'register' attribute only applies to cbuffer/tbuffer and external global variables}}
 static MyTemplatedSRV<float> U : register(u5);
@@ -61,9 +53,6 @@ void foo2() {
   extern MyTemplatedSRV<float> U2 : register(u5);
 }
 
-// expected-error@+1 {{binding type 'u' only applies to UAV resources}}
-float b : register(u0, space1);
-
 // expected-error@+1 {{'register' attribute only applies to cbuffer/tbuffer and external global variables}}
 void bar(MyTemplatedSRV<float> U : register(u3)) {
 
diff --git a/clang/test/SemaHLSL/resource_binding_attr_error_basic.hlsl b/clang/test/SemaHLSL/resource_binding_attr_error_basic.hlsl
index 0a547ed66af0a2..760c057630a7fa 100644
--- a/clang/test/SemaHLSL/resource_binding_attr_error_basic.hlsl
+++ b/clang/test/SemaHLSL/resource_binding_attr_error_basic.hlsl
@@ -3,37 +3,40 @@
 // expected-error@+1{{binding type 't' only applies to SRV resources}}
 float f1 : register(t0);
 
-
-float f2 : register(c0);
+// expected-error@+1 {{binding type 'u' only applies to UAV resources}}
+float f2 : register(u0);
 
 // expected-error@+1{{binding type 'b' only applies to constant buffers. The 'bool constant' binding type is no longer supported}}
 float f3 : register(b9);
 
+// expected-error@+1 {{binding type 's' only applies to sampler state}}
+float f4 : register(s0);
+
 // expected-error@+1{{binding type 'i' ignored. The 'integer constant' binding type is no longer supported}}
-float f4 : register(i9);
+float f5 : register(i9);
 
 // expected-error@+1{{binding type 'x' is invalid}}
-float f5 : register(x9);
+float f6 : register(x9);
 
 cbuffer g_cbuffer1 {
 // expected-error@+1{{binding type 'c' ignored in buffer declaration. Did you mean 'packoffset'?}}
-    float f6 : register(c2);
+    float f7 : register(c2);
 };
 
 tbuffer g_tbuffer1 {
 // expected-error@+1{{binding type 'c' ignored in buffer declaration. Did you mean 'packoffset'?}}
-    float f7 : register(c2);
+    float f8 : register(c2);
 };
 
 cbuffer g_cbuffer2 {
 // expected-error@+1{{binding type 'b' only applies to constant buffer resources}}
-    float f8 : register(b2);
+    float f9 : register(b2);
 };
 
 tbuffer g_tbuffer2 {
 // expected-error@+1{{binding type 'i' ignored. The 'integer constant' binding type is no longer supported}}
-    float f9 : register(i2);
+    float f10 : register(i2);
 };
 
 // expected-error@+1{{binding type 'c' only applies to numeric variables in the global scope}}
-RWBuffer<float> f10 : register(c3);
+RWBuffer<float> f11 : register(c3);
diff --git a/clang/test/SemaHLSL/resource_binding_attr_error_space.hlsl b/clang/test/SemaHLSL/resource_binding_attr_error_space.hlsl
new file mode 100644
index 00000000000000..70e64e6ca75280
--- /dev/null
+++ b/clang/test/SemaHLSL/resource_binding_attr_error_space.hlsl
@@ -0,0 +1,62 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -o - -fsyntax-only %s -verify
+
+// valid
+cbuffer cbuf {
+    RWBuffer<int> r : register(u0, space0);
+}
+
+cbuffer cbuf2 {
+    struct x {
+        // this test validates that no diagnostic is emitted on the space parameter, because
+        // this register annotation is not in the global scope.
+        // expected-error@+1 {{'register' attribute only applies to cbuffer/tbuffer and external global variables}}
+        RWBuffer<int> E : register(u2, space3);
+    };
+}
+
+struct MyStruct {
+    RWBuffer<int> E;
+};
+
+cbuffer cbuf3 {
+  // valid
+  MyStruct E : register(u2, space3);
+}
+
+// valid
+MyStruct F : register(u3, space4);
+
+cbuffer cbuf4 {
+  // this test validates that no diagnostic is emitted on the space parameter, because
+  // this register annotation is not in the global scope.
+  // expected-error@+1 {{binding type 'u' only applies to UAV resources}}
+  float a : register(u2, space3); 
+}
+
+// expected-error@+1 {{invalid space specifier 's2' used; expected 'space' followed by an integer, like space1}}
+cbuffer a : register(b0, s2) {
+
+}
+
+// expected-error@+1 {{invalid space specifier 'spaces' used; expected 'space' followed by an integer, like space1}}
+cbuffer b : register(b2, spaces) {
+
+}
+
+// expected-error@+1 {{wrong argument format for hlsl attribute, use space3 instead}}
+cbuffer c : register(b2, space 3) {}
+
+// expected-error@+1 {{register space cannot be specified on global constants}}
+int d : register(c2, space3);
+
+// expected-error@+1 {{register space cannot be specified on global constants}}
+int e : register(c2, space0);
+
+// expected-error@+1 {{register space cannot be specified on global constants}}
+int f : register(c2, space00);
+
+// valid
+RWBuffer<int> g : register(u2, space0);
+
+// valid
+RWBuffer<int> h : register(u2, space0);
diff --git a/clang/test/utils/update-verify-tests/lit.local.cfg b/clang/test/utils/update-verify-tests/lit.local.cfg
index a0b6afccc25010..b0eebf337da5c9 100644
--- a/clang/test/utils/update-verify-tests/lit.local.cfg
+++ b/clang/test/utils/update-verify-tests/lit.local.cfg
@@ -23,3 +23,6 @@ else:
             "%s %s" % (python, shell_quote(script_path)),
         )
     )
+    # AIX 'diff' command doesn't support --strip-trailing-cr, but the internal
+    # python implementation does, so use that for cross platform compatibility
+    config.test_format = lit.formats.ShTest()
diff --git a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
index 028392f499da3b..ec0be27774d8b2 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
@@ -5052,6 +5052,19 @@ TEST(ForEachConstructorInitializer, MatchesInitializers) {
     cxxConstructorDecl(forEachConstructorInitializer(cxxCtorInitializer()))));
 }
 
+TEST(LambdaCapture, InvalidLambdaCapture) {
+  // not crash
+  EXPECT_FALSE(matches(
+      R"(int main() {
+        struct A { A()=default; A(A const&)=delete; };
+        A a; [a]() -> void {}();
+        return 0;
+      })",
+      traverse(TK_IgnoreUnlessSpelledInSource,
+               lambdaExpr(has(lambdaCapture()))),
+      langCxx11OrLater()));
+}
+
 TEST(ForEachLambdaCapture, MatchesCaptures) {
   EXPECT_TRUE(matches(
       "int main() { int x, y; auto f = [x, y]() { return x + y; }; }",
diff --git a/clang/www/c_status.html b/clang/www/c_status.html
index 6e37dd65ccbdee..148405ec31a9ba 100644
--- a/clang/www/c_status.html
+++ b/clang/www/c_status.html
@@ -532,7 +532,7 @@ <h2 id="c2x">C23 implementation status</h2>
     <tr>
       <td>Contradiction about INFINITY macro</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n2848.pdf">N2848</a></td>
-      <td class="unreleased" align="center">Clang 19</td>
+      <td class="full" align="center">Clang 19</td>
     </tr>
     <tr>
       <td>Require exact-width integer type interfaces</td>
@@ -603,7 +603,7 @@ <h2 id="c2x">C23 implementation status</h2>
     <tr>
       <td>char8_t: A type for UTF-8 characters and strings</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n2653.htm">N2653</a></td>
-      <td class="unreleased" align="center">Clang 19</td>
+      <td class="full" align="center">Clang 19</td>
     </tr>
     <tr>
       <td>Clarification for max exponent macros-update</td>
@@ -727,7 +727,7 @@ <h2 id="c2x">C23 implementation status</h2>
     <tr>
       <td>constexpr for object definitions</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3018.htm">N3018</a></td>
-      <td class="unreleased" align="center">Clang 19</td>
+      <td class="full" align="center">Clang 19</td>
     </tr>
     <tr>
       <td>Introduce storage class specifiers for compound literals</td>
@@ -757,7 +757,7 @@ <h2 id="c2x">C23 implementation status</h2>
     <tr>
       <td>#embed</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3017.htm">N3017</a></td>
-      <td class="unreleased" align="center">Clang 19</td>
+      <td class="full" align="center">Clang 19</td>
     </tr>
 </table>
 </details>
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index f036fc5add2413..e5c5e50104fdaf 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -945,7 +945,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/150.html">150</a></td>
     <td>C++17</td>
     <td>Template template parameters and default arguments</td>
-    <td class="unreleased" align="center">Clang 19</td>
+    <td class="full" align="center">Clang 19</td>
   </tr>
   <tr id="151">
     <td><a href="https://cplusplus.github.io/CWG/issues/151.html">151</a></td>
@@ -1017,7 +1017,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/162.html">162</a></td>
     <td>CD1</td>
     <td>(<TT>&amp;C::f)()</TT> with nonstatic members</td>
-    <td class="unreleased" align="center">Clang 19</td>
+    <td class="full" align="center">Clang 19</td>
   </tr>
   <tr id="163">
     <td><a href="https://cplusplus.github.io/CWG/issues/163.html">163</a></td>
@@ -7827,7 +7827,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/1334.html">1334</a></td>
     <td>NAD</td>
     <td>Layout compatibility and cv-qualification</td>
-    <td class="unreleased-superseded" align="center">Superseded by <a href="#1719">1719</a></td>
+    <td class="full-superseded" align="center">Superseded by <a href="#1719">1719</a></td>
   </tr>
   <tr id="1335">
     <td><a href="https://cplusplus.github.io/CWG/issues/1335.html">1335</a></td>
@@ -10141,7 +10141,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/1719.html">1719</a></td>
     <td>CD4</td>
     <td>Layout compatibility and cv-qualification revisited</td>
-    <td class="unreleased" align="center">Clang 19</td>
+    <td class="full" align="center">Clang 19</td>
   </tr>
   <tr id="1720">
     <td><a href="https://cplusplus.github.io/CWG/issues/1720.html">1720</a></td>
@@ -11809,11 +11809,11 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td>Reference list-initialization ignores conversion functions</td>
     <td align="center">Not resolved</td>
   </tr>
-  <tr class="open" id="1997">
+  <tr id="1997">
     <td><a href="https://cplusplus.github.io/CWG/issues/1997.html">1997</a></td>
-    <td>drafting</td>
+    <td>DRWP</td>
     <td>Placement new and previous initialization</td>
-    <td align="center">Not resolved</td>
+    <td class="unknown" align="center">Unknown</td>
   </tr>
   <tr id="1998">
     <td><a href="https://cplusplus.github.io/CWG/issues/1998.html">1998</a></td>
@@ -12977,7 +12977,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2191.html">2191</a></td>
     <td>C++17</td>
     <td>Incorrect result for <TT>noexcept(typeid(v))</TT></td>
-    <td class="unreleased" align="center">Clang 19</td>
+    <td class="full" align="center">Clang 19</td>
   </tr>
   <tr class="open" id="2192">
     <td><a href="https://cplusplus.github.io/CWG/issues/2192.html">2192</a></td>
@@ -14403,7 +14403,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2428.html">2428</a></td>
     <td>C++23</td>
     <td>Deprecating a concept</td>
-    <td class="unreleased" align="center">Clang 19</td>
+    <td class="full" align="center">Clang 19</td>
   </tr>
   <tr id="2429">
     <td><a href="https://cplusplus.github.io/CWG/issues/2429.html">2429</a></td>
@@ -14505,7 +14505,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2445.html">2445</a></td>
     <td>C++20</td>
     <td>Partial ordering with rewritten candidates</td>
-    <td class="unreleased" align="center">Clang 19</td>
+    <td class="full" align="center">Clang 19</td>
   </tr>
   <tr id="2446">
     <td><a href="https://cplusplus.github.io/CWG/issues/2446.html">2446</a></td>
@@ -15345,7 +15345,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2583.html">2583</a></td>
     <td>C++23</td>
     <td>Common initial sequence should consider over-alignment</td>
-    <td class="unreleased" align="center">Clang 19</td>
+    <td class="full" align="center">Clang 19</td>
   </tr>
   <tr class="open" id="2584">
     <td><a href="https://cplusplus.github.io/CWG/issues/2584.html">2584</a></td>
@@ -15573,7 +15573,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2621.html">2621</a></td>
     <td>C++23</td>
     <td>Kind of lookup for <TT>using enum</TT> declarations</td>
-    <td class="unreleased-superseded" align="center">Superseded by <a href="#2877">2877</a></td>
+    <td class="full-superseded" align="center">Superseded by <a href="#2877">2877</a></td>
   </tr>
   <tr id="2622">
     <td><a href="https://cplusplus.github.io/CWG/issues/2622.html">2622</a></td>
@@ -15999,7 +15999,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2692.html">2692</a></td>
     <td>C++23</td>
     <td>Static and explicit object member functions with the same parameter-type-lists</td>
-    <td class="unreleased" align="center">Clang 19</td>
+    <td class="full" align="center">Clang 19</td>
   </tr>
   <tr class="open" id="2693">
     <td><a href="https://cplusplus.github.io/CWG/issues/2693.html">2693</a></td>
@@ -16401,7 +16401,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2759.html">2759</a></td>
     <td>DRWP</td>
     <td>[[no_unique_address] and common initial sequence</td>
-    <td class="unreleased" align="center">Clang 19</td>
+    <td class="full" align="center">Clang 19</td>
   </tr>
   <tr id="2760">
     <td><a href="https://cplusplus.github.io/CWG/issues/2760.html">2760</a></td>
@@ -16762,7 +16762,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2819.html">2819</a></td>
     <td>accepted</td>
     <td>Cast from null pointer value in a constant expression</td>
-    <td class="unreleased" align="center">Clang 19</td>
+    <td class="full" align="center">Clang 19</td>
   </tr>
   <tr id="2820">
     <td><a href="https://cplusplus.github.io/CWG/issues/2820.html">2820</a></td>
@@ -17000,7 +17000,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2858.html">2858</a></td>
     <td>accepted</td>
     <td>Declarative <I>nested-name-specifier</I>s and <I>pack-index-specifier</I>s</td>
-    <td class="unreleased" align="center">Clang 19</td>
+    <td class="full" align="center">Clang 19</td>
   </tr>
   <tr id="2859">
     <td><a href="https://cplusplus.github.io/CWG/issues/2859.html">2859</a></td>
@@ -17114,7 +17114,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2877.html">2877</a></td>
     <td>DR</td>
     <td>Type-only lookup for <I>using-enum-declarator</I></td>
-    <td class="unreleased" align="center">Clang 19</td>
+    <td class="full" align="center">Clang 19</td>
   </tr>
   <tr class="open" id="2878">
     <td><a href="https://cplusplus.github.io/CWG/issues/2878.html">2878</a></td>
@@ -17138,7 +17138,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2881.html">2881</a></td>
     <td>DR</td>
     <td>Type restrictions for the explicit object parameter of a lambda</td>
-    <td class="unreleased" align="center">Clang 19</td>
+    <td class="full" align="center">Clang 19</td>
   </tr>
   <tr id="2882">
     <td><a href="https://cplusplus.github.io/CWG/issues/2882.html">2882</a></td>
@@ -17346,11 +17346,15 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2915.html">2915</a></td>
     <td>tentatively ready</td>
     <td>Explicit object parameters of type <TT>void</TT></td>
-    <td align="center">Not resolved</td>
+    <td align="center">
+      <details>
+        <summary>Not resolved</summary>
+        Clang 20 implements 2024-08-16 resolution
+      </details></td>
   </tr>
   <tr class="open" id="2916">
     <td><a href="https://cplusplus.github.io/CWG/issues/2916.html">2916</a></td>
-    <td>tentatively ready</td>
+    <td>review</td>
     <td>Variable template partial specializations should not be declared <TT>static</TT></td>
     <td align="center">Not resolved</td>
   </tr>
@@ -17427,6 +17431,60 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td>open</td>
     <td>Unclear status of translation unit with <TT>module</TT> keyword</td>
     <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2928">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2928.html">2928</a></td>
+    <td>open</td>
+    <td>No ordering for initializing thread-local variables</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2929">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2929.html">2929</a></td>
+    <td>open</td>
+    <td>Lifetime of trivially-destructible static or thread-local objects</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2930">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2930.html">2930</a></td>
+    <td>open</td>
+    <td>Unclear term "copy/move operation" in specification of copy elision</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2931">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2931.html">2931</a></td>
+    <td>open</td>
+    <td>Restrictions on operator functions that are explicit object member functions</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2932">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2932.html">2932</a></td>
+    <td>open</td>
+    <td>Value range of empty enumeration</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2933">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2933.html">2933</a></td>
+    <td>open</td>
+    <td>Dangling references</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2934">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2934.html">2934</a></td>
+    <td>open</td>
+    <td>Unclear semantics of exception escaping from <TT>unhandled_exception</TT></td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2935">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2935.html">2935</a></td>
+    <td>open</td>
+    <td>Destroying the coroutine state when initial-await-resume-called is false</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2936">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2936.html">2936</a></td>
+    <td>open</td>
+    <td>Local classes of templated functions should be part of the current instantiation</td>
+    <td align="center">Not resolved</td>
   </tr></table>
 
 </div>
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index 58bbb12a76dd75..3f6a46c08c8514 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -156,7 +156,7 @@ <h2 id="cxx26">C++2c implementation status</h2>
  <tr>
   <td>Pack Indexing</td>
   <td><a href="https://wg21.link/P2662R3">P2662R3</a></td>
-  <td class="unreleased" align="center">Clang 19</td>
+  <td class="full" align="center">Clang 19</td>
  </tr>
  <tr>
   <td>Remove Deprecated Arithmetic Conversion on Enumerations</td>
@@ -167,7 +167,7 @@ <h2 id="cxx26">C++2c implementation status</h2>
  <tr>
   <td>Disallow Binding a Returned Glvalue to a Temporary</td>
   <td><a href="https://wg21.link/P2748R5">P2748R5</a></td>
-  <td class="unreleased" align="center">Clang 19</td>
+  <td class="full" align="center">Clang 19</td>
  </tr>
  <tr>
   <td>Clarifying rules for brace elision in aggregate initialization</td>
@@ -177,7 +177,7 @@ <h2 id="cxx26">C++2c implementation status</h2>
  <tr>
   <td>Attributes for Structured Bindings</td>
   <td><a href="https://wg21.link/P0609R3">P0609R3</a></td>
-  <td class="unreleased" align="center">Clang 19</td>
+  <td class="full" align="center">Clang 19</td>
  </tr>
  <tr>
   <td>Module Declarations Shouldn’t be Macros</td>
@@ -187,7 +187,7 @@ <h2 id="cxx26">C++2c implementation status</h2>
  <tr>
   <td>Trivial infinite loops are not Undefined Behavior</td>
   <td><a href="https://wg21.link/P2809R3">P2809R3</a> (<a href="#dr">DR</a>)</td>
-  <td class="unreleased" align="center">Clang 19</td>
+  <td class="full" align="center">Clang 19</td>
  </tr>
  <tr>
   <td>Erroneous behaviour for uninitialized reads</td>
@@ -197,7 +197,7 @@ <h2 id="cxx26">C++2c implementation status</h2>
  <tr>
   <td><tt>= delete("should have a reason");</tt></td>
   <td><a href="https://wg21.link/P2573R2">P2573R2</a></td>
-  <td class="unreleased" align="center">Clang 19</td>
+  <td class="full" align="center">Clang 19</td>
  </tr>
  <tr>
   <td>Variadic friends</td>
@@ -213,12 +213,12 @@ <h2 id="cxx26">C++2c implementation status</h2>
  <tr>
   <td>Deleting a Pointer to an Incomplete Type Should be Ill-formed</td>
   <td><a href="https://wg21.link/P3144">P3144R2</a></td>
-  <td class="unreleased" align="center">Clang 19</td>
+  <td class="full" align="center">Clang 19</td>
  </tr>
  <tr>
   <td>Ordering of constraints involving fold expressions</td>
   <td><a href="https://wg21.link/P2963R3">P2963R3</a></td>
-  <td class="unreleased" align="center">Clang 19</td>
+  <td class="full" align="center">Clang 19</td>
  </tr>
  <tr>
   <td>Structured binding declaration as a condition</td>
@@ -304,7 +304,7 @@ <h2 id="cxx23">C++23 implementation status</h2>
     </tr>
     <tr>
       <td><a href="https://wg21.link/P2797R0">P2797R0</a></td>
-      <td class="unreleased" align="center">Clang 19</td>
+      <td class="full" align="center">Clang 19</td>
     </tr>
     <tr>
       <td rowspan=2>Change scope of lambda trailing-return-type</td>
@@ -368,7 +368,7 @@ <h2 id="cxx23">C++23 implementation status</h2>
     <tr>
       <td>Type trait to determine if a reference binds to a temporary</td>
       <td><a href="https://wg21.link/P2255R2">P2255R2</a></td>
-      <td class="unreleased" align="center">Clang 19</td>
+      <td class="full" align="center">Clang 19</td>
     </tr>
     <!-- July 2022 papers -->
     <tr>
@@ -409,7 +409,7 @@ <h2 id="cxx23">C++23 implementation status</h2>
     <tr>
       <td>Relaxing some constexpr restrictions</td>
       <td><a href="https://wg21.link/P2448R2">P2448R2</a></td>
-      <td class="unreleased" align="center">Clang 19</td>
+      <td class="full" align="center">Clang 19</td>
     </tr>
     <tr>
       <td>Using unknown pointers and references in constant expressions</td>
@@ -434,7 +434,7 @@ <h2 id="cxx23">C++23 implementation status</h2>
     <tr>
       <td>Portable assumptions</td>
       <td><a href="https://wg21.link/P1774R8">P1774R8</a></td>
-      <td class="unreleased" align="center">Clang 19</td>
+      <td class="full" align="center">Clang 19</td>
     </tr>
     <tr>
       <td>Support for UTF-8 as a portable source file encoding</td>
@@ -564,7 +564,7 @@ <h2 id="cxx20">C++20 implementation status</h2>
       </tr>
       <tr> <!-- from Cologne -->
         <td><a href="https://wg21.link/p0848r3">P0848R3</a></td>
-        <td rowspan="1" class="unreleased" align="center">Clang 19</td>
+        <td rowspan="1" class="full" align="center">Clang 19</td>
       </tr>
       <tr>
         <td><a href="https://wg21.link/p1616r1">P1616R1</a></td>
@@ -1154,7 +1154,7 @@ <h2 id="cxx17">C++17 implementation status</h2>
     <tr>
       <td>Matching template template parameters to compatible arguments</td>
       <td><a href="https://wg21.link/p0522r0">P0522R0</a> (<a href="#dr">DR</a>)</td>
-      <td class="unreleased" align="center">Clang 19 <a href="#p0522">(10)</a></td>
+      <td class="full" align="center">Clang 19 <a href="#p0522">(10)</a></td>
     </tr>
     <tr>
       <td>Removing deprecated dynamic exception specifications</td>
diff --git a/clang/www/make_cxx_dr_status b/clang/www/make_cxx_dr_status
index e245d0447baafc..f9a35c61c12dea 100755
--- a/clang/www/make_cxx_dr_status
+++ b/clang/www/make_cxx_dr_status
@@ -1,7 +1,7 @@
 #! /usr/bin/env python3
 import sys, os, re, urllib.request
 
-latest_release = 18
+latest_release = 19
 
 clang_www_dir = os.path.dirname(__file__)
 default_issue_list_path = os.path.join(clang_www_dir, 'cwg_index.html')
diff --git a/compiler-rt/lib/asan/asan_linux.cpp b/compiler-rt/lib/asan/asan_linux.cpp
index 0b470db86748fb..4cabca388ca9a1 100644
--- a/compiler-rt/lib/asan/asan_linux.cpp
+++ b/compiler-rt/lib/asan/asan_linux.cpp
@@ -187,10 +187,7 @@ void AsanCheckIncompatibleRT() {
       MemoryMappedSegment segment(filename, sizeof(filename));
       while (proc_maps.Next(&segment)) {
         if (IsDynamicRTName(segment.filename)) {
-          Report(
-              "Your application is linked against "
-              "incompatible ASan runtimes.\n");
-          Die();
+          ReportIncompatibleRT();
         }
       }
       __asan_rt_version = ASAN_RT_VERSION_STATIC;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp
index 3966d82d51ee96..684720963a8dcb 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp
@@ -171,7 +171,7 @@ void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name,
         "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
         "Perhaps you're using ulimit -v or ulimit -d\n",
         size);
-    Abort();
+    Die();
   }
   if (madvise_shadow && common_flags()->use_madv_dontdump)
     DontDumpShadowMemory(beg, size);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
index 51c1f9ff111de9..53add5a9b16423 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
@@ -40,6 +40,10 @@
 #  include <sys/resource.h>
 #  include <syslog.h>
 
+#  if SANITIZER_GLIBC
+#    include <gnu/libc-version.h>
+#  endif
+
 #  if !defined(ElfW)
 #    define ElfW(type) Elf_##type
 #  endif
@@ -196,27 +200,6 @@ bool SetEnv(const char *name, const char *value) {
 }
 #  endif
 
-__attribute__((unused)) static bool GetLibcVersion(int *major, int *minor,
-                                                   int *patch) {
-#  ifdef _CS_GNU_LIBC_VERSION
-  char buf[64];
-  uptr len = confstr(_CS_GNU_LIBC_VERSION, buf, sizeof(buf));
-  if (len >= sizeof(buf))
-    return false;
-  buf[len] = 0;
-  static const char kGLibC[] = "glibc ";
-  if (internal_strncmp(buf, kGLibC, sizeof(kGLibC) - 1) != 0)
-    return false;
-  const char *p = buf + sizeof(kGLibC) - 1;
-  *major = internal_simple_strtoll(p, &p, 10);
-  *minor = (*p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
-  *patch = (*p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
-  return true;
-#  else
-  return false;
-#  endif
-}
-
 // True if we can use dlpi_tls_data. glibc before 2.25 may leave NULL (BZ
 // #19826) so dlpi_tls_data cannot be used.
 //
@@ -226,11 +209,22 @@ __attribute__((unused)) static bool GetLibcVersion(int *major, int *minor,
 __attribute__((unused)) static int g_use_dlpi_tls_data;
 
 #  if SANITIZER_GLIBC && !SANITIZER_GO
+
+static void GetGLibcVersion(int *major, int *minor, int *patch) {
+  const char *p = gnu_get_libc_version();
+  *major = internal_simple_strtoll(p, &p, 10);
+  // Caller does not expect anything else.
+  CHECK_EQ(*major, 2);
+  *minor = (*p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
+  *patch = (*p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
+}
+
 __attribute__((unused)) static size_t g_tls_size;
+
 void InitTlsSize() {
   int major, minor, patch;
-  g_use_dlpi_tls_data =
-      GetLibcVersion(&major, &minor, &patch) && major == 2 && minor >= 25;
+  GetGLibcVersion(&major, &minor, &patch);
+  g_use_dlpi_tls_data = major == 2 && minor >= 25;
 
 #    if defined(__aarch64__) || defined(__x86_64__) || \
         defined(__powerpc64__) || defined(__loongarch__)
@@ -246,77 +240,87 @@ void InitTlsSize() {}
 // On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage
 // of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan
 // to get the pointer to thread-specific data keys in the thread control block.
-#  if (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_SOLARIS) && \
-      !SANITIZER_ANDROID && !SANITIZER_GO
+#  if (SANITIZER_FREEBSD || SANITIZER_GLIBC) && !SANITIZER_GO
 // sizeof(struct pthread) from glibc.
 static atomic_uintptr_t thread_descriptor_size;
 
+// FIXME: Implementation is very GLIBC specific, but it's used by FREEBSD.
 static uptr ThreadDescriptorSizeFallback() {
-  uptr val = 0;
-#    if defined(__x86_64__) || defined(__i386__) || defined(__arm__)
+#    if defined(__x86_64__) || defined(__i386__) || defined(__arm__) || \
+        SANITIZER_RISCV64
+#      if SANITIZER_GLIBC
   int major;
   int minor;
   int patch;
-  if (GetLibcVersion(&major, &minor, &patch) && major == 2) {
-    /* sizeof(struct pthread) values from various glibc versions.  */
-    if (SANITIZER_X32)
-      val = 1728;  // Assume only one particular version for x32.
-    // For ARM sizeof(struct pthread) changed in Glibc 2.23.
-    else if (SANITIZER_ARM)
-      val = minor <= 22 ? 1120 : 1216;
-    else if (minor <= 3)
-      val = FIRST_32_SECOND_64(1104, 1696);
-    else if (minor == 4)
-      val = FIRST_32_SECOND_64(1120, 1728);
-    else if (minor == 5)
-      val = FIRST_32_SECOND_64(1136, 1728);
-    else if (minor <= 9)
-      val = FIRST_32_SECOND_64(1136, 1712);
-    else if (minor == 10)
-      val = FIRST_32_SECOND_64(1168, 1776);
-    else if (minor == 11 || (minor == 12 && patch == 1))
-      val = FIRST_32_SECOND_64(1168, 2288);
-    else if (minor <= 14)
-      val = FIRST_32_SECOND_64(1168, 2304);
-    else if (minor < 32)  // Unknown version
-      val = FIRST_32_SECOND_64(1216, 2304);
-    else  // minor == 32
-      val = FIRST_32_SECOND_64(1344, 2496);
-  }
-#    elif defined(__s390__) || defined(__sparc__)
+  GetGLibcVersion(&major, &minor, &patch);
+#      else   // SANITIZER_GLIBC
+  return 0;
+#      endif  // SANITIZER_GLIBC
+#    endif
+
+#    if defined(__x86_64__) || defined(__i386__) || defined(__arm__)
+  /* sizeof(struct pthread) values from various glibc versions.  */
+  if (SANITIZER_X32)
+    return 1728;  // Assume only one particular version for x32.
+  // For ARM sizeof(struct pthread) changed in Glibc 2.23.
+  if (SANITIZER_ARM)
+    return minor <= 22 ? 1120 : 1216;
+  if (minor <= 3)
+    return FIRST_32_SECOND_64(1104, 1696);
+  if (minor == 4)
+    return FIRST_32_SECOND_64(1120, 1728);
+  if (minor == 5)
+    return FIRST_32_SECOND_64(1136, 1728);
+  if (minor <= 9)
+    return FIRST_32_SECOND_64(1136, 1712);
+  if (minor == 10)
+    return FIRST_32_SECOND_64(1168, 1776);
+  if (minor == 11 || (minor == 12 && patch == 1))
+    return FIRST_32_SECOND_64(1168, 2288);
+  if (minor <= 14)
+    return FIRST_32_SECOND_64(1168, 2304);
+  if (minor < 32)  // Unknown version
+    return FIRST_32_SECOND_64(1216, 2304);
+  // minor == 32
+  return FIRST_32_SECOND_64(1344, 2496);
+#    endif
+
+#    if SANITIZER_RISCV64
+  // TODO: consider adding an optional runtime check for an unknown (untested)
+  // glibc version
+  if (minor <= 28)  // WARNING: the highest tested version is 2.29
+    return 1772;    // no guarantees for this one
+  if (minor <= 31)
+    return 1772;  // tested against glibc 2.29, 2.31
+  return 1936;    // tested against glibc 2.32
+#    endif
+
+#    if defined(__s390__) || defined(__sparc__)
   // The size of a prefix of TCB including pthread::{specific_1stblock,specific}
   // suffices. Just return offsetof(struct pthread, specific_used), which hasn't
   // changed since 2007-05. Technically this applies to i386/x86_64 as well but
   // we call _dl_get_tls_static_info and need the precise size of struct
   // pthread.
   return FIRST_32_SECOND_64(524, 1552);
-#    elif defined(__mips__)
+#    endif
+
+#    if defined(__mips__)
   // TODO(sagarthakur): add more values as per different glibc versions.
-  val = FIRST_32_SECOND_64(1152, 1776);
-#    elif SANITIZER_LOONGARCH64
-  val = 1856;  // from glibc 2.36
-#    elif SANITIZER_RISCV64
-  int major;
-  int minor;
-  int patch;
-  if (GetLibcVersion(&major, &minor, &patch) && major == 2) {
-    // TODO: consider adding an optional runtime check for an unknown (untested)
-    // glibc version
-    if (minor <= 28)  // WARNING: the highest tested version is 2.29
-      val = 1772;     // no guarantees for this one
-    else if (minor <= 31)
-      val = 1772;  // tested against glibc 2.29, 2.31
-    else
-      val = 1936;  // tested against glibc 2.32
-  }
+  return FIRST_32_SECOND_64(1152, 1776);
+#    endif
 
-#    elif defined(__aarch64__)
+#    if SANITIZER_LOONGARCH64
+  return 1856;  // from glibc 2.36
+#    endif
+
+#    if defined(__aarch64__)
   // The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.
-  val = 1776;
-#    elif defined(__powerpc64__)
-  val = 1776;  // from glibc.ppc64le 2.20-8.fc21
+  return 1776;
+#    endif
+
+#    if defined(__powerpc64__)
+  return 1776;  // from glibc.ppc64le 2.20-8.fc21
 #    endif
-  return val;
 }
 
 uptr ThreadDescriptorSize() {
@@ -355,6 +359,10 @@ static uptr TlsPreTcbSize() {
 }
 #    endif
 
+#  endif
+
+#  if (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_SOLARIS) && \
+      !SANITIZER_ANDROID && !SANITIZER_GO
 namespace {
 struct TlsBlock {
   uptr begin, end, align;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
index b4a5d687dbdf4c..2a36104e6f9f29 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
@@ -575,7 +575,7 @@ uptr TlsSize() {
 void GetThreadStackAndTls(bool main, uptr *stk_begin, uptr *stk_end,
                           uptr *tls_begin, uptr *tls_end) {
 #  if !SANITIZER_GO
-  GetThreadStackTopAndBottom(main, stk_begin, stk_end);
+  GetThreadStackTopAndBottom(main, stk_end, stk_begin);
   *tls_begin = TlsBaseAddr();
   *tls_end = *tls_begin + TlsSize();
 #  else
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
index a17a14882d0e15..0db8547268f4e2 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
@@ -14,6 +14,7 @@
 
 #include "sanitizer_allocator_interface.h"
 #include "sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_flags.h"
 #include "sanitizer_platform_interceptors.h"
 
@@ -115,7 +116,6 @@ SANITIZER_INTERFACE_WEAK_DEF(uptr, __sanitizer_get_dtls_size,
   const void *start = __sanitizer_get_allocated_begin(tls_begin);
   if (!start)
     return 0;
-  CHECK_EQ(start, tls_begin);
   uptr tls_size = __sanitizer_get_allocated_size(start);
   VReport(2, "__tls_get_addr: glibc DTLS suspected; tls={%p,0x%zx}\n",
           tls_begin, tls_size);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
index d8f51bf020e242..2c8f8343519ed8 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
@@ -884,7 +884,7 @@ void GetThreadStackAndTls(bool main, uptr *stk_begin, uptr *stk_end,
   *tls_begin = 0;
   *tls_end = 0;
 #  else
-  GetThreadStackTopAndBottom(main, stk_begin, stk_end);
+  GetThreadStackTopAndBottom(main, stk_end, stk_begin);
   *tls_begin = 0;
   *tls_end = 0;
 #  endif
diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h
index 9be9a63fa951e0..2fae29e5a21687 100644
--- a/compiler-rt/lib/scudo/standalone/secondary.h
+++ b/compiler-rt/lib/scudo/standalone/secondary.h
@@ -247,7 +247,6 @@ class MapAllocatorCache {
     // The cache is initially empty
     LRUHead = CachedBlock::InvalidEntry;
     LRUTail = CachedBlock::InvalidEntry;
-    LastUnreleasedEntry = CachedBlock::InvalidEntry;
 
     // Available entries will be retrieved starting from the beginning of the
     // Entries array
@@ -322,10 +321,9 @@ class MapAllocatorCache {
         }
         CachedBlock PrevEntry = Quarantine[QuarantinePos];
         Quarantine[QuarantinePos] = Entry;
+        if (OldestTime == 0)
+          OldestTime = Entry.Time;
         Entry = PrevEntry;
-        // Update the entry time to reflect the time that the
-        // quarantined memory is placed in the Entries array
-        Entry.Time = Time;
       }
 
       // All excess entries are evicted from the cache
@@ -336,6 +334,9 @@ class MapAllocatorCache {
       }
 
       insert(Entry);
+
+      if (OldestTime == 0)
+        OldestTime = Entry.Time;
     } while (0);
 
     for (MemMapT &EvictMemMap : EvictionMemMaps)
@@ -534,9 +535,6 @@ class MapAllocatorCache {
       Entries[LRUHead].Prev = static_cast<u16>(FreeIndex);
     }
 
-    if (LastUnreleasedEntry == CachedBlock::InvalidEntry)
-      LastUnreleasedEntry = static_cast<u16>(FreeIndex);
-
     Entries[FreeIndex] = Entry;
     Entries[FreeIndex].Next = LRUHead;
     Entries[FreeIndex].Prev = CachedBlock::InvalidEntry;
@@ -554,9 +552,6 @@ class MapAllocatorCache {
 
     Entries[I].invalidate();
 
-    if (I == LastUnreleasedEntry)
-      LastUnreleasedEntry = Entries[LastUnreleasedEntry].Prev;
-
     if (I == LRUHead)
       LRUHead = Entries[I].Next;
     else
@@ -598,37 +593,35 @@ class MapAllocatorCache {
     }
   }
 
-  inline void release(CachedBlock &Entry) {
-    DCHECK(Entry.Time != 0);
+  void releaseIfOlderThan(CachedBlock &Entry, u64 Time) REQUIRES(Mutex) {
+    if (!Entry.isValid() || !Entry.Time)
+      return;
+    if (Entry.Time > Time) {
+      if (OldestTime == 0 || Entry.Time < OldestTime)
+        OldestTime = Entry.Time;
+      return;
+    }
     Entry.MemMap.releaseAndZeroPagesToOS(Entry.CommitBase, Entry.CommitSize);
     Entry.Time = 0;
   }
 
   void releaseOlderThan(u64 Time) EXCLUDES(Mutex) {
     ScopedLock L(Mutex);
-    if (!EntriesCount)
+    if (!EntriesCount || OldestTime == 0 || OldestTime > Time)
       return;
-
-    for (uptr I = 0; I < Config::getQuarantineSize(); I++) {
-      CachedBlock &ReleaseEntry = Quarantine[I];
-      if (!ReleaseEntry.isValid() || !ReleaseEntry.Time ||
-          ReleaseEntry.Time > Time)
-        continue;
-      release(ReleaseEntry);
-    }
-
-    // Release oldest entries first by releasing from decommit base
-    while (LastUnreleasedEntry != CachedBlock::InvalidEntry &&
-           Entries[LastUnreleasedEntry].Time <= Time) {
-      release(Entries[LastUnreleasedEntry]);
-      LastUnreleasedEntry = Entries[LastUnreleasedEntry].Prev;
-    }
+    OldestTime = 0;
+    for (uptr I = 0; I < Config::getQuarantineSize(); I++)
+      releaseIfOlderThan(Quarantine[I], Time);
+    for (uptr I = 0; I < Config::getEntriesArraySize(); I++)
+      releaseIfOlderThan(Entries[I], Time);
   }
+
   HybridMutex Mutex;
   u32 EntriesCount GUARDED_BY(Mutex) = 0;
   u32 QuarantinePos GUARDED_BY(Mutex) = 0;
   atomic_u32 MaxEntriesCount = {};
   atomic_uptr MaxEntrySize = {};
+  u64 OldestTime GUARDED_BY(Mutex) = 0;
   atomic_s32 ReleaseToOsIntervalMs = {};
   u32 CallsToRetrieve GUARDED_BY(Mutex) = 0;
   u32 SuccessfulRetrieves GUARDED_BY(Mutex) = 0;
@@ -643,9 +636,6 @@ class MapAllocatorCache {
   u16 LRUTail GUARDED_BY(Mutex) = 0;
   // The AvailableHead is the top of the stack of available entries
   u16 AvailableHead GUARDED_BY(Mutex) = 0;
-  // The LastUnreleasedEntry is the least recently used entry that has not
-  // been released
-  u16 LastUnreleasedEntry GUARDED_BY(Mutex) = 0;
 };
 
 template <typename Config> class MapAllocator {
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
index 460cbacf3408cf..53c876f4f9175f 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
@@ -12,14 +12,15 @@
 // sanitizer_common/sanitizer_common_interceptors.inc
 //===----------------------------------------------------------------------===//
 
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_glibc_version.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_linux.h"
 #include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
-#include "sanitizer_common/sanitizer_placement_new.h"
 #include "sanitizer_common/sanitizer_posix.h"
 #include "sanitizer_common/sanitizer_stacktrace.h"
 #include "sanitizer_common/sanitizer_tls_get_addr.h"
@@ -252,6 +253,13 @@ SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionBegin() {}
 SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() {}
 #endif
 
+// FIXME: Use for `in_symbolizer()` as well. As-is we can't use
+// `DlSymAllocator`, because it uses the primary allocator only. Symbolizer
+// requires support of the secondary allocator for larger blocks.
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+  static bool UseImpl() { return (ctx && !ctx->initialized); }
+};
+
 }  // namespace __tsan
 
 static ThreadSignalContext *SigCtx(ThreadState *thr) {
@@ -661,6 +669,8 @@ TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) {
 TSAN_INTERCEPTOR(void*, malloc, uptr size) {
   if (in_symbolizer())
     return InternalAlloc(size);
+  if (DlsymAlloc::Use())
+    return DlsymAlloc::Allocate(size);
   void *p = 0;
   {
     SCOPED_INTERCEPTOR_RAW(malloc, size);
@@ -681,6 +691,8 @@ TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
 TSAN_INTERCEPTOR(void *, calloc, uptr n, uptr size) {
   if (in_symbolizer())
     return InternalCalloc(n, size);
+  if (DlsymAlloc::Use())
+    return DlsymAlloc::Callocate(n, size);
   void *p = 0;
   {
     SCOPED_INTERCEPTOR_RAW(calloc, n, size);
@@ -693,6 +705,8 @@ TSAN_INTERCEPTOR(void *, calloc, uptr n, uptr size) {
 TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) {
   if (in_symbolizer())
     return InternalRealloc(p, size);
+  if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(p))
+    return DlsymAlloc::Realloc(p, size);
   if (p)
     invoke_free_hook(p);
   {
@@ -717,20 +731,24 @@ TSAN_INTERCEPTOR(void *, reallocarray, void *p, uptr n, uptr size) {
 }
 
 TSAN_INTERCEPTOR(void, free, void *p) {
-  if (p == 0)
+  if (UNLIKELY(!p))
     return;
   if (in_symbolizer())
     return InternalFree(p);
+  if (DlsymAlloc::PointerIsMine(p))
+    return DlsymAlloc::Free(p);
   invoke_free_hook(p);
   SCOPED_INTERCEPTOR_RAW(free, p);
   user_free(thr, pc, p);
 }
 
 TSAN_INTERCEPTOR(void, cfree, void *p) {
-  if (p == 0)
+  if (UNLIKELY(!p))
     return;
   if (in_symbolizer())
     return InternalFree(p);
+  if (DlsymAlloc::PointerIsMine(p))
+    return DlsymAlloc::Free(p);
   invoke_free_hook(p);
   SCOPED_INTERCEPTOR_RAW(cfree, p);
   user_free(thr, pc, p);
diff --git a/compiler-rt/test/sanitizer_common/TestCases/dlsym_alloc.c b/compiler-rt/test/sanitizer_common/TestCases/dlsym_alloc.c
index 7b5b9cf34a90f9..4aa87afe47f4ea 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/dlsym_alloc.c
+++ b/compiler-rt/test/sanitizer_common/TestCases/dlsym_alloc.c
@@ -1,7 +1,5 @@
 // RUN: %clang -O0 %s -o %t && %run %t
 
-// FIXME: TSAN does not use DlsymAlloc.
-// UNSUPPORTED: tsan
 // FIXME: investigate why this fails on macos
 // UNSUPPORTED: darwin
 
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td
index 49bd4f5349a754..925ada0f9d3507 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.td
+++ b/flang/include/flang/Optimizer/Transforms/Passes.td
@@ -278,7 +278,7 @@ def ConstantArgumentGlobalisationOpt : Pass<"constant-argument-globalisation-opt
   let dependentDialects = [ "fir::FIROpsDialect" ];
 }
 
-def StackArrays : Pass<"stack-arrays", "mlir::ModuleOp"> {
+def StackArrays : Pass<"stack-arrays", "mlir::func::FuncOp"> {
   let summary = "Move local array allocations from heap memory into stack memory";
   let description = [{
     Convert heap allocations for arrays, even those of unknown size, into stack
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 37c3370b48a085..0b4ebcbaa24c40 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -875,7 +875,7 @@ class ParseTreeDumper {
       ss << x;
     }
     if (ss.tell()) {
-      return ss.str();
+      return buf;
     }
     if constexpr (std::is_same_v<T, Name>) {
       return x.source.ToString();
diff --git a/flang/lib/Evaluate/formatting.cpp b/flang/lib/Evaluate/formatting.cpp
index 0870d56549f74d..3581b9c96c19bf 100644
--- a/flang/lib/Evaluate/formatting.cpp
+++ b/flang/lib/Evaluate/formatting.cpp
@@ -563,7 +563,7 @@ std::string ExpressionBase<RESULT>::AsFortran() const {
   std::string buf;
   llvm::raw_string_ostream ss{buf};
   AsFortran(ss);
-  return ss.str();
+  return buf;
 }
 
 template <typename RESULT>
@@ -604,7 +604,7 @@ static std::string DerivedTypeSpecAsFortran(
   if (ch != '(') {
     ss << ')';
   }
-  return ss.str();
+  return buf;
 }
 
 llvm::raw_ostream &StructureConstructor::AsFortran(llvm::raw_ostream &o) const {
diff --git a/flang/lib/Evaluate/real.cpp b/flang/lib/Evaluate/real.cpp
index 642490646dcaf3..2c0f2833f07dc5 100644
--- a/flang/lib/Evaluate/real.cpp
+++ b/flang/lib/Evaluate/real.cpp
@@ -770,11 +770,13 @@ template <typename W, int P> Real<W, P> Real<W, P>::SPACING() const {
   } else if (IsInfinite()) {
     return NotANumber();
   } else if (IsZero() || IsSubnormal()) {
-    return TINY(); // mandated by standard
+    return TINY(); // standard & 100% portable
   } else {
     Real result;
     result.Normalize(false, Exponent(), Fraction::MASKR(1));
-    return result.IsZero() ? TINY() : result;
+    // Can the result be less than TINY()?  No, with five commonly
+    // used compilers; yes, with two less commonly used ones.
+    return result.IsZero() || result.IsSubnormal() ? TINY() : result;
   }
 }
 
diff --git a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
index 46e70d7ef9180a..a921d91d16fe48 100644
--- a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
+++ b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
@@ -271,6 +271,9 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
         typeGen.convertType(resTy, fileAttr, cuAttr, /*declOp=*/nullptr);
     types.push_back(tyAttr);
   }
+  // If no return type then add a null type as a place holder for that.
+  if (types.empty())
+    types.push_back(mlir::LLVM::DINullTypeAttr::get(context));
   for (auto inTy : funcOp.getArgumentTypes()) {
     auto tyAttr = typeGen.convertType(fir::unwrapRefType(inTy), fileAttr,
                                       cuAttr, /*declOp=*/nullptr);
diff --git a/flang/lib/Optimizer/Transforms/CufOpConversion.cpp b/flang/lib/Optimizer/Transforms/CufOpConversion.cpp
index c22c74d3f78af7..03a1eb74343b43 100644
--- a/flang/lib/Optimizer/Transforms/CufOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CufOpConversion.cpp
@@ -15,6 +15,7 @@
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "flang/Optimizer/Support/DataLayout.h"
 #include "flang/Runtime/CUDA/descriptor.h"
+#include "flang/Runtime/CUDA/memory.h"
 #include "flang/Runtime/allocatable.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
@@ -255,6 +256,171 @@ struct CufFreeOpConversion : public mlir::OpRewritePattern<cuf::FreeOp> {
   }
 };
 
+static int computeWidth(mlir::Location loc, mlir::Type type,
+                        fir::KindMapping &kindMap) {
+  auto eleTy = fir::unwrapSequenceType(type);
+  int width = 0;
+  if (auto t{mlir::dyn_cast<mlir::IntegerType>(eleTy)}) {
+    width = t.getWidth() / 8;
+  } else if (auto t{mlir::dyn_cast<mlir::FloatType>(eleTy)}) {
+    width = t.getWidth() / 8;
+  } else if (eleTy.isInteger(1)) {
+    width = 1;
+  } else if (auto t{mlir::dyn_cast<fir::LogicalType>(eleTy)}) {
+    int kind = t.getFKind();
+    width = kindMap.getLogicalBitsize(kind) / 8;
+  } else if (auto t{mlir::dyn_cast<fir::ComplexType>(eleTy)}) {
+    int kind = t.getFKind();
+    int elemSize = kindMap.getRealBitsize(kind) / 8;
+    width = 2 * elemSize;
+  } else {
+    llvm::report_fatal_error("unsupported type");
+  }
+  return width;
+}
+
+static mlir::Value createConvertOp(mlir::PatternRewriter &rewriter,
+                                   mlir::Location loc, mlir::Type toTy,
+                                   mlir::Value val) {
+  if (val.getType() != toTy)
+    return rewriter.create<fir::ConvertOp>(loc, toTy, val);
+  return val;
+}
+
+struct CufDataTransferOpConversion
+    : public mlir::OpRewritePattern<cuf::DataTransferOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cuf::DataTransferOp op,
+                  mlir::PatternRewriter &rewriter) const override {
+
+    mlir::Type srcTy = fir::unwrapRefType(op.getSrc().getType());
+    mlir::Type dstTy = fir::unwrapRefType(op.getDst().getType());
+
+    // Only convert cuf.data_transfer with at least one descripor.
+    if (!mlir::isa<fir::BaseBoxType>(srcTy) &&
+        !mlir::isa<fir::BaseBoxType>(dstTy))
+      return failure();
+
+    unsigned mode;
+    if (op.getTransferKind() == cuf::DataTransferKind::HostDevice) {
+      mode = kHostToDevice;
+    } else if (op.getTransferKind() == cuf::DataTransferKind::DeviceHost) {
+      mode = kDeviceToHost;
+    } else if (op.getTransferKind() == cuf::DataTransferKind::DeviceDevice) {
+      mode = kDeviceToDevice;
+    }
+
+    auto mod = op->getParentOfType<mlir::ModuleOp>();
+    fir::FirOpBuilder builder(rewriter, mod);
+    mlir::Location loc = op.getLoc();
+
+    if (mlir::isa<fir::BaseBoxType>(srcTy) &&
+        mlir::isa<fir::BaseBoxType>(dstTy)) {
+      // Transfer between two descriptor.
+      mlir::func::FuncOp func =
+          fir::runtime::getRuntimeFunc<mkRTKey(CUFDataTransferDescDesc)>(
+              loc, builder);
+
+      auto fTy = func.getFunctionType();
+      mlir::Value modeValue =
+          builder.createIntegerConstant(loc, builder.getI32Type(), mode);
+      mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc);
+      mlir::Value sourceLine =
+          fir::factory::locationToLineNo(builder, loc, fTy.getInput(4));
+      mlir::Value dst = builder.loadIfRef(loc, op.getDst());
+      mlir::Value src = builder.loadIfRef(loc, op.getSrc());
+      llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
+          builder, loc, fTy, dst, src, modeValue, sourceFile, sourceLine)};
+      builder.create<fir::CallOp>(loc, func, args);
+      rewriter.eraseOp(op);
+    } else if (mlir::isa<fir::BaseBoxType>(dstTy) && fir::isa_trivial(srcTy)) {
+      // Scalar to descriptor transfer.
+      mlir::Value val = op.getSrc();
+      if (op.getSrc().getDefiningOp() &&
+          mlir::isa<mlir::arith::ConstantOp>(op.getSrc().getDefiningOp())) {
+        mlir::Value alloc = builder.createTemporary(loc, srcTy);
+        builder.create<fir::StoreOp>(loc, op.getSrc(), alloc);
+        val = alloc;
+      }
+
+      mlir::func::FuncOp func =
+          fir::runtime::getRuntimeFunc<mkRTKey(CUFMemsetDescriptor)>(loc,
+                                                                     builder);
+      auto fTy = func.getFunctionType();
+      mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc);
+      mlir::Value sourceLine =
+          fir::factory::locationToLineNo(builder, loc, fTy.getInput(3));
+      mlir::Value dst = builder.loadIfRef(loc, op.getDst());
+      llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
+          builder, loc, fTy, dst, val, sourceFile, sourceLine)};
+      builder.create<fir::CallOp>(loc, func, args);
+      rewriter.eraseOp(op);
+    } else {
+      mlir::Value modeValue =
+          builder.createIntegerConstant(loc, builder.getI32Type(), mode);
+      // Type used to compute the width.
+      mlir::Type computeType = dstTy;
+      auto seqTy = mlir::dyn_cast<fir::SequenceType>(dstTy);
+      bool dstIsDesc = false;
+      if (mlir::isa<fir::BaseBoxType>(dstTy)) {
+        dstIsDesc = true;
+        computeType = srcTy;
+        seqTy = mlir::dyn_cast<fir::SequenceType>(srcTy);
+      }
+      fir::KindMapping kindMap{fir::getKindMapping(mod)};
+      int width = computeWidth(loc, computeType, kindMap);
+
+      mlir::Value nbElement;
+      mlir::Type idxTy = rewriter.getIndexType();
+      if (!op.getShape()) {
+        nbElement = rewriter.create<mlir::arith::ConstantOp>(
+            loc, idxTy,
+            rewriter.getIntegerAttr(idxTy, seqTy.getConstantArraySize()));
+      } else {
+        auto shapeOp =
+            mlir::dyn_cast<fir::ShapeOp>(op.getShape().getDefiningOp());
+        nbElement =
+            createConvertOp(rewriter, loc, idxTy, shapeOp.getExtents()[0]);
+        for (unsigned i = 1; i < shapeOp.getExtents().size(); ++i) {
+          auto operand =
+              createConvertOp(rewriter, loc, idxTy, shapeOp.getExtents()[i]);
+          nbElement =
+              rewriter.create<mlir::arith::MulIOp>(loc, nbElement, operand);
+        }
+      }
+
+      mlir::Value widthValue = rewriter.create<mlir::arith::ConstantOp>(
+          loc, idxTy, rewriter.getIntegerAttr(idxTy, width));
+      mlir::Value bytes =
+          rewriter.create<mlir::arith::MulIOp>(loc, nbElement, widthValue);
+
+      mlir::func::FuncOp func =
+          dstIsDesc
+              ? fir::runtime::getRuntimeFunc<mkRTKey(CUFDataTransferDescPtr)>(
+                    loc, builder)
+              : fir::runtime::getRuntimeFunc<mkRTKey(CUFDataTransferPtrDesc)>(
+                    loc, builder);
+      auto fTy = func.getFunctionType();
+      mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc);
+      mlir::Value sourceLine =
+          fir::factory::locationToLineNo(builder, loc, fTy.getInput(5));
+      mlir::Value dst =
+          dstIsDesc ? builder.loadIfRef(loc, op.getDst()) : op.getDst();
+      mlir::Value src = mlir::isa<fir::BaseBoxType>(srcTy)
+                            ? builder.loadIfRef(loc, op.getSrc())
+                            : op.getSrc();
+      llvm::SmallVector<mlir::Value> args{
+          fir::runtime::createArguments(builder, loc, fTy, dst, src, bytes,
+                                        modeValue, sourceFile, sourceLine)};
+      builder.create<fir::CallOp>(loc, func, args);
+      rewriter.eraseOp(op);
+    }
+    return mlir::success();
+  }
+};
+
 class CufOpConversion : public fir::impl::CufOpConversionBase<CufOpConversion> {
 public:
   void runOnOperation() override {
@@ -285,10 +451,17 @@ class CufOpConversion : public fir::impl::CufOpConversionBase<CufOpConversion> {
         [](::cuf::AllocateOp op) { return needDoubleDescriptor(op); });
     target.addDynamicallyLegalOp<cuf::DeallocateOp>(
         [](::cuf::DeallocateOp op) { return needDoubleDescriptor(op); });
-    target.addLegalDialect<fir::FIROpsDialect>();
+    target.addDynamicallyLegalOp<cuf::DataTransferOp>(
+        [](::cuf::DataTransferOp op) {
+          mlir::Type srcTy = fir::unwrapRefType(op.getSrc().getType());
+          mlir::Type dstTy = fir::unwrapRefType(op.getDst().getType());
+          return !mlir::isa<fir::BaseBoxType>(srcTy) &&
+                 !mlir::isa<fir::BaseBoxType>(dstTy);
+        });
+    target.addLegalDialect<fir::FIROpsDialect, mlir::arith::ArithDialect>();
     patterns.insert<CufAllocOpConversion>(ctx, &*dl, &typeConverter);
     patterns.insert<CufAllocateOpConversion, CufDeallocateOpConversion,
-                    CufFreeOpConversion>(ctx);
+                    CufFreeOpConversion, CufDataTransferOpConversion>(ctx);
     if (mlir::failed(mlir::applyPartialConversion(getOperation(), target,
                                                   std::move(patterns)))) {
       mlir::emitError(mlir::UnknownLoc::get(ctx),
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index 7c4382079fd6d4..82c6a6618e0ed8 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -213,23 +213,26 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertSequenceType(
       convertType(seqTy.getEleTy(), fileAttr, scope, declOp);
 
   unsigned index = 0;
+  auto intTy = mlir::IntegerType::get(context, 64);
   for (fir::SequenceType::Extent dim : seqTy.getShape()) {
+    int64_t shift = 1;
+    if (declOp && declOp.getShift().size() > index) {
+      if (std::optional<std::int64_t> optint =
+              getIntIfConstant(declOp.getShift()[index]))
+        shift = *optint;
+    }
     if (dim == seqTy.getUnknownExtent()) {
+      mlir::IntegerAttr lowerAttr = nullptr;
+      if (declOp && declOp.getShift().size() > index)
+        lowerAttr = mlir::IntegerAttr::get(intTy, llvm::APInt(64, shift));
       // FIXME: This path is taken for assumed size arrays but also for arrays
       // with non constant extent. For the latter case, the DISubrangeAttr
       // should point to a variable which will have the extent at runtime.
       auto subrangeTy = mlir::LLVM::DISubrangeAttr::get(
-          context, /*count=*/nullptr, /*lowerBound=*/nullptr,
-          /*upperBound*/ nullptr, /*stride*/ nullptr);
+          context, /*count=*/nullptr, lowerAttr, /*upperBound*/ nullptr,
+          /*stride*/ nullptr);
       elements.push_back(subrangeTy);
     } else {
-      auto intTy = mlir::IntegerType::get(context, 64);
-      int64_t shift = 1;
-      if (declOp && declOp.getShift().size() > index) {
-        if (std::optional<std::int64_t> optint =
-                getIntIfConstant(declOp.getShift()[index]))
-          shift = *optint;
-      }
       auto countAttr = mlir::IntegerAttr::get(intTy, llvm::APInt(64, dim));
       auto lowerAttr = mlir::IntegerAttr::get(intTy, llvm::APInt(64, shift));
       auto subrangeTy = mlir::LLVM::DISubrangeAttr::get(
diff --git a/flang/lib/Optimizer/Transforms/StackArrays.cpp b/flang/lib/Optimizer/Transforms/StackArrays.cpp
index 1b929928dc7164..d9e7bd684d4600 100644
--- a/flang/lib/Optimizer/Transforms/StackArrays.cpp
+++ b/flang/lib/Optimizer/Transforms/StackArrays.cpp
@@ -225,7 +225,6 @@ class StackArraysPass : public fir::impl::StackArraysBase<StackArraysPass> {
   llvm::StringRef getDescription() const override;
 
   void runOnOperation() override;
-  void runOnFunc(mlir::Operation *func);
 
 private:
   Statistic runCount{this, "stackArraysRunCount",
@@ -766,13 +765,7 @@ llvm::StringRef StackArraysPass::getDescription() const {
 }
 
 void StackArraysPass::runOnOperation() {
-  mlir::ModuleOp mod = getOperation();
-
-  mod.walk([this](mlir::func::FuncOp func) { runOnFunc(func); });
-}
-
-void StackArraysPass::runOnFunc(mlir::Operation *func) {
-  assert(mlir::isa<mlir::func::FuncOp>(func));
+  mlir::func::FuncOp func = getOperation();
 
   auto &analysis = getAnalysis<StackArraysAnalysisWrapper>();
   const StackArraysAnalysisWrapper::AllocMemMap *candidateOps =
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index b594df8ecb6858..b231c5859cf873 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -1098,7 +1098,7 @@ void Prescanner::FortranInclude(const char *firstQuote) {
   const SourceFile *included{
       allSources_.Open(path, error, std::move(prependPath))};
   if (!included) {
-    Say(provenance, "INCLUDE: %s"_err_en_US, error.str());
+    Say(provenance, "INCLUDE: %s"_err_en_US, buf);
   } else if (included->bytes() > 0) {
     ProvenanceRange includeLineRange{
         provenance, static_cast<std::size_t>(p - nextLine_)};
diff --git a/flang/lib/Semantics/check-do-forall.cpp b/flang/lib/Semantics/check-do-forall.cpp
index d798244ff1ef2d..b9778e96080a62 100644
--- a/flang/lib/Semantics/check-do-forall.cpp
+++ b/flang/lib/Semantics/check-do-forall.cpp
@@ -122,6 +122,10 @@ class DoConcurrentBodyEnforce {
     }
     return true;
   }
+  bool Pre(const parser::ConcurrentHeader &) {
+    // handled in CheckConcurrentHeader
+    return false;
+  }
   template <typename T> void Post(const T &) {}
 
   // C1140 -- Can't deallocate a polymorphic entity in a DO CONCURRENT.
@@ -375,8 +379,13 @@ class DoConcurrentVariableEnforce {
 // Find a DO or FORALL and enforce semantics checks on its body
 class DoContext {
 public:
-  DoContext(SemanticsContext &context, IndexVarKind kind, bool isNested)
-      : context_{context}, kind_{kind}, isNested_{isNested} {}
+  DoContext(SemanticsContext &context, IndexVarKind kind,
+      const std::list<IndexVarKind> nesting)
+      : context_{context}, kind_{kind} {
+    if (!nesting.empty()) {
+      concurrentNesting_ = nesting.back();
+    }
+  }
 
   // Mark this DO construct as a point of definition for the DO variables
   // or index-names it contains.  If they're already defined, emit an error
@@ -439,8 +448,8 @@ class DoContext {
             common::visitors{[&](const auto &x) { return GetAssignment(x); }},
             stmt.u)}) {
       CheckForallIndexesUsed(*assignment);
-      CheckForImpureCall(assignment->lhs);
-      CheckForImpureCall(assignment->rhs);
+      CheckForImpureCall(assignment->lhs, kind_);
+      CheckForImpureCall(assignment->rhs, kind_);
 
       if (IsVariable(assignment->lhs)) {
         if (const Symbol * symbol{GetLastSymbol(assignment->lhs)}) {
@@ -455,23 +464,23 @@ class DoContext {
 
       if (const auto *proc{
               std::get_if<evaluate::ProcedureRef>(&assignment->u)}) {
-        CheckForImpureCall(*proc);
+        CheckForImpureCall(*proc, kind_);
       }
       common::visit(
           common::visitors{
               [](const evaluate::Assignment::Intrinsic &) {},
               [&](const evaluate::ProcedureRef &proc) {
-                CheckForImpureCall(proc);
+                CheckForImpureCall(proc, kind_);
               },
               [&](const evaluate::Assignment::BoundsSpec &bounds) {
                 for (const auto &bound : bounds) {
-                  CheckForImpureCall(SomeExpr{bound});
+                  CheckForImpureCall(SomeExpr{bound}, kind_);
                 }
               },
               [&](const evaluate::Assignment::BoundsRemapping &bounds) {
                 for (const auto &bound : bounds) {
-                  CheckForImpureCall(SomeExpr{bound.first});
-                  CheckForImpureCall(SomeExpr{bound.second});
+                  CheckForImpureCall(SomeExpr{bound.first}, kind_);
+                  CheckForImpureCall(SomeExpr{bound.second}, kind_);
                 }
               },
           },
@@ -754,12 +763,10 @@ class DoContext {
       if (indexName.symbol) {
         indexNames.insert(*indexName.symbol);
       }
-      if (isNested_) {
-        CheckForImpureCall(std::get<1>(control.t));
-        CheckForImpureCall(std::get<2>(control.t));
-        if (const auto &stride{std::get<3>(control.t)}) {
-          CheckForImpureCall(*stride);
-        }
+      CheckForImpureCall(std::get<1>(control.t), concurrentNesting_);
+      CheckForImpureCall(std::get<2>(control.t), concurrentNesting_);
+      if (const auto &stride{std::get<3>(control.t)}) {
+        CheckForImpureCall(*stride, concurrentNesting_);
       }
     }
     if (!indexNames.empty()) {
@@ -819,20 +826,29 @@ class DoContext {
     CheckConcurrentHeader(std::get<parser::ConcurrentHeader>(concurrent.t));
   }
 
-  template <typename T> void CheckForImpureCall(const T &x) const {
+  template <typename T>
+  void CheckForImpureCall(
+      const T &x, std::optional<IndexVarKind> nesting) const {
     if (auto bad{FindImpureCall(context_.foldingContext(), x)}) {
-      context_.Say(
-          "Impure procedure '%s' may not be referenced in a %s"_err_en_US, *bad,
-          LoopKindName());
+      if (nesting) {
+        context_.Say(
+            "Impure procedure '%s' may not be referenced in a %s"_err_en_US,
+            *bad, LoopKindName(*nesting));
+      } else {
+        context_.Say(
+            "Impure procedure '%s' should not be referenced in a %s header"_warn_en_US,
+            *bad, LoopKindName(kind_));
+      }
     }
   }
-  void CheckForImpureCall(const parser::ScalarIntExpr &x) const {
+  void CheckForImpureCall(const parser::ScalarIntExpr &x,
+      std::optional<IndexVarKind> nesting) const {
     const auto &parsedExpr{x.thing.thing.value()};
     auto oldLocation{context_.location()};
     context_.set_location(parsedExpr.source);
     if (const auto &typedExpr{parsedExpr.typedExpr}) {
       if (const auto &expr{typedExpr->v}) {
-        CheckForImpureCall(*expr);
+        CheckForImpureCall(*expr, nesting);
       }
     }
     context_.set_location(oldLocation);
@@ -885,54 +901,59 @@ class DoContext {
   }
 
   // For messages where the DO loop must be DO CONCURRENT, make that explicit.
-  const char *LoopKindName() const {
-    return kind_ == IndexVarKind::DO ? "DO CONCURRENT" : "FORALL";
+  const char *LoopKindName(IndexVarKind kind) const {
+    return kind == IndexVarKind::DO ? "DO CONCURRENT" : "FORALL";
   }
+  const char *LoopKindName() const { return LoopKindName(kind_); }
 
   SemanticsContext &context_;
   const IndexVarKind kind_;
   parser::CharBlock currentStatementSourcePosition_;
-  bool isNested_{false};
+  std::optional<IndexVarKind> concurrentNesting_;
 }; // class DoContext
 
 void DoForallChecker::Enter(const parser::DoConstruct &doConstruct) {
-  DoContext doContext{context_, IndexVarKind::DO, constructNesting_ > 0};
+  DoContext doContext{context_, IndexVarKind::DO, nestedWithinConcurrent_};
+  if (doConstruct.IsDoConcurrent()) {
+    nestedWithinConcurrent_.push_back(IndexVarKind::DO);
+  }
   doContext.DefineDoVariables(doConstruct);
+  doContext.Check(doConstruct);
 }
 
 void DoForallChecker::Leave(const parser::DoConstruct &doConstruct) {
-  DoContext doContext{context_, IndexVarKind::DO, constructNesting_ > 0};
-  ++constructNesting_;
-  doContext.Check(doConstruct);
+  DoContext doContext{context_, IndexVarKind::DO, nestedWithinConcurrent_};
   doContext.ResetDoVariables(doConstruct);
-  --constructNesting_;
+  if (doConstruct.IsDoConcurrent()) {
+    nestedWithinConcurrent_.pop_back();
+  }
 }
 
 void DoForallChecker::Enter(const parser::ForallConstruct &construct) {
-  DoContext doContext{context_, IndexVarKind::FORALL, constructNesting_ > 0};
+  DoContext doContext{context_, IndexVarKind::FORALL, nestedWithinConcurrent_};
   doContext.ActivateIndexVars(GetControls(construct));
-  ++constructNesting_;
+  nestedWithinConcurrent_.push_back(IndexVarKind::FORALL);
   doContext.Check(construct);
 }
 void DoForallChecker::Leave(const parser::ForallConstruct &construct) {
-  DoContext doContext{context_, IndexVarKind::FORALL, constructNesting_ > 0};
+  DoContext doContext{context_, IndexVarKind::FORALL, nestedWithinConcurrent_};
   doContext.DeactivateIndexVars(GetControls(construct));
-  --constructNesting_;
+  nestedWithinConcurrent_.pop_back();
 }
 
 void DoForallChecker::Enter(const parser::ForallStmt &stmt) {
-  DoContext doContext{context_, IndexVarKind::FORALL, constructNesting_ > 0};
-  ++constructNesting_;
+  DoContext doContext{context_, IndexVarKind::FORALL, nestedWithinConcurrent_};
+  nestedWithinConcurrent_.push_back(IndexVarKind::FORALL);
   doContext.Check(stmt);
   doContext.ActivateIndexVars(GetControls(stmt));
 }
 void DoForallChecker::Leave(const parser::ForallStmt &stmt) {
-  DoContext doContext{context_, IndexVarKind::FORALL, constructNesting_ > 0};
+  DoContext doContext{context_, IndexVarKind::FORALL, nestedWithinConcurrent_};
   doContext.DeactivateIndexVars(GetControls(stmt));
-  --constructNesting_;
+  nestedWithinConcurrent_.pop_back();
 }
 void DoForallChecker::Leave(const parser::ForallAssignmentStmt &stmt) {
-  DoContext doContext{context_, IndexVarKind::FORALL, constructNesting_ > 0};
+  DoContext doContext{context_, IndexVarKind::FORALL, nestedWithinConcurrent_};
   doContext.Check(stmt);
 }
 
diff --git a/flang/lib/Semantics/check-do-forall.h b/flang/lib/Semantics/check-do-forall.h
index f08ff3467d4cc9..4a65818364d764 100644
--- a/flang/lib/Semantics/check-do-forall.h
+++ b/flang/lib/Semantics/check-do-forall.h
@@ -60,7 +60,7 @@ class DoForallChecker : public virtual BaseChecker {
 private:
   SemanticsContext &context_;
   int exprDepth_{0};
-  int constructNesting_{0};
+  std::list<SemanticsContext::IndexVarKind> nestedWithinConcurrent_;
 
   void SayBadLeave(
       StmtType, const char *enclosingStmt, const ConstructNode &) const;
diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index 072ebe172f45a9..182ea5d441956c 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -3223,7 +3223,7 @@ void ExpressionAnalyzer::Analyze(const parser::CallStmt &callStmt) {
       llvm::raw_string_ostream dump{buf};
       parser::DumpTree(dump, callStmt);
       Say("Internal error: Expression analysis failed on CALL statement: %s"_err_en_US,
-          dump.str());
+          buf);
     }
   }
 }
@@ -3847,8 +3847,7 @@ MaybeExpr ExpressionAnalyzer::ExprOrVariable(
       std::string buf;
       llvm::raw_string_ostream dump{buf};
       parser::DumpTree(dump, x);
-      Say("Internal error: Expression analysis failed on: %s"_err_en_US,
-          dump.str());
+      Say("Internal error: Expression analysis failed on: %s"_err_en_US, buf);
     }
     return std::nullopt;
   }
diff --git a/flang/lib/Semantics/pointer-assignment.cpp b/flang/lib/Semantics/pointer-assignment.cpp
index 4948fce77a2502..2813a0cf968cbd 100644
--- a/flang/lib/Semantics/pointer-assignment.cpp
+++ b/flang/lib/Semantics/pointer-assignment.cpp
@@ -353,7 +353,7 @@ bool PointerAssignmentChecker::Check(const evaluate::Designator<T> &d) {
       std::string buf;
       llvm::raw_string_ostream ss{buf};
       d.AsFortran(ss);
-      Say(*m, description_, ss.str());
+      Say(*m, description_, buf);
     } else {
       Say(std::get<MessageFormattedText>(*msg));
     }
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 723596ad6ce454..45fe17c0122c09 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -19,6 +19,7 @@
 #include "flang/Parser/parse-tree.h"
 #include "flang/Parser/tools.h"
 #include "flang/Semantics/expression.h"
+#include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
 #include <list>
 #include <map>
@@ -717,7 +718,7 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
   void CheckDataCopyingClause(
       const parser::Name &, const Symbol &, Symbol::Flag);
   void CheckAssocLoopLevel(std::int64_t level, const parser::OmpClause *clause);
-  void CheckObjectInNamelist(
+  void CheckObjectInNamelistOrAssociate(
       const parser::Name &, const Symbol &, Symbol::Flag);
   void CheckSourceLabel(const parser::Label &);
   void CheckLabelContext(const parser::CharBlock, const parser::CharBlock,
@@ -2356,7 +2357,7 @@ void OmpAttributeVisitor::ResolveOmpObject(
                     CheckMultipleAppearances(*name, *symbol, ompFlag);
                   }
                   if (privateDataSharingAttributeFlags.test(ompFlag)) {
-                    CheckObjectInNamelist(*name, *symbol, ompFlag);
+                    CheckObjectInNamelistOrAssociate(*name, *symbol, ompFlag);
                   }
 
                   if (ompFlag == Symbol::Flag::OmpAllocate) {
@@ -2713,7 +2714,7 @@ void OmpAttributeVisitor::CheckDataCopyingClause(
   }
 }
 
-void OmpAttributeVisitor::CheckObjectInNamelist(
+void OmpAttributeVisitor::CheckObjectInNamelistOrAssociate(
     const parser::Name &name, const Symbol &symbol, Symbol::Flag ompFlag) {
   const auto &ultimateSymbol{symbol.GetUltimate()};
   llvm::StringRef clauseName{"PRIVATE"};
@@ -2728,6 +2729,12 @@ void OmpAttributeVisitor::CheckObjectInNamelist(
         "Variable '%s' in NAMELIST cannot be in a %s clause"_err_en_US,
         name.ToString(), clauseName.str());
   }
+
+  if (ultimateSymbol.has<AssocEntityDetails>()) {
+    context_.Say(name.source,
+        "Variable '%s' in ASSOCIATE cannot be in a %s clause"_err_en_US,
+        name.ToString(), clauseName.str());
+  }
 }
 
 void OmpAttributeVisitor::CheckSourceLabel(const parser::Label &label) {
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index d8f601212d8d01..b99f308e1c7fab 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -474,15 +474,15 @@ class FuncResultStack {
   ~FuncResultStack();
 
   struct FuncInfo {
-    explicit FuncInfo(const Scope &s) : scope{s} {}
+    FuncInfo(const Scope &s, SourceName at) : scope{s}, source{at} {}
     const Scope &scope;
+    SourceName source;
     // Parse tree of the type specification in the FUNCTION prefix
     const parser::DeclarationTypeSpec *parsedType{nullptr};
     // Name of the function RESULT in the FUNCTION suffix, if any
     const parser::Name *resultName{nullptr};
     // Result symbol
     Symbol *resultSymbol{nullptr};
-    std::optional<SourceName> source;
     bool inFunctionStmt{false}; // true between Pre/Post of FunctionStmt
   };
 
@@ -492,7 +492,9 @@ class FuncResultStack {
   void CompleteTypeIfFunctionResult(Symbol &);
 
   FuncInfo *Top() { return stack_.empty() ? nullptr : &stack_.back(); }
-  FuncInfo &Push(const Scope &scope) { return stack_.emplace_back(scope); }
+  FuncInfo &Push(const Scope &scope, SourceName at) {
+    return stack_.emplace_back(scope, at);
+  }
   void Pop();
 
 private:
@@ -3799,11 +3801,13 @@ bool SubprogramVisitor::Pre(const parser::PrefixSpec &x) {
   if (const auto *parsedType{std::get_if<parser::DeclarationTypeSpec>(&x.u)}) {
     if (FuncResultStack::FuncInfo * info{funcResultStack().Top()}) {
       if (info->parsedType) { // C1543
-        Say(currStmtSource().value(),
+        Say(currStmtSource().value_or(info->source),
             "FUNCTION prefix cannot specify the type more than once"_err_en_US);
       } else {
         info->parsedType = parsedType;
-        info->source = currStmtSource();
+        if (auto at{currStmtSource()}) {
+          info->source = *at;
+        }
       }
     } else {
       Say(currStmtSource().value(),
@@ -3978,6 +3982,9 @@ bool SubprogramVisitor::Pre(const parser::FunctionStmt &) {
   FuncResultStack::FuncInfo &info{DEREF(funcResultStack().Top())};
   CHECK(!info.inFunctionStmt);
   info.inFunctionStmt = true;
+  if (auto at{currStmtSource()}) {
+    info.source = *at;
+  }
   return BeginAttrs();
 }
 bool SubprogramVisitor::Pre(const parser::EntryStmt &) { return BeginAttrs(); }
@@ -4507,7 +4514,7 @@ Symbol &SubprogramVisitor::PushSubprogramScope(const parser::Name &name,
   symbol->set(subpFlag);
   PushScope(Scope::Kind::Subprogram, symbol);
   if (subpFlag == Symbol::Flag::Function) {
-    funcResultStack().Push(currScope());
+    funcResultStack().Push(currScope(), name.source);
   }
   if (inInterfaceBlock()) {
     auto &details{symbol->get<SubprogramDetails>()};
diff --git a/flang/lib/Semantics/scope.cpp b/flang/lib/Semantics/scope.cpp
index 89128e4a5049b1..6ee53cde56866c 100644
--- a/flang/lib/Semantics/scope.cpp
+++ b/flang/lib/Semantics/scope.cpp
@@ -47,7 +47,7 @@ std::string EquivalenceObject::AsFortran() const {
   if (substringStart) {
     ss << '(' << *substringStart << ":)";
   }
-  return ss.str();
+  return buf;
 }
 
 Scope &Scope::MakeScope(Kind kind, Symbol *symbol) {
diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index 8d16ab71008766..3723b28fecef52 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -440,7 +440,7 @@ static void CheckMissingAnalysis(
     llvm::raw_string_ostream ss{buf};
     ss << "node has not been analyzed:\n";
     parser::DumpTree(ss, x);
-    common::die(ss.str().c_str());
+    common::die(buf.c_str());
   }
 }
 
diff --git a/flang/lib/Semantics/type.cpp b/flang/lib/Semantics/type.cpp
index aa6e8973ebd304..810b9829b0b8db 100644
--- a/flang/lib/Semantics/type.cpp
+++ b/flang/lib/Semantics/type.cpp
@@ -658,7 +658,7 @@ std::string DerivedTypeSpec::VectorTypeAsFortran() const {
   case (Fortran::semantics::DerivedTypeSpec::Category::DerivedType):
     Fortran::common::die("Vector element type not implemented");
   }
-  return ss.str();
+  return buf;
 }
 
 std::string DerivedTypeSpec::AsFortran() const {
@@ -694,7 +694,7 @@ std::string DerivedTypeSpec::AsFortran() const {
     }
     ss << ')';
   }
-  return ss.str();
+  return buf;
 }
 
 llvm::raw_ostream &operator<<(llvm::raw_ostream &o, const DerivedTypeSpec &x) {
@@ -771,7 +771,7 @@ std::string ParamValue::AsFortran() const {
       std::string buf;
       llvm::raw_string_ostream ss{buf};
       expr_->AsFortran(ss);
-      return ss.str();
+      return buf;
     } else {
       return "";
     }
@@ -795,7 +795,7 @@ static std::string KindAsFortran(const KindExpr &kind) {
   } else {
     kind.AsFortran(ss);
   }
-  return ss.str();
+  return buf;
 }
 
 std::string IntrinsicTypeSpec::AsFortran() const {
diff --git a/flang/runtime/numeric-templates.h b/flang/runtime/numeric-templates.h
index 1b43498a6bfd12..8cc63859ab7d4f 100644
--- a/flang/runtime/numeric-templates.h
+++ b/flang/runtime/numeric-templates.h
@@ -88,12 +88,16 @@ struct MaxOrMinIdentity<TypeCategory::Real, 16, IS_MAXVAL,
 
 // Minimum finite representable value.
 // For floating-point types, returns minimum positive normalized value.
-template <typename T> struct MinValue {
+template <int PREC, typename T> struct MinValue {
   static RT_API_ATTRS T get() { return std::numeric_limits<T>::min(); }
 };
+template <typename T> struct MinValue<11, T> {
+  // TINY(0._2)
+  static constexpr RT_API_ATTRS T get() { return 0.00006103515625E-04; }
+};
 
 #if HAS_FLOAT128
-template <> struct MinValue<CppTypeFor<TypeCategory::Real, 16>> {
+template <> struct MinValue<113, CppTypeFor<TypeCategory::Real, 16>> {
   using Type = CppTypeFor<TypeCategory::Real, 16>;
   static RT_API_ATTRS Type get() {
     // Create a buffer to store binary representation of __float128 constant.
@@ -167,8 +171,8 @@ template <> struct MAXTy<CppTypeFor<TypeCategory::Real, 16>> {
 };
 #endif
 
-template <typename T> struct MINTy {
-  static constexpr RT_API_ATTRS T compute() { return MinValue<T>::get(); }
+template <int PREC, typename T> struct MINTy {
+  static constexpr RT_API_ATTRS T compute() { return MinValue<PREC, T>::get(); }
 };
 
 template <typename T> struct QNANTy {
@@ -339,23 +343,20 @@ template <int PREC, typename T> inline RT_API_ATTRS T RRSpacing(T x) {
 
 // SPACING (16.9.180)
 template <int PREC, typename T> inline RT_API_ATTRS T Spacing(T x) {
+  T tiny{MINTy<PREC, T>::compute()};
   if (ISNANTy<T>::compute(x)) {
     return x; // NaN -> same NaN
   } else if (ISINFTy<T>::compute(x)) {
     return QNANTy<T>::compute(); // +/-Inf -> NaN
   } else if (x == 0) { // 0 -> TINY(x)
-    // The standard-mandated behavior seems broken, since TINY() can't be
-    // subnormal.
-    if constexpr (PREC == 11) { // REAL(2)
-      return 0.00006103515625E-04; // TINY(0._2)
-    } else {
-      // N.B. TINY(0._3) == TINY(0._4) so this works even if no std::bfloat16_t.
-      return MINTy<T>::compute();
-    }
+    return tiny;
   } else {
     T result{LDEXPTy<T>::compute(
         static_cast<T>(1.0), ILOGBTy<T>::compute(x) + 1 - PREC)}; // 2**(e-p)
-    return result == 0 ? /*TINY(x)*/ MINTy<T>::compute() : result;
+    // All compilers return TINY(x) for |x| <= TINY(x), but differ over whether
+    // SPACING(x) can be < TINY(x) for |x| > TINY(x).  The most common precedent
+    // is to never return a value < TINY(x).
+    return result <= tiny ? tiny : result;
   }
 }
 
diff --git a/flang/test/Driver/nostdlib.f90 b/flang/test/Driver/nostdlib.f90
new file mode 100644
index 00000000000000..cd707e632a45ff
--- /dev/null
+++ b/flang/test/Driver/nostdlib.f90
@@ -0,0 +1,29 @@
+! Check that the libraries do not appear when using -nostdlib and -nodefaultlibs
+
+! RUN: %flang -### -nostdlib --target=ppc64le-linux-gnu %s 2>&1 | FileCheck %s
+! RUN: %flang -### -nostdlib --target=aarch64-apple-darwin %s 2>&1 | FileCheck %s
+! RUN: %flang -### -nostdlib --target=sparc-sun-solaris2.11 %s 2>&1 | FileCheck %s
+! RUN: %flang -### -nostdlib --target=x86_64-unknown-freebsd %s 2>&1 | FileCheck %s
+! RUN: %flang -### -nostdlib --target=x86_64-unknown-netbsd %s 2>&1 | FileCheck %s
+! RUN: %flang -### -nostdlib --target=x86_64-unknown-openbsd %s 2>&1 | FileCheck %s
+! RUN: %flang -### -nostdlib --target=x86_64-unknown-dragonfly %s 2>&1 | FileCheck %s
+! RUN: %flang -### -nostdlib --target=x86_64-unknown-haiku %s 2>&1 | FileCheck %s
+! RUN: %flang -### -nostdlib --target=x86_64-windows-gnu %s 2>&1 | FileCheck %s
+
+! RUN: %flang -### -nodefaultlibs --target=ppc64le-linux-gnu %s 2>&1 | FileCheck %s
+! RUN: %flang -### -nodefaultlibs --target=aarch64-apple-darwin %s 2>&1 | FileCheck %s
+! RUN: %flang -### -nodefaultlibs --target=sparc-sun-solaris2.11 %s 2>&1 | FileCheck %s
+! RUN: %flang -### -nodefaultlibs --target=x86_64-unknown-freebsd %s 2>&1 | FileCheck %s
+! RUN: %flang -### -nodefaultlibs --target=x86_64-unknown-netbsd %s 2>&1 | FileCheck %s
+! RUN: %flang -### -nodefaultlibs --target=x86_64-unknown-openbsd %s 2>&1 | FileCheck %s
+! RUN: %flang -### -nodefaultlibs --target=x86_64-unknown-dragonfly %s 2>&1 | FileCheck %s
+! RUN: %flang -### -nodefaultlibs --target=x86_64-unknown-haiku %s 2>&1 | FileCheck %s
+! RUN: %flang -### -nodefaultlibs --target=x86_64-windows-gnu %s 2>&1 | FileCheck %s
+
+! -lgcc will not be linked on all platforms, so checking for that is redundant
+! in certain cases. But it is not clear that it is worth checking for each
+! platform individually.
+
+! CHECK-NOT: "-lFortranRuntime"
+! CHECK-NOT: "-lFortranDecimal"
+! CHECK-NOT: "-lgcc"
diff --git a/flang/test/Evaluate/fold-spacing.f90 b/flang/test/Evaluate/fold-spacing.f90
index 1d7b58081d70fe..cfae6eab8b9b8b 100644
--- a/flang/test/Evaluate/fold-spacing.f90
+++ b/flang/test/Evaluate/fold-spacing.f90
@@ -5,7 +5,9 @@ module m
   logical, parameter :: test_2 = spacing(-3.0) == scale(1.0, -22)
   logical, parameter :: test_3 = spacing(3.0d0) == scale(1.0, -51)
   logical, parameter :: test_4 = spacing(0.) == tiny(0.)
-  logical, parameter :: test_5 = spacing(tiny(0.)) == 1.e-45
+  logical, parameter :: test_5a = spacing(tiny(0.)) == tiny(0.)
+  logical, parameter :: test_5b = spacing(tiny(0.)/2) == tiny(0.)
+  logical, parameter :: test_5c = spacing(tiny(0.)*2) == tiny(0.)
   logical, parameter :: test_6 = spacing(8388608.) == 1.
   logical, parameter :: test_7 = spacing(spacing(tiny(.0))) == tiny(0.)
   logical, parameter :: test_11 = rrspacing(3.0) == scale(0.75, 24)
diff --git a/flang/test/Fir/CUDA/cuda-data-transfer.fir b/flang/test/Fir/CUDA/cuda-data-transfer.fir
new file mode 100644
index 00000000000000..f639a6c22b76d0
--- /dev/null
+++ b/flang/test/Fir/CUDA/cuda-data-transfer.fir
@@ -0,0 +1,140 @@
+// RUN: fir-opt --cuf-convert %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>} {
+
+func.func @_QPsub1() {
+  %0 = cuf.alloc !fir.box<!fir.heap<!fir.array<?xi32>>> {bindc_name = "adev", data_attr = #cuf.cuda<device>, uniq_name = "_QFsub1Eadev"} -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  %4:2 = hlfir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Eadev"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+  %5 = fir.alloca !fir.box<!fir.heap<!fir.array<?xi32>>> {bindc_name = "ahost", uniq_name = "_QFsub1Eahost"}
+  %9:2 = hlfir.declare %5 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Eahost"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+  cuf.data_transfer %4#0 to %9#0 {transfer_kind = #cuf.cuda_transfer<device_host>} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  cuf.free %4#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+  return
+}
+
+// CHECK-LABEL: func.func @_QPsub1()
+// CHECK: %[[ADEV:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Eadev"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+// CHECK: %[[AHOST:.*]]:2 = hlfir.declare %{{.*}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Eahost"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+// CHECK: %[[AHOST_LOAD:.*]] = fir.load %[[AHOST]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+// CHECK: %[[ADEV_LOAD:.*]] = fir.load %[[ADEV]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+// CHECK: %[[AHOST_BOX:.*]] = fir.convert %[[AHOST_LOAD]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.box<none>
+// CHECK: %[[ADEV_BOX:.*]] = fir.convert %[[ADEV_LOAD]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.box<none>
+// CHECK: fir.call @_FortranACUFDataTransferDescDesc(%[[AHOST_BOX]], %[[ADEV_BOX]], %c1{{.*}}, %{{.*}}, %{{.*}}) : (!fir.box<none>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> none
+
+func.func @_QPsub2() {
+  %0 = cuf.alloc !fir.box<!fir.heap<!fir.array<?xi32>>> {bindc_name = "adev", data_attr = #cuf.cuda<device>, uniq_name = "_QFsub2Eadev"} -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  %4:2 = hlfir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub2Eadev"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+  %c2_i32 = arith.constant 2 : i32
+  cuf.data_transfer %c2_i32 to %4#0 {transfer_kind = #cuf.cuda_transfer<host_device>} : i32, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  cuf.free %4#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+  return
+}
+
+// CHECK-LABEL: func.func @_QPsub2()
+// CHECK: %[[TEMP:.*]] = fir.alloca i32
+// CHECK: %[[ADEV:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub2Eadev"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+// CHECK: %[[C2:.*]] = arith.constant 2 : i32
+// CHECK: fir.store %[[C2]] to %[[TEMP]] : !fir.ref<i32>
+// CHECK: %[[ADEV_LOAD:.*]] = fir.load %[[ADEV]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+// CHECK: %[[ADEV_BOX:.*]] = fir.convert %[[ADEV_LOAD]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.box<none>
+// CHECK: %[[TEMP_CONV:.*]] = fir.convert %[[TEMP]] : (!fir.ref<i32>) -> !fir.llvm_ptr<i8>
+// CHECK: fir.call @_FortranACUFMemsetDescriptor(%[[ADEV_BOX]], %[[TEMP_CONV]], %{{.*}}, %{{.*}}) : (!fir.box<none>, !fir.llvm_ptr<i8>, !fir.ref<i8>, i32) -> none
+
+func.func @_QPsub3() {
+  %0 = cuf.alloc !fir.box<!fir.heap<!fir.array<?xi32>>> {bindc_name = "adev", data_attr = #cuf.cuda<device>, uniq_name = "_QFsub3Eadev"} -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  %4:2 = hlfir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub3Eadev"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+  %5 = fir.alloca i32 {bindc_name = "v", uniq_name = "_QFsub3Ev"}
+  %6:2 = hlfir.declare %5 {uniq_name = "_QFsub3Ev"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  cuf.data_transfer %6#0 to %4#0 {transfer_kind = #cuf.cuda_transfer<host_device>} : !fir.ref<i32>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  cuf.free %4#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+  return
+}
+
+// CHECK-LABEL: func.func @_QPsub3()
+// CHECK: %[[ADEV:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub3Eadev"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+// CHECK: %[[V:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsub3Ev"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK: %[[ADEV_LOAD:.*]] = fir.load %[[ADEV]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+// CHECK: %[[ADEV_BOX:.*]] = fir.convert %[[ADEV_LOAD]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.box<none>
+// CHECK: %[[V_CONV:.*]] = fir.convert %[[V]]#0 : (!fir.ref<i32>) -> !fir.llvm_ptr<i8>
+// CHECK: fir.call @_FortranACUFMemsetDescriptor(%[[ADEV_BOX]], %[[V_CONV]], %{{.*}}, %{{.*}}) : (!fir.box<none>, !fir.llvm_ptr<i8>, !fir.ref<i8>, i32) -> none
+
+func.func @_QPsub4() {
+  %0 = cuf.alloc !fir.box<!fir.heap<!fir.array<?xi32>>> {bindc_name = "adev", data_attr = #cuf.cuda<device>, uniq_name = "_QFsub4Eadev"} -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  %4:2 = hlfir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub4Eadev"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+  %c10 = arith.constant 10 : index
+  %5 = fir.alloca !fir.array<10xi32> {bindc_name = "ahost", uniq_name = "_QFsub4Eahost"}
+  %6 = fir.shape %c10 : (index) -> !fir.shape<1>
+  %7:2 = hlfir.declare %5(%6) {uniq_name = "_QFsub4Eahost"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+  cuf.data_transfer %7#0 to %4#0 {transfer_kind = #cuf.cuda_transfer<host_device>} : !fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+  cuf.data_transfer %4#0 to %7#0 {transfer_kind = #cuf.cuda_transfer<device_host>} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.array<10xi32>>
+  cuf.free %4#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+  return
+}
+
+// CHECK-LABEL: func.func @_QPsub4()
+// CHECK: %[[ADEV:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub4Eadev"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+// CHECK: %[[AHOST:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFsub4Eahost"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+// CHECK: %[[NBELEM:.*]] = arith.constant 10 : index
+// CHECK: %[[WIDTH:.*]] = arith.constant 4 : index
+// CHECK: %[[BYTES:.*]] = arith.muli %[[NBELEM]], %[[WIDTH]] : index
+// CHECK: %[[ADEV_LOAD:.*]] = fir.load %[[ADEV]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+// CHECK: %[[ADEV_BOX:.*]] = fir.convert %[[ADEV_LOAD]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.box<none>
+// CHECK: %[[AHOST_PTR:.*]] = fir.convert %[[AHOST]]#0 : (!fir.ref<!fir.array<10xi32>>) -> !fir.llvm_ptr<i8>
+// CHECK: %[[BYTES_CONV:.*]] = fir.convert %[[BYTES]] : (index) -> i64
+// CHECK: fir.call @_FortranACUFDataTransferDescPtr(%[[ADEV_BOX]], %[[AHOST_PTR]], %[[BYTES_CONV]], %c0{{.*}}, %{{.*}}, %{{.*}}) : (!fir.box<none>, !fir.llvm_ptr<i8>, i64, i32, !fir.ref<i8>, i32) -> none
+// CHECK: %[[NBELEM:.*]] = arith.constant 10 : index
+// CHECK: %[[WIDTH:.*]] = arith.constant 4 : index
+// CHECK: %[[BYTES:.*]] = arith.muli %[[NBELEM]], %[[WIDTH]] : index
+// CHECK: %[[ADEV_LOAD:.*]] = fir.load %[[ADEV]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+// CHECK: %[[AHOST_PTR:.*]] = fir.convert %[[AHOST]]#0 : (!fir.ref<!fir.array<10xi32>>) -> !fir.llvm_ptr<i8>
+// CHECK: %[[ADEV_BOX:.*]] = fir.convert %[[ADEV_LOAD]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.box<none>
+// CHECK: %[[BYTES_CONV:.*]] = fir.convert %[[BYTES]] : (index) -> i64
+// CHECK: fir.call @_FortranACUFDataTransferPtrDesc(%[[AHOST_PTR]], %[[ADEV_BOX]], %[[BYTES_CONV]], %c1{{.*}}, %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.box<none>, i64, i32, !fir.ref<i8>, i32) -> none
+
+func.func @_QPsub5(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}) {
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = cuf.alloc !fir.box<!fir.heap<!fir.array<?x?xi32>>> {bindc_name = "adev", data_attr = #cuf.cuda<device>, uniq_name = "_QFsub5Eadev"} -> !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+  %5:2 = hlfir.declare %1 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub5Eadev"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
+  %6:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFsub5En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %7 = fir.load %6#0 : !fir.ref<i32>
+  %8 = fir.convert %7 : (i32) -> i64
+  %9 = fir.convert %8 : (i64) -> index
+  %c0_0 = arith.constant 0 : index
+  %10 = arith.cmpi sgt, %9, %c0_0 : index
+  %11 = arith.select %10, %9, %c0_0 : index
+  %12 = fir.load %6#0 : !fir.ref<i32>
+  %13 = fir.convert %12 : (i32) -> i64
+  %14 = fir.convert %13 : (i64) -> index
+  %c0_1 = arith.constant 0 : index
+  %15 = arith.cmpi sgt, %14, %c0_1 : index
+  %16 = arith.select %15, %14, %c0_1 : index
+  %17 = fir.alloca !fir.array<?x?xi32>, %11, %16 {bindc_name = "ahost", uniq_name = "_QFsub5Eahost"}
+  %18 = fir.shape %11, %16 : (index, index) -> !fir.shape<2>
+  %19:2 = hlfir.declare %17(%18) {uniq_name = "_QFsub5Eahost"} : (!fir.ref<!fir.array<?x?xi32>>, !fir.shape<2>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.ref<!fir.array<?x?xi32>>)
+  cuf.data_transfer %19#1 to %5#0, %18 : !fir.shape<2> {transfer_kind = #cuf.cuda_transfer<host_device>} : !fir.ref<!fir.array<?x?xi32>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+  cuf.data_transfer %5#0 to %19#1, %18 : !fir.shape<2> {transfer_kind = #cuf.cuda_transfer<device_host>} : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.array<?x?xi32>>
+  return
+}
+
+// CHECK-LABEL: func.func @_QPsub5
+// CHECK: %[[ADEV:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub5Eadev"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
+// CHECK: %[[SHAPE:.*]] = fir.shape %[[I1:.*]], %[[I2:.*]] : (index, index) -> !fir.shape<2>
+// CHECK: %[[AHOST:.*]]:2 = hlfir.declare %{{.*}}(%[[SHAPE]]) {uniq_name = "_QFsub5Eahost"} : (!fir.ref<!fir.array<?x?xi32>>, !fir.shape<2>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.ref<!fir.array<?x?xi32>>)
+// CHECK: %[[NBELEM:.*]] = arith.muli %[[I1]], %[[I2]] : index
+// CHECK: %[[WIDTH:.*]] = arith.constant 4 : index
+// CHECK: %[[BYTES:.*]] = arith.muli %[[NBELEM]], %[[WIDTH]] : index
+// CHECK: %[[ADEV_LOAD:.*]] = fir.load %[[ADEV]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+// CHECK: %[[ADEV_BOX:.*]] = fir.convert %[[ADEV_LOAD]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>) -> !fir.box<none>
+// CHECK: %[[AHOST_PTR:.*]] = fir.convert %[[AHOST]]#1 : (!fir.ref<!fir.array<?x?xi32>>) -> !fir.llvm_ptr<i8>
+// CHECK: %[[BYTES_CONV:.*]] = fir.convert %[[BYTES]] : (index) -> i64
+// CHECK: fir.call @_FortranACUFDataTransferDescPtr(%[[ADEV_BOX]], %[[AHOST_PTR]], %[[BYTES_CONV]], %c0{{.*}}, %{{.*}}, %{{.*}}) : (!fir.box<none>, !fir.llvm_ptr<i8>, i64, i32, !fir.ref<i8>, i32) -> none
+// CHECK: %[[NBELEM:.*]] = arith.muli %[[I1]], %[[I2]] : index
+// CHECK: %[[WIDTH:.*]] = arith.constant 4 : index
+// CHECK: %[[BYTES:.*]] = arith.muli %[[NBELEM]], %[[WIDTH]] : index
+// CHECK: %[[ADEV_LOAD:.*]] = fir.load %[[ADEV]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+// CHECK: %[[AHOST_PTR:.*]] = fir.convert %[[AHOST]]#1 : (!fir.ref<!fir.array<?x?xi32>>) -> !fir.llvm_ptr<i8>
+// CHECK: %[[ADEV_BOX:.*]] = fir.convert %[[ADEV_LOAD]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>) -> !fir.box<none>
+// CHECK: %[[BYTES_CONV:.*]] = fir.convert %[[BYTES]] : (index) -> i64
+// CHECK: fir.call @_FortranACUFDataTransferPtrDesc(%[[AHOST_PTR]], %[[ADEV_BOX]], %[[BYTES_CONV]], %c1{{.*}}, %{{.*}}, %{{.*}}) : (!fir.llvm_ptr<i8>, !fir.box<none>, i64, i32, !fir.ref<i8>, i32) -> none
+
+} // end of module
diff --git a/flang/test/Semantics/OpenMP/private-assoc.f90 b/flang/test/Semantics/OpenMP/private-assoc.f90
new file mode 100644
index 00000000000000..bf50cd11de1724
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/private-assoc.f90
@@ -0,0 +1,32 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+
+! The ASSOCIATE name preserves the association with the selector established
+! in the associate statement. Therefore it is incorrect to change the
+! data-sharing attribute of the name.
+
+subroutine assoc_private(x)
+  integer :: x
+  associate(z => x)
+  !ERROR: Variable 'z' in ASSOCIATE cannot be in a PRIVATE clause
+  !$omp parallel private(z)
+  !$omp end parallel
+  end associate
+end subroutine
+
+subroutine assoc_firstprivate(x)
+  integer :: x
+  associate(z => x)
+  !ERROR: Variable 'z' in ASSOCIATE cannot be in a FIRSTPRIVATE clause
+  !$omp parallel firstprivate(z)
+  !$omp end parallel
+  end associate
+end subroutine
+
+subroutine assoc_lastprivate(x)
+  integer :: x
+  associate(z => x)
+  !ERROR: Variable 'z' in ASSOCIATE cannot be in a LASTPRIVATE clause
+  !$omp parallel sections lastprivate(z)
+  !$omp end parallel sections
+  end associate
+end subroutine
diff --git a/flang/test/Semantics/OpenMP/workshare02.f90 b/flang/test/Semantics/OpenMP/workshare02.f90
index e099ecb9f1e614..11f33d63a3eb80 100644
--- a/flang/test/Semantics/OpenMP/workshare02.f90
+++ b/flang/test/Semantics/OpenMP/workshare02.f90
@@ -40,6 +40,7 @@ subroutine workshare(aa, bb, cc, dd, ee, ff, n)
     cc = ee + my_func()
   end where
 
+  !WARNING: Impure procedure 'my_func' should not be referenced in a FORALL header
   !ERROR: User defined non-ELEMENTAL function 'my_func' is not allowed in a WORKSHARE construct
   forall (j = 1:my_func()) aa(j) = aa(j) + bb(j)
 
diff --git a/flang/test/Semantics/call11.f90 b/flang/test/Semantics/call11.f90
index 7bc4931890dee5..5358c741e2998e 100644
--- a/flang/test/Semantics/call11.f90
+++ b/flang/test/Semantics/call11.f90
@@ -42,30 +42,36 @@ subroutine test
       !ERROR: Impure procedure 'impure' may not be referenced in DO CONCURRENT
       a(j) = impure(j) ! C1139
     end do
-    do concurrent (k=impure(1):1); end do ! ok
-    do concurrent (k=1:impure(1)); end do ! ok
-    do concurrent (k=1:1:impure(1)); end do ! ok
-    forall (k=impure(1):1); end forall ! ok
-    forall (k=1:impure(1)); end forall ! ok
-    forall (k=1:1:impure(1)); end forall ! ok
+    !WARNING: Impure procedure 'impure' should not be referenced in a DO CONCURRENT header
+    do concurrent (k=impure(1):1); end do
+    !WARNING: Impure procedure 'impure' should not be referenced in a DO CONCURRENT header
+    do concurrent (k=1:impure(1)); end do
+    !WARNING: Impure procedure 'impure' should not be referenced in a DO CONCURRENT header
+    do concurrent (k=1:1:impure(1)); end do
+    !WARNING: Impure procedure 'impure' should not be referenced in a FORALL header
+    forall (k=impure(1):1); end forall
+    !WARNING: Impure procedure 'impure' should not be referenced in a FORALL header
+    forall (k=1:impure(1)); end forall
+    !WARNING: Impure procedure 'impure' should not be referenced in a FORALL header
+    forall (k=1:1:impure(1)); end forall
     do concurrent (j=1:1)
-      !ERROR: Impure procedure 'impure' may not be referenced in DO CONCURRENT
+      !ERROR: Impure procedure 'impure' may not be referenced in a DO CONCURRENT
       do concurrent (k=impure(1):1); end do
-      !ERROR: Impure procedure 'impure' may not be referenced in DO CONCURRENT
+      !ERROR: Impure procedure 'impure' may not be referenced in a DO CONCURRENT
       do concurrent (k=1:impure(1)); end do
-      !ERROR: Impure procedure 'impure' may not be referenced in DO CONCURRENT
+      !ERROR: Impure procedure 'impure' may not be referenced in a DO CONCURRENT
       do concurrent (k=1:1:impure(1)); end do
-      !ERROR: Impure procedure 'impure' may not be referenced in DO CONCURRENT
+      !ERROR: Impure procedure 'impure' may not be referenced in a DO CONCURRENT
       forall (k=impure(1):1); end forall
-      !ERROR: Impure procedure 'impure' may not be referenced in DO CONCURRENT
+      !ERROR: Impure procedure 'impure' may not be referenced in a DO CONCURRENT
       forall (k=1:impure(1)); end forall
-      !ERROR: Impure procedure 'impure' may not be referenced in DO CONCURRENT
+      !ERROR: Impure procedure 'impure' may not be referenced in a DO CONCURRENT
       forall (k=1:1:impure(1)); end forall
-      !ERROR: Impure procedure 'impure' may not be referenced in DO CONCURRENT
+      !ERROR: Impure procedure 'impure' may not be referenced in a DO CONCURRENT
       forall (k=impure(1):1) a(k) = 0.
-      !ERROR: Impure procedure 'impure' may not be referenced in DO CONCURRENT
+      !ERROR: Impure procedure 'impure' may not be referenced in a DO CONCURRENT
       forall (k=1:impure(1)) a(k) = 0.
-      !ERROR: Impure procedure 'impure' may not be referenced in DO CONCURRENT
+      !ERROR: Impure procedure 'impure' may not be referenced in a DO CONCURRENT
       forall (k=1:1:impure(1)) a(k) = 0.
     end do
     forall (j=1:1)
diff --git a/flang/test/Transforms/debug-assumed-size-array.fir b/flang/test/Transforms/debug-assumed-size-array.fir
index d25224fb1b5ec3..9adf711a0a6f7b 100644
--- a/flang/test/Transforms/debug-assumed-size-array.fir
+++ b/flang/test/Transforms/debug-assumed-size-array.fir
@@ -4,10 +4,11 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
   func.func @_QMhelperPfn(%arg0: !fir.ref<!fir.array<5x?xi32>> {fir.bindc_name = "a1"}, %arg1: !fir.ref<!fir.array<?xi32>> {fir.bindc_name = "a2"}, %arg2: !fir.ref<!fir.array<2x?xi32>> {fir.bindc_name = "a3"}) {
     %c5 = arith.constant 5 : index
     %c1 = arith.constant 1 : index
+    %c2 = arith.constant 2 : index
     %c-1 = arith.constant -1 : index
     %0 = fir.undefined !fir.dscope
     %1 = fircg.ext_declare %arg0(%c5, %c-1) dummy_scope %0 {uniq_name = "_QMhelperFfnEa1"} : (!fir.ref<!fir.array<5x?xi32>>, index, index, !fir.dscope) -> !fir.ref<!fir.array<5x?xi32>> loc(#loc1)
-    %2 = fircg.ext_declare %arg1(%c-1) dummy_scope %0 {uniq_name = "_QMhelperFfnEa2"} : (!fir.ref<!fir.array<?xi32>>, index, !fir.dscope) -> !fir.ref<!fir.array<?xi32>> loc(#loc2)
+    %2 = fircg.ext_declare %arg1(%c-1) origin %c2 dummy_scope %0 {uniq_name = "_QMhelperFfnEa2"} : (!fir.ref<!fir.array<?xi32>>, index, index, !fir.dscope) -> !fir.ref<!fir.array<?xi32>> loc(#loc2)
     return
   } loc(#loc3)
 }
@@ -16,6 +17,6 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
 #loc2 = loc("test.f90":4:1)
 
 // CHECK-DAG: #[[TY1:.*]] = #llvm.di_composite_type<tag = DW_TAG_array_type{{.*}}elements = #llvm.di_subrange<count = 5 : i64, lowerBound = 1 : i64>, #llvm.di_subrange<>>
-// CHECK-DAG: #[[TY2:.*]] = #llvm.di_composite_type<tag = DW_TAG_array_type{{.*}}elements = #llvm.di_subrange<>>
+// CHECK-DAG: #[[TY2:.*]] = #llvm.di_composite_type<tag = DW_TAG_array_type{{.*}}elements = #llvm.di_subrange<lowerBound = 2 : i64>>
 // CHECK-DAG: #llvm.di_local_variable<{{.*}}name = "a1"{{.*}}type = #[[TY1]]>
 // CHECK-DAG: #llvm.di_local_variable<{{.*}}name = "a2"{{.*}}type = #[[TY2]]>
diff --git a/flang/test/Transforms/debug-fn-info.fir b/flang/test/Transforms/debug-fn-info.fir
index 5433e088a648d0..f23a1a27e13ebd 100644
--- a/flang/test/Transforms/debug-fn-info.fir
+++ b/flang/test/Transforms/debug-fn-info.fir
@@ -52,10 +52,16 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
     %11 = fir.load %5 : !fir.ref<i32>
     return %11 : i32
   } loc(#loc3)
+  func.func private @_QFPfn3(%arg0: !fir.ref<i32> {fir.bindc_name = "abc"}) attributes {fir.host_symbol = @_QQmain, llvm.linkage = #llvm.linkage<internal>} {
+    %0 = fir.undefined !fir.dscope
+    %1 = fircg.ext_declare %arg0 dummy_scope %0 {uniq_name = "_QFFfn3Eabc"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+    return
+  } loc(#loc4)
 }
 #loc1 = loc("test.f90":15:1)
 #loc2 = loc("test.f90":26:22)
 #loc3 = loc("test2.f90":43:22)
+#loc4 = loc("test2.f90":53:22)
 
 
 // CHECK-DAG: #[[INT8:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 64, encoding = DW_ATE_signed>
@@ -64,12 +70,14 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
 // CHECK-DAG: #[[LOG1:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "logical", sizeInBits = 8, encoding = DW_ATE_boolean>
 // CHECK-DAG: #[[REAL4:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "real", sizeInBits = 32, encoding = DW_ATE_float>
 // CHECK-DAG: #[[LOG4:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "logical", sizeInBits = 32, encoding = DW_ATE_boolean>
-// CHECK: #[[TY0:.*]] = #llvm.di_subroutine_type<callingConvention = DW_CC_program>
+// CHECK: #[[TY0:.*]] = #llvm.di_subroutine_type<callingConvention = DW_CC_program, types = #di_null_type>
 // CHECK: #[[TY1:.*]] = #llvm.di_subroutine_type<callingConvention = DW_CC_normal, types = #[[INT8]], #[[INT4]], #[[REAL8]], #[[LOG1]]>
 // CHECK: #[[TY2:.*]] = #llvm.di_subroutine_type<callingConvention = DW_CC_normal, types = #[[INT4]], #[[INT8]], #[[REAL4]], #[[LOG4]]>
+// CHECK: #[[TY3:.*]] = #llvm.di_subroutine_type<callingConvention = DW_CC_normal, types = #di_null_type, #[[INT4]]>
 
 // Line numbers should match the number in corresponding loc entry.
 // CHECK: #llvm.di_subprogram<{{.*}}name = "_QQmain", linkageName = "_QQmain", file = {{.*}}, line = 15, scopeLine = 15, subprogramFlags = Definition, type = #[[TY0]]>
 // CHECK: #llvm.di_subprogram<{{.*}}name = "fn1", linkageName = "_QFPfn1", file = {{.*}}, line = 26, scopeLine = 26, subprogramFlags = Definition, type = #[[TY1]]>
 // CHECK: #llvm.di_subprogram<{{.*}}name = "fn2", linkageName = "_QFPfn2", file = {{.*}}, line = 43, scopeLine = 43, subprogramFlags = Definition, type = #[[TY2]]>
+// CHECK: #llvm.di_subprogram<{{.*}}name = "fn3", linkageName = "_QFPfn3", file = {{.*}}, line = 53, scopeLine = 53, subprogramFlags = Definition, type = #[[TY3]]>
 
diff --git a/flang/unittests/Evaluate/real.cpp b/flang/unittests/Evaluate/real.cpp
index ccbaf81f054cb5..a6152d346762fa 100644
--- a/flang/unittests/Evaluate/real.cpp
+++ b/flang/unittests/Evaluate/real.cpp
@@ -158,10 +158,9 @@ template <typename R> void basicTests(int rm, Rounding rounding) {
       TEST(ivf.flags.empty())(ldesc);
       MATCH(x, ivf.value.ToUInt64())(ldesc);
       if (rounding.mode == RoundingMode::TiesToEven) { // to match stold()
-        std::string buf;
-        llvm::raw_string_ostream ss{buf};
+        std::string decimal;
+        llvm::raw_string_ostream ss{decimal};
         vr.value.AsFortran(ss, kind, false /*exact*/);
-        std::string decimal{ss.str()};
         const char *p{decimal.data()};
         MATCH(x, static_cast<std::uint64_t>(std::stold(decimal)))
         ("%s %s", ldesc, p);
@@ -424,14 +423,13 @@ void subsetTests(int pass, Rounding rounding, std::uint32_t opds) {
       ("%d IsInfinite(0x%jx)", pass, static_cast<std::intmax_t>(rj));
 
       static constexpr int kind{REAL::bits / 8};
-      std::string ssBuf, cssBuf;
-      llvm::raw_string_ostream ss{ssBuf};
+      std::string s, cssBuf;
+      llvm::raw_string_ostream ss{s};
       llvm::raw_string_ostream css{cssBuf};
       x.AsFortran(ss, kind, false /*exact*/);
-      std::string s{ss.str()};
       if (IsNaN(rj)) {
         css << "(0._" << kind << "/0.)";
-        MATCH(css.str(), s)
+        MATCH(cssBuf, s)
         ("%d invalid(0x%jx)", pass, static_cast<std::intmax_t>(rj));
       } else if (IsInfinite(rj)) {
         css << '(';
@@ -439,7 +437,7 @@ void subsetTests(int pass, Rounding rounding, std::uint32_t opds) {
           css << '-';
         }
         css << "1._" << kind << "/0.)";
-        MATCH(css.str(), s)
+        MATCH(cssBuf, s)
         ("%d overflow(0x%jx)", pass, static_cast<std::intmax_t>(rj));
       } else {
         const char *p = s.data();
diff --git a/flang/unittests/Frontend/CodeGenActionTest.cpp b/flang/unittests/Frontend/CodeGenActionTest.cpp
index 9d798c7678ad15..5d75de03d4e55c 100644
--- a/flang/unittests/Frontend/CodeGenActionTest.cpp
+++ b/flang/unittests/Frontend/CodeGenActionTest.cpp
@@ -103,7 +103,7 @@ TEST(CodeGenAction, GracefullyHandleLLVMConversionFailure) {
   action.setCurrentInput(file);
 
   consumeError(action.execute());
-  ASSERT_EQ(diagnosticsOS.str(),
+  ASSERT_EQ(diagnosticOutput,
       "error: Lowering to LLVM IR failed\n"
       "error: failed to create the LLVM module\n");
 }
diff --git a/flang/unittests/Frontend/CompilerInstanceTest.cpp b/flang/unittests/Frontend/CompilerInstanceTest.cpp
index 35f1ec1748a3f6..3fe2f063e996a8 100644
--- a/flang/unittests/Frontend/CompilerInstanceTest.cpp
+++ b/flang/unittests/Frontend/CompilerInstanceTest.cpp
@@ -90,6 +90,6 @@ TEST(CompilerInstance, AllowDiagnosticLogWithUnownedDiagnosticConsumer) {
 
   // 6. Verify that the reported diagnostic wasn't lost and did end up in the
   // output stream
-  ASSERT_EQ(diagnosticsOS.str(), "error: expected no crash\n");
+  ASSERT_EQ(diagnosticOutput, "error: expected no crash\n");
 }
 } // namespace
diff --git a/libc/benchmarks/MemorySizeDistributions.cpp b/libc/benchmarks/MemorySizeDistributions.cpp
index f003cebd9c3762..c3590297445dd1 100644
--- a/libc/benchmarks/MemorySizeDistributions.cpp
+++ b/libc/benchmarks/MemorySizeDistributions.cpp
@@ -197,7 +197,7 @@ getDistributionOrDie(ArrayRef<MemorySizeDistribution> Distributions,
          << "', available distributions:\n";
   for (const auto &MSD : Distributions)
     Stream << "'" << MSD.Name << "'\n";
-  report_fatal_error(Stream.str());
+  report_fatal_error(Message);
 }
 
 } // namespace libc_benchmarks
diff --git a/libc/benchmarks/automemcpy/unittests/CodeGenTest.cpp b/libc/benchmarks/automemcpy/unittests/CodeGenTest.cpp
index 17bc189fa3f127..a7fc8570a73b01 100644
--- a/libc/benchmarks/automemcpy/unittests/CodeGenTest.cpp
+++ b/libc/benchmarks/automemcpy/unittests/CodeGenTest.cpp
@@ -50,7 +50,7 @@ TEST(Automemcpy, Codegen) {
   raw_string_ostream OutputStream(Output);
   Serialize(OutputStream, kDescriptors);
 
-  EXPECT_STREQ(OutputStream.str().c_str(),
+  EXPECT_STREQ(Output.c_str(),
                R"(// This file is auto-generated by libc/benchmarks/automemcpy.
 // Functions : 6
 
diff --git a/libc/docs/date_and_time.rst b/libc/docs/date_and_time.rst
index db77d2c5ba61a5..b745a3b416f802 100644
--- a/libc/docs/date_and_time.rst
+++ b/libc/docs/date_and_time.rst
@@ -55,6 +55,10 @@ Implementation Status
 +---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
 | asctime_r           | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |
 +---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ctime               | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ctime_r             | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |
++---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
 | clock               | |check| | |check| |         |     |check|     |         |         |         |         |         |         |         |         |
 +---------------------+---------+---------+---------+-----------------+---------+---------+---------+---------+---------+---------+---------+---------+
 | clock_getcpuclockid |         |         |         |                 |         |         |         |         |         |         |         |         |
diff --git a/libcxx/docs/Contributing.rst b/libcxx/docs/Contributing.rst
index f81c537e00c243..b15fc88943d59b 100644
--- a/libcxx/docs/Contributing.rst
+++ b/libcxx/docs/Contributing.rst
@@ -168,7 +168,7 @@ sure you don't forget anything:
 
 - Did you add the relevant feature test macro(s) for your feature? Did you update the ``generate_feature_test_macro_components.py`` script with it?
 - Did you run the ``libcxx-generate-files`` target and verify its output?
-- If needed, did you add `_LIBCPP_PUSH_MACROS` and `_LIBCPP_POP_MACROS` to the relevant headers?
+- If needed, did you add ``_LIBCPP_PUSH_MACROS`` and ``_LIBCPP_POP_MACROS`` to the relevant headers?
 
 The review process
 ==================
diff --git a/libcxx/docs/Status/Cxx20Papers.csv b/libcxx/docs/Status/Cxx20Papers.csv
index 4a4d75b21fd408..cc75d28f14aac2 100644
--- a/libcxx/docs/Status/Cxx20Papers.csv
+++ b/libcxx/docs/Status/Cxx20Papers.csv
@@ -118,7 +118,7 @@
 "`P1361R2 <https://wg21.link/P1361R2>`__","Integration of chrono with text formatting","2019-07 (Cologne)","|Partial|","",""
 "`P1423R3 <https://wg21.link/P1423R3>`__","char8_t backward compatibility remediation","2019-07 (Cologne)","|Complete|","15.0",""
 "`P1424R1 <https://wg21.link/P1424R1>`__","'constexpr' feature macro concerns","2019-07 (Cologne)","|Nothing To Do|","","Superseded by `P1902 <https://wg21.link/P1902>`__"
-"`P1466R3 <https://wg21.link/P1466R3>`__","Miscellaneous minor fixes for chrono","2019-07 (Cologne)","","",""
+"`P1466R3 <https://wg21.link/P1466R3>`__","Miscellaneous minor fixes for chrono","2019-07 (Cologne)","|Partial|","",""
 "`P1474R1 <https://wg21.link/P1474R1>`__","Helpful pointers for ContiguousIterator","2019-07 (Cologne)","|Complete|","15.0",""
 "`P1502R1 <https://wg21.link/P1502R1>`__","Standard library header units for C++20","2019-07 (Cologne)","","",""
 "`P1522R1 <https://wg21.link/P1522R1>`__","Iterator Difference Type and Integer Overflow","2019-07 (Cologne)","|Complete|","15.0",""
diff --git a/libcxx/include/__assert b/libcxx/include/__assert
index 49769fb4d44978..90eaa6023587b9 100644
--- a/libcxx/include/__assert
+++ b/libcxx/include/__assert
@@ -23,10 +23,10 @@
        : _LIBCPP_ASSERTION_HANDLER(__FILE__ ":" _LIBCPP_TOSTRING(__LINE__) ": assertion " _LIBCPP_TOSTRING(            \
              expression) " failed: " message "\n"))
 
-// TODO: __builtin_assume can currently inhibit optimizations. Until this has been fixed and we can add
-//       assumptions without a clear optimization intent, disable that to avoid worsening the code generation.
-//       See https://discourse.llvm.org/t/llvm-assume-blocks-optimization/71609 for a discussion.
-#if 0 && __has_builtin(__builtin_assume)
+// WARNING: __builtin_assume can currently inhibit optimizations. Only add assumptions with a clear
+// optimization intent. See https://discourse.llvm.org/t/llvm-assume-blocks-optimization/71609 for a
+// discussion.
+#if __has_builtin(__builtin_assume)
 #  define _LIBCPP_ASSUME(expression)                                                                                   \
     (_LIBCPP_DIAGNOSTIC_PUSH _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wassume")                                              \
          __builtin_assume(static_cast<bool>(expression)) _LIBCPP_DIAGNOSTIC_POP)
@@ -44,18 +44,18 @@
 #  define _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(expression, message)    _LIBCPP_ASSERT(expression, message)
 // Disabled checks.
 // On most modern platforms, dereferencing a null pointer does not lead to an actual memory access.
-#  define _LIBCPP_ASSERT_NON_NULL(expression, message)                _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_NON_NULL(expression, message)                ((void)0)
 // Overlapping ranges will make algorithms produce incorrect results but don't directly lead to a security
 // vulnerability.
-#  define _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(expression, message)  _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_VALID_DEALLOCATION(expression, message)      _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(expression, message) _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(expression, message)    _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(expression, message)  _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_PEDANTIC(expression, message)                _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(expression, message)    _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_INTERNAL(expression, message)                _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_UNCATEGORIZED(expression, message)           _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(expression, message)  ((void)0)
+#  define _LIBCPP_ASSERT_VALID_DEALLOCATION(expression, message)      ((void)0)
+#  define _LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(expression, message) ((void)0)
+#  define _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(expression, message)    ((void)0)
+#  define _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(expression, message)  ((void)0)
+#  define _LIBCPP_ASSERT_PEDANTIC(expression, message)                ((void)0)
+#  define _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(expression, message)    ((void)0)
+#  define _LIBCPP_ASSERT_INTERNAL(expression, message)                ((void)0)
+#  define _LIBCPP_ASSERT_UNCATEGORIZED(expression, message)           ((void)0)
 
 // Extensive hardening mode checks.
 
@@ -73,8 +73,8 @@
 #  define _LIBCPP_ASSERT_PEDANTIC(expression, message)                _LIBCPP_ASSERT(expression, message)
 #  define _LIBCPP_ASSERT_UNCATEGORIZED(expression, message)           _LIBCPP_ASSERT(expression, message)
 // Disabled checks.
-#  define _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(expression, message)    _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_INTERNAL(expression, message)                _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(expression, message)    ((void)0)
+#  define _LIBCPP_ASSERT_INTERNAL(expression, message)                ((void)0)
 
 // Debug hardening mode checks.
 
@@ -99,18 +99,18 @@
 #else
 
 // All checks disabled.
-#  define _LIBCPP_ASSERT_VALID_INPUT_RANGE(expression, message)       _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(expression, message)    _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_NON_NULL(expression, message)                _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(expression, message)  _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_VALID_DEALLOCATION(expression, message)      _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(expression, message) _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(expression, message)    _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(expression, message)  _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_PEDANTIC(expression, message)                _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(expression, message)    _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_INTERNAL(expression, message)                _LIBCPP_ASSUME(expression)
-#  define _LIBCPP_ASSERT_UNCATEGORIZED(expression, message)           _LIBCPP_ASSUME(expression)
+#  define _LIBCPP_ASSERT_VALID_INPUT_RANGE(expression, message)       ((void)0)
+#  define _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(expression, message)    ((void)0)
+#  define _LIBCPP_ASSERT_NON_NULL(expression, message)                ((void)0)
+#  define _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(expression, message)  ((void)0)
+#  define _LIBCPP_ASSERT_VALID_DEALLOCATION(expression, message)      ((void)0)
+#  define _LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(expression, message) ((void)0)
+#  define _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(expression, message)    ((void)0)
+#  define _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(expression, message)  ((void)0)
+#  define _LIBCPP_ASSERT_PEDANTIC(expression, message)                ((void)0)
+#  define _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(expression, message)    ((void)0)
+#  define _LIBCPP_ASSERT_INTERNAL(expression, message)                ((void)0)
+#  define _LIBCPP_ASSERT_UNCATEGORIZED(expression, message)           ((void)0)
 
 #endif // _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_FAST
 // clang-format on
diff --git a/libcxx/include/__locale_dir/locale_base_api/ibm.h b/libcxx/include/__locale_dir/locale_base_api/ibm.h
index 01af20194428b9..fa3bc1c3633f5d 100644
--- a/libcxx/include/__locale_dir/locale_base_api/ibm.h
+++ b/libcxx/include/__locale_dir/locale_base_api/ibm.h
@@ -82,7 +82,7 @@ strtoull_l(const char* __nptr, char** __endptr, int __base, locale_t locale) {
 inline _LIBCPP_HIDE_FROM_ABI
 _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 2, 0) int vasprintf(char** strp, const char* fmt, va_list ap) {
   const size_t buff_size = 256;
-  if ((*strp = (char*)malloc(buff_size)) == NULL) {
+  if ((*strp = (char*)malloc(buff_size)) == nullptr) {
     return -1;
   }
 
@@ -97,7 +97,7 @@ _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 2, 0) int vasprintf(char** strp, const char
   va_end(ap_copy);
 
   if ((size_t)str_size >= buff_size) {
-    if ((*strp = (char*)realloc(*strp, str_size + 1)) == NULL) {
+    if ((*strp = (char*)realloc(*strp, str_size + 1)) == nullptr) {
       return -1;
     }
     str_size = vsnprintf(*strp, str_size + 1, fmt, ap);
diff --git a/libcxx/include/__support/xlocale/__nop_locale_mgmt.h b/libcxx/include/__support/xlocale/__nop_locale_mgmt.h
index b9ffcbe1622d51..5aaf3eaa6441d9 100644
--- a/libcxx/include/__support/xlocale/__nop_locale_mgmt.h
+++ b/libcxx/include/__support/xlocale/__nop_locale_mgmt.h
@@ -15,13 +15,13 @@
 // Patch over lack of extended locale support
 typedef void* locale_t;
 
-inline _LIBCPP_HIDE_FROM_ABI locale_t duplocale(locale_t) { return NULL; }
+inline _LIBCPP_HIDE_FROM_ABI locale_t duplocale(locale_t) { return nullptr; }
 
 inline _LIBCPP_HIDE_FROM_ABI void freelocale(locale_t) {}
 
-inline _LIBCPP_HIDE_FROM_ABI locale_t newlocale(int, const char*, locale_t) { return NULL; }
+inline _LIBCPP_HIDE_FROM_ABI locale_t newlocale(int, const char*, locale_t) { return nullptr; }
 
-inline _LIBCPP_HIDE_FROM_ABI locale_t uselocale(locale_t) { return NULL; }
+inline _LIBCPP_HIDE_FROM_ABI locale_t uselocale(locale_t) { return nullptr; }
 
 #define LC_COLLATE_MASK (1 << LC_COLLATE)
 #define LC_CTYPE_MASK (1 << LC_CTYPE)
diff --git a/libcxx/include/string b/libcxx/include/string
index 76359022f3650c..fdb189016bfbac 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -749,6 +749,14 @@ struct __can_be_converted_to_string_view
 struct __uninitialized_size_tag {};
 struct __init_with_sentinel_tag {};
 
+template <size_t _PaddingSize>
+struct __padding {
+  char __padding_[_PaddingSize];
+};
+
+template <>
+struct __padding<0> {};
+
 template <class _CharT, class _Traits, class _Allocator>
 class basic_string {
 private:
@@ -853,7 +861,7 @@ private:
 
   struct __short {
     value_type __data_[__min_cap];
-    unsigned char __padding_[sizeof(value_type) - 1];
+    _LIBCPP_NO_UNIQUE_ADDRESS __padding<sizeof(value_type) - 1> __padding_;
     unsigned char __size_    : 7;
     unsigned char __is_long_ : 1;
   };
@@ -905,7 +913,7 @@ private:
       unsigned char __is_long_ : 1;
       unsigned char __size_    : 7;
     };
-    char __padding_[sizeof(value_type) - 1];
+    _LIBCPP_NO_UNIQUE_ADDRESS __padding<sizeof(value_type) - 1> __padding_;
     value_type __data_[__min_cap];
   };
 
diff --git a/libcxx/src/atomic.cpp b/libcxx/src/atomic.cpp
index ae062155661e2f..c1af8d6f95aaef 100644
--- a/libcxx/src/atomic.cpp
+++ b/libcxx/src/atomic.cpp
@@ -94,11 +94,11 @@ static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const vo
 
 static void
 __libcpp_platform_wait_on_address(__cxx_atomic_contention_t const volatile* __ptr, __cxx_contention_t __val) {
-  _umtx_op(const_cast<__cxx_atomic_contention_t*>(__ptr), UMTX_OP_WAIT, __val, NULL, NULL);
+  _umtx_op(const_cast<__cxx_atomic_contention_t*>(__ptr), UMTX_OP_WAIT, __val, nullptr, nullptr);
 }
 
 static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const volatile* __ptr, bool __notify_one) {
-  _umtx_op(const_cast<__cxx_atomic_contention_t*>(__ptr), UMTX_OP_WAKE, __notify_one ? 1 : INT_MAX, NULL, NULL);
+  _umtx_op(const_cast<__cxx_atomic_contention_t*>(__ptr), UMTX_OP_WAKE, __notify_one ? 1 : INT_MAX, nullptr, nullptr);
 }
 
 #else // <- Add other operating systems here
diff --git a/libcxx/src/locale.cpp b/libcxx/src/locale.cpp
index 0f87c7099fe37e..484963dcede9d5 100644
--- a/libcxx/src/locale.cpp
+++ b/libcxx/src/locale.cpp
@@ -1004,7 +1004,7 @@ const ctype<char>::mask* ctype<char>::classic_table() noexcept {
 #    warning ctype<char>::classic_table() is not implemented
   printf("ctype<char>::classic_table() is not implemented\n");
   abort();
-  return NULL;
+  return nullptr;
 #  endif
 }
 #endif
diff --git a/libcxx/src/support/ibm/mbsnrtowcs.cpp b/libcxx/src/support/ibm/mbsnrtowcs.cpp
index 122dc5ed892f35..d0006a8468aad5 100644
--- a/libcxx/src/support/ibm/mbsnrtowcs.cpp
+++ b/libcxx/src/support/ibm/mbsnrtowcs.cpp
@@ -48,7 +48,7 @@ _LIBCPP_EXPORTED_FROM_ABI size_t mbsnrtowcs(
     size_t dest_remaining   = max_dest_chars - dest_converted;
 
     if (dst == nullptr) {
-      result = mbrtowc(NULL, *src + source_converted, source_remaining, ps);
+      result = mbrtowc(nullptr, *src + source_converted, source_remaining, ps);
     } else if (dest_remaining >= source_remaining) {
       // dst has enough space to translate in-place.
       result = mbrtowc(dst + dest_converted, *src + source_converted, source_remaining, ps);
@@ -86,7 +86,7 @@ _LIBCPP_EXPORTED_FROM_ABI size_t mbsnrtowcs(
 
   if (dst) {
     if (result == terminated_sequence)
-      *src = NULL;
+      *src = nullptr;
     else
       *src += source_converted;
   }
diff --git a/libcxx/src/support/ibm/wcsnrtombs.cpp b/libcxx/src/support/ibm/wcsnrtombs.cpp
index f0372152339fc2..df87b9ea07f85f 100644
--- a/libcxx/src/support/ibm/wcsnrtombs.cpp
+++ b/libcxx/src/support/ibm/wcsnrtombs.cpp
@@ -41,7 +41,7 @@ _LIBCPP_EXPORTED_FROM_ABI size_t wcsnrtombs(
     size_t dest_remaining = dst_size_bytes - dest_converted;
 
     if (dst == nullptr) {
-      result = wcrtomb(NULL, c, ps);
+      result = wcrtomb(nullptr, c, ps);
     } else if (dest_remaining >= static_cast<size_t>(MB_CUR_MAX)) {
       // dst has enough space to translate in-place.
       result = wcrtomb(dst + dest_converted, c, ps);
@@ -82,7 +82,7 @@ _LIBCPP_EXPORTED_FROM_ABI size_t wcsnrtombs(
 
     if (c == L'\0') {
       if (dst)
-        *src = NULL;
+        *src = nullptr;
       return dest_converted;
     }
   }
diff --git a/libcxx/src/support/ibm/xlocale_zos.cpp b/libcxx/src/support/ibm/xlocale_zos.cpp
index 9a90e08e11cf9c..136999ec0b02f8 100644
--- a/libcxx/src/support/ibm/xlocale_zos.cpp
+++ b/libcxx/src/support/ibm/xlocale_zos.cpp
@@ -20,12 +20,12 @@ locale_t newlocale(int category_mask, const char* locale, locale_t base) {
   std::string current_loc_name(setlocale(LC_ALL, 0));
 
   // Check for errors.
-  if (category_mask == LC_ALL_MASK && setlocale(LC_ALL, locale) == NULL) {
+  if (category_mask == LC_ALL_MASK && setlocale(LC_ALL, locale) == nullptr) {
     errno = EINVAL;
     return (locale_t)0;
   } else {
     for (int _Cat = 0; _Cat <= _LC_MAX; ++_Cat) {
-      if ((_CATMASK(_Cat) & category_mask) != 0 && setlocale(_Cat, locale) == NULL) {
+      if ((_CATMASK(_Cat) & category_mask) != 0 && setlocale(_Cat, locale) == nullptr) {
         setlocale(LC_ALL, current_loc_name.c_str());
         errno = EINVAL;
         return (locale_t)0;
@@ -74,12 +74,12 @@ locale_t uselocale(locale_t newloc) {
   if (newloc) {
     // Set locales and check for errors.
     bool is_error =
-        (newloc->category_mask & LC_COLLATE_MASK && setlocale(LC_COLLATE, newloc->lc_collate.c_str()) == NULL) ||
-        (newloc->category_mask & LC_CTYPE_MASK && setlocale(LC_CTYPE, newloc->lc_ctype.c_str()) == NULL) ||
-        (newloc->category_mask & LC_MONETARY_MASK && setlocale(LC_MONETARY, newloc->lc_monetary.c_str()) == NULL) ||
-        (newloc->category_mask & LC_NUMERIC_MASK && setlocale(LC_NUMERIC, newloc->lc_numeric.c_str()) == NULL) ||
-        (newloc->category_mask & LC_TIME_MASK && setlocale(LC_TIME, newloc->lc_time.c_str()) == NULL) ||
-        (newloc->category_mask & LC_MESSAGES_MASK && setlocale(LC_MESSAGES, newloc->lc_messages.c_str()) == NULL);
+        (newloc->category_mask & LC_COLLATE_MASK && setlocale(LC_COLLATE, newloc->lc_collate.c_str()) == nullptr) ||
+        (newloc->category_mask & LC_CTYPE_MASK && setlocale(LC_CTYPE, newloc->lc_ctype.c_str()) == nullptr) ||
+        (newloc->category_mask & LC_MONETARY_MASK && setlocale(LC_MONETARY, newloc->lc_monetary.c_str()) == nullptr) ||
+        (newloc->category_mask & LC_NUMERIC_MASK && setlocale(LC_NUMERIC, newloc->lc_numeric.c_str()) == nullptr) ||
+        (newloc->category_mask & LC_TIME_MASK && setlocale(LC_TIME, newloc->lc_time.c_str()) == nullptr) ||
+        (newloc->category_mask & LC_MESSAGES_MASK && setlocale(LC_MESSAGES, newloc->lc_messages.c_str()) == nullptr);
 
     if (is_error) {
       setlocale(LC_ALL, current_loc_name.c_str());
diff --git a/libcxx/src/support/win32/support.cpp b/libcxx/src/support/win32/support.cpp
index fed51d8b0bede9..ccf5ff5acb4581 100644
--- a/libcxx/src/support/win32/support.cpp
+++ b/libcxx/src/support/win32/support.cpp
@@ -17,13 +17,13 @@
 // a pointer to a malloc'd string in *sptr.
 // If return >= 0, use free to delete *sptr.
 int __libcpp_vasprintf(char** sptr, const char* __restrict format, va_list ap) {
-  *sptr = NULL;
+  *sptr = nullptr;
   // Query the count required.
   va_list ap_copy;
   va_copy(ap_copy, ap);
   _LIBCPP_DIAGNOSTIC_PUSH
   _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wformat-nonliteral")
-  int count = vsnprintf(NULL, 0, format, ap_copy);
+  int count = vsnprintf(nullptr, 0, format, ap_copy);
   _LIBCPP_DIAGNOSTIC_POP
   va_end(ap_copy);
   if (count < 0)
@@ -81,7 +81,7 @@ size_t mbsnrtowcs(wchar_t* __restrict dst,
     // if result > 0, it's the size in bytes of that character.
     // othewise if result is zero it indicates the null character has been found.
     // otherwise it's an error and errno may be set.
-    size_t char_size = mbrtowc(dst ? dst + dest_converted : NULL, *src + source_converted, source_remaining, ps);
+    size_t char_size = mbrtowc(dst ? dst + dest_converted : nullptr, *src + source_converted, source_remaining, ps);
     // Don't do anything to change errno from here on.
     if (char_size > 0) {
       source_remaining -= char_size;
@@ -95,7 +95,7 @@ size_t mbsnrtowcs(wchar_t* __restrict dst,
   }
   if (dst) {
     if (have_result && result == terminated_sequence)
-      *src = NULL;
+      *src = nullptr;
     else
       *src += source_converted;
   }
@@ -141,7 +141,7 @@ size_t wcsnrtombs(char* __restrict dst,
     if (dst)
       result = wcrtomb_s(&char_size, dst + dest_converted, dest_remaining, c, ps);
     else
-      result = wcrtomb_s(&char_size, NULL, 0, c, ps);
+      result = wcrtomb_s(&char_size, nullptr, 0, c, ps);
     // If result is zero there is no error and char_size contains the
     // size of the multi-byte-sequence converted.
     // Otherwise result indicates an errno type error.
@@ -161,7 +161,7 @@ size_t wcsnrtombs(char* __restrict dst,
   }
   if (dst) {
     if (terminator_found)
-      *src = NULL;
+      *src = nullptr;
     else
       *src = *src + source_converted;
   }
diff --git a/libcxx/src/support/win32/thread_win32.cpp b/libcxx/src/support/win32/thread_win32.cpp
index a2585c0b89f287..3a67d759f0f5e8 100644
--- a/libcxx/src/support/win32/thread_win32.cpp
+++ b/libcxx/src/support/win32/thread_win32.cpp
@@ -129,7 +129,7 @@ __libcpp_init_once_execute_once_thunk(PINIT_ONCE __init_once, PVOID __parameter,
 
 int __libcpp_execute_once(__libcpp_exec_once_flag* __flag, void (*__init_routine)(void)) {
   if (!InitOnceExecuteOnce(
-          (PINIT_ONCE)__flag, __libcpp_init_once_execute_once_thunk, reinterpret_cast<void*>(__init_routine), NULL))
+          (PINIT_ONCE)__flag, __libcpp_init_once_execute_once_thunk, reinterpret_cast<void*>(__init_routine), nullptr))
     return GetLastError();
   return 0;
 }
diff --git a/libcxx/test/libcxx/feature_test_macro/ftm_metadata.sh.py b/libcxx/test/libcxx/feature_test_macro/ftm_metadata.sh.py
index 1dec1ae612ecb1..8c47448a800415 100644
--- a/libcxx/test/libcxx/feature_test_macro/ftm_metadata.sh.py
+++ b/libcxx/test/libcxx/feature_test_macro/ftm_metadata.sh.py
@@ -47,5 +47,10 @@ def test(output, expected):
             "test_suite_guard": None,
             "libcxx_guard": None,
         },
+        "__cpp_lib_missing_FTM_in_older_standard": {
+            "headers": [],
+            "test_suite_guard": None,
+            "libcxx_guard": None,
+        },
     },
 )
diff --git a/libcxx/test/libcxx/feature_test_macro/implemented_ftms.sh.py b/libcxx/test/libcxx/feature_test_macro/implemented_ftms.sh.py
index 62a3c46b15db3c..942cd4b6087608 100644
--- a/libcxx/test/libcxx/feature_test_macro/implemented_ftms.sh.py
+++ b/libcxx/test/libcxx/feature_test_macro/implemented_ftms.sh.py
@@ -50,5 +50,10 @@ def test(output, expected):
             "c++23": "202102L",
             "c++26": "202102L",
         },
+        "__cpp_lib_missing_FTM_in_older_standard": {
+            "c++17": None,
+            "c++20": None,
+            "c++26": None,
+        },
     },
 )
diff --git a/libcxx/test/libcxx/feature_test_macro/standard_ftms.sh.py b/libcxx/test/libcxx/feature_test_macro/standard_ftms.sh.py
index 231e6072898752..95c9b9baaf2fa4 100644
--- a/libcxx/test/libcxx/feature_test_macro/standard_ftms.sh.py
+++ b/libcxx/test/libcxx/feature_test_macro/standard_ftms.sh.py
@@ -50,5 +50,11 @@ def test(output, expected):
             "c++23": "202106L",
             "c++26": "202306L",
         },
+        "__cpp_lib_missing_FTM_in_older_standard": {
+            "c++17": "2017L",
+            "c++20": "2020L",
+            "c++23": "2020L",
+            "c++26": "2026L",
+        },
     },
 )
diff --git a/libcxx/test/libcxx/feature_test_macro/test_data.json b/libcxx/test/libcxx/feature_test_macro/test_data.json
index 18c8836fa5149f..f7ae74fb83286c 100644
--- a/libcxx/test/libcxx/feature_test_macro/test_data.json
+++ b/libcxx/test/libcxx/feature_test_macro/test_data.json
@@ -154,5 +154,35 @@
     "headers": [
       "variant"
     ]
+  },
+  {
+    "name": "__cpp_lib_missing_FTM_in_older_standard",
+    "values": {
+      "c++17": {
+        "2017": [
+          {
+            "title": "Some FTM missing a paper in an older Standard mode, which should result in the FTM never being defined.",
+            "implemented": false
+          }
+        ]
+      },
+      "c++20": {
+        "2020": [
+          {
+            "title": "",
+            "implemented": true
+          }
+        ]
+      },
+      "c++26": {
+        "2026": [
+          {
+            "title": "",
+            "implemented": true
+          }
+        ]
+      }
+    },
+    "headers": []
   }
 ]
diff --git a/libcxx/test/libcxx/feature_test_macro/version_header.sh.py b/libcxx/test/libcxx/feature_test_macro/version_header.sh.py
index bbb863371f7639..634f43b83cafaa 100644
--- a/libcxx/test/libcxx/feature_test_macro/version_header.sh.py
+++ b/libcxx/test/libcxx/feature_test_macro/version_header.sh.py
@@ -43,6 +43,7 @@ def test(output, expected):
 #  define __cpp_lib_any 201606L
 #  define __cpp_lib_parallel_algorithm 201603L
 #  define __cpp_lib_variant 202102L
+// define __cpp_lib_missing_FTM_in_older_standard 2017L
 #endif // _LIBCPP_STD_VER >= 17
 
 #if _LIBCPP_STD_VER >= 20
@@ -50,8 +51,8 @@ def test(output, expected):
 #    define __cpp_lib_barrier 201907L
 #  endif
 // define __cpp_lib_format 202110L
-#  undef __cpp_lib_variant
-#  define __cpp_lib_variant 202106L
+// define __cpp_lib_variant 202106L
+// define __cpp_lib_missing_FTM_in_older_standard 2020L
 #endif // _LIBCPP_STD_VER >= 20
 
 #if _LIBCPP_STD_VER >= 23
@@ -64,8 +65,8 @@ def test(output, expected):
 #    define __cpp_lib_barrier 299900L
 #  endif
 // define __cpp_lib_format 202311L
-#  undef __cpp_lib_variant
-#  define __cpp_lib_variant 202306L
+// define __cpp_lib_variant 202306L
+// define __cpp_lib_missing_FTM_in_older_standard 2026L
 #endif // _LIBCPP_STD_VER >= 26
 
 #endif // _LIBCPP_VERSION
diff --git a/libcxx/test/libcxx/feature_test_macro/version_header_implementation.sh.py b/libcxx/test/libcxx/feature_test_macro/version_header_implementation.sh.py
index db90206ae770f3..a2fc1c5682f63b 100644
--- a/libcxx/test/libcxx/feature_test_macro/version_header_implementation.sh.py
+++ b/libcxx/test/libcxx/feature_test_macro/version_header_implementation.sh.py
@@ -9,106 +9,137 @@
 # RUN: %{python} %s %{libcxx-dir}/utils %{libcxx-dir}/test/libcxx/feature_test_macro/test_data.json
 
 import sys
+import unittest
 
-sys.path.append(sys.argv[1])
-from generate_feature_test_macro_components import FeatureTestMacros
-
+UTILS = sys.argv[1]
+TEST_DATA = sys.argv[2]
+del sys.argv[1:3]
 
-def test(output, expected):
-    assert output == expected, f"expected\n{expected}\n\noutput\n{output}"
+sys.path.append(UTILS)
+from generate_feature_test_macro_components import FeatureTestMacros
 
+class Test(unittest.TestCase):
+    def setUp(self):
+        self.ftm = FeatureTestMacros(TEST_DATA)
+        self.maxDiff = None # This causes the diff to be printed when the test fails
 
-ftm = FeatureTestMacros(sys.argv[2])
-test(
-    ftm.version_header_implementation,
-    {
-        "17": [
-            {
-                "__cpp_lib_any": {
-                    "value": "201606L",
-                    "implemented": True,
-                    "need_undef": False,
-                    "condition": None,
+    def test_implementation(self):
+        expected = {
+            "17": [
+                {
+                    "__cpp_lib_any": {
+                        "value": "201606L",
+                        "implemented": True,
+                        "need_undef": False,
+                        "condition": None,
+                    },
+                },
+                {
+                    "__cpp_lib_parallel_algorithm": {
+                        "value": "201603L",
+                        "implemented": True,
+                        "need_undef": False,
+                        "condition": None,
+                    },
                 },
-            },
-            {
-                "__cpp_lib_parallel_algorithm": {
-                    "value": "201603L",
-                    "implemented": True,
-                    "need_undef": False,
-                    "condition": None,
+                {
+                    "__cpp_lib_variant": {
+                        "value": "202102L",
+                        "implemented": True,
+                        "need_undef": False,
+                        "condition": None,
+                    },
                 },
-            },
-            {
-                "__cpp_lib_variant": {
-                    "value": "202102L",
-                    "implemented": True,
-                    "need_undef": False,
-                    "condition": None,
+                {
+                    "__cpp_lib_missing_FTM_in_older_standard": {
+                        "value": "2017L",
+                        "implemented": False,
+                        "need_undef": False,
+                        "condition": None,
+                    },
                 },
-            },
-        ],
-        "20": [
-            {
-                "__cpp_lib_barrier": {
-                    "value": "201907L",
-                    "implemented": True,
-                    "need_undef": False,
-                    "condition": "!defined(_LIBCPP_HAS_NO_THREADS) && _LIBCPP_AVAILABILITY_HAS_SYNC",
+            ],
+            "20": [
+                {
+                    "__cpp_lib_barrier": {
+                        "value": "201907L",
+                        "implemented": True,
+                        "need_undef": False,
+                        "condition": "!defined(_LIBCPP_HAS_NO_THREADS) && _LIBCPP_AVAILABILITY_HAS_SYNC",
+                    },
                 },
-            },
-            {
-                "__cpp_lib_format": {
-                    "value": "202110L",
-                    "implemented": False,
-                    "need_undef": False,
-                    "condition": None,
+                {
+                    "__cpp_lib_format": {
+                        "value": "202110L",
+                        "implemented": False,
+                        "need_undef": False,
+                        "condition": None,
+                    },
                 },
-            },
-            {
-                "__cpp_lib_variant": {
-                    "value": "202106L",
-                    "implemented": True,
-                    "need_undef": True,
-                    "condition": None,
+                {
+                    "__cpp_lib_variant": {
+                        "value": "202106L",
+                        "implemented": False,
+                        "need_undef": False,
+                        "condition": None,
+                    },
                 },
-            },
-        ],
-        "23": [
-            {
-                "__cpp_lib_format": {
-                    "value": "202207L",
-                    "implemented": False,
-                    "need_undef": False,
-                    "condition": None,
+                {
+                    "__cpp_lib_missing_FTM_in_older_standard": {
+                        "value": "2020L",
+                        "implemented": False,
+                        "need_undef": False,
+                        "condition": None,
+                    },
                 },
-            },
-        ],
-        "26": [
-            {
-                "__cpp_lib_barrier": {
-                    "value": "299900L",
-                    "implemented": True,
-                    "need_undef": True,
-                    "condition": "!defined(_LIBCPP_HAS_NO_THREADS) && _LIBCPP_AVAILABILITY_HAS_SYNC",
+            ],
+            "23": [
+                {
+                    "__cpp_lib_format": {
+                        "value": "202207L",
+                        "implemented": False,
+                        "need_undef": False,
+                        "condition": None,
+                    },
                 },
-            },
-            {
-                "__cpp_lib_format": {
-                    "value": "202311L",
-                    "implemented": False,
-                    "need_undef": False,
-                    "condition": None,
+            ],
+            "26": [
+                {
+                    "__cpp_lib_barrier": {
+                        "value": "299900L",
+                        "implemented": True,
+                        "need_undef": True,
+                        "condition": "!defined(_LIBCPP_HAS_NO_THREADS) && _LIBCPP_AVAILABILITY_HAS_SYNC",
+                    },
                 },
-            },
-            {
-                "__cpp_lib_variant": {
-                    "value": "202306L",
-                    "implemented": True,
-                    "need_undef": True,
-                    "condition": None,
+                {
+                    "__cpp_lib_format": {
+                        "value": "202311L",
+                        "implemented": False,
+                        "need_undef": False,
+                        "condition": None,
+                    },
                 },
-            },
-        ],
-    },
-)
+                {
+                    "__cpp_lib_variant": {
+                        "value": "202306L",
+                        "implemented": False,
+                        "need_undef": False,
+                        "condition": None,
+                    },
+                },
+                {
+                    "__cpp_lib_missing_FTM_in_older_standard": {
+                        "value": "2026L",
+                        "implemented": False,
+                        "need_undef": False,
+                        "condition": None,
+                    },
+                },
+            ],
+        }
+
+        self.assertEqual(self.ftm.version_header_implementation, expected)
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index cb5ff770b1196e..3b8a52362ede68 100755
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -2102,7 +2102,8 @@ class FeatureTestMacros:
 
     def __init__(self, filename: str):
         """Initializes the class with the JSON data in the file 'filename'."""
-        self.__data = json.load(open(filename))
+        with open(filename) as f:
+            self.__data = json.load(f)
 
     @functools.cached_property
     def std_dialects(self) -> List[str]:
@@ -2187,8 +2188,8 @@ def version_header_implementation(self) -> Dict[str, List[Dict[str, Any]]]:
             result[get_std_number(std)] = list()
 
         for ftm, values in self.standard_ftms.items():
-            need_undef = False
             last_value = None
+            last_entry = None
             for std, value in values.items():
                 # When a newer Standard does not change the value of the macro
                 # there is no need to redefine it with the same value.
@@ -2198,12 +2199,11 @@ def version_header_implementation(self) -> Dict[str, List[Dict[str, Any]]]:
 
                 entry = dict()
                 entry["value"] = value
-                entry["implemented"] = self.implemented_ftms[ftm][std] != None
-                entry["need_undef"] = need_undef
+                entry["implemented"] = self.implemented_ftms[ftm][std] == self.standard_ftms[ftm][std]
+                entry["need_undef"] = last_entry is not None and last_entry["implemented"] and entry["implemented"]
                 entry["condition"] = self.ftm_metadata[ftm]["libcxx_guard"]
 
-                need_undef = entry["implemented"]
-
+                last_entry = entry
                 result[get_std_number(std)].append(dict({ftm: entry}))
 
         return result
diff --git a/libcxx/utils/synchronize_csv_status_files.py b/libcxx/utils/synchronize_csv_status_files.py
index f87e938484c898..8317dde770baf6 100755
--- a/libcxx/utils/synchronize_csv_status_files.py
+++ b/libcxx/utils/synchronize_csv_status_files.py
@@ -128,9 +128,6 @@ def to_csv_entry(self) -> str:
         }
         return self._original if self._original is not None else mapping[self._status]
 
-    def is_done(self) -> bool:
-        return self._status == PaperStatus.DONE or self._status == PaperStatus.NOTHING_TO_DO
-
 class PaperInfo:
     paper_number: str
     """
@@ -250,21 +247,25 @@ def merge(paper: PaperInfo, gh: PaperInfo) -> PaperInfo:
     row with the newer status. Otherwise, report an error if they have a different status because
     something must be wrong.
 
+    We don't update issues from 'To Do' to 'In Progress', since that only creates churn and the
+    status files aim to document user-facing functionality in releases, for which 'In Progress'
+    is not useful.
+
     In case we don't update the CSV row's status, we still take any updated notes coming
     from the Github issue.
     """
-    if paper.status < gh.status:
-        return gh
-    elif paper.status != gh.status:
-        print(f"We found a CSV row and a Github issue with different statuses:\nrow: {paper}\nGithub issue: {gh}")
-        return paper
+    if paper.status == PaperStatus(PaperStatus.TODO) and gh.status == PaperStatus(PaperStatus.IN_PROGRESS):
+        result = copy.deepcopy(paper)
+        result.notes = gh.notes
+    elif paper.status < gh.status:
+        result = copy.deepcopy(gh)
+    elif paper.status == gh.status:
+        result = copy.deepcopy(paper)
+        result.notes = gh.notes
     else:
-        # Retain the notes from the Github issue, if any
-        if gh.notes is not None:
-            cp = copy.deepcopy(paper)
-            cp.notes = gh.notes
-            return cp
-        return paper
+        print(f"We found a CSV row and a Github issue with different statuses:\nrow: {paper}\nGithub issue: {gh}")
+        result = copy.deepcopy(paper)
+    return result
 
 def load_csv(file: pathlib.Path) -> List[Tuple]:
     rows = []
diff --git a/lld/COFF/DLL.cpp b/lld/COFF/DLL.cpp
index b00a59433319a7..39dcce9fe84837 100644
--- a/lld/COFF/DLL.cpp
+++ b/lld/COFF/DLL.cpp
@@ -730,13 +730,19 @@ void IdataContents::create(COFFLinkerContext &ctx) {
         auto chunk = make<AuxImportChunk>(s->file);
         auxIat.push_back(chunk);
         s->file->impECSym->setLocation(chunk);
+
+        chunk = make<AuxImportChunk>(s->file);
+        auxIatCopy.push_back(chunk);
+        s->file->auxImpCopySym->setLocation(chunk);
       }
     }
     // Terminate with null values.
     lookups.push_back(make<NullChunk>(ctx.config.wordsize));
     addresses.push_back(make<NullChunk>(ctx.config.wordsize));
-    if (ctx.config.machine == ARM64EC)
+    if (ctx.config.machine == ARM64EC) {
       auxIat.push_back(make<NullChunk>(ctx.config.wordsize));
+      auxIatCopy.push_back(make<NullChunk>(ctx.config.wordsize));
+    }
 
     for (int i = 0, e = syms.size(); i < e; ++i)
       syms[i]->setLocation(addresses[base + i]);
diff --git a/lld/COFF/DLL.h b/lld/COFF/DLL.h
index 48b0f17ca62aeb..afb46f22ec9e14 100644
--- a/lld/COFF/DLL.h
+++ b/lld/COFF/DLL.h
@@ -32,6 +32,7 @@ class IdataContents {
   std::vector<Chunk *> hints;
   std::vector<Chunk *> dllNames;
   std::vector<Chunk *> auxIat;
+  std::vector<Chunk *> auxIatCopy;
 };
 
 // Windows-specific.
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index 9994639d5d364a..1b94f10acf80e5 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -2448,6 +2448,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
     ctx.symtab.addAbsolute("__arm64x_redirection_metadata", 0);
     ctx.symtab.addAbsolute("__arm64x_redirection_metadata_count", 0);
     ctx.symtab.addAbsolute("__hybrid_auxiliary_iat", 0);
+    ctx.symtab.addAbsolute("__hybrid_auxiliary_iat_copy", 0);
     ctx.symtab.addAbsolute("__hybrid_code_map", 0);
     ctx.symtab.addAbsolute("__hybrid_code_map_count", 0);
     ctx.symtab.addAbsolute("__x64_code_ranges_to_entry_points", 0);
diff --git a/lld/COFF/InputFiles.cpp b/lld/COFF/InputFiles.cpp
index 94ad7f3ceb306a..d9184b04735e83 100644
--- a/lld/COFF/InputFiles.cpp
+++ b/lld/COFF/InputFiles.cpp
@@ -1091,6 +1091,12 @@ void ImportFile::parse() {
     }
     if (!impECSym)
       return;
+
+    StringRef auxImpCopyName = saver().save("__auximpcopy_" + name);
+    auxImpCopySym =
+        ctx.symtab.addImportData(auxImpCopyName, this, auxCopyLocation);
+    if (!auxImpCopySym)
+      return;
   }
   // If this was a duplicate, we logged an error but may continue;
   // in this case, impSym is nullptr.
diff --git a/lld/COFF/InputFiles.h b/lld/COFF/InputFiles.h
index acf221d85ae8f2..5fa93f57ef9e3a 100644
--- a/lld/COFF/InputFiles.h
+++ b/lld/COFF/InputFiles.h
@@ -362,10 +362,12 @@ class ImportFile : public InputFile {
   const coff_import_header *hdr;
   Chunk *location = nullptr;
 
-  // Auxiliary IAT symbol and chunk on ARM64EC.
+  // Auxiliary IAT symbols and chunks on ARM64EC.
   DefinedImportData *impECSym = nullptr;
   Chunk *auxLocation = nullptr;
   Symbol *auxThunkSym = nullptr;
+  DefinedImportData *auxImpCopySym = nullptr;
+  Chunk *auxCopyLocation = nullptr;
 
   // We want to eliminate dllimported symbols if no one actually refers to them.
   // These "Live" bits are used to keep track of which import library members
diff --git a/lld/COFF/SymbolTable.cpp b/lld/COFF/SymbolTable.cpp
index fa40335c26943e..efea16ccbbfea0 100644
--- a/lld/COFF/SymbolTable.cpp
+++ b/lld/COFF/SymbolTable.cpp
@@ -20,6 +20,7 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/Parallel.h"
 #include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include <utility>
@@ -562,6 +563,9 @@ void SymbolTable::addExitThunk(Symbol *from, Symbol *to) {
 }
 
 void SymbolTable::initializeECThunks() {
+  if (!isArm64EC(ctx.config.machine))
+    return;
+
   for (auto it : entryThunks) {
     auto *to = dyn_cast<Defined>(it.second);
     if (!to)
@@ -587,6 +591,19 @@ void SymbolTable::initializeECThunks() {
       sym = exitThunks.lookup(file->impECSym);
     file->impchkThunk->exitThunk = dyn_cast_or_null<Defined>(sym);
   }
+
+  // On ARM64EC, the __imp_ symbol references the auxiliary IAT, while the
+  // __imp_aux_ symbol references the regular IAT. However, x86_64 code expects
+  // both to reference the regular IAT, so adjust the symbol if necessary.
+  parallelForEach(ctx.objFileInstances, [&](ObjFile *file) {
+    if (file->getMachineType() != AMD64)
+      return;
+    for (auto &sym : file->getMutableSymbols()) {
+      auto impSym = dyn_cast_or_null<DefinedImportData>(sym);
+      if (impSym && impSym->file->impchkThunk && sym == impSym->file->impECSym)
+        sym = impSym->file->impSym;
+    }
+  });
 }
 
 Symbol *SymbolTable::addUndefined(StringRef name, InputFile *f,
diff --git a/lld/COFF/Writer.cpp b/lld/COFF/Writer.cpp
index 216db652c10aa8..c2765453aa964e 100644
--- a/lld/COFF/Writer.cpp
+++ b/lld/COFF/Writer.cpp
@@ -916,6 +916,8 @@ void Writer::addSyntheticIdata() {
   add(".idata$7", idata.dllNames);
   if (!idata.auxIat.empty())
     add(".idata$9", idata.auxIat);
+  if (!idata.auxIatCopy.empty())
+    add(".idata$a", idata.auxIatCopy);
 }
 
 void Writer::appendECImportTables() {
@@ -2279,6 +2281,11 @@ void Writer::setECSymbols() {
   replaceSymbol<DefinedSynthetic>(iatSym, "__hybrid_auxiliary_iat",
                                   idata.auxIat.empty() ? nullptr
                                                        : idata.auxIat.front());
+
+  Symbol *iatCopySym = ctx.symtab.findUnderscore("__hybrid_auxiliary_iat_copy");
+  replaceSymbol<DefinedSynthetic>(
+      iatCopySym, "__hybrid_auxiliary_iat_copy",
+      idata.auxIatCopy.empty() ? nullptr : idata.auxIatCopy.front());
 }
 
 // Write section contents to a mmap'ed file.
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index 11bf0ec4d173ba..7cae8677ef5ce1 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -48,6 +48,7 @@ class BitcodeCompiler;
 class OutputSection;
 class LinkerScript;
 class TargetInfo;
+struct Ctx;
 struct Partition;
 struct PhdrEntry;
 
@@ -149,11 +150,14 @@ struct VersionDefinition {
 
 class LinkerDriver {
 public:
+  LinkerDriver(Ctx &ctx);
+  LinkerDriver(LinkerDriver &) = delete;
   void linkerMain(ArrayRef<const char *> args);
   void addFile(StringRef path, bool withLOption);
   void addLibrary(StringRef name);
 
 private:
+  Ctx &ctx;
   void createFiles(llvm::opt::InputArgList &args);
   void inferMachineType();
   template <class ELFT> void link(llvm::opt::InputArgList &args);
@@ -540,6 +544,7 @@ struct InStruct {
 };
 
 struct Ctx {
+  Config &arg;
   LinkerDriver driver;
   LinkerScript *script;
   TargetInfo *target;
@@ -652,6 +657,7 @@ struct Ctx {
   // STT_SECTION symbol associated to the .toc input section.
   llvm::DenseSet<std::pair<const Symbol *, uint64_t>> ppc64noTocRelax;
 
+  Ctx();
   void reset();
 
   llvm::raw_fd_ostream openAuxiliaryFile(llvm::StringRef, std::error_code &);
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 3712d28f227f69..e25db0e4951275 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -81,8 +81,8 @@ using namespace lld::elf;
 ConfigWrapper elf::config;
 Ctx elf::ctx;
 
-static void setConfigs(opt::InputArgList &args);
-static void readConfigs(opt::InputArgList &args);
+static void setConfigs(Ctx &ctx, opt::InputArgList &args);
+static void readConfigs(Ctx &ctx, opt::InputArgList &args);
 
 void elf::errorOrWarn(const Twine &msg) {
   if (config->noinhibitExec)
@@ -91,8 +91,11 @@ void elf::errorOrWarn(const Twine &msg) {
     error(msg);
 }
 
+Ctx::Ctx() : arg(config.c), driver(*this) {}
+
 void Ctx::reset() {
-  driver = LinkerDriver();
+  driver.~LinkerDriver();
+  new (&driver) LinkerDriver(*this);
   script = nullptr;
   target = nullptr;
 
@@ -103,7 +106,7 @@ void Ctx::reset() {
   outputSections.clear();
   partitions.clear();
 
-  ctx.in.reset();
+  in.reset();
   sym = ElfSym{};
 
   memoryBuffers.clear();
@@ -262,7 +265,7 @@ static bool isBitcode(MemoryBufferRef mb) {
 
 bool LinkerDriver::tryAddFatLTOFile(MemoryBufferRef mb, StringRef archiveName,
                                     uint64_t offsetInArchive, bool lazy) {
-  if (!config->fatLTOObjects)
+  if (!ctx.arg.fatLTOObjects)
     return false;
   Expected<MemoryBufferRef> fatLTOData =
       IRObjectFile::findBitcodeInMemBuffer(mb);
@@ -282,7 +285,7 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) {
     return;
   MemoryBufferRef mbref = *buffer;
 
-  if (config->formatBinary) {
+  if (ctx.arg.formatBinary) {
     files.push_back(make<BinaryFile>(mbref));
     return;
   }
@@ -336,7 +339,7 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) {
     return;
   }
   case file_magic::elf_shared_object: {
-    if (config->isStatic) {
+    if (ctx.arg.isStatic) {
       error("attempted static link of dynamic object " + path);
       return;
     }
@@ -385,109 +388,109 @@ static void initLLVM() {
 
 // Some command line options or some combinations of them are not allowed.
 // This function checks for such errors.
-static void checkOptions() {
+static void checkOptions(Ctx &ctx) {
   // The MIPS ABI as of 2016 does not support the GNU-style symbol lookup
   // table which is a relatively new feature.
-  if (config->emachine == EM_MIPS && config->gnuHash)
+  if (ctx.arg.emachine == EM_MIPS && ctx.arg.gnuHash)
     error("the .gnu.hash section is not compatible with the MIPS target");
 
-  if (config->emachine == EM_ARM) {
-    if (!config->cmseImplib) {
-      if (!config->cmseInputLib.empty())
+  if (ctx.arg.emachine == EM_ARM) {
+    if (!ctx.arg.cmseImplib) {
+      if (!ctx.arg.cmseInputLib.empty())
         error("--in-implib may not be used without --cmse-implib");
-      if (!config->cmseOutputLib.empty())
+      if (!ctx.arg.cmseOutputLib.empty())
         error("--out-implib may not be used without --cmse-implib");
     }
   } else {
-    if (config->cmseImplib)
+    if (ctx.arg.cmseImplib)
       error("--cmse-implib is only supported on ARM targets");
-    if (!config->cmseInputLib.empty())
+    if (!ctx.arg.cmseInputLib.empty())
       error("--in-implib is only supported on ARM targets");
-    if (!config->cmseOutputLib.empty())
+    if (!ctx.arg.cmseOutputLib.empty())
       error("--out-implib is only supported on ARM targets");
   }
 
-  if (config->fixCortexA53Errata843419 && config->emachine != EM_AARCH64)
+  if (ctx.arg.fixCortexA53Errata843419 && ctx.arg.emachine != EM_AARCH64)
     error("--fix-cortex-a53-843419 is only supported on AArch64 targets");
 
-  if (config->fixCortexA8 && config->emachine != EM_ARM)
+  if (ctx.arg.fixCortexA8 && ctx.arg.emachine != EM_ARM)
     error("--fix-cortex-a8 is only supported on ARM targets");
 
-  if (config->armBe8 && config->emachine != EM_ARM)
+  if (ctx.arg.armBe8 && ctx.arg.emachine != EM_ARM)
     error("--be8 is only supported on ARM targets");
 
-  if (config->fixCortexA8 && !config->isLE)
+  if (ctx.arg.fixCortexA8 && !ctx.arg.isLE)
     error("--fix-cortex-a8 is not supported on big endian targets");
 
-  if (config->tocOptimize && config->emachine != EM_PPC64)
+  if (ctx.arg.tocOptimize && ctx.arg.emachine != EM_PPC64)
     error("--toc-optimize is only supported on PowerPC64 targets");
 
-  if (config->pcRelOptimize && config->emachine != EM_PPC64)
+  if (ctx.arg.pcRelOptimize && ctx.arg.emachine != EM_PPC64)
     error("--pcrel-optimize is only supported on PowerPC64 targets");
 
-  if (config->relaxGP && config->emachine != EM_RISCV)
+  if (ctx.arg.relaxGP && ctx.arg.emachine != EM_RISCV)
     error("--relax-gp is only supported on RISC-V targets");
 
-  if (config->pie && config->shared)
+  if (ctx.arg.pie && ctx.arg.shared)
     error("-shared and -pie may not be used together");
 
-  if (!config->shared && !config->filterList.empty())
+  if (!ctx.arg.shared && !ctx.arg.filterList.empty())
     error("-F may not be used without -shared");
 
-  if (!config->shared && !config->auxiliaryList.empty())
+  if (!ctx.arg.shared && !ctx.arg.auxiliaryList.empty())
     error("-f may not be used without -shared");
 
-  if (config->strip == StripPolicy::All && config->emitRelocs)
+  if (ctx.arg.strip == StripPolicy::All && ctx.arg.emitRelocs)
     error("--strip-all and --emit-relocs may not be used together");
 
-  if (config->zText && config->zIfuncNoplt)
+  if (ctx.arg.zText && ctx.arg.zIfuncNoplt)
     error("-z text and -z ifunc-noplt may not be used together");
 
-  if (config->relocatable) {
-    if (config->shared)
+  if (ctx.arg.relocatable) {
+    if (ctx.arg.shared)
       error("-r and -shared may not be used together");
-    if (config->gdbIndex)
+    if (ctx.arg.gdbIndex)
       error("-r and --gdb-index may not be used together");
-    if (config->icf != ICFLevel::None)
+    if (ctx.arg.icf != ICFLevel::None)
       error("-r and --icf may not be used together");
-    if (config->pie)
+    if (ctx.arg.pie)
       error("-r and -pie may not be used together");
-    if (config->exportDynamic)
+    if (ctx.arg.exportDynamic)
       error("-r and --export-dynamic may not be used together");
-    if (config->debugNames)
+    if (ctx.arg.debugNames)
       error("-r and --debug-names may not be used together");
-    if (!config->zSectionHeader)
+    if (!ctx.arg.zSectionHeader)
       error("-r and -z nosectionheader may not be used together");
   }
 
-  if (config->executeOnly) {
-    if (config->emachine != EM_AARCH64)
+  if (ctx.arg.executeOnly) {
+    if (ctx.arg.emachine != EM_AARCH64)
       error("--execute-only is only supported on AArch64 targets");
 
-    if (config->singleRoRx && !ctx.script->hasSectionsCommand)
+    if (ctx.arg.singleRoRx && !ctx.script->hasSectionsCommand)
       error("--execute-only and --no-rosegment cannot be used together");
   }
 
-  if (config->zRetpolineplt && config->zForceIbt)
+  if (ctx.arg.zRetpolineplt && ctx.arg.zForceIbt)
     error("-z force-ibt may not be used with -z retpolineplt");
 
-  if (config->emachine != EM_AARCH64) {
-    if (config->zPacPlt)
+  if (ctx.arg.emachine != EM_AARCH64) {
+    if (ctx.arg.zPacPlt)
       error("-z pac-plt only supported on AArch64");
-    if (config->zForceBti)
+    if (ctx.arg.zForceBti)
       error("-z force-bti only supported on AArch64");
-    if (config->zBtiReport != "none")
+    if (ctx.arg.zBtiReport != "none")
       error("-z bti-report only supported on AArch64");
-    if (config->zPauthReport != "none")
+    if (ctx.arg.zPauthReport != "none")
       error("-z pauth-report only supported on AArch64");
-    if (config->zGcsReport != "none")
+    if (ctx.arg.zGcsReport != "none")
       error("-z gcs-report only supported on AArch64");
-    if (config->zGcs != GcsPolicy::Implicit)
+    if (ctx.arg.zGcs != GcsPolicy::Implicit)
       error("-z gcs only supported on AArch64");
   }
 
-  if (config->emachine != EM_386 && config->emachine != EM_X86_64 &&
-      config->zCetReport != "none")
+  if (ctx.arg.emachine != EM_386 && ctx.arg.emachine != EM_X86_64 &&
+      ctx.arg.zCetReport != "none")
     error("-z cet-report only supported on X86 and X86_64");
 }
 
@@ -613,6 +616,8 @@ constexpr const char *saveTempsValues[] = {
     "resolution", "preopt",     "promote", "internalize",  "import",
     "opt",        "precodegen", "prelink", "combinedindex"};
 
+LinkerDriver::LinkerDriver(Ctx &ctx) : ctx(ctx) {}
+
 void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
   ELFOptTable parser;
   opt::InputArgList args = parser.parse(argsArr.slice(1));
@@ -664,7 +669,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
     }
   }
 
-  readConfigs(args);
+  readConfigs(ctx, args);
   checkZOptions(args);
 
   // The behavior of -v or --version is a bit strange, but this is
@@ -675,8 +680,8 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
     return;
 
   // Initialize time trace profiler.
-  if (config->timeTraceEnabled)
-    timeTraceProfilerInitialize(config->timeTraceGranularity, config->progName);
+  if (ctx.arg.timeTraceEnabled)
+    timeTraceProfilerInitialize(ctx.arg.timeTraceGranularity, ctx.arg.progName);
 
   {
     llvm::TimeTraceScope timeScope("ExecuteLinker");
@@ -687,17 +692,17 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
       return;
 
     inferMachineType();
-    setConfigs(args);
-    checkOptions();
+    setConfigs(ctx, args);
+    checkOptions(ctx);
     if (errorCount())
       return;
 
     invokeELFT(link, args);
   }
 
-  if (config->timeTraceEnabled) {
+  if (ctx.arg.timeTraceEnabled) {
     checkError(timeTraceProfilerWrite(
-        args.getLastArgValue(OPT_time_trace_eq).str(), config->outputFile));
+        args.getLastArgValue(OPT_time_trace_eq).str(), ctx.arg.outputFile));
     timeTraceProfilerCleanup();
   }
 }
@@ -709,14 +714,14 @@ static std::string getRpath(opt::InputArgList &args) {
 
 // Determines what we should do if there are remaining unresolved
 // symbols after the name resolution.
-static void setUnresolvedSymbolPolicy(opt::InputArgList &args) {
+static void setUnresolvedSymbolPolicy(Ctx &ctx, opt::InputArgList &args) {
   UnresolvedPolicy errorOrWarn = args.hasFlag(OPT_error_unresolved_symbols,
                                               OPT_warn_unresolved_symbols, true)
                                      ? UnresolvedPolicy::ReportError
                                      : UnresolvedPolicy::Warn;
   // -shared implies --unresolved-symbols=ignore-all because missing
   // symbols are likely to be resolved at runtime.
-  bool diagRegular = !config->shared, diagShlib = !config->shared;
+  bool diagRegular = !ctx.arg.shared, diagShlib = !ctx.arg.shared;
 
   for (const opt::Arg *arg : args) {
     switch (arg->getOption().getID()) {
@@ -760,9 +765,9 @@ static void setUnresolvedSymbolPolicy(opt::InputArgList &args) {
     }
   }
 
-  config->unresolvedSymbols =
+  ctx.arg.unresolvedSymbols =
       diagRegular ? errorOrWarn : UnresolvedPolicy::Ignore;
-  config->unresolvedSymbolsInShlib =
+  ctx.arg.unresolvedSymbolsInShlib =
       diagShlib ? errorOrWarn : UnresolvedPolicy::Ignore;
 }
 
@@ -799,25 +804,25 @@ static DiscardPolicy getDiscard(opt::InputArgList &args) {
   return DiscardPolicy::None;
 }
 
-static StringRef getDynamicLinker(opt::InputArgList &args) {
+static StringRef getDynamicLinker(Ctx &ctx, opt::InputArgList &args) {
   auto *arg = args.getLastArg(OPT_dynamic_linker, OPT_no_dynamic_linker);
   if (!arg)
     return "";
   if (arg->getOption().getID() == OPT_no_dynamic_linker) {
     // --no-dynamic-linker suppresses undefined weak symbols in .dynsym
-    config->noDynamicLinker = true;
+    ctx.arg.noDynamicLinker = true;
     return "";
   }
   return arg->getValue();
 }
 
-static int getMemtagMode(opt::InputArgList &args) {
+static int getMemtagMode(Ctx &ctx, opt::InputArgList &args) {
   StringRef memtagModeArg = args.getLastArgValue(OPT_android_memtag_mode);
   if (memtagModeArg.empty()) {
-    if (config->androidMemtagStack)
+    if (ctx.arg.androidMemtagStack)
       warn("--android-memtag-mode is unspecified, leaving "
            "--android-memtag-stack a no-op");
-    else if (config->androidMemtagHeap)
+    else if (ctx.arg.androidMemtagHeap)
       warn("--android-memtag-mode is unspecified, leaving "
            "--android-memtag-heap a no-op");
     return ELF::NT_MEMTAG_LEVEL_NONE;
@@ -1242,140 +1247,141 @@ static bool remapInputs(StringRef line, const Twine &location) {
 }
 
 // Initializes Config members by the command line options.
-static void readConfigs(opt::InputArgList &args) {
+static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
   errorHandler().verbose = args.hasArg(OPT_verbose);
   errorHandler().vsDiagnostics =
       args.hasArg(OPT_visual_studio_diagnostics_format, false);
 
-  config->allowMultipleDefinition =
+  ctx.arg.allowMultipleDefinition =
       hasZOption(args, "muldefs") ||
       args.hasFlag(OPT_allow_multiple_definition,
                    OPT_no_allow_multiple_definition, false);
-  config->androidMemtagHeap =
+  ctx.arg.androidMemtagHeap =
       args.hasFlag(OPT_android_memtag_heap, OPT_no_android_memtag_heap, false);
-  config->androidMemtagStack = args.hasFlag(OPT_android_memtag_stack,
+  ctx.arg.androidMemtagStack = args.hasFlag(OPT_android_memtag_stack,
                                             OPT_no_android_memtag_stack, false);
-  config->fatLTOObjects =
+  ctx.arg.fatLTOObjects =
       args.hasFlag(OPT_fat_lto_objects, OPT_no_fat_lto_objects, false);
-  config->androidMemtagMode = getMemtagMode(args);
-  config->auxiliaryList = args::getStrings(args, OPT_auxiliary);
-  config->armBe8 = args.hasArg(OPT_be8);
+  ctx.arg.androidMemtagMode = getMemtagMode(ctx, args);
+  ctx.arg.auxiliaryList = args::getStrings(args, OPT_auxiliary);
+  ctx.arg.armBe8 = args.hasArg(OPT_be8);
   if (opt::Arg *arg = args.getLastArg(
           OPT_Bno_symbolic, OPT_Bsymbolic_non_weak_functions,
           OPT_Bsymbolic_functions, OPT_Bsymbolic_non_weak, OPT_Bsymbolic)) {
     if (arg->getOption().matches(OPT_Bsymbolic_non_weak_functions))
-      config->bsymbolic = BsymbolicKind::NonWeakFunctions;
+      ctx.arg.bsymbolic = BsymbolicKind::NonWeakFunctions;
     else if (arg->getOption().matches(OPT_Bsymbolic_functions))
-      config->bsymbolic = BsymbolicKind::Functions;
+      ctx.arg.bsymbolic = BsymbolicKind::Functions;
     else if (arg->getOption().matches(OPT_Bsymbolic_non_weak))
-      config->bsymbolic = BsymbolicKind::NonWeak;
+      ctx.arg.bsymbolic = BsymbolicKind::NonWeak;
     else if (arg->getOption().matches(OPT_Bsymbolic))
-      config->bsymbolic = BsymbolicKind::All;
+      ctx.arg.bsymbolic = BsymbolicKind::All;
   }
-  config->callGraphProfileSort = getCGProfileSortKind(args);
-  config->checkSections =
+  ctx.arg.callGraphProfileSort = getCGProfileSortKind(args);
+  ctx.arg.checkSections =
       args.hasFlag(OPT_check_sections, OPT_no_check_sections, true);
-  config->chroot = args.getLastArgValue(OPT_chroot);
+  ctx.arg.chroot = args.getLastArgValue(OPT_chroot);
   if (auto *arg = args.getLastArg(OPT_compress_debug_sections)) {
-    config->compressDebugSections =
+    ctx.arg.compressDebugSections =
         getCompressionType(arg->getValue(), "--compress-debug-sections");
   }
-  config->cref = args.hasArg(OPT_cref);
-  config->optimizeBBJumps =
+  ctx.arg.cref = args.hasArg(OPT_cref);
+  ctx.arg.optimizeBBJumps =
       args.hasFlag(OPT_optimize_bb_jumps, OPT_no_optimize_bb_jumps, false);
-  config->debugNames = args.hasFlag(OPT_debug_names, OPT_no_debug_names, false);
-  config->demangle = args.hasFlag(OPT_demangle, OPT_no_demangle, true);
-  config->dependencyFile = args.getLastArgValue(OPT_dependency_file);
-  config->dependentLibraries = args.hasFlag(OPT_dependent_libraries, OPT_no_dependent_libraries, true);
-  config->disableVerify = args.hasArg(OPT_disable_verify);
-  config->discard = getDiscard(args);
-  config->dwoDir = args.getLastArgValue(OPT_plugin_opt_dwo_dir_eq);
-  config->dynamicLinker = getDynamicLinker(args);
-  config->ehFrameHdr =
+  ctx.arg.debugNames = args.hasFlag(OPT_debug_names, OPT_no_debug_names, false);
+  ctx.arg.demangle = args.hasFlag(OPT_demangle, OPT_no_demangle, true);
+  ctx.arg.dependencyFile = args.getLastArgValue(OPT_dependency_file);
+  ctx.arg.dependentLibraries =
+      args.hasFlag(OPT_dependent_libraries, OPT_no_dependent_libraries, true);
+  ctx.arg.disableVerify = args.hasArg(OPT_disable_verify);
+  ctx.arg.discard = getDiscard(args);
+  ctx.arg.dwoDir = args.getLastArgValue(OPT_plugin_opt_dwo_dir_eq);
+  ctx.arg.dynamicLinker = getDynamicLinker(ctx, args);
+  ctx.arg.ehFrameHdr =
       args.hasFlag(OPT_eh_frame_hdr, OPT_no_eh_frame_hdr, false);
-  config->emitLLVM = args.hasArg(OPT_lto_emit_llvm);
-  config->emitRelocs = args.hasArg(OPT_emit_relocs);
-  config->enableNewDtags =
+  ctx.arg.emitLLVM = args.hasArg(OPT_lto_emit_llvm);
+  ctx.arg.emitRelocs = args.hasArg(OPT_emit_relocs);
+  ctx.arg.enableNewDtags =
       args.hasFlag(OPT_enable_new_dtags, OPT_disable_new_dtags, true);
-  config->enableNonContiguousRegions =
+  ctx.arg.enableNonContiguousRegions =
       args.hasArg(OPT_enable_non_contiguous_regions);
-  config->entry = args.getLastArgValue(OPT_entry);
+  ctx.arg.entry = args.getLastArgValue(OPT_entry);
 
   errorHandler().errorHandlingScript =
       args.getLastArgValue(OPT_error_handling_script);
 
-  config->executeOnly =
+  ctx.arg.executeOnly =
       args.hasFlag(OPT_execute_only, OPT_no_execute_only, false);
-  config->exportDynamic =
+  ctx.arg.exportDynamic =
       args.hasFlag(OPT_export_dynamic, OPT_no_export_dynamic, false) ||
       args.hasArg(OPT_shared);
-  config->filterList = args::getStrings(args, OPT_filter);
-  config->fini = args.getLastArgValue(OPT_fini, "_fini");
-  config->fixCortexA53Errata843419 = args.hasArg(OPT_fix_cortex_a53_843419) &&
-                                     !args.hasArg(OPT_relocatable);
-  config->cmseImplib = args.hasArg(OPT_cmse_implib);
-  config->cmseInputLib = args.getLastArgValue(OPT_in_implib);
-  config->cmseOutputLib = args.getLastArgValue(OPT_out_implib);
-  config->fixCortexA8 =
+  ctx.arg.filterList = args::getStrings(args, OPT_filter);
+  ctx.arg.fini = args.getLastArgValue(OPT_fini, "_fini");
+  ctx.arg.fixCortexA53Errata843419 =
+      args.hasArg(OPT_fix_cortex_a53_843419) && !args.hasArg(OPT_relocatable);
+  ctx.arg.cmseImplib = args.hasArg(OPT_cmse_implib);
+  ctx.arg.cmseInputLib = args.getLastArgValue(OPT_in_implib);
+  ctx.arg.cmseOutputLib = args.getLastArgValue(OPT_out_implib);
+  ctx.arg.fixCortexA8 =
       args.hasArg(OPT_fix_cortex_a8) && !args.hasArg(OPT_relocatable);
-  config->fortranCommon =
+  ctx.arg.fortranCommon =
       args.hasFlag(OPT_fortran_common, OPT_no_fortran_common, false);
-  config->gcSections = args.hasFlag(OPT_gc_sections, OPT_no_gc_sections, false);
-  config->gnuUnique = args.hasFlag(OPT_gnu_unique, OPT_no_gnu_unique, true);
-  config->gdbIndex = args.hasFlag(OPT_gdb_index, OPT_no_gdb_index, false);
-  config->icf = getICF(args);
-  config->ignoreDataAddressEquality =
+  ctx.arg.gcSections = args.hasFlag(OPT_gc_sections, OPT_no_gc_sections, false);
+  ctx.arg.gnuUnique = args.hasFlag(OPT_gnu_unique, OPT_no_gnu_unique, true);
+  ctx.arg.gdbIndex = args.hasFlag(OPT_gdb_index, OPT_no_gdb_index, false);
+  ctx.arg.icf = getICF(args);
+  ctx.arg.ignoreDataAddressEquality =
       args.hasArg(OPT_ignore_data_address_equality);
-  config->ignoreFunctionAddressEquality =
+  ctx.arg.ignoreFunctionAddressEquality =
       args.hasArg(OPT_ignore_function_address_equality);
-  config->init = args.getLastArgValue(OPT_init, "_init");
-  config->ltoAAPipeline = args.getLastArgValue(OPT_lto_aa_pipeline);
-  config->ltoCSProfileGenerate = args.hasArg(OPT_lto_cs_profile_generate);
-  config->ltoCSProfileFile = args.getLastArgValue(OPT_lto_cs_profile_file);
-  config->ltoPGOWarnMismatch = args.hasFlag(OPT_lto_pgo_warn_mismatch,
+  ctx.arg.init = args.getLastArgValue(OPT_init, "_init");
+  ctx.arg.ltoAAPipeline = args.getLastArgValue(OPT_lto_aa_pipeline);
+  ctx.arg.ltoCSProfileGenerate = args.hasArg(OPT_lto_cs_profile_generate);
+  ctx.arg.ltoCSProfileFile = args.getLastArgValue(OPT_lto_cs_profile_file);
+  ctx.arg.ltoPGOWarnMismatch = args.hasFlag(OPT_lto_pgo_warn_mismatch,
                                             OPT_no_lto_pgo_warn_mismatch, true);
-  config->ltoDebugPassManager = args.hasArg(OPT_lto_debug_pass_manager);
-  config->ltoEmitAsm = args.hasArg(OPT_lto_emit_asm);
-  config->ltoNewPmPasses = args.getLastArgValue(OPT_lto_newpm_passes);
-  config->ltoWholeProgramVisibility =
+  ctx.arg.ltoDebugPassManager = args.hasArg(OPT_lto_debug_pass_manager);
+  ctx.arg.ltoEmitAsm = args.hasArg(OPT_lto_emit_asm);
+  ctx.arg.ltoNewPmPasses = args.getLastArgValue(OPT_lto_newpm_passes);
+  ctx.arg.ltoWholeProgramVisibility =
       args.hasFlag(OPT_lto_whole_program_visibility,
                    OPT_no_lto_whole_program_visibility, false);
-  config->ltoValidateAllVtablesHaveTypeInfos =
+  ctx.arg.ltoValidateAllVtablesHaveTypeInfos =
       args.hasFlag(OPT_lto_validate_all_vtables_have_type_infos,
                    OPT_no_lto_validate_all_vtables_have_type_infos, false);
-  config->ltoo = args::getInteger(args, OPT_lto_O, 2);
-  if (config->ltoo > 3)
-    error("invalid optimization level for LTO: " + Twine(config->ltoo));
+  ctx.arg.ltoo = args::getInteger(args, OPT_lto_O, 2);
+  if (ctx.arg.ltoo > 3)
+    error("invalid optimization level for LTO: " + Twine(ctx.arg.ltoo));
   unsigned ltoCgo =
-      args::getInteger(args, OPT_lto_CGO, args::getCGOptLevel(config->ltoo));
+      args::getInteger(args, OPT_lto_CGO, args::getCGOptLevel(ctx.arg.ltoo));
   if (auto level = CodeGenOpt::getLevel(ltoCgo))
-    config->ltoCgo = *level;
+    ctx.arg.ltoCgo = *level;
   else
     error("invalid codegen optimization level for LTO: " + Twine(ltoCgo));
-  config->ltoObjPath = args.getLastArgValue(OPT_lto_obj_path_eq);
-  config->ltoPartitions = args::getInteger(args, OPT_lto_partitions, 1);
-  config->ltoSampleProfile = args.getLastArgValue(OPT_lto_sample_profile);
-  config->ltoBBAddrMap =
+  ctx.arg.ltoObjPath = args.getLastArgValue(OPT_lto_obj_path_eq);
+  ctx.arg.ltoPartitions = args::getInteger(args, OPT_lto_partitions, 1);
+  ctx.arg.ltoSampleProfile = args.getLastArgValue(OPT_lto_sample_profile);
+  ctx.arg.ltoBBAddrMap =
       args.hasFlag(OPT_lto_basic_block_address_map,
                    OPT_no_lto_basic_block_address_map, false);
-  config->ltoBasicBlockSections =
+  ctx.arg.ltoBasicBlockSections =
       args.getLastArgValue(OPT_lto_basic_block_sections);
-  config->ltoUniqueBasicBlockSectionNames =
+  ctx.arg.ltoUniqueBasicBlockSectionNames =
       args.hasFlag(OPT_lto_unique_basic_block_section_names,
                    OPT_no_lto_unique_basic_block_section_names, false);
-  config->mapFile = args.getLastArgValue(OPT_Map);
-  config->mipsGotSize = args::getInteger(args, OPT_mips_got_size, 0xfff0);
-  config->mergeArmExidx =
+  ctx.arg.mapFile = args.getLastArgValue(OPT_Map);
+  ctx.arg.mipsGotSize = args::getInteger(args, OPT_mips_got_size, 0xfff0);
+  ctx.arg.mergeArmExidx =
       args.hasFlag(OPT_merge_exidx_entries, OPT_no_merge_exidx_entries, true);
-  config->mmapOutputFile =
+  ctx.arg.mmapOutputFile =
       args.hasFlag(OPT_mmap_output_file, OPT_no_mmap_output_file, true);
-  config->nmagic = args.hasFlag(OPT_nmagic, OPT_no_nmagic, false);
-  config->noinhibitExec = args.hasArg(OPT_noinhibit_exec);
-  config->nostdlib = args.hasArg(OPT_nostdlib);
-  config->oFormatBinary = isOutputFormatBinary(args);
-  config->omagic = args.hasFlag(OPT_omagic, OPT_no_omagic, false);
-  config->optRemarksFilename = args.getLastArgValue(OPT_opt_remarks_filename);
-  config->optStatsFilename = args.getLastArgValue(OPT_plugin_opt_stats_file);
+  ctx.arg.nmagic = args.hasFlag(OPT_nmagic, OPT_no_nmagic, false);
+  ctx.arg.noinhibitExec = args.hasArg(OPT_noinhibit_exec);
+  ctx.arg.nostdlib = args.hasArg(OPT_nostdlib);
+  ctx.arg.oFormatBinary = isOutputFormatBinary(args);
+  ctx.arg.omagic = args.hasFlag(OPT_omagic, OPT_no_omagic, false);
+  ctx.arg.optRemarksFilename = args.getLastArgValue(OPT_opt_remarks_filename);
+  ctx.arg.optStatsFilename = args.getLastArgValue(OPT_plugin_opt_stats_file);
 
   // Parse remarks hotness threshold. Valid value is either integer or 'auto'.
   if (auto *arg = args.getLastArg(OPT_opt_remarks_hotness_threshold)) {
@@ -1384,77 +1390,77 @@ static void readConfigs(opt::InputArgList &args) {
       error(arg->getSpelling() + ": invalid argument '" + arg->getValue() +
             "', only integer or 'auto' is supported");
     else
-      config->optRemarksHotnessThreshold = *resultOrErr;
-  }
-
-  config->optRemarksPasses = args.getLastArgValue(OPT_opt_remarks_passes);
-  config->optRemarksWithHotness = args.hasArg(OPT_opt_remarks_with_hotness);
-  config->optRemarksFormat = args.getLastArgValue(OPT_opt_remarks_format);
-  config->optimize = args::getInteger(args, OPT_O, 1);
-  config->orphanHandling = getOrphanHandling(args);
-  config->outputFile = args.getLastArgValue(OPT_o);
-  config->packageMetadata = args.getLastArgValue(OPT_package_metadata);
-  config->pie = args.hasFlag(OPT_pie, OPT_no_pie, false);
-  config->printIcfSections =
+      ctx.arg.optRemarksHotnessThreshold = *resultOrErr;
+  }
+
+  ctx.arg.optRemarksPasses = args.getLastArgValue(OPT_opt_remarks_passes);
+  ctx.arg.optRemarksWithHotness = args.hasArg(OPT_opt_remarks_with_hotness);
+  ctx.arg.optRemarksFormat = args.getLastArgValue(OPT_opt_remarks_format);
+  ctx.arg.optimize = args::getInteger(args, OPT_O, 1);
+  ctx.arg.orphanHandling = getOrphanHandling(args);
+  ctx.arg.outputFile = args.getLastArgValue(OPT_o);
+  ctx.arg.packageMetadata = args.getLastArgValue(OPT_package_metadata);
+  ctx.arg.pie = args.hasFlag(OPT_pie, OPT_no_pie, false);
+  ctx.arg.printIcfSections =
       args.hasFlag(OPT_print_icf_sections, OPT_no_print_icf_sections, false);
-  config->printGcSections =
+  ctx.arg.printGcSections =
       args.hasFlag(OPT_print_gc_sections, OPT_no_print_gc_sections, false);
-  config->printMemoryUsage = args.hasArg(OPT_print_memory_usage);
-  config->printArchiveStats = args.getLastArgValue(OPT_print_archive_stats);
-  config->printSymbolOrder =
-      args.getLastArgValue(OPT_print_symbol_order);
-  config->rejectMismatch = !args.hasArg(OPT_no_warn_mismatch);
-  config->relax = args.hasFlag(OPT_relax, OPT_no_relax, true);
-  config->relaxGP = args.hasFlag(OPT_relax_gp, OPT_no_relax_gp, false);
-  config->rpath = getRpath(args);
-  config->relocatable = args.hasArg(OPT_relocatable);
-  config->resolveGroups =
+  ctx.arg.printMemoryUsage = args.hasArg(OPT_print_memory_usage);
+  ctx.arg.printArchiveStats = args.getLastArgValue(OPT_print_archive_stats);
+  ctx.arg.printSymbolOrder = args.getLastArgValue(OPT_print_symbol_order);
+  ctx.arg.rejectMismatch = !args.hasArg(OPT_no_warn_mismatch);
+  ctx.arg.relax = args.hasFlag(OPT_relax, OPT_no_relax, true);
+  ctx.arg.relaxGP = args.hasFlag(OPT_relax_gp, OPT_no_relax_gp, false);
+  ctx.arg.rpath = getRpath(args);
+  ctx.arg.relocatable = args.hasArg(OPT_relocatable);
+  ctx.arg.resolveGroups =
       !args.hasArg(OPT_relocatable) || args.hasArg(OPT_force_group_allocation);
 
   if (args.hasArg(OPT_save_temps)) {
     // --save-temps implies saving all temps.
     for (const char *s : saveTempsValues)
-      config->saveTempsArgs.insert(s);
+      ctx.arg.saveTempsArgs.insert(s);
   } else {
     for (auto *arg : args.filtered(OPT_save_temps_eq)) {
       StringRef s = arg->getValue();
       if (llvm::is_contained(saveTempsValues, s))
-        config->saveTempsArgs.insert(s);
+        ctx.arg.saveTempsArgs.insert(s);
       else
         error("unknown --save-temps value: " + s);
     }
   }
 
-  config->searchPaths = args::getStrings(args, OPT_library_path);
-  config->sectionStartMap = getSectionStartMap(args);
-  config->shared = args.hasArg(OPT_shared);
-  config->singleRoRx = !args.hasFlag(OPT_rosegment, OPT_no_rosegment, true);
-  config->soName = args.getLastArgValue(OPT_soname);
-  config->sortSection = getSortSection(args);
-  config->splitStackAdjustSize = args::getInteger(args, OPT_split_stack_adjust_size, 16384);
-  config->zSectionHeader =
+  ctx.arg.searchPaths = args::getStrings(args, OPT_library_path);
+  ctx.arg.sectionStartMap = getSectionStartMap(args);
+  ctx.arg.shared = args.hasArg(OPT_shared);
+  ctx.arg.singleRoRx = !args.hasFlag(OPT_rosegment, OPT_no_rosegment, true);
+  ctx.arg.soName = args.getLastArgValue(OPT_soname);
+  ctx.arg.sortSection = getSortSection(args);
+  ctx.arg.splitStackAdjustSize =
+      args::getInteger(args, OPT_split_stack_adjust_size, 16384);
+  ctx.arg.zSectionHeader =
       getZFlag(args, "sectionheader", "nosectionheader", true);
-  config->strip = getStrip(args); // needs zSectionHeader
-  config->sysroot = args.getLastArgValue(OPT_sysroot);
-  config->target1Rel = args.hasFlag(OPT_target1_rel, OPT_target1_abs, false);
-  config->target2 = getTarget2(args);
-  config->thinLTOCacheDir = args.getLastArgValue(OPT_thinlto_cache_dir);
-  config->thinLTOCachePolicy = CHECK(
+  ctx.arg.strip = getStrip(args); // needs zSectionHeader
+  ctx.arg.sysroot = args.getLastArgValue(OPT_sysroot);
+  ctx.arg.target1Rel = args.hasFlag(OPT_target1_rel, OPT_target1_abs, false);
+  ctx.arg.target2 = getTarget2(args);
+  ctx.arg.thinLTOCacheDir = args.getLastArgValue(OPT_thinlto_cache_dir);
+  ctx.arg.thinLTOCachePolicy = CHECK(
       parseCachePruningPolicy(args.getLastArgValue(OPT_thinlto_cache_policy)),
       "--thinlto-cache-policy: invalid cache policy");
-  config->thinLTOEmitImportsFiles = args.hasArg(OPT_thinlto_emit_imports_files);
-  config->thinLTOEmitIndexFiles = args.hasArg(OPT_thinlto_emit_index_files) ||
+  ctx.arg.thinLTOEmitImportsFiles = args.hasArg(OPT_thinlto_emit_imports_files);
+  ctx.arg.thinLTOEmitIndexFiles = args.hasArg(OPT_thinlto_emit_index_files) ||
                                   args.hasArg(OPT_thinlto_index_only) ||
                                   args.hasArg(OPT_thinlto_index_only_eq);
-  config->thinLTOIndexOnly = args.hasArg(OPT_thinlto_index_only) ||
+  ctx.arg.thinLTOIndexOnly = args.hasArg(OPT_thinlto_index_only) ||
                              args.hasArg(OPT_thinlto_index_only_eq);
-  config->thinLTOIndexOnlyArg = args.getLastArgValue(OPT_thinlto_index_only_eq);
-  config->thinLTOObjectSuffixReplace =
+  ctx.arg.thinLTOIndexOnlyArg = args.getLastArgValue(OPT_thinlto_index_only_eq);
+  ctx.arg.thinLTOObjectSuffixReplace =
       getOldNewOptions(args, OPT_thinlto_object_suffix_replace_eq);
-  std::tie(config->thinLTOPrefixReplaceOld, config->thinLTOPrefixReplaceNew,
-           config->thinLTOPrefixReplaceNativeObject) =
+  std::tie(ctx.arg.thinLTOPrefixReplaceOld, ctx.arg.thinLTOPrefixReplaceNew,
+           ctx.arg.thinLTOPrefixReplaceNativeObject) =
       getOldNewOptionsExtra(args, OPT_thinlto_prefix_replace_eq);
-  if (config->thinLTOEmitIndexFiles && !config->thinLTOIndexOnly) {
+  if (ctx.arg.thinLTOEmitIndexFiles && !ctx.arg.thinLTOIndexOnly) {
     if (args.hasArg(OPT_thinlto_object_suffix_replace_eq))
       error("--thinlto-object-suffix-replace is not supported with "
             "--thinlto-emit-index-files");
@@ -1462,69 +1468,69 @@ static void readConfigs(opt::InputArgList &args) {
       error("--thinlto-prefix-replace is not supported with "
             "--thinlto-emit-index-files");
   }
-  if (!config->thinLTOPrefixReplaceNativeObject.empty() &&
-      config->thinLTOIndexOnlyArg.empty()) {
+  if (!ctx.arg.thinLTOPrefixReplaceNativeObject.empty() &&
+      ctx.arg.thinLTOIndexOnlyArg.empty()) {
     error("--thinlto-prefix-replace=old_dir;new_dir;obj_dir must be used with "
           "--thinlto-index-only=");
   }
-  config->thinLTOModulesToCompile =
+  ctx.arg.thinLTOModulesToCompile =
       args::getStrings(args, OPT_thinlto_single_module_eq);
-  config->timeTraceEnabled = args.hasArg(OPT_time_trace_eq);
-  config->timeTraceGranularity =
+  ctx.arg.timeTraceEnabled = args.hasArg(OPT_time_trace_eq);
+  ctx.arg.timeTraceGranularity =
       args::getInteger(args, OPT_time_trace_granularity, 500);
-  config->trace = args.hasArg(OPT_trace);
-  config->undefined = args::getStrings(args, OPT_undefined);
-  config->undefinedVersion =
+  ctx.arg.trace = args.hasArg(OPT_trace);
+  ctx.arg.undefined = args::getStrings(args, OPT_undefined);
+  ctx.arg.undefinedVersion =
       args.hasFlag(OPT_undefined_version, OPT_no_undefined_version, false);
-  config->unique = args.hasArg(OPT_unique);
-  config->useAndroidRelrTags = args.hasFlag(
+  ctx.arg.unique = args.hasArg(OPT_unique);
+  ctx.arg.useAndroidRelrTags = args.hasFlag(
       OPT_use_android_relr_tags, OPT_no_use_android_relr_tags, false);
-  config->warnBackrefs =
+  ctx.arg.warnBackrefs =
       args.hasFlag(OPT_warn_backrefs, OPT_no_warn_backrefs, false);
-  config->warnCommon = args.hasFlag(OPT_warn_common, OPT_no_warn_common, false);
-  config->warnSymbolOrdering =
+  ctx.arg.warnCommon = args.hasFlag(OPT_warn_common, OPT_no_warn_common, false);
+  ctx.arg.warnSymbolOrdering =
       args.hasFlag(OPT_warn_symbol_ordering, OPT_no_warn_symbol_ordering, true);
-  config->whyExtract = args.getLastArgValue(OPT_why_extract);
-  config->zCombreloc = getZFlag(args, "combreloc", "nocombreloc", true);
-  config->zCopyreloc = getZFlag(args, "copyreloc", "nocopyreloc", true);
-  config->zForceBti = hasZOption(args, "force-bti");
-  config->zForceIbt = hasZOption(args, "force-ibt");
-  config->zGcs = getZGcs(args);
-  config->zGlobal = hasZOption(args, "global");
-  config->zGnustack = getZGnuStack(args);
-  config->zHazardplt = hasZOption(args, "hazardplt");
-  config->zIfuncNoplt = hasZOption(args, "ifunc-noplt");
-  config->zInitfirst = hasZOption(args, "initfirst");
-  config->zInterpose = hasZOption(args, "interpose");
-  config->zKeepTextSectionPrefix = getZFlag(
+  ctx.arg.whyExtract = args.getLastArgValue(OPT_why_extract);
+  ctx.arg.zCombreloc = getZFlag(args, "combreloc", "nocombreloc", true);
+  ctx.arg.zCopyreloc = getZFlag(args, "copyreloc", "nocopyreloc", true);
+  ctx.arg.zForceBti = hasZOption(args, "force-bti");
+  ctx.arg.zForceIbt = hasZOption(args, "force-ibt");
+  ctx.arg.zGcs = getZGcs(args);
+  ctx.arg.zGlobal = hasZOption(args, "global");
+  ctx.arg.zGnustack = getZGnuStack(args);
+  ctx.arg.zHazardplt = hasZOption(args, "hazardplt");
+  ctx.arg.zIfuncNoplt = hasZOption(args, "ifunc-noplt");
+  ctx.arg.zInitfirst = hasZOption(args, "initfirst");
+  ctx.arg.zInterpose = hasZOption(args, "interpose");
+  ctx.arg.zKeepTextSectionPrefix = getZFlag(
       args, "keep-text-section-prefix", "nokeep-text-section-prefix", false);
-  config->zLrodataAfterBss =
+  ctx.arg.zLrodataAfterBss =
       getZFlag(args, "lrodata-after-bss", "nolrodata-after-bss", false);
-  config->zNodefaultlib = hasZOption(args, "nodefaultlib");
-  config->zNodelete = hasZOption(args, "nodelete");
-  config->zNodlopen = hasZOption(args, "nodlopen");
-  config->zNow = getZFlag(args, "now", "lazy", false);
-  config->zOrigin = hasZOption(args, "origin");
-  config->zPacPlt = hasZOption(args, "pac-plt");
-  config->zRelro = getZFlag(args, "relro", "norelro", true);
-  config->zRetpolineplt = hasZOption(args, "retpolineplt");
-  config->zRodynamic = hasZOption(args, "rodynamic");
-  config->zSeparate = getZSeparate(args);
-  config->zShstk = hasZOption(args, "shstk");
-  config->zStackSize = args::getZOptionValue(args, OPT_z, "stack-size", 0);
-  config->zStartStopGC =
+  ctx.arg.zNodefaultlib = hasZOption(args, "nodefaultlib");
+  ctx.arg.zNodelete = hasZOption(args, "nodelete");
+  ctx.arg.zNodlopen = hasZOption(args, "nodlopen");
+  ctx.arg.zNow = getZFlag(args, "now", "lazy", false);
+  ctx.arg.zOrigin = hasZOption(args, "origin");
+  ctx.arg.zPacPlt = hasZOption(args, "pac-plt");
+  ctx.arg.zRelro = getZFlag(args, "relro", "norelro", true);
+  ctx.arg.zRetpolineplt = hasZOption(args, "retpolineplt");
+  ctx.arg.zRodynamic = hasZOption(args, "rodynamic");
+  ctx.arg.zSeparate = getZSeparate(args);
+  ctx.arg.zShstk = hasZOption(args, "shstk");
+  ctx.arg.zStackSize = args::getZOptionValue(args, OPT_z, "stack-size", 0);
+  ctx.arg.zStartStopGC =
       getZFlag(args, "start-stop-gc", "nostart-stop-gc", true);
-  config->zStartStopVisibility = getZStartStopVisibility(args);
-  config->zText = getZFlag(args, "text", "notext", true);
-  config->zWxneeded = hasZOption(args, "wxneeded");
-  setUnresolvedSymbolPolicy(args);
-  config->power10Stubs = args.getLastArgValue(OPT_power10_stubs_eq) != "no";
+  ctx.arg.zStartStopVisibility = getZStartStopVisibility(args);
+  ctx.arg.zText = getZFlag(args, "text", "notext", true);
+  ctx.arg.zWxneeded = hasZOption(args, "wxneeded");
+  setUnresolvedSymbolPolicy(ctx, args);
+  ctx.arg.power10Stubs = args.getLastArgValue(OPT_power10_stubs_eq) != "no";
 
   if (opt::Arg *arg = args.getLastArg(OPT_eb, OPT_el)) {
     if (arg->getOption().matches(OPT_eb))
-      config->optEB = true;
+      ctx.arg.optEB = true;
     else
-      config->optEL = true;
+      ctx.arg.optEL = true;
   }
 
   for (opt::Arg *arg : args.filtered(OPT_remap_inputs)) {
@@ -1555,15 +1561,15 @@ static void readConfigs(opt::InputArgList &args) {
     if (!to_integer(kv.second, v))
       error(errPrefix + "expected an integer, but got '" + kv.second + "'");
     else if (Expected<GlobPattern> pat = GlobPattern::create(kv.first))
-      config->shuffleSections.emplace_back(std::move(*pat), uint32_t(v));
+      ctx.arg.shuffleSections.emplace_back(std::move(*pat), uint32_t(v));
     else
       error(errPrefix + toString(pat.takeError()) + ": " + kv.first);
   }
 
-  auto reports = {std::make_pair("bti-report", &config->zBtiReport),
-                  std::make_pair("cet-report", &config->zCetReport),
-                  std::make_pair("gcs-report", &config->zGcsReport),
-                  std::make_pair("pauth-report", &config->zPauthReport)};
+  auto reports = {std::make_pair("bti-report", &ctx.arg.zBtiReport),
+                  std::make_pair("cet-report", &ctx.arg.zCetReport),
+                  std::make_pair("gcs-report", &ctx.arg.zGcsReport),
+                  std::make_pair("pauth-report", &ctx.arg.zPauthReport)};
   for (opt::Arg *arg : args.filtered(OPT_z)) {
     std::pair<StringRef, StringRef> option =
         StringRef(arg->getValue()).split('=');
@@ -1598,7 +1604,7 @@ static void readConfigs(opt::InputArgList &args) {
             levelStr + "'");
     }
     if (Expected<GlobPattern> pat = GlobPattern::create(fields[0])) {
-      config->compressSections.emplace_back(std::move(*pat), type, level);
+      ctx.arg.compressSections.emplace_back(std::move(*pat), type, level);
     } else {
       error(arg->getSpelling() + ": " + toString(pat.takeError()));
       continue;
@@ -1622,7 +1628,7 @@ static void readConfigs(opt::InputArgList &args) {
       error(errPrefix + "expected a non-negative integer, but got '" +
             kv.second + "'");
     else if (Expected<GlobPattern> pat = GlobPattern::create(kv.first))
-      config->deadRelocInNonAlloc.emplace_back(std::move(*pat), v);
+      ctx.arg.deadRelocInNonAlloc.emplace_back(std::move(*pat), v);
     else
       error(errPrefix + toString(pat.takeError()) + ": " + kv.first);
   }
@@ -1648,23 +1654,23 @@ static void readConfigs(opt::InputArgList &args) {
             "'");
   }
 
-  config->passPlugins = args::getStrings(args, OPT_load_pass_plugins);
+  ctx.arg.passPlugins = args::getStrings(args, OPT_load_pass_plugins);
 
   // Parse -mllvm options.
   for (const auto *arg : args.filtered(OPT_mllvm)) {
     parseClangOption(arg->getValue(), arg->getSpelling());
-    config->mllvmOpts.emplace_back(arg->getValue());
+    ctx.arg.mllvmOpts.emplace_back(arg->getValue());
   }
 
-  config->ltoKind = LtoKind::Default;
+  ctx.arg.ltoKind = LtoKind::Default;
   if (auto *arg = args.getLastArg(OPT_lto)) {
     StringRef s = arg->getValue();
     if (s == "thin")
-      config->ltoKind = LtoKind::UnifiedThin;
+      ctx.arg.ltoKind = LtoKind::UnifiedThin;
     else if (s == "full")
-      config->ltoKind = LtoKind::UnifiedRegular;
+      ctx.arg.ltoKind = LtoKind::UnifiedRegular;
     else if (s == "default")
-      config->ltoKind = LtoKind::Default;
+      ctx.arg.ltoKind = LtoKind::Default;
     else
       error("unknown LTO mode: " + s);
   }
@@ -1680,21 +1686,21 @@ static void readConfigs(opt::InputArgList &args) {
       error(arg->getSpelling() + ": expected a positive integer, but got '" +
             arg->getValue() + "'");
     parallel::strategy = hardware_concurrency(threads);
-    config->thinLTOJobs = v;
+    ctx.arg.thinLTOJobs = v;
   } else if (parallel::strategy.compute_thread_count() > 16) {
     log("set maximum concurrency to 16, specify --threads= to change");
     parallel::strategy = hardware_concurrency(16);
   }
   if (auto *arg = args.getLastArg(OPT_thinlto_jobs_eq))
-    config->thinLTOJobs = arg->getValue();
-  config->threadCount = parallel::strategy.compute_thread_count();
+    ctx.arg.thinLTOJobs = arg->getValue();
+  ctx.arg.threadCount = parallel::strategy.compute_thread_count();
 
-  if (config->ltoPartitions == 0)
+  if (ctx.arg.ltoPartitions == 0)
     error("--lto-partitions: number of threads must be > 0");
-  if (!get_threadpool_strategy(config->thinLTOJobs))
-    error("--thinlto-jobs: invalid job count: " + config->thinLTOJobs);
+  if (!get_threadpool_strategy(ctx.arg.thinLTOJobs))
+    error("--thinlto-jobs: invalid job count: " + ctx.arg.thinLTOJobs);
 
-  if (config->splitStackAdjustSize < 0)
+  if (ctx.arg.splitStackAdjustSize < 0)
     error("--split-stack-adjust-size: size must be >= 0");
 
   // The text segment is traditionally the first segment, whose address equals
@@ -1708,42 +1714,42 @@ static void readConfigs(opt::InputArgList &args) {
   // Parse ELF{32,64}{LE,BE} and CPU type.
   if (auto *arg = args.getLastArg(OPT_m)) {
     StringRef s = arg->getValue();
-    std::tie(config->ekind, config->emachine, config->osabi) =
+    std::tie(ctx.arg.ekind, ctx.arg.emachine, ctx.arg.osabi) =
         parseEmulation(s);
-    config->mipsN32Abi =
+    ctx.arg.mipsN32Abi =
         (s.starts_with("elf32btsmipn32") || s.starts_with("elf32ltsmipn32"));
-    config->emulation = s;
+    ctx.arg.emulation = s;
   }
 
   // Parse --hash-style={sysv,gnu,both}.
   if (auto *arg = args.getLastArg(OPT_hash_style)) {
     StringRef s = arg->getValue();
     if (s == "sysv")
-      config->sysvHash = true;
+      ctx.arg.sysvHash = true;
     else if (s == "gnu")
-      config->gnuHash = true;
+      ctx.arg.gnuHash = true;
     else if (s == "both")
-      config->sysvHash = config->gnuHash = true;
+      ctx.arg.sysvHash = ctx.arg.gnuHash = true;
     else
       error("unknown --hash-style: " + s);
   }
 
   if (args.hasArg(OPT_print_map))
-    config->mapFile = "-";
+    ctx.arg.mapFile = "-";
 
   // Page alignment can be disabled by the -n (--nmagic) and -N (--omagic).
   // As PT_GNU_RELRO relies on Paging, do not create it when we have disabled
   // it. Also disable RELRO for -r.
-  if (config->nmagic || config->omagic || config->relocatable)
-    config->zRelro = false;
+  if (ctx.arg.nmagic || ctx.arg.omagic || ctx.arg.relocatable)
+    ctx.arg.zRelro = false;
 
-  std::tie(config->buildId, config->buildIdVector) = getBuildId(args);
+  std::tie(ctx.arg.buildId, ctx.arg.buildIdVector) = getBuildId(args);
 
   if (getZFlag(args, "pack-relative-relocs", "nopack-relative-relocs", false)) {
-    config->relrGlibc = true;
-    config->relrPackDynRelocs = true;
+    ctx.arg.relrGlibc = true;
+    ctx.arg.relrPackDynRelocs = true;
   } else {
-    std::tie(config->androidPackDynRelocs, config->relrPackDynRelocs) =
+    std::tie(ctx.arg.androidPackDynRelocs, ctx.arg.relrPackDynRelocs) =
         getPackDynRelocs(args);
   }
 
@@ -1752,34 +1758,34 @@ static void readConfigs(opt::InputArgList &args) {
       error("--symbol-ordering-file and --call-graph-order-file "
             "may not be used together");
     if (std::optional<MemoryBufferRef> buffer = readFile(arg->getValue())) {
-      config->symbolOrderingFile = getSymbolOrderingFile(*buffer);
+      ctx.arg.symbolOrderingFile = getSymbolOrderingFile(*buffer);
       // Also need to disable CallGraphProfileSort to prevent
       // LLD order symbols with CGProfile
-      config->callGraphProfileSort = CGProfileSortKind::None;
+      ctx.arg.callGraphProfileSort = CGProfileSortKind::None;
     }
   }
 
-  assert(config->versionDefinitions.empty());
-  config->versionDefinitions.push_back(
+  assert(ctx.arg.versionDefinitions.empty());
+  ctx.arg.versionDefinitions.push_back(
       {"local", (uint16_t)VER_NDX_LOCAL, {}, {}});
-  config->versionDefinitions.push_back(
+  ctx.arg.versionDefinitions.push_back(
       {"global", (uint16_t)VER_NDX_GLOBAL, {}, {}});
 
   // If --retain-symbol-file is used, we'll keep only the symbols listed in
   // the file and discard all others.
   if (auto *arg = args.getLastArg(OPT_retain_symbols_file)) {
-    config->versionDefinitions[VER_NDX_LOCAL].nonLocalPatterns.push_back(
+    ctx.arg.versionDefinitions[VER_NDX_LOCAL].nonLocalPatterns.push_back(
         {"*", /*isExternCpp=*/false, /*hasWildcard=*/true});
     if (std::optional<MemoryBufferRef> buffer = readFile(arg->getValue()))
       for (StringRef s : args::getLines(*buffer))
-        config->versionDefinitions[VER_NDX_GLOBAL].nonLocalPatterns.push_back(
+        ctx.arg.versionDefinitions[VER_NDX_GLOBAL].nonLocalPatterns.push_back(
             {s, /*isExternCpp=*/false, /*hasWildcard=*/false});
   }
 
   for (opt::Arg *arg : args.filtered(OPT_warn_backrefs_exclude)) {
     StringRef pattern(arg->getValue());
     if (Expected<GlobPattern> pat = GlobPattern::create(pattern))
-      config->warnBackrefsExclude.push_back(std::move(*pat));
+      ctx.arg.warnBackrefsExclude.push_back(std::move(*pat));
     else
       error(arg->getSpelling() + ": " + toString(pat.takeError()) + ": " +
             pattern);
@@ -1791,15 +1797,15 @@ static void readConfigs(opt::InputArgList &args) {
   // even if other options express a symbolic intention: -Bsymbolic,
   // -Bsymbolic-functions (if STT_FUNC), --dynamic-list.
   for (auto *arg : args.filtered(OPT_export_dynamic_symbol))
-    config->dynamicList.push_back(
+    ctx.arg.dynamicList.push_back(
         {arg->getValue(), /*isExternCpp=*/false,
          /*hasWildcard=*/hasWildcard(arg->getValue())});
 
   // --export-dynamic-symbol-list specifies a list of --export-dynamic-symbol
   // patterns. --dynamic-list is --export-dynamic-symbol-list plus -Bsymbolic
   // like semantics.
-  config->symbolic =
-      config->bsymbolic == BsymbolicKind::All || args.hasArg(OPT_dynamic_list);
+  ctx.arg.symbolic =
+      ctx.arg.bsymbolic == BsymbolicKind::All || args.hasArg(OPT_dynamic_list);
   for (auto *arg :
        args.filtered(OPT_dynamic_list, OPT_export_dynamic_symbol_list))
     if (std::optional<MemoryBufferRef> buffer = readFile(arg->getValue()))
@@ -1818,18 +1824,18 @@ static void readConfigs(opt::InputArgList &args) {
 // command line options, but computed based on other Config values.
 // This function initialize such members. See Config.h for the details
 // of these values.
-static void setConfigs(opt::InputArgList &args) {
-  ELFKind k = config->ekind;
-  uint16_t m = config->emachine;
-
-  config->copyRelocs = (config->relocatable || config->emitRelocs);
-  config->is64 = (k == ELF64LEKind || k == ELF64BEKind);
-  config->isLE = (k == ELF32LEKind || k == ELF64LEKind);
-  config->endianness = config->isLE ? endianness::little : endianness::big;
-  config->isMips64EL = (k == ELF64LEKind && m == EM_MIPS);
-  config->isPic = config->pie || config->shared;
-  config->picThunk = args.hasArg(OPT_pic_veneer, config->isPic);
-  config->wordsize = config->is64 ? 8 : 4;
+static void setConfigs(Ctx &ctx, opt::InputArgList &args) {
+  ELFKind k = ctx.arg.ekind;
+  uint16_t m = ctx.arg.emachine;
+
+  ctx.arg.copyRelocs = (ctx.arg.relocatable || ctx.arg.emitRelocs);
+  ctx.arg.is64 = (k == ELF64LEKind || k == ELF64BEKind);
+  ctx.arg.isLE = (k == ELF32LEKind || k == ELF64LEKind);
+  ctx.arg.endianness = ctx.arg.isLE ? endianness::little : endianness::big;
+  ctx.arg.isMips64EL = (k == ELF64LEKind && m == EM_MIPS);
+  ctx.arg.isPic = ctx.arg.pie || ctx.arg.shared;
+  ctx.arg.picThunk = args.hasArg(OPT_pic_veneer, ctx.arg.isPic);
+  ctx.arg.wordsize = ctx.arg.is64 ? 8 : 4;
 
   // ELF defines two different ways to store relocation addends as shown below:
   //
@@ -1845,16 +1851,16 @@ static void setConfigs(opt::InputArgList &args) {
   // We pick the format for dynamic relocations according to the psABI for each
   // processor, but a contrary choice can be made if the dynamic loader
   // supports.
-  config->isRela = getIsRela(args);
+  ctx.arg.isRela = getIsRela(args);
 
   // If the output uses REL relocations we must store the dynamic relocation
   // addends to the output sections. We also store addends for RELA relocations
   // if --apply-dynamic-relocs is used.
   // We default to not writing the addends when using RELA relocations since
   // any standard conforming tool can find it in r_addend.
-  config->writeAddends = args.hasFlag(OPT_apply_dynamic_relocs,
+  ctx.arg.writeAddends = args.hasFlag(OPT_apply_dynamic_relocs,
                                       OPT_no_apply_dynamic_relocs, false) ||
-                         !config->isRela;
+                         !ctx.arg.isRela;
   // Validation of dynamic relocation addends is on by default for assertions
   // builds and disabled otherwise. This check is enabled when writeAddends is
   // true.
@@ -1863,42 +1869,42 @@ static void setConfigs(opt::InputArgList &args) {
 #else
   bool checkDynamicRelocsDefault = false;
 #endif
-  config->checkDynamicRelocs =
+  ctx.arg.checkDynamicRelocs =
       args.hasFlag(OPT_check_dynamic_relocations,
                    OPT_no_check_dynamic_relocations, checkDynamicRelocsDefault);
-  config->tocOptimize =
+  ctx.arg.tocOptimize =
       args.hasFlag(OPT_toc_optimize, OPT_no_toc_optimize, m == EM_PPC64);
-  config->pcRelOptimize =
+  ctx.arg.pcRelOptimize =
       args.hasFlag(OPT_pcrel_optimize, OPT_no_pcrel_optimize, m == EM_PPC64);
 
   if (!args.hasArg(OPT_hash_style)) {
-    if (config->emachine == EM_MIPS)
-      config->sysvHash = true;
+    if (ctx.arg.emachine == EM_MIPS)
+      ctx.arg.sysvHash = true;
     else
-      config->sysvHash = config->gnuHash = true;
+      ctx.arg.sysvHash = ctx.arg.gnuHash = true;
   }
 
   // Set default entry point and output file if not specified by command line or
   // linker scripts.
-  config->warnMissingEntry =
-      (!config->entry.empty() || (!config->shared && !config->relocatable));
-  if (config->entry.empty() && !config->relocatable)
-    config->entry = config->emachine == EM_MIPS ? "__start" : "_start";
-  if (config->outputFile.empty())
-    config->outputFile = "a.out";
+  ctx.arg.warnMissingEntry =
+      (!ctx.arg.entry.empty() || (!ctx.arg.shared && !ctx.arg.relocatable));
+  if (ctx.arg.entry.empty() && !ctx.arg.relocatable)
+    ctx.arg.entry = ctx.arg.emachine == EM_MIPS ? "__start" : "_start";
+  if (ctx.arg.outputFile.empty())
+    ctx.arg.outputFile = "a.out";
 
   // Fail early if the output file or map file is not writable. If a user has a
   // long link, e.g. due to a large LTO link, they do not wish to run it and
   // find that it failed because there was a mistake in their command-line.
   {
     llvm::TimeTraceScope timeScope("Create output files");
-    if (auto e = tryCreateFile(config->outputFile))
-      error("cannot open output file " + config->outputFile + ": " +
+    if (auto e = tryCreateFile(ctx.arg.outputFile))
+      error("cannot open output file " + ctx.arg.outputFile + ": " +
             e.message());
-    if (auto e = tryCreateFile(config->mapFile))
-      error("cannot open map file " + config->mapFile + ": " + e.message());
-    if (auto e = tryCreateFile(config->whyExtract))
-      error("cannot open --why-extract= file " + config->whyExtract + ": " +
+    if (auto e = tryCreateFile(ctx.arg.mapFile))
+      error("cannot open map file " + ctx.arg.mapFile + ": " + e.message());
+    if (auto e = tryCreateFile(ctx.arg.whyExtract))
+      error("cannot open --why-extract= file " + ctx.arg.whyExtract + ": " +
             e.message());
   }
 }
@@ -1919,7 +1925,7 @@ void LinkerDriver::createFiles(opt::InputArgList &args) {
   std::vector<std::tuple<bool, bool, bool>> stack;
 
   // -r implies -Bstatic and has precedence over -Bdynamic.
-  config->isStatic = config->relocatable;
+  ctx.arg.isStatic = ctx.arg.relocatable;
 
   // Iterate over argv to process input files and positional arguments.
   std::optional<MemoryBufferRef> defaultScript;
@@ -1955,22 +1961,22 @@ void LinkerDriver::createFiles(opt::InputArgList &args) {
       error(Twine("cannot find linker script ") + arg->getValue());
       break;
     case OPT_as_needed:
-      config->asNeeded = true;
+      ctx.arg.asNeeded = true;
       break;
     case OPT_format:
-      config->formatBinary = isFormatBinary(arg->getValue());
+      ctx.arg.formatBinary = isFormatBinary(arg->getValue());
       break;
     case OPT_no_as_needed:
-      config->asNeeded = false;
+      ctx.arg.asNeeded = false;
       break;
     case OPT_Bstatic:
     case OPT_omagic:
     case OPT_nmagic:
-      config->isStatic = true;
+      ctx.arg.isStatic = true;
       break;
     case OPT_Bdynamic:
-      if (!config->relocatable)
-        config->isStatic = false;
+      if (!ctx.arg.relocatable)
+        ctx.arg.isStatic = false;
       break;
     case OPT_whole_archive:
       inWholeArchive = true;
@@ -2017,14 +2023,15 @@ void LinkerDriver::createFiles(opt::InputArgList &args) {
       ++InputFile::nextGroupId;
       break;
     case OPT_push_state:
-      stack.emplace_back(config->asNeeded, config->isStatic, inWholeArchive);
+      stack.emplace_back(ctx.arg.asNeeded, ctx.arg.isStatic, inWholeArchive);
       break;
     case OPT_pop_state:
       if (stack.empty()) {
         error("unbalanced --push-state/--pop-state");
         break;
       }
-      std::tie(config->asNeeded, config->isStatic, inWholeArchive) = stack.back();
+      std::tie(ctx.arg.asNeeded, ctx.arg.isStatic, inWholeArchive) =
+          stack.back();
       stack.pop_back();
       break;
     }
@@ -2038,7 +2045,7 @@ void LinkerDriver::createFiles(opt::InputArgList &args) {
 
 // If -m <machine_type> was not given, infer it from object files.
 void LinkerDriver::inferMachineType() {
-  if (config->ekind != ELFNoneKind)
+  if (ctx.arg.ekind != ELFNoneKind)
     return;
 
   bool inferred = false;
@@ -2047,11 +2054,11 @@ void LinkerDriver::inferMachineType() {
       continue;
     if (!inferred) {
       inferred = true;
-      config->ekind = f->ekind;
-      config->emachine = f->emachine;
-      config->mipsN32Abi = config->emachine == EM_MIPS && isMipsN32Abi(f);
+      ctx.arg.ekind = f->ekind;
+      ctx.arg.emachine = f->emachine;
+      ctx.arg.mipsN32Abi = ctx.arg.emachine == EM_MIPS && isMipsN32Abi(f);
     }
-    config->osabi = f->osabi;
+    ctx.arg.osabi = f->osabi;
     if (f->osabi != ELFOSABI_NONE)
       return;
   }
@@ -2061,14 +2068,14 @@ void LinkerDriver::inferMachineType() {
 
 // Parse -z max-page-size=<value>. The default value is defined by
 // each target.
-static uint64_t getMaxPageSize(opt::InputArgList &args) {
+static uint64_t getMaxPageSize(Ctx &ctx, opt::InputArgList &args) {
   uint64_t val = args::getZOptionValue(args, OPT_z, "max-page-size",
                                        ctx.target->defaultMaxPageSize);
   if (!isPowerOf2_64(val)) {
     error("max-page-size: value isn't a power of 2");
     return ctx.target->defaultMaxPageSize;
   }
-  if (config->nmagic || config->omagic) {
+  if (ctx.arg.nmagic || ctx.arg.omagic) {
     if (val != ctx.target->defaultMaxPageSize)
       warn("-z max-page-size set, but paging disabled by omagic or nmagic");
     return 1;
@@ -2078,7 +2085,7 @@ static uint64_t getMaxPageSize(opt::InputArgList &args) {
 
 // Parse -z common-page-size=<value>. The default value is defined by
 // each target.
-static uint64_t getCommonPageSize(opt::InputArgList &args) {
+static uint64_t getCommonPageSize(Ctx &ctx, opt::InputArgList &args) {
   uint64_t val = args::getZOptionValue(args, OPT_z, "common-page-size",
                                        ctx.target->defaultCommonPageSize);
   if (!isPowerOf2_64(val)) {
@@ -2097,7 +2104,7 @@ static uint64_t getCommonPageSize(opt::InputArgList &args) {
 }
 
 // Parses --image-base option.
-static std::optional<uint64_t> getImageBase(opt::InputArgList &args) {
+static std::optional<uint64_t> getImageBase(Ctx &ctx, opt::InputArgList &args) {
   // Because we are using "Config->maxPageSize" here, this function has to be
   // called after the variable is initialized.
   auto *arg = args.getLastArg(OPT_image_base);
@@ -2110,7 +2117,7 @@ static std::optional<uint64_t> getImageBase(opt::InputArgList &args) {
     error("--image-base: number expected, but got " + s);
     return 0;
   }
-  if ((v % config->maxPageSize) != 0)
+  if ((v % ctx.arg.maxPageSize) != 0)
     warn("--image-base: address isn't multiple of page size: " + s);
   return v;
 }
@@ -2163,7 +2170,7 @@ static void excludeLibs(opt::InputArgList &args) {
 }
 
 // Force Sym to be entered in the output.
-static void handleUndefined(Symbol *sym, const char *option) {
+static void handleUndefined(Ctx &ctx, Symbol *sym, const char *option) {
   // Since a symbol may not be used inside the program, LTO may
   // eliminate it. Mark the symbol as "used" to prevent it.
   sym->isUsedInRegularObj = true;
@@ -2171,14 +2178,14 @@ static void handleUndefined(Symbol *sym, const char *option) {
   if (!sym->isLazy())
     return;
   sym->extract();
-  if (!config->whyExtract.empty())
+  if (!ctx.arg.whyExtract.empty())
     ctx.whyExtractRecords.emplace_back(option, sym->file, *sym);
 }
 
 // As an extension to GNU linkers, lld supports a variant of `-u`
 // which accepts wildcard patterns. All symbols that match a given
 // pattern are handled as if they were given by `-u`.
-static void handleUndefinedGlob(StringRef arg) {
+static void handleUndefinedGlob(Ctx &ctx, StringRef arg) {
   Expected<GlobPattern> pat = GlobPattern::create(arg);
   if (!pat) {
     error("--undefined-glob: " + toString(pat.takeError()) + ": " + arg);
@@ -2193,26 +2200,26 @@ static void handleUndefinedGlob(StringRef arg) {
       syms.push_back(sym);
 
   for (Symbol *sym : syms)
-    handleUndefined(sym, "--undefined-glob");
+    handleUndefined(ctx, sym, "--undefined-glob");
 }
 
-static void handleLibcall(StringRef name) {
+static void handleLibcall(Ctx &ctx, StringRef name) {
   Symbol *sym = symtab.find(name);
   if (sym && sym->isLazy() && isa<BitcodeFile>(sym->file)) {
-    if (!config->whyExtract.empty())
+    if (!ctx.arg.whyExtract.empty())
       ctx.whyExtractRecords.emplace_back("<libcall>", sym->file, *sym);
     sym->extract();
   }
 }
 
-static void writeArchiveStats() {
-  if (config->printArchiveStats.empty())
+static void writeArchiveStats(Ctx &ctx) {
+  if (ctx.arg.printArchiveStats.empty())
     return;
 
   std::error_code ec;
-  raw_fd_ostream os = ctx.openAuxiliaryFile(config->printArchiveStats, ec);
+  raw_fd_ostream os = ctx.openAuxiliaryFile(ctx.arg.printArchiveStats, ec);
   if (ec) {
-    error("--print-archive-stats=: cannot open " + config->printArchiveStats +
+    error("--print-archive-stats=: cannot open " + ctx.arg.printArchiveStats +
           ": " + ec.message());
     return;
   }
@@ -2235,14 +2242,14 @@ static void writeArchiveStats() {
   }
 }
 
-static void writeWhyExtract() {
-  if (config->whyExtract.empty())
+static void writeWhyExtract(Ctx &ctx) {
+  if (ctx.arg.whyExtract.empty())
     return;
 
   std::error_code ec;
-  raw_fd_ostream os = ctx.openAuxiliaryFile(config->whyExtract, ec);
+  raw_fd_ostream os = ctx.openAuxiliaryFile(ctx.arg.whyExtract, ec);
   if (ec) {
-    error("cannot open --why-extract= file " + config->whyExtract + ": " +
+    error("cannot open --why-extract= file " + ctx.arg.whyExtract + ": " +
           ec.message());
     return;
   }
@@ -2254,7 +2261,7 @@ static void writeWhyExtract() {
   }
 }
 
-static void reportBackrefs() {
+static void reportBackrefs(Ctx &ctx) {
   for (auto &ref : ctx.backwardReferences) {
     const Symbol &sym = *ref.first;
     std::string to = toString(ref.second.second);
@@ -2262,7 +2269,7 @@ static void reportBackrefs() {
     // with --warn-backrefs-exclude=. The value may look like (for --start-lib)
     // *.o or (archive member) *.a(*.o).
     bool exclude = false;
-    for (const llvm::GlobPattern &pat : config->warnBackrefsExclude)
+    for (const llvm::GlobPattern &pat : ctx.arg.warnBackrefsExclude)
       if (pat.match(to)) {
         exclude = true;
         break;
@@ -2297,11 +2304,11 @@ static void reportBackrefs() {
 // part of your program. By using --dependency-file option, you can make
 // lld to dump dependency info so that you can maintain exact dependencies
 // easily.
-static void writeDependencyFile() {
+static void writeDependencyFile(Ctx &ctx) {
   std::error_code ec;
-  raw_fd_ostream os = ctx.openAuxiliaryFile(config->dependencyFile, ec);
+  raw_fd_ostream os = ctx.openAuxiliaryFile(ctx.arg.dependencyFile, ec);
   if (ec) {
-    error("cannot open " + config->dependencyFile + ": " + ec.message());
+    error("cannot open " + ctx.arg.dependencyFile + ": " + ec.message());
     return;
   }
 
@@ -2328,14 +2335,14 @@ static void writeDependencyFile() {
     }
   };
 
-  os << config->outputFile << ":";
-  for (StringRef path : config->dependencyFiles) {
+  os << ctx.arg.outputFile << ":";
+  for (StringRef path : ctx.arg.dependencyFiles) {
     os << " \\\n ";
     printFilename(os, path);
   }
   os << "\n";
 
-  for (StringRef path : config->dependencyFiles) {
+  for (StringRef path : ctx.arg.dependencyFiles) {
     os << "\n";
     printFilename(os, path);
     os << ":\n";
@@ -2346,7 +2353,7 @@ static void writeDependencyFile() {
 // This function is called after all symbol names are resolved. As a
 // result, the passes after the symbol resolution won't see any
 // symbols of type CommonSymbol.
-static void replaceCommonSymbols() {
+static void replaceCommonSymbols(Ctx &ctx) {
   llvm::TimeTraceScope timeScope("Replace common symbols");
   for (ELFFileBase *file : ctx.objectFiles) {
     if (!file->hasCommonSyms)
@@ -2381,7 +2388,7 @@ static void markAddrsig(Symbol *s) {
 // and symbols referred to by address-significance tables. These sections are
 // ineligible for ICF.
 template <class ELFT>
-static void findKeepUniqueSections(opt::InputArgList &args) {
+static void findKeepUniqueSections(Ctx &ctx, opt::InputArgList &args) {
   for (auto *arg : args.filtered(OPT_keep_unique)) {
     StringRef name = arg->getValue();
     auto *d = dyn_cast_or_null<Defined>(symtab.find(name));
@@ -2394,7 +2401,7 @@ static void findKeepUniqueSections(opt::InputArgList &args) {
 
   // --icf=all --ignore-data-address-equality means that we can ignore
   // the dynsym and address-significance tables entirely.
-  if (config->icf == ICFLevel::All && config->ignoreDataAddressEquality)
+  if (ctx.arg.icf == ICFLevel::All && ctx.arg.ignoreDataAddressEquality)
     return;
 
   // Symbols in the dynsym could be address-significant in other executables
@@ -2532,7 +2539,7 @@ void LinkerDriver::compileBitcodeFiles(bool skipLinkedOutput) {
     obj->parse(/*ignoreComdats=*/true);
 
     // Parse '@' in symbol names for non-relocatable output.
-    if (!config->relocatable)
+    if (!ctx.arg.relocatable)
       for (Symbol *sym : obj->getGlobalSymbols())
         if (sym->hasVersionSuffix)
           sym->parseSymbolVersion();
@@ -2663,7 +2670,7 @@ static void combineVersionedSymbol(Symbol &sym,
 // When this function is executed, only InputFiles and symbol table
 // contain pointers to symbol objects. We visit them to replace pointers,
 // so that wrapped symbols are swapped as instructed by the command line.
-static void redirectSymbols(ArrayRef<WrappedSymbol> wrapped) {
+static void redirectSymbols(Ctx &ctx, ArrayRef<WrappedSymbol> wrapped) {
   llvm::TimeTraceScope timeScope("Redirect symbols");
   DenseMap<Symbol *, Symbol *> map;
   for (const WrappedSymbol &w : wrapped) {
@@ -2674,7 +2681,7 @@ static void redirectSymbols(ArrayRef<WrappedSymbol> wrapped) {
   // If there are version definitions (versionDefinitions.size() > 2), enumerate
   // symbols with a non-default version (foo@v1) and check whether it should be
   // combined with foo or foo@@v1.
-  if (config->versionDefinitions.size() > 2)
+  if (ctx.arg.versionDefinitions.size() > 2)
     for (Symbol *sym : symtab.getSymbols())
       if (sym->hasVersionSuffix)
         combineVersionedSymbol(*sym, map);
@@ -2867,15 +2874,15 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   parseFiles(files, armCmseImpLib);
 
   // Create dynamic sections for dynamic linking and static PIE.
-  config->hasDynSymTab = !ctx.sharedFiles.empty() || config->isPic;
+  ctx.arg.hasDynSymTab = !ctx.sharedFiles.empty() || ctx.arg.isPic;
 
   // If an entry symbol is in a static archive, pull out that file now.
-  if (Symbol *sym = symtab.find(config->entry))
-    handleUndefined(sym, "--entry");
+  if (Symbol *sym = symtab.find(ctx.arg.entry))
+    handleUndefined(ctx, sym, "--entry");
 
   // Handle the `--undefined-glob <pattern>` options.
   for (StringRef pat : args::getStrings(args, OPT_undefined_glob))
-    handleUndefinedGlob(pat);
+    handleUndefinedGlob(ctx, pat);
 
   // After potential archive member extraction involving ENTRY and
   // -u/--undefined-glob, check whether PROVIDE symbols should be defined (the
@@ -2883,14 +2890,14 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   ctx.script->addScriptReferencedSymbolsToSymTable();
 
   // Prevent LTO from removing any definition referenced by -u.
-  for (StringRef name : config->undefined)
+  for (StringRef name : ctx.arg.undefined)
     if (Defined *sym = dyn_cast_or_null<Defined>(symtab.find(name)))
       sym->isUsedInRegularObj = true;
 
   // Mark -init and -fini symbols so that the LTO doesn't eliminate them.
-  if (Symbol *sym = dyn_cast_or_null<Defined>(symtab.find(config->init)))
+  if (Symbol *sym = dyn_cast_or_null<Defined>(symtab.find(ctx.arg.init)))
     sym->isUsedInRegularObj = true;
-  if (Symbol *sym = dyn_cast_or_null<Defined>(symtab.find(config->fini)))
+  if (Symbol *sym = dyn_cast_or_null<Defined>(symtab.find(ctx.arg.fini)))
     sym->isUsedInRegularObj = true;
 
   // If any of our inputs are bitcode files, the LTO code generator may create
@@ -2913,7 +2920,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   if (!ctx.bitcodeFiles.empty()) {
     llvm::Triple TT(ctx.bitcodeFiles.front()->obj->getTargetTriple());
     for (auto *s : lto::LTO::getRuntimeLibcallSymbols(TT))
-      handleLibcall(s);
+      handleLibcall(ctx, s);
   }
 
   // Archive members defining __wrap symbols may be extracted.
@@ -2961,7 +2968,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   ctx.out.elfHeader = make<OutputSection>("", 0, SHF_ALLOC);
 
   // We need to create some reserved symbols such as _end. Create them.
-  if (!config->relocatable)
+  if (!ctx.arg.relocatable)
     addReservedSymbols();
 
   // Apply version scripts.
@@ -2969,7 +2976,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   // For a relocatable output, version scripts don't make sense, and
   // parsing a symbol version string (e.g. dropping "@ver1" from a symbol
   // name "foo@ver1") rather do harm, so we don't call this if -r is given.
-  if (!config->relocatable) {
+  if (!ctx.arg.relocatable) {
     llvm::TimeTraceScope timeScope("Process symbol versions");
     symtab.scanVersionScript();
   }
@@ -2981,12 +2988,12 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   // --plugin-opt=emit-asm create output files in bitcode or assembly code,
   // respectively. When only certain thinLTO modules are specified for
   // compilation, the intermediate object file are the expected output.
-  const bool skipLinkedOutput = config->thinLTOIndexOnly || config->emitLLVM ||
-                                config->ltoEmitAsm ||
-                                !config->thinLTOModulesToCompile.empty();
+  const bool skipLinkedOutput = ctx.arg.thinLTOIndexOnly || ctx.arg.emitLLVM ||
+                                ctx.arg.ltoEmitAsm ||
+                                !ctx.arg.thinLTOModulesToCompile.empty();
 
   // Handle --lto-validate-all-vtables-have-type-infos.
-  if (config->ltoValidateAllVtablesHaveTypeInfos)
+  if (ctx.arg.ltoValidateAllVtablesHaveTypeInfos)
     ltoValidateAllVtablesHaveTypeInfos<ELFT>(args);
 
   // Do link-time optimization if given files are LLVM bitcode files.
@@ -3000,9 +3007,9 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
 
   // Symbol resolution finished. Report backward reference problems,
   // --print-archive-stats=, and --why-extract=.
-  reportBackrefs();
-  writeArchiveStats();
-  writeWhyExtract();
+  reportBackrefs(ctx);
+  writeArchiveStats(ctx);
+  writeWhyExtract(ctx);
   if (errorCount())
     return;
 
@@ -3044,10 +3051,10 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   processArmCmseSymbols();
 
   // Apply symbol renames for --wrap and combine foo@v1 and foo@@v1.
-  redirectSymbols(wrapped);
+  redirectSymbols(ctx, wrapped);
 
   // Replace common symbols with regular symbols.
-  replaceCommonSymbols();
+  replaceCommonSymbols(ctx);
 
   {
     llvm::TimeTraceScope timeScope("Aggregate sections");
@@ -3081,7 +3088,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
     }
     // We do not want to emit debug sections if --strip-all
     // or --strip-debug are given.
-    if (config->strip != StripPolicy::None) {
+    if (ctx.arg.strip != StripPolicy::None) {
       llvm::erase_if(ctx.inputSections, [](InputSectionBase *s) {
         if (isDebugSection(*s))
           return true;
@@ -3097,8 +3104,8 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
 
   // Since we now have a complete set of input files, we can create
   // a .d file to record build dependencies.
-  if (!config->dependencyFile.empty())
-    writeDependencyFile();
+  if (!ctx.arg.dependencyFile.empty())
+    writeDependencyFile(ctx);
 
   // Now that the number of partitions is fixed, save a pointer to the main
   // partition.
@@ -3113,23 +3120,23 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   // values such as a default image base address.
   ctx.target = getTarget();
 
-  config->eflags = ctx.target->calcEFlags();
+  ctx.arg.eflags = ctx.target->calcEFlags();
   // maxPageSize (sometimes called abi page size) is the maximum page size that
   // the output can be run on. For example if the OS can use 4k or 64k page
   // sizes then maxPageSize must be 64k for the output to be useable on both.
   // All important alignment decisions must use this value.
-  config->maxPageSize = getMaxPageSize(args);
+  ctx.arg.maxPageSize = getMaxPageSize(ctx, args);
   // commonPageSize is the most common page size that the output will be run on.
   // For example if an OS can use 4k or 64k page sizes and 4k is more common
   // than 64k then commonPageSize is set to 4k. commonPageSize can be used for
   // optimizations such as DATA_SEGMENT_ALIGN in linker scripts. LLD's use of it
   // is limited to writing trap instructions on the last executable segment.
-  config->commonPageSize = getCommonPageSize(args);
+  ctx.arg.commonPageSize = getCommonPageSize(ctx, args);
 
-  config->imageBase = getImageBase(args);
+  ctx.arg.imageBase = getImageBase(ctx, args);
 
   // This adds a .comment section containing a version string.
-  if (!config->relocatable)
+  if (!ctx.arg.relocatable)
     ctx.inputSections.push_back(createCommentSection());
 
   // Split SHF_MERGE and .eh_frame sections into pieces in preparation for garbage collection.
@@ -3154,11 +3161,11 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   // Some input sections that are used for exception handling need to be moved
   // into synthetic sections. Do that now so that they aren't assigned to
   // output sections in the usual way.
-  if (!config->relocatable)
+  if (!ctx.arg.relocatable)
     combineEhSections();
 
   // Merge .riscv.attributes sections.
-  if (config->emachine == EM_RISCV)
+  if (ctx.arg.emachine == EM_RISCV)
     mergeRISCVAttributesSections();
 
   {
@@ -3188,13 +3195,13 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
 
   // Two input sections with different output sections should not be folded.
   // ICF runs after processSectionCommands() so that we know the output sections.
-  if (config->icf != ICFLevel::None) {
-    findKeepUniqueSections<ELFT>(args);
+  if (ctx.arg.icf != ICFLevel::None) {
+    findKeepUniqueSections<ELFT>(ctx, args);
     doIcf<ELFT>();
   }
 
   // Read the callgraph now that we know what was gced or icfed
-  if (config->callGraphProfileSort != CGProfileSortKind::None) {
+  if (ctx.arg.callGraphProfileSort != CGProfileSortKind::None) {
     if (auto *arg = args.getLastArg(OPT_call_graph_ordering_file))
       if (std::optional<MemoryBufferRef> buffer = readFile(arg->getValue()))
         readCallGraph(*buffer);
diff --git a/lld/test/COFF/Inputs/loadconfig-arm64ec.s b/lld/test/COFF/Inputs/loadconfig-arm64ec.s
index 8d59d29bbd0212..cb79b5c257e6ec 100644
--- a/lld/test/COFF/Inputs/loadconfig-arm64ec.s
+++ b/lld/test/COFF/Inputs/loadconfig-arm64ec.s
@@ -84,7 +84,7 @@ __chpe_metadata:
         .rva __arm64x_extra_rfe_table
         .word __arm64x_extra_rfe_table_size
         .rva __os_arm64x_dispatch_fptr
-        .word 0 // __hybrid_auxiliary_iat_copy
+        .rva __hybrid_auxiliary_iat_copy
         .rva __os_arm64x_helper0
         .rva __os_arm64x_helper1
         .rva __os_arm64x_helper2
diff --git a/lld/test/COFF/arm64ec-import.test b/lld/test/COFF/arm64ec-import.test
index e403daa41f368c..92d7f5517bd426 100644
--- a/lld/test/COFF/arm64ec-import.test
+++ b/lld/test/COFF/arm64ec-import.test
@@ -4,6 +4,7 @@ RUN: split-file %s %t.dir && cd %t.dir
 RUN: llvm-mc -filetype=obj -triple=arm64ec-windows test.s -o test.obj
 RUN: llvm-mc -filetype=obj -triple=arm64ec-windows icall.s -o icall.obj
 RUN: llvm-mc -filetype=obj -triple=arm64ec-windows hybmp.s -o hybmp.obj
+RUN: llvm-mc -filetype=obj -triple=x86_64-windows test.s -o test-x86_64.obj
 RUN: llvm-mc -filetype=obj -triple=arm64ec-windows %S/Inputs/loadconfig-arm64ec.s -o loadconfig-arm64ec.obj
 RUN: llvm-lib -machine:arm64ec -def:test.def -out:test-arm64ec.lib
 RUN: llvm-lib -machine:arm64ec -def:test2.def -out:test2-arm64ec.lib
@@ -17,11 +18,16 @@ Link using x86_64 import library:
 RUN: lld-link -machine:arm64ec -dll -noentry -out:out2.dll loadconfig-arm64ec.obj icall.obj hybmp.obj \
 RUN:          test.obj test-x86_64.lib test2-arm64ec.lib
 
+Link using x86_64 object file:
+RUN: lld-link -machine:arm64ec -dll -noentry -out:out3.dll loadconfig-arm64ec.obj icall.obj hybmp.obj \
+RUN:          test-x86_64.obj test-arm64ec.lib test2-arm64ec.lib
+
 RUN: llvm-readobj --coff-imports out.dll | FileCheck --check-prefix=IMPORTS %s
 RUN: llvm-readobj --coff-imports out2.dll | FileCheck --check-prefix=IMPORTS %s
+RUN: llvm-readobj --coff-imports out3.dll | FileCheck -check-prefix=IMPORTS %s
 IMPORTS:      Import {
 IMPORTS-NEXT:   Name: test.dll
-IMPORTS-NEXT:   ImportLookupTableRVA:
+IMPORTS-NEXT:   ImportLookupTableRVA: 0x4230
 IMPORTS-NEXT:   ImportAddressTableRVA: 0x3000
 IMPORTS-NEXT:   Symbol: data (0)
 IMPORTS-NEXT:   Symbol: func (0)
@@ -36,6 +42,7 @@ IMPORTS-NEXT: }
 
 RUN: llvm-objdump -d out.dll | FileCheck --check-prefix=DISASM %s
 RUN: llvm-objdump -d out2.dll | FileCheck --check-prefix=DISASM %s
+RUN: llvm-objdump -d out3.dll | FileCheck -check-prefix=DISASM %s
 
 DISASM:      180001000: 52800000     mov     w0, #0x0                // =0
 DISASM-NEXT: 180001004: d65f03c0     ret
@@ -72,23 +79,54 @@ RUN: llvm-readobj --hex-dump=.test out2.dll | FileCheck --check-prefix=TESTSEC %
 TESTSEC:      0x180007000 08500000 00300000 10500000 20500000
 TESTSEC-NEXT: 0x180007010 08300000 00500000 10300000 20300000
 TESTSEC-NEXT: 0x180007020 14100000 28100000 00200000 08100000
-TESTSEC-NEXT: 0x180007030 3c100000
+TESTSEC-NEXT: 0x180007030 3c100000 a0420000
+
+RUN: llvm-readobj --hex-dump=.test out3.dll | FileCheck -check-prefix=TESTSEC-X64 %s
+TESTSEC-X64:      0x180007000 08300000 00300000 10300000 20300000
+TESTSEC-X64-NEXT: 0x180007010 08300000 00500000 10300000 20300000
+TESTSEC-X64-NEXT: 0x180007020 14100000 28100000 00200000 08100000
+TESTSEC-X64-NEXT: 0x180007030 3c100000 a0420000
 
 RUN: llvm-readobj --headers out.dll | FileCheck -check-prefix=HEADERS %s
+RUN: llvm-readobj --headers out2.dll | FileCheck -check-prefix=HEADERS %s
+RUN: llvm-readobj --headers out3.dll | FileCheck -check-prefix=HEADERS %s
 HEADERS:  LoadConfigTableRVA: 0x4010
 HEADERS:  IATRVA: 0x3000
 HEADERS:  IATSize: 0x1000
 
 RUN: llvm-readobj --coff-load-config out.dll | FileCheck -check-prefix=LOADCONFIG %s
+RUN: llvm-readobj --coff-load-config out2.dll | FileCheck -check-prefix=LOADCONFIG %s
+RUN: llvm-readobj --coff-load-config out3.dll | FileCheck -check-prefix=LOADCONFIG %s
 LOADCONFIG: AuxiliaryIAT: 0x5000
+LOADCONFIG: AuxiliaryIATCopy: 0x4298
 
 RUN: llvm-readobj --hex-dump=.rdata out.dll | FileCheck -check-prefix=RDATA %s
+RUN: llvm-readobj --hex-dump=.rdata out2.dll | FileCheck -check-prefix=RDATA %s
+RUN: llvm-readobj --hex-dump=.rdata out3.dll | FileCheck -check-prefix=RDATA %s
+RDATA:      0x180004290 2e646c6c 00000000 00000000 00000000
+RDATA-NEXT: 0x1800042a0 14100080 01000000 28100080 01000000
+RDATA-NEXT: 0x1800042b0 00000000 00000000 48100080 01000000
+RDATA-NEXT: 0x1800042c0 00000000 00000000 00000000 00000000
 RDATA:      0x180005000 00000000 00000000 14100080 01000000
 RDATA-NEXT: 0x180005010 28100080 01000000 00000000 00000000
 RDATA-NEXT: 0x180005020 48100080 01000000 00000000 00000000
 
 RUN: llvm-readobj --coff-basereloc out.dll | FileCheck -check-prefix=BASERELOC %s
+RUN: llvm-readobj --coff-basereloc out2.dll | FileCheck -check-prefix=BASERELOC %s
+RUN: llvm-readobj --coff-basereloc out3.dll | FileCheck -check-prefix=BASERELOC %s
 BASERELOC:      BaseReloc [
+Aux IAT copy:
+BASERELOC:          Address: 0x42A0
+BASERELOC-NEXT:   }
+BASERELOC-NEXT:   Entry {
+BASERELOC-NEXT:     Type: DIR64
+BASERELOC-NEXT:     Address: 0x42A8
+BASERELOC-NEXT:   }
+BASERELOC-NEXT:   Entry {
+BASERELOC-NEXT:     Type: DIR64
+BASERELOC-NEXT:     Address: 0x42B8
+BASERELOC-NEXT:   }
+Aux IAT:
 BASERELOC-NOT:      Address: 0x5000
 BASERELOC:          Address: 0x5008
 BASERELOC-NEXT:   }
@@ -119,6 +157,7 @@ arm64ec_data_sym:
     .rva func
     .rva "#func"
     .rva "#t2func"
+    .rva __auximpcopy_func
 
 #--- icall.s
     .text
diff --git a/lldb/include/lldb/Target/CoreFileMemoryRanges.h b/lldb/include/lldb/Target/CoreFileMemoryRanges.h
index 503ecd691e5948..0cc5433525ddc4 100644
--- a/lldb/include/lldb/Target/CoreFileMemoryRanges.h
+++ b/lldb/include/lldb/Target/CoreFileMemoryRanges.h
@@ -8,6 +8,7 @@
 
 #include "lldb/Utility/RangeMap.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/Utility/StreamString.h"
 
 #include "llvm/ADT/AddressRanges.h"
 
@@ -35,6 +36,16 @@ struct CoreFileMemoryRange {
       return lldb_permissions < rhs.lldb_permissions;
     return false;
   }
+
+  std::string Dump() const {
+    lldb_private::StreamString stream;
+    stream << "[";
+    stream.PutHex64(range.start());
+    stream << '-';
+    stream.PutHex64(range.end());
+    stream << ")";
+    return stream.GetString().str();
+  }
 };
 
 class CoreFileMemoryRanges
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
index c6417760f17a2b..bbd701efa3a666 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
@@ -691,6 +691,19 @@ def request_disassemble(
         for inst in instructions:
             self.disassembled_instructions[inst["address"]] = inst
 
+    def request_readMemory(self, memoryReference, offset, count):
+        args_dict = {
+            "memoryReference": memoryReference,
+            "offset": offset,
+            "count": count,
+        }
+        command_dict = {
+            "command": "readMemory",
+            "type": "request",
+            "arguments": args_dict,
+        }
+        return self.send_recv(command_dict)
+
     def request_evaluate(self, expression, frameIndex=0, threadId=None, context=None):
         stackFrame = self.get_stackFrame(frameIndex=frameIndex, threadId=threadId)
         if stackFrame is None:
@@ -1096,6 +1109,17 @@ def request_setVariable(self, containingVarRef, name, value, id=None):
         }
         return self.send_recv(command_dict)
 
+    def request_locations(self, locationReference):
+        args_dict = {
+            "locationReference": locationReference,
+        }
+        command_dict = {
+            "command": "locations",
+            "type": "request",
+            "arguments": args_dict,
+        }
+        return self.send_recv(command_dict)
+
     def request_testGetTargetBreakpoints(self):
         """A request packet used in the LLDB test suite to get all currently
         set breakpoint infos for all breakpoints currently set in the
diff --git a/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp b/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp
index ca22dacb2ba6c9..3f1e25f730a184 100644
--- a/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp
+++ b/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp
@@ -837,15 +837,17 @@ Status MinidumpFileBuilder::AddMemoryList() {
   error = m_process_sp->CalculateCoreFileSaveRanges(m_save_core_options,
                                                     all_core_memory_ranges);
 
+  if (error.Fail())
+    return error;
+
+  lldb_private::Progress progress("Saving Minidump File", "",
+                                  all_core_memory_ranges.GetSize());
   std::vector<CoreFileMemoryRange> all_core_memory_vec;
   // Extract all the data into just a vector of data. So we can mutate this in
   // place.
   for (const auto &core_range : all_core_memory_ranges)
     all_core_memory_vec.push_back(core_range.data);
 
-  if (error.Fail())
-    return error;
-
   // Start by saving all of the stacks and ensuring they fit under the 32b
   // limit.
   uint64_t total_size = GetCurrentDataEndOffset();
@@ -892,13 +894,13 @@ Status MinidumpFileBuilder::AddMemoryList() {
     }
   }
 
-  error = AddMemoryList_32(ranges_32);
+  error = AddMemoryList_32(ranges_32, progress);
   if (error.Fail())
     return error;
 
   // Add the remaining memory as a 64b range.
   if (!ranges_64.empty()) {
-    error = AddMemoryList_64(ranges_64);
+    error = AddMemoryList_64(ranges_64, progress);
     if (error.Fail())
       return error;
   }
@@ -972,8 +974,9 @@ GetLargestRangeSize(const std::vector<CoreFileMemoryRange> &ranges) {
   return max_size;
 }
 
-Status MinidumpFileBuilder::AddMemoryList_32(
-    std::vector<CoreFileMemoryRange> &ranges) {
+Status
+MinidumpFileBuilder::AddMemoryList_32(std::vector<CoreFileMemoryRange> &ranges,
+                                      Progress &progress) {
   std::vector<MemoryDescriptor> descriptors;
   Status error;
   if (ranges.size() == 0)
@@ -996,6 +999,7 @@ Status MinidumpFileBuilder::AddMemoryList_32(
               region_index, ranges.size(), size, addr, addr + size);
     ++region_index;
 
+    progress.Increment(1, "Adding Memory Range " + core_range.Dump());
     const size_t bytes_read =
         m_process_sp->ReadMemory(addr, data_up->GetBytes(), size, error);
     if (error.Fail() || bytes_read == 0) {
@@ -1049,8 +1053,9 @@ Status MinidumpFileBuilder::AddMemoryList_32(
   return error;
 }
 
-Status MinidumpFileBuilder::AddMemoryList_64(
-    std::vector<CoreFileMemoryRange> &ranges) {
+Status
+MinidumpFileBuilder::AddMemoryList_64(std::vector<CoreFileMemoryRange> &ranges,
+                                      Progress &progress) {
   Status error;
   if (ranges.empty())
     return error;
@@ -1111,6 +1116,7 @@ Status MinidumpFileBuilder::AddMemoryList_64(
               region_index, ranges.size(), size, addr, addr + size);
     ++region_index;
 
+    progress.Increment(1, "Adding Memory Range " + core_range.Dump());
     const size_t bytes_read =
         m_process_sp->ReadMemory(addr, data_up->GetBytes(), size, error);
     if (error.Fail()) {
diff --git a/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.h b/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.h
index 72e5658718b3c4..d5eac9015ac422 100644
--- a/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.h
+++ b/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.h
@@ -23,6 +23,7 @@
 #include <utility>
 #include <variant>
 
+#include "lldb/Core/Progress.h"
 #include "lldb/Symbol/SaveCoreOptions.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/Target.h"
@@ -79,7 +80,7 @@ class MinidumpFileBuilder {
                       const lldb::ProcessSP &process_sp,
                       lldb_private::SaveCoreOptions &save_core_options)
       : m_process_sp(process_sp), m_core_file(std::move(core_file)),
-        m_save_core_options(save_core_options){}
+        m_save_core_options(save_core_options) {}
 
   MinidumpFileBuilder(const MinidumpFileBuilder &) = delete;
   MinidumpFileBuilder &operator=(const MinidumpFileBuilder &) = delete;
@@ -124,9 +125,11 @@ class MinidumpFileBuilder {
   lldb_private::Status AddData(const void *data, uint64_t size);
   // Add MemoryList stream, containing dumps of important memory segments
   lldb_private::Status
-  AddMemoryList_64(std::vector<lldb_private::CoreFileMemoryRange> &ranges);
+  AddMemoryList_64(std::vector<lldb_private::CoreFileMemoryRange> &ranges,
+                   lldb_private::Progress &progress);
   lldb_private::Status
-  AddMemoryList_32(std::vector<lldb_private::CoreFileMemoryRange> &ranges);
+  AddMemoryList_32(std::vector<lldb_private::CoreFileMemoryRange> &ranges,
+                   lldb_private::Progress &progress);
   // Update the thread list on disk with the newly emitted stack RVAs.
   lldb_private::Status FixThreadStacks();
   lldb_private::Status FlushBufferToDisk();
diff --git a/lldb/source/Plugins/Trace/intel-pt/CommandObjectTraceStartIntelPT.cpp b/lldb/source/Plugins/Trace/intel-pt/CommandObjectTraceStartIntelPT.cpp
index 81f7c228561a6d..ea28c3251d7472 100644
--- a/lldb/source/Plugins/Trace/intel-pt/CommandObjectTraceStartIntelPT.cpp
+++ b/lldb/source/Plugins/Trace/intel-pt/CommandObjectTraceStartIntelPT.cpp
@@ -78,7 +78,7 @@ bool CommandObjectThreadTraceStartIntelPT::DoExecuteOnThreads(
     llvm::ArrayRef<lldb::tid_t> tids) {
   if (Error err = m_trace.Start(tids, m_options.m_ipt_trace_size,
                                 m_options.m_enable_tsc, m_options.m_psb_period))
-    result.SetError(Status(std::move(err)));
+    result.SetError(std::move(err));
   else
     result.SetStatus(eReturnStatusSuccessFinishResult);
 
@@ -164,7 +164,7 @@ void CommandObjectProcessTraceStartIntelPT::DoExecute(
           m_options.m_ipt_trace_size, m_options.m_process_buffer_size_limit,
           m_options.m_enable_tsc, m_options.m_psb_period,
           m_options.m_per_cpu_tracing, m_options.m_disable_cgroup_filtering))
-    result.SetError(Status(std::move(err)));
+    result.SetError(std::move(err));
   else
     result.SetStatus(eReturnStatusSuccessFinishResult);
 }
diff --git a/lldb/test/API/tools/lldb-dap/locations/Makefile b/lldb/test/API/tools/lldb-dap/locations/Makefile
new file mode 100644
index 00000000000000..10495940055b63
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/locations/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules
diff --git a/lldb/test/API/tools/lldb-dap/locations/TestDAP_locations.py b/lldb/test/API/tools/lldb-dap/locations/TestDAP_locations.py
new file mode 100644
index 00000000000000..76d938d3908492
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/locations/TestDAP_locations.py
@@ -0,0 +1,40 @@
+"""
+Test lldb-dap locations request
+"""
+
+
+import dap_server
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+import lldbdap_testcase
+import os
+
+
+class TestDAP_locations(lldbdap_testcase.DAPTestCaseBase):
+    @skipIfWindows
+    def test_locations(self):
+        """
+        Tests the 'locations' request.
+        """
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program)
+        source = "main.c"
+        self.source_path = os.path.join(os.getcwd(), source)
+        self.set_source_breakpoints(
+            source,
+            [line_number(source, "// BREAK HERE")],
+        )
+        self.continue_to_next_stop()
+
+        locals = {l["name"]: l for l in self.dap_server.get_local_variables()}
+
+        # var1 has a declarationLocation but no valueLocation
+        self.assertIn("declarationLocationReference", locals["var1"].keys())
+        self.assertNotIn("valueLocationReference", locals["var1"].keys())
+        loc_var1 = self.dap_server.request_locations(
+            locals["var1"]["declarationLocationReference"]
+        )
+        self.assertTrue(loc_var1["success"])
+        self.assertTrue(loc_var1["body"]["source"]["path"].endswith("main.c"))
+        self.assertEqual(loc_var1["body"]["line"], 2)
diff --git a/lldb/test/API/tools/lldb-dap/locations/main.c b/lldb/test/API/tools/lldb-dap/locations/main.c
new file mode 100644
index 00000000000000..6a8c86d00cb562
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/locations/main.c
@@ -0,0 +1,5 @@
+int main(void) {
+  int var1 = 1;
+  // BREAK HERE
+  return 0;
+}
diff --git a/lldb/test/API/tools/lldb-dap/memory/Makefile b/lldb/test/API/tools/lldb-dap/memory/Makefile
new file mode 100644
index 00000000000000..99998b20bcb050
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/memory/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/tools/lldb-dap/memory/TestDAP_memory.py b/lldb/test/API/tools/lldb-dap/memory/TestDAP_memory.py
new file mode 100644
index 00000000000000..3d8aaeda7f4b85
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/memory/TestDAP_memory.py
@@ -0,0 +1,120 @@
+"""
+Test lldb-dap memory support
+"""
+
+from base64 import b64decode
+import dap_server
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+import lldbdap_testcase
+import os
+
+
+class TestDAP_memory(lldbdap_testcase.DAPTestCaseBase):
+    def test_memory_refs_variables(self):
+        """
+        Tests memory references for evaluate
+        """
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program)
+        source = "main.cpp"
+        self.source_path = os.path.join(os.getcwd(), source)
+        self.set_source_breakpoints(
+            source,
+            [line_number(source, "// Breakpoint")],
+        )
+        self.continue_to_next_stop()
+
+        locals = {l["name"]: l for l in self.dap_server.get_local_variables()}
+
+        # Pointers should have memory-references
+        self.assertIn("memoryReference", locals["rawptr"].keys())
+        # Non-pointers should also have memory-references
+        self.assertIn("memoryReference", locals["not_a_ptr"].keys())
+
+    def test_memory_refs_evaluate(self):
+        """
+        Tests memory references for evaluate
+        """
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program)
+        source = "main.cpp"
+        self.source_path = os.path.join(os.getcwd(), source)
+        self.set_source_breakpoints(
+            source,
+            [line_number(source, "// Breakpoint")],
+        )
+        self.continue_to_next_stop()
+
+        self.assertIn(
+            "memoryReference",
+            self.dap_server.request_evaluate("rawptr")["body"].keys(),
+        )
+
+    def test_memory_refs_set_variable(self):
+        """
+        Tests memory references for `setVariable`
+        """
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program)
+        source = "main.cpp"
+        self.source_path = os.path.join(os.getcwd(), source)
+        self.set_source_breakpoints(
+            source,
+            [line_number(source, "// Breakpoint")],
+        )
+        self.continue_to_next_stop()
+
+        ptr_value = self.get_local_as_int("rawptr")
+        self.assertIn(
+            "memoryReference",
+            self.dap_server.request_setVariable(1, "rawptr", ptr_value + 2)[
+                "body"
+            ].keys(),
+        )
+
+    # lldb-dap assumes that all reads will be within the same region. On Windows
+    # the target string is at the very start of a region so the -1 offset causes
+    # the read to only read from the previous region and only return 1 byte.
+    @skipIfWindows
+    def test_readMemory(self):
+        """
+        Tests the 'readMemory' request
+        """
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program)
+        source = "main.cpp"
+        self.source_path = os.path.join(os.getcwd(), source)
+        self.set_source_breakpoints(
+            source,
+            [line_number(source, "// Breakpoint")],
+        )
+        self.continue_to_next_stop()
+
+        ptr_deref = self.dap_server.request_evaluate("*rawptr")["body"]
+        memref = ptr_deref["memoryReference"]
+
+        # We can read the complete string
+        mem = self.dap_server.request_readMemory(memref, 0, 5)["body"]
+        self.assertEqual(mem["unreadableBytes"], 0)
+        self.assertEqual(b64decode(mem["data"]), b"dead\0")
+
+        # Use an offset
+        mem = self.dap_server.request_readMemory(memref, 2, 3)["body"]
+        self.assertEqual(b64decode(mem["data"]), b"ad\0")
+
+        # Use a negative offset
+        mem = self.dap_server.request_readMemory(memref, -1, 6)["body"]
+        self.assertEqual(b64decode(mem["data"])[1:], b"dead\0")
+
+        # Reads of size 0 are successful
+        # VS-Code sends those in order to check if a `memoryReference` can actually be dereferenced.
+        mem = self.dap_server.request_readMemory(memref, 0, 0)
+        self.assertEqual(mem["success"], True)
+        self.assertEqual(mem["body"]["data"], "")
+
+        # Reads at offset 0x0 fail
+        mem = self.dap_server.request_readMemory("0x0", 0, 6)
+        self.assertEqual(mem["success"], False)
+        self.assertEqual(mem["message"], "Memory region is not readable")
diff --git a/lldb/test/API/tools/lldb-dap/memory/main.cpp b/lldb/test/API/tools/lldb-dap/memory/main.cpp
new file mode 100644
index 00000000000000..0db7b78e93383d
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/memory/main.cpp
@@ -0,0 +1,6 @@
+int main() {
+  int not_a_ptr = 666;
+  const char *rawptr = "dead";
+  // Breakpoint
+  return 0;
+}
diff --git a/lldb/tools/lldb-dap/DAP.cpp b/lldb/tools/lldb-dap/DAP.cpp
index fe1ca185778b8e..4be9bb055dd1f4 100644
--- a/lldb/tools/lldb-dap/DAP.cpp
+++ b/lldb/tools/lldb-dap/DAP.cpp
@@ -822,7 +822,7 @@ void Variables::Clear() {
   locals.Clear();
   globals.Clear();
   registers.Clear();
-  expandable_variables.clear();
+  referenced_variables.clear();
 }
 
 int64_t Variables::GetNewVariableReference(bool is_permanent) {
@@ -837,24 +837,23 @@ bool Variables::IsPermanentVariableReference(int64_t var_ref) {
 
 lldb::SBValue Variables::GetVariable(int64_t var_ref) const {
   if (IsPermanentVariableReference(var_ref)) {
-    auto pos = expandable_permanent_variables.find(var_ref);
-    if (pos != expandable_permanent_variables.end())
+    auto pos = referenced_permanent_variables.find(var_ref);
+    if (pos != referenced_permanent_variables.end())
       return pos->second;
   } else {
-    auto pos = expandable_variables.find(var_ref);
-    if (pos != expandable_variables.end())
+    auto pos = referenced_variables.find(var_ref);
+    if (pos != referenced_variables.end())
       return pos->second;
   }
   return lldb::SBValue();
 }
 
-int64_t Variables::InsertExpandableVariable(lldb::SBValue variable,
-                                            bool is_permanent) {
+int64_t Variables::InsertVariable(lldb::SBValue variable, bool is_permanent) {
   int64_t var_ref = GetNewVariableReference(is_permanent);
   if (is_permanent)
-    expandable_permanent_variables.insert(std::make_pair(var_ref, variable));
+    referenced_permanent_variables.insert(std::make_pair(var_ref, variable));
   else
-    expandable_variables.insert(std::make_pair(var_ref, variable));
+    referenced_variables.insert(std::make_pair(var_ref, variable));
   return var_ref;
 }
 
diff --git a/lldb/tools/lldb-dap/DAP.h b/lldb/tools/lldb-dap/DAP.h
index f4fdec6e895ad1..e28fd593ed74ce 100644
--- a/lldb/tools/lldb-dap/DAP.h
+++ b/lldb/tools/lldb-dap/DAP.h
@@ -111,12 +111,12 @@ struct Variables {
   int64_t next_temporary_var_ref{VARREF_FIRST_VAR_IDX};
   int64_t next_permanent_var_ref{PermanentVariableStartIndex};
 
-  /// Expandable variables that are alive in this stop state.
+  /// Variables that are alive in this stop state.
   /// Will be cleared when debuggee resumes.
-  llvm::DenseMap<int64_t, lldb::SBValue> expandable_variables;
-  /// Expandable variables that persist across entire debug session.
+  llvm::DenseMap<int64_t, lldb::SBValue> referenced_variables;
+  /// Variables that persist across entire debug session.
   /// These are the variables evaluated from debug console REPL.
-  llvm::DenseMap<int64_t, lldb::SBValue> expandable_permanent_variables;
+  llvm::DenseMap<int64_t, lldb::SBValue> referenced_permanent_variables;
 
   /// Check if \p var_ref points to a variable that should persist for the
   /// entire duration of the debug session, e.g. repl expandable variables
@@ -134,7 +134,7 @@ struct Variables {
 
   /// Insert a new \p variable.
   /// \return variableReference assigned to this expandable variable.
-  int64_t InsertExpandableVariable(lldb::SBValue variable, bool is_permanent);
+  int64_t InsertVariable(lldb::SBValue variable, bool is_permanent);
 
   /// Clear all scope variables and non-permanent expandable variables.
   void Clear();
diff --git a/lldb/tools/lldb-dap/JSONUtils.cpp b/lldb/tools/lldb-dap/JSONUtils.cpp
index f175079c6f1fb5..4f9c9c01cf4b6b 100644
--- a/lldb/tools/lldb-dap/JSONUtils.cpp
+++ b/lldb/tools/lldb-dap/JSONUtils.cpp
@@ -112,6 +112,22 @@ bool ObjectContainsKey(const llvm::json::Object &obj, llvm::StringRef key) {
   return obj.find(key) != obj.end();
 }
 
+std::string EncodeMemoryReference(lldb::addr_t addr) {
+  return "0x" + llvm::utohexstr(addr);
+}
+
+std::optional<lldb::addr_t>
+DecodeMemoryReference(llvm::StringRef memoryReference) {
+  if (!memoryReference.starts_with("0x"))
+    return std::nullopt;
+
+  lldb::addr_t addr;
+  if (memoryReference.consumeInteger(0, addr))
+    return std::nullopt;
+
+  return addr;
+}
+
 std::vector<std::string> GetStrings(const llvm::json::Object *obj,
                                     llvm::StringRef key) {
   std::vector<std::string> strs;
@@ -614,9 +630,8 @@ CreateExceptionBreakpointFilter(const ExceptionBreakpoint &bp) {
 //     }
 //   }
 // }
-llvm::json::Value CreateSource(lldb::SBLineEntry &line_entry) {
+llvm::json::Value CreateSource(const lldb::SBFileSpec &file) {
   llvm::json::Object object;
-  lldb::SBFileSpec file = line_entry.GetFileSpec();
   if (file.IsValid()) {
     const char *name = file.GetFilename();
     if (name)
@@ -630,6 +645,10 @@ llvm::json::Value CreateSource(lldb::SBLineEntry &line_entry) {
   return llvm::json::Value(std::move(object));
 }
 
+llvm::json::Value CreateSource(const lldb::SBLineEntry &line_entry) {
+  return CreateSource(line_entry.GetFileSpec());
+}
+
 llvm::json::Value CreateSource(llvm::StringRef source_path) {
   llvm::json::Object source;
   llvm::StringRef name = llvm::sys::path::filename(source_path);
@@ -690,8 +709,7 @@ std::optional<llvm::json::Value> CreateSource(lldb::SBFrame &frame) {
 //     "instructionPointerReference": {
 // 	     "type": "string",
 // 	     "description": "A memory reference for the current instruction
-// pointer
-//                       in this frame."
+//                         pointer in this frame."
 //     },
 //     "moduleId": {
 //       "type": ["integer", "string"],
@@ -1238,65 +1256,85 @@ std::string VariableDescription::GetResult(llvm::StringRef context) {
 //       "description": "The number of indexed child variables. The client
 //                       can use this optional information to present the
 //                       children in a paged UI and fetch them in chunks."
-//     }
-//
+//     },
+//     "memoryReference": {
+//        "type": "string",
+//        "description": "A memory reference associated with this variable.
+//                        For pointer type variables, this is generally a
+//                        reference to the memory address contained in the
+//                        pointer. For executable data, this reference may later
+//                        be used in a `disassemble` request. This attribute may
+//                        be returned by a debug adapter if corresponding
+//                        capability `supportsMemoryReferences` is true."
+//     },
+//     "declarationLocationReference": {
+//       "type": "integer",
+//       "description": "A reference that allows the client to request the
+//                       location where the variable is declared. This should be
+//                       present only if the adapter is likely to be able to
+//                       resolve the location.\n\nThis reference shares the same
+//                       lifetime as the `variablesReference`. See 'Lifetime of
+//                       Object References' in the Overview section for
+//                       details."
+//     },
 //
 //     "$__lldb_extensions": {
 //       "description": "Unofficial extensions to the protocol",
 //       "properties": {
 //         "declaration": {
-//         "type": "object",
-//         "description": "The source location where the variable was declared.
-//                         This value won't be present if no declaration is
-//                         available.",
-//         "properties": {
-//           "path": {
-//             "type": "string",
-//             "description": "The source file path where the variable was
-//                            declared."
-//           },
-//           "line": {
-//             "type": "number",
-//             "description": "The 1-indexed source line where the variable was
-//                             declared."
-//           },
-//           "column": {
-//             "type": "number",
-//             "description": "The 1-indexed source column where the variable
-//                             was declared."
+//           "type": "object",
+//           "description": "The source location where the variable was
+//                           declared. This value won't be present if no
+//                           declaration is available.
+//                           Superseded by `declarationLocationReference`",
+//           "properties": {
+//             "path": {
+//               "type": "string",
+//               "description": "The source file path where the variable was
+//                              declared."
+//             },
+//             "line": {
+//               "type": "number",
+//               "description": "The 1-indexed source line where the variable
+//                               was declared."
+//             },
+//             "column": {
+//               "type": "number",
+//               "description": "The 1-indexed source column where the variable
+//                               was declared."
+//             }
 //           }
+//         },
+//         "value": {
+//           "type": "string",
+//           "description": "The internal value of the variable as returned by
+//                            This is effectively SBValue.GetValue(). The other
+//                            `value` entry in the top-level variable response
+//                            is, on the other hand, just a display string for
+//                            the variable."
+//         },
+//         "summary": {
+//           "type": "string",
+//           "description": "The summary string of the variable. This is
+//                           effectively SBValue.GetSummary()."
+//         },
+//         "autoSummary": {
+//           "type": "string",
+//           "description": "The auto generated summary if using
+//                           `enableAutoVariableSummaries`."
+//         },
+//         "error": {
+//           "type": "string",
+//           "description": "An error message generated if LLDB couldn't inspect
+//                           the variable."
 //         }
-//       },
-//       "value":
-//         "type": "string",
-//         "description": "The internal value of the variable as returned by
-//                         This is effectively SBValue.GetValue(). The other
-//                         `value` entry in the top-level variable response is,
-//                          on the other hand, just a display string for the
-//                          variable."
-//       },
-//       "summary":
-//         "type": "string",
-//         "description": "The summary string of the variable. This is
-//                         effectively SBValue.GetSummary()."
-//       },
-//       "autoSummary":
-//         "type": "string",
-//         "description": "The auto generated summary if using
-//                         `enableAutoVariableSummaries`."
-//       },
-//       "error":
-//         "type": "string",
-//         "description": "An error message generated if LLDB couldn't inspect
-//                         the variable."
 //       }
 //     }
 //   },
 //   "required": [ "name", "value", "variablesReference" ]
 // }
-llvm::json::Value CreateVariable(lldb::SBValue v, int64_t variablesReference,
-                                 int64_t varID, bool format_hex,
-                                 bool is_name_duplicated,
+llvm::json::Value CreateVariable(lldb::SBValue v, int64_t var_ref,
+                                 bool format_hex, bool is_name_duplicated,
                                  std::optional<std::string> custom_name) {
   VariableDescription desc(v, format_hex, is_name_duplicated, custom_name);
   llvm::json::Object object;
@@ -1341,12 +1379,21 @@ llvm::json::Value CreateVariable(lldb::SBValue v, int64_t variablesReference,
     }
   }
   EmplaceSafeString(object, "type", desc.display_type_name);
-  if (varID != INT64_MAX)
-    object.try_emplace("id", varID);
+
+  // A unique variable identifier to help in properly identifying variables with
+  // the same name. This is an extension to the VS protocol.
+  object.try_emplace("id", var_ref);
+
   if (v.MightHaveChildren())
-    object.try_emplace("variablesReference", variablesReference);
+    object.try_emplace("variablesReference", var_ref);
   else
-    object.try_emplace("variablesReference", (int64_t)0);
+    object.try_emplace("variablesReference", 0);
+
+  if (v.GetDeclaration().IsValid())
+    object.try_emplace("declarationLocationReference", var_ref);
+
+  if (lldb::addr_t addr = v.GetLoadAddress(); addr != LLDB_INVALID_ADDRESS)
+    object.try_emplace("memoryReference", EncodeMemoryReference(addr));
 
   object.try_emplace("$__lldb_extensions", desc.GetVariableExtensionsJSON());
   return llvm::json::Value(std::move(object));
diff --git a/lldb/tools/lldb-dap/JSONUtils.h b/lldb/tools/lldb-dap/JSONUtils.h
index f8fec22d7aa0ea..13018458ffe0ad 100644
--- a/lldb/tools/lldb-dap/JSONUtils.h
+++ b/lldb/tools/lldb-dap/JSONUtils.h
@@ -131,6 +131,13 @@ int64_t GetSigned(const llvm::json::Object *obj, llvm::StringRef key,
 ///     \b True if the key exists in the \a obj, \b False otherwise.
 bool ObjectContainsKey(const llvm::json::Object &obj, llvm::StringRef key);
 
+/// Encodes a memory reference
+std::string EncodeMemoryReference(lldb::addr_t addr);
+
+/// Decodes a memory reference
+std::optional<lldb::addr_t>
+DecodeMemoryReference(llvm::StringRef memoryReference);
+
 /// Extract an array of strings for the specified key from an object.
 ///
 /// String values in the array will be extracted without any quotes
@@ -282,6 +289,16 @@ llvm::json::Value CreateScope(const llvm::StringRef name,
                               int64_t variablesReference,
                               int64_t namedVariables, bool expensive);
 
+/// Create a "Source" JSON object as described in the debug adaptor definition.
+///
+/// \param[in] file
+///     The SBFileSpec to use when populating out the "Source" object
+///
+/// \return
+///     A "Source" JSON object that follows the formal JSON
+///     definition outlined by Microsoft.
+llvm::json::Value CreateSource(const lldb::SBFileSpec &file);
+
 /// Create a "Source" JSON object as described in the debug adaptor definition.
 ///
 /// \param[in] line_entry
@@ -289,9 +306,9 @@ llvm::json::Value CreateScope(const llvm::StringRef name,
 ///     object
 ///
 /// \return
-///     A "Source" JSON object with that follows the formal JSON
+///     A "Source" JSON object that follows the formal JSON
 ///     definition outlined by Microsoft.
-llvm::json::Value CreateSource(lldb::SBLineEntry &line_entry);
+llvm::json::Value CreateSource(const lldb::SBLineEntry &line_entry);
 
 /// Create a "Source" object for a given source path.
 ///
@@ -463,15 +480,10 @@ struct VariableDescription {
 ///     The LLDB value to use when populating out the "Variable"
 ///     object.
 ///
-/// \param[in] variablesReference
-///     The variable reference. Zero if this value isn't structured
-///     and has no children, non-zero if it does have children and
-///     might be asked to expand itself.
-///
-/// \param[in] varID
-///     A unique variable identifier to help in properly identifying
-///     variables with the same name. This is an extension to the
-///     VS protocol.
+/// \param[in] var_ref
+///     The variable reference. Used to identify the value, e.g.
+///     in the `variablesReference` or `declarationLocationReference`
+///     properties.
 ///
 /// \param[in] format_hex
 ///     It set to true the variable will be formatted as hex in
@@ -492,8 +504,8 @@ struct VariableDescription {
 /// \return
 ///     A "Variable" JSON object with that follows the formal JSON
 ///     definition outlined by Microsoft.
-llvm::json::Value CreateVariable(lldb::SBValue v, int64_t variablesReference,
-                                 int64_t varID, bool format_hex,
+llvm::json::Value CreateVariable(lldb::SBValue v, int64_t var_ref,
+                                 bool format_hex,
                                  bool is_name_duplicated = false,
                                  std::optional<std::string> custom_name = {});
 
diff --git a/lldb/tools/lldb-dap/lldb-dap.cpp b/lldb/tools/lldb-dap/lldb-dap.cpp
index c2ebc9a96a9a29..2fb86f675b4516 100644
--- a/lldb/tools/lldb-dap/lldb-dap.cpp
+++ b/lldb/tools/lldb-dap/lldb-dap.cpp
@@ -8,7 +8,9 @@
 
 #include "DAP.h"
 #include "Watchpoint.h"
+#include "lldb/API/SBDeclaration.h"
 #include "lldb/API/SBMemoryRegionInfo.h"
+#include "llvm/Support/Base64.h"
 
 #include <cassert>
 #include <climits>
@@ -1541,6 +1543,16 @@ void request_completions(const llvm::json::Object &request) {
 //                              present the variables in a paged UI and fetch
 //                              them in chunks."
 //            }
+//            "memoryReference": {
+//               "type": "string",
+//                "description": "A memory reference to a location appropriate
+//                                for this result. For pointer type eval
+//                                results, this is generally a reference to the
+//                                memory address contained in the pointer. This
+//                                attribute may be returned by a debug adapter
+//                                if corresponding capability
+//                                `supportsMemoryReferences` is true."
+//             },
 //          },
 //          "required": [ "result", "variablesReference" ]
 //        }
@@ -1600,12 +1612,15 @@ void request_evaluate(const llvm::json::Object &request) {
       EmplaceSafeString(body, "result", desc.GetResult(context));
       EmplaceSafeString(body, "type", desc.display_type_name);
       if (value.MightHaveChildren()) {
-        auto variableReference = g_dap.variables.InsertExpandableVariable(
+        auto variableReference = g_dap.variables.InsertVariable(
             value, /*is_permanent=*/context == "repl");
         body.try_emplace("variablesReference", variableReference);
       } else {
         body.try_emplace("variablesReference", (int64_t)0);
       }
+      if (lldb::addr_t addr = value.GetLoadAddress();
+          addr != LLDB_INVALID_ADDRESS)
+        body.try_emplace("memoryReference", EncodeMemoryReference(addr));
     }
   }
   response.try_emplace("body", std::move(body));
@@ -1875,6 +1890,8 @@ void request_initialize(const llvm::json::Object &request) {
   // The debug adapter supports stepping granularities (argument `granularity`)
   // for the stepping requests.
   body.try_emplace("supportsSteppingGranularity", true);
+  // The debug adapter support for instruction breakpoint.
+  body.try_emplace("supportsInstructionBreakpoints", true);
 
   llvm::json::Array completion_characters;
   completion_characters.emplace_back(".");
@@ -1916,8 +1933,8 @@ void request_initialize(const llvm::json::Object &request) {
   body.try_emplace("supportsLogPoints", true);
   // The debug adapter supports data watchpoints.
   body.try_emplace("supportsDataBreakpoints", true);
-  // The debug adapter support for instruction breakpoint.
-  body.try_emplace("supportsInstructionBreakpoints", true);
+  // The debug adapter supports the `readMemory` request.
+  body.try_emplace("supportsReadMemoryRequest", true);
 
   // Put in non-DAP specification lldb specific information.
   llvm::json::Object lldb_json;
@@ -3773,10 +3790,13 @@ void request_setVariable(const llvm::json::Object &request) {
       // is_permanent is false because debug console does not support
       // setVariable request.
       if (variable.MightHaveChildren())
-        newVariablesReference = g_dap.variables.InsertExpandableVariable(
-            variable, /*is_permanent=*/false);
-
+        newVariablesReference =
+            g_dap.variables.InsertVariable(variable, /*is_permanent=*/false);
       body.try_emplace("variablesReference", newVariablesReference);
+
+      if (lldb::addr_t addr = variable.GetLoadAddress();
+          addr != LLDB_INVALID_ADDRESS)
+        body.try_emplace("memoryReference", EncodeMemoryReference(addr));
     } else {
       EmplaceSafeString(body, "message", std::string(error.GetCString()));
     }
@@ -3945,13 +3965,10 @@ void request_variables(const llvm::json::Object &request) {
       if (!variable.IsValid())
         break;
 
-      int64_t var_ref = 0;
-      if (variable.MightHaveChildren() || variable.IsSynthetic()) {
-        var_ref = g_dap.variables.InsertExpandableVariable(
-            variable, /*is_permanent=*/false);
-      }
+      int64_t var_ref =
+          g_dap.variables.InsertVariable(variable, /*is_permanent=*/false);
       variables.emplace_back(CreateVariable(
-          variable, var_ref, var_ref != 0 ? var_ref : UINT64_MAX, hex,
+          variable, var_ref, hex,
           variable_name_counts[GetNonNullVariableName(variable)] > 1));
     }
   } else {
@@ -3963,19 +3980,11 @@ void request_variables(const llvm::json::Object &request) {
                           std::optional<std::string> custom_name = {}) {
         if (!child.IsValid())
           return;
-        if (child.MightHaveChildren()) {
-          auto is_permanent =
-              g_dap.variables.IsPermanentVariableReference(variablesReference);
-          auto childVariablesReferences =
-              g_dap.variables.InsertExpandableVariable(child, is_permanent);
-          variables.emplace_back(CreateVariable(
-              child, childVariablesReferences, childVariablesReferences, hex,
-              /*is_name_duplicated=*/false, custom_name));
-        } else {
-          variables.emplace_back(CreateVariable(child, 0, INT64_MAX, hex,
-                                                /*is_name_duplicated=*/false,
-                                                custom_name));
-        }
+        bool is_permanent =
+            g_dap.variables.IsPermanentVariableReference(variablesReference);
+        int64_t var_ref = g_dap.variables.InsertVariable(child, is_permanent);
+        variables.emplace_back(CreateVariable(
+            child, var_ref, hex, /*is_name_duplicated=*/false, custom_name));
       };
       const int64_t num_children = variable.GetNumChildren();
       int64_t end_idx = start + ((count == 0) ? num_children : count);
@@ -3998,6 +4007,117 @@ void request_variables(const llvm::json::Object &request) {
   g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
+// "LocationsRequest": {
+//   "allOf": [ { "$ref": "#/definitions/Request" }, {
+//     "type": "object",
+//     "description": "Looks up information about a location reference
+//                     previously returned by the debug adapter.",
+//     "properties": {
+//       "command": {
+//         "type": "string",
+//         "enum": [ "locations" ]
+//       },
+//       "arguments": {
+//         "$ref": "#/definitions/LocationsArguments"
+//       }
+//     },
+//     "required": [ "command", "arguments" ]
+//   }]
+// },
+// "LocationsArguments": {
+//   "type": "object",
+//   "description": "Arguments for `locations` request.",
+//   "properties": {
+//     "locationReference": {
+//       "type": "integer",
+//       "description": "Location reference to resolve."
+//     }
+//   },
+//   "required": [ "locationReference" ]
+// },
+// "LocationsResponse": {
+//   "allOf": [ { "$ref": "#/definitions/Response" }, {
+//     "type": "object",
+//     "description": "Response to `locations` request.",
+//     "properties": {
+//       "body": {
+//         "type": "object",
+//         "properties": {
+//           "source": {
+//             "$ref": "#/definitions/Source",
+//             "description": "The source containing the location; either
+//                             `source.path` or `source.sourceReference` must be
+//                             specified."
+//           },
+//           "line": {
+//             "type": "integer",
+//             "description": "The line number of the location. The client
+//                             capability `linesStartAt1` determines whether it
+//                             is 0- or 1-based."
+//           },
+//           "column": {
+//             "type": "integer",
+//             "description": "Position of the location within the `line`. It is
+//                             measured in UTF-16 code units and the client
+//                             capability `columnsStartAt1` determines whether
+//                             it is 0- or 1-based. If no column is given, the
+//                             first position in the start line is assumed."
+//           },
+//           "endLine": {
+//             "type": "integer",
+//             "description": "End line of the location, present if the location
+//                             refers to a range.  The client capability
+//                             `linesStartAt1` determines whether it is 0- or
+//                             1-based."
+//           },
+//           "endColumn": {
+//             "type": "integer",
+//             "description": "End position of the location within `endLine`,
+//                             present if the location refers to a range. It is
+//                             measured in UTF-16 code units and the client
+//                             capability `columnsStartAt1` determines whether
+//                             it is 0- or 1-based."
+//           }
+//         },
+//         "required": [ "source", "line" ]
+//       }
+//     }
+//   }]
+// },
+void request_locations(const llvm::json::Object &request) {
+  llvm::json::Object response;
+  FillResponse(request, response);
+  auto arguments = request.getObject("arguments");
+
+  uint64_t reference_id = GetUnsigned(arguments, "locationReference", 0);
+  lldb::SBValue variable = g_dap.variables.GetVariable(reference_id);
+  if (!variable.IsValid()) {
+    response["success"] = false;
+    response["message"] = "Invalid variable reference";
+    g_dap.SendJSON(llvm::json::Value(std::move(response)));
+    return;
+  }
+
+  // Get the declaration location
+  lldb::SBDeclaration decl = variable.GetDeclaration();
+  if (!decl.IsValid()) {
+    response["success"] = false;
+    response["message"] = "No declaration location available";
+    g_dap.SendJSON(llvm::json::Value(std::move(response)));
+    return;
+  }
+
+  llvm::json::Object body;
+  body.try_emplace("source", CreateSource(decl.GetFileSpec()));
+  if (int line = decl.GetLine())
+    body.try_emplace("line", line);
+  if (int column = decl.GetColumn())
+    body.try_emplace("column", column);
+
+  response.try_emplace("body", std::move(body));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
+}
+
 // "DisassembleRequest": {
 //   "allOf": [ { "$ref": "#/definitions/Request" }, {
 //     "type": "object",
@@ -4073,17 +4193,18 @@ void request_variables(const llvm::json::Object &request) {
 void request_disassemble(const llvm::json::Object &request) {
   llvm::json::Object response;
   FillResponse(request, response);
-  auto arguments = request.getObject("arguments");
+  auto *arguments = request.getObject("arguments");
 
-  auto memoryReference = GetString(arguments, "memoryReference");
-  lldb::addr_t addr_ptr;
-  if (memoryReference.consumeInteger(0, addr_ptr)) {
+  llvm::StringRef memoryReference = GetString(arguments, "memoryReference");
+  auto addr_opt = DecodeMemoryReference(memoryReference);
+  if (!addr_opt.has_value()) {
     response["success"] = false;
     response["message"] =
         "Malformed memory reference: " + memoryReference.str();
     g_dap.SendJSON(llvm::json::Value(std::move(response)));
     return;
   }
+  lldb::addr_t addr_ptr = *addr_opt;
 
   addr_ptr += GetSigned(arguments, "instructionOffset", 0);
   lldb::SBAddress addr(addr_ptr, g_dap.target);
@@ -4201,6 +4322,161 @@ void request_disassemble(const llvm::json::Object &request) {
   response.try_emplace("body", std::move(body));
   g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
+
+// "ReadMemoryRequest": {
+//   "allOf": [ { "$ref": "#/definitions/Request" }, {
+//     "type": "object",
+//     "description": "Reads bytes from memory at the provided location. Clients
+//                     should only call this request if the corresponding
+//                     capability `supportsReadMemoryRequest` is true.",
+//     "properties": {
+//       "command": {
+//         "type": "string",
+//         "enum": [ "readMemory" ]
+//       },
+//       "arguments": {
+//         "$ref": "#/definitions/ReadMemoryArguments"
+//       }
+//     },
+//     "required": [ "command", "arguments" ]
+//   }]
+// },
+// "ReadMemoryArguments": {
+//   "type": "object",
+//   "description": "Arguments for `readMemory` request.",
+//   "properties": {
+//     "memoryReference": {
+//       "type": "string",
+//       "description": "Memory reference to the base location from which data
+//                       should be read."
+//     },
+//     "offset": {
+//       "type": "integer",
+//       "description": "Offset (in bytes) to be applied to the reference
+//                       location before reading data. Can be negative."
+//     },
+//     "count": {
+//       "type": "integer",
+//       "description": "Number of bytes to read at the specified location and
+//                       offset."
+//     }
+//   },
+//   "required": [ "memoryReference", "count" ]
+// },
+// "ReadMemoryResponse": {
+//   "allOf": [ { "$ref": "#/definitions/Response" }, {
+//     "type": "object",
+//     "description": "Response to `readMemory` request.",
+//     "properties": {
+//       "body": {
+//         "type": "object",
+//         "properties": {
+//           "address": {
+//             "type": "string",
+//             "description": "The address of the first byte of data returned.
+//                             Treated as a hex value if prefixed with `0x`, or
+//                             as a decimal value otherwise."
+//           },
+//           "unreadableBytes": {
+//             "type": "integer",
+//             "description": "The number of unreadable bytes encountered after
+//                             the last successfully read byte.\nThis can be
+//                             used to determine the number of bytes that should
+//                             be skipped before a subsequent
+//             `readMemory` request succeeds."
+//           },
+//           "data": {
+//             "type": "string",
+//             "description": "The bytes read from memory, encoded using base64.
+//                             If the decoded length of `data` is less than the
+//                             requested `count` in the original `readMemory`
+//                             request, and `unreadableBytes` is zero or
+//                             omitted, then the client should assume it's
+//                             reached the end of readable memory."
+//           }
+//         },
+//         "required": [ "address" ]
+//       }
+//     }
+//   }]
+// },
+void request_readMemory(const llvm::json::Object &request) {
+  llvm::json::Object response;
+  FillResponse(request, response);
+  auto *arguments = request.getObject("arguments");
+
+  lldb::SBProcess process = g_dap.target.GetProcess();
+  if (!process.IsValid()) {
+    response["success"] = false;
+    response["message"] = "No process running";
+    g_dap.SendJSON(llvm::json::Value(std::move(response)));
+    return;
+  }
+
+  llvm::StringRef memoryReference = GetString(arguments, "memoryReference");
+  auto addr_opt = DecodeMemoryReference(memoryReference);
+  if (!addr_opt.has_value()) {
+    response["success"] = false;
+    response["message"] =
+        "Malformed memory reference: " + memoryReference.str();
+    g_dap.SendJSON(llvm::json::Value(std::move(response)));
+    return;
+  }
+  lldb::addr_t addr = *addr_opt;
+
+  addr += GetSigned(arguments, "offset", 0);
+  const uint64_t requested_count = GetUnsigned(arguments, "count", 0);
+  lldb::SBMemoryRegionInfo region_info;
+  lldb::SBError memreg_error = process.GetMemoryRegionInfo(addr, region_info);
+  if (memreg_error.Fail()) {
+    response["success"] = false;
+    EmplaceSafeString(response, "message",
+                      "Unable to find memory region: " +
+                          std::string(memreg_error.GetCString()));
+    g_dap.SendJSON(llvm::json::Value(std::move(response)));
+    return;
+  }
+  if (!region_info.IsReadable()) {
+    response["success"] = false;
+    response.try_emplace("message", "Memory region is not readable");
+    g_dap.SendJSON(llvm::json::Value(std::move(response)));
+    return;
+  }
+  const uint64_t available_count =
+      std::min(requested_count, region_info.GetRegionEnd() - addr);
+  const uint64_t unavailable_count = requested_count - available_count;
+
+  std::vector<uint8_t> buf;
+  buf.resize(available_count);
+  if (available_count > 0) {
+    lldb::SBError memread_error;
+    uint64_t bytes_read =
+        process.ReadMemory(addr, buf.data(), available_count, memread_error);
+    if (memread_error.Fail()) {
+      response["success"] = false;
+      EmplaceSafeString(response, "message",
+                        "Unable to read memory: " +
+                            std::string(memread_error.GetCString()));
+      g_dap.SendJSON(llvm::json::Value(std::move(response)));
+      return;
+    }
+    if (bytes_read != available_count) {
+      response["success"] = false;
+      EmplaceSafeString(response, "message", "Unexpected, short read");
+      g_dap.SendJSON(llvm::json::Value(std::move(response)));
+      return;
+    }
+  }
+
+  llvm::json::Object body;
+  std::string formatted_addr = "0x" + llvm::utohexstr(addr);
+  body.try_emplace("address", formatted_addr);
+  body.try_emplace("data", llvm::encodeBase64(buf));
+  body.try_emplace("unreadableBytes", unavailable_count);
+  response.try_emplace("body", std::move(body));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
+}
+
 // A request used in testing to get the details on all breakpoints that are
 // currently set in the target. This helps us to test "setBreakpoints" and
 // "setFunctionBreakpoints" requests to verify we have the correct set of
@@ -4498,8 +4774,9 @@ void RegisterRequestCallbacks() {
   g_dap.RegisterRequestCallback("stepOut", request_stepOut);
   g_dap.RegisterRequestCallback("threads", request_threads);
   g_dap.RegisterRequestCallback("variables", request_variables);
+  g_dap.RegisterRequestCallback("locations", request_locations);
   g_dap.RegisterRequestCallback("disassemble", request_disassemble);
-  // Instruction breakpoint request
+  g_dap.RegisterRequestCallback("readMemory", request_readMemory);
   g_dap.RegisterRequestCallback("setInstructionBreakpoints",
                                 request_setInstructionBreakpoints);
   // Custom requests
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index f256a72aae700e..4b48b54b18bb99 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -492,6 +492,8 @@ Every processor supports every OS ABI (see :ref:`amdgpu-os`) with the following
                                                                         work-item                       Add product
                                                                         IDs                             names.
 
+     **GCN GFX11 (RDNA 3.5)** [AMD-GCN-GFX11-RDNA3.5]_
+     -----------------------------------------------------------------------------------------------------------------------
      ``gfx1150``                 ``amdgcn``   APU   - cumode          - Architected                   *TBA*
                                                     - wavefrontsize64   flat
                                                                         scratch                       .. TODO::
@@ -17546,8 +17548,8 @@ combinations of operands, refer to one of instruction set architecture manuals
 [AMD-GCN-GFX6]_, [AMD-GCN-GFX7]_, [AMD-GCN-GFX8]_,
 [AMD-GCN-GFX900-GFX904-VEGA]_, [AMD-GCN-GFX906-VEGA7NM]_,
 [AMD-GCN-GFX908-CDNA1]_, [AMD-GCN-GFX90A-CDNA2]_,
-[AMD-GCN-GFX940-GFX942-CDNA3]_, [AMD-GCN-GFX10-RDNA1]_, [AMD-GCN-GFX10-RDNA2]_
-and [AMD-GCN-GFX11-RDNA3]_.
+[AMD-GCN-GFX940-GFX942-CDNA3]_, [AMD-GCN-GFX10-RDNA1]_, [AMD-GCN-GFX10-RDNA2]_,
+[AMD-GCN-GFX11-RDNA3]_ and [AMD-GCN-GFX11-RDNA3.5]_.
 
 Operands
 ~~~~~~~~
@@ -18345,6 +18347,7 @@ Additional Documentation
 .. [AMD-GCN-GFX10-RDNA1] `AMD RDNA 1.0 Instruction Set Architecture <https://gpuopen.com/wp-content/uploads/2019/08/RDNA_Shader_ISA_5August2019.pdf>`__
 .. [AMD-GCN-GFX10-RDNA2] `AMD RDNA 2 Instruction Set Architecture <https://developer.amd.com/wp-content/resources/RDNA2_Shader_ISA_November2020.pdf>`__
 .. [AMD-GCN-GFX11-RDNA3] `AMD RDNA 3 Instruction Set Architecture <https://developer.amd.com/wp-content/resources/RDNA3_Shader_ISA_December2022.pdf>`__
+.. [AMD-GCN-GFX11-RDNA3.5] `AMD RDNA 3.5 Instruction Set Architecture <https://www.amd.com/content/dam/amd/en/documents/radeon-tech-docs/instruction-set-architectures/rdna35_instruction_set_architecture.pdf>`__
 .. [AMD-RADEON-HD-2000-3000] `AMD R6xx shader ISA <http://developer.amd.com/wordpress/media/2012/10/R600_Instruction_Set_Architecture.pdf>`__
 .. [AMD-RADEON-HD-4000] `AMD R7xx shader ISA <http://developer.amd.com/wordpress/media/2012/10/R700-Family_Instruction_Set_Architecture.pdf>`__
 .. [AMD-RADEON-HD-5000] `AMD Evergreen shader ISA <http://developer.amd.com/wordpress/media/2012/10/AMD_Evergreen-Family_Instruction_Set_Architecture.pdf>`__
diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst
index b06ce1415b2bbe..c42adc10b10a29 100644
--- a/llvm/docs/GlobalISel/GenericOpcode.rst
+++ b/llvm/docs/GlobalISel/GenericOpcode.rst
@@ -706,6 +706,8 @@ vector must be valid indices of that vector. If this condition cannot be
 determined statically but is false at runtime, then the result vector is
 undefined.
 
+Mixing scalable vectors and fixed vectors are not allowed.
+
 .. code-block:: none
 
   %3:_(<vscale x 4 x i64>) = G_EXTRACT_SUBVECTOR %2:_(<vscale x 8 x i64>), 2
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index 55ad8400ca16a5..f4240048574541 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -429,6 +429,7 @@ Global Pointer (GP) Relaxation and the Small Data Limit
 Some of the RISC-V psABI variants reserve ``gp`` (``x3``) for use as a "Global Pointer", to make generating data addresses more efficient.
 
 To use this functionality, you need to be doing all of the following:
+
 * Use the ``medlow`` (aka ``small``) code model;
 * Not use the ``gp`` register for any other uses (some platforms use it for the shadow stack and others as a temporary -- as denoted by the ``Tag_RISCV_x3_reg_usage`` build attribute);
 * Compile your objects with Clang's ``-mrelax`` option, to enable relaxation annotations on relocatable objects (this is the default, but ``-mno-relax`` disables these relaxation annotations);
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index b2124c6106198e..3411163549de2f 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -882,6 +882,8 @@ class TargetTransformInfo {
   ///  should use coldcc calling convention.
   bool useColdCCForColdCall(Function &F) const;
 
+  bool isTargetIntrinsicTriviallyScalarizable(Intrinsic::ID ID) const;
+
   /// Estimate the overhead of scalarizing an instruction. Insert and Extract
   /// are set if the demanded result elements need to be inserted and/or
   /// extracted from vectors.
@@ -1928,6 +1930,7 @@ class TargetTransformInfo::Concept {
   virtual bool shouldBuildLookupTablesForConstant(Constant *C) = 0;
   virtual bool shouldBuildRelLookupTables() = 0;
   virtual bool useColdCCForColdCall(Function &F) = 0;
+  virtual bool isTargetIntrinsicTriviallyScalarizable(Intrinsic::ID ID) = 0;
   virtual InstructionCost getScalarizationOverhead(VectorType *Ty,
                                                    const APInt &DemandedElts,
                                                    bool Insert, bool Extract,
@@ -2467,7 +2470,9 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
   bool useColdCCForColdCall(Function &F) override {
     return Impl.useColdCCForColdCall(F);
   }
-
+  bool isTargetIntrinsicTriviallyScalarizable(Intrinsic::ID ID) override {
+    return Impl.isTargetIntrinsicTriviallyScalarizable(ID);
+  }
   InstructionCost getScalarizationOverhead(VectorType *Ty,
                                            const APInt &DemandedElts,
                                            bool Insert, bool Extract,
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 90eef93a2a54d5..2819af30cd1704 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -373,6 +373,10 @@ class TargetTransformInfoImplBase {
 
   bool useColdCCForColdCall(Function &F) const { return false; }
 
+  bool isTargetIntrinsicTriviallyScalarizable(Intrinsic::ID ID) const {
+    return false;
+  }
+
   InstructionCost getScalarizationOverhead(VectorType *Ty,
                                            const APInt &DemandedElts,
                                            bool Insert, bool Extract,
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index caa3a57ebabc2e..2f2a6a09ffc44d 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -789,6 +789,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return Cost;
   }
 
+  bool isTargetIntrinsicTriviallyScalarizable(Intrinsic::ID ID) const {
+    return false;
+  }
+
   /// Helper wrapper for the DemandedElts variant of getScalarizationOverhead.
   InstructionCost getScalarizationOverhead(VectorType *InTy, bool Insert,
                                            bool Extract,
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 37c9422d192754..3261b26e74cd71 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -911,6 +911,7 @@ class CombinerHelper {
                               const MachineInstr &BVMI, BuildFnTy &MatchInfo);
 
   bool matchCanonicalizeICmp(const MachineInstr &MI, BuildFnTy &MatchInfo);
+  bool matchCanonicalizeFCmp(const MachineInstr &MI, BuildFnTy &MatchInfo);
 
 private:
   /// Checks for legality of an indexed variant of \p LdSt.
@@ -1029,6 +1030,8 @@ class CombinerHelper {
 
   bool constantFoldICmp(const GICmp &ICmp, const GIConstant &LHSCst,
                         const GIConstant &RHSCst, BuildFnTy &MatchInfo);
+  bool constantFoldFCmp(const GFCmp &FCmp, const GFConstant &LHSCst,
+                        const GFConstant &RHSCst, BuildFnTy &MatchInfo);
 };
 } // namespace llvm
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index 76e0954357a5d7..95a8234d3c6080 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -632,5 +632,44 @@ class GIConstant {
                                                const MachineRegisterInfo &MRI);
 };
 
+/// An floating-point-like constant.
+///
+/// It abstracts over scalar, fixed-length vectors, and scalable vectors.
+/// In the common case, it provides a common API and feels like an APFloat,
+/// while still providing low-level access.
+/// It can be used for constant-folding.
+///
+/// bool isZero()
+/// abstracts over the kind.
+///
+/// switch(const.getKind())
+/// {
+/// }
+/// provides low-level access.
+class GFConstant {
+public:
+  enum class GFConstantKind { Scalar, FixedVector, ScalableVector };
+
+private:
+  GFConstantKind Kind;
+  SmallVector<APFloat> Values;
+
+public:
+  GFConstant(ArrayRef<APFloat> Values)
+      : Kind(GFConstantKind::FixedVector), Values(Values) {};
+  GFConstant(const APFloat &Value, GFConstantKind Kind) : Kind(Kind) {
+    Values.push_back(Value);
+  }
+
+  /// Returns the kind of of this constant, e.g, Scalar.
+  GFConstantKind getKind() const { return Kind; }
+
+  /// Returns the value, if this constant is a scalar.
+  APFloat getScalarValue() const;
+
+  static std::optional<GFConstant> getConstant(Register Const,
+                                               const MachineRegisterInfo &MRI);
+};
+
 } // End namespace llvm.
 #endif
diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index 8d6df1026e53a3..76a7b8662bae66 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -728,7 +728,7 @@ class MachineInstr
     return make_range(operands_begin(),
                       operands_begin() + getNumExplicitDefs());
   }
-  /// Returns a range that includes all operands that are register uses.
+  /// Returns a range that includes all operands which may be register uses.
   /// This may include unrelated operands which are not register uses.
   iterator_range<mop_iterator> uses() {
     return make_range(operands_begin() + getNumExplicitDefs(), operands_end());
diff --git a/llvm/include/llvm/CodeGen/VirtRegMap.h b/llvm/include/llvm/CodeGen/VirtRegMap.h
index 96c28c191e8bd5..52221762fed5d0 100644
--- a/llvm/include/llvm/CodeGen/VirtRegMap.h
+++ b/llvm/include/llvm/CodeGen/VirtRegMap.h
@@ -31,13 +31,6 @@ class raw_ostream;
 class TargetInstrInfo;
 
   class VirtRegMap : public MachineFunctionPass {
-  public:
-    enum {
-      NO_STACK_SLOT = (1L << 30)-1,
-      MAX_STACK_SLOT = (1L << 18)-1
-    };
-
-  private:
     MachineRegisterInfo *MRI = nullptr;
     const TargetInstrInfo *TII = nullptr;
     const TargetRegisterInfo *TRI = nullptr;
@@ -70,6 +63,8 @@ class TargetInstrInfo;
   public:
     static char ID;
 
+    static constexpr int NO_STACK_SLOT = INT_MAX;
+
     VirtRegMap() : MachineFunctionPass(ID), Virt2StackSlotMap(NO_STACK_SLOT) {}
     VirtRegMap(const VirtRegMap &) = delete;
     VirtRegMap &operator=(const VirtRegMap &) = delete;
diff --git a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
index 30ca43d2bde020..09b9d947464ae9 100644
--- a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
+++ b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
@@ -104,6 +104,8 @@ enum class PrimitiveKind {
   Double,
   Ldouble,
   Nullptr,
+  Auto,
+  DecltypeAuto,
 };
 
 enum class CharKind {
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 1bc895eee60f1a..0a74a217a5f010 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -1632,12 +1632,16 @@ def int_experimental_patchpoint_void : Intrinsic<[],
                                                  [llvm_i64_ty, llvm_i32_ty,
                                                   llvm_ptr_ty, llvm_i32_ty,
                                                   llvm_vararg_ty],
-                                                  [Throws]>;
+                                                  [Throws, ImmArg<ArgIndex<0>>,
+                                                   ImmArg<ArgIndex<1>>,
+                                                   ImmArg<ArgIndex<3>>]>;
 def int_experimental_patchpoint : Intrinsic<[llvm_any_ty],
                                             [llvm_i64_ty, llvm_i32_ty,
                                              llvm_ptr_ty, llvm_i32_ty,
                                              llvm_vararg_ty],
-                                            [Throws]>;
+                                            [Throws, ImmArg<ArgIndex<0>>,
+                                             ImmArg<ArgIndex<1>>,
+                                             ImmArg<ArgIndex<3>>]>;
 
 
 //===------------------------ Garbage Collection Intrinsics ---------------===//
diff --git a/llvm/include/llvm/SandboxIR/SandboxIR.h b/llvm/include/llvm/SandboxIR/SandboxIR.h
index 0cc0f3a4c87035..c01516aa9d31ac 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIR.h
+++ b/llvm/include/llvm/SandboxIR/SandboxIR.h
@@ -129,6 +129,8 @@ class ConstantTokenNone;
 class GlobalValue;
 class GlobalObject;
 class GlobalIFunc;
+class GlobalVariable;
+class GlobalAlias;
 class Context;
 class Function;
 class Instruction;
@@ -334,6 +336,8 @@ class Value {
   friend class DSOLocalEquivalent;    // For `Val`.
   friend class GlobalObject;          // For `Val`.
   friend class GlobalIFunc;           // For `Val`.
+  friend class GlobalVariable;        // For `Val`.
+  friend class GlobalAlias;           // For `Val`.
 
   /// All values point to the context.
   Context &Ctx;
@@ -1370,6 +1374,194 @@ class GlobalIFunc final
 #endif
 };
 
+class GlobalVariable final
+    : public GlobalWithNodeAPI<GlobalVariable, llvm::GlobalVariable,
+                               GlobalObject, llvm::GlobalObject> {
+  GlobalVariable(llvm::GlobalObject *C, Context &Ctx)
+      : GlobalWithNodeAPI(ClassID::GlobalVariable, C, Ctx) {}
+  friend class Context; // For constructor.
+
+  /// Helper for mapped_iterator.
+  struct LLVMGVToGV {
+    Context &Ctx;
+    LLVMGVToGV(Context &Ctx) : Ctx(Ctx) {}
+    GlobalVariable &operator()(llvm::GlobalVariable &LLVMGV) const;
+  };
+
+public:
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::GlobalVariable;
+  }
+
+  /// Definitions have initializers, declarations don't.
+  ///
+  inline bool hasInitializer() const {
+    return cast<llvm::GlobalVariable>(Val)->hasInitializer();
+  }
+
+  /// hasDefinitiveInitializer - Whether the global variable has an initializer,
+  /// and any other instances of the global (this can happen due to weak
+  /// linkage) are guaranteed to have the same initializer.
+  ///
+  /// Note that if you want to transform a global, you must use
+  /// hasUniqueInitializer() instead, because of the *_odr linkage type.
+  ///
+  /// Example:
+  ///
+  /// @a = global SomeType* null - Initializer is both definitive and unique.
+  ///
+  /// @b = global weak SomeType* null - Initializer is neither definitive nor
+  /// unique.
+  ///
+  /// @c = global weak_odr SomeType* null - Initializer is definitive, but not
+  /// unique.
+  inline bool hasDefinitiveInitializer() const {
+    return cast<llvm::GlobalVariable>(Val)->hasDefinitiveInitializer();
+  }
+
+  /// hasUniqueInitializer - Whether the global variable has an initializer, and
+  /// any changes made to the initializer will turn up in the final executable.
+  inline bool hasUniqueInitializer() const {
+    return cast<llvm::GlobalVariable>(Val)->hasUniqueInitializer();
+  }
+
+  /// getInitializer - Return the initializer for this global variable.  It is
+  /// illegal to call this method if the global is external, because we cannot
+  /// tell what the value is initialized to!
+  ///
+  Constant *getInitializer() const;
+  /// setInitializer - Sets the initializer for this global variable, removing
+  /// any existing initializer if InitVal==NULL. The initializer must have the
+  /// type getValueType().
+  void setInitializer(Constant *InitVal);
+
+  // TODO: Add missing replaceInitializer(). Requires special tracker
+
+  /// If the value is a global constant, its value is immutable throughout the
+  /// runtime execution of the program.  Assigning a value into the constant
+  /// leads to undefined behavior.
+  ///
+  bool isConstant() const {
+    return cast<llvm::GlobalVariable>(Val)->isConstant();
+  }
+  void setConstant(bool V);
+
+  bool isExternallyInitialized() const {
+    return cast<llvm::GlobalVariable>(Val)->isExternallyInitialized();
+  }
+  void setExternallyInitialized(bool Val);
+
+  // TODO: Missing copyAttributesFrom()
+
+  // TODO: Missing removeFromParent(), eraseFromParent(), dropAllReferences()
+
+  // TODO: Missing addDebugInfo(), getDebugInfo()
+
+  // TODO: Missing attribute setter functions: addAttribute(), setAttributes().
+  //       There seems to be no removeAttribute() so we can't undo them.
+
+  /// Return true if the attribute exists.
+  bool hasAttribute(Attribute::AttrKind Kind) const {
+    return cast<llvm::GlobalVariable>(Val)->hasAttribute(Kind);
+  }
+
+  /// Return true if the attribute exists.
+  bool hasAttribute(StringRef Kind) const {
+    return cast<llvm::GlobalVariable>(Val)->hasAttribute(Kind);
+  }
+
+  /// Return true if any attributes exist.
+  bool hasAttributes() const {
+    return cast<llvm::GlobalVariable>(Val)->hasAttributes();
+  }
+
+  /// Return the attribute object.
+  Attribute getAttribute(Attribute::AttrKind Kind) const {
+    return cast<llvm::GlobalVariable>(Val)->getAttribute(Kind);
+  }
+
+  /// Return the attribute object.
+  Attribute getAttribute(StringRef Kind) const {
+    return cast<llvm::GlobalVariable>(Val)->getAttribute(Kind);
+  }
+
+  /// Return the attribute set for this global
+  AttributeSet getAttributes() const {
+    return cast<llvm::GlobalVariable>(Val)->getAttributes();
+  }
+
+  /// Return attribute set as list with index.
+  /// FIXME: This may not be required once ValueEnumerators
+  /// in bitcode-writer can enumerate attribute-set.
+  AttributeList getAttributesAsList(unsigned Index) const {
+    return cast<llvm::GlobalVariable>(Val)->getAttributesAsList(Index);
+  }
+
+  /// Check if section name is present
+  bool hasImplicitSection() const {
+    return cast<llvm::GlobalVariable>(Val)->hasImplicitSection();
+  }
+
+  /// Get the custom code model raw value of this global.
+  ///
+  unsigned getCodeModelRaw() const {
+    return cast<llvm::GlobalVariable>(Val)->getCodeModelRaw();
+  }
+
+  /// Get the custom code model of this global if it has one.
+  ///
+  /// If this global does not have a custom code model, the empty instance
+  /// will be returned.
+  std::optional<CodeModel::Model> getCodeModel() const {
+    return cast<llvm::GlobalVariable>(Val)->getCodeModel();
+  }
+
+  // TODO: Missing setCodeModel(). Requires custom tracker.
+
+#ifndef NDEBUG
+  void verify() const override {
+    assert(isa<llvm::GlobalVariable>(Val) && "Expected a GlobalVariable!");
+  }
+  void dumpOS(raw_ostream &OS) const override {
+    dumpCommonPrefix(OS);
+    dumpCommonSuffix(OS);
+  }
+#endif
+};
+
+class GlobalAlias final
+    : public GlobalWithNodeAPI<GlobalAlias, llvm::GlobalAlias, GlobalValue,
+                               llvm::GlobalValue> {
+  GlobalAlias(llvm::GlobalAlias *C, Context &Ctx)
+      : GlobalWithNodeAPI(ClassID::GlobalAlias, C, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::GlobalAlias;
+  }
+
+  // TODO: Missing create() due to unimplemented sandboxir::Module.
+
+  // TODO: Missing copyAttributresFrom().
+  // TODO: Missing removeFromParent(), eraseFromParent().
+
+  void setAliasee(Constant *Aliasee);
+  Constant *getAliasee() const;
+
+  const GlobalObject *getAliaseeObject() const;
+  GlobalObject *getAliaseeObject() {
+    return const_cast<GlobalObject *>(
+        static_cast<const GlobalAlias *>(this)->getAliaseeObject());
+  }
+
+  static bool isValidLinkage(LinkageTypes L) {
+    return llvm::GlobalAlias::isValidLinkage(L);
+  }
+};
+
 class BlockAddress final : public Constant {
   BlockAddress(llvm::BlockAddress *C, Context &Ctx)
       : Constant(ClassID::BlockAddress, C, Ctx) {}
@@ -1501,6 +1693,8 @@ class BBIterator {
   /// \Returns the SBInstruction that corresponds to this iterator, or null if
   /// the instruction is not found in the IR-to-SandboxIR tables.
   pointer get() const { return getInstr(It); }
+  /// \Returns the parent BB.
+  BasicBlock *getNodeParent() const;
 };
 
 /// Contains a list of sandboxir::Instruction's.
diff --git a/llvm/include/llvm/Support/GlobPattern.h b/llvm/include/llvm/Support/GlobPattern.h
index 1a722e65bed802..eff8957d1b682e 100644
--- a/llvm/include/llvm/Support/GlobPattern.h
+++ b/llvm/include/llvm/Support/GlobPattern.h
@@ -34,8 +34,8 @@ namespace llvm {
 ///   expansions are not supported. If \p MaxSubPatterns is empty then
 ///   brace expansions are not supported and characters \p "{,}" are treated as
 ///   literals.
-/// * \p "\" escapes the next character so it is treated as a literal.
-///
+/// * \p "\\" (a single backslash) escapes the next character so it is treated
+///   as a literal.
 ///
 /// Some known edge cases are:
 /// * \p "]" is allowed as the first character in a character class, i.e.,
@@ -45,9 +45,8 @@ namespace llvm {
 /// * \p "}" and \p "," that are not inside a brace expansion are taken as
 ///   literals, e.g., \p ",}" is valid but \p "{" is not.
 ///
-///
-/// For example, \p "*[/\\]foo.{c,cpp}" will match (unix or windows) paths to
-/// all files named \p "foo.c" or \p "foo.cpp".
+/// For example, \p "*[/\\\\]foo.{c,cpp}" (with two backslashes) will match
+/// (unix or windows) paths to all files named \p "foo.c" or \p "foo.cpp".
 class GlobPattern {
 public:
   /// \param Pat the pattern to match against
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index a55d9d3b04e62f..f5e62dda6fd043 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -1541,7 +1541,7 @@ def G_INSERT_SUBVECTOR : GenericInstruction {
 // Generic extract subvector.
 def G_EXTRACT_SUBVECTOR : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
-  let InOperandList = (ins type0:$src, untyped_imm_0:$idx);
+  let InOperandList = (ins type1:$src, untyped_imm_0:$idx);
   let hasSideEffects = false;
 }
 
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index c66212d2ab12c8..e75cf0b7d4afc1 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -1921,8 +1921,15 @@ def canonicalize_icmp : GICombineRule<
          [{ return Helper.matchCanonicalizeICmp(*${cmp}, ${matchinfo}); }]),
   (apply [{ Helper.applyBuildFn(*${cmp}, ${matchinfo}); }])>;
 
-def icmp_combines: GICombineGroup<[
+def canonicalize_fcmp : GICombineRule<
+  (defs root:$root, build_fn_matchinfo:$matchinfo),
+  (match (G_FCMP $root, $pred, $lhs, $rhs):$cmp,
+         [{ return Helper.matchCanonicalizeFCmp(*${cmp}, ${matchinfo}); }]),
+  (apply [{ Helper.applyBuildFn(*${cmp}, ${matchinfo}); }])>;
+
+def cmp_combines: GICombineGroup<[
   canonicalize_icmp,
+  canonicalize_fcmp,
   icmp_to_true_false_known_bits,
   icmp_to_lhs_known_bits,
   double_icmp_zero_and_combine,
@@ -1995,7 +2002,7 @@ def all_combines : GICombineGroup<[integer_reassoc_combines, trivial_combines,
     combine_extracted_vector_load,
     undef_combines, identity_combines, phi_combines,
     simplify_add_to_sub, hoist_logic_op_with_same_opcode_hands, shifts_too_big,
-    reassocs, ptr_add_immed_chain, icmp_combines,
+    reassocs, ptr_add_immed_chain, cmp_combines,
     shl_ashr_to_sext_inreg, sext_inreg_of_load,
     width_reduction_combines, select_combines,
     known_bits_simplifications,
diff --git a/llvm/include/llvm/Transforms/IPO/FunctionImport.h b/llvm/include/llvm/Transforms/IPO/FunctionImport.h
index 70739709a810ab..4b29d3f40ab7b5 100644
--- a/llvm/include/llvm/Transforms/IPO/FunctionImport.h
+++ b/llvm/include/llvm/Transforms/IPO/FunctionImport.h
@@ -270,13 +270,20 @@ class FunctionImporter {
   // A map from destination modules to lists of imports.
   class ImportListsTy {
   public:
-    ImportListsTy() = default;
-    ImportListsTy(size_t Size) : ListsImpl(Size) {}
+    ImportListsTy() : EmptyList(ImportIDs) {}
+    ImportListsTy(size_t Size) : EmptyList(ImportIDs), ListsImpl(Size) {}
 
     ImportMapTy &operator[](StringRef DestMod) {
       return ListsImpl.try_emplace(DestMod, ImportIDs).first->second;
     }
 
+    const ImportMapTy &lookup(StringRef DestMod) const {
+      auto It = ListsImpl.find(DestMod);
+      if (It != ListsImpl.end())
+        return It->second;
+      return EmptyList;
+    }
+
     size_t size() const { return ListsImpl.size(); }
 
     using const_iterator = DenseMap<StringRef, ImportMapTy>::const_iterator;
@@ -284,6 +291,7 @@ class FunctionImporter {
     const_iterator end() const { return ListsImpl.end(); }
 
   private:
+    ImportMapTy EmptyList;
     DenseMap<StringRef, ImportMapTy> ListsImpl;
     ImportIDTable ImportIDs;
   };
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
new file mode 100644
index 00000000000000..8a2021a5e6ba60
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
@@ -0,0 +1,86 @@
+//===- DependencyGraph.h ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the dependency graph used by the vectorizer's instruction
+// scheduler.
+//
+// The nodes of the graph are objects of the `DGNode` class. Each `DGNode`
+// object points to an instruction.
+// The edges between `DGNode`s are implicitly defined by an ordered set of
+// predecessor nodes, to save memory.
+// Finally the whole dependency graph is an object of the `DependencyGraph`
+// class, which also provides the API for creating/extending the graph from
+// input Sandbox IR.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_DEPENDENCYGRAPH_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_DEPENDENCYGRAPH_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/SandboxIR/SandboxIR.h"
+
+namespace llvm::sandboxir {
+
+/// A DependencyGraph Node that points to an Instruction and contains memory
+/// dependency edges.
+class DGNode {
+  Instruction *I;
+  /// Memory predecessors.
+  DenseSet<DGNode *> MemPreds;
+
+public:
+  DGNode(Instruction *I) : I(I) {}
+  Instruction *getInstruction() const { return I; }
+  void addMemPred(DGNode *PredN) { MemPreds.insert(PredN); }
+  /// \Returns all memory dependency predecessors.
+  iterator_range<DenseSet<DGNode *>::const_iterator> memPreds() const {
+    return make_range(MemPreds.begin(), MemPreds.end());
+  }
+  /// \Returns true if there is a memory dependency N->this.
+  bool hasMemPred(DGNode *N) const { return MemPreds.count(N); }
+#ifndef NDEBUG
+  void print(raw_ostream &OS, bool PrintDeps = true) const;
+  friend raw_ostream &operator<<(DGNode &N, raw_ostream &OS) {
+    N.print(OS);
+    return OS;
+  }
+  LLVM_DUMP_METHOD void dump() const;
+#endif // NDEBUG
+};
+
+class DependencyGraph {
+private:
+  DenseMap<Instruction *, std::unique_ptr<DGNode>> InstrToNodeMap;
+
+public:
+  DependencyGraph() {}
+
+  DGNode *getNode(Instruction *I) const {
+    auto It = InstrToNodeMap.find(I);
+    return It != InstrToNodeMap.end() ? It->second.get() : nullptr;
+  }
+  DGNode *getOrCreateNode(Instruction *I) {
+    auto [It, NotInMap] = InstrToNodeMap.try_emplace(I);
+    if (NotInMap)
+      It->second = std::make_unique<DGNode>(I);
+    return It->second.get();
+  }
+  // TODO: extend() should work with intervals not the whole BB.
+  /// Build the dependency graph for \p BB.
+  void extend(BasicBlock *BB);
+#ifndef NDEBUG
+  void print(raw_ostream &OS) const;
+  LLVM_DUMP_METHOD void dump() const;
+#endif // NDEBUG
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_DEPENDENCYGRAPH_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
new file mode 100644
index 00000000000000..78c1c0e4c04649
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
@@ -0,0 +1,62 @@
+//===- Legality.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Legality checks for the Sandbox Vectorizer.
+//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_LEGALITY_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_LEGALITY_H
+
+#include "llvm/SandboxIR/SandboxIR.h"
+
+namespace llvm::sandboxir {
+
+class LegalityAnalysis;
+
+enum class LegalityResultID {
+  Widen, ///> Vectorize by combining scalars to a vector.
+};
+
+/// The legality outcome is represented by a class rather than an enum class
+/// because in some cases the legality checks are expensive and look for a
+/// particular instruction that can be passed along to the vectorizer to avoid
+/// repeating the same expensive computation.
+class LegalityResult {
+protected:
+  LegalityResultID ID;
+  /// Only Legality can create LegalityResults.
+  LegalityResult(LegalityResultID ID) : ID(ID) {}
+  friend class LegalityAnalysis;
+
+public:
+  LegalityResultID getSubclassID() const { return ID; }
+};
+
+class Widen final : public LegalityResult {
+  friend class LegalityAnalysis;
+  Widen() : LegalityResult(LegalityResultID::Widen) {}
+
+public:
+  static bool classof(const LegalityResult *From) {
+    return From->getSubclassID() == LegalityResultID::Widen;
+  }
+};
+
+/// Performs the legality analysis and returns a LegalityResult object.
+class LegalityAnalysis {
+public:
+  LegalityAnalysis() = default;
+  LegalityResult canVectorize(ArrayRef<Value *> Bndl) {
+    // TODO: For now everything is legal.
+    return Widen();
+  }
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_LEGALITY_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
index 5b3d1a50aa1ec0..99582e3e0e0233 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
@@ -12,11 +12,18 @@
 #ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_BOTTOMUPVEC_H
 #define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_BOTTOMUPVEC_H
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/SandboxIR/Pass.h"
+#include "llvm/SandboxIR/SandboxIR.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
 
 namespace llvm::sandboxir {
 
 class BottomUpVec final : public FunctionPass {
+  bool Change = false;
+  LegalityAnalysis Legality;
+  void vectorizeRec(ArrayRef<Value *> Bndl);
+  void tryVectorize(ArrayRef<Value *> Seeds);
 
 public:
   BottomUpVec() : FunctionPass("bottom-up-vec") {}
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 57e03f667ba6ff..e06863b6deb902 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -8594,8 +8594,7 @@ const SCEV *ScalarEvolution::BackedgeTakenInfo::getExact(
     Ops.push_back(BECount);
 
     if (Preds)
-      for (const auto *P : ENT.Predicates)
-        Preds->push_back(P);
+      append_range(*Preds, ENT.Predicates);
 
     assert((Preds || ENT.hasAlwaysTruePredicate()) &&
            "Predicate should be always true!");
@@ -8616,8 +8615,7 @@ ScalarEvolution::BackedgeTakenInfo::getExitNotTaken(
       if (ENT.hasAlwaysTruePredicate())
         return &ENT;
       else if (Predicates) {
-        for (const auto *P : ENT.Predicates)
-          Predicates->push_back(P);
+        append_range(*Predicates, ENT.Predicates);
         return &ENT;
       }
     }
@@ -8659,8 +8657,7 @@ const SCEV *ScalarEvolution::BackedgeTakenInfo::getSymbolicMax(
                "dominate latch!");
         ExitCounts.push_back(ExitCount);
         if (Predicates)
-          for (const auto *P : ENT.Predicates)
-            Predicates->push_back(P);
+          append_range(*Predicates, ENT.Predicates);
 
         assert((Predicates || ENT.hasAlwaysTruePredicate()) &&
                "Predicate should be always true!");
@@ -14804,8 +14801,7 @@ const SCEVAddRecExpr *ScalarEvolution::convertSCEVToAddRecWithPredicates(
 
   // Since the transformation was successful, we can now transfer the SCEV
   // predicates.
-  for (const auto *P : TransformPreds)
-    Preds.insert(P);
+  Preds.insert(TransformPreds.begin(), TransformPreds.end());
 
   return AddRec;
 }
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 2c26493bd3f1ca..67b626f300a101 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -587,6 +587,11 @@ bool TargetTransformInfo::useColdCCForColdCall(Function &F) const {
   return TTIImpl->useColdCCForColdCall(F);
 }
 
+bool TargetTransformInfo::isTargetIntrinsicTriviallyScalarizable(
+    Intrinsic::ID ID) const {
+  return TTIImpl->isTargetIntrinsicTriviallyScalarizable(ID);
+}
+
 InstructionCost TargetTransformInfo::getScalarizationOverhead(
     VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
     TTI::TargetCostKind CostKind) const {
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 148b620c2b62b7..e9649f9ff81658 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -2061,8 +2061,10 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) {
   unsigned LastAsmLine =
       Asm->OutStreamer->getContext().getCurrentDwarfLoc().getLine();
 
-  bool PrevInstInDiffBB = PrevInstBB && PrevInstBB != MI->getParent();
-  if (DL == PrevInstLoc && !PrevInstInDiffBB) {
+  bool PrevInstInSameSection =
+      (!PrevInstBB ||
+       PrevInstBB->getSectionID() == MI->getParent()->getSectionID());
+  if (DL == PrevInstLoc && PrevInstInSameSection) {
     // If we have an ongoing unspecified location, nothing to do here.
     if (!DL)
       return;
@@ -2091,7 +2093,8 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) {
     //   possibly debug information; we want it to have a source location.
     // - Instruction is at the top of a block; we don't want to inherit the
     //   location from the physically previous (maybe unrelated) block.
-    if (UnknownLocations == Enable || PrevLabel || PrevInstInDiffBB) {
+    if (UnknownLocations == Enable || PrevLabel ||
+        (PrevInstBB && PrevInstBB != MI->getParent())) {
       // Preserve the file and column numbers, if we can, to save space in
       // the encoded line table.
       // Do not update PrevInstLoc, it remembers the last non-0 line.
@@ -2116,11 +2119,9 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) {
     PrologEndLoc = nullptr;
   }
   // If the line changed, we call that a new statement; unless we went to
-  // line 0 and came back, in which case it is not a new statement. We also
-  // mark is_stmt for the first non-0 line in each BB, in case a predecessor BB
-  // ends with a different line.
+  // line 0 and came back, in which case it is not a new statement.
   unsigned OldLine = PrevInstLoc ? PrevInstLoc.getLine() : LastAsmLine;
-  if (DL.getLine() && (DL.getLine() != OldLine || PrevInstInDiffBB))
+  if (DL.getLine() && DL.getLine() != OldLine)
     Flags |= DWARF2_FLAG_IS_STMT;
 
   const MDNode *Scope = DL.getScope();
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelperCompares.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelperCompares.cpp
index 025cd2dc9f87f1..7a4cfd4b1a7bb5 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelperCompares.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelperCompares.cpp
@@ -60,6 +60,37 @@ bool CombinerHelper::constantFoldICmp(const GICmp &ICmp,
   return true;
 }
 
+bool CombinerHelper::constantFoldFCmp(const GFCmp &FCmp,
+                                      const GFConstant &LHSCst,
+                                      const GFConstant &RHSCst,
+                                      BuildFnTy &MatchInfo) {
+  if (LHSCst.getKind() != GFConstant::GFConstantKind::Scalar)
+    return false;
+
+  Register Dst = FCmp.getReg(0);
+  LLT DstTy = MRI.getType(Dst);
+
+  if (!isConstantLegalOrBeforeLegalizer(DstTy))
+    return false;
+
+  CmpInst::Predicate Pred = FCmp.getCond();
+  APFloat LHS = LHSCst.getScalarValue();
+  APFloat RHS = RHSCst.getScalarValue();
+
+  bool Result = FCmpInst::compare(LHS, RHS, Pred);
+
+  MatchInfo = [=](MachineIRBuilder &B) {
+    if (Result)
+      B.buildConstant(Dst, getICmpTrueVal(getTargetLowering(),
+                                          /*IsVector=*/DstTy.isVector(),
+                                          /*IsFP=*/true));
+    else
+      B.buildConstant(Dst, 0);
+  };
+
+  return true;
+}
+
 bool CombinerHelper::matchCanonicalizeICmp(const MachineInstr &MI,
                                            BuildFnTy &MatchInfo) {
   const GICmp *Cmp = cast<GICmp>(&MI);
@@ -84,3 +115,31 @@ bool CombinerHelper::matchCanonicalizeICmp(const MachineInstr &MI,
 
   return false;
 }
+
+bool CombinerHelper::matchCanonicalizeFCmp(const MachineInstr &MI,
+                                           BuildFnTy &MatchInfo) {
+  const GFCmp *Cmp = cast<GFCmp>(&MI);
+
+  Register Dst = Cmp->getReg(0);
+  Register LHS = Cmp->getLHSReg();
+  Register RHS = Cmp->getRHSReg();
+
+  CmpInst::Predicate Pred = Cmp->getCond();
+  assert(CmpInst::isFPPredicate(Pred) && "Not an FP compare!");
+
+  if (auto CLHS = GFConstant::getConstant(LHS, MRI)) {
+    if (auto CRHS = GFConstant::getConstant(RHS, MRI))
+      return constantFoldFCmp(*Cmp, *CLHS, *CRHS, MatchInfo);
+
+    // If we have a constant, make sure it is on the RHS.
+    std::swap(LHS, RHS);
+    Pred = CmpInst::getSwappedPredicate(Pred);
+
+    MatchInfo = [=](MachineIRBuilder &B) {
+      B.buildFCmp(Pred, Dst, LHS, RHS, Cmp->getFlags());
+    };
+    return true;
+  }
+
+  return false;
+}
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index 925a1c7cf6aacc..59f2fc633f5de7 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -942,7 +942,7 @@ MachineInstrBuilder MachineIRBuilder::buildInsertSubvector(const DstOp &Res,
 MachineInstrBuilder MachineIRBuilder::buildExtractSubvector(const DstOp &Res,
                                                             const SrcOp &Src,
                                                             unsigned Idx) {
-  return buildInstr(TargetOpcode::G_INSERT_SUBVECTOR, Res,
+  return buildInstr(TargetOpcode::G_EXTRACT_SUBVECTOR, Res,
                     {Src, uint64_t(Idx)});
 }
 
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 15d3aa427d568d..9574464207d99f 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -2008,3 +2008,43 @@ llvm::GIConstant::getConstant(Register Const, const MachineRegisterInfo &MRI) {
 
   return GIConstant(MayBeConstant->Value, GIConstantKind::Scalar);
 }
+
+APFloat llvm::GFConstant::getScalarValue() const {
+  assert(Kind == GFConstantKind::Scalar && "Expected scalar constant");
+
+  return Values[0];
+}
+
+std::optional<GFConstant>
+llvm::GFConstant::getConstant(Register Const, const MachineRegisterInfo &MRI) {
+  MachineInstr *Constant = getDefIgnoringCopies(Const, MRI);
+
+  if (GSplatVector *Splat = dyn_cast<GSplatVector>(Constant)) {
+    std::optional<FPValueAndVReg> MayBeConstant =
+        getFConstantVRegValWithLookThrough(Splat->getScalarReg(), MRI);
+    if (!MayBeConstant)
+      return std::nullopt;
+    return GFConstant(MayBeConstant->Value, GFConstantKind::ScalableVector);
+  }
+
+  if (GBuildVector *Build = dyn_cast<GBuildVector>(Constant)) {
+    SmallVector<APFloat> Values;
+    unsigned NumSources = Build->getNumSources();
+    for (unsigned I = 0; I < NumSources; ++I) {
+      Register SrcReg = Build->getSourceReg(I);
+      std::optional<FPValueAndVReg> MayBeConstant =
+          getFConstantVRegValWithLookThrough(SrcReg, MRI);
+      if (!MayBeConstant)
+        return std::nullopt;
+      Values.push_back(MayBeConstant->Value);
+    }
+    return GFConstant(Values);
+  }
+
+  std::optional<FPValueAndVReg> MayBeConstant =
+      getFConstantVRegValWithLookThrough(Const, MRI);
+  if (!MayBeConstant)
+    return std::nullopt;
+
+  return GFConstant(MayBeConstant->Value, GFConstantKind::Scalar);
+}
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index 759201ed9dadc7..1fcbeeec6f64cc 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -1739,7 +1739,7 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
     }
 
     if (IndexOp.getImm() != 0 &&
-        Src1Ty.getElementCount().getKnownMinValue() % IndexOp.getImm() != 0) {
+        IndexOp.getImm() % Src1Ty.getElementCount().getKnownMinValue() != 0) {
       report("Index must be a multiple of the second source vector's "
              "minimum vector length",
              MI);
@@ -1778,10 +1778,25 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
       break;
     }
 
-    if (IndexOp.getImm() != 0 &&
-        SrcTy.getElementCount().getKnownMinValue() % IndexOp.getImm() != 0) {
-      report("Index must be a multiple of the source vector's minimum vector "
-             "length",
+    if (SrcTy.isScalable() != DstTy.isScalable()) {
+      report("Vector types must both be fixed or both be scalable", MI);
+      break;
+    }
+
+    uint64_t Idx = IndexOp.getImm();
+    uint64_t DstMinLen = DstTy.getElementCount().getKnownMinValue();
+    if (Idx % DstMinLen != 0) {
+      report("Index must be a multiple of the destination vector's minimum "
+             "vector length",
+             MI);
+      break;
+    }
+
+    uint64_t SrcMinLen = SrcTy.getElementCount().getKnownMinValue();
+    if (SrcTy.isScalable() == DstTy.isScalable() &&
+        (Idx >= SrcMinLen || Idx + DstMinLen > SrcMinLen)) {
+      report("Source type and index must not cause extract to overrun to the "
+             "destination type",
              MI);
       break;
     }
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index cd39cb08f8de92..37fcd09d4f5626 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27010,6 +27010,19 @@ SDValue DAGCombiner::visitVECREDUCE(SDNode *N) {
       return DAG.getNode(Opcode, SDLoc(N), N->getValueType(0), Subvec);
   }
 
+  // vecreduce_or(sext(x)) -> sext(vecreduce_or(x))
+  // Same for zext and anyext, and for and/or/xor reductions.
+  if ((Opcode == ISD::VECREDUCE_OR || Opcode == ISD::VECREDUCE_AND ||
+       Opcode == ISD::VECREDUCE_XOR) &&
+      (N0.getOpcode() == ISD::SIGN_EXTEND ||
+       N0.getOpcode() == ISD::ZERO_EXTEND ||
+       N0.getOpcode() == ISD::ANY_EXTEND) &&
+      TLI.isOperationLegalOrCustom(Opcode, N0.getOperand(0).getValueType())) {
+    SDValue Red = DAG.getNode(Opcode, SDLoc(N),
+                              N0.getOperand(0).getValueType().getScalarType(),
+                              N0.getOperand(0));
+    return DAG.getNode(N0.getOpcode(), SDLoc(N), N->getValueType(0), Red);
+  }
   return SDValue();
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index b5c80005a0ecc1..2c81c829e75cbb 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -2385,32 +2385,26 @@ SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
 //
 
 static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
-  if (OpVT == MVT::f16) {
+  if (OpVT == MVT::f16)
     return ISD::FP16_TO_FP;
-  } else if (RetVT == MVT::f16) {
+  if (RetVT == MVT::f16)
     return ISD::FP_TO_FP16;
-  } else if (OpVT == MVT::bf16) {
+  if (OpVT == MVT::bf16)
     return ISD::BF16_TO_FP;
-  } else if (RetVT == MVT::bf16) {
+  if (RetVT == MVT::bf16)
     return ISD::FP_TO_BF16;
-  }
-
   report_fatal_error("Attempt at an invalid promotion-related conversion");
 }
 
 static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
   if (OpVT == MVT::f16)
     return ISD::STRICT_FP16_TO_FP;
-
   if (RetVT == MVT::f16)
     return ISD::STRICT_FP_TO_FP16;
-
   if (OpVT == MVT::bf16)
     return ISD::STRICT_BF16_TO_FP;
-
   if (RetVT == MVT::bf16)
     return ISD::STRICT_FP_TO_BF16;
-
   report_fatal_error("Attempt at an invalid promotion-related conversion");
 }
 
@@ -3138,6 +3132,8 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
     break;
   case ISD::SELECT:      R = SoftPromoteHalfRes_SELECT(N); break;
   case ISD::SELECT_CC:   R = SoftPromoteHalfRes_SELECT_CC(N); break;
+  case ISD::STRICT_SINT_TO_FP:
+  case ISD::STRICT_UINT_TO_FP:
   case ISD::SINT_TO_FP:
   case ISD::UINT_TO_FP:  R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
   case ISD::UNDEF:       R = SoftPromoteHalfRes_UNDEF(N); break;
@@ -3288,19 +3284,13 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
 
 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
   EVT RVT = N->getValueType(0);
-  EVT SVT = N->getOperand(0).getValueType();
+  bool IsStrict = N->isStrictFPOpcode();
+  SDValue Op = N->getOperand(IsStrict ? 1 : 0);
+  EVT SVT = Op.getValueType();
 
-  if (N->isStrictFPOpcode()) {
-    // FIXME: assume we only have two f16 variants for now.
-    unsigned Opcode;
-    if (RVT == MVT::f16)
-      Opcode = ISD::STRICT_FP_TO_FP16;
-    else if (RVT == MVT::bf16)
-      Opcode = ISD::STRICT_FP_TO_BF16;
-    else
-      llvm_unreachable("unknown half type");
-    SDValue Res = DAG.getNode(Opcode, SDLoc(N), {MVT::i16, MVT::Other},
-                              {N->getOperand(0), N->getOperand(1)});
+  if (IsStrict) {
+    SDValue Res = DAG.getNode(GetPromotionOpcodeStrict(SVT, RVT), SDLoc(N),
+                              {MVT::i16, MVT::Other}, {N->getOperand(0), Op});
     ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
     return Res;
   }
@@ -3359,6 +3349,15 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
   SDLoc dl(N);
 
+  if (N->isStrictFPOpcode()) {
+    SDValue Op = DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other},
+                             {N->getOperand(0), N->getOperand(1)});
+    Op = DAG.getNode(GetPromotionOpcodeStrict(NVT, OVT), dl,
+                     {MVT::i16, MVT::Other}, {Op.getValue(1), Op});
+    ReplaceValueWith(SDValue(N, 1), Op.getValue(1));
+    return Op;
+  }
+
   SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
 
   // Round the value to the softened type.
@@ -3447,6 +3446,8 @@ bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
     Res = SoftPromoteHalfOp_FAKE_USE(N, OpNo);
     break;
   case ISD::FCOPYSIGN:  Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
+  case ISD::STRICT_FP_TO_SINT:
+  case ISD::STRICT_FP_TO_UINT:
   case ISD::FP_TO_SINT:
   case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
   case ISD::FP_TO_SINT_SAT:
@@ -3517,16 +3518,8 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
   Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
 
   if (IsStrict) {
-    unsigned Opcode;
-    if (SVT == MVT::f16)
-      Opcode = ISD::STRICT_FP16_TO_FP;
-    else if (SVT == MVT::bf16)
-      Opcode = ISD::STRICT_BF16_TO_FP;
-    else
-      llvm_unreachable("unknown half type");
-    SDValue Res =
-        DAG.getNode(Opcode, SDLoc(N), {N->getValueType(0), MVT::Other},
-                    {N->getOperand(0), Op});
+    SDValue Res = DAG.getNode(GetPromotionOpcodeStrict(SVT, RVT), SDLoc(N),
+                              {RVT, MVT::Other}, {N->getOperand(0), Op});
     ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
     ReplaceValueWith(SDValue(N, 0), Res);
     return SDValue();
@@ -3537,17 +3530,26 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
 
 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
   EVT RVT = N->getValueType(0);
-  SDValue Op = N->getOperand(0);
+  bool IsStrict = N->isStrictFPOpcode();
+  SDValue Op = N->getOperand(IsStrict ? 1 : 0);
   EVT SVT = Op.getValueType();
   SDLoc dl(N);
 
-  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
-
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
   Op = GetSoftPromotedHalf(Op);
 
-  SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
+  if (IsStrict) {
+    Op = DAG.getNode(GetPromotionOpcodeStrict(SVT, RVT), dl, {NVT, MVT::Other},
+                     {N->getOperand(0), Op});
+    Op = DAG.getNode(N->getOpcode(), dl, {RVT, MVT::Other},
+                     {Op.getValue(1), Op});
+    ReplaceValueWith(SDValue(N, 1), Op.getValue(1));
+    ReplaceValueWith(SDValue(N, 0), Op);
+    return SDValue();
+  }
 
-  return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
+  SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
+  return DAG.getNode(N->getOpcode(), dl, RVT, Res);
 }
 
 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 29dae4e27c7689..3dc5affacc5a76 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -146,7 +146,7 @@ class VectorLegalizer {
   SDValue ExpandFCOPYSIGN(SDNode *Node);
   void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
   void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
-  void ExpandBITREVERSE(SDNode *Node, SmallVectorImpl<SDValue> &Results);
+  SDValue ExpandBITREVERSE(SDNode *Node);
   void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
   void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
   void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
@@ -867,8 +867,11 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
       Results.push_back(Node->getOperand(i));
     return;
   case ISD::SIGN_EXTEND_INREG:
-    Results.push_back(ExpandSEXTINREG(Node));
-    return;
+    if (SDValue Expanded = ExpandSEXTINREG(Node)) {
+      Results.push_back(Expanded);
+      return;
+    }
+    break;
   case ISD::ANY_EXTEND_VECTOR_INREG:
     Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(Node));
     return;
@@ -879,17 +882,26 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
     Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(Node));
     return;
   case ISD::BSWAP:
-    Results.push_back(ExpandBSWAP(Node));
-    return;
+    if (SDValue Expanded = ExpandBSWAP(Node)) {
+      Results.push_back(Expanded);
+      return;
+    }
+    break;
   case ISD::VP_BSWAP:
     Results.push_back(TLI.expandVPBSWAP(Node, DAG));
     return;
   case ISD::VSELECT:
-    Results.push_back(ExpandVSELECT(Node));
-    return;
+    if (SDValue Expanded = ExpandVSELECT(Node)) {
+      Results.push_back(Expanded);
+      return;
+    }
+    break;
   case ISD::VP_SELECT:
-    Results.push_back(ExpandVP_SELECT(Node));
-    return;
+    if (SDValue Expanded = ExpandVP_SELECT(Node)) {
+      Results.push_back(Expanded);
+      return;
+    }
+    break;
   case ISD::VP_SREM:
   case ISD::VP_UREM:
     if (SDValue Expanded = ExpandVP_REM(Node)) {
@@ -916,8 +928,11 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
     }
     break;
   case ISD::SELECT:
-    Results.push_back(ExpandSELECT(Node));
-    return;
+    if (SDValue Expanded = ExpandSELECT(Node)) {
+      Results.push_back(Expanded);
+      return;
+    }
+    break;
   case ISD::SELECT_CC: {
     if (Node->getValueType(0).isScalableVector()) {
       EVT CondVT = TLI.getSetCCResultType(
@@ -986,8 +1001,11 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
     }
     break;
   case ISD::BITREVERSE:
-    ExpandBITREVERSE(Node, Results);
-    return;
+    if (SDValue Expanded = ExpandBITREVERSE(Node)) {
+      Results.push_back(Expanded);
+      return;
+    }
+    break;
   case ISD::VP_BITREVERSE:
     if (SDValue Expanded = TLI.expandVPBITREVERSE(Node, DAG)) {
       Results.push_back(Expanded);
@@ -1160,8 +1178,11 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
     ExpandREM(Node, Results);
     return;
   case ISD::VP_MERGE:
-    Results.push_back(ExpandVP_MERGE(Node));
-    return;
+    if (SDValue Expanded = ExpandVP_MERGE(Node)) {
+      Results.push_back(Expanded);
+      return;
+    }
+    break;
   case ISD::FREM:
     if (tryExpandVecMathCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
                              RTLIB::REM_F80, RTLIB::REM_F128,
@@ -1217,7 +1238,7 @@ SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {
       TLI.getOperationAction(VT.isFixedLengthVector() ? ISD::BUILD_VECTOR
                                                       : ISD::SPLAT_VECTOR,
                              VT) == TargetLowering::Expand)
-    return DAG.UnrollVectorOp(Node);
+    return SDValue();
 
   // Generate a mask operand.
   EVT MaskTy = VT.changeVectorElementTypeToInteger();
@@ -1251,7 +1272,7 @@ SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) {
   // Make sure that the SRA and SHL instructions are available.
   if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand ||
       TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand)
-    return DAG.UnrollVectorOp(Node);
+    return SDValue();
 
   SDLoc DL(Node);
   EVT OrigTy = cast<VTSDNode>(Node->getOperand(1))->getVT();
@@ -1396,26 +1417,20 @@ SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) {
       TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT))
     return TLI.expandBSWAP(Node, DAG);
 
-  // Otherwise unroll.
-  return DAG.UnrollVectorOp(Node);
+  // Otherwise let the caller unroll.
+  return SDValue();
 }
 
-void VectorLegalizer::ExpandBITREVERSE(SDNode *Node,
-                                       SmallVectorImpl<SDValue> &Results) {
+SDValue VectorLegalizer::ExpandBITREVERSE(SDNode *Node) {
   EVT VT = Node->getValueType(0);
 
   // We can't unroll or use shuffles for scalable vectors.
-  if (VT.isScalableVector()) {
-    Results.push_back(TLI.expandBITREVERSE(Node, DAG));
-    return;
-  }
+  if (VT.isScalableVector())
+    return TLI.expandBITREVERSE(Node, DAG);
 
   // If we have the scalar operation, it's probably cheaper to unroll it.
-  if (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, VT.getScalarType())) {
-    SDValue Tmp = DAG.UnrollVectorOp(Node);
-    Results.push_back(Tmp);
-    return;
-  }
+  if (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, VT.getScalarType()))
+    return SDValue();
 
   // If the vector element width is a whole number of bytes, test if its legal
   // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte
@@ -1438,8 +1453,7 @@ void VectorLegalizer::ExpandBITREVERSE(SDNode *Node,
                                 BSWAPMask);
       Op = DAG.getNode(ISD::BITREVERSE, DL, ByteVT, Op);
       Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
-      Results.push_back(Op);
-      return;
+      return Op;
     }
   }
 
@@ -1448,14 +1462,11 @@ void VectorLegalizer::ExpandBITREVERSE(SDNode *Node,
   if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
       TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
       TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
-      TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT)) {
-    Results.push_back(TLI.expandBITREVERSE(Node, DAG));
-    return;
-  }
+      TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT))
+    return TLI.expandBITREVERSE(Node, DAG);
 
   // Otherwise unroll.
-  SDValue Tmp = DAG.UnrollVectorOp(Node);
-  Results.push_back(Tmp);
+  return SDValue();
 }
 
 SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
@@ -1476,7 +1487,7 @@ SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
   if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
       TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
       TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand)
-    return DAG.UnrollVectorOp(Node);
+    return SDValue();
 
   // This operation also isn't safe with AND, OR, XOR when the boolean type is
   // 0/1 and the select operands aren't also booleans, as we need an all-ones
@@ -1486,13 +1497,13 @@ SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
   if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
       !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
         Op1.getValueType().getVectorElementType() == MVT::i1))
-    return DAG.UnrollVectorOp(Node);
+    return SDValue();
 
   // If the mask and the type are different sizes, unroll the vector op. This
   // can occur when getSetCCResultType returns something that is different in
   // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
   if (VT.getSizeInBits() != Op1.getValueSizeInBits())
-    return DAG.UnrollVectorOp(Node);
+    return SDValue();
 
   // Bitcast the operands to be the same type as the mask.
   // This is needed when we select between FP types because
@@ -1525,11 +1536,11 @@ SDValue VectorLegalizer::ExpandVP_SELECT(SDNode *Node) {
   if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand ||
       TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand ||
       TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand)
-    return DAG.UnrollVectorOp(Node);
+    return SDValue();
 
   // This operation also isn't safe when the operands aren't also booleans.
   if (Op1.getValueType().getVectorElementType() != MVT::i1)
-    return DAG.UnrollVectorOp(Node);
+    return SDValue();
 
   SDValue Ones = DAG.getAllOnesConstant(DL, VT);
   SDValue NotMask = DAG.getNode(ISD::VP_XOR, DL, VT, Mask, Ones, Ones, EVL);
@@ -1563,13 +1574,13 @@ SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) {
       (!IsFixedLen &&
        (!TLI.isOperationLegalOrCustom(ISD::STEP_VECTOR, EVLVecVT) ||
         !TLI.isOperationLegalOrCustom(ISD::SPLAT_VECTOR, EVLVecVT))))
-    return DAG.UnrollVectorOp(Node);
+    return SDValue();
 
   // If using a SETCC would result in a different type than the mask type,
   // unroll.
   if (TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
                              EVLVecVT) != MaskVT)
-    return DAG.UnrollVectorOp(Node);
+    return SDValue();
 
   SDValue StepVec = DAG.getStepVector(DL, EVLVecVT);
   SDValue SplatEVL = DAG.getSplat(EVLVecVT, DL, EVL);
diff --git a/llvm/lib/Demangle/MicrosoftDemangle.cpp b/llvm/lib/Demangle/MicrosoftDemangle.cpp
index c5835e8c2e989d..aa65f3be29da77 100644
--- a/llvm/lib/Demangle/MicrosoftDemangle.cpp
+++ b/llvm/lib/Demangle/MicrosoftDemangle.cpp
@@ -2035,6 +2035,10 @@ Demangler::demanglePrimitiveType(std::string_view &MangledName) {
       return Arena.alloc<PrimitiveTypeNode>(PrimitiveKind::Char16);
     case 'U':
       return Arena.alloc<PrimitiveTypeNode>(PrimitiveKind::Char32);
+    case 'P':
+      return Arena.alloc<PrimitiveTypeNode>(PrimitiveKind::Auto);
+    case 'T':
+      return Arena.alloc<PrimitiveTypeNode>(PrimitiveKind::DecltypeAuto);
     }
     break;
   }
diff --git a/llvm/lib/Demangle/MicrosoftDemangleNodes.cpp b/llvm/lib/Demangle/MicrosoftDemangleNodes.cpp
index 9a9c34ec6d348c..ec6e67058c6836 100644
--- a/llvm/lib/Demangle/MicrosoftDemangleNodes.cpp
+++ b/llvm/lib/Demangle/MicrosoftDemangleNodes.cpp
@@ -149,6 +149,8 @@ void PrimitiveTypeNode::outputPre(OutputBuffer &OB, OutputFlags Flags) const {
     OUTPUT_ENUM_CLASS_VALUE(PrimitiveKind, Double, "double");
     OUTPUT_ENUM_CLASS_VALUE(PrimitiveKind, Ldouble, "long double");
     OUTPUT_ENUM_CLASS_VALUE(PrimitiveKind, Nullptr, "std::nullptr_t");
+    OUTPUT_ENUM_CLASS_VALUE(PrimitiveKind, Auto, "auto");
+    OUTPUT_ENUM_CLASS_VALUE(PrimitiveKind, DecltypeAuto, "decltype(auto)");
   }
   outputQualifiers(OB, Quals, true, false);
 }
diff --git a/llvm/lib/SandboxIR/SandboxIR.cpp b/llvm/lib/SandboxIR/SandboxIR.cpp
index 03d3e9e607f01a..f8faef5a386a93 100644
--- a/llvm/lib/SandboxIR/SandboxIR.cpp
+++ b/llvm/lib/SandboxIR/SandboxIR.cpp
@@ -306,6 +306,11 @@ BBIterator &BBIterator::operator--() {
   return *this;
 }
 
+BasicBlock *BBIterator::getNodeParent() const {
+  llvm::BasicBlock *Parent = const_cast<BBIterator *>(this)->It.getNodeParent();
+  return cast<BasicBlock>(Ctx->getValue(Parent));
+}
+
 const char *Instruction::getOpcodeName(Opcode Opc) {
   switch (Opc) {
 #define OP(OPC)                                                                \
@@ -2532,6 +2537,10 @@ template class GlobalWithNodeAPI<GlobalIFunc, llvm::GlobalIFunc, GlobalObject,
                                  llvm::GlobalObject>;
 template class GlobalWithNodeAPI<Function, llvm::Function, GlobalObject,
                                  llvm::GlobalObject>;
+template class GlobalWithNodeAPI<GlobalVariable, llvm::GlobalVariable,
+                                 GlobalObject, llvm::GlobalObject>;
+template class GlobalWithNodeAPI<GlobalAlias, llvm::GlobalAlias, GlobalValue,
+                                 llvm::GlobalValue>;
 } // namespace llvm::sandboxir
 
 void GlobalIFunc::setResolver(Constant *Resolver) {
@@ -2552,6 +2561,57 @@ Function *GlobalIFunc::getResolverFunction() {
       cast<llvm::GlobalIFunc>(Val)->getResolverFunction()));
 }
 
+GlobalVariable &
+GlobalVariable::LLVMGVToGV::operator()(llvm::GlobalVariable &LLVMGV) const {
+  return cast<GlobalVariable>(*Ctx.getValue(&LLVMGV));
+}
+
+Constant *GlobalVariable::getInitializer() const {
+  return Ctx.getOrCreateConstant(
+      cast<llvm::GlobalVariable>(Val)->getInitializer());
+}
+
+void GlobalVariable::setInitializer(Constant *InitVal) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&GlobalVariable::getInitializer,
+                                       &GlobalVariable::setInitializer>>(this);
+  cast<llvm::GlobalVariable>(Val)->setInitializer(
+      cast<llvm::Constant>(InitVal->Val));
+}
+
+void GlobalVariable::setConstant(bool V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&GlobalVariable::isConstant,
+                                       &GlobalVariable::setConstant>>(this);
+  cast<llvm::GlobalVariable>(Val)->setConstant(V);
+}
+
+void GlobalVariable::setExternallyInitialized(bool V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&GlobalVariable::isExternallyInitialized,
+                        &GlobalVariable::setExternallyInitialized>>(this);
+  cast<llvm::GlobalVariable>(Val)->setExternallyInitialized(V);
+}
+
+void GlobalAlias::setAliasee(Constant *Aliasee) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&GlobalAlias::getAliasee, &GlobalAlias::setAliasee>>(
+          this);
+  cast<llvm::GlobalAlias>(Val)->setAliasee(cast<llvm::Constant>(Aliasee->Val));
+}
+
+Constant *GlobalAlias::getAliasee() const {
+  return cast<Constant>(
+      Ctx.getOrCreateConstant(cast<llvm::GlobalAlias>(Val)->getAliasee()));
+}
+
+const GlobalObject *GlobalAlias::getAliaseeObject() const {
+  return cast<GlobalObject>(Ctx.getOrCreateConstant(
+      cast<llvm::GlobalAlias>(Val)->getAliaseeObject()));
+}
+
 void GlobalValue::setUnnamedAddr(UnnamedAddr V) {
   Ctx.getTracker()
       .emplaceIfTracking<GenericSetter<&GlobalValue::getUnnamedAddr,
@@ -2764,6 +2824,14 @@ Value *Context::getOrCreateValueInternal(llvm::Value *LLVMV, llvm::User *U) {
       It->second = std::unique_ptr<GlobalIFunc>(
           new GlobalIFunc(cast<llvm::GlobalIFunc>(C), *this));
       break;
+    case llvm::Value::GlobalVariableVal:
+      It->second = std::unique_ptr<GlobalVariable>(
+          new GlobalVariable(cast<llvm::GlobalVariable>(C), *this));
+      break;
+    case llvm::Value::GlobalAliasVal:
+      It->second = std::unique_ptr<GlobalAlias>(
+          new GlobalAlias(cast<llvm::GlobalAlias>(C), *this));
+      break;
     default:
       It->second = std::unique_ptr<Constant>(new Constant(C, *this));
       break;
diff --git a/llvm/lib/Support/FormatVariadic.cpp b/llvm/lib/Support/FormatVariadic.cpp
index 3240b1c0f4e413..7eb10887940513 100644
--- a/llvm/lib/Support/FormatVariadic.cpp
+++ b/llvm/lib/Support/FormatVariadic.cpp
@@ -148,6 +148,7 @@ formatv_object_base::parseFormatString(StringRef Fmt, size_t NumArgs,
 #if ENABLE_VALIDATION
   const StringRef SavedFmtStr = Fmt;
   unsigned NumExpectedArgs = 0;
+  bool HasExplicitIndex = false;
 #endif
 
   while (!Fmt.empty()) {
@@ -155,14 +156,17 @@ formatv_object_base::parseFormatString(StringRef Fmt, size_t NumArgs,
     std::tie(I, Fmt) = splitLiteralAndReplacement(Fmt);
     if (!I)
       continue;
-    if (I->Index == ~0U)
-      I->Index = NextAutomaticIndex++;
-
-    Replacements.emplace_back(*I);
+    if (I->Type == ReplacementType::Format) {
+      if (I->Index == ~0U)
+        I->Index = NextAutomaticIndex++;
 #if ENABLE_VALIDATION
-    if (I->Type == ReplacementType::Format)
+      else
+        HasExplicitIndex = true;
       NumExpectedArgs = std::max(NumExpectedArgs, I->Index + 1);
 #endif
+    }
+
+    Replacements.emplace_back(*I);
   }
 
 #if ENABLE_VALIDATION
@@ -208,9 +212,8 @@ formatv_object_base::parseFormatString(StringRef Fmt, size_t NumArgs,
     return getErrorReplacements("Replacement indices have holes");
   }
 
-  // If we had automatic numbering of replacement indices, verify that all
-  // indices used automatic numbering.
-  if (NextAutomaticIndex != 0 && NextAutomaticIndex != Count) {
+  // Fail validation if we see both automatic index and explicit index.
+  if (NextAutomaticIndex != 0 && HasExplicitIndex) {
     errs() << formatv(
         "Cannot mix automatic and explicit indices for format string '{}'\n",
         SavedFmtStr);
diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index 8031f19b63239e..1a9e5899892a1b 100644
--- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -1321,6 +1321,17 @@ AArch64LoadStoreOpt::promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
               .add(StMO)
               .addImm(AndMaskEncoded)
               .setMIFlags(LoadI->getFlags());
+    } else if (IsStoreXReg && Imms == 31) {
+      // Use the 32 bit variant of UBFM if it's the LSR alias of the
+      // instruction.
+      assert(Immr <= Imms && "Expected LSR alias of UBFM");
+      BitExtMI = BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
+                         TII->get(AArch64::UBFMWri),
+                         TRI->getSubReg(DestReg, AArch64::sub_32))
+                     .addReg(TRI->getSubReg(StRt, AArch64::sub_32))
+                     .addImm(Immr)
+                     .addImm(Imms)
+                     .setMIFlags(LoadI->getFlags());
     } else {
       BitExtMI =
           BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
diff --git a/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp b/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp
index 6a18c4516418bd..94a49bde74fd62 100644
--- a/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp
+++ b/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp
@@ -64,6 +64,9 @@
 // 8. Remove redundant CSELs that select between identical registers, by
 //    replacing them with unconditional moves.
 //
+// 9. Replace UBFMXri with UBFMWri if the instruction is equivalent to a 32 bit
+//    LSR or LSL alias of UBFM.
+//
 //===----------------------------------------------------------------------===//
 
 #include "AArch64ExpandImm.h"
@@ -132,6 +135,7 @@ struct AArch64MIPeepholeOpt : public MachineFunctionPass {
   bool visitINSviGPR(MachineInstr &MI, unsigned Opc);
   bool visitINSvi64lane(MachineInstr &MI);
   bool visitFMOVDr(MachineInstr &MI);
+  bool visitUBFMXri(MachineInstr &MI);
   bool visitCopy(MachineInstr &MI);
   bool runOnMachineFunction(MachineFunction &MF) override;
 
@@ -715,6 +719,57 @@ bool AArch64MIPeepholeOpt::visitFMOVDr(MachineInstr &MI) {
   return true;
 }
 
+bool AArch64MIPeepholeOpt::visitUBFMXri(MachineInstr &MI) {
+  // Check if the instruction is equivalent to a 32 bit LSR or LSL alias of
+  // UBFM, and replace the UBFMXri instruction with its 32 bit variant, UBFMWri.
+  int64_t Immr = MI.getOperand(2).getImm();
+  int64_t Imms = MI.getOperand(3).getImm();
+
+  bool IsLSR = Imms == 31 && Immr <= Imms;
+  bool IsLSL = Immr == Imms + 33;
+  if (!IsLSR && !IsLSL)
+    return false;
+
+  if (IsLSL) {
+    Immr -= 32;
+  }
+
+  const TargetRegisterClass *DstRC64 =
+      TII->getRegClass(TII->get(MI.getOpcode()), 0, TRI, *MI.getMF());
+  const TargetRegisterClass *DstRC32 =
+      TRI->getSubRegisterClass(DstRC64, AArch64::sub_32);
+  assert(DstRC32 && "Destination register class of UBFMXri doesn't have a "
+                    "sub_32 subregister class");
+
+  const TargetRegisterClass *SrcRC64 =
+      TII->getRegClass(TII->get(MI.getOpcode()), 1, TRI, *MI.getMF());
+  const TargetRegisterClass *SrcRC32 =
+      TRI->getSubRegisterClass(SrcRC64, AArch64::sub_32);
+  assert(SrcRC32 && "Source register class of UBFMXri doesn't have a sub_32 "
+                    "subregister class");
+
+  Register DstReg64 = MI.getOperand(0).getReg();
+  Register DstReg32 = MRI->createVirtualRegister(DstRC32);
+  Register SrcReg64 = MI.getOperand(1).getReg();
+  Register SrcReg32 = MRI->createVirtualRegister(SrcRC32);
+
+  BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(AArch64::COPY),
+          SrcReg32)
+      .addReg(SrcReg64, 0, AArch64::sub_32);
+  BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(AArch64::UBFMWri),
+          DstReg32)
+      .addReg(SrcReg32)
+      .addImm(Immr)
+      .addImm(Imms);
+  BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
+          TII->get(AArch64::SUBREG_TO_REG), DstReg64)
+      .addImm(0)
+      .addReg(DstReg32)
+      .addImm(AArch64::sub_32);
+  MI.eraseFromParent();
+  return true;
+}
+
 // Across a basic-block we might have in i32 extract from a value that only
 // operates on upper bits (for example a sxtw). We can replace the COPY with a
 // new version skipping the sxtw.
@@ -865,6 +920,9 @@ bool AArch64MIPeepholeOpt::runOnMachineFunction(MachineFunction &MF) {
       case AArch64::FMOVDr:
         Changed |= visitFMOVDr(MI);
         break;
+      case AArch64::UBFMXri:
+        Changed |= visitUBFMXri(MI);
+        break;
       case AArch64::COPY:
         Changed |= visitCopy(MI);
         break;
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 7b3b73b83bd377..65e0a1fc49d462 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -218,6 +218,16 @@ def AArch64fsub_p_contract : PatFrag<(ops node:$op1, node:$op2, node:$op3),
   return N->getFlags().hasAllowContract();
 }]>;
 
+def AArch64fminnm_p_nnan : PatFrag<(ops node:$op1, node:$op2, node:$op3),
+                                   (AArch64fminnm_p node:$op1, node:$op2, node:$op3), [{
+  return N->getFlags().hasNoNaNs();
+}]>;
+
+def AArch64fmaxnm_p_nnan : PatFrag<(ops node:$op1, node:$op2, node:$op3),
+                                   (AArch64fmaxnm_p node:$op1, node:$op2, node:$op3), [{
+  return N->getFlags().hasNoNaNs();
+}]>;
+
 def SDT_AArch64Arith_Imm : SDTypeProfile<1, 3, [
   SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVT<3,i32>,
   SDTCVecEltisVT<1,i1>, SDTCisSameAs<0,2>
@@ -469,6 +479,24 @@ def AArch64fmls_m1 : PatFrags<(ops node:$pg, node:$za, node:$zn, node:$zm),
                                (vselect node:$pg, (AArch64fsub_p_contract (SVEAllActive), node:$za, (AArch64fmul_p_oneuse (SVEAllActive), node:$zn, node:$zm)), node:$za),
                                (vselect node:$pg, (AArch64fma_p (SVEAllActive), (AArch64fneg_mt (SVEAllActive), node:$zn, (undef)), node:$zm, node:$za), node:$za)]>;
 
+def AArch64famin_p : PatFrags<(ops node:$pred, node:$op1, node:$op2),
+                              [(int_aarch64_sve_famin_u node:$pred, node:$op1, node:$op2),
+                               (AArch64fmin_p node:$pred,
+                                  (AArch64fabs_mt node:$pred, node:$op1, undef),
+                                  (AArch64fabs_mt node:$pred, node:$op2, undef)),
+                               (AArch64fminnm_p_nnan node:$pred,
+                                  (AArch64fabs_mt node:$pred, node:$op1, undef),
+                                  (AArch64fabs_mt node:$pred, node:$op2, undef))]>;
+
+def AArch64famax_p : PatFrags<(ops node:$pred, node:$op1, node:$op2),
+                              [(int_aarch64_sve_famax_u node:$pred, node:$op1, node:$op2),
+                               (AArch64fmax_p node:$pred,
+                                  (AArch64fabs_mt node:$pred, node:$op1, undef),
+                                  (AArch64fabs_mt node:$pred, node:$op2, undef)),
+                               (AArch64fmaxnm_p_nnan node:$pred,
+                                  (AArch64fabs_mt node:$pred, node:$op1, undef),
+                                  (AArch64fabs_mt node:$pred, node:$op2, undef))]>;
+
 def AArch64add_m1 : VSelectUnpredOrPassthruPatFrags<int_aarch64_sve_add, add>;
 def AArch64sub_m1 : VSelectUnpredOrPassthruPatFrags<int_aarch64_sve_sub, sub>;
 def AArch64mul_m1 : VSelectCommPredOrPassthruPatFrags<int_aarch64_sve_mul, AArch64mul_p>;
@@ -4184,8 +4212,8 @@ let Predicates = [HasSVE2orSME2, HasFAMINMAX] in {
 defm FAMIN_ZPmZ : sve_fp_2op_p_zds<0b1111, "famin", "FAMIN_ZPZZ", int_aarch64_sve_famin, DestructiveBinaryComm>;
 defm FAMAX_ZPmZ : sve_fp_2op_p_zds<0b1110, "famax", "FAMAX_ZPZZ", int_aarch64_sve_famax, DestructiveBinaryComm>;
 
-defm FAMAX_ZPZZ : sve_fp_bin_pred_hfd<int_aarch64_sve_famax_u>;
-defm FAMIN_ZPZZ : sve_fp_bin_pred_hfd<int_aarch64_sve_famin_u>;
+defm FAMAX_ZPZZ : sve_fp_bin_pred_hfd<AArch64famax_p>;
+defm FAMIN_ZPZZ : sve_fp_bin_pred_hfd<AArch64famin_p>;
 } // End HasSVE2orSME2, HasFAMINMAX
 
 let Predicates = [HasSSVE_FP8FMA] in {
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index b855b6b3107029..fd9fe1196b7853 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -304,7 +304,8 @@ class WaitcntBrackets {
 
   RegInterval getRegInterval(const MachineInstr *MI,
                              const MachineRegisterInfo *MRI,
-                             const SIRegisterInfo *TRI, unsigned OpNo) const;
+                             const SIRegisterInfo *TRI,
+                             const MachineOperand &Op) const;
 
   bool counterOutOfOrder(InstCounterType T) const;
   void simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const;
@@ -405,9 +406,9 @@ class WaitcntBrackets {
     }
   }
 
-  void setExpScore(const MachineInstr *MI, const SIInstrInfo *TII,
-                   const SIRegisterInfo *TRI, const MachineRegisterInfo *MRI,
-                   unsigned OpNo, unsigned Val);
+  void setExpScore(const MachineInstr *MI, const SIRegisterInfo *TRI,
+                   const MachineRegisterInfo *MRI, const MachineOperand &Op,
+                   unsigned Val);
 
   const GCNSubtarget *ST = nullptr;
   InstCounterType MaxCounter = NUM_EXTENDED_INST_CNTS;
@@ -734,8 +735,7 @@ class SIInsertWaitcnts : public MachineFunctionPass {
 RegInterval WaitcntBrackets::getRegInterval(const MachineInstr *MI,
                                             const MachineRegisterInfo *MRI,
                                             const SIRegisterInfo *TRI,
-                                            unsigned OpNo) const {
-  const MachineOperand &Op = MI->getOperand(OpNo);
+                                            const MachineOperand &Op) const {
   if (!TRI->isInAllocatableClass(Op.getReg()))
     return {-1, -1};
 
@@ -773,12 +773,11 @@ RegInterval WaitcntBrackets::getRegInterval(const MachineInstr *MI,
 }
 
 void WaitcntBrackets::setExpScore(const MachineInstr *MI,
-                                  const SIInstrInfo *TII,
                                   const SIRegisterInfo *TRI,
-                                  const MachineRegisterInfo *MRI, unsigned OpNo,
-                                  unsigned Val) {
-  RegInterval Interval = getRegInterval(MI, MRI, TRI, OpNo);
-  assert(TRI->isVectorRegister(*MRI, MI->getOperand(OpNo).getReg()));
+                                  const MachineRegisterInfo *MRI,
+                                  const MachineOperand &Op, unsigned Val) {
+  RegInterval Interval = getRegInterval(MI, MRI, TRI, Op);
+  assert(TRI->isVectorRegister(*MRI, Op.getReg()));
   for (int RegNo = Interval.first; RegNo < Interval.second; ++RegNo) {
     setRegScore(RegNo, EXP_CNT, Val);
   }
@@ -804,79 +803,60 @@ void WaitcntBrackets::updateByEvent(const SIInstrInfo *TII,
     // Put score on the source vgprs. If this is a store, just use those
     // specific register(s).
     if (TII->isDS(Inst) && (Inst.mayStore() || Inst.mayLoad())) {
-      int AddrOpIdx =
-          AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::addr);
       // All GDS operations must protect their address register (same as
       // export.)
-      if (AddrOpIdx != -1) {
-        setExpScore(&Inst, TII, TRI, MRI, AddrOpIdx, CurrScore);
-      }
+      if (const auto *AddrOp = TII->getNamedOperand(Inst, AMDGPU::OpName::addr))
+        setExpScore(&Inst, TRI, MRI, *AddrOp, CurrScore);
 
       if (Inst.mayStore()) {
-        if (AMDGPU::hasNamedOperand(Inst.getOpcode(), AMDGPU::OpName::data0)) {
-          setExpScore(
-              &Inst, TII, TRI, MRI,
-              AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data0),
-              CurrScore);
-        }
-        if (AMDGPU::hasNamedOperand(Inst.getOpcode(), AMDGPU::OpName::data1)) {
-          setExpScore(&Inst, TII, TRI, MRI,
-                      AMDGPU::getNamedOperandIdx(Inst.getOpcode(),
-                                                 AMDGPU::OpName::data1),
-                      CurrScore);
-        }
+        if (const auto *Data0 =
+                TII->getNamedOperand(Inst, AMDGPU::OpName::data0))
+          setExpScore(&Inst, TRI, MRI, *Data0, CurrScore);
+        if (const auto *Data1 =
+                TII->getNamedOperand(Inst, AMDGPU::OpName::data1))
+          setExpScore(&Inst, TRI, MRI, *Data1, CurrScore);
       } else if (SIInstrInfo::isAtomicRet(Inst) && !SIInstrInfo::isGWS(Inst) &&
                  Inst.getOpcode() != AMDGPU::DS_APPEND &&
                  Inst.getOpcode() != AMDGPU::DS_CONSUME &&
                  Inst.getOpcode() != AMDGPU::DS_ORDERED_COUNT) {
-        for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
-          const MachineOperand &Op = Inst.getOperand(I);
-          if (Op.isReg() && !Op.isDef() &&
-              TRI->isVectorRegister(*MRI, Op.getReg())) {
-            setExpScore(&Inst, TII, TRI, MRI, I, CurrScore);
-          }
+        for (const MachineOperand &Op : Inst.all_uses()) {
+          if (Op.isReg() && TRI->isVectorRegister(*MRI, Op.getReg()))
+            setExpScore(&Inst, TRI, MRI, Op, CurrScore);
         }
       }
     } else if (TII->isFLAT(Inst)) {
       if (Inst.mayStore()) {
-        setExpScore(
-            &Inst, TII, TRI, MRI,
-            AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data),
-            CurrScore);
+        setExpScore(&Inst, TRI, MRI,
+                    *TII->getNamedOperand(Inst, AMDGPU::OpName::data),
+                    CurrScore);
       } else if (SIInstrInfo::isAtomicRet(Inst)) {
-        setExpScore(
-            &Inst, TII, TRI, MRI,
-            AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data),
-            CurrScore);
+        setExpScore(&Inst, TRI, MRI,
+                    *TII->getNamedOperand(Inst, AMDGPU::OpName::data),
+                    CurrScore);
       }
     } else if (TII->isMIMG(Inst)) {
       if (Inst.mayStore()) {
-        setExpScore(&Inst, TII, TRI, MRI, 0, CurrScore);
+        setExpScore(&Inst, TRI, MRI, Inst.getOperand(0), CurrScore);
       } else if (SIInstrInfo::isAtomicRet(Inst)) {
-        setExpScore(
-            &Inst, TII, TRI, MRI,
-            AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data),
-            CurrScore);
+        setExpScore(&Inst, TRI, MRI,
+                    *TII->getNamedOperand(Inst, AMDGPU::OpName::data),
+                    CurrScore);
       }
     } else if (TII->isMTBUF(Inst)) {
-      if (Inst.mayStore()) {
-        setExpScore(&Inst, TII, TRI, MRI, 0, CurrScore);
-      }
+      if (Inst.mayStore())
+        setExpScore(&Inst, TRI, MRI, Inst.getOperand(0), CurrScore);
     } else if (TII->isMUBUF(Inst)) {
       if (Inst.mayStore()) {
-        setExpScore(&Inst, TII, TRI, MRI, 0, CurrScore);
+        setExpScore(&Inst, TRI, MRI, Inst.getOperand(0), CurrScore);
       } else if (SIInstrInfo::isAtomicRet(Inst)) {
-        setExpScore(
-            &Inst, TII, TRI, MRI,
-            AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data),
-            CurrScore);
+        setExpScore(&Inst, TRI, MRI,
+                    *TII->getNamedOperand(Inst, AMDGPU::OpName::data),
+                    CurrScore);
       }
     } else if (TII->isLDSDIR(Inst)) {
       // LDSDIR instructions attach the score to the destination.
-      setExpScore(
-          &Inst, TII, TRI, MRI,
-          AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::vdst),
-          CurrScore);
+      setExpScore(&Inst, TRI, MRI,
+                  *TII->getNamedOperand(Inst, AMDGPU::OpName::vdst), CurrScore);
     } else {
       if (TII->isEXP(Inst)) {
         // For export the destination registers are really temps that
@@ -891,12 +871,9 @@ void WaitcntBrackets::updateByEvent(const SIInstrInfo *TII,
           }
         }
       }
-      for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
-        MachineOperand &MO = Inst.getOperand(I);
-        if (MO.isReg() && !MO.isDef() &&
-            TRI->isVectorRegister(*MRI, MO.getReg())) {
-          setExpScore(&Inst, TII, TRI, MRI, I, CurrScore);
-        }
+      for (const MachineOperand &Op : Inst.all_uses()) {
+        if (Op.isReg() && TRI->isVectorRegister(*MRI, Op.getReg()))
+          setExpScore(&Inst, TRI, MRI, Op, CurrScore);
       }
     }
   } else /* LGKM_CNT || EXP_CNT || VS_CNT || NUM_INST_CNTS */ {
@@ -907,14 +884,10 @@ void WaitcntBrackets::updateByEvent(const SIInstrInfo *TII,
     // artificial dependency, while these are there only for register liveness
     // accounting purposes.
     //
-    // Special cases where implicit register defs and uses exists, such as
-    // M0, FLAT_SCR or VCC, but the wait will be generated earlier in the
-    // generateWaitcntInstBefore() if that was loaded from memory.
-    for (unsigned I = 0, E = Inst.getNumExplicitOperands(); I != E; ++I) {
-      auto &Op = Inst.getOperand(I);
-      if (!Op.isReg() || !Op.isDef())
-        continue;
-      RegInterval Interval = getRegInterval(&Inst, MRI, TRI, I);
+    // Special cases where implicit register defs exists, such as M0 or VCC,
+    // but none with memory instructions.
+    for (const MachineOperand &Op : Inst.defs()) {
+      RegInterval Interval = getRegInterval(&Inst, MRI, TRI, Op);
       if (T == LOAD_CNT || T == SAMPLE_CNT || T == BVH_CNT) {
         if (Interval.first >= NUM_ALL_VGPRS)
           continue;
@@ -1692,22 +1665,19 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
       // load). We also need to check WAW dependency with saved PC.
       Wait = AMDGPU::Waitcnt();
 
-      int CallAddrOpIdx =
-          AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
-
-      if (MI.getOperand(CallAddrOpIdx).isReg()) {
+      const auto &CallAddrOp = *TII->getNamedOperand(MI, AMDGPU::OpName::src0);
+      if (CallAddrOp.isReg()) {
         RegInterval CallAddrOpInterval =
-            ScoreBrackets.getRegInterval(&MI, MRI, TRI, CallAddrOpIdx);
+            ScoreBrackets.getRegInterval(&MI, MRI, TRI, CallAddrOp);
 
         for (int RegNo = CallAddrOpInterval.first;
              RegNo < CallAddrOpInterval.second; ++RegNo)
           ScoreBrackets.determineWait(SmemAccessCounter, RegNo, Wait);
 
-        int RtnAddrOpIdx =
-          AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::dst);
-        if (RtnAddrOpIdx != -1) {
+        if (const auto *RtnAddrOp =
+                TII->getNamedOperand(MI, AMDGPU::OpName::dst)) {
           RegInterval RtnAddrOpInterval =
-              ScoreBrackets.getRegInterval(&MI, MRI, TRI, RtnAddrOpIdx);
+              ScoreBrackets.getRegInterval(&MI, MRI, TRI, *RtnAddrOp);
 
           for (int RegNo = RtnAddrOpInterval.first;
                RegNo < RtnAddrOpInterval.second; ++RegNo)
@@ -1769,8 +1739,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
       }
 
       // Loop over use and def operands.
-      for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
-        MachineOperand &Op = MI.getOperand(I);
+      for (const MachineOperand &Op : MI.operands()) {
         if (!Op.isReg())
           continue;
 
@@ -1778,7 +1747,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
         if (Op.isTied() && Op.isUse() && TII->doesNotReadTiedSource(MI))
           continue;
 
-        RegInterval Interval = ScoreBrackets.getRegInterval(&MI, MRI, TRI, I);
+        RegInterval Interval = ScoreBrackets.getRegInterval(&MI, MRI, TRI, Op);
 
         const bool IsVGPR = TRI->isVectorRegister(*MRI, Op.getReg());
         for (int RegNo = Interval.first; RegNo < Interval.second; ++RegNo) {
@@ -2357,34 +2326,35 @@ bool SIInsertWaitcnts::shouldFlushVmCnt(MachineLoop *ML,
         if (MI.mayStore())
           HasVMemStore = true;
       }
-      for (unsigned I = 0; I < MI.getNumOperands(); I++) {
-        MachineOperand &Op = MI.getOperand(I);
+      for (const MachineOperand &Op : MI.all_uses()) {
         if (!Op.isReg() || !TRI->isVectorRegister(*MRI, Op.getReg()))
           continue;
-        RegInterval Interval = Brackets.getRegInterval(&MI, MRI, TRI, I);
+        RegInterval Interval = Brackets.getRegInterval(&MI, MRI, TRI, Op);
         // Vgpr use
-        if (Op.isUse()) {
-          for (int RegNo = Interval.first; RegNo < Interval.second; ++RegNo) {
-            // If we find a register that is loaded inside the loop, 1. and 2.
-            // are invalidated and we can exit.
-            if (VgprDef.contains(RegNo))
-              return false;
-            VgprUse.insert(RegNo);
-            // If at least one of Op's registers is in the score brackets, the
-            // value is likely loaded outside of the loop.
-            if (Brackets.getRegScore(RegNo, LOAD_CNT) >
-                    Brackets.getScoreLB(LOAD_CNT) ||
-                Brackets.getRegScore(RegNo, SAMPLE_CNT) >
-                    Brackets.getScoreLB(SAMPLE_CNT) ||
-                Brackets.getRegScore(RegNo, BVH_CNT) >
-                    Brackets.getScoreLB(BVH_CNT)) {
-              UsesVgprLoadedOutside = true;
-              break;
-            }
+        for (int RegNo = Interval.first; RegNo < Interval.second; ++RegNo) {
+          // If we find a register that is loaded inside the loop, 1. and 2.
+          // are invalidated and we can exit.
+          if (VgprDef.contains(RegNo))
+            return false;
+          VgprUse.insert(RegNo);
+          // If at least one of Op's registers is in the score brackets, the
+          // value is likely loaded outside of the loop.
+          if (Brackets.getRegScore(RegNo, LOAD_CNT) >
+                  Brackets.getScoreLB(LOAD_CNT) ||
+              Brackets.getRegScore(RegNo, SAMPLE_CNT) >
+                  Brackets.getScoreLB(SAMPLE_CNT) ||
+              Brackets.getRegScore(RegNo, BVH_CNT) >
+                  Brackets.getScoreLB(BVH_CNT)) {
+            UsesVgprLoadedOutside = true;
+            break;
           }
         }
-        // VMem load vgpr def
-        else if (isVMEMOrFlatVMEM(MI) && MI.mayLoad() && Op.isDef())
+      }
+
+      // VMem load vgpr def
+      if (isVMEMOrFlatVMEM(MI) && MI.mayLoad()) {
+        for (const MachineOperand &Op : MI.all_defs()) {
+          RegInterval Interval = Brackets.getRegInterval(&MI, MRI, TRI, Op);
           for (int RegNo = Interval.first; RegNo < Interval.second; ++RegNo) {
             // If we find a register that is loaded inside the loop, 1. and 2.
             // are invalidated and we can exit.
@@ -2392,6 +2362,7 @@ bool SIInsertWaitcnts::shouldFlushVmCnt(MachineLoop *ML,
               return false;
             VgprDef.insert(RegNo);
           }
+        }
       }
     }
   }
diff --git a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
index c016b2dd91dc67..193722fa356114 100644
--- a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
+++ b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
@@ -68,9 +68,9 @@ class AVRAsmParser : public MCTargetAsmParser {
   ParseStatus parseMemriOperand(OperandVector &Operands);
 
   bool parseOperand(OperandVector &Operands, bool maybeReg);
-  int parseRegisterName(MCRegister (*matchFn)(StringRef));
-  int parseRegisterName();
-  int parseRegister(bool RestoreOnFailure = false);
+  MCRegister parseRegisterName(MCRegister (*matchFn)(StringRef));
+  MCRegister parseRegisterName();
+  MCRegister parseRegister(bool RestoreOnFailure = false);
   bool tryParseRegisterOperand(OperandVector &Operands);
   bool tryParseExpression(OperandVector &Operands, int64_t offset);
   bool tryParseRelocExpression(OperandVector &Operands);
@@ -79,7 +79,7 @@ class AVRAsmParser : public MCTargetAsmParser {
   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
                                       unsigned Kind) override;
 
-  unsigned toDREG(unsigned Reg, unsigned From = AVR::sub_lo) {
+  MCRegister toDREG(MCRegister Reg, unsigned From = AVR::sub_lo) {
     MCRegisterClass const *Class = &AVRMCRegisterClasses[AVR::DREGSRegClassID];
     return MRI->getMatchingSuperReg(Reg, From, Class);
   }
@@ -113,15 +113,15 @@ class AVROperand : public MCParsedAsmOperand {
 public:
   AVROperand(StringRef Tok, SMLoc const &S)
       : Kind(k_Token), Tok(Tok), Start(S), End(S) {}
-  AVROperand(unsigned Reg, SMLoc const &S, SMLoc const &E)
+  AVROperand(MCRegister Reg, SMLoc const &S, SMLoc const &E)
       : Kind(k_Register), RegImm({Reg, nullptr}), Start(S), End(E) {}
   AVROperand(MCExpr const *Imm, SMLoc const &S, SMLoc const &E)
       : Kind(k_Immediate), RegImm({0, Imm}), Start(S), End(E) {}
-  AVROperand(unsigned Reg, MCExpr const *Imm, SMLoc const &S, SMLoc const &E)
+  AVROperand(MCRegister Reg, MCExpr const *Imm, SMLoc const &S, SMLoc const &E)
       : Kind(k_Memri), RegImm({Reg, Imm}), Start(S), End(E) {}
 
   struct RegisterImmediate {
-    unsigned Reg;
+    MCRegister Reg;
     MCExpr const *Imm;
   };
   union {
@@ -210,9 +210,9 @@ class AVROperand : public MCParsedAsmOperand {
     return std::make_unique<AVROperand>(Str, S);
   }
 
-  static std::unique_ptr<AVROperand> CreateReg(unsigned RegNum, SMLoc S,
+  static std::unique_ptr<AVROperand> CreateReg(MCRegister Reg, SMLoc S,
                                                SMLoc E) {
-    return std::make_unique<AVROperand>(RegNum, S, E);
+    return std::make_unique<AVROperand>(Reg, S, E);
   }
 
   static std::unique_ptr<AVROperand> CreateImm(const MCExpr *Val, SMLoc S,
@@ -221,8 +221,8 @@ class AVROperand : public MCParsedAsmOperand {
   }
 
   static std::unique_ptr<AVROperand>
-  CreateMemri(unsigned RegNum, const MCExpr *Val, SMLoc S, SMLoc E) {
-    return std::make_unique<AVROperand>(RegNum, Val, S, E);
+  CreateMemri(MCRegister Reg, const MCExpr *Val, SMLoc S, SMLoc E) {
+    return std::make_unique<AVROperand>(Reg, Val, S, E);
   }
 
   void makeToken(StringRef Token) {
@@ -230,9 +230,9 @@ class AVROperand : public MCParsedAsmOperand {
     Tok = Token;
   }
 
-  void makeReg(unsigned RegNo) {
+  void makeReg(MCRegister Reg) {
     Kind = k_Register;
-    RegImm = {RegNo, nullptr};
+    RegImm = {Reg, nullptr};
   }
 
   void makeImm(MCExpr const *Ex) {
@@ -240,9 +240,9 @@ class AVROperand : public MCParsedAsmOperand {
     RegImm = {0, Ex};
   }
 
-  void makeMemri(unsigned RegNo, MCExpr const *Imm) {
+  void makeMemri(MCRegister Reg, MCExpr const *Imm) {
     Kind = k_Memri;
-    RegImm = {RegNo, Imm};
+    RegImm = {Reg, Imm};
   }
 
   SMLoc getStartLoc() const override { return Start; }
@@ -346,36 +346,36 @@ bool AVRAsmParser::MatchAndEmitInstruction(SMLoc Loc, unsigned &Opcode,
 
 /// Parses a register name using a given matching function.
 /// Checks for lowercase or uppercase if necessary.
-int AVRAsmParser::parseRegisterName(MCRegister (*matchFn)(StringRef)) {
+MCRegister AVRAsmParser::parseRegisterName(MCRegister (*matchFn)(StringRef)) {
   StringRef Name = Parser.getTok().getString();
 
-  int RegNum = matchFn(Name);
+  MCRegister Reg = matchFn(Name);
 
   // GCC supports case insensitive register names. Some of the AVR registers
   // are all lower case, some are all upper case but non are mixed. We prefer
   // to use the original names in the register definitions. That is why we
   // have to test both upper and lower case here.
-  if (RegNum == AVR::NoRegister) {
-    RegNum = matchFn(Name.lower());
+  if (!Reg) {
+    Reg = matchFn(Name.lower());
   }
-  if (RegNum == AVR::NoRegister) {
-    RegNum = matchFn(Name.upper());
+  if (!Reg) {
+    Reg = matchFn(Name.upper());
   }
 
-  return RegNum;
+  return Reg;
 }
 
-int AVRAsmParser::parseRegisterName() {
-  int RegNum = parseRegisterName(&MatchRegisterName);
+MCRegister AVRAsmParser::parseRegisterName() {
+  MCRegister Reg = parseRegisterName(&MatchRegisterName);
 
-  if (RegNum == AVR::NoRegister)
-    RegNum = parseRegisterName(&MatchRegisterAltName);
+  if (!Reg)
+    Reg = parseRegisterName(&MatchRegisterAltName);
 
-  return RegNum;
+  return Reg;
 }
 
-int AVRAsmParser::parseRegister(bool RestoreOnFailure) {
-  int RegNum = AVR::NoRegister;
+MCRegister AVRAsmParser::parseRegister(bool RestoreOnFailure) {
+  MCRegister Reg;
 
   if (Parser.getTok().is(AsmToken::Identifier)) {
     // Check for register pair syntax
@@ -387,32 +387,32 @@ int AVRAsmParser::parseRegister(bool RestoreOnFailure) {
 
       if (Parser.getTok().is(AsmToken::Identifier)) {
         // Convert lower (even) register to DREG
-        RegNum = toDREG(parseRegisterName());
+        Reg = toDREG(parseRegisterName());
       }
-      if (RegNum == AVR::NoRegister && RestoreOnFailure) {
+      if (!Reg && RestoreOnFailure) {
         getLexer().UnLex(std::move(ColonTok));
         getLexer().UnLex(std::move(HighTok));
       }
     } else {
-      RegNum = parseRegisterName();
+      Reg = parseRegisterName();
     }
   }
-  return RegNum;
+  return Reg;
 }
 
 bool AVRAsmParser::tryParseRegisterOperand(OperandVector &Operands) {
-  int RegNo = parseRegister();
+  MCRegister Reg = parseRegister();
 
-  if (RegNo == AVR::NoRegister)
+  if (!Reg)
     return true;
 
   // Reject R0~R15 on avrtiny.
-  if (AVR::R0 <= RegNo && RegNo <= AVR::R15 &&
+  if (AVR::R0 <= Reg && Reg <= AVR::R15 &&
       STI.hasFeature(AVR::FeatureTinyEncoding))
     return Error(Parser.getTok().getLoc(), "invalid register on avrtiny");
 
   AsmToken const &T = Parser.getTok();
-  Operands.push_back(AVROperand::CreateReg(RegNo, T.getLoc(), T.getEndLoc()));
+  Operands.push_back(AVROperand::CreateReg(Reg, T.getLoc(), T.getEndLoc()));
   Parser.Lex(); // Eat register token.
 
   return false;
@@ -569,13 +569,13 @@ ParseStatus AVRAsmParser::parseMemriOperand(OperandVector &Operands) {
 
   SMLoc E, S;
   MCExpr const *Expression;
-  int RegNo;
+  MCRegister Reg;
 
   // Parse register.
   {
-    RegNo = parseRegister();
+    Reg = parseRegister();
 
-    if (RegNo == AVR::NoRegister)
+    if (!Reg)
       return ParseStatus::Failure;
 
     S = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
@@ -590,7 +590,7 @@ ParseStatus AVRAsmParser::parseMemriOperand(OperandVector &Operands) {
     E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
   }
 
-  Operands.push_back(AVROperand::CreateMemri(RegNo, Expression, S, E));
+  Operands.push_back(AVROperand::CreateMemri(Reg, Expression, S, E));
 
   return ParseStatus::Success;
 }
@@ -761,9 +761,8 @@ unsigned AVRAsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp,
 
       std::ostringstream RegName;
       RegName << "r" << RegNum;
-      RegNum = MatchRegisterName(RegName.str());
-      if (RegNum != AVR::NoRegister) {
-        Op.makeReg(RegNum);
+      if (MCRegister Reg = MatchRegisterName(RegName.str())) {
+        Op.makeReg(Reg);
         if (validateOperandClass(Op, Expected) == Match_Success) {
           return Match_Success;
         }
@@ -776,9 +775,9 @@ unsigned AVRAsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp,
     // If the instruction uses a register pair but we got a single, lower
     // register we perform a "class cast".
     if (isSubclass(Expected, MCK_DREGS)) {
-      unsigned correspondingDREG = toDREG(Op.getReg());
+      MCRegister correspondingDREG = toDREG(Op.getReg());
 
-      if (correspondingDREG != AVR::NoRegister) {
+      if (correspondingDREG) {
         Op.makeReg(correspondingDREG);
         return validateOperandClass(Op, Expected);
       }
diff --git a/llvm/lib/Target/DirectX/CMakeLists.txt b/llvm/lib/Target/DirectX/CMakeLists.txt
index f7ae09957996b5..a9c5d81391b8d7 100644
--- a/llvm/lib/Target/DirectX/CMakeLists.txt
+++ b/llvm/lib/Target/DirectX/CMakeLists.txt
@@ -18,6 +18,7 @@ add_llvm_target(DirectXCodeGen
   DirectXRegisterInfo.cpp
   DirectXSubtarget.cpp
   DirectXTargetMachine.cpp
+  DirectXTargetTransformInfo.cpp
   DXContainerGlobals.cpp
   DXILFinalizeLinkage.cpp
   DXILIntrinsicExpansion.cpp
diff --git a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
new file mode 100644
index 00000000000000..1a59f04b214042
--- /dev/null
+++ b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
@@ -0,0 +1,25 @@
+//===- DirectXTargetTransformInfo.cpp - DirectX TTI ---------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+//===----------------------------------------------------------------------===//
+
+#include "DirectXTargetTransformInfo.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsDirectX.h"
+
+bool llvm::DirectXTTIImpl::isTargetIntrinsicTriviallyScalarizable(
+    Intrinsic::ID ID) const {
+  switch (ID) {
+  case Intrinsic::dx_frac:
+  case Intrinsic::dx_rsqrt:
+    return true;
+  default:
+    return false;
+  }
+}
diff --git a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.h b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.h
index ed98355fad002d..48414549f83495 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.h
+++ b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.h
@@ -34,6 +34,7 @@ class DirectXTTIImpl : public BasicTTIImplBase<DirectXTTIImpl> {
       : BaseT(TM, F.getDataLayout()), ST(TM->getSubtargetImpl(F)),
         TLI(ST->getTargetLowering()) {}
   unsigned getMinVectorRegisterBitWidth() const { return 32; }
+  bool isTargetIntrinsicTriviallyScalarizable(Intrinsic::ID ID) const;
 };
 } // namespace llvm
 
diff --git a/llvm/lib/Target/RISCV/CMakeLists.txt b/llvm/lib/Target/RISCV/CMakeLists.txt
index 124bc239451aed..7d02d630a28199 100644
--- a/llvm/lib/Target/RISCV/CMakeLists.txt
+++ b/llvm/lib/Target/RISCV/CMakeLists.txt
@@ -24,6 +24,8 @@ tablegen(LLVM RISCVGenPreLegalizeGICombiner.inc -gen-global-isel-combiner
               -combiners="RISCVPreLegalizerCombiner")
 tablegen(LLVM RISCVGenPostLegalizeGICombiner.inc -gen-global-isel-combiner
               -combiners="RISCVPostLegalizerCombiner")
+tablegen(LLVM RISCVGenPostLegalizeGILowering.inc -gen-global-isel-combiner
+              -combiners="RISCVPostLegalizerLowering")
 
 add_public_tablegen_target(RISCVCommonTableGen)
 
@@ -63,6 +65,7 @@ add_llvm_target(RISCVCodeGen
   GISel/RISCVInstructionSelector.cpp
   GISel/RISCVLegalizerInfo.cpp
   GISel/RISCVPostLegalizerCombiner.cpp
+  GISel/RISCVPostLegalizerLowering.cpp
   GISel/RISCVO0PreLegalizerCombiner.cpp
   GISel/RISCVPreLegalizerCombiner.cpp
   GISel/RISCVRegisterBankInfo.cpp
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 8de908d20d245c..c204683f4e79f8 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -450,9 +450,9 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
 
   if (ST.hasStdExtM()) {
     getActionDefinitionsBuilder({G_UDIV, G_SDIV, G_UREM, G_SREM})
-        .legalFor({sXLen})
+        .legalFor({s32, sXLen})
         .libcallFor({sDoubleXLen})
-        .clampScalar(0, sXLen, sDoubleXLen)
+        .clampScalar(0, s32, sDoubleXLen)
         .widenScalarToNextPow2(0);
   } else {
     getActionDefinitionsBuilder({G_UDIV, G_SDIV, G_UREM, G_SREM})
@@ -466,7 +466,7 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
 
   auto &AbsActions = getActionDefinitionsBuilder(G_ABS);
   if (ST.hasStdExtZbb())
-    AbsActions.customFor({sXLen}).minScalar(0, sXLen);
+    AbsActions.customFor({s32, sXLen}).minScalar(0, sXLen);
   AbsActions.lower();
 
   auto &MinMaxActions =
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVPostLegalizerLowering.cpp b/llvm/lib/Target/RISCV/GISel/RISCVPostLegalizerLowering.cpp
new file mode 100644
index 00000000000000..66db15e3a2e28c
--- /dev/null
+++ b/llvm/lib/Target/RISCV/GISel/RISCVPostLegalizerLowering.cpp
@@ -0,0 +1,154 @@
+//===--------------- RISCVPostLegalizerLowering.cpp -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Post-legalization lowering for instructions.
+///
+/// This is used to offload pattern matching from the selector.
+///
+/// General optimization combines should be handled by either the
+/// RISCVPostLegalizerCombiner or the RISCVPreLegalizerCombiner.
+///
+//===----------------------------------------------------------------------===//
+
+#include "RISCVSubtarget.h"
+
+#include "llvm/CodeGen/GlobalISel/Combiner.h"
+#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
+#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
+#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Support/Debug.h"
+
+#define GET_GICOMBINER_DEPS
+#include "RISCVGenPostLegalizeGILowering.inc"
+#undef GET_GICOMBINER_DEPS
+
+#define DEBUG_TYPE "riscv-postlegalizer-lowering"
+
+using namespace llvm;
+
+namespace {
+
+#define GET_GICOMBINER_TYPES
+#include "RISCVGenPostLegalizeGILowering.inc"
+#undef GET_GICOMBINER_TYPES
+
+class RISCVPostLegalizerLoweringImpl : public Combiner {
+protected:
+  // TODO: Make CombinerHelper methods const.
+  mutable CombinerHelper Helper;
+  const RISCVPostLegalizerLoweringImplRuleConfig &RuleConfig;
+  const RISCVSubtarget &STI;
+
+public:
+  RISCVPostLegalizerLoweringImpl(
+      MachineFunction &MF, CombinerInfo &CInfo, const TargetPassConfig *TPC,
+      GISelCSEInfo *CSEInfo,
+      const RISCVPostLegalizerLoweringImplRuleConfig &RuleConfig,
+      const RISCVSubtarget &STI);
+
+  static const char *getName() { return "RISCVPreLegalizerCombiner"; }
+
+  bool tryCombineAll(MachineInstr &I) const override;
+
+private:
+#define GET_GICOMBINER_CLASS_MEMBERS
+#include "RISCVGenPostLegalizeGILowering.inc"
+#undef GET_GICOMBINER_CLASS_MEMBERS
+};
+
+#define GET_GICOMBINER_IMPL
+#include "RISCVGenPostLegalizeGILowering.inc"
+#undef GET_GICOMBINER_IMPL
+
+RISCVPostLegalizerLoweringImpl::RISCVPostLegalizerLoweringImpl(
+    MachineFunction &MF, CombinerInfo &CInfo, const TargetPassConfig *TPC,
+    GISelCSEInfo *CSEInfo,
+    const RISCVPostLegalizerLoweringImplRuleConfig &RuleConfig,
+    const RISCVSubtarget &STI)
+    : Combiner(MF, CInfo, TPC, /*KB*/ nullptr, CSEInfo),
+      Helper(Observer, B, /*IsPreLegalize*/ true), RuleConfig(RuleConfig),
+      STI(STI),
+#define GET_GICOMBINER_CONSTRUCTOR_INITS
+#include "RISCVGenPostLegalizeGILowering.inc"
+#undef GET_GICOMBINER_CONSTRUCTOR_INITS
+{
+}
+
+class RISCVPostLegalizerLowering : public MachineFunctionPass {
+public:
+  static char ID;
+
+  RISCVPostLegalizerLowering();
+
+  StringRef getPassName() const override {
+    return "RISCVPostLegalizerLowering";
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+private:
+  RISCVPostLegalizerLoweringImplRuleConfig RuleConfig;
+};
+} // end anonymous namespace
+
+void RISCVPostLegalizerLowering::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.addRequired<TargetPassConfig>();
+  AU.setPreservesCFG();
+  getSelectionDAGFallbackAnalysisUsage(AU);
+  MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+RISCVPostLegalizerLowering::RISCVPostLegalizerLowering()
+    : MachineFunctionPass(ID) {
+  if (!RuleConfig.parseCommandLineOption())
+    report_fatal_error("Invalid rule identifier");
+}
+
+bool RISCVPostLegalizerLowering::runOnMachineFunction(MachineFunction &MF) {
+  if (MF.getProperties().hasProperty(
+          MachineFunctionProperties::Property::FailedISel))
+    return false;
+  assert(MF.getProperties().hasProperty(
+             MachineFunctionProperties::Property::Legalized) &&
+         "Expected a legalized function?");
+  auto *TPC = &getAnalysis<TargetPassConfig>();
+  const Function &F = MF.getFunction();
+
+  const RISCVSubtarget &ST = MF.getSubtarget<RISCVSubtarget>();
+  CombinerInfo CInfo(/*AllowIllegalOps*/ true, /*ShouldLegalizeIllegal*/ false,
+                     /*LegalizerInfo*/ nullptr, /*OptEnabled=*/true,
+                     F.hasOptSize(), F.hasMinSize());
+  // Disable fixed-point iteration to reduce compile-time
+  CInfo.MaxIterations = 1;
+  CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
+  // PostLegalizerCombiner performs DCE, so a full DCE pass is unnecessary.
+  CInfo.EnableFullDCE = false;
+  RISCVPostLegalizerLoweringImpl Impl(MF, CInfo, TPC, /*CSEInfo*/ nullptr,
+                                      RuleConfig, ST);
+  return Impl.combineMachineInstrs();
+}
+
+char RISCVPostLegalizerLowering::ID = 0;
+INITIALIZE_PASS_BEGIN(RISCVPostLegalizerLowering, DEBUG_TYPE,
+                      "Lower RISC-V MachineInstrs after legalization", false,
+                      false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_END(RISCVPostLegalizerLowering, DEBUG_TYPE,
+                    "Lower RISC-V MachineInstrs after legalization", false,
+                    false)
+
+namespace llvm {
+FunctionPass *createRISCVPostLegalizerLowering() {
+  return new RISCVPostLegalizerLowering();
+}
+} // end namespace llvm
diff --git a/llvm/lib/Target/RISCV/RISCV.h b/llvm/lib/Target/RISCV/RISCV.h
index 5a94ada8f8dd46..561e4895e19428 100644
--- a/llvm/lib/Target/RISCV/RISCV.h
+++ b/llvm/lib/Target/RISCV/RISCV.h
@@ -99,6 +99,9 @@ void initializeRISCVO0PreLegalizerCombinerPass(PassRegistry &);
 
 FunctionPass *createRISCVPreLegalizerCombiner();
 void initializeRISCVPreLegalizerCombinerPass(PassRegistry &);
+
+FunctionPass *createRISCVPostLegalizerLowering();
+void initializeRISCVPostLegalizerLoweringPass(PassRegistry &);
 } // namespace llvm
 
 #endif
diff --git a/llvm/lib/Target/RISCV/RISCV.td b/llvm/lib/Target/RISCV/RISCV.td
index c58ebeeafe13f5..00c3d702e12a22 100644
--- a/llvm/lib/Target/RISCV/RISCV.td
+++ b/llvm/lib/Target/RISCV/RISCV.td
@@ -52,6 +52,7 @@ include "RISCVSchedSiFiveP400.td"
 include "RISCVSchedSiFiveP600.td"
 include "RISCVSchedSyntacoreSCR1.td"
 include "RISCVSchedSyntacoreSCR345.td"
+include "RISCVSchedSyntacoreSCR7.td"
 include "RISCVSchedXiangShanNanHu.td"
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/RISCV/RISCVCombine.td b/llvm/lib/Target/RISCV/RISCVCombine.td
index 3a5afb1b075c9e..d48698ae6f2bfb 100644
--- a/llvm/lib/Target/RISCV/RISCVCombine.td
+++ b/llvm/lib/Target/RISCV/RISCVCombine.td
@@ -19,6 +19,13 @@ def RISCVO0PreLegalizerCombiner: GICombiner<
   "RISCVO0PreLegalizerCombinerImpl", [optnone_combines]> {
 }
 
+// Post-legalization combines which should happen at all optimization levels.
+// (E.g. ones that facilitate matching for the selector) For example, matching
+// pseudos.
+def RISCVPostLegalizerLowering
+    : GICombiner<"RISCVPostLegalizerLoweringImpl", []> {
+}
+
 // Post-legalization combines which are primarily optimizations.
 // TODO: Add more combines.
 def RISCVPostLegalizerCombiner
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 7d2a7b20ba2508..d5b3cccda02d3b 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -531,6 +531,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
                        Subtarget.isSoftFPABI() ? LibCall : Custom);
     setOperationAction(ISD::FP_TO_FP16, MVT::f32, Custom);
     setOperationAction(ISD::FP16_TO_FP, MVT::f32, Custom);
+    setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f32, Custom);
+    setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f32, Custom);
 
     if (Subtarget.hasStdExtZfa()) {
       setOperationAction(ISD::ConstantFP, MVT::f32, Custom);
@@ -581,6 +583,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
                        Subtarget.isSoftFPABI() ? LibCall : Custom);
     setOperationAction(ISD::FP_TO_FP16, MVT::f64, Custom);
     setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
+    setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f64, Custom);
+    setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f64, Expand);
   }
 
   if (Subtarget.is64Bit()) {
@@ -6889,33 +6893,45 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
       return DAG.getNode(ISD::FP_EXTEND, DL, VT, Res);
     return Res;
   }
+  case ISD::STRICT_FP_TO_FP16:
   case ISD::FP_TO_FP16: {
     // Custom lower to ensure the libcall return is passed in an FPR on hard
     // float ABIs.
     assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalisation");
     SDLoc DL(Op);
     MakeLibCallOptions CallOptions;
-    RTLIB::Libcall LC =
-        RTLIB::getFPROUND(Op.getOperand(0).getValueType(), MVT::f16);
-    SDValue Res =
-        makeLibCall(DAG, LC, MVT::f32, Op.getOperand(0), CallOptions, DL).first;
+    bool IsStrict = Op->isStrictFPOpcode();
+    SDValue Op0 = IsStrict ? Op.getOperand(1) : Op.getOperand(0);
+    SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
+    RTLIB::Libcall LC = RTLIB::getFPROUND(Op0.getValueType(), MVT::f16);
+    SDValue Res;
+    std::tie(Res, Chain) =
+        makeLibCall(DAG, LC, MVT::f32, Op0, CallOptions, DL, Chain);
     if (Subtarget.is64Bit())
       return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Res);
-    return DAG.getBitcast(MVT::i32, Res);
+    SDValue Result = DAG.getBitcast(MVT::i32, IsStrict ? Res.getValue(0) : Res);
+    if (IsStrict)
+      return DAG.getMergeValues({Result, Chain}, DL);
+    return Result;
   }
+  case ISD::STRICT_FP16_TO_FP:
   case ISD::FP16_TO_FP: {
     // Custom lower to ensure the libcall argument is passed in an FPR on hard
     // float ABIs.
     assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalisation");
     SDLoc DL(Op);
     MakeLibCallOptions CallOptions;
+    bool IsStrict = Op->isStrictFPOpcode();
+    SDValue Op0 = IsStrict ? Op.getOperand(1) : Op.getOperand(0);
+    SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
     SDValue Arg = Subtarget.is64Bit()
-                      ? DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32,
-                                    Op.getOperand(0))
-                      : DAG.getBitcast(MVT::f32, Op.getOperand(0));
-    SDValue Res =
-        makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg, CallOptions, DL)
-            .first;
+                      ? DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Op0)
+                      : DAG.getBitcast(MVT::f32, Op0);
+    SDValue Res;
+    std::tie(Res, Chain) = makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg,
+                                       CallOptions, DL, Chain);
+    if (IsStrict)
+      return DAG.getMergeValues({Res, Chain}, DL);
     return Res;
   }
   case ISD::FTRUNC:
@@ -14740,6 +14756,19 @@ struct NodeExtensionHelper {
     EnforceOneUse = false;
   }
 
+  bool isSupportedFPExtend(SDNode *Root, MVT NarrowEltVT,
+                           const RISCVSubtarget &Subtarget) {
+    // Any f16 extension will neeed zvfh
+    if (NarrowEltVT == MVT::f16 && !Subtarget.hasVInstructionsF16())
+      return false;
+    // The only bf16 extension we can do is vfmadd_vl -> vfwmadd_vl with
+    // zvfbfwma
+    if (NarrowEltVT == MVT::bf16 && (!Subtarget.hasStdExtZvfbfwma() ||
+                                     Root->getOpcode() != RISCVISD::VFMADD_VL))
+      return false;
+    return true;
+  }
+
   /// Helper method to set the various fields of this struct based on the
   /// type of \p Root.
   void fillUpExtensionSupport(SDNode *Root, SelectionDAG &DAG,
@@ -14775,9 +14804,14 @@ struct NodeExtensionHelper {
     case RISCVISD::VSEXT_VL:
       SupportsSExt = true;
       break;
-    case RISCVISD::FP_EXTEND_VL:
+    case RISCVISD::FP_EXTEND_VL: {
+      MVT NarrowEltVT =
+          OrigOperand.getOperand(0).getSimpleValueType().getVectorElementType();
+      if (!isSupportedFPExtend(Root, NarrowEltVT, Subtarget))
+        break;
       SupportsFPExt = true;
       break;
+    }
     case ISD::SPLAT_VECTOR:
     case RISCVISD::VMV_V_X_VL:
       fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
@@ -14792,6 +14826,10 @@ struct NodeExtensionHelper {
       if (Op.getOpcode() != ISD::FP_EXTEND)
         break;
 
+      if (!isSupportedFPExtend(Root, Op.getOperand(0).getSimpleValueType(),
+                               Subtarget))
+        break;
+
       unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
       unsigned ScalarBits = Op.getOperand(0).getValueSizeInBits();
       if (NarrowSize != ScalarBits)
@@ -15774,10 +15812,6 @@ static SDValue performVFMADD_VLCombine(SDNode *N,
   if (SDValue V = combineVFMADD_VLWithVFNEG_VL(N, DAG))
     return V;
 
-  if (N->getValueType(0).getVectorElementType() == MVT::f32 &&
-      !Subtarget.hasVInstructionsF16() && !Subtarget.hasStdExtZvfbfwma())
-    return SDValue();
-
   // FIXME: Ignore strict opcodes for now.
   if (N->isTargetStrictFPOpcode())
     return SDValue();
@@ -17522,12 +17556,8 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
   case RISCVISD::FSUB_VL:
   case RISCVISD::FMUL_VL:
   case RISCVISD::VFWADD_W_VL:
-  case RISCVISD::VFWSUB_W_VL: {
-    if (N->getValueType(0).getVectorElementType() == MVT::f32 &&
-        !Subtarget.hasVInstructionsF16())
-      return SDValue();
+  case RISCVISD::VFWSUB_W_VL:
     return combineOp_VLToVWOp_VL(N, DCI, Subtarget);
-  }
   case ISD::LOAD:
   case ISD::STORE: {
     if (DCI.isAfterLegalizeDAG())
diff --git a/llvm/lib/Target/RISCV/RISCVProcessors.td b/llvm/lib/Target/RISCV/RISCVProcessors.td
index c4e1a1457e8d30..364aa35f09453b 100644
--- a/llvm/lib/Target/RISCV/RISCVProcessors.td
+++ b/llvm/lib/Target/RISCV/RISCVProcessors.td
@@ -384,7 +384,7 @@ def SYNTACORE_SCR5_RV64 : RISCVProcessorModel<"syntacore-scr5-rv64",
                                               [TuneNoDefaultUnroll, FeaturePostRAScheduler]>;
 
 def SYNTACORE_SCR7 : RISCVProcessorModel<"syntacore-scr7",
-                                              NoSchedModel,
+                                              SyntacoreSCR7Model,
                                               [Feature64Bit,
                                                FeatureStdExtI,
                                                FeatureStdExtZicsr,
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSyntacoreSCR7.td b/llvm/lib/Target/RISCV/RISCVSchedSyntacoreSCR7.td
new file mode 100644
index 00000000000000..2818c2f60f3f9c
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVSchedSyntacoreSCR7.td
@@ -0,0 +1,379 @@
+//==- RISCVSchedSyntacoreSCR7.td - Syntacore SCR7 Sched Defs -*- tablegen -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+
+// This file covers scheduling model for rv64imafdcv_zba_zbb_zbc_zbs
+// configuration of Syntacore SCR7 processor.
+// Overview: https://syntacore.com/products/scr7
+
+// SCR7 is an out-of-order superscalar dual-issue core.
+// FIXME: add V and Zkn extensions scheduling model
+def SyntacoreSCR7Model : SchedMachineModel {
+  let MicroOpBufferSize = 36;
+  let IssueWidth = 2;
+  let MispredictPenalty = 9;
+  let LoadLatency = 3;
+  let CompleteModel = 0;
+  let UnsupportedFeatures = [HasStdExtZksed, HasStdExtZksh, HasStdExtZkr,
+                             HasStdExtZcmt, HasVInstructions];
+}
+
+// Branching
+multiclass SCR7_Branching<ProcResourceKind BRU> {
+  def : WriteRes<WriteJmp, [BRU]>;
+  def : WriteRes<WriteJal, [BRU]>;
+  def : WriteRes<WriteJalr, [BRU]>;
+}
+
+// Single-cycle integer arithmetic and logic
+multiclass SCR7_IntALU<ProcResourceKind ALU> {
+  def : WriteRes<WriteIALU, [ALU]>;
+  def : WriteRes<WriteIALU32, [ALU]>;
+  def : WriteRes<WriteShiftImm, [ALU]>;
+  def : WriteRes<WriteShiftImm32, [ALU]>;
+  def : WriteRes<WriteShiftReg, [ALU]>;
+  def : WriteRes<WriteShiftReg32, [ALU]>;
+}
+
+// Pipelined integer multiplication
+multiclass SCR7_IntMul<list<ProcResourceKind> Resources> {
+  let Latency = 3 in {
+    def : WriteRes<WriteIMul, Resources>;
+    def : WriteRes<WriteIMul32, Resources>;
+  }
+}
+
+// Common implementation for WriteIDiv and WriteIDiv32 sched writes.
+multiclass SCR7_IntDivImpl<list<ProcResourceKind> Resources,
+                           list<int> ReleaseCycles, int DivLatency,
+                           SchedWrite DivWrite, SchedWrite RemWrite> {
+  let Latency = DivLatency, ReleaseAtCycles = ReleaseCycles in {
+   def : WriteRes<DivWrite, Resources>;
+   def : WriteRes<RemWrite, Resources>;
+  }
+}
+
+// Non-pipelined integer division
+multiclass SCR7_IntDiv<list<ProcResourceKind> Resources,
+                       list<int> ReleaseCycles,
+                       int DivLatency> {
+  defm : SCR7_IntDivImpl<Resources,
+                         ReleaseCycles,
+                         DivLatency,
+                         WriteIDiv,
+                         WriteIRem>;
+}
+
+multiclass SCR7_IntDiv32<list<ProcResourceKind> Resources,
+                         list<int> ReleaseCycles,
+                         int DivLatency> {
+  defm : SCR7_IntDivImpl<Resources,
+                         ReleaseCycles,
+                         DivLatency,
+                         WriteIDiv32,
+                         WriteIRem32>;
+}
+
+multiclass SCR7_Bitmanip<ProcResourceKind BMU> {
+  let Latency = 1 in {
+    // Zba
+    def : WriteRes<WriteSHXADD, [BMU]>;
+    def : WriteRes<WriteSHXADD32, [BMU]>;
+    // Zbb
+    def : WriteRes<WriteRotateImm, [BMU]>;
+    def : WriteRes<WriteRotateImm32, [BMU]>;
+    def : WriteRes<WriteRotateReg, [BMU]>;
+    def : WriteRes<WriteRotateReg32, [BMU]>;
+    def : WriteRes<WriteCLZ, [BMU]>;
+    def : WriteRes<WriteCLZ32, [BMU]>;
+    def : WriteRes<WriteCTZ, [BMU]>;
+    def : WriteRes<WriteCTZ32, [BMU]>;
+    def : WriteRes<WriteCPOP, [BMU]>;
+    def : WriteRes<WriteCPOP32, [BMU]>;
+    def : WriteRes<WriteREV8, [BMU]>;
+    def : WriteRes<WriteORCB, [BMU]>;
+    def : WriteRes<WriteIMinMax, [BMU]>;
+    // Zbs
+    def : WriteRes<WriteSingleBit, [BMU]>;
+    def : WriteRes<WriteSingleBitImm, [BMU]>;
+    // Zbc
+    def : WriteRes<WriteCLMUL, [BMU]>;
+    def : WriteRes<WriteBEXT, [BMU]>;
+    def : WriteRes<WriteBEXTI, [BMU]>;
+  }
+}
+
+multiclass SCR7_ScalarCrypto<ProcResourceKind SCU> {
+  let Latency = 1 in {
+    // Zbkb
+    def : WriteRes<WriteBREV8, [SCU]>;
+    def : WriteRes<WritePACK, [SCU]>;
+    def : WriteRes<WritePACK32, [SCU]>;
+    def : WriteRes<WriteZIP, [SCU]>;
+    // Zbkx
+    def : WriteRes<WriteXPERM, [SCU]>;
+  }
+}
+
+multiclass SCR7_IntPipeline<ProcResourceKind ALU_Any,
+                            ProcResourceKind ALU_DIV_IS,
+                            ProcResourceKind DIV,
+                            ProcResourceKind ALU_MUL_IS,
+                            ProcResourceKind MUL> {
+  defm : SCR7_Branching<ALU_Any>;
+  defm : SCR7_Bitmanip<ALU_Any>;
+  defm : SCR7_ScalarCrypto<ALU_Any>;
+  defm : SCR7_IntALU<ALU_Any>;
+  defm : SCR7_IntMul<[ALU_MUL_IS, MUL]>;
+  defm : SCR7_IntDiv<[ALU_DIV_IS, DIV],
+                      /* ReleaseAtCycles */[1, 35],
+                      /* Latency */ 35>;
+  defm : SCR7_IntDiv32<[ALU_DIV_IS, DIV],
+                        /* ReleaseAtCycles */[1, 19],
+                        /* Latency */ 19>;
+}
+
+// Load/store instructions
+multiclass SCR7_BasicMemory<ProcResourceKind LSU> {
+  let Latency = 3 in {
+    def : WriteRes<WriteSTB, [LSU]>;
+    def : WriteRes<WriteSTH, [LSU]>;
+    def : WriteRes<WriteSTW, [LSU]>;
+    def : WriteRes<WriteSTD, [LSU]>;
+    def : WriteRes<WriteLDB, [LSU]>;
+    def : WriteRes<WriteLDH, [LSU]>;
+    def : WriteRes<WriteLDW, [LSU]>;
+    def : WriteRes<WriteLDD, [LSU]>;
+    def : WriteRes<WriteFST32, [LSU]>;
+    def : WriteRes<WriteFST64, [LSU]>;
+    def : WriteRes<WriteFLD32, [LSU]>;
+    def : WriteRes<WriteFLD64, [LSU]>;
+  }
+}
+
+// Atomic memory
+multiclass SCR7_AtomicMemory<ProcResourceKind LSU> {
+  let Latency = 19 in {
+    def : WriteRes<WriteAtomicLDW, [LSU]>;
+    def : WriteRes<WriteAtomicLDD, [LSU]>;
+  }
+  let Latency = 21 in {
+    def : WriteRes<WriteAtomicW, [LSU]>;
+    def : WriteRes<WriteAtomicD, [LSU]>;
+    def : WriteRes<WriteAtomicSTW, [LSU]>;
+    def : WriteRes<WriteAtomicSTD, [LSU]>;
+  }
+}
+multiclass SCR7_FPU<ProcResourceKind FPU_IS, ProcResourceKind FALU,
+                    ProcResourceKind FMA, ProcResourceKind FDIVSQRT> {
+  // FALU operations
+  let Latency = 4 in {
+    def : WriteRes<WriteFAdd32, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFAdd64, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFSGNJ32, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFSGNJ64, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFMinMax32, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFMinMax64, [FPU_IS, FALU]>;
+
+    def : WriteRes<WriteFCvtI32ToF32, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFCvtI32ToF64, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFCvtI64ToF32, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFCvtI64ToF64, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFCvtF32ToF64, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFCvtF64ToF32, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFCvtF32ToI32, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFCvtF32ToI64, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFCvtF64ToI32, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFCvtF64ToI64, [FPU_IS, FALU]>;
+
+    def : WriteRes<WriteFClass32, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFClass64, [FPU_IS, FALU]>;
+
+    def : WriteRes<WriteFCmp32, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFCmp64, [FPU_IS, FALU]>;
+
+    def : WriteRes<WriteFMovI32ToF32, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFMovF32ToI32, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFMovI64ToF64, [FPU_IS, FALU]>;
+    def : WriteRes<WriteFMovF64ToI64, [FPU_IS, FALU]>;
+  }
+
+  // FMA operations
+  let Latency = 6 in {
+    def : WriteRes<WriteFMul32, [FPU_IS, FMA]>;
+    def : WriteRes<WriteFMul64, [FPU_IS, FMA]>;
+    def : WriteRes<WriteFMA32, [FPU_IS, FMA]>;
+    def : WriteRes<WriteFMA64, [FPU_IS, FMA]>;
+  }
+
+  def : WriteRes<WriteFDiv32, [FPU_IS, FDIVSQRT]> {
+    let Latency = 16;
+    let ReleaseAtCycles = [1, 15];
+  }
+  def : WriteRes<WriteFDiv64, [FPU_IS, FDIVSQRT]> {
+    let Latency = 30;
+    let ReleaseAtCycles = [1, 29];
+  }
+
+  def : WriteRes<WriteFSqrt32, [FPU_IS, FDIVSQRT]> {
+    let Latency = 18;
+    let ReleaseAtCycles = [1, 16];
+  }
+  def : WriteRes<WriteFSqrt64, [FPU_IS, FDIVSQRT]> {
+    let Latency = 32;
+    let ReleaseAtCycles = [1, 30];
+  }
+}
+
+// Others
+multiclass SCR7_Other {
+  def : WriteRes<WriteCSR, []>;
+  def : WriteRes<WriteNop, []>;
+
+  def : InstRW<[WriteIALU], (instrs COPY)>;
+}
+
+// Unsupported scheduling classes for SCR7.
+multiclass SCR7_Unsupported {
+  defm : UnsupportedSchedSFB;
+  defm : UnsupportedSchedV;
+  defm : UnsupportedSchedXsfvcp;
+  defm : UnsupportedSchedZabha;
+  defm : UnsupportedSchedZfa;
+  defm : UnsupportedSchedZfh;
+  defm : UnsupportedSchedZvk;
+}
+
+
+// Bypasses (none)
+multiclass SCR7_NoReadAdvances {
+  def : ReadAdvance<ReadJmp, 0>;
+  def : ReadAdvance<ReadJalr, 0>;
+  def : ReadAdvance<ReadCSR, 0>;
+  def : ReadAdvance<ReadStoreData, 0>;
+  def : ReadAdvance<ReadMemBase, 0>;
+  def : ReadAdvance<ReadIALU, 0>;
+  def : ReadAdvance<ReadIALU32, 0>;
+  def : ReadAdvance<ReadShiftImm, 0>;
+  def : ReadAdvance<ReadShiftImm32, 0>;
+  def : ReadAdvance<ReadShiftReg, 0>;
+  def : ReadAdvance<ReadShiftReg32, 0>;
+  def : ReadAdvance<ReadIDiv, 0>;
+  def : ReadAdvance<ReadIDiv32, 0>;
+  def : ReadAdvance<ReadIRem, 0>;
+  def : ReadAdvance<ReadIRem32, 0>;
+  def : ReadAdvance<ReadIMul, 0>;
+  def : ReadAdvance<ReadIMul32, 0>;
+  def : ReadAdvance<ReadAtomicWA, 0>;
+  def : ReadAdvance<ReadAtomicWD, 0>;
+  def : ReadAdvance<ReadAtomicDA, 0>;
+  def : ReadAdvance<ReadAtomicDD, 0>;
+  def : ReadAdvance<ReadAtomicLDW, 0>;
+  def : ReadAdvance<ReadAtomicLDD, 0>;
+  def : ReadAdvance<ReadAtomicSTW, 0>;
+  def : ReadAdvance<ReadAtomicSTD, 0>;
+  def : ReadAdvance<ReadSHXADD, 0>;
+  def : ReadAdvance<ReadSHXADD32, 0>;
+  def : ReadAdvance<ReadRotateImm, 0>;
+  def : ReadAdvance<ReadRotateImm32, 0>;
+  def : ReadAdvance<ReadRotateReg, 0>;
+  def : ReadAdvance<ReadRotateReg32, 0>;
+  def : ReadAdvance<ReadCLZ, 0>;
+  def : ReadAdvance<ReadCLZ32, 0>;
+  def : ReadAdvance<ReadCTZ, 0>;
+  def : ReadAdvance<ReadCTZ32, 0>;
+  def : ReadAdvance<ReadCPOP, 0>;
+  def : ReadAdvance<ReadCPOP32, 0>;
+  def : ReadAdvance<ReadREV8, 0>;
+  def : ReadAdvance<ReadORCB, 0>;
+  def : ReadAdvance<ReadIMinMax, 0>;
+  def : ReadAdvance<ReadCLMUL, 0>;
+  def : ReadAdvance<ReadBREV8, 0>;
+  def : ReadAdvance<ReadPACK, 0>;
+  def : ReadAdvance<ReadPACK32, 0>;
+  def : ReadAdvance<ReadZIP, 0>;
+  def : ReadAdvance<ReadXPERM, 0>;
+  def : ReadAdvance<ReadSingleBit, 0>;
+  def : ReadAdvance<ReadSingleBitImm, 0>;
+  def : ReadAdvance<ReadFStoreData, 0>;
+  def : ReadAdvance<ReadFMemBase, 0>;
+  def : ReadAdvance<ReadFAdd32, 0>;
+  def : ReadAdvance<ReadFAdd64, 0>;
+  def : ReadAdvance<ReadFMul32, 0>;
+  def : ReadAdvance<ReadFMul64, 0>;
+  def : ReadAdvance<ReadFMA32, 0>;
+  def : ReadAdvance<ReadFMA32Addend, 0>;
+  def : ReadAdvance<ReadFMA64, 0>;
+  def : ReadAdvance<ReadFMA64Addend, 0>;
+  def : ReadAdvance<ReadFDiv32, 0>;
+  def : ReadAdvance<ReadFDiv64, 0>;
+  def : ReadAdvance<ReadFSqrt32, 0>;
+  def : ReadAdvance<ReadFSqrt64, 0>;
+  def : ReadAdvance<ReadFCmp32, 0>;
+  def : ReadAdvance<ReadFCmp64, 0>;
+  def : ReadAdvance<ReadFSGNJ32, 0>;
+  def : ReadAdvance<ReadFSGNJ64, 0>;
+  def : ReadAdvance<ReadFMinMax32, 0>;
+  def : ReadAdvance<ReadFMinMax64, 0>;
+  def : ReadAdvance<ReadFCvtF32ToI32, 0>;
+  def : ReadAdvance<ReadFCvtF32ToI64, 0>;
+  def : ReadAdvance<ReadFCvtF64ToI32, 0>;
+  def : ReadAdvance<ReadFCvtF64ToI64, 0>;
+  def : ReadAdvance<ReadFCvtI32ToF32, 0>;
+  def : ReadAdvance<ReadFCvtI32ToF64, 0>;
+  def : ReadAdvance<ReadFCvtI64ToF32, 0>;
+  def : ReadAdvance<ReadFCvtI64ToF64, 0>;
+  def : ReadAdvance<ReadFCvtF32ToF64, 0>;
+  def : ReadAdvance<ReadFCvtF64ToF32, 0>;
+  def : ReadAdvance<ReadFMovF32ToI32, 0>;
+  def : ReadAdvance<ReadFMovI32ToF32, 0>;
+  def : ReadAdvance<ReadFMovF64ToI64, 0>;
+  def : ReadAdvance<ReadFMovI64ToF64, 0>;
+  def : ReadAdvance<ReadFClass32, 0>;
+  def : ReadAdvance<ReadFClass64, 0>;
+}
+
+let SchedModel = SyntacoreSCR7Model in {
+  // Integer pipeline has two reservation stations with single issue port
+  // each. Every station has eight entries:
+  // First station:
+  //   - ALU (+ bitmanip and scalar crypto)
+  //   - Pipelined Multiplier (3 stage)
+  // Second station:
+  //   - ALU (+ bitmanip and scalar crypto)
+  //   - Non-pipelined divider (other units are not blocked)
+  def SCR7_ALU_MUL_IS : ProcResource<1> { let BufferSize = 8; }
+  def SCR7_ALU_DIV_IS : ProcResource<1> { let BufferSize = 8; }
+  def SCR7_ALU_Any : ProcResGroup<[SCR7_ALU_MUL_IS, SCR7_ALU_DIV_IS]>;
+  def SCR7_MUL : ProcResource<1> { let BufferSize = 1; }
+  def SCR7_DIV : ProcResource<1> { let BufferSize = 1; }
+
+  defm : SCR7_IntPipeline<SCR7_ALU_Any,
+                         SCR7_ALU_DIV_IS, SCR7_DIV,
+                         SCR7_ALU_MUL_IS, SCR7_MUL>;
+
+  // SCR7 single-issue LSU with sixteen entries.
+  def SCR7_LSU : ProcResource<1> { let BufferSize = 16; }
+  defm : SCR7_BasicMemory<SCR7_LSU>;
+  defm : SCR7_AtomicMemory<SCR7_LSU>;
+
+  // FPU has one issue slot with eight entries:
+  //   - FP ALU
+  //   - FMA
+  //   - Non-pipelined FDIV/FSQRT
+  def SCR7_FPU_IS : ProcResource<1> { let BufferSize = 8; }
+  def SCR7_FALU : ProcResource<1> { let BufferSize = 1; }
+  def SCR7_FMA : ProcResource<1> { let BufferSize = 1; }
+  def SCR7_FDIVSQRT : ProcResource<1> { let BufferSize = 1; }
+  defm : SCR7_FPU<SCR7_FPU_IS, SCR7_FALU, SCR7_FMA, SCR7_FDIVSQRT>;
+
+  defm : SCR7_Other;
+  defm : SCR7_Unsupported;
+  defm : SCR7_NoReadAdvances;
+}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 794df2212dfa53..5cb9fbc8a9eb98 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -111,6 +111,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
   initializeRISCVO0PreLegalizerCombinerPass(*PR);
   initializeRISCVPreLegalizerCombinerPass(*PR);
   initializeRISCVPostLegalizerCombinerPass(*PR);
+  initializeRISCVPostLegalizerLoweringPass(*PR);
   initializeKCFIPass(*PR);
   initializeRISCVDeadRegisterDefinitionsPass(*PR);
   initializeRISCVMakeCompressibleOptPass(*PR);
@@ -482,6 +483,7 @@ bool RISCVPassConfig::addLegalizeMachineIR() {
 void RISCVPassConfig::addPreRegBankSelect() {
   if (getOptLevel() != CodeGenOptLevel::None)
     addPass(createRISCVPostLegalizerCombiner());
+  addPass(createRISCVPostLegalizerLowering());
 }
 
 bool RISCVPassConfig::addRegBankSelect() {
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 2b5e7c47279284..5d280b44630aef 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1756,6 +1756,14 @@ InstructionCost RISCVTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
       Index = Index % Width;
     }
 
+    // If exact VLEN is known, we will insert/extract into the appropriate
+    // subvector with no additional subvector insert/extract cost.
+    if (auto VLEN = ST->getRealVLen()) {
+      unsigned EltSize = LT.second.getScalarSizeInBits();
+      unsigned M1Max = *VLEN / EltSize;
+      Index = Index % M1Max;
+    }
+
     // We could extract/insert the first element without vslidedown/vslideup.
     if (Index == 0)
       SlideCost = 0;
@@ -2030,8 +2038,15 @@ void RISCVTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
 }
 
 unsigned RISCVTTIImpl::getRegUsageForType(Type *Ty) {
-  TypeSize Size = DL.getTypeSizeInBits(Ty);
   if (Ty->isVectorTy()) {
+    // f16 with only zvfhmin and bf16 will be promoted to f32
+    Type *EltTy = cast<VectorType>(Ty)->getElementType();
+    if ((EltTy->isHalfTy() && !ST->hasVInstructionsF16()) ||
+        EltTy->isBFloatTy())
+      Ty = VectorType::get(Type::getFloatTy(Ty->getContext()),
+                           cast<VectorType>(Ty));
+
+    TypeSize Size = DL.getTypeSizeInBits(Ty);
     if (Size.isScalable() && ST->hasVInstructions())
       return divideCeil(Size.getKnownMinValue(), RISCV::RVVBitsPerBlock);
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp
index b37c7c1a6ee044..b78f1c3f060a23 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp
@@ -4,7 +4,9 @@
 #include "SPIRVGlobalRegistry.h"
 #include "SPIRVRegisterInfo.h"
 #include "SPIRVTargetMachine.h"
+#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -13,7 +15,10 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/Register.h"
 #include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/DebugProgramInstruction.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/PassRegistry.h"
 #include "llvm/Support/Casting.h"
@@ -57,6 +62,17 @@ SPIRVEmitNonSemanticDI::SPIRVEmitNonSemanticDI() : MachineFunctionPass(ID) {
   initializeSPIRVEmitNonSemanticDIPass(*PassRegistry::getPassRegistry());
 }
 
+enum BaseTypeAttributeEncoding {
+  Unspecified = 0,
+  Address = 1,
+  Boolean = 2,
+  Float = 3,
+  Signed = 4,
+  SignedChar = 5,
+  Unsigned = 6,
+  UnsignedChar = 7
+};
+
 bool SPIRVEmitNonSemanticDI::emitGlobalDI(MachineFunction &MF) {
   // If this MachineFunction doesn't have any BB repeat procedure
   // for the next
@@ -71,7 +87,7 @@ bool SPIRVEmitNonSemanticDI::emitGlobalDI(MachineFunction &MF) {
   unsigned SourceLanguage = 0;
   int64_t DwarfVersion = 0;
   int64_t DebugInfoVersion = 0;
-
+  SmallPtrSet<DIBasicType *, 12> BasicTypes;
   // Searching through the Module metadata to find nescessary
   // information like DwarfVersion or SourceLanguage
   {
@@ -104,6 +120,21 @@ bool SPIRVEmitNonSemanticDI::emitGlobalDI(MachineFunction &MF) {
                 cast<ConstantAsMetadata>(Op->getOperand(2))->getValue())
                 ->getSExtValue();
     }
+
+    // This traversal is the only supported way to access
+    // instruction related DI metadata like DIBasicType
+    for (auto &F : *M) {
+      for (auto &BB : F) {
+        for (auto &I : BB) {
+          for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
+            DILocalVariable *LocalVariable = DVR.getVariable();
+            if (auto *BasicType =
+                    dyn_cast<DIBasicType>(LocalVariable->getType()))
+              BasicTypes.insert(BasicType);
+          }
+        }
+      }
+    }
   }
   // NonSemantic.Shader.DebugInfo.100 global DI instruction emitting
   {
@@ -120,29 +151,45 @@ bool SPIRVEmitNonSemanticDI::emitGlobalDI(MachineFunction &MF) {
     // and before first terminator
     MachineIRBuilder MIRBuilder(MBB, MBB.getFirstTerminator());
 
+    const auto EmitOpString = [&](StringRef SR) {
+      const Register StrReg = MRI.createVirtualRegister(&SPIRV::IDRegClass);
+      MRI.setType(StrReg, LLT::scalar(32));
+      MachineInstrBuilder MIB = MIRBuilder.buildInstr(SPIRV::OpString);
+      MIB.addDef(StrReg);
+      addStringImm(SR, MIB);
+      return StrReg;
+    };
+
     // Emit OpString with FilePath which is required by DebugSource
-    const Register StrReg = MRI.createVirtualRegister(&SPIRV::IDRegClass);
-    MRI.setType(StrReg, LLT::scalar(32));
-    MachineInstrBuilder MIB = MIRBuilder.buildInstr(SPIRV::OpString);
-    MIB.addDef(StrReg);
-    addStringImm(FilePath, MIB);
+    const Register FilePathStrReg = EmitOpString(FilePath);
 
     const SPIRVType *VoidTy =
         GR->getOrCreateSPIRVType(Type::getVoidTy(*Context), MIRBuilder);
 
+    const auto EmitDIInstruction =
+        [&](SPIRV::NonSemanticExtInst::NonSemanticExtInst Inst,
+            std::initializer_list<Register> Registers) {
+          const Register InstReg =
+              MRI.createVirtualRegister(&SPIRV::IDRegClass);
+          MRI.setType(InstReg, LLT::scalar(32));
+          MachineInstrBuilder MIB =
+              MIRBuilder.buildInstr(SPIRV::OpExtInst)
+                  .addDef(InstReg)
+                  .addUse(GR->getSPIRVTypeID(VoidTy))
+                  .addImm(static_cast<int64_t>(
+                      SPIRV::InstructionSet::NonSemantic_Shader_DebugInfo_100))
+                  .addImm(Inst);
+          for (auto Reg : Registers) {
+            MIB.addUse(Reg);
+          }
+          MIB.constrainAllUses(*TII, *TRI, *RBI);
+          GR->assignSPIRVTypeToVReg(VoidTy, InstReg, MF);
+          return InstReg;
+        };
+
     // Emit DebugSource which is required by DebugCompilationUnit
-    const Register DebugSourceResIdReg =
-        MRI.createVirtualRegister(&SPIRV::IDRegClass);
-    MRI.setType(DebugSourceResIdReg, LLT::scalar(32));
-    MIB = MIRBuilder.buildInstr(SPIRV::OpExtInst)
-              .addDef(DebugSourceResIdReg)
-              .addUse(GR->getSPIRVTypeID(VoidTy))
-              .addImm(static_cast<int64_t>(
-                  SPIRV::InstructionSet::NonSemantic_Shader_DebugInfo_100))
-              .addImm(SPIRV::NonSemanticExtInst::DebugSource)
-              .addUse(StrReg);
-    MIB.constrainAllUses(*TII, *TRI, *RBI);
-    GR->assignSPIRVTypeToVReg(VoidTy, DebugSourceResIdReg, MF);
+    const Register DebugSourceResIdReg = EmitDIInstruction(
+        SPIRV::NonSemanticExtInst::DebugSource, {FilePathStrReg});
 
     const SPIRVType *I32Ty =
         GR->getOrCreateSPIRVType(Type::getInt32Ty(*Context), MIRBuilder);
@@ -156,22 +203,56 @@ bool SPIRVEmitNonSemanticDI::emitGlobalDI(MachineFunction &MF) {
     const Register SourceLanguageReg =
         GR->buildConstantInt(SourceLanguage, MIRBuilder, I32Ty, false);
 
-    // Emit DebugCompilationUnit
+    [[maybe_unused]]
     const Register DebugCompUnitResIdReg =
-        MRI.createVirtualRegister(&SPIRV::IDRegClass);
-    MRI.setType(DebugCompUnitResIdReg, LLT::scalar(32));
-    MIB = MIRBuilder.buildInstr(SPIRV::OpExtInst)
-              .addDef(DebugCompUnitResIdReg)
-              .addUse(GR->getSPIRVTypeID(VoidTy))
-              .addImm(static_cast<int64_t>(
-                  SPIRV::InstructionSet::NonSemantic_Shader_DebugInfo_100))
-              .addImm(SPIRV::NonSemanticExtInst::DebugCompilationUnit)
-              .addUse(DebugInfoVersionReg)
-              .addUse(DwarfVersionReg)
-              .addUse(DebugSourceResIdReg)
-              .addUse(SourceLanguageReg);
-    MIB.constrainAllUses(*TII, *TRI, *RBI);
-    GR->assignSPIRVTypeToVReg(VoidTy, DebugCompUnitResIdReg, MF);
+        EmitDIInstruction(SPIRV::NonSemanticExtInst::DebugCompilationUnit,
+                          {DebugInfoVersionReg, DwarfVersionReg,
+                           DebugSourceResIdReg, SourceLanguageReg});
+
+    // We aren't extracting any DebugInfoFlags now so we
+    // emitting zero to use as <id>Flags argument for DebugBasicType
+    const Register I32ZeroReg =
+        GR->buildConstantInt(0, MIRBuilder, I32Ty, false);
+
+    for (auto *BasicType : BasicTypes) {
+      const Register BasicTypeStrReg = EmitOpString(BasicType->getName());
+
+      const Register ConstIntBitwidthReg = GR->buildConstantInt(
+          BasicType->getSizeInBits(), MIRBuilder, I32Ty, false);
+
+      uint64_t AttributeEncoding = BaseTypeAttributeEncoding::Unspecified;
+      switch (BasicType->getEncoding()) {
+      case dwarf::DW_ATE_signed:
+        AttributeEncoding = BaseTypeAttributeEncoding::Signed;
+        break;
+      case dwarf::DW_ATE_unsigned:
+        AttributeEncoding = BaseTypeAttributeEncoding::Unsigned;
+        break;
+      case dwarf::DW_ATE_unsigned_char:
+        AttributeEncoding = BaseTypeAttributeEncoding::UnsignedChar;
+        break;
+      case dwarf::DW_ATE_signed_char:
+        AttributeEncoding = BaseTypeAttributeEncoding::SignedChar;
+        break;
+      case dwarf::DW_ATE_float:
+        AttributeEncoding = BaseTypeAttributeEncoding::Float;
+        break;
+      case dwarf::DW_ATE_boolean:
+        AttributeEncoding = BaseTypeAttributeEncoding::Boolean;
+        break;
+      case dwarf::DW_ATE_address:
+        AttributeEncoding = BaseTypeAttributeEncoding::Address;
+      }
+
+      const Register AttributeEncodingReg =
+          GR->buildConstantInt(AttributeEncoding, MIRBuilder, I32Ty, false);
+
+      [[maybe_unused]]
+      const Register BasicTypeReg =
+          EmitDIInstruction(SPIRV::NonSemanticExtInst::DebugTypeBasic,
+                            {BasicTypeStrReg, ConstIntBitwidthReg,
+                             AttributeEncodingReg, I32ZeroReg});
+    }
   }
   return true;
 }
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index df42b6de193bda..fa71223a341b16 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -435,7 +435,8 @@ void SPIRVModuleAnalysis::processOtherInstrs(const Module &M) {
           MachineOperand Ins = MI.getOperand(3);
           namespace NS = SPIRV::NonSemanticExtInst;
           static constexpr int64_t GlobalNonSemanticDITy[] = {
-              NS::DebugSource, NS::DebugCompilationUnit};
+              NS::DebugSource, NS::DebugCompilationUnit, NS::DebugInfoNone,
+              NS::DebugTypeBasic};
           bool IsGlobalDI = false;
           for (unsigned Idx = 0; Idx < std::size(GlobalNonSemanticDITy); ++Idx)
             IsGlobalDI |= Ins.getImm() == GlobalNonSemanticDITy[Idx];
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
index 5299e6ea06f0bd..4fef5fa0ef2208 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
@@ -69,12 +69,23 @@ struct WebAssemblyOperand : public MCParsedAsmOperand {
     std::vector<unsigned> List;
   };
 
+  struct CaLOpElem {
+    uint8_t Opcode;
+    const MCExpr *Tag;
+    unsigned Dest;
+  };
+
+  struct CaLOp {
+    std::vector<CaLOpElem> List;
+  };
+
   union {
     struct TokOp Tok;
     struct IntOp Int;
     struct FltOp Flt;
     struct SymOp Sym;
     struct BrLOp BrL;
+    struct CaLOp CaL;
   };
 
   WebAssemblyOperand(SMLoc Start, SMLoc End, TokOp T)
@@ -85,12 +96,16 @@ struct WebAssemblyOperand : public MCParsedAsmOperand {
       : Kind(Float), StartLoc(Start), EndLoc(End), Flt(F) {}
   WebAssemblyOperand(SMLoc Start, SMLoc End, SymOp S)
       : Kind(Symbol), StartLoc(Start), EndLoc(End), Sym(S) {}
-  WebAssemblyOperand(SMLoc Start, SMLoc End)
-      : Kind(BrList), StartLoc(Start), EndLoc(End), BrL() {}
+  WebAssemblyOperand(SMLoc Start, SMLoc End, BrLOp B)
+      : Kind(BrList), StartLoc(Start), EndLoc(End), BrL(B) {}
+  WebAssemblyOperand(SMLoc Start, SMLoc End, CaLOp C)
+      : Kind(CatchList), StartLoc(Start), EndLoc(End), CaL(C) {}
 
   ~WebAssemblyOperand() {
     if (isBrList())
       BrL.~BrLOp();
+    if (isCatchList())
+      CaL.~CaLOp();
   }
 
   bool isToken() const override { return Kind == Token; }
@@ -153,7 +168,15 @@ struct WebAssemblyOperand : public MCParsedAsmOperand {
   }
 
   void addCatchListOperands(MCInst &Inst, unsigned N) const {
-    // TODO
+    assert(N == 1 && isCatchList() && "Invalid CatchList!");
+    Inst.addOperand(MCOperand::createImm(CaL.List.size()));
+    for (auto Ca : CaL.List) {
+      Inst.addOperand(MCOperand::createImm(Ca.Opcode));
+      if (Ca.Opcode == wasm::WASM_OPCODE_CATCH ||
+          Ca.Opcode == wasm::WASM_OPCODE_CATCH_REF)
+        Inst.addOperand(MCOperand::createExpr(Ca.Tag));
+      Inst.addOperand(MCOperand::createImm(Ca.Dest));
+    }
   }
 
   void print(raw_ostream &OS) const override {
@@ -174,7 +197,7 @@ struct WebAssemblyOperand : public MCParsedAsmOperand {
       OS << "BrList:" << BrL.List.size();
       break;
     case CatchList:
-      // TODO
+      OS << "CaList:" << CaL.List.size();
       break;
     }
   }
@@ -228,6 +251,7 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
     Loop,
     Try,
     CatchAll,
+    TryTable,
     If,
     Else,
     Undefined,
@@ -304,6 +328,8 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
       return {"try", "end_try/delegate"};
     case CatchAll:
       return {"catch_all", "end_try"};
+    case TryTable:
+      return {"try_table", "end_try_table"};
     case If:
       return {"if", "end_if"};
     case Else:
@@ -571,6 +597,7 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
     // proper nesting.
     bool ExpectBlockType = false;
     bool ExpectFuncType = false;
+    bool ExpectCatchList = false;
     std::unique_ptr<WebAssemblyOperand> FunctionTable;
     if (Name == "block") {
       push(Block);
@@ -593,12 +620,19 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
     } else if (Name == "catch_all") {
       if (popAndPushWithSameSignature(Name, Try, CatchAll))
         return true;
+    } else if (Name == "try_table") {
+      push(TryTable);
+      ExpectBlockType = true;
+      ExpectCatchList = true;
     } else if (Name == "end_if") {
       if (pop(Name, If, Else))
         return true;
     } else if (Name == "end_try") {
       if (pop(Name, Try, CatchAll))
         return true;
+    } else if (Name == "end_try_table") {
+      if (pop(Name, TryTable))
+        return true;
     } else if (Name == "delegate") {
       if (pop(Name, Try))
         return true;
@@ -622,7 +656,18 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
       ExpectFuncType = true;
     }
 
-    if (ExpectFuncType || (ExpectBlockType && Lexer.is(AsmToken::LParen))) {
+    // Returns true if the next tokens are a catch clause
+    auto PeekCatchList = [&]() {
+      if (Lexer.isNot(AsmToken::LParen))
+        return false;
+      AsmToken NextTok = Lexer.peekTok();
+      return NextTok.getKind() == AsmToken::Identifier &&
+             NextTok.getIdentifier().starts_with("catch");
+    };
+
+    // Parse a multivalue block type
+    if (ExpectFuncType ||
+        (Lexer.is(AsmToken::LParen) && ExpectBlockType && !PeekCatchList())) {
       // This has a special TYPEINDEX operand which in text we
       // represent as a signature, such that we can re-build this signature,
       // attach it to an anonymous symbol, which is what WasmObjectWriter
@@ -648,6 +693,23 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
           Loc.getLoc(), Loc.getEndLoc(), WebAssemblyOperand::SymOp{Expr}));
     }
 
+    // If we are expecting a catch clause list, try to parse it here.
+    //
+    // If there is a multivalue block return type before this catch list, it
+    // should have been parsed above. If there is no return type before
+    // encountering this catch list, this means the type is void.
+    // The case when there is a single block return value and then a catch list
+    // will be handled below in the 'while' loop.
+    if (ExpectCatchList && PeekCatchList()) {
+      if (ExpectBlockType) {
+        ExpectBlockType = false;
+        addBlockTypeOperand(Operands, NameLoc, WebAssembly::BlockType::Void);
+      }
+      if (parseCatchList(Operands))
+        return true;
+      ExpectCatchList = false;
+    }
+
     while (Lexer.isNot(AsmToken::EndOfStatement)) {
       auto &Tok = Lexer.getTok();
       switch (Tok.getKind()) {
@@ -661,7 +723,15 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
           if (BT == WebAssembly::BlockType::Invalid)
             return error("Unknown block type: ", Id);
           addBlockTypeOperand(Operands, NameLoc, BT);
+          ExpectBlockType = false;
           Parser.Lex();
+          // Now that we've parsed a single block return type, if we are
+          // expecting a catch clause list, try to parse it.
+          if (ExpectCatchList && PeekCatchList()) {
+            if (parseCatchList(Operands))
+              return true;
+            ExpectCatchList = false;
+          }
         } else {
           // Assume this identifier is a label.
           const MCExpr *Val;
@@ -703,8 +773,8 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
       }
       case AsmToken::LCurly: {
         Parser.Lex();
-        auto Op =
-            std::make_unique<WebAssemblyOperand>(Tok.getLoc(), Tok.getEndLoc());
+        auto Op = std::make_unique<WebAssemblyOperand>(
+            Tok.getLoc(), Tok.getEndLoc(), WebAssemblyOperand::BrLOp{});
         if (!Lexer.is(AsmToken::RCurly))
           for (;;) {
             Op->BrL.List.push_back(Lexer.getTok().getIntVal());
@@ -724,10 +794,18 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
           return true;
       }
     }
-    if (ExpectBlockType && Operands.size() == 1) {
-      // Support blocks with no operands as default to void.
+
+    // If we are still expecting to parse a block type or a catch list at this
+    // point, we set them to the default/empty state.
+
+    // Support blocks with no operands as default to void.
+    if (ExpectBlockType)
       addBlockTypeOperand(Operands, NameLoc, WebAssembly::BlockType::Void);
-    }
+    // If no catch list has been parsed, add an empty catch list operand.
+    if (ExpectCatchList)
+      Operands.push_back(std::make_unique<WebAssemblyOperand>(
+          NameLoc, NameLoc, WebAssemblyOperand::CaLOp{}));
+
     if (FunctionTable)
       Operands.push_back(std::move(FunctionTable));
     Parser.Lex();
@@ -752,6 +830,55 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
     return false;
   }
 
+  bool parseCatchList(OperandVector &Operands) {
+    auto Op = std::make_unique<WebAssemblyOperand>(
+        Lexer.getTok().getLoc(), SMLoc(), WebAssemblyOperand::CaLOp{});
+    SMLoc EndLoc;
+
+    while (Lexer.is(AsmToken::LParen)) {
+      if (expect(AsmToken::LParen, "("))
+        return true;
+
+      auto CatchStr = expectIdent();
+      if (CatchStr.empty())
+        return true;
+      uint8_t CatchOpcode =
+          StringSwitch<uint8_t>(CatchStr)
+              .Case("catch", wasm::WASM_OPCODE_CATCH)
+              .Case("catch_ref", wasm::WASM_OPCODE_CATCH_REF)
+              .Case("catch_all", wasm::WASM_OPCODE_CATCH_ALL)
+              .Case("catch_all_ref", wasm::WASM_OPCODE_CATCH_ALL_REF)
+              .Default(0xff);
+      if (CatchOpcode == 0xff)
+        return error(
+            "Expected catch/catch_ref/catch_all/catch_all_ref, instead got: " +
+            CatchStr);
+
+      const MCExpr *Tag = nullptr;
+      if (CatchOpcode == wasm::WASM_OPCODE_CATCH ||
+          CatchOpcode == wasm::WASM_OPCODE_CATCH_REF) {
+        if (Parser.parseExpression(Tag))
+          return error("Cannot parse symbol: ", Lexer.getTok());
+      }
+
+      auto &DestTok = Lexer.getTok();
+      if (DestTok.isNot(AsmToken::Integer))
+        return error("Expected integer constant, instead got: ", DestTok);
+      unsigned Dest = DestTok.getIntVal();
+      Parser.Lex();
+
+      EndLoc = Lexer.getTok().getEndLoc();
+      if (expect(AsmToken::RParen, ")"))
+        return true;
+
+      Op->CaL.List.push_back({CatchOpcode, Tag, Dest});
+    }
+
+    Op->EndLoc = EndLoc;
+    Operands.push_back(std::move(Op));
+    return false;
+  }
+
   bool CheckDataSection() {
     if (CurrentState != DataSection) {
       auto WS = cast<MCSectionWasm>(getStreamer().getCurrentSectionOnly());
diff --git a/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp b/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
index 3585b5f4a5c9ad..a66ee2b6ac1af7 100644
--- a/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
+++ b/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
@@ -289,6 +289,24 @@ MCDisassembler::DecodeStatus WebAssemblyDisassembler::getInstruction(
         return MCDisassembler::Fail;
       break;
     }
+    case WebAssembly::OPERAND_CATCH_LIST: {
+      if (!parseLEBImmediate(MI, Size, Bytes, false))
+        return MCDisassembler::Fail;
+      int64_t NumCatches = MI.getOperand(MI.getNumOperands() - 1).getImm();
+      for (int64_t I = 0; I < NumCatches; I++) {
+        if (!parseImmediate<uint8_t>(MI, Size, Bytes))
+          return MCDisassembler::Fail;
+        int64_t CatchOpcode = MI.getOperand(MI.getNumOperands() - 1).getImm();
+        if (CatchOpcode == wasm::WASM_OPCODE_CATCH ||
+            CatchOpcode == wasm::WASM_OPCODE_CATCH_REF) {
+          if (!parseLEBImmediate(MI, Size, Bytes, false)) // tag index
+            return MCDisassembler::Fail;
+        }
+        if (!parseLEBImmediate(MI, Size, Bytes, false)) // destination
+          return MCDisassembler::Fail;
+      }
+      break;
+    }
     case MCOI::OPERAND_REGISTER:
       // The tablegen header currently does not have any register operands since
       // we use only the stack (_S) instructions.
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp
index 903dbcd21ea967..4c29b59b3302e4 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp
@@ -110,6 +110,20 @@ void WebAssemblyInstPrinter::printInst(const MCInst *MI, uint64_t Address,
   // Print any added annotation.
   printAnnotation(OS, Annot);
 
+  auto PrintBranchAnnotation = [&](const MCOperand &Op,
+                                   SmallSet<uint64_t, 8> &Printed) {
+    uint64_t Depth = Op.getImm();
+    if (!Printed.insert(Depth).second)
+      return;
+    if (Depth >= ControlFlowStack.size()) {
+      printAnnotation(OS, "Invalid depth argument!");
+    } else {
+      const auto &Pair = ControlFlowStack.rbegin()[Depth];
+      printAnnotation(OS, utostr(Depth) + ": " + (Pair.second ? "up" : "down") +
+                              " to label" + utostr(Pair.first));
+    }
+  };
+
   if (CommentStream) {
     // Observe any effects on the control flow stack, for use in annotating
     // control flow label references.
@@ -136,6 +150,23 @@ void WebAssemblyInstPrinter::printInst(const MCInst *MI, uint64_t Address,
       EHInstStack.push_back(TRY);
       return;
 
+    case WebAssembly::TRY_TABLE:
+    case WebAssembly::TRY_TABLE_S: {
+      SmallSet<uint64_t, 8> Printed;
+      unsigned OpIdx = 1;
+      const MCOperand &Op = MI->getOperand(OpIdx++);
+      unsigned NumCatches = Op.getImm();
+      for (unsigned I = 0; I < NumCatches; I++) {
+        int64_t CatchOpcode = MI->getOperand(OpIdx++).getImm();
+        if (CatchOpcode == wasm::WASM_OPCODE_CATCH ||
+            CatchOpcode == wasm::WASM_OPCODE_CATCH_REF)
+          OpIdx++; // Skip tag
+        PrintBranchAnnotation(MI->getOperand(OpIdx++), Printed);
+      }
+      ControlFlowStack.push_back(std::make_pair(ControlFlowCounter++, false));
+      return;
+    }
+
     case WebAssembly::END_LOOP:
     case WebAssembly::END_LOOP_S:
       if (ControlFlowStack.empty()) {
@@ -147,6 +178,8 @@ void WebAssemblyInstPrinter::printInst(const MCInst *MI, uint64_t Address,
 
     case WebAssembly::END_BLOCK:
     case WebAssembly::END_BLOCK_S:
+    case WebAssembly::END_TRY_TABLE:
+    case WebAssembly::END_TRY_TABLE_S:
       if (ControlFlowStack.empty()) {
         printAnnotation(OS, "End marker mismatch!");
       } else {
@@ -251,17 +284,7 @@ void WebAssemblyInstPrinter::printInst(const MCInst *MI, uint64_t Address,
         if (!MI->getOperand(I).isImm())
           continue;
       }
-      uint64_t Depth = MI->getOperand(I).getImm();
-      if (!Printed.insert(Depth).second)
-        continue;
-      if (Depth >= ControlFlowStack.size()) {
-        printAnnotation(OS, "Invalid depth argument!");
-      } else {
-        const auto &Pair = ControlFlowStack.rbegin()[Depth];
-        printAnnotation(OS, utostr(Depth) + ": " +
-                                (Pair.second ? "up" : "down") + " to label" +
-                                utostr(Pair.first));
-      }
+      PrintBranchAnnotation(MI->getOperand(I), Printed);
     }
   }
 }
@@ -377,10 +400,15 @@ void WebAssemblyInstPrinter::printCatchList(const MCInst *MI, unsigned OpNo,
   auto PrintTagOp = [&](const MCOperand &Op) {
     const MCSymbolRefExpr *TagExpr = nullptr;
     const MCSymbolWasm *TagSym = nullptr;
-    assert(Op.isExpr());
-    TagExpr = dyn_cast<MCSymbolRefExpr>(Op.getExpr());
-    TagSym = cast<MCSymbolWasm>(&TagExpr->getSymbol());
-    O << TagSym->getName() << " ";
+    if (Op.isExpr()) {
+      TagExpr = cast<MCSymbolRefExpr>(Op.getExpr());
+      TagSym = cast<MCSymbolWasm>(&TagExpr->getSymbol());
+      O << TagSym->getName() << " ";
+    } else {
+      // When instructions are parsed from the disassembler, we have an
+      // immediate tag index and not a tag expr
+      O << Op.getImm() << " ";
+    }
   };
 
   for (unsigned I = 0; I < NumCatches; I++) {
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
index aaca213c4afe9b..3e7487dbd8f54e 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
@@ -82,14 +82,15 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
   // For br_table instructions, encode the size of the table. In the MCInst,
   // there's an index operand (if not a stack instruction), one operand for
   // each table entry, and the default operand.
-  if (MI.getOpcode() == WebAssembly::BR_TABLE_I32_S ||
-      MI.getOpcode() == WebAssembly::BR_TABLE_I64_S)
+  unsigned Opcode = MI.getOpcode();
+  if (Opcode == WebAssembly::BR_TABLE_I32_S ||
+      Opcode == WebAssembly::BR_TABLE_I64_S)
     encodeULEB128(MI.getNumOperands() - 1, OS);
-  if (MI.getOpcode() == WebAssembly::BR_TABLE_I32 ||
-      MI.getOpcode() == WebAssembly::BR_TABLE_I64)
+  if (Opcode == WebAssembly::BR_TABLE_I32 ||
+      Opcode == WebAssembly::BR_TABLE_I64)
     encodeULEB128(MI.getNumOperands() - 2, OS);
 
-  const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
+  const MCInstrDesc &Desc = MCII.get(Opcode);
   for (unsigned I = 0, E = MI.getNumOperands(); I < E; ++I) {
     const MCOperand &MO = MI.getOperand(I);
     if (MO.isReg()) {
@@ -136,7 +137,12 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
           encodeULEB128(uint64_t(MO.getImm()), OS);
         }
       } else {
-        encodeULEB128(uint64_t(MO.getImm()), OS);
+        // Variadic immediate operands are br_table's destination operands or
+        // try_table's operands (# of catch clauses, catch sub-opcodes, or catch
+        // clause destinations)
+        assert(WebAssembly::isBrTable(Opcode) ||
+               Opcode == WebAssembly::TRY_TABLE_S);
+        encodeULEB128(uint32_t(MO.getImm()), OS);
       }
 
     } else if (MO.isSFPImm()) {
@@ -146,32 +152,38 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
       uint64_t D = MO.getDFPImm();
       support::endian::write<uint64_t>(OS, D, llvm::endianness::little);
     } else if (MO.isExpr()) {
-      const MCOperandInfo &Info = Desc.operands()[I];
       llvm::MCFixupKind FixupKind;
       size_t PaddedSize = 5;
-      switch (Info.OperandType) {
-      case WebAssembly::OPERAND_I32IMM:
-        FixupKind = MCFixupKind(WebAssembly::fixup_sleb128_i32);
-        break;
-      case WebAssembly::OPERAND_I64IMM:
-        FixupKind = MCFixupKind(WebAssembly::fixup_sleb128_i64);
-        PaddedSize = 10;
-        break;
-      case WebAssembly::OPERAND_FUNCTION32:
-      case WebAssembly::OPERAND_TABLE:
-      case WebAssembly::OPERAND_OFFSET32:
-      case WebAssembly::OPERAND_SIGNATURE:
-      case WebAssembly::OPERAND_TYPEINDEX:
-      case WebAssembly::OPERAND_GLOBAL:
-      case WebAssembly::OPERAND_TAG:
+      if (I < Desc.getNumOperands()) {
+        const MCOperandInfo &Info = Desc.operands()[I];
+        switch (Info.OperandType) {
+        case WebAssembly::OPERAND_I32IMM:
+          FixupKind = MCFixupKind(WebAssembly::fixup_sleb128_i32);
+          break;
+        case WebAssembly::OPERAND_I64IMM:
+          FixupKind = MCFixupKind(WebAssembly::fixup_sleb128_i64);
+          PaddedSize = 10;
+          break;
+        case WebAssembly::OPERAND_FUNCTION32:
+        case WebAssembly::OPERAND_TABLE:
+        case WebAssembly::OPERAND_OFFSET32:
+        case WebAssembly::OPERAND_SIGNATURE:
+        case WebAssembly::OPERAND_TYPEINDEX:
+        case WebAssembly::OPERAND_GLOBAL:
+        case WebAssembly::OPERAND_TAG:
+          FixupKind = MCFixupKind(WebAssembly::fixup_uleb128_i32);
+          break;
+        case WebAssembly::OPERAND_OFFSET64:
+          FixupKind = MCFixupKind(WebAssembly::fixup_uleb128_i64);
+          PaddedSize = 10;
+          break;
+        default:
+          llvm_unreachable("unexpected symbolic operand kind");
+        }
+      } else {
+        // Variadic expr operands are try_table's catch/catch_ref clauses' tags.
+        assert(Opcode == WebAssembly::TRY_TABLE_S);
         FixupKind = MCFixupKind(WebAssembly::fixup_uleb128_i32);
-        break;
-      case WebAssembly::OPERAND_OFFSET64:
-        FixupKind = MCFixupKind(WebAssembly::fixup_uleb128_i64);
-        PaddedSize = 10;
-        break;
-      default:
-        llvm_unreachable("unexpected symbolic operand kind");
       }
       Fixups.push_back(MCFixup::create(OS.tell() - Start, MO.getExpr(),
                                        FixupKind, MI.getLoc()));
diff --git a/llvm/lib/Target/X86/X86DomainReassignment.cpp b/llvm/lib/Target/X86/X86DomainReassignment.cpp
index 831944cce3afdd..4823183113989a 100644
--- a/llvm/lib/Target/X86/X86DomainReassignment.cpp
+++ b/llvm/lib/Target/X86/X86DomainReassignment.cpp
@@ -636,21 +636,21 @@ void X86DomainReassignment::initConverters() {
   createReplacer(X86::MOV16rm, GET_EGPR_IF_ENABLED(X86::KMOVWkm));
   createReplacer(X86::MOV16mr, GET_EGPR_IF_ENABLED(X86::KMOVWmk));
   createReplacer(X86::MOV16rr, GET_EGPR_IF_ENABLED(X86::KMOVWkk));
-  createReplacer(X86::SHR16ri, X86::KSHIFTRWri);
-  createReplacer(X86::SHL16ri, X86::KSHIFTLWri);
-  createReplacer(X86::NOT16r, X86::KNOTWrr);
-  createReplacer(X86::OR16rr, X86::KORWrr);
-  createReplacer(X86::AND16rr, X86::KANDWrr);
-  createReplacer(X86::XOR16rr, X86::KXORWrr);
+  createReplacer(X86::SHR16ri, X86::KSHIFTRWki);
+  createReplacer(X86::SHL16ri, X86::KSHIFTLWki);
+  createReplacer(X86::NOT16r, X86::KNOTWkk);
+  createReplacer(X86::OR16rr, X86::KORWkk);
+  createReplacer(X86::AND16rr, X86::KANDWkk);
+  createReplacer(X86::XOR16rr, X86::KXORWkk);
 
   bool HasNDD = STI->hasNDD();
   if (HasNDD) {
-    createReplacer(X86::SHR16ri_ND, X86::KSHIFTRWri);
-    createReplacer(X86::SHL16ri_ND, X86::KSHIFTLWri);
-    createReplacer(X86::NOT16r_ND, X86::KNOTWrr);
-    createReplacer(X86::OR16rr_ND, X86::KORWrr);
-    createReplacer(X86::AND16rr_ND, X86::KANDWrr);
-    createReplacer(X86::XOR16rr_ND, X86::KXORWrr);
+    createReplacer(X86::SHR16ri_ND, X86::KSHIFTRWki);
+    createReplacer(X86::SHL16ri_ND, X86::KSHIFTLWki);
+    createReplacer(X86::NOT16r_ND, X86::KNOTWkk);
+    createReplacer(X86::OR16rr_ND, X86::KORWkk);
+    createReplacer(X86::AND16rr_ND, X86::KANDWkk);
+    createReplacer(X86::XOR16rr_ND, X86::KXORWkk);
   }
 
   if (STI->hasBWI()) {
@@ -663,86 +663,86 @@ void X86DomainReassignment::initConverters() {
     createReplacer(X86::MOV32rr, GET_EGPR_IF_ENABLED(X86::KMOVDkk));
     createReplacer(X86::MOV64rr, GET_EGPR_IF_ENABLED(X86::KMOVQkk));
 
-    createReplacer(X86::SHR32ri, X86::KSHIFTRDri);
-    createReplacer(X86::SHR64ri, X86::KSHIFTRQri);
+    createReplacer(X86::SHR32ri, X86::KSHIFTRDki);
+    createReplacer(X86::SHR64ri, X86::KSHIFTRQki);
 
-    createReplacer(X86::SHL32ri, X86::KSHIFTLDri);
-    createReplacer(X86::SHL64ri, X86::KSHIFTLQri);
+    createReplacer(X86::SHL32ri, X86::KSHIFTLDki);
+    createReplacer(X86::SHL64ri, X86::KSHIFTLQki);
 
-    createReplacer(X86::ADD32rr, X86::KADDDrr);
-    createReplacer(X86::ADD64rr, X86::KADDQrr);
+    createReplacer(X86::ADD32rr, X86::KADDDkk);
+    createReplacer(X86::ADD64rr, X86::KADDQkk);
 
-    createReplacer(X86::NOT32r, X86::KNOTDrr);
-    createReplacer(X86::NOT64r, X86::KNOTQrr);
+    createReplacer(X86::NOT32r, X86::KNOTDkk);
+    createReplacer(X86::NOT64r, X86::KNOTQkk);
 
-    createReplacer(X86::OR32rr, X86::KORDrr);
-    createReplacer(X86::OR64rr, X86::KORQrr);
+    createReplacer(X86::OR32rr, X86::KORDkk);
+    createReplacer(X86::OR64rr, X86::KORQkk);
 
-    createReplacer(X86::AND32rr, X86::KANDDrr);
-    createReplacer(X86::AND64rr, X86::KANDQrr);
+    createReplacer(X86::AND32rr, X86::KANDDkk);
+    createReplacer(X86::AND64rr, X86::KANDQkk);
 
-    createReplacer(X86::ANDN32rr, X86::KANDNDrr);
-    createReplacer(X86::ANDN64rr, X86::KANDNQrr);
+    createReplacer(X86::ANDN32rr, X86::KANDNDkk);
+    createReplacer(X86::ANDN64rr, X86::KANDNQkk);
 
-    createReplacer(X86::XOR32rr, X86::KXORDrr);
-    createReplacer(X86::XOR64rr, X86::KXORQrr);
+    createReplacer(X86::XOR32rr, X86::KXORDkk);
+    createReplacer(X86::XOR64rr, X86::KXORQkk);
 
     if (HasNDD) {
-      createReplacer(X86::SHR32ri_ND, X86::KSHIFTRDri);
-      createReplacer(X86::SHL32ri_ND, X86::KSHIFTLDri);
-      createReplacer(X86::ADD32rr_ND, X86::KADDDrr);
-      createReplacer(X86::NOT32r_ND, X86::KNOTDrr);
-      createReplacer(X86::OR32rr_ND, X86::KORDrr);
-      createReplacer(X86::AND32rr_ND, X86::KANDDrr);
-      createReplacer(X86::XOR32rr_ND, X86::KXORDrr);
-      createReplacer(X86::SHR64ri_ND, X86::KSHIFTRQri);
-      createReplacer(X86::SHL64ri_ND, X86::KSHIFTLQri);
-      createReplacer(X86::ADD64rr_ND, X86::KADDQrr);
-      createReplacer(X86::NOT64r_ND, X86::KNOTQrr);
-      createReplacer(X86::OR64rr_ND, X86::KORQrr);
-      createReplacer(X86::AND64rr_ND, X86::KANDQrr);
-      createReplacer(X86::XOR64rr_ND, X86::KXORQrr);
+      createReplacer(X86::SHR32ri_ND, X86::KSHIFTRDki);
+      createReplacer(X86::SHL32ri_ND, X86::KSHIFTLDki);
+      createReplacer(X86::ADD32rr_ND, X86::KADDDkk);
+      createReplacer(X86::NOT32r_ND, X86::KNOTDkk);
+      createReplacer(X86::OR32rr_ND, X86::KORDkk);
+      createReplacer(X86::AND32rr_ND, X86::KANDDkk);
+      createReplacer(X86::XOR32rr_ND, X86::KXORDkk);
+      createReplacer(X86::SHR64ri_ND, X86::KSHIFTRQki);
+      createReplacer(X86::SHL64ri_ND, X86::KSHIFTLQki);
+      createReplacer(X86::ADD64rr_ND, X86::KADDQkk);
+      createReplacer(X86::NOT64r_ND, X86::KNOTQkk);
+      createReplacer(X86::OR64rr_ND, X86::KORQkk);
+      createReplacer(X86::AND64rr_ND, X86::KANDQkk);
+      createReplacer(X86::XOR64rr_ND, X86::KXORQkk);
     }
 
     // TODO: KTEST is not a replacement for TEST due to flag differences. Need
     // to prove only Z flag is used.
-    // createReplacer(X86::TEST32rr, X86::KTESTDrr);
-    // createReplacer(X86::TEST64rr, X86::KTESTQrr);
+    // createReplacer(X86::TEST32rr, X86::KTESTDkk);
+    // createReplacer(X86::TEST64rr, X86::KTESTQkk);
   }
 
   if (STI->hasDQI()) {
-    createReplacer(X86::ADD8rr, X86::KADDBrr);
-    createReplacer(X86::ADD16rr, X86::KADDWrr);
+    createReplacer(X86::ADD8rr, X86::KADDBkk);
+    createReplacer(X86::ADD16rr, X86::KADDWkk);
 
-    createReplacer(X86::AND8rr, X86::KANDBrr);
+    createReplacer(X86::AND8rr, X86::KANDBkk);
 
     createReplacer(X86::MOV8rm, GET_EGPR_IF_ENABLED(X86::KMOVBkm));
     createReplacer(X86::MOV8mr, GET_EGPR_IF_ENABLED(X86::KMOVBmk));
     createReplacer(X86::MOV8rr, GET_EGPR_IF_ENABLED(X86::KMOVBkk));
 
-    createReplacer(X86::NOT8r, X86::KNOTBrr);
+    createReplacer(X86::NOT8r, X86::KNOTBkk);
 
-    createReplacer(X86::OR8rr, X86::KORBrr);
+    createReplacer(X86::OR8rr, X86::KORBkk);
 
-    createReplacer(X86::SHR8ri, X86::KSHIFTRBri);
-    createReplacer(X86::SHL8ri, X86::KSHIFTLBri);
+    createReplacer(X86::SHR8ri, X86::KSHIFTRBki);
+    createReplacer(X86::SHL8ri, X86::KSHIFTLBki);
 
     // TODO: KTEST is not a replacement for TEST due to flag differences. Need
     // to prove only Z flag is used.
-    // createReplacer(X86::TEST8rr, X86::KTESTBrr);
-    // createReplacer(X86::TEST16rr, X86::KTESTWrr);
+    // createReplacer(X86::TEST8rr, X86::KTESTBkk);
+    // createReplacer(X86::TEST16rr, X86::KTESTWkk);
 
-    createReplacer(X86::XOR8rr, X86::KXORBrr);
+    createReplacer(X86::XOR8rr, X86::KXORBkk);
 
     if (HasNDD) {
-      createReplacer(X86::ADD8rr_ND, X86::KADDBrr);
-      createReplacer(X86::ADD16rr_ND, X86::KADDWrr);
-      createReplacer(X86::AND8rr_ND, X86::KANDBrr);
-      createReplacer(X86::NOT8r_ND, X86::KNOTBrr);
-      createReplacer(X86::OR8rr_ND, X86::KORBrr);
-      createReplacer(X86::SHR8ri_ND, X86::KSHIFTRBri);
-      createReplacer(X86::SHL8ri_ND, X86::KSHIFTLBri);
-      createReplacer(X86::XOR8rr_ND, X86::KXORBrr);
+      createReplacer(X86::ADD8rr_ND, X86::KADDBkk);
+      createReplacer(X86::ADD16rr_ND, X86::KADDWkk);
+      createReplacer(X86::AND8rr_ND, X86::KANDBkk);
+      createReplacer(X86::NOT8r_ND, X86::KNOTBkk);
+      createReplacer(X86::OR8rr_ND, X86::KORBkk);
+      createReplacer(X86::SHR8ri_ND, X86::KSHIFTRBki);
+      createReplacer(X86::SHL8ri_ND, X86::KSHIFTLBki);
+      createReplacer(X86::XOR8rr_ND, X86::KXORBkk);
     }
   }
 #undef GET_EGPR_IF_ENABLED
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index d0a54ab8993c26..e8a9da1f17d962 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -1647,10 +1647,10 @@ void X86DAGToDAGISel::PostprocessISelDAG() {
     // used. We're doing this late so we can prefer to fold the AND into masked
     // comparisons. Doing that can be better for the live range of the mask
     // register.
-    case X86::KORTESTBrr:
-    case X86::KORTESTWrr:
-    case X86::KORTESTDrr:
-    case X86::KORTESTQrr: {
+    case X86::KORTESTBkk:
+    case X86::KORTESTWkk:
+    case X86::KORTESTDkk:
+    case X86::KORTESTQkk: {
       SDValue Op0 = N->getOperand(0);
       if (Op0 != N->getOperand(1) || !N->isOnlyUserOf(Op0.getNode()) ||
           !Op0.isMachineOpcode() || !onlyUsesZeroFlag(SDValue(N, 0)))
@@ -1661,10 +1661,10 @@ void X86DAGToDAGISel::PostprocessISelDAG() {
       switch (Op0.getMachineOpcode()) {
       default:
         continue;
-        CASE(KANDBrr)
-        CASE(KANDWrr)
-        CASE(KANDDrr)
-        CASE(KANDQrr)
+        CASE(KANDBkk)
+        CASE(KANDWkk)
+        CASE(KANDDkk)
+        CASE(KANDQkk)
       }
       unsigned NewOpc;
 #define FROM_TO(A, B)                                                          \
@@ -1672,14 +1672,14 @@ void X86DAGToDAGISel::PostprocessISelDAG() {
     NewOpc = X86::B;                                                           \
     break;
       switch (Opc) {
-        FROM_TO(KORTESTBrr, KTESTBrr)
-        FROM_TO(KORTESTWrr, KTESTWrr)
-        FROM_TO(KORTESTDrr, KTESTDrr)
-        FROM_TO(KORTESTQrr, KTESTQrr)
+        FROM_TO(KORTESTBkk, KTESTBkk)
+        FROM_TO(KORTESTWkk, KTESTWkk)
+        FROM_TO(KORTESTDkk, KTESTDkk)
+        FROM_TO(KORTESTQkk, KTESTQkk)
       }
       // KANDW is legal with AVX512F, but KTESTW requires AVX512DQ. The other
       // KAND instructions and KTEST use the same ISA feature.
-      if (NewOpc == X86::KTESTWrr && !Subtarget->hasDQI())
+      if (NewOpc == X86::KTESTWkk && !Subtarget->hasDQI())
         continue;
 #undef FROM_TO
       MachineSDNode *KTest = CurDAG->getMachineNode(
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 2946c2b43112b0..f6d42ade600885 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -39710,13 +39710,23 @@ static SDValue combineX86ShuffleChainWithExtract(
   unsigned RootEltSizeInBits = RootSizeInBits / NumMaskElts;
   assert((RootSizeInBits % NumMaskElts) == 0 && "Unexpected root shuffle mask");
 
-  // Peek through extract_subvector to find widest legal vector.
+  // Peek through subvectors to find widest legal vector.
   // TODO: Handle ISD::TRUNCATE
   unsigned WideSizeInBits = RootSizeInBits;
-  for (unsigned I = 0; I != NumInputs; ++I) {
-    SDValue Input = peekThroughBitcasts(Inputs[I]);
-    while (Input.getOpcode() == ISD::EXTRACT_SUBVECTOR)
-      Input = peekThroughBitcasts(Input.getOperand(0));
+  for (SDValue Input : Inputs) {
+    Input = peekThroughBitcasts(Input);
+    while (1) {
+      if (Input.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
+        Input = peekThroughBitcasts(Input.getOperand(0));
+        continue;
+      }
+      if (Input.getOpcode() == ISD::INSERT_SUBVECTOR &&
+          Input.getOperand(0).isUndef()) {
+        Input = peekThroughBitcasts(Input.getOperand(1));
+        continue;
+      }
+      break;
+    }
     if (DAG.getTargetLoweringInfo().isTypeLegal(Input.getValueType()) &&
         WideSizeInBits < Input.getValueSizeInBits())
       WideSizeInBits = Input.getValueSizeInBits();
@@ -39744,21 +39754,32 @@ static SDValue combineX86ShuffleChainWithExtract(
   for (unsigned I = 0; I != NumInputs; ++I) {
     SDValue &Input = WideInputs[I];
     Input = peekThroughBitcasts(Input);
-    while (Input.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
-           Input.getOperand(0).getValueSizeInBits() <= WideSizeInBits) {
-      uint64_t Idx = Input.getConstantOperandVal(1);
-      if (Idx != 0) {
-        ++AdjustedMasks;
-        unsigned InputEltSizeInBits = Input.getScalarValueSizeInBits();
-        Idx = (Idx * InputEltSizeInBits) / RootEltSizeInBits;
-
-        int lo = I * WideMask.size();
-        int hi = (I + 1) * WideMask.size();
-        for (int &M : WideMask)
-          if (lo <= M && M < hi)
-            M += Idx;
+    while (1) {
+      if (Input.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
+          Input.getOperand(0).getValueSizeInBits() <= WideSizeInBits) {
+        uint64_t Idx = Input.getConstantOperandVal(1);
+        if (Idx != 0) {
+          ++AdjustedMasks;
+          unsigned InputEltSizeInBits = Input.getScalarValueSizeInBits();
+          Idx = (Idx * InputEltSizeInBits) / RootEltSizeInBits;
+
+          int lo = I * WideMask.size();
+          int hi = (I + 1) * WideMask.size();
+          for (int &M : WideMask)
+            if (lo <= M && M < hi)
+              M += Idx;
+        }
+        Input = peekThroughBitcasts(Input.getOperand(0));
+        continue;
       }
-      Input = peekThroughBitcasts(Input.getOperand(0));
+      // TODO: Handle insertions into upper subvectors.
+      if (Input.getOpcode() == ISD::INSERT_SUBVECTOR &&
+          Input.getOperand(0).isUndef() &&
+          isNullConstant(Input.getOperand(2))) {
+        Input = peekThroughBitcasts(Input.getOperand(1));
+        continue;
+      }
+      break;
     }
   }
 
@@ -56929,6 +56950,23 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
         }
       }
       break;
+    case ISD::ANY_EXTEND:
+    case ISD::SIGN_EXTEND:
+    case ISD::ZERO_EXTEND:
+      // TODO: Handle ANY_EXTEND combos with SIGN/ZERO_EXTEND.
+      if (!IsSplat && NumOps == 2 &&
+          ((VT.is256BitVector() && Subtarget.hasInt256()) ||
+           (VT.is512BitVector() && Subtarget.useAVX512Regs() &&
+            (EltSizeInBits >= 32 || Subtarget.useBWIRegs())))) {
+        EVT SrcVT = Ops[0].getOperand(0).getValueType();
+        if (SrcVT.isSimple() && SrcVT.is128BitVector() &&
+            SrcVT == Ops[1].getOperand(0).getValueType()) {
+          EVT NewSrcVT = SrcVT.getDoubleNumVectorElementsVT(Ctx);
+          return DAG.getNode(Op0.getOpcode(), DL, VT,
+                             ConcatSubOperand(NewSrcVT, Ops, 0));
+        }
+      }
+      break;
     case X86ISD::VSHLI:
     case X86ISD::VSRLI:
       // Special case: SHL/SRL AVX1 V4i64 by 32-bits can lower as a shuffle.
diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index b6bf34a8a0d31c..9ed59803c1f9d9 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -2782,7 +2782,7 @@ multiclass avx512_mask_unop<bits<8> opc, string OpcodeStr,
                             RegisterClass KRC, SDPatternOperator OpNode,
                             X86FoldableSchedWrite sched, Predicate prd> {
   let Predicates = [prd] in
-    def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src),
+    def kk : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src),
                !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
                [(set KRC:$dst, (OpNode KRC:$src))]>,
                Sched<[sched]>;
@@ -2807,14 +2807,14 @@ defm KNOT : avx512_mask_unop_all<0x44, "knot", vnot, SchedWriteVecLogic.XMM>;
 // KNL does not support KMOVB, 8-bit mask is promoted to 16-bit
 let Predicates = [HasAVX512, NoDQI] in
 def : Pat<(vnot VK8:$src),
-          (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$src, VK16)), VK8)>;
+          (COPY_TO_REGCLASS (KNOTWkk (COPY_TO_REGCLASS VK8:$src, VK16)), VK8)>;
 
 def : Pat<(vnot VK4:$src),
-          (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK4:$src, VK16)), VK4)>;
+          (COPY_TO_REGCLASS (KNOTWkk (COPY_TO_REGCLASS VK4:$src, VK16)), VK4)>;
 def : Pat<(vnot VK2:$src),
-          (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK2:$src, VK16)), VK2)>;
+          (COPY_TO_REGCLASS (KNOTWkk (COPY_TO_REGCLASS VK2:$src, VK16)), VK2)>;
 def : Pat<(vnot VK1:$src),
-          (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK1:$src, VK16)), VK2)>;
+          (COPY_TO_REGCLASS (KNOTWkk (COPY_TO_REGCLASS VK1:$src, VK16)), VK2)>;
 
 // Mask binary operation
 // - KAND, KANDN, KOR, KXNOR, KXOR
@@ -2823,7 +2823,7 @@ multiclass avx512_mask_binop<bits<8> opc, string OpcodeStr,
                            X86FoldableSchedWrite sched, Predicate prd,
                            bit IsCommutable> {
   let Predicates = [prd], isCommutable = IsCommutable in
-    def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2),
+    def kk : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2),
                !strconcat(OpcodeStr,
                           "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                [(set KRC:$dst, (OpNode KRC:$src1, KRC:$src2))]>,
@@ -2877,11 +2877,11 @@ multiclass avx512_binop_pat<SDPatternOperator VOpNode,
                            (COPY_TO_REGCLASS VK4:$src2, VK16)), VK4)>;
 }
 
-defm : avx512_binop_pat<and,   KANDWrr>;
-defm : avx512_binop_pat<vandn, KANDNWrr>;
-defm : avx512_binop_pat<or,    KORWrr>;
-defm : avx512_binop_pat<vxnor, KXNORWrr>;
-defm : avx512_binop_pat<xor,   KXORWrr>;
+defm : avx512_binop_pat<and,   KANDWkk>;
+defm : avx512_binop_pat<vandn, KANDNWkk>;
+defm : avx512_binop_pat<or,    KORWkk>;
+defm : avx512_binop_pat<vxnor, KXNORWkk>;
+defm : avx512_binop_pat<xor,   KXORWkk>;
 
 // Mask unpacking
 multiclass avx512_mask_unpck<string Suffix, X86KVectorVTInfo Dst,
@@ -2889,13 +2889,13 @@ multiclass avx512_mask_unpck<string Suffix, X86KVectorVTInfo Dst,
                              Predicate prd> {
   let Predicates = [prd] in {
     let hasSideEffects = 0 in
-    def rr : I<0x4b, MRMSrcReg, (outs Dst.KRC:$dst),
+    def kk : I<0x4b, MRMSrcReg, (outs Dst.KRC:$dst),
                (ins Src.KRC:$src1, Src.KRC:$src2),
                "kunpck"#Suffix#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
                VEX, VVVV, VEX_L, Sched<[sched]>;
 
     def : Pat<(Dst.KVT (concat_vectors Src.KRC:$src1, Src.KRC:$src2)),
-              (!cast<Instruction>(NAME#rr) Src.KRC:$src2, Src.KRC:$src1)>;
+              (!cast<Instruction>(NAME#kk) Src.KRC:$src2, Src.KRC:$src1)>;
   }
 }
 
@@ -2908,7 +2908,7 @@ multiclass avx512_mask_testop<bits<8> opc, string OpcodeStr, RegisterClass KRC,
                               SDNode OpNode, X86FoldableSchedWrite sched,
                               Predicate prd> {
   let Predicates = [prd], Defs = [EFLAGS] in
-    def rr : I<opc, MRMSrcReg, (outs), (ins KRC:$src1, KRC:$src2),
+    def kk : I<opc, MRMSrcReg, (outs), (ins KRC:$src1, KRC:$src2),
                !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"),
                [(set EFLAGS, (OpNode KRC:$src1, KRC:$src2))]>,
                Sched<[sched]>;
@@ -2935,7 +2935,7 @@ defm KTEST   : avx512_mask_testop_w<0x99, "ktest", X86ktest, SchedWriteVecLogic.
 multiclass avx512_mask_shiftop<bits<8> opc, string OpcodeStr, RegisterClass KRC,
                                SDNode OpNode, X86FoldableSchedWrite sched> {
   let Predicates = [HasAVX512] in
-    def ri : Ii8<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src, u8imm:$imm),
+    def ki : Ii8<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src, u8imm:$imm),
                  !strconcat(OpcodeStr,
                             "\t{$imm, $src, $dst|$dst, $src, $imm}"),
                             [(set KRC:$dst, (OpNode KRC:$src, (i8 timm:$imm)))]>,
@@ -3463,12 +3463,12 @@ def VMOVUPSZ256mr_NOVLX : I<0, Pseudo, (outs), (ins f256mem:$dst, VR256X:$src),
 
 def : Pat<(v8i64 (vselect VK8WM:$mask, (v8i64 immAllZerosV),
                           (v8i64 VR512:$src))),
-   (VMOVDQA64Zrrkz (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$mask, VK16)),
+   (VMOVDQA64Zrrkz (COPY_TO_REGCLASS (KNOTWkk (COPY_TO_REGCLASS VK8:$mask, VK16)),
                                               VK8), VR512:$src)>;
 
 def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 immAllZerosV),
                            (v16i32 VR512:$src))),
-                  (VMOVDQA32Zrrkz (KNOTWrr VK16WM:$mask), VR512:$src)>;
+                  (VMOVDQA32Zrrkz (KNOTWkk VK16WM:$mask), VR512:$src)>;
 
 // These patterns exist to prevent the above patterns from introducing a second
 // mask inversion when one already exists.
@@ -10425,7 +10425,7 @@ defm VSCATTERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qpd
                      VK8WM, vz512mem>, EVEX_V512, REX_W, EVEX_CD8<64, CD8VT1>;
 
 multiclass cvt_by_vec_width<bits<8> opc, X86VectorVTInfo Vec, string OpcodeStr, SchedWrite Sched> {
-def rr : AVX512XS8I<opc, MRMSrcReg, (outs Vec.RC:$dst), (ins Vec.KRC:$src),
+def rk : AVX512XS8I<opc, MRMSrcReg, (outs Vec.RC:$dst), (ins Vec.KRC:$src),
                   !strconcat(OpcodeStr#Vec.Suffix, "\t{$src, $dst|$dst, $src}"),
                   [(set Vec.RC:$dst, (Vec.VT (sext Vec.KRC:$src)))]>,
                   EVEX, Sched<[Sched]>;
@@ -10448,7 +10448,7 @@ defm VPMOVM2D : cvt_mask_by_elt_width<0x38, avx512vl_i32_info, "vpmovm2", HasDQI
 defm VPMOVM2Q : cvt_mask_by_elt_width<0x38, avx512vl_i64_info, "vpmovm2", HasDQI> , REX_W;
 
 multiclass convert_vector_to_mask_common<bits<8> opc, X86VectorVTInfo _, string OpcodeStr > {
-    def rr : AVX512XS8I<opc, MRMSrcReg, (outs _.KRC:$dst), (ins _.RC:$src),
+    def kr : AVX512XS8I<opc, MRMSrcReg, (outs _.KRC:$dst), (ins _.RC:$src),
                         !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
                         [(set _.KRC:$dst, (X86pcmpgtm _.ImmAllZerosV, (_.VT _.RC:$src)))]>,
                         EVEX, Sched<[WriteMove]>;
@@ -10461,7 +10461,7 @@ multiclass convert_vector_to_mask_lowering<X86VectorVTInfo ExtendInfo,
 
   def : Pat<(_.KVT (X86pcmpgtm _.ImmAllZerosV, (_.VT _.RC:$src))),
             (_.KVT (COPY_TO_REGCLASS
-                     (!cast<Instruction>(Name#"Zrr")
+                     (!cast<Instruction>(Name#"Zkr")
                        (INSERT_SUBREG (ExtendInfo.VT (IMPLICIT_DEF)),
                                       _.RC:$src, _.SubRegIdx)),
                    _.KRC))>;
@@ -10499,14 +10499,14 @@ defm VPMOVQ2M : avx512_convert_vector_to_mask<0x39, "vpmovq2m",
 // a target independent DAG combine likes to combine sext and trunc.
 let Predicates = [HasDQI, NoBWI] in {
   def : Pat<(v16i8 (sext (v16i1 VK16:$src))),
-            (VPMOVDBZrr (v16i32 (VPMOVM2DZrr VK16:$src)))>;
+            (VPMOVDBZrr (v16i32 (VPMOVM2DZrk VK16:$src)))>;
   def : Pat<(v16i16 (sext (v16i1 VK16:$src))),
-            (VPMOVDWZrr (v16i32 (VPMOVM2DZrr VK16:$src)))>;
+            (VPMOVDWZrr (v16i32 (VPMOVM2DZrk VK16:$src)))>;
 }
 
 let Predicates = [HasDQI, NoBWI, HasVLX] in {
   def : Pat<(v8i16 (sext (v8i1 VK8:$src))),
-            (VPMOVDWZ256rr (v8i32 (VPMOVM2DZ256rr VK8:$src)))>;
+            (VPMOVDWZ256rr (v8i32 (VPMOVM2DZ256rk VK8:$src)))>;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index a74da000af0cee..57a894b09e0445 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -6323,17 +6323,17 @@ bool X86InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
   // FIXME: A more advanced approach would be to choose the best input mask
   // register based on context.
   case X86::KSET0W:
-    return Expand2AddrKreg(MIB, get(X86::KXORWrr), X86::K0);
+    return Expand2AddrKreg(MIB, get(X86::KXORWkk), X86::K0);
   case X86::KSET0D:
-    return Expand2AddrKreg(MIB, get(X86::KXORDrr), X86::K0);
+    return Expand2AddrKreg(MIB, get(X86::KXORDkk), X86::K0);
   case X86::KSET0Q:
-    return Expand2AddrKreg(MIB, get(X86::KXORQrr), X86::K0);
+    return Expand2AddrKreg(MIB, get(X86::KXORQkk), X86::K0);
   case X86::KSET1W:
-    return Expand2AddrKreg(MIB, get(X86::KXNORWrr), X86::K0);
+    return Expand2AddrKreg(MIB, get(X86::KXNORWkk), X86::K0);
   case X86::KSET1D:
-    return Expand2AddrKreg(MIB, get(X86::KXNORDrr), X86::K0);
+    return Expand2AddrKreg(MIB, get(X86::KXNORDkk), X86::K0);
   case X86::KSET1Q:
-    return Expand2AddrKreg(MIB, get(X86::KXNORQrr), X86::K0);
+    return Expand2AddrKreg(MIB, get(X86::KXNORQkk), X86::K0);
   case TargetOpcode::LOAD_STACK_GUARD:
     expandLoadStackGuard(MIB, *this);
     return true;
@@ -9805,22 +9805,22 @@ bool X86InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
   case X86::VXORPSZ256rr:
   case X86::VXORPDZrr:
   case X86::VXORPSZrr:
-  case X86::KADDBrr:
-  case X86::KADDWrr:
-  case X86::KADDDrr:
-  case X86::KADDQrr:
-  case X86::KANDBrr:
-  case X86::KANDWrr:
-  case X86::KANDDrr:
-  case X86::KANDQrr:
-  case X86::KORBrr:
-  case X86::KORWrr:
-  case X86::KORDrr:
-  case X86::KORQrr:
-  case X86::KXORBrr:
-  case X86::KXORWrr:
-  case X86::KXORDrr:
-  case X86::KXORQrr:
+  case X86::KADDBkk:
+  case X86::KADDWkk:
+  case X86::KADDDkk:
+  case X86::KADDQkk:
+  case X86::KANDBkk:
+  case X86::KANDWkk:
+  case X86::KANDDkk:
+  case X86::KANDQkk:
+  case X86::KORBkk:
+  case X86::KORWkk:
+  case X86::KORDkk:
+  case X86::KORQkk:
+  case X86::KXORBkk:
+  case X86::KXORWkk:
+  case X86::KXORDkk:
+  case X86::KXORQkk:
   case X86::VPADDBrr:
   case X86::VPADDWrr:
   case X86::VPADDDrr:
@@ -10729,7 +10729,7 @@ void X86InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
       return;
 
     // KXOR is safe to use because it doesn't affect flags.
-    unsigned Op = ST.hasBWI() ? X86::KXORQrr : X86::KXORWrr;
+    unsigned Op = ST.hasBWI() ? X86::KXORQkk : X86::KXORWkk;
     BuildMI(MBB, Iter, DL, get(Op), Reg)
         .addReg(Reg, RegState::Undef)
         .addReg(Reg, RegState::Undef);
diff --git a/llvm/lib/Target/X86/X86InstrVecCompiler.td b/llvm/lib/Target/X86/X86InstrVecCompiler.td
index 461b2badc13134..122627ca45d31f 100644
--- a/llvm/lib/Target/X86/X86InstrVecCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrVecCompiler.td
@@ -280,24 +280,24 @@ let Predicates = [HasBWI, HasVLX] in {
 let Predicates = [HasAVX512] in {
   def : Pat<(v16i1 (insert_subvector (v16i1 immAllZerosV),
                                      (v1i1 VK1:$mask), (iPTR 0))),
-            (KSHIFTRWri (KSHIFTLWri (COPY_TO_REGCLASS VK1:$mask, VK16),
+            (KSHIFTRWki (KSHIFTLWki (COPY_TO_REGCLASS VK1:$mask, VK16),
                                     (i8 15)), (i8 15))>;
 
   def : Pat<(v16i1 (insert_subvector (v16i1 immAllZerosV),
                                      (v2i1 VK2:$mask), (iPTR 0))),
-            (KSHIFTRWri (KSHIFTLWri (COPY_TO_REGCLASS VK2:$mask, VK16),
+            (KSHIFTRWki (KSHIFTLWki (COPY_TO_REGCLASS VK2:$mask, VK16),
                                     (i8 14)), (i8 14))>;
 
   def : Pat<(v16i1 (insert_subvector (v16i1 immAllZerosV),
                                      (v4i1 VK4:$mask), (iPTR 0))),
-            (KSHIFTRWri (KSHIFTLWri (COPY_TO_REGCLASS VK4:$mask, VK16),
+            (KSHIFTRWki (KSHIFTLWki (COPY_TO_REGCLASS VK4:$mask, VK16),
                                     (i8 12)), (i8 12))>;
 }
 
 let Predicates = [HasAVX512, NoDQI] in {
   def : Pat<(v16i1 (insert_subvector (v16i1 immAllZerosV),
                                      (v8i1 VK8:$mask), (iPTR 0))),
-            (KSHIFTRWri (KSHIFTLWri (COPY_TO_REGCLASS VK8:$mask, VK16),
+            (KSHIFTRWki (KSHIFTLWki (COPY_TO_REGCLASS VK8:$mask, VK16),
                                     (i8 8)), (i8 8))>;
 }
 
@@ -308,15 +308,15 @@ let Predicates = [HasDQI] in {
 
   def : Pat<(v8i1 (insert_subvector (v8i1 immAllZerosV),
                                     (v1i1 VK1:$mask), (iPTR 0))),
-            (KSHIFTRBri (KSHIFTLBri (COPY_TO_REGCLASS VK1:$mask, VK8),
+            (KSHIFTRBki (KSHIFTLBki (COPY_TO_REGCLASS VK1:$mask, VK8),
                                     (i8 7)), (i8 7))>;
   def : Pat<(v8i1 (insert_subvector (v8i1 immAllZerosV),
                                     (v2i1 VK2:$mask), (iPTR 0))),
-            (KSHIFTRBri (KSHIFTLBri (COPY_TO_REGCLASS VK2:$mask, VK8),
+            (KSHIFTRBki (KSHIFTLBki (COPY_TO_REGCLASS VK2:$mask, VK8),
                                     (i8 6)), (i8 6))>;
   def : Pat<(v8i1 (insert_subvector (v8i1 immAllZerosV),
                                     (v4i1 VK4:$mask), (iPTR 0))),
-            (KSHIFTRBri (KSHIFTLBri (COPY_TO_REGCLASS VK4:$mask, VK8),
+            (KSHIFTRBki (KSHIFTLBki (COPY_TO_REGCLASS VK4:$mask, VK8),
                                     (i8 4)), (i8 4))>;
 }
 
@@ -336,12 +336,12 @@ let Predicates = [HasBWI] in {
 let Predicates = [HasBWI, NoDQI] in {
   def : Pat<(v32i1 (insert_subvector (v32i1 immAllZerosV),
                                      (v8i1 VK8:$mask), (iPTR 0))),
-            (KSHIFTRDri (KSHIFTLDri (COPY_TO_REGCLASS VK8:$mask, VK32),
+            (KSHIFTRDki (KSHIFTLDki (COPY_TO_REGCLASS VK8:$mask, VK32),
                                     (i8 24)), (i8 24))>;
 
   def : Pat<(v64i1 (insert_subvector (v64i1 immAllZerosV),
                                      (v8i1 VK8:$mask), (iPTR 0))),
-            (KSHIFTRQri (KSHIFTLQri (COPY_TO_REGCLASS VK8:$mask, VK64),
+            (KSHIFTRQki (KSHIFTLQki (COPY_TO_REGCLASS VK8:$mask, VK64),
                                     (i8 56)), (i8 56))>;
 }
 
@@ -358,28 +358,28 @@ let Predicates = [HasBWI, HasDQI] in {
 let Predicates = [HasBWI] in {
   def : Pat<(v32i1 (insert_subvector (v32i1 immAllZerosV),
                                      (v1i1 VK1:$mask), (iPTR 0))),
-            (KSHIFTRDri (KSHIFTLDri (COPY_TO_REGCLASS VK1:$mask, VK32),
+            (KSHIFTRDki (KSHIFTLDki (COPY_TO_REGCLASS VK1:$mask, VK32),
                                     (i8 31)), (i8 31))>;
   def : Pat<(v32i1 (insert_subvector (v32i1 immAllZerosV),
                                      (v2i1 VK2:$mask), (iPTR 0))),
-            (KSHIFTRDri (KSHIFTLDri (COPY_TO_REGCLASS VK2:$mask, VK32),
+            (KSHIFTRDki (KSHIFTLDki (COPY_TO_REGCLASS VK2:$mask, VK32),
                                     (i8 30)), (i8 30))>;
   def : Pat<(v32i1 (insert_subvector (v32i1 immAllZerosV),
                                      (v4i1 VK4:$mask), (iPTR 0))),
-            (KSHIFTRDri (KSHIFTLDri (COPY_TO_REGCLASS VK4:$mask, VK32),
+            (KSHIFTRDki (KSHIFTLDki (COPY_TO_REGCLASS VK4:$mask, VK32),
                                     (i8 28)), (i8 28))>;
 
   def : Pat<(v64i1 (insert_subvector (v64i1 immAllZerosV),
                                      (v1i1 VK1:$mask), (iPTR 0))),
-            (KSHIFTRQri (KSHIFTLQri (COPY_TO_REGCLASS VK1:$mask, VK64),
+            (KSHIFTRQki (KSHIFTLQki (COPY_TO_REGCLASS VK1:$mask, VK64),
                                     (i8 63)), (i8 63))>;
   def : Pat<(v64i1 (insert_subvector (v64i1 immAllZerosV),
                                      (v2i1 VK2:$mask), (iPTR 0))),
-            (KSHIFTRQri (KSHIFTLQri (COPY_TO_REGCLASS VK2:$mask, VK64),
+            (KSHIFTRQki (KSHIFTLQki (COPY_TO_REGCLASS VK2:$mask, VK64),
                                     (i8 62)), (i8 62))>;
   def : Pat<(v64i1 (insert_subvector (v64i1 immAllZerosV),
                                      (v4i1 VK4:$mask), (iPTR 0))),
-            (KSHIFTRQri (KSHIFTLQri (COPY_TO_REGCLASS VK4:$mask, VK64),
+            (KSHIFTRQki (KSHIFTLQki (COPY_TO_REGCLASS VK4:$mask, VK64),
                                     (i8 60)), (i8 60))>;
 }
 
diff --git a/llvm/lib/Target/X86/X86SchedIceLake.td b/llvm/lib/Target/X86/X86SchedIceLake.td
index b32db53ff08cde..72fbcc5598108f 100644
--- a/llvm/lib/Target/X86/X86SchedIceLake.td
+++ b/llvm/lib/Target/X86/X86SchedIceLake.td
@@ -633,13 +633,13 @@ def ICXWriteResGroup1 : SchedWriteRes<[ICXPort0]> {
   let NumMicroOps = 1;
   let ReleaseAtCycles = [1];
 }
-def: InstRW<[ICXWriteResGroup1], (instregex "KAND(B|D|Q|W)rr",
-                                            "KANDN(B|D|Q|W)rr",
+def: InstRW<[ICXWriteResGroup1], (instregex "KAND(B|D|Q|W)kk",
+                                            "KANDN(B|D|Q|W)kk",
                                             "KMOV(B|D|Q|W)kk",
-                                            "KNOT(B|D|Q|W)rr",
-                                            "KOR(B|D|Q|W)rr",
-                                            "KXNOR(B|D|Q|W)rr",
-                                            "KXOR(B|D|Q|W)rr",
+                                            "KNOT(B|D|Q|W)kk",
+                                            "KOR(B|D|Q|W)kk",
+                                            "KXNOR(B|D|Q|W)kk",
+                                            "KXOR(B|D|Q|W)kk",
                                             "KSET0(B|D|Q|W)", // Same as KXOR
                                             "KSET1(B|D|Q|W)", // Same as KXNOR
                                             "MMX_PADDS(B|W)rr",
@@ -651,10 +651,10 @@ def: InstRW<[ICXWriteResGroup1], (instregex "KAND(B|D|Q|W)rr",
                                             "MMX_P(MAX|MIN)UBrr",
                                             "MMX_PSUBS(B|W)rr",
                                             "MMX_PSUBUS(B|W)rr",
-                                            "VPMOVB2M(Z|Z128|Z256)rr",
-                                            "VPMOVD2M(Z|Z128|Z256)rr",
-                                            "VPMOVQ2M(Z|Z128|Z256)rr",
-                                            "VPMOVW2M(Z|Z128|Z256)rr")>;
+                                            "VPMOVB2M(Z|Z128|Z256)kr",
+                                            "VPMOVD2M(Z|Z128|Z256)kr",
+                                            "VPMOVQ2M(Z|Z128|Z256)kr",
+                                            "VPMOVW2M(Z|Z128|Z256)kr")>;
 
 def ICXWriteResGroup3 : SchedWriteRes<[ICXPort5]> {
   let Latency = 1;
@@ -826,8 +826,8 @@ def ICXWriteResGroup30 : SchedWriteRes<[ICXPort0]> {
   let ReleaseAtCycles = [1];
 }
 def: InstRW<[ICXWriteResGroup30], (instregex "KMOV(B|D|Q|W)rk",
-                                             "KORTEST(B|D|Q|W)rr",
-                                             "KTEST(B|D|Q|W)rr")>;
+                                             "KORTEST(B|D|Q|W)kk",
+                                             "KTEST(B|D|Q|W)kk")>;
 
 def ICXWriteResGroup31 : SchedWriteRes<[ICXPort1]> {
   let Latency = 3;
@@ -854,10 +854,10 @@ def ICXWriteResGroup33 : SchedWriteRes<[ICXPort5]> {
   let NumMicroOps = 1;
   let ReleaseAtCycles = [1];
 }
-def: InstRW<[ICXWriteResGroup33], (instregex "KADD(B|D|Q|W)rr",
-                                             "KSHIFTL(B|D|Q|W)ri",
-                                             "KSHIFTR(B|D|Q|W)ri",
-                                             "KUNPCK(BW|DQ|WD)rr",
+def: InstRW<[ICXWriteResGroup33], (instregex "KADD(B|D|Q|W)kk",
+                                             "KSHIFTL(B|D|Q|W)ki",
+                                             "KSHIFTR(B|D|Q|W)ki",
+                                             "KUNPCK(BW|DQ|WD)kk",
                                              "VCMPPD(Z|Z128|Z256)rri",
                                              "VCMPPS(Z|Z128|Z256)rri",
                                              "VCMP(SD|SS)Zrr",
diff --git a/llvm/lib/Target/X86/X86SchedSapphireRapids.td b/llvm/lib/Target/X86/X86SchedSapphireRapids.td
index 1fb3c7560a5724..9818f4c01ea678 100644
--- a/llvm/lib/Target/X86/X86SchedSapphireRapids.td
+++ b/llvm/lib/Target/X86/X86SchedSapphireRapids.td
@@ -1318,17 +1318,17 @@ def : InstRW<[SPRWriteResGroup90], (instrs VZEROUPPER)>;
 def SPRWriteResGroup91 : SchedWriteRes<[SPRPort05]> {
   let Latency = 4;
 }
-def : InstRW<[SPRWriteResGroup91], (instregex "^KADD(B|D|Q|W)rr$",
-                                              "^KSHIFT(LB|RD|RQ|RW)ri$",
-                                              "^KSHIFT(LD|RB)ri$",
-                                              "^KSHIFTL(Q|W)ri$",
-                                              "^KUNPCK(BW|DQ|WD)rr$")>;
+def : InstRW<[SPRWriteResGroup91], (instregex "^KADD(B|D|Q|W)kk",
+                                              "^KSHIFT(LB|RD|RQ|RW)ki$",
+                                              "^KSHIFT(LD|RB)ki$",
+                                              "^KSHIFTL(Q|W)ki$",
+                                              "^KUNPCK(BW|DQ|WD)kk$")>;
 
 def SPRWriteResGroup92 : SchedWriteRes<[SPRPort00]>;
-def : InstRW<[SPRWriteResGroup92], (instregex "^KAND(B|D|Q|W|ND|NQ|NW)rr$",
+def : InstRW<[SPRWriteResGroup92], (instregex "^KAND(B|D|Q|W|ND|NQ|NW)kk$",
                                               "^KMOV(B|D|Q|W)kk$",
-                                              "^KNOT(B|D|Q|W)rr$",
-                                              "^K((X|XN)?)OR(B|D|Q|W)rr$",
+                                              "^KNOT(B|D|Q|W)kk$",
+                                              "^K((X|XN)?)OR(B|D|Q|W)kk$",
                                               "^VP(A|SU)BSBZrr$",
                                               "^VPABS(D|Q|W)Zrr$",
                                               "^VPABS(D|Q)Zrrk(z?)$",
@@ -1342,7 +1342,7 @@ def : InstRW<[SPRWriteResGroup92], (instregex "^KAND(B|D|Q|W|ND|NQ|NW)rr$",
                                               "^VPSH(L|R)DV(D|Q|W)Zr$",
                                               "^VPSH(L|R)DV(D|Q)Zrk(z?)$",
                                               "^VPSUB(U?)SWZrr$")>;
-def : InstRW<[SPRWriteResGroup92], (instrs KANDNBrr,
+def : InstRW<[SPRWriteResGroup92], (instrs KANDNBkk,
                                            VPSUBUSBZrr)>;
 
 def SPRWriteResGroup93 : SchedWriteRes<[SPRPort02_03_11, SPRPort05]> {
@@ -1378,7 +1378,7 @@ def : InstRW<[SPRWriteResGroup95], (instrs KMOVQkr,
 def SPRWriteResGroup96 : SchedWriteRes<[SPRPort00]> {
   let Latency = 3;
 }
-def : InstRW<[SPRWriteResGroup96], (instregex "^K((OR)?)TEST(B|D|Q|W)rr$",
+def : InstRW<[SPRWriteResGroup96], (instregex "^K((OR)?)TEST(B|D|Q|W)kk$",
                                               "^VP(A|SU)BS(B|W)Zrrk(z?)$",
                                               "^VPADD(U?)S(B|W)Zrrk(z?)$",
                                               "^VPAVG(B|W)Zrrk(z?)$",
@@ -1867,7 +1867,7 @@ def : InstRW<[SPRWriteResGroup160], (instregex "^(V?)MOVS(D|S)rr((_REV)?)$",
                                                "^VP(ADD|SUB)(B|D|Q|W)Yrr$",
                                                "^VP(ADD|SUB)(B|D|Q|W)Z(128|256)rr$",
                                                "^VP(ADD|SUB)(D|Q)Z(128|256)rrk(z?)$",
-                                               "^VPMOVM2(D|Q)Z128rr$",
+                                               "^VPMOVM2(D|Q)Z128rk$",
                                                "^VPTERNLOG(D|Q)Z(128|256)rri((k|kz)?)$")>;
 def : InstRW<[SPRWriteResGroup160], (instrs VPBLENDDrri)>;
 
@@ -2534,7 +2534,7 @@ def : InstRW<[SPRWriteResGroup250], (instregex "^V(ADD|SUB)P(D|S)Zrr(b?)$",
                                                "^VMOVDQU(8|16)Zrrk(z?)((_REV)?)$",
                                                "^VP(ADD|SUB)(B|W)Zrrk(z?)$",
                                                "^VPBLENDM(B|W)Zrrk(z?)$",
-                                               "^VPMOVM2(B|W)Zrr$")>;
+                                               "^VPMOVM2(B|W)Zrk$")>;
 
 def SPRWriteResGroup251 : SchedWriteRes<[SPRPort00_01]> {
   let Latency = 6;
@@ -4001,7 +4001,7 @@ def : InstRW<[SPRWriteResGroup418], (instregex "^VMOVDQU(8|16)Z(128|256)rrk(z?)(
                                                "^VMOVSHZrrk(z?)((_REV)?)$",
                                                "^VP(ADD|SUB)(B|W)Z(128|256)rrk(z?)$",
                                                "^VPBLENDM(B|W)Z(128|256)rrk(z?)$",
-                                               "^VPMOVM2(B|W)Z(128|256)rr$")>;
+                                               "^VPMOVM2(B|W)Z(128|256)rk$")>;
 
 def SPRWriteResGroup419 : SchedWriteRes<[SPRPort00_01_05_06_10, SPRPort04_09, SPRPort07_08]> {
   let ReleaseAtCycles = [1, 2, 2];
diff --git a/llvm/lib/Target/X86/X86SchedSkylakeServer.td b/llvm/lib/Target/X86/X86SchedSkylakeServer.td
index 649d38de185a80..26e290a2250c9f 100644
--- a/llvm/lib/Target/X86/X86SchedSkylakeServer.td
+++ b/llvm/lib/Target/X86/X86SchedSkylakeServer.td
@@ -628,13 +628,13 @@ def SKXWriteResGroup1 : SchedWriteRes<[SKXPort0]> {
   let NumMicroOps = 1;
   let ReleaseAtCycles = [1];
 }
-def: InstRW<[SKXWriteResGroup1], (instregex "KAND(B|D|Q|W)rr",
-                                            "KANDN(B|D|Q|W)rr",
+def: InstRW<[SKXWriteResGroup1], (instregex "KAND(B|D|Q|W)kk",
+                                            "KANDN(B|D|Q|W)kk",
                                             "KMOV(B|D|Q|W)kk",
-                                            "KNOT(B|D|Q|W)rr",
-                                            "KOR(B|D|Q|W)rr",
-                                            "KXNOR(B|D|Q|W)rr",
-                                            "KXOR(B|D|Q|W)rr",
+                                            "KNOT(B|D|Q|W)kk",
+                                            "KOR(B|D|Q|W)kk",
+                                            "KXNOR(B|D|Q|W)kk",
+                                            "KXOR(B|D|Q|W)kk",
                                             "KSET0(B|D|Q|W)", // Same as KXOR
                                             "KSET1(B|D|Q|W)", // Same as KXNOR
                                             "MMX_PADDS(B|W)rr",
@@ -646,10 +646,10 @@ def: InstRW<[SKXWriteResGroup1], (instregex "KAND(B|D|Q|W)rr",
                                             "MMX_P(MAX|MIN)UBrr",
                                             "MMX_PSUBS(B|W)rr",
                                             "MMX_PSUBUS(B|W)rr",
-                                            "VPMOVB2M(Z|Z128|Z256)rr",
-                                            "VPMOVD2M(Z|Z128|Z256)rr",
-                                            "VPMOVQ2M(Z|Z128|Z256)rr",
-                                            "VPMOVW2M(Z|Z128|Z256)rr")>;
+                                            "VPMOVB2M(Z|Z128|Z256)kr",
+                                            "VPMOVD2M(Z|Z128|Z256)kr",
+                                            "VPMOVQ2M(Z|Z128|Z256)kr",
+                                            "VPMOVW2M(Z|Z128|Z256)kr")>;
 
 def SKXWriteResGroup3 : SchedWriteRes<[SKXPort5]> {
   let Latency = 1;
@@ -812,8 +812,8 @@ def SKXWriteResGroup30 : SchedWriteRes<[SKXPort0]> {
   let ReleaseAtCycles = [1];
 }
 def: InstRW<[SKXWriteResGroup30], (instregex "KMOV(B|D|Q|W)rk",
-                                             "KORTEST(B|D|Q|W)rr",
-                                             "KTEST(B|D|Q|W)rr")>;
+                                             "KORTEST(B|D|Q|W)kk",
+                                             "KTEST(B|D|Q|W)kk")>;
 
 def SKXWriteResGroup31 : SchedWriteRes<[SKXPort1]> {
   let Latency = 3;
@@ -839,10 +839,10 @@ def SKXWriteResGroup33 : SchedWriteRes<[SKXPort5]> {
   let NumMicroOps = 1;
   let ReleaseAtCycles = [1];
 }
-def: InstRW<[SKXWriteResGroup33], (instregex "KADD(B|D|Q|W)rr",
-                                             "KSHIFTL(B|D|Q|W)ri",
-                                             "KSHIFTR(B|D|Q|W)ri",
-                                             "KUNPCK(BW|DQ|WD)rr",
+def: InstRW<[SKXWriteResGroup33], (instregex "KADD(B|D|Q|W)kk",
+                                             "KSHIFTL(B|D|Q|W)ki",
+                                             "KSHIFTR(B|D|Q|W)ki",
+                                             "KUNPCK(BW|DQ|WD)kk",
                                              "VCMPPD(Z|Z128|Z256)rri",
                                              "VCMPPS(Z|Z128|Z256)rri",
                                              "VCMP(SD|SS)Zrr",
diff --git a/llvm/lib/Target/X86/X86ScheduleZnver4.td b/llvm/lib/Target/X86/X86ScheduleZnver4.td
index f82f9a88bb25e0..9625306d716b53 100644
--- a/llvm/lib/Target/X86/X86ScheduleZnver4.td
+++ b/llvm/lib/Target/X86/X86ScheduleZnver4.td
@@ -1081,18 +1081,18 @@ def Zn4WriteVecOpMask : SchedWriteRes<[Zn4FPOpMask01]> {
   let ReleaseAtCycles = [1];
   let NumMicroOps = 1;
 }
-def : InstRW<[Zn4WriteVecOpMask], (instrs   KADDBrr, KADDDrr, KADDQrr, KADDWrr,
-                                            KANDBrr, KANDDrr, KANDQrr, KANDWrr,
-                                            KANDNBrr, KANDNDrr, KANDNQrr, KANDNWrr,
+def : InstRW<[Zn4WriteVecOpMask], (instrs   KADDBkk, KADDDkk, KADDQkk, KADDWkk,
+                                            KANDBkk, KANDDkk, KANDQkk, KANDWkk,
+                                            KANDNBkk, KANDNDkk, KANDNQkk, KANDNWkk,
                                             KMOVBkk, KMOVDkk, KMOVQkk, KMOVWkk,
                                             KMOVBrk, KMOVDrk, KMOVQrk, KMOVWrk,
-                                            KNOTBrr, KNOTDrr, KNOTQrr, KNOTWrr,
-                                            KORBrr, KORDrr, KORQrr, KORWrr,
-                                            KORTESTBrr, KORTESTDrr, KORTESTQrr, KORTESTWrr,
-                                            KTESTBrr, KTESTDrr, KTESTQrr, KTESTWrr,
-                                            KUNPCKBWrr, KUNPCKDQrr, KUNPCKWDrr,
-                                            KXNORBrr, KXNORDrr, KXNORQrr, KXNORWrr,
-                                            KXORBrr, KXORDrr, KXORQrr, KXORWrr)>;
+                                            KNOTBkk, KNOTDkk, KNOTQkk, KNOTWkk,
+                                            KORBkk, KORDkk, KORQkk, KORWkk,
+                                            KORTESTBkk, KORTESTDkk, KORTESTQkk, KORTESTWkk,
+                                            KTESTBkk, KTESTDkk, KTESTQkk, KTESTWkk,
+                                            KUNPCKBWkk, KUNPCKDQkk, KUNPCKWDkk,
+                                            KXNORBkk, KXNORDkk, KXNORQkk, KXNORWkk,
+                                            KXORBkk, KXORDkk, KXORQkk, KXORWkk)>;
 
 def Zn4WriteVecOpMaskMemMov : SchedWriteRes<[Zn4FPOpMask4]> {
   let Latency = 1;
@@ -1839,35 +1839,59 @@ def Zn4WriteFZeroIdiom : SchedWriteVariant<[
 ]>;
 // NOTE: XORPSrr, XORPDrr are not zero-cycle!
 def : InstRW<[Zn4WriteFZeroIdiom], (instrs VXORPSrr, VXORPDrr,
-                                           VANDNPSrr, VANDNPDrr)>;
+                                           VXORPSZ128rr,
+                                           VXORPDZ128rr,
+                                           VANDNPSrr, VANDNPDrr,
+                                           VANDNPSZ128rr,
+                                           VANDNPDZ128rr)>;
 
 def Zn4WriteFZeroIdiomY : SchedWriteVariant<[
     SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [Zn4WriteZeroLatency]>,
     SchedVar<NoSchedPred,                          [WriteFLogicY]>
 ]>;
 def : InstRW<[Zn4WriteFZeroIdiomY], (instrs VXORPSYrr, VXORPDYrr,
-                                            VANDNPSYrr, VANDNPDYrr)>;
+                                            VXORPSZ256rr,
+                                            VXORPDZ256rr,
+                                            VANDNPSYrr, VANDNPDYrr,
+                                            VANDNPSZ256rr,
+                                            VANDNPDZ256rr)>;
+
+def Zn4WriteFZeroIdiomZ : SchedWriteVariant<[
+    SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [Zn4WriteZeroLatency]>,
+    SchedVar<NoSchedPred,                          [WriteFLogicZ]>
+]>;
+def : InstRW<[Zn4WriteFZeroIdiomZ], (instrs VXORPSZrr, VXORPDZrr,
+                                            VANDNPSZrr, VANDNPDZrr)>;
 
 def Zn4WriteVZeroIdiomLogicX : SchedWriteVariant<[
     SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [Zn4WriteZeroLatency]>,
     SchedVar<NoSchedPred,                          [WriteVecLogicX]>
 ]>;
 // NOTE: PXORrr,PANDNrr are not zero-cycle!
-def : InstRW<[Zn4WriteVZeroIdiomLogicX], (instrs VPXORrr, VPANDNrr)>;
+def : InstRW<[Zn4WriteVZeroIdiomLogicX], (instrs VPXORrr,
+                                                 VPXORDZ128rr,
+                                                 VPXORQZ128rr,
+                                                 VPANDNrr,
+                                                 VPANDNDZ128rr,
+                                                 VPANDNQZ128rr)>;
 
-// TODO: This should be extended to incorporate all of the AVX512 zeroing
-// idioms that can be executed by the renamer.
-def Zn4WriteVZeroIdiomLogicZ : SchedWriteVariant<[
+def Zn4WriteVZeroIdiomLogicY : SchedWriteVariant<[
     SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [Zn4WriteZeroLatency]>,
-    SchedVar<NoSchedPred,                          [WriteVecLogicZ]>
+    SchedVar<NoSchedPred,                          [WriteVecLogicY]>
 ]>;
-def : InstRW<[Zn4WriteVZeroIdiomLogicZ], (instrs VPXORDZrr)>;
+def : InstRW<[Zn4WriteVZeroIdiomLogicY], (instrs VPXORYrr,
+                                                 VPXORDZ256rr,
+                                                 VPXORQZ256rr,
+                                                 VPANDNYrr,
+                                                 VPANDNDZ256rr,
+                                                 VPANDNQZ256rr)>;
 
-def Zn4WriteVZeroIdiomLogicY : SchedWriteVariant<[
+def Zn4WriteVZeroIdiomLogicZ : SchedWriteVariant<[
     SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [Zn4WriteZeroLatency]>,
-    SchedVar<NoSchedPred,                          [WriteVecLogicY]>
+    SchedVar<NoSchedPred,                          [WriteVecLogicZ]>
 ]>;
-def : InstRW<[Zn4WriteVZeroIdiomLogicY], (instrs VPXORYrr, VPANDNYrr)>;
+def : InstRW<[Zn4WriteVZeroIdiomLogicZ], (instrs VPXORDZrr, VPXORQZrr,
+                                                 VPANDNDZrr, VPANDNQZrr)>;
 
 def Zn4WriteVZeroIdiomALUX : SchedWriteVariant<[
     SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [Zn4WriteZeroLatency]>,
@@ -1877,7 +1901,10 @@ def Zn4WriteVZeroIdiomALUX : SchedWriteVariant<[
 //       PCMPGTBrr, PCMPGTWrr, PCMPGTDrr, PCMPGTQrr are not zero-cycle!
 def : InstRW<[Zn4WriteVZeroIdiomALUX],
              (instrs VPSUBBrr, VPSUBWrr, VPSUBDrr, VPSUBQrr,
-                     VPCMPGTBrr, VPCMPGTWrr, VPCMPGTDrr, VPCMPGTQrr)>;
+                     VPSUBBZ128rr, VPSUBWZ128rr, VPSUBDZ128rr, VPSUBQZ128rr,
+                     VPCMPGTBrr, VPCMPGTWrr, VPCMPGTDrr, VPCMPGTQrr,
+                     VPCMPGTBZ128rr, VPCMPGTWZ128rr,
+                     VPCMPGTDZ128rr, VPCMPGTQZ128rr)>;
 
 def Zn4WriteVZeroIdiomALUY : SchedWriteVariant<[
     SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [Zn4WriteZeroLatency]>,
@@ -1885,7 +1912,18 @@ def Zn4WriteVZeroIdiomALUY : SchedWriteVariant<[
 ]>;
 def : InstRW<[Zn4WriteVZeroIdiomALUY],
              (instrs VPSUBBYrr, VPSUBWYrr, VPSUBDYrr, VPSUBQYrr,
-                     VPCMPGTBYrr, VPCMPGTWYrr, VPCMPGTDYrr, VPCMPGTQYrr)>;
+                     VPSUBBZ256rr, VPSUBWZ256rr, VPSUBDZ256rr, VPSUBQZ256rr,
+                     VPCMPGTBYrr, VPCMPGTWYrr, VPCMPGTDYrr, VPCMPGTQYrr,
+                     VPCMPGTBZ256rr, VPCMPGTWZ256rr,
+                     VPCMPGTDZ256rr, VPCMPGTQZ256rr)>;
+
+def Zn4WriteVZeroIdiomALUZ : SchedWriteVariant<[
+    SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [Zn4WriteZeroLatency]>,
+    SchedVar<NoSchedPred,                          [WriteVecALUZ]>
+]>;
+def : InstRW<[Zn4WriteVZeroIdiomALUY],
+             (instrs VPSUBBZrr, VPSUBWZrr, VPSUBDZrr, VPSUBQZrr,
+                     VPCMPGTBZrr, VPCMPGTWZrr, VPCMPGTDZrr, VPCMPGTQZrr)>;
 
 def : IsZeroIdiomFunction<[
   // GPR Zero-idioms.
@@ -1940,9 +1978,24 @@ def : IsZeroIdiomFunction<[
   ], ZeroIdiomPredicate>,
 
   // AVX ZMM Zero-idioms.
-  // TODO: This should be expanded to incorporate all AVX512 zeroing idioms.
   DepBreakingClass<[
-    VPXORDZrr
+    // fp variants.
+    VXORPSZrr, VXORPDZrr,
+    VXORPSZ128rr, VXORPDZ128rr, VXORPSZ256rr, VXORPDZ256rr,
+    VANDNPSZrr, VANDNPDZrr,
+    VANDNPSZ128rr, VANDNPDZ128rr, VANDNPSZ256rr, VANDNPDZ256rr,
+
+    // int variants.
+    VPCMPGTBZrr, VPCMPGTWZrr, VPCMPGTDZrr, VPCMPGTQZrr,
+    VPCMPGTBZ128rr, VPCMPGTWZ128rr, VPCMPGTDZ128rr, VPCMPGTQZ128rr,
+    VPCMPGTBZ256rr, VPCMPGTWZ256rr, VPCMPGTDZ256rr, VPCMPGTQZ256rr,
+    VPANDNDZrr, VPANDNQZrr,
+    VPANDNDZ128rr, VPANDNQZ128rr, VPANDNDZ256rr, VPANDNQZ256rr,
+    VPXORDZrr, VPXORQZrr,
+    VPXORDZ128rr, VPXORQZ128rr, VPXORDZ256rr, VPXORQZ256rr,
+    VPSUBBZrr, VPSUBWZrr, VPSUBDZrr, VPSUBQZrr,
+    VPSUBBZ128rr, VPSUBWZ128rr, VPSUBDZ128rr, VPSUBQZ128rr,
+    VPSUBBZ256rr, VPSUBWZ256rr, VPSUBDZ256rr, VPSUBQZ256rr,
   ], ZeroIdiomPredicate>,
 ]>;
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
index 5c9faa9449f539..ea51d779045718 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -2323,6 +2323,11 @@ static Value *optimizeIntegerToVectorInsertions(BitCastInst &CI,
   auto *DestVecTy = cast<FixedVectorType>(CI.getType());
   Value *IntInput = CI.getOperand(0);
 
+  // if the int input is just an undef value do not try to optimize to vector
+  // insertions as it will prevent undef propagation
+  if (isa<UndefValue>(IntInput))
+    return nullptr;
+
   SmallVector<Value*, 8> Elements(DestVecTy->getNumElements());
   if (!collectInsertionElements(IntInput, 0, Elements,
                                 DestVecTy->getElementType(),
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 16e43bef1524de..e8a95f5dfd435c 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -2960,6 +2960,10 @@ bool AddressSanitizer::instrumentFunction(Function &F,
 
   bool FunctionModified = false;
 
+  // Do not apply any instrumentation for naked functions.
+  if (F.hasFnAttribute(Attribute::Naked))
+    return FunctionModified;
+
   // If needed, insert __asan_init before checking for SanitizeAddress attr.
   // This function needs to be called even if the function body is not
   // instrumented.
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index da7dc180de50d8..7a5c690c7ea512 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -1598,6 +1598,10 @@ void HWAddressSanitizer::sanitizeFunction(Function &F,
   if (&F == HwasanCtorFunction)
     return;
 
+  // Do not apply any instrumentation for naked functions.
+  if (F.hasFnAttribute(Attribute::Naked))
+    return;
+
   if (!F.hasFnAttribute(Attribute::SanitizeHWAddress))
     return;
 
diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp
index fc78d8c60ec050..e76689e2f5f0a5 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp
@@ -26,6 +26,7 @@
 #include "llvm/IR/Analysis.h"
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/Dominators.h"
+#include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PassManager.h"
@@ -34,6 +35,7 @@
 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
 #include "llvm/Transforms/Scalar/DCE.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include <deque>
 
 using namespace llvm;
 
@@ -51,6 +53,11 @@ class ProfileAnnotator final {
 
   class BBInfo {
     std::optional<uint64_t> Count;
+    // OutEdges is dimensioned to match the number of terminator operands.
+    // Entries in the vector match the index in the terminator operand list. In
+    // some cases - see `shouldExcludeEdge` and its implementation - an entry
+    // will be nullptr.
+    // InEdges doesn't have the above constraint.
     SmallVector<EdgeInfo *> OutEdges;
     SmallVector<EdgeInfo *> InEdges;
     size_t UnknownCountOutEdges = 0;
@@ -58,22 +65,30 @@ class ProfileAnnotator final {
 
     // Pass AssumeAllKnown when we try to propagate counts from edges to BBs -
     // because all the edge counters must be known.
-    uint64_t getEdgeSum(const SmallVector<EdgeInfo *> &Edges,
-                        bool AssumeAllKnown) const {
-      uint64_t Sum = 0;
-      for (const auto *E : Edges)
-        if (E)
-          Sum += AssumeAllKnown ? *E->Count : E->Count.value_or(0U);
+    // Return std::nullopt if there were no edges to sum. The user can decide
+    // how to interpret that.
+    std::optional<uint64_t> getEdgeSum(const SmallVector<EdgeInfo *> &Edges,
+                                       bool AssumeAllKnown) const {
+      std::optional<uint64_t> Sum;
+      for (const auto *E : Edges) {
+        // `Edges` may be `OutEdges`, case in which `E` could be nullptr.
+        if (E) {
+          if (!Sum.has_value())
+            Sum = 0;
+          *Sum += (AssumeAllKnown ? *E->Count : E->Count.value_or(0U));
+        }
+      }
       return Sum;
     }
 
-    void computeCountFrom(const SmallVector<EdgeInfo *> &Edges) {
+    bool computeCountFrom(const SmallVector<EdgeInfo *> &Edges) {
       assert(!Count.has_value());
       Count = getEdgeSum(Edges, true);
+      return Count.has_value();
     }
 
     void setSingleUnknownEdgeCount(SmallVector<EdgeInfo *> &Edges) {
-      uint64_t KnownSum = getEdgeSum(Edges, false);
+      uint64_t KnownSum = getEdgeSum(Edges, false).value_or(0U);
       uint64_t EdgeVal = *Count > KnownSum ? *Count - KnownSum : 0U;
       EdgeInfo *E = nullptr;
       for (auto *I : Edges)
@@ -110,17 +125,15 @@ class ProfileAnnotator final {
     }
 
     bool tryTakeCountFromKnownOutEdges(const BasicBlock &BB) {
-      if (!succ_empty(&BB) && !UnknownCountOutEdges) {
-        computeCountFrom(OutEdges);
-        return true;
+      if (!UnknownCountOutEdges) {
+        return computeCountFrom(OutEdges);
       }
       return false;
     }
 
     bool tryTakeCountFromKnownInEdges(const BasicBlock &BB) {
-      if (!BB.isEntryBlock() && !UnknownCountInEdges) {
-        computeCountFrom(InEdges);
-        return true;
+      if (!UnknownCountInEdges) {
+        return computeCountFrom(InEdges);
       }
       return false;
     }
@@ -178,7 +191,7 @@ class ProfileAnnotator final {
     bool KeepGoing = true;
     while (KeepGoing) {
       KeepGoing = false;
-      for (const auto &BB : reverse(F)) {
+      for (const auto &BB : F) {
         auto &Info = getBBInfo(BB);
         if (!Info.hasCount())
           KeepGoing |= Info.tryTakeCountFromKnownOutEdges(BB) ||
@@ -198,6 +211,52 @@ class ProfileAnnotator final {
 
   BBInfo &getBBInfo(const BasicBlock &BB) { return BBInfos.find(&BB)->second; }
 
+  const BBInfo &getBBInfo(const BasicBlock &BB) const {
+    return BBInfos.find(&BB)->second;
+  }
+
+  // validation function after we propagate the counters: all BBs and edges'
+  // counters must have a value.
+  bool allCountersAreAssigned() const {
+    for (const auto &BBInfo : BBInfos)
+      if (!BBInfo.second.hasCount())
+        return false;
+    for (const auto &EdgeInfo : EdgeInfos)
+      if (!EdgeInfo.Count.has_value())
+        return false;
+    return true;
+  }
+
+  /// Check that all paths from the entry basic block that use edges with
+  /// non-zero counts arrive at a basic block with no successors (i.e. "exit")
+  bool allTakenPathsExit() const {
+    std::deque<const BasicBlock *> Worklist;
+    DenseSet<const BasicBlock *> Visited;
+    Worklist.push_back(&F.getEntryBlock());
+    Visited.insert(&F.getEntryBlock());
+    while (!Worklist.empty()) {
+      const auto *BB = Worklist.front();
+      Worklist.pop_front();
+      if (succ_size(BB) <= 1)
+        continue;
+      const auto &BBInfo = getBBInfo(*BB);
+      bool Inserted = false;
+      for (auto I = 0U; I < BB->getTerminator()->getNumSuccessors(); ++I) {
+        const auto *Succ = BB->getTerminator()->getSuccessor(I);
+        if (!shouldExcludeEdge(*BB, *Succ)) {
+          if (BBInfo.getEdgeCount(I) > 0)
+            if (Visited.insert(Succ).second) {
+              Worklist.push_back(Succ);
+              Inserted = true;
+            }
+        }
+      }
+      if (!Inserted)
+        return false;
+    }
+    return true;
+  }
+
 public:
   ProfileAnnotator(Function &F, const SmallVectorImpl<uint64_t> &Counters,
                    InstrProfSummaryBuilder &PB)
@@ -216,6 +275,9 @@ class ProfileAnnotator final {
                "profile is managed by IPO transforms");
         (void)Index;
         Count = Counters[Ins->getIndex()->getZExtValue()];
+      } else if (isa<UnreachableInst>(BB.getTerminator())) {
+        // The program presumably didn't crash.
+        Count = 0;
       }
       auto [It, Ins] =
           BBInfos.insert({&BB, {pred_size(&BB), succ_size(&BB), Count}});
@@ -268,14 +330,16 @@ class ProfileAnnotator final {
         PB.addInternalCount(EdgeCount);
       }
 
-      if (MaxCount == 0)
-        F.getContext().emitError(
-            "[ctx-prof] Encountered a BB with more than one successor, where "
-            "all outgoing edges have a 0 count. This occurs in non-exiting "
-            "functions (message pumps, usually) which are not supported in the "
-            "contextual profiling case");
-      setProfMetadata(F.getParent(), Term, EdgeCounts, MaxCount);
+      if (MaxCount != 0)
+        setProfMetadata(F.getParent(), Term, EdgeCounts, MaxCount);
     }
+    assert(allCountersAreAssigned() &&
+           "Expected all counters have been assigned.");
+    assert(allTakenPathsExit() &&
+           "[ctx-prof] Encountered a BB with more than one successor, where "
+           "all outgoing edges have a 0 count. This occurs in non-exiting "
+           "functions (message pumps, usually) which are not supported in the "
+           "contextual profiling case");
   }
 };
 
diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerBinaryMetadata.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerBinaryMetadata.cpp
index e326f30ad88eeb..eb346254f30dd7 100644
--- a/llvm/lib/Transforms/Instrumentation/SanitizerBinaryMetadata.cpp
+++ b/llvm/lib/Transforms/Instrumentation/SanitizerBinaryMetadata.cpp
@@ -258,6 +258,9 @@ bool SanitizerBinaryMetadata::run() {
 void SanitizerBinaryMetadata::runOn(Function &F, MetadataInfoSet &MIS) {
   if (F.empty())
     return;
+  // Do not apply any instrumentation for naked functions.
+  if (F.hasFnAttribute(Attribute::Naked))
+    return;
   if (F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))
     return;
   if (Ignorelist && Ignorelist->inSection("metadata", "fun", F.getName()))
diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
index 6a89cee9aaf6cc..db4bf709c9cc9c 100644
--- a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
+++ b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
@@ -629,6 +629,9 @@ void ModuleSanitizerCoverage::instrumentFunction(Function &F) {
     return;
   if (Blocklist && Blocklist->inSection("coverage", "fun", F.getName()))
     return;
+  // Do not apply any instrumentation for naked functions.
+  if (F.hasFnAttribute(Attribute::Naked))
+    return;
   if (F.hasFnAttribute(Attribute::NoSanitizeCoverage))
     return;
   if (F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))
diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index 613597b0878814..7e76e4a0f387eb 100644
--- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -1788,6 +1788,13 @@ bool IndVarSimplify::predicateLoopExits(Loop *L, SCEVExpander &Rewriter) {
     return false;
   };
 
+  // Make sure all exits dominate the latch. This means there is a linear chain
+  // of exits. We check this before sorting so we have a total order.
+  BasicBlock *Latch = L->getLoopLatch();
+  for (BasicBlock *ExitingBB : ExitingBlocks)
+    if (!DT->dominates(ExitingBB, Latch))
+      return false;
+
   // If we have any exits which can't be predicated themselves, than we can't
   // predicate any exit which isn't guaranteed to execute before it.  Consider
   // two exits (a) and (b) which would both exit on the same iteration.  If we
@@ -1795,21 +1802,23 @@ bool IndVarSimplify::predicateLoopExits(Loop *L, SCEVExpander &Rewriter) {
   // we could convert a loop from exiting through (a) to one exiting through
   // (b).  Note that this problem exists only for exits with the same exit
   // count, and we could be more aggressive when exit counts are known inequal.
-  llvm::sort(ExitingBlocks,
-            [&](BasicBlock *A, BasicBlock *B) {
-              // std::sort sorts in ascending order, so we want the inverse of
-              // the normal dominance relation, plus a tie breaker for blocks
-              // unordered by dominance.
-              if (DT->properlyDominates(A, B)) return true;
-              if (DT->properlyDominates(B, A)) return false;
-              return A->getName() < B->getName();
-            });
-  // Check to see if our exit blocks are a total order (i.e. a linear chain of
-  // exits before the backedge).  If they aren't, reasoning about reachability
-  // is complicated and we choose not to for now.
-  for (unsigned i = 1; i < ExitingBlocks.size(); i++)
-    if (!DT->dominates(ExitingBlocks[i-1], ExitingBlocks[i]))
+  llvm::sort(ExitingBlocks, [&](BasicBlock *A, BasicBlock *B) {
+    // llvm::sort sorts in ascending order, so we want the inverse of
+    // the normal dominance relation.
+    if (A == B)
+      return false;
+    if (DT->properlyDominates(A, B))
+      return true;
+    if (DT->properlyDominates(B, A))
       return false;
+    llvm_unreachable("Should have total dominance order");
+  });
+
+  // Make sure our exit blocks are really a total order (i.e. a linear chain of
+  // exits before the backedge).
+  for (unsigned i = 1; i < ExitingBlocks.size(); i++)
+    assert(DT->dominates(ExitingBlocks[i - 1], ExitingBlocks[i]) &&
+           "Not sorted by dominance");
 
   // Given our sorted total order, we know that exit[j] must be evaluated
   // after all exit[i] such j > i.
@@ -1822,14 +1831,6 @@ bool IndVarSimplify::predicateLoopExits(Loop *L, SCEVExpander &Rewriter) {
   if (ExitingBlocks.empty())
     return false;
 
-  // We rely on not being able to reach an exiting block on a later iteration
-  // then it's statically compute exit count.  The implementaton of
-  // getExitCount currently has this invariant, but assert it here so that
-  // breakage is obvious if this ever changes..
-  assert(llvm::all_of(ExitingBlocks, [&](BasicBlock *ExitingBB) {
-        return DT->dominates(ExitingBB, L->getLoopLatch());
-      }));
-
   // At this point, ExitingBlocks consists of only those blocks which are
   // predicatable.  Given that, we know we have at least one exit we can
   // predicate if the loop is doesn't have side effects and doesn't have any
diff --git a/llvm/lib/Transforms/Scalar/Scalarizer.cpp b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
index 01d24335df2262..d464e49990b3b3 100644
--- a/llvm/lib/Transforms/Scalar/Scalarizer.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
@@ -18,6 +18,7 @@
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/BasicBlock.h"
@@ -281,10 +282,11 @@ T getWithDefaultOverride(const cl::opt<T> &ClOption,
 
 class ScalarizerVisitor : public InstVisitor<ScalarizerVisitor, bool> {
 public:
-  ScalarizerVisitor(DominatorTree *DT, ScalarizerPassOptions Options)
-      : DT(DT), ScalarizeVariableInsertExtract(getWithDefaultOverride(
-                    ClScalarizeVariableInsertExtract,
-                    Options.ScalarizeVariableInsertExtract)),
+  ScalarizerVisitor(DominatorTree *DT, const TargetTransformInfo *TTI,
+                    ScalarizerPassOptions Options)
+      : DT(DT), TTI(TTI), ScalarizeVariableInsertExtract(getWithDefaultOverride(
+                              ClScalarizeVariableInsertExtract,
+                              Options.ScalarizeVariableInsertExtract)),
         ScalarizeLoadStore(getWithDefaultOverride(ClScalarizeLoadStore,
                                                   Options.ScalarizeLoadStore)),
         ScalarizeMinBits(getWithDefaultOverride(ClScalarizeMinBits,
@@ -292,6 +294,8 @@ class ScalarizerVisitor : public InstVisitor<ScalarizerVisitor, bool> {
 
   bool visit(Function &F);
 
+  bool isTriviallyScalarizable(Intrinsic::ID ID);
+
   // InstVisitor methods.  They return true if the instruction was scalarized,
   // false if nothing changed.
   bool visitInstruction(Instruction &I) { return false; }
@@ -335,6 +339,7 @@ class ScalarizerVisitor : public InstVisitor<ScalarizerVisitor, bool> {
   SmallVector<WeakTrackingVH, 32> PotentiallyDeadInstrs;
 
   DominatorTree *DT;
+  const TargetTransformInfo *TTI;
 
   const bool ScalarizeVariableInsertExtract;
   const bool ScalarizeLoadStore;
@@ -358,6 +363,7 @@ ScalarizerLegacyPass::ScalarizerLegacyPass(const ScalarizerPassOptions &Options)
 
 void ScalarizerLegacyPass::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<DominatorTreeWrapperPass>();
+  AU.addRequired<TargetTransformInfoWrapperPass>();
   AU.addPreserved<DominatorTreeWrapperPass>();
 }
 
@@ -445,7 +451,9 @@ bool ScalarizerLegacyPass::runOnFunction(Function &F) {
     return false;
 
   DominatorTree *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
-  ScalarizerVisitor Impl(DT, Options);
+  const TargetTransformInfo *TTI =
+      &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
+  ScalarizerVisitor Impl(DT, TTI, Options);
   return Impl.visit(F);
 }
 
@@ -689,8 +697,11 @@ bool ScalarizerVisitor::splitBinary(Instruction &I, const Splitter &Split) {
   return true;
 }
 
-static bool isTriviallyScalariable(Intrinsic::ID ID) {
-  return isTriviallyVectorizable(ID);
+bool ScalarizerVisitor::isTriviallyScalarizable(Intrinsic::ID ID) {
+  if (isTriviallyVectorizable(ID))
+    return true;
+  return Function::isTargetIntrinsic(ID) &&
+         TTI->isTargetIntrinsicTriviallyScalarizable(ID);
 }
 
 /// If a call to a vector typed intrinsic function, split into a scalar call per
@@ -705,7 +716,8 @@ bool ScalarizerVisitor::splitCall(CallInst &CI) {
     return false;
 
   Intrinsic::ID ID = F->getIntrinsicID();
-  if (ID == Intrinsic::not_intrinsic || !isTriviallyScalariable(ID))
+
+  if (ID == Intrinsic::not_intrinsic || !isTriviallyScalarizable(ID))
     return false;
 
   // unsigned NumElems = VT->getNumElements();
@@ -1249,7 +1261,8 @@ bool ScalarizerVisitor::finish() {
 
 PreservedAnalyses ScalarizerPass::run(Function &F, FunctionAnalysisManager &AM) {
   DominatorTree *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-  ScalarizerVisitor Impl(DT, Options);
+  const TargetTransformInfo *TTI = &AM.getResult<TargetIRAnalysis>(F);
+  ScalarizerVisitor Impl(DT, TTI, Options);
   bool Changed = Impl.visit(F);
   PreservedAnalyses PA;
   PA.preserve<DominatorTreeAnalysis>();
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 5a694b5e7f204b..c63618d9dd1297 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -1954,6 +1954,9 @@ static bool isLifeTimeMarker(const Instruction *I) {
 // into variables.
 static bool replacingOperandWithVariableIsCheap(const Instruction *I,
                                                 int OpIdx) {
+  // Divide/Remainder by constant is typically much cheaper than by variable.
+  if (I->isIntDivRem())
+    return OpIdx != 1;
   return !isa<IntrinsicInst>(I);
 }
 
@@ -3334,7 +3337,7 @@ bool SimplifyCFGOpt::speculativelyExecuteBB(BranchInst *BI,
     // extended for vector types in the future.
     assert(!getLoadStoreType(I)->isVectorTy() && "not implemented");
     auto *Op0 = I->getOperand(0);
-    Instruction *MaskedLoadStore = nullptr;
+    CallInst *MaskedLoadStore = nullptr;
     if (auto *LI = dyn_cast<LoadInst>(I)) {
       // Handle Load.
       auto *Ty = I->getType();
@@ -3367,8 +3370,9 @@ bool SimplifyCFGOpt::speculativelyExecuteBB(BranchInst *BI,
     //         vector specifies a per-element range, so the semantics stay the
     //         same. Keep it.
     // !annotation: Not impact semantics. Keep it.
-    I->dropUBImplyingAttrsAndUnknownMetadata(
-        {LLVMContext::MD_range, LLVMContext::MD_annotation});
+    if (const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range))
+      MaskedLoadStore->addRangeRetAttr(getConstantRangeFromMetadata(*Ranges));
+    I->dropUBImplyingAttrsAndUnknownMetadata({LLVMContext::MD_annotation});
     // FIXME: DIAssignID is not supported for masked store yet.
     // (Verifier::visitDIAssignIDMetadata)
     at::deleteAssignmentMarkers(I);
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index b11631350e8b4e..59d04ac3cecd00 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -3,6 +3,7 @@ add_llvm_component_library(LLVMVectorize
   LoopIdiomVectorize.cpp
   LoopVectorizationLegality.cpp
   LoopVectorize.cpp
+  SandboxVectorizer/DependencyGraph.cpp
   SandboxVectorizer/Passes/BottomUpVec.cpp
   SandboxVectorizer/SandboxVectorizer.cpp
   SLPVectorizer.cpp
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
new file mode 100644
index 00000000000000..41e50953a4ec8a
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
@@ -0,0 +1,64 @@
+//===- DependencyGraph.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h"
+
+using namespace llvm::sandboxir;
+
+#ifndef NDEBUG
+void DGNode::print(raw_ostream &OS, bool PrintDeps) const {
+  I->dumpOS(OS);
+  if (PrintDeps) {
+    OS << "\n";
+    // Print memory preds.
+    static constexpr const unsigned Indent = 4;
+    for (auto *Pred : MemPreds) {
+      OS.indent(Indent) << "<-";
+      Pred->print(OS, false);
+      OS << "\n";
+    }
+  }
+}
+void DGNode::dump() const {
+  print(dbgs());
+  dbgs() << "\n";
+}
+#endif // NDEBUG
+
+void DependencyGraph::extend(BasicBlock *BB) {
+  if (BB->empty())
+    return;
+  // TODO: For now create a chain of dependencies.
+  DGNode *LastN = getOrCreateNode(&*BB->begin());
+  for (auto &I : drop_begin(*BB)) {
+    auto *N = getOrCreateNode(&I);
+    N->addMemPred(LastN);
+    LastN = N;
+  }
+}
+
+#ifndef NDEBUG
+void DependencyGraph::print(raw_ostream &OS) const {
+  // InstrToNodeMap is unordered so we need to create an ordered vector.
+  SmallVector<DGNode *> Nodes;
+  Nodes.reserve(InstrToNodeMap.size());
+  for (const auto &Pair : InstrToNodeMap)
+    Nodes.push_back(Pair.second.get());
+  // Sort them based on which one comes first in the BB.
+  sort(Nodes, [](DGNode *N1, DGNode *N2) {
+    return N1->getInstruction()->comesBefore(N2->getInstruction());
+  });
+  for (auto *N : Nodes)
+    N->print(OS, /*PrintDeps=*/true);
+}
+
+void DependencyGraph::dump() const {
+  print(dbgs());
+  dbgs() << "\n";
+}
+#endif // NDEBUG
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
index c4870b70fd52da..0c44d05f0474d5 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
@@ -7,7 +7,58 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h"
+#include "llvm/ADT/SmallVector.h"
 
 using namespace llvm::sandboxir;
 
-bool BottomUpVec::runOnFunction(Function &F) { return false; }
+namespace llvm::sandboxir {
+// TODO: This is a temporary function that returns some seeds.
+//       Replace this with SeedCollector's function when it lands.
+static llvm::SmallVector<Value *, 4> collectSeeds(BasicBlock &BB) {
+  llvm::SmallVector<Value *, 4> Seeds;
+  for (auto &I : BB)
+    if (auto *SI = llvm::dyn_cast<StoreInst>(&I))
+      Seeds.push_back(SI);
+  return Seeds;
+}
+
+static SmallVector<Value *, 4> getOperand(ArrayRef<Value *> Bndl,
+                                          unsigned OpIdx) {
+  SmallVector<Value *, 4> Operands;
+  for (Value *BndlV : Bndl) {
+    auto *BndlI = cast<Instruction>(BndlV);
+    Operands.push_back(BndlI->getOperand(OpIdx));
+  }
+  return Operands;
+}
+
+} // namespace llvm::sandboxir
+
+void BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl) {
+  auto LegalityRes = Legality.canVectorize(Bndl);
+  switch (LegalityRes.getSubclassID()) {
+  case LegalityResultID::Widen: {
+    auto *I = cast<Instruction>(Bndl[0]);
+    for (auto OpIdx : seq<unsigned>(I->getNumOperands())) {
+      auto OperandBndl = getOperand(Bndl, OpIdx);
+      vectorizeRec(OperandBndl);
+    }
+    break;
+  }
+  }
+}
+
+void BottomUpVec::tryVectorize(ArrayRef<Value *> Bndl) { vectorizeRec(Bndl); }
+
+bool BottomUpVec::runOnFunction(Function &F) {
+  Change = false;
+  // TODO: Start from innermost BBs first
+  for (auto &BB : F) {
+    // TODO: Replace with proper SeedCollector function.
+    auto Seeds = collectSeeds(BB);
+    // TODO: Slice Seeds into smaller chunks.
+    if (Seeds.size() >= 2)
+      tryVectorize(Seeds);
+  }
+  return Change;
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index a310756793a515..41e281f3fa9973 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -981,16 +981,16 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
 /// have a single predecessor, which is rewired to the new VPIRBasicBlock. All
 /// successors of VPBB, if any, are rewired to the new VPIRBasicBlock.
 static void replaceVPBBWithIRVPBB(VPBasicBlock *VPBB, BasicBlock *IRBB) {
-  VPIRBasicBlock *IRMiddleVPBB = createVPIRBasicBlockFor(IRBB);
+  VPIRBasicBlock *IRVPBB = createVPIRBasicBlockFor(IRBB);
   for (auto &R : make_early_inc_range(*VPBB)) {
     assert(!R.isPhi() && "Tried to move phi recipe to end of block");
-    R.moveBefore(*IRMiddleVPBB, IRMiddleVPBB->end());
+    R.moveBefore(*IRVPBB, IRVPBB->end());
   }
   VPBlockBase *PredVPBB = VPBB->getSinglePredecessor();
   VPBlockUtils::disconnectBlocks(PredVPBB, VPBB);
-  VPBlockUtils::connectBlocks(PredVPBB, IRMiddleVPBB);
+  VPBlockUtils::connectBlocks(PredVPBB, IRVPBB);
   for (auto *Succ : to_vector(VPBB->getSuccessors())) {
-    VPBlockUtils::connectBlocks(IRMiddleVPBB, Succ);
+    VPBlockUtils::connectBlocks(IRVPBB, Succ);
     VPBlockUtils::disconnectBlocks(VPBB, Succ);
   }
   delete VPBB;
diff --git a/llvm/test/Analysis/CostModel/AArch64/extract_float.ll b/llvm/test/Analysis/CostModel/AArch64/extract_float.ll
new file mode 100644
index 00000000000000..dd3d0289bbb1cf
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/extract_float.ll
@@ -0,0 +1,205 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64-unknown-linux \
+; RUN:       -mattr=-fullfp16 | FileCheck %s --check-prefixes=CHECK,NOFP16
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64-unknown-linux \
+; RUN:       -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,FULLFP16
+
+; res = lane 0 * lane 1
+define double @extract_case1(<2 x double> %a) {
+; CHECK-LABEL: 'extract_case1'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %0 = extractelement <2 x double> %a, i32 0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = extractelement <2 x double> %a, i32 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul double %0, %1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %res
+entry:
+  %1 = extractelement <2 x double> %a, i32 0
+  %2 = extractelement <2 x double> %a, i32 1
+  %res = fmul double %1, %2
+  ret double %res
+}
+
+; res = lane 1 * lane 1
+define double @extract_case2(<2 x double> %a) {
+; CHECK-LABEL: 'extract_case2'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %0 = extractelement <2 x double> %a, i32 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul double %0, %0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %res
+entry:
+  %1 = extractelement <2 x double> %a, i32 1
+  %res = fmul double %1, %1
+  ret double %res
+}
+
+; res = lane 0 * lane 0
+define double @extract_case3(<2 x double> %a) {
+; CHECK-LABEL: 'extract_case3'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %0 = extractelement <2 x double> %a, i32 0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul double %0, %0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %res
+entry:
+  %1 = extractelement <2 x double> %a, i32 0
+  %res = fmul double %1, %1
+  ret double %res
+}
+
+; res = lane 0 * scalar
+define double @extract_case4(<2 x double> %a, double %b) {
+; CHECK-LABEL: 'extract_case4'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %0 = extractelement <2 x double> %a, i32 0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul double %0, %b
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %res
+entry:
+  %1 = extractelement <2 x double> %a, i32 0
+  %res = fmul double %1, %b
+  ret double %res
+}
+
+; res = lane 1 * scalar
+define double @extract_case5(<2 x double> %a, double %b) {
+; CHECK-LABEL: 'extract_case5'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %0 = extractelement <2 x double> %a, i32 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul double %0, %b
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %res
+entry:
+  %1 = extractelement <2 x double> %a, i32 1
+  %res = fmul double %1, %b
+  ret double %res
+}
+
+; Input vector = <3 x double> (i.e. odd length vector)
+; res = lane 0 * lane 1
+define double @extract_case6(<3 x double> %a) {
+; CHECK-LABEL: 'extract_case6'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %0 = extractelement <3 x double> %a, i32 0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = extractelement <3 x double> %a, i32 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul double %0, %1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %res
+entry:
+  %1 = extractelement <3 x double> %a, i32 0
+  %2 = extractelement <3 x double> %a, i32 1
+  %res = fmul double %1, %2
+  ret double %res
+}
+
+; res = lane 1 * lane 2
+; Extract from lane 2 is equivalent to extract from lane 0 of other 128-bit
+; register. But for other register sizes, this is not the case.
+define double @extract_case7(<4 x double> %a) {
+; CHECK-LABEL: 'extract_case7'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %0 = extractelement <4 x double> %a, i32 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %1 = extractelement <4 x double> %a, i32 2
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul double %0, %1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %res
+entry:
+  %1 = extractelement <4 x double> %a, i32 1
+  %2 = extractelement <4 x double> %a, i32 2
+  %res = fmul double %1, %2
+  ret double %res
+}
+
+; res = lane 0 * lane 1
+; Additional insert of extract from lane 1.
+define double @extract_case8(<2 x double> %a) {
+; CHECK-LABEL: 'extract_case8'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %0 = extractelement <2 x double> %a, i32 0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = extractelement <2 x double> %a, i32 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %2 = insertelement <2 x double> %a, double %1, i32 0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = call double @llvm.vector.reduce.fmul.v2f64(double 0.000000e+00, <2 x double> %2)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = fmul double %0, %1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = fmul double %3, %4
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %5
+entry:
+  %1 = extractelement <2 x double> %a, i32 0
+  %2 = extractelement <2 x double> %a, i32 1
+  %3 = insertelement <2 x double> %a, double %2, i32 0
+  %4 = call double @llvm.vector.reduce.fmul.v2f64(double 0.0, <2 x double> %3)
+  %5 = fmul double %1, %2
+  %6 = fmul double %4, %5
+  ret double %6
+}
+
+; res = lane 0 * lane 1
+; Additional insert of extract from lane 1.
+define double @extract_case9(<2 x double> %a) {
+; CHECK-LABEL: 'extract_case9'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %0 = extractelement <2 x double> %a, i32 0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = extractelement <2 x double> %a, i32 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %2 = insertelement <2 x double> %a, double %1, i32 0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %3 = call double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> %2)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = fmul double %0, %1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = fmul double %3, %4
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %5
+entry:
+  %1 = extractelement <2 x double> %a, i32 0
+  %2 = extractelement <2 x double> %a, i32 1
+  %3 = insertelement <2 x double> %a, double %2, i32 0
+  %4 = call double @llvm.vector.reduce.fadd.v2f64(double 0.0, <2 x double> %3)
+  %5 = fmul double %1, %2
+  %6 = fmul double %4, %5
+  ret double %6
+}
+
+; res = lane 0 * lane 1
+; Extract from lane 1 passed as function param.
+define double @extract_case10(<4 x double> %a) {
+; CHECK-LABEL: 'extract_case10'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %0 = extractelement <4 x double> %a, i32 0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = extractelement <4 x double> %a, i32 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @foo(double %1)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = fmul double %0, %1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %2
+entry:
+  %1 = extractelement <4 x double> %a, i32 0
+  %2 = extractelement <4 x double> %a, i32 1
+  call void @foo(double %2)
+  %3 = fmul double %1, %2
+  ret double %3
+}
+
+; res = lane 0 * lane 1
+define half @extract_case11(<2 x half> %a) {
+; CHECK-LABEL: 'extract_case11'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %0 = extractelement <2 x half> %a, i32 0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = extractelement <2 x half> %a, i32 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul half %0, %1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret half %res
+entry:
+  %1 = extractelement <2 x half> %a, i32 0
+  %2 = extractelement <2 x half> %a, i32 1
+  %res = fmul half %1, %2
+  ret half %res
+}
+
+; res = lane 0 * lane 1
+define float @extract_case12(<2 x float> %a) {
+; CHECK-LABEL: 'extract_case12'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %0 = extractelement <2 x float> %a, i32 0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = extractelement <2 x float> %a, i32 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul float %0, %1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %res
+entry:
+  %1 = extractelement <2 x float> %a, i32 0
+  %2 = extractelement <2 x float> %a, i32 1
+  %res = fmul float %1, %2
+  ret float %res
+}
+
+; res = lane 0 + lane 1
+; Use of bin-op other than fmul.
+define double @extract_case13(<2 x double> %a) {
+; CHECK-LABEL: 'extract_case13'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %0 = extractelement <2 x double> %a, i32 0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = extractelement <2 x double> %a, i32 1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = fadd double %0, %1
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %res
+entry:
+  %1 = extractelement <2 x double> %a, i32 0
+  %2 = extractelement <2 x double> %a, i32 1
+  %res = fadd double %1, %2
+  ret double %res
+}
+
+declare void @foo(double)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; FULLFP16: {{.*}}
+; NOFP16: {{.*}}
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-extractelement.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-extractelement.ll
index 4a9e30888cdd1f..618b7bc8945a50 100644
--- a/llvm/test/Analysis/CostModel/RISCV/rvv-extractelement.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-extractelement.ll
@@ -1501,3 +1501,55 @@ define void @extractelement_int_nonpoweroftwo(i32 %x) {
 
   ret void
 }
+
+define void @extractelement_vls(i32 %x) vscale_range(2,2) {
+; RV32V-LABEL: 'extractelement_vls'
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_0 = extractelement <32 x i32> undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_4 = extractelement <32 x i32> undef, i32 4
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_5 = extractelement <32 x i32> undef, i32 5
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_8 = extractelement <32 x i32> undef, i32 8
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_9 = extractelement <32 x i32> undef, i32 9
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_11 = extractelement <32 x i32> undef, i32 11
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_12 = extractelement <32 x i32> undef, i32 12
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; RV64V-LABEL: 'extractelement_vls'
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_0 = extractelement <32 x i32> undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_4 = extractelement <32 x i32> undef, i32 4
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_5 = extractelement <32 x i32> undef, i32 5
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_8 = extractelement <32 x i32> undef, i32 8
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_9 = extractelement <32 x i32> undef, i32 9
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_11 = extractelement <32 x i32> undef, i32 11
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_12 = extractelement <32 x i32> undef, i32 12
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; RV32ZVE64X-LABEL: 'extractelement_vls'
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_0 = extractelement <32 x i32> undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_4 = extractelement <32 x i32> undef, i32 4
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_5 = extractelement <32 x i32> undef, i32 5
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_8 = extractelement <32 x i32> undef, i32 8
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_9 = extractelement <32 x i32> undef, i32 9
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_11 = extractelement <32 x i32> undef, i32 11
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_12 = extractelement <32 x i32> undef, i32 12
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; RV64ZVE64X-LABEL: 'extractelement_vls'
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_0 = extractelement <32 x i32> undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_4 = extractelement <32 x i32> undef, i32 4
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_5 = extractelement <32 x i32> undef, i32 5
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_8 = extractelement <32 x i32> undef, i32 8
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_9 = extractelement <32 x i32> undef, i32 9
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_11 = extractelement <32 x i32> undef, i32 11
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_12 = extractelement <32 x i32> undef, i32 12
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %v32i32_0 = extractelement <32 x i32> undef, i32 0
+  %v32i32_4 = extractelement <32 x i32> undef, i32 4
+  %v32i32_5 = extractelement <32 x i32> undef, i32 5
+  %v32i32_8 = extractelement <32 x i32> undef, i32 8
+  %v32i32_9 = extractelement <32 x i32> undef, i32 9
+  %v32i32_11 = extractelement <32 x i32> undef, i32 11
+  %v32i32_12 = extractelement <32 x i32> undef, i32 12
+
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll
index 0616e0919b9d9a..c240a75066b10c 100644
--- a/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll
@@ -1491,3 +1491,55 @@ define void @insertelement_int_nonpoweroftwo(i32 %x) {
 
   ret void
 }
+
+define void @insertelement_vls(i32 %x) vscale_range(2,2) {
+; RV32V-LABEL: 'insertelement_vls'
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_0 = insertelement <32 x i32> undef, i32 undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_4 = insertelement <32 x i32> undef, i32 undef, i32 4
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_5 = insertelement <32 x i32> undef, i32 undef, i32 5
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_8 = insertelement <32 x i32> undef, i32 undef, i32 8
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_9 = insertelement <32 x i32> undef, i32 undef, i32 9
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_11 = insertelement <32 x i32> undef, i32 undef, i32 11
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_12 = insertelement <32 x i32> undef, i32 undef, i32 12
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; RV64V-LABEL: 'insertelement_vls'
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_0 = insertelement <32 x i32> undef, i32 undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_4 = insertelement <32 x i32> undef, i32 undef, i32 4
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_5 = insertelement <32 x i32> undef, i32 undef, i32 5
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_8 = insertelement <32 x i32> undef, i32 undef, i32 8
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_9 = insertelement <32 x i32> undef, i32 undef, i32 9
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_11 = insertelement <32 x i32> undef, i32 undef, i32 11
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_12 = insertelement <32 x i32> undef, i32 undef, i32 12
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; RV32ZVE64X-LABEL: 'insertelement_vls'
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_0 = insertelement <32 x i32> undef, i32 undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_4 = insertelement <32 x i32> undef, i32 undef, i32 4
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_5 = insertelement <32 x i32> undef, i32 undef, i32 5
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_8 = insertelement <32 x i32> undef, i32 undef, i32 8
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_9 = insertelement <32 x i32> undef, i32 undef, i32 9
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_11 = insertelement <32 x i32> undef, i32 undef, i32 11
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_12 = insertelement <32 x i32> undef, i32 undef, i32 12
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; RV64ZVE64X-LABEL: 'insertelement_vls'
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_0 = insertelement <32 x i32> undef, i32 undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_4 = insertelement <32 x i32> undef, i32 undef, i32 4
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_5 = insertelement <32 x i32> undef, i32 undef, i32 5
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_8 = insertelement <32 x i32> undef, i32 undef, i32 8
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_9 = insertelement <32 x i32> undef, i32 undef, i32 9
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i32_11 = insertelement <32 x i32> undef, i32 undef, i32 11
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i32_12 = insertelement <32 x i32> undef, i32 undef, i32 12
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %v32i32_0 = insertelement <32 x i32> undef, i32 undef, i32 0
+  %v32i32_4 = insertelement <32 x i32> undef, i32 undef, i32 4
+  %v32i32_5 = insertelement <32 x i32> undef, i32 undef, i32 5
+  %v32i32_8 = insertelement <32 x i32> undef, i32 undef, i32 8
+  %v32i32_9 = insertelement <32 x i32> undef, i32 undef, i32 9
+  %v32i32_11 = insertelement <32 x i32> undef, i32 undef, i32 11
+  %v32i32_12 = insertelement <32 x i32> undef, i32 undef, i32 12
+
+  ret void
+}
diff --git a/llvm/test/Analysis/CtxProfAnalysis/flatten-zero-path.ll b/llvm/test/Analysis/CtxProfAnalysis/flatten-zero-path.ll
new file mode 100644
index 00000000000000..7eea1c36afc37a
--- /dev/null
+++ b/llvm/test/Analysis/CtxProfAnalysis/flatten-zero-path.ll
@@ -0,0 +1,55 @@
+; Check that flattened profile lowering handles cold subgraphs that end in "unreachable"
+; RUN: split-file %s %t
+; RUN: llvm-ctxprof-util fromJSON --input=%t/profile.json --output=%t/profile.ctxprofdata
+; RUN: opt -passes=ctx-prof-flatten %t/example.ll -use-ctx-profile=%t/profile.ctxprofdata -S -o - | FileCheck %s
+
+; CHECK-LABEL: entry:
+; CHECK:          br i1 %t, label %yes, label %no, !prof ![[C1:[0-9]+]]
+; CHECK-LABEL: no:
+; CHECK-NOT:   !prof
+; CHECK-LABEL: no1:
+; CHECK-NOT:   !prof
+; CHECK-LABEL: no2:
+; CHECK-NOT:   !prof
+; CHECK-LABEL: yes:
+; CHECK:          br i1 %t3, label %yes1, label %yes2, !prof ![[C1]]
+; CHECK-NOT:   !prof
+; CHECK: ![[C1]] = !{!"branch_weights", i32 6, i32 0}
+
+;--- example.ll
+define void @f1(i32 %cond) !guid !0 {
+entry:
+  call void @llvm.instrprof.increment(ptr @f1, i64 42, i32 42, i32 0)
+  %t = icmp eq i32 %cond, 1
+  br i1 %t, label %yes, label %no
+
+no:
+  %t2 = icmp eq i32 %cond, 2
+  br i1 %t2, label %no1, label %no2
+no1:
+  unreachable
+no2:
+  call void @llvm.instrprof.increment(ptr @f1, i64 42, i32 42, i32 1)
+  unreachable
+yes:
+  %t3 = icmp eq i32 %cond, 3
+  br i1 %t3, label %yes1, label %yes2
+yes1:
+  br label %exit
+yes2:
+  call void @llvm.instrprof.increment(ptr @f1, i64 42, i32 42, i32 2)
+  %t4 = icmp eq i32 %cond, 4
+  br i1 %t4, label %yes3, label %yes4
+yes3:
+  br label %exit
+yes4:
+  call void @llvm.instrprof.increment(ptr @f1, i64 42, i32 42, i32 3)
+  unreachable
+exit:
+  ret void
+}
+
+!0 = !{i64 1234}
+
+;--- profile.json
+[{"Guid":1234, "Counters":[6,0,0,0]}]
diff --git a/llvm/test/Analysis/ValueTracking/shift-recurrence-knownbits.ll b/llvm/test/Analysis/ValueTracking/recurrence-knownbits.ll
similarity index 65%
rename from llvm/test/Analysis/ValueTracking/shift-recurrence-knownbits.ll
rename to llvm/test/Analysis/ValueTracking/recurrence-knownbits.ll
index 5e9084aac31a38..3355328cad9ecf 100644
--- a/llvm/test/Analysis/ValueTracking/shift-recurrence-knownbits.ll
+++ b/llvm/test/Analysis/ValueTracking/recurrence-knownbits.ll
@@ -21,6 +21,106 @@ exit:
   ret i64 %res
 }
 
+define i64 @test_add(i1 %c) {
+; CHECK-LABEL: @test_add(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i64 0
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [8, %entry], [%iv.next, %loop]
+  %iv.next = add nuw i64 %iv, 4
+  br i1 %c, label %exit, label %loop
+exit:
+  %res = and i64 %iv, 1
+  ret i64 %res
+}
+
+define i64 @test_sub(i1 %c) {
+; CHECK-LABEL: @test_sub(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i64 0
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [8, %entry], [%iv.next, %loop]
+  %iv.next = sub nuw i64 %iv, 4
+  br i1 %c, label %exit, label %loop
+exit:
+  %res = and i64 %iv, 1
+  ret i64 %res
+}
+
+define i64 @test_mul(i1 %c) {
+; CHECK-LABEL: @test_mul(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i64 0
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [8, %entry], [%iv.next, %loop]
+  %iv.next = mul i64 %iv, 2
+  br i1 %c, label %exit, label %loop
+exit:
+  %res = and i64 %iv, 2
+  ret i64 %res
+}
+
+define i64 @test_and(i1 %c) {
+; CHECK-LABEL: @test_and(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i64 2047
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [1025, %entry], [%iv.next, %loop]
+  %iv.next = and i64 %iv, 1024
+  br i1 %c, label %exit, label %loop
+exit:
+  %res = or i64 %iv, 1023
+  ret i64 %res
+}
+
+define i64 @test_or(i1 %c) {
+; CHECK-LABEL: @test_or(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i64 2047
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [1025, %entry], [%iv.next, %loop]
+  %iv.next = or i64 %iv, 1024
+  br i1 %c, label %exit, label %loop
+exit:
+  %res = or i64 %iv, 1023
+  ret i64 %res
+}
+
 define i64 @test_ashr_zeros(i1 %c) {
 ; CHECK-LABEL: @test_ashr_zeros(
 ; CHECK-NEXT:  entry:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-cannonicalize-fcmp.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-cannonicalize-fcmp.mir
new file mode 100644
index 00000000000000..94204611095db0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-cannonicalize-fcmp.mir
@@ -0,0 +1,119 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -o - -mtriple=aarch64-unknown-unknown -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs  %s | FileCheck %s --check-prefixes=CHECK
+
+---
+name:            test_fcmp_canon
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: test_fcmp_canon
+    ; CHECK: %lhs:_(s64) = G_FCONSTANT double 1.000000e+00
+    ; CHECK-NEXT: %rhs:_(s64) = COPY $x0
+    ; CHECK-NEXT: %res:_(s32) = afn G_FCMP floatpred(ole), %rhs(s64), %lhs
+    ; CHECK-NEXT: $w0 = COPY %res(s32)
+    %lhs:_(s64) = G_FCONSTANT double 1.0
+    %rhs:_(s64) = COPY $x0
+    %res:_(s32) = afn G_FCMP floatpred(oge), %lhs(s64), %rhs
+    $w0 = COPY %res(s32)
+...
+---
+name:            test_fcmp_no_canon
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: test_fcmp_no_canon
+    ; CHECK: %lhs:_(s64) = COPY $x0
+    ; CHECK-NEXT: %rhs:_(s64) = G_FCONSTANT double 1.000000e+00
+    ; CHECK-NEXT: %res:_(s32) = afn G_FCMP floatpred(oge), %lhs(s64), %rhs
+    ; CHECK-NEXT: $w0 = COPY %res(s32)
+    %lhs:_(s64) = COPY $x0
+    %rhs:_(s64) = G_FCONSTANT double 1.0
+    %res:_(s32) = afn G_FCMP floatpred(oge), %lhs(s64), %rhs
+    $w0 = COPY %res(s32)
+...
+---
+name:            test_fcmp_no_canon_bv
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: test_fcmp_no_canon_bv
+    ; CHECK: %opaque1:_(s64) = COPY $x0
+    ; CHECK-NEXT: %opaque2:_(s64) = COPY $x0
+    ; CHECK-NEXT: %const1:_(s64) = G_FCONSTANT double 1.000000e+00
+    ; CHECK-NEXT: %const2:_(s64) = G_FCONSTANT double 2.000000e+00
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = G_BUILD_VECTOR %const1(s64), %opaque2(s64)
+    ; CHECK-NEXT: %rhs:_(<2 x s64>) = G_BUILD_VECTOR %opaque1(s64), %const2(s64)
+    ; CHECK-NEXT: %res:_(<2 x s32>) = afn G_FCMP floatpred(oge), %lhs(<2 x s64>), %rhs
+    ; CHECK-NEXT: $x0 = COPY %res(<2 x s32>)
+    %opaque1:_(s64) = COPY $x0
+    %opaque2:_(s64) = COPY $x0
+    %const1:_(s64) = G_FCONSTANT double 1.0
+    %const2:_(s64) = G_FCONSTANT double 2.0
+    %lhs:_(<2 x s64>) = G_BUILD_VECTOR %const1(s64), %opaque2(s64)
+    %rhs:_(<2 x s64>) = G_BUILD_VECTOR %opaque1(s64), %const2(s64)
+    %res:_(<2 x s32>) = afn G_FCMP floatpred(oge), %lhs(<2 x s64>), %rhs
+    $x0 = COPY %res(<2 x s32>)
+...
+---
+name:            test_fcmp_canon_bv
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: test_fcmp_canon_bv
+    ; CHECK: %opaque1:_(s64) = COPY $x0
+    ; CHECK-NEXT: %opaque2:_(s64) = COPY $x0
+    ; CHECK-NEXT: %const1:_(s64) = G_FCONSTANT double 1.000000e+00
+    ; CHECK-NEXT: %const2:_(s64) = G_FCONSTANT double 2.000000e+00
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = G_BUILD_VECTOR %const1(s64), %const2(s64)
+    ; CHECK-NEXT: %rhs:_(<2 x s64>) = G_BUILD_VECTOR %opaque1(s64), %opaque2(s64)
+    ; CHECK-NEXT: %res:_(<2 x s32>) = afn G_FCMP floatpred(ole), %rhs(<2 x s64>), %lhs
+    ; CHECK-NEXT: $x0 = COPY %res(<2 x s32>)
+    %opaque1:_(s64) = COPY $x0
+    %opaque2:_(s64) = COPY $x0
+    %const1:_(s64) = G_FCONSTANT double 1.0
+    %const2:_(s64) = G_FCONSTANT double 2.0
+    %lhs:_(<2 x s64>) = G_BUILD_VECTOR %const1(s64), %const2(s64)
+    %rhs:_(<2 x s64>) = G_BUILD_VECTOR %opaque1(s64), %opaque2(s64)
+    %res:_(<2 x s32>) = afn G_FCMP floatpred(oge), %lhs(<2 x s64>), %rhs
+    $x0 = COPY %res(<2 x s32>)
+...
+---
+name:            test_fcmp_canon_splat
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: test_fcmp_canon_splat
+    ; CHECK: %const:_(s64) = G_FCONSTANT double 1.000000e+00
+    ; CHECK-NEXT: %lhs:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR %const(s64)
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x1
+    ; CHECK-NEXT: %rhs:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR [[COPY]](s64)
+    ; CHECK-NEXT: %res:_(<vscale x 2 x s32>) = afn G_FCMP floatpred(ole), %rhs(<vscale x 2 x s64>), %lhs
+    ; CHECK-NEXT: %z:_(<vscale x 2 x s64>) = G_ZEXT %res(<vscale x 2 x s32>)
+    ; CHECK-NEXT: $z0 = COPY %z(<vscale x 2 x s64>)
+    %const:_(s64) = G_FCONSTANT double 1.0
+    %lhs:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR %const:_(s64)
+    %1:_(s64) = COPY $x1
+    %rhs:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR %1:_(s64)
+    %res:_(<vscale x 2 x s32>) = afn G_FCMP floatpred(oge), %lhs(<vscale x 2 x s64>), %rhs
+    %z:_(<vscale x 2 x s64>) = G_ZEXT  %res
+    $z0 = COPY %z(<vscale x 2 x s64>)
+...
+---
+name:            test_fcmp_const
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: test_fcmp_const
+    ; CHECK: %res:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: $w0 = COPY %res(s32)
+    %lhs:_(s64) = G_FCONSTANT double 1.0
+    %rhs:_(s64) = G_FCONSTANT double 2.0
+    %res:_(s32) = afn G_FCMP floatpred(oge), %lhs(s64), %rhs
+    $w0 = COPY %res(s32)
+...
+---
+name:            test_fcmp_const_other
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: test_fcmp_const_other
+    ; CHECK: %res:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: $w0 = COPY %res(s32)
+    %lhs:_(s64) = G_FCONSTANT double 2.0
+    %rhs:_(s64) = G_FCONSTANT double 1.0
+    %res:_(s32) = afn G_FCMP floatpred(oge), %lhs(s64), %rhs
+    $w0 = COPY %res(s32)
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index ee3087a81d2713..db2412de48b56a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -651,7 +651,7 @@
 # DEBUG-NEXT: G_INSERT_SUBVECTOR (opcode {{[0-9]+}}): 2 type indices, 1 imm index
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: G_EXTRACT_SUBVECTOR (opcode {{[0-9]+}}): 1 type index, 1 imm index
+# DEBUG-NEXT: G_EXTRACT_SUBVECTOR (opcode {{[0-9]+}}): 2 type indices, 1 imm index
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_INSERT_VECTOR_ELT (opcode {{[0-9]+}}): 3 type indices, 0 imm indices
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sve2-faminmax.ll b/llvm/test/CodeGen/AArch64/aarch64-sve2-faminmax.ll
new file mode 100644
index 00000000000000..05be619f95b0e8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-sve2-faminmax.ll
@@ -0,0 +1,661 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64 -mattr=+faminmax -mattr=+sve2 -verify-machineinstrs %s -o - | FileCheck %s
+; RUN: llc -mtriple=aarch64 -mattr=+faminmax -mattr=+sme2 -force-streaming -verify-machineinstrs %s -o - | FileCheck %s
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK-NO-FAMINMAX
+
+; Replace pattern max(abs(a), abs(b)) by famax(a,b)
+; Replace pattern min(abs(a), abs(b)) by famin(a,b)
+
+; When the fastmath flag 'nnan' (no nan) is enabled, we may also replace:
+; minnm(abs(a), abs(b)) with famin(a, b)
+; maxnm(abs(a), abs(b)) with famax(a, b)
+
+define <vscale x 4 x half> @famax_max_nx4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: famax_max_nx4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    famax z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famax_max_nx4f16:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.h, p0/m, z0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.h, p0/m, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    fmax z0.h, p0/m, z0.h, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half> %a)
+    %bb = call <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half> %b)
+    %r = call <vscale x 4 x half> @llvm.maximum.nxv4f16(<vscale x 4 x half> %aa, <vscale x 4 x half> %bb)
+    ret <vscale x 4 x half> %r
+}
+
+define <vscale x 4 x half> @famax_maxnm_nnan_nx4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: famax_maxnm_nnan_nx4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    famax z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famax_maxnm_nnan_nx4f16:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.h, p0/m, z0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.h, p0/m, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    fmaxnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half> %a)
+    %bb = call <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half> %b)
+    %r = call nnan <vscale x 4 x half> @llvm.maxnum.nxv4f16(<vscale x 4 x half> %aa, <vscale x 4 x half> %bb)
+    ret <vscale x 4 x half> %r
+}
+
+define <vscale x 4 x half> @famax_maxnm_noflag_nx4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: famax_maxnm_noflag_nx4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fabs z0.h, p0/m, z0.h
+; CHECK-NEXT:    fabs z1.h, p0/m, z1.h
+; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famax_maxnm_noflag_nx4f16:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.h, p0/m, z0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.h, p0/m, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    fmaxnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half> %a)
+    %bb = call <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half> %b)
+    %r = call <vscale x 4 x half> @llvm.maxnum.nxv4f16(<vscale x 4 x half> %aa, <vscale x 4 x half> %bb)
+    ret <vscale x 4 x half> %r
+}
+
+define <vscale x 4 x half> @famin_min_nx4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: famin_min_nx4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    famin z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famin_min_nx4f16:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.h, p0/m, z0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.h, p0/m, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    fmin z0.h, p0/m, z0.h, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half> %a)
+    %bb = call <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half> %b)
+    %r = call <vscale x 4 x half> @llvm.minimum.nxv4f16(<vscale x 4 x half> %aa, <vscale x 4 x half> %bb)
+    ret <vscale x 4 x half> %r
+}
+
+define <vscale x 4 x half> @famin_minnm_nnan_nx4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: famin_minnm_nnan_nx4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    famin z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famin_minnm_nnan_nx4f16:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.h, p0/m, z0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.h, p0/m, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    fminnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half> %a)
+    %bb = call <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half> %b)
+    %r = call nnan <vscale x 4 x half> @llvm.minnum.nxv4f16(<vscale x 4 x half> %aa, <vscale x 4 x half> %bb)
+    ret <vscale x 4 x half> %r
+}
+
+define <vscale x 4 x half> @famin_minnm_noflag_nx4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: famin_minnm_noflag_nx4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fabs z0.h, p0/m, z0.h
+; CHECK-NEXT:    fabs z1.h, p0/m, z1.h
+; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famin_minnm_noflag_nx4f16:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.h, p0/m, z0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.h, p0/m, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    fminnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half> %a)
+    %bb = call <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half> %b)
+    %r = call <vscale x 4 x half> @llvm.minnum.nxv4f16(<vscale x 4 x half> %aa, <vscale x 4 x half> %bb)
+    ret <vscale x 4 x half> %r
+}
+
+define <vscale x 8 x half> @famax_max_nx8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: famax_max_nx8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    famax z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famax_max_nx8f16:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.h, p0/m, z0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.h, p0/m, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    fmax z0.h, p0/m, z0.h, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half> %a)
+    %bb = call <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half> %b)
+    %r = call <vscale x 8 x half> @llvm.maximum.nxv8f16(<vscale x 8 x half> %aa, <vscale x 8 x half> %bb)
+    ret <vscale x 8 x half> %r
+}
+
+define <vscale x 8 x half> @famax_maxnm_nnan_nx8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: famax_maxnm_nnan_nx8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    famax z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famax_maxnm_nnan_nx8f16:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.h, p0/m, z0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.h, p0/m, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    fmaxnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half> %a)
+    %bb = call <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half> %b)
+    %r = call nnan <vscale x 8 x half> @llvm.maxnum.nxv8f16(<vscale x 8 x half> %aa, <vscale x 8 x half> %bb)
+    ret <vscale x 8 x half> %r
+}
+
+define <vscale x 8 x half> @famax_maxnm_noflag_nx8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: famax_maxnm_noflag_nx8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    fabs z0.h, p0/m, z0.h
+; CHECK-NEXT:    fabs z1.h, p0/m, z1.h
+; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famax_maxnm_noflag_nx8f16:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.h, p0/m, z0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.h, p0/m, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    fmaxnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half> %a)
+    %bb = call <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half> %b)
+    %r = call <vscale x 8 x half> @llvm.maxnum.nxv8f16(<vscale x 8 x half> %aa, <vscale x 8 x half> %bb)
+    ret <vscale x 8 x half> %r
+}
+
+define <vscale x 8 x half> @famin_min_nx8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: famin_min_nx8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    famin z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famin_min_nx8f16:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.h, p0/m, z0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.h, p0/m, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    fmin z0.h, p0/m, z0.h, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half> %a)
+    %bb = call <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half> %b)
+    %r = call <vscale x 8 x half> @llvm.minimum.nxv8f16(<vscale x 8 x half> %aa, <vscale x 8 x half> %bb)
+    ret <vscale x 8 x half> %r
+}
+
+define <vscale x 8 x half> @famin_minnm_nnan_nx8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: famin_minnm_nnan_nx8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    famin z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famin_minnm_nnan_nx8f16:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.h, p0/m, z0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.h, p0/m, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    fminnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half> %a)
+    %bb = call <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half> %b)
+    %r = call nnan <vscale x 8 x half> @llvm.minnum.nxv8f16(<vscale x 8 x half> %aa, <vscale x 8 x half> %bb)
+    ret <vscale x 8 x half> %r
+}
+
+define <vscale x 8 x half> @famin_minnm_noflag_nx8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: famin_minnm_noflag_nx8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    fabs z0.h, p0/m, z0.h
+; CHECK-NEXT:    fabs z1.h, p0/m, z1.h
+; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famin_minnm_noflag_nx8f16:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.h, p0/m, z0.h
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.h, p0/m, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    fminnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half> %a)
+    %bb = call <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half> %b)
+    %r = call  <vscale x 8 x half> @llvm.minnum.nxv8f16(<vscale x 8 x half> %aa, <vscale x 8 x half> %bb)
+    ret <vscale x 8 x half> %r
+}
+
+define <vscale x 2 x float> @famax_max_nx2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: famax_max_nx2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    famax z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famax_max_nx2f32:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.s, p0/m, z0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.s, p0/m, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    fmax z0.s, p0/m, z0.s, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 2 x float> @llvm.fabs.nx2f32(<vscale x 2 x float> %a)
+    %bb = call <vscale x 2 x float> @llvm.fabs.nx2f32(<vscale x 2 x float> %b)
+    %r = call <vscale x 2 x float> @llvm.maximum.nx2f32(<vscale x 2 x float> %aa, <vscale x  2 x float> %bb)
+    ret <vscale x 2 x float> %r
+}
+
+define <vscale x 2 x float> @famax_maxnm_nnan_nx2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: famax_maxnm_nnan_nx2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    famax z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famax_maxnm_nnan_nx2f32:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.s, p0/m, z0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.s, p0/m, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    fmaxnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 2 x float> @llvm.fabs.nx2f32(<vscale x 2 x float> %a)
+    %bb = call <vscale x 2 x float> @llvm.fabs.nx2f32(<vscale x 2 x float> %b)
+    %r = call nnan <vscale x 2 x float> @llvm.maxnum.nx2f32(<vscale x 2 x float> %aa, <vscale x  2 x float> %bb)
+    ret <vscale x 2 x float> %r
+}
+
+define <vscale x 2 x float> @famax_maxnm_noflag_nx2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: famax_maxnm_noflag_nx2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fabs z0.s, p0/m, z0.s
+; CHECK-NEXT:    fabs z1.s, p0/m, z1.s
+; CHECK-NEXT:    fmaxnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famax_maxnm_noflag_nx2f32:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.s, p0/m, z0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.s, p0/m, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    fmaxnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 2 x float> @llvm.fabs.nx2f32(<vscale x 2 x float> %a)
+    %bb = call <vscale x 2 x float> @llvm.fabs.nx2f32(<vscale x 2 x float> %b)
+    %r = call <vscale x 2 x float> @llvm.maxnum.nx2f32(<vscale x 2 x float> %aa, <vscale x  2 x float> %bb)
+    ret <vscale x 2 x float> %r
+}
+
+define <vscale x 2 x float> @famin_min_nx2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: famin_min_nx2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    famin z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famin_min_nx2f32:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.s, p0/m, z0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.s, p0/m, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    fmin z0.s, p0/m, z0.s, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 2 x float> @llvm.fabs.nx2f32(<vscale x 2 x float> %a)
+    %bb = call <vscale x 2 x float> @llvm.fabs.nx2f32(<vscale x 2 x float> %b)
+    %r = call <vscale x 2 x float> @llvm.minimum.nx2f32(<vscale x 2 x float> %aa, <vscale x  2 x float> %bb)
+    ret <vscale x 2 x float> %r
+}
+
+define <vscale x 2 x float> @famin_minnm_nnan_nx2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: famin_minnm_nnan_nx2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    famin z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famin_minnm_nnan_nx2f32:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.s, p0/m, z0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.s, p0/m, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    fminnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 2 x float> @llvm.fabs.nx2f32(<vscale x 2 x float> %a)
+    %bb = call <vscale x 2 x float> @llvm.fabs.nx2f32(<vscale x 2 x float> %b)
+    %r = call nnan <vscale x 2 x float> @llvm.minnum.nx2f32(<vscale x 2 x float> %aa, <vscale x  2 x float> %bb)
+    ret <vscale x 2 x float> %r
+}
+
+define <vscale x 2 x float> @famin_minnm_noflag_nx2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: famin_minnm_noflag_nx2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fabs z0.s, p0/m, z0.s
+; CHECK-NEXT:    fabs z1.s, p0/m, z1.s
+; CHECK-NEXT:    fminnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famin_minnm_noflag_nx2f32:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.s, p0/m, z0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.s, p0/m, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    fminnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 2 x float> @llvm.fabs.nx2f32(<vscale x 2 x float> %a)
+    %bb = call <vscale x 2 x float> @llvm.fabs.nx2f32(<vscale x 2 x float> %b)
+    %r = call <vscale x 2 x float> @llvm.minnum.nx2f32(<vscale x 2 x float> %aa, <vscale x  2 x float> %bb)
+    ret <vscale x 2 x float> %r
+}
+
+define <vscale x 4 x float> @famax_max_nx4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: famax_max_nx4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    famax z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famax_max_nx4f32:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.s, p0/m, z0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.s, p0/m, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    fmax z0.s, p0/m, z0.s, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 4 x float> @llvm.fabs.nx4f32(<vscale x 4 x float> %a)
+    %bb = call <vscale x 4 x float> @llvm.fabs.nx4f32(<vscale x 4 x float> %b)
+    %r = call <vscale x 4 x float> @llvm.maximum.nx4f32(<vscale x 4 x float> %aa, <vscale x  4 x float> %bb)
+    ret <vscale x 4 x float> %r
+}
+
+define <vscale x 4 x float> @famax_maxnm_nnan_nx4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: famax_maxnm_nnan_nx4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    famax z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famax_maxnm_nnan_nx4f32:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.s, p0/m, z0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.s, p0/m, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    fmaxnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 4 x float> @llvm.fabs.nx4f32(<vscale x 4 x float> %a)
+    %bb = call <vscale x 4 x float> @llvm.fabs.nx4f32(<vscale x 4 x float> %b)
+    %r = call nnan <vscale x 4 x float> @llvm.maxnum.nx4f32(<vscale x 4 x float> %aa, <vscale x  4 x float> %bb)
+    ret <vscale x 4 x float> %r
+}
+
+define <vscale x 4 x float> @famax_maxnm_noflag_nx4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: famax_maxnm_noflag_nx4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fabs z0.s, p0/m, z0.s
+; CHECK-NEXT:    fabs z1.s, p0/m, z1.s
+; CHECK-NEXT:    fmaxnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famax_maxnm_noflag_nx4f32:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.s, p0/m, z0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.s, p0/m, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    fmaxnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 4 x float> @llvm.fabs.nx4f32(<vscale x 4 x float> %a)
+    %bb = call <vscale x 4 x float> @llvm.fabs.nx4f32(<vscale x 4 x float> %b)
+    %r = call <vscale x 4 x float> @llvm.maxnum.nx4f32(<vscale x 4 x float> %aa, <vscale x  4 x float> %bb)
+    ret <vscale x 4 x float> %r
+}
+
+define <vscale x 4 x float> @famin_min_nx4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: famin_min_nx4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    famin z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famin_min_nx4f32:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.s, p0/m, z0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.s, p0/m, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    fmin z0.s, p0/m, z0.s, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 4 x float> @llvm.fabs.nx4f32(<vscale x 4 x float> %a)
+    %bb = call <vscale x 4 x float> @llvm.fabs.nx4f32(<vscale x 4 x float> %b)
+    %r = call <vscale x 4 x float> @llvm.minimum.nx4f32(<vscale x 4 x float> %aa, <vscale x  4 x float> %bb)
+    ret <vscale x 4 x float> %r
+}
+
+define <vscale x 4 x float> @famin_minnm_nnan_nx4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: famin_minnm_nnan_nx4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    famin z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famin_minnm_nnan_nx4f32:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.s, p0/m, z0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.s, p0/m, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    fminnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 4 x float> @llvm.fabs.nx4f32(<vscale x 4 x float> %a)
+    %bb = call <vscale x 4 x float> @llvm.fabs.nx4f32(<vscale x 4 x float> %b)
+    %r = call nnan <vscale x 4 x float> @llvm.minnum.nx4f32(<vscale x 4 x float> %aa, <vscale x  4 x float> %bb)
+    ret <vscale x 4 x float> %r
+}
+
+define <vscale x 4 x float> @famin_minnm_noflag_nx4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: famin_minnm_noflag_nx4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fabs z0.s, p0/m, z0.s
+; CHECK-NEXT:    fabs z1.s, p0/m, z1.s
+; CHECK-NEXT:    fminnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famin_minnm_noflag_nx4f32:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.s, p0/m, z0.s
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.s, p0/m, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    fminnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 4 x float> @llvm.fabs.nx4f32(<vscale x 4 x float> %a)
+    %bb = call <vscale x 4 x float> @llvm.fabs.nx4f32(<vscale x 4 x float> %b)
+    %r = call <vscale x 4 x float> @llvm.minnum.nx4f32(<vscale x 4 x float> %aa, <vscale x  4 x float> %bb)
+    ret <vscale x 4 x float> %r
+}
+
+define <vscale x 2 x double> @famax_max_nx2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: famax_max_nx2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    famax z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famax_max_nx2f64:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.d, p0/m, z0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.d, p0/m, z1.d
+; CHECK-NO-FAMINMAX-NEXT:    fmax z0.d, p0/m, z0.d, z1.d
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 2 x double> @llvm.fabs.nx2f64(<vscale x 2 x double> %a)
+    %bb = call <vscale x 2 x double> @llvm.fabs.nx2f64(<vscale x 2 x double> %b)
+    %r = call <vscale x 2 x double> @llvm.maximum.nx2f64(<vscale x 2 x double> %aa, <vscale x 2 x double> %bb)
+    ret <vscale x 2 x double> %r
+}
+
+define <vscale x 2 x double> @famax_maxnm_nnan_nx2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: famax_maxnm_nnan_nx2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    famax z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famax_maxnm_nnan_nx2f64:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.d, p0/m, z0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.d, p0/m, z1.d
+; CHECK-NO-FAMINMAX-NEXT:    fmaxnm z0.d, p0/m, z0.d, z1.d
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 2 x double> @llvm.fabs.nx2f64(<vscale x 2 x double> %a)
+    %bb = call <vscale x 2 x double> @llvm.fabs.nx2f64(<vscale x 2 x double> %b)
+    %r = call nnan <vscale x 2 x double> @llvm.maxnum.nx2f64(<vscale x 2 x double> %aa, <vscale x 2 x double> %bb)
+    ret <vscale x 2 x double> %r
+}
+
+define <vscale x 2 x double> @famax_maxnm_noflag_nx2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: famax_maxnm_noflag_nx2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fabs z0.d, p0/m, z0.d
+; CHECK-NEXT:    fabs z1.d, p0/m, z1.d
+; CHECK-NEXT:    fmaxnm z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famax_maxnm_noflag_nx2f64:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.d, p0/m, z0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.d, p0/m, z1.d
+; CHECK-NO-FAMINMAX-NEXT:    fmaxnm z0.d, p0/m, z0.d, z1.d
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 2 x double> @llvm.fabs.nx2f64(<vscale x 2 x double> %a)
+    %bb = call <vscale x 2 x double> @llvm.fabs.nx2f64(<vscale x 2 x double> %b)
+    %r = call <vscale x 2 x double> @llvm.maxnum.nx2f64(<vscale x 2 x double> %aa, <vscale x 2 x double> %bb)
+    ret <vscale x 2 x double> %r
+}
+
+define <vscale x 2 x double> @famin_min_nx2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: famin_min_nx2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    famin z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famin_min_nx2f64:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.d, p0/m, z0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.d, p0/m, z1.d
+; CHECK-NO-FAMINMAX-NEXT:    fmin z0.d, p0/m, z0.d, z1.d
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 2 x double> @llvm.fabs.nx2f64(<vscale x 2 x double> %a)
+    %bb = call <vscale x 2 x double> @llvm.fabs.nx2f64(<vscale x 2 x double> %b)
+    %r = call <vscale x 2 x double> @llvm.minimum.nx2f64(<vscale x 2 x double> %aa, <vscale x 2 x double> %bb)
+    ret <vscale x 2 x double> %r
+}
+
+define <vscale x 2 x double> @famin_minnm_nnan_nx2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: famin_minnm_nnan_nx2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    famin z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famin_minnm_nnan_nx2f64:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.d, p0/m, z0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.d, p0/m, z1.d
+; CHECK-NO-FAMINMAX-NEXT:    fminnm z0.d, p0/m, z0.d, z1.d
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 2 x double> @llvm.fabs.nx2f64(<vscale x 2 x double> %a)
+    %bb = call <vscale x 2 x double> @llvm.fabs.nx2f64(<vscale x 2 x double> %b)
+    %r = call nnan <vscale x 2 x double> @llvm.minnum.nx2f64(<vscale x 2 x double> %aa, <vscale x 2 x double> %bb)
+    ret <vscale x 2 x double> %r
+}
+
+define <vscale x 2 x double> @famin_minnm_noflag_nx2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: famin_minnm_noflag_nx2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fabs z0.d, p0/m, z0.d
+; CHECK-NEXT:    fabs z1.d, p0/m, z1.d
+; CHECK-NEXT:    fminnm z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+;
+; CHECK-NO-FAMINMAX-LABEL: famin_minnm_noflag_nx2f64:
+; CHECK-NO-FAMINMAX:       // %bb.0:
+; CHECK-NO-FAMINMAX-NEXT:    ptrue p0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z0.d, p0/m, z0.d
+; CHECK-NO-FAMINMAX-NEXT:    fabs z1.d, p0/m, z1.d
+; CHECK-NO-FAMINMAX-NEXT:    fminnm z0.d, p0/m, z0.d, z1.d
+; CHECK-NO-FAMINMAX-NEXT:    ret
+    %aa = call <vscale x 2 x double> @llvm.fabs.nx2f64(<vscale x 2 x double> %a)
+    %bb = call <vscale x 2 x double> @llvm.fabs.nx2f64(<vscale x 2 x double> %b)
+    %r = call <vscale x 2 x double> @llvm.minnum.nx2f64(<vscale x 2 x double> %aa, <vscale x 2 x double> %bb)
+    ret <vscale x 2 x double> %r
+}
+
+declare <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.fabs.nx2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.fabs.nx4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.fabs.nx2f64(<vscale x 2 x double>)
+
+declare <vscale x 4 x half> @llvm.maximum.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.maximum.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.maximum.nx2f32(<vscale x 2 x float>, <vscale x  2 x float>)
+declare <vscale x 4 x float> @llvm.maximum.nx4f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.maximum.nx2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 4 x half> @llvm.minimum.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.minimum.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.minimum.nx2f32(<vscale x 2 x float>, <vscale x  2 x float>)
+declare <vscale x 4 x float> @llvm.minimum.nx4f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.minimum.nx2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+
+declare <vscale x 4 x half> @llvm.maxnum.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.maxnum.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half> )
+declare <vscale x 2 x float> @llvm.maxnum.nx2f32(<vscale x 2 x float>, <vscale x  2 x float>)
+declare <vscale x 2 x double> @llvm.maxnum.nx2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+declare <vscale x 4 x float> @llvm.maxnum.nx4f32(<vscale x 4 x float>, <vscale x  4 x float>)
+
+declare <vscale x 4 x half> @llvm.minnum.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.minnum.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.minnum.nx2f32(<vscale x 2 x float>, <vscale x  2 x float>)
+declare <vscale x 4 x float> @llvm.minnum.nx4f32(<vscale x 4 x float>, <vscale x  4 x float>)
+declare <vscale x 2 x double> @llvm.minnum.nx2f64(<vscale x 2 x double>, <vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/addsub-24bit-imm.mir b/llvm/test/CodeGen/AArch64/addsub-24bit-imm.mir
index 98a73bdd932034..f6c0ec083096ec 100644
--- a/llvm/test/CodeGen/AArch64/addsub-24bit-imm.mir
+++ b/llvm/test/CodeGen/AArch64/addsub-24bit-imm.mir
@@ -36,8 +36,10 @@ body: |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
     ; CHECK-NEXT: [[ADDXri:%[0-9]+]]:gpr64sp = ADDXri [[COPY]], 273, 12
     ; CHECK-NEXT: [[ADDXri1:%[0-9]+]]:gpr64common = ADDXri [[ADDXri]], 3549, 0
-    ; CHECK-NEXT: [[UBFMXri:%[0-9]+]]:gpr64 = UBFMXri [[ADDXri1]], 28, 31
-    ; CHECK-NEXT: $x0 = COPY [[UBFMXri]]
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY [[ADDXri1]].sub_32
+    ; CHECK-NEXT: [[UBFMWri:%[0-9]+]]:gpr32 = UBFMWri [[COPY1]], 28, 31
+    ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gpr64 = SUBREG_TO_REG 0, [[UBFMWri]], %subreg.sub_32
+    ; CHECK-NEXT: $x0 = COPY [[SUBREG_TO_REG]]
     ; CHECK-NEXT: RET_ReallyLR implicit $x0
     %0:gpr64 = COPY $x0
     %1:gpr32 = MOVi32imm 1121757
@@ -58,8 +60,10 @@ body: |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
     ; CHECK-NEXT: [[SUBXri:%[0-9]+]]:gpr64sp = SUBXri [[COPY]], 273, 12
     ; CHECK-NEXT: [[SUBXri1:%[0-9]+]]:gpr64common = SUBXri [[SUBXri]], 3549, 0
-    ; CHECK-NEXT: [[UBFMXri:%[0-9]+]]:gpr64 = UBFMXri [[SUBXri1]], 28, 31
-    ; CHECK-NEXT: $x0 = COPY [[UBFMXri]]
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY [[SUBXri1]].sub_32
+    ; CHECK-NEXT: [[UBFMWri:%[0-9]+]]:gpr32 = UBFMWri [[COPY1]], 28, 31
+    ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gpr64 = SUBREG_TO_REG 0, [[UBFMWri]], %subreg.sub_32
+    ; CHECK-NEXT: $x0 = COPY [[SUBREG_TO_REG]]
     ; CHECK-NEXT: RET_ReallyLR implicit $x0
     %0:gpr64 = COPY $x0
     %1:gpr64 = MOVi64imm -1121757
diff --git a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
index 2b42a3f29a72be..bdbff0563a22b9 100644
--- a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
@@ -810,7 +810,7 @@ define i32 @fct19(i64 %arg1) nounwind readonly ssp  {
 ; LLC-NEXT:    add w0, w8, #16
 ; LLC-NEXT:    ret
 ; LLC-NEXT:  .LBB26_4: // %if.end13
-; LLC-NEXT:    ubfx x8, x0, #16, #16
+; LLC-NEXT:    lsr w8, w0, #16
 ; LLC-NEXT:    cbz w8, .LBB26_6
 ; LLC-NEXT:  // %bb.5: // %if.then17
 ; LLC-NEXT:    adrp x9, first_ones
diff --git a/llvm/test/CodeGen/AArch64/arm64-ld-from-st.ll b/llvm/test/CodeGen/AArch64/arm64-ld-from-st.ll
index 44e7bdc37f449a..533a436868300f 100644
--- a/llvm/test/CodeGen/AArch64/arm64-ld-from-st.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-ld-from-st.ll
@@ -64,7 +64,7 @@ define i16 @Str64Ldr16_1(ptr nocapture %P, i64 %v, i64 %n) {
 ; CHECK-LABEL: Str64Ldr16_1:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x1, [x0, #8]
-; CHECK-NEXT:    ubfx x0, x1, #16, #16
+; CHECK-NEXT:    lsr w0, w1, #16
 ; CHECK-NEXT:    ret
 entry:
   %arrayidx0 = getelementptr inbounds i64, ptr %P, i64 1
@@ -149,7 +149,7 @@ define i8 @Str64Ldr8_3(ptr nocapture %P, i64 %v, i64 %n) {
 ; CHECK-LABEL: Str64Ldr8_3:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x1, [x0, #8]
-; CHECK-NEXT:    ubfx x0, x1, #24, #8
+; CHECK-NEXT:    lsr w0, w1, #24
 ; CHECK-NEXT:    ret
 entry:
   %arrayidx0 = getelementptr inbounds i64, ptr %P, i64 1
@@ -424,7 +424,7 @@ define i16 @Unscaled_Str64Ldr16_1(ptr nocapture %P, i64 %v, i64 %n) {
 ; CHECK-LABEL: Unscaled_Str64Ldr16_1:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    stur x1, [x0, #-8]
-; CHECK-NEXT:    ubfx x0, x1, #16, #16
+; CHECK-NEXT:    lsr w0, w1, #16
 ; CHECK-NEXT:    ret
 entry:
   %arrayidx0 = getelementptr inbounds i64, ptr %P, i64 -1
@@ -509,7 +509,7 @@ define i8 @Unscaled_Str64Ldr8_3(ptr nocapture %P, i64 %v, i64 %n) {
 ; CHECK-LABEL: Unscaled_Str64Ldr8_3:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    stur x1, [x0, #-8]
-; CHECK-NEXT:    ubfx x0, x1, #24, #8
+; CHECK-NEXT:    lsr w0, w1, #24
 ; CHECK-NEXT:    ret
 entry:
   %arrayidx0 = getelementptr inbounds i64, ptr %P, i64 -1
diff --git a/llvm/test/CodeGen/AArch64/arm64_32.ll b/llvm/test/CodeGen/AArch64/arm64_32.ll
index c63edf0ceeea37..cddadcab9cde18 100644
--- a/llvm/test/CodeGen/AArch64/arm64_32.ll
+++ b/llvm/test/CodeGen/AArch64/arm64_32.ll
@@ -677,7 +677,7 @@ declare i64 @get_int()
 
 define i1 @test_icmp_ptr(ptr %in) {
 ; CHECK-LABEL: test_icmp_ptr
-; CHECK: ubfx x0, x0, #31, #1
+; CHECK: lsr w0, w0, #31
   %res = icmp slt ptr %in, null
   ret i1 %res
 }
diff --git a/llvm/test/CodeGen/AArch64/bitfield-extract.ll b/llvm/test/CodeGen/AArch64/bitfield-extract.ll
index 9c8871af4a5e9e..9caf34860051c7 100644
--- a/llvm/test/CodeGen/AArch64/bitfield-extract.ll
+++ b/llvm/test/CodeGen/AArch64/bitfield-extract.ll
@@ -99,7 +99,7 @@ declare void @use(i16 signext, i64)
 
 ; CHECK-LABEL: test_complex_node:
 ; CHECK: ldr d0, [x0], #8
-; CHECK: ubfx x[[VAL:[0-9]+]], x0, #5, #27
+; CHECK: lsr w[[VAL:[0-9]+]], w0, #5
 ; CHECK: str w[[VAL]], [x2]
 define <2 x i32> @test_complex_node(ptr %addr, ptr %addr2, ptr %bf ) {
   %vec = load <2 x i32>, ptr %addr
@@ -113,3 +113,11 @@ define <2 x i32> @test_complex_node(ptr %addr, ptr %addr2, ptr %bf ) {
 
   ret <2 x i32> %vec
 }
+
+; CHECK-LABEL: @test12
+; CHECK: lsr w0, w0, #10
+define i32 @test12(i64 %a) {
+  %tmp = trunc i64 %a to i32
+  %res = lshr i32 %tmp, 10
+  ret i32 %res
+}
diff --git a/llvm/test/CodeGen/AArch64/fast-isel-int-ext3.ll b/llvm/test/CodeGen/AArch64/fast-isel-int-ext3.ll
index 7602ce3fb1c7f8..32a8a943945fc9 100644
--- a/llvm/test/CodeGen/AArch64/fast-isel-int-ext3.ll
+++ b/llvm/test/CodeGen/AArch64/fast-isel-int-ext3.ll
@@ -52,7 +52,7 @@ define i64 @load_unscaled_zext_i16_to_i64(i64 %a) {
 define i64 @load_unscaled_zext_i32_to_i64(i64 %a) {
 ; CHECK-LABEL: load_unscaled_zext_i32_to_i64
 ; CHECK:       ldur w[[REG:[0-9]+]], [x0, #-8]
-; CHECK:       ubfx x0, x[[REG]], #0, #32
+; CHECK:       lsr w0, w[[REG]], #0
   %1 = sub i64 %a, 8
   %2 = inttoptr i64 %1 to ptr addrspace(256)
   %3 = load i32, ptr addrspace(256) %2
diff --git a/llvm/test/CodeGen/AArch64/fast-isel-shift.ll b/llvm/test/CodeGen/AArch64/fast-isel-shift.ll
index 95891db80bc4ed..76f81719b4547e 100644
--- a/llvm/test/CodeGen/AArch64/fast-isel-shift.ll
+++ b/llvm/test/CodeGen/AArch64/fast-isel-shift.ll
@@ -681,7 +681,7 @@ define i64 @shl_zext_zero(i32 %a) {
 ; CHECK-LABEL: shl_zext_zero:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    ; kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    ubfx x0, x0, #0, #32
+; CHECK-NEXT:    lsr w0, w0, #0
 ; CHECK-NEXT:    ret
   %1 = zext i32 %a to i64
   %2 = shl i64 %1, 0
@@ -692,7 +692,7 @@ define i64 @lshr_zext_zero(i32 %a) {
 ; CHECK-LABEL: lshr_zext_zero:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    ; kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    ubfx x0, x0, #0, #32
+; CHECK-NEXT:    lsr w0, w0, #0
 ; CHECK-NEXT:    ret
   %1 = zext i32 %a to i64
   %2 = lshr i64 %1, 0
@@ -703,7 +703,7 @@ define i64 @ashr_zext_zero(i32 %a) {
 ; CHECK-LABEL: ashr_zext_zero:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    ; kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    ubfx x0, x0, #0, #32
+; CHECK-NEXT:    lsr w0, w0, #0
 ; CHECK-NEXT:    ret
   %1 = zext i32 %a to i64
   %2 = ashr i64 %1, 0
diff --git a/llvm/test/CodeGen/AArch64/machine_cse_impdef_killflags.ll b/llvm/test/CodeGen/AArch64/machine_cse_impdef_killflags.ll
index be1c0f21cc77ab..398a2ac24e5d03 100644
--- a/llvm/test/CodeGen/AArch64/machine_cse_impdef_killflags.ll
+++ b/llvm/test/CodeGen/AArch64/machine_cse_impdef_killflags.ll
@@ -12,7 +12,7 @@ define i64 @csed_impdef_killflag(i64 %a) {
 ; CHECK-NEXT:    mov x9, #2 ; =0x2
 ; CHECK-NEXT:    csel w8, wzr, w8, ne
 ; CHECK-NEXT:    mov x10, #3 ; =0x3
-; CHECK-NEXT:    ubfx x8, x8, #0, #32
+; CHECK-NEXT:    lsr w8, w8, #0
 ; CHECK-NEXT:    csel x9, x9, x10, ne
 ; CHECK-NEXT:    add x0, x9, x8
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll b/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll
index c4a58ba12dc6be..c838ffb0a6576e 100644
--- a/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll
@@ -575,7 +575,7 @@ define void @trunc_v8i19_to_v8i8_in_loop(ptr %A, ptr %dst) {
 ; CHECK-NEXT:    ldrb w14, [x0, #18]
 ; CHECK-NEXT:    ldrh w15, [x0, #16]
 ; CHECK-NEXT:    add x0, x0, #32
-; CHECK-NEXT:    ubfx x12, x10, #12, #20
+; CHECK-NEXT:    lsr w12, w10, #12
 ; CHECK-NEXT:    fmov s1, w9
 ; CHECK-NEXT:    lsr x11, x9, #19
 ; CHECK-NEXT:    lsr x13, x10, #31
@@ -586,7 +586,7 @@ define void @trunc_v8i19_to_v8i8_in_loop(ptr %A, ptr %dst) {
 ; CHECK-NEXT:    orr x11, x15, x14, lsl #16
 ; CHECK-NEXT:    mov.s v0[1], w13
 ; CHECK-NEXT:    extr x13, x11, x10, #50
-; CHECK-NEXT:    ubfx x10, x11, #5, #27
+; CHECK-NEXT:    lsr w10, w11, #5
 ; CHECK-NEXT:    mov.s v1[2], w12
 ; CHECK-NEXT:    mov.s v0[2], w13
 ; CHECK-NEXT:    mov.s v1[3], w9
@@ -616,14 +616,14 @@ define void @trunc_v8i19_to_v8i8_in_loop(ptr %A, ptr %dst) {
 ; CHECK-BE-NEXT:    lsr x15, x10, #40
 ; CHECK-BE-NEXT:    extr x12, x12, x11, #57
 ; CHECK-BE-NEXT:    fmov s0, w13
-; CHECK-BE-NEXT:    ubfx x13, x10, #7, #25
+; CHECK-BE-NEXT:    lsr w13, w10, #7
 ; CHECK-BE-NEXT:    extr x14, x15, x14, #50
-; CHECK-BE-NEXT:    ubfx x15, x9, #14, #18
+; CHECK-BE-NEXT:    lsr w15, w9, #14
 ; CHECK-BE-NEXT:    extr x9, x10, x9, #40
 ; CHECK-BE-NEXT:    fmov s1, w12
 ; CHECK-BE-NEXT:    orr w12, w17, w16, lsl #8
 ; CHECK-BE-NEXT:    mov v0.s[1], w14
-; CHECK-BE-NEXT:    ubfx x9, x9, #12, #20
+; CHECK-BE-NEXT:    lsr w9, w9, #12
 ; CHECK-BE-NEXT:    orr w11, w12, w11
 ; CHECK-BE-NEXT:    mov v1.s[1], w15
 ; CHECK-BE-NEXT:    lsr w11, w11, #19
@@ -657,14 +657,14 @@ define void @trunc_v8i19_to_v8i8_in_loop(ptr %A, ptr %dst) {
 ; CHECK-DISABLE-NEXT:    lsr x15, x10, #40
 ; CHECK-DISABLE-NEXT:    extr x12, x12, x11, #57
 ; CHECK-DISABLE-NEXT:    fmov s0, w13
-; CHECK-DISABLE-NEXT:    ubfx x13, x10, #7, #25
+; CHECK-DISABLE-NEXT:    lsr w13, w10, #7
 ; CHECK-DISABLE-NEXT:    extr x14, x15, x14, #50
-; CHECK-DISABLE-NEXT:    ubfx x15, x9, #14, #18
+; CHECK-DISABLE-NEXT:    lsr w15, w9, #14
 ; CHECK-DISABLE-NEXT:    extr x9, x10, x9, #40
 ; CHECK-DISABLE-NEXT:    fmov s1, w12
 ; CHECK-DISABLE-NEXT:    orr w12, w17, w16, lsl #8
 ; CHECK-DISABLE-NEXT:    mov v0.s[1], w14
-; CHECK-DISABLE-NEXT:    ubfx x9, x9, #12, #20
+; CHECK-DISABLE-NEXT:    lsr w9, w9, #12
 ; CHECK-DISABLE-NEXT:    orr w11, w12, w11
 ; CHECK-DISABLE-NEXT:    mov v1.s[1], w15
 ; CHECK-DISABLE-NEXT:    lsr w11, w11, #19
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-bitext.ll b/llvm/test/CodeGen/AArch64/vecreduce-bitext.ll
new file mode 100644
index 00000000000000..cc65f17b718649
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vecreduce-bitext.ll
@@ -0,0 +1,1202 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s
+
+define zeroext i16 @and_sext_v8i8_i16(<8 x i8> %x) {
+; CHECK-LABEL: and_sext_v8i8_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    and w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb w8, w8
+; CHECK-NEXT:    and w0, w8, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <8 x i8> %x to <8 x i16>
+  %z = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> %y)
+  ret i16 %z
+}
+
+define zeroext i16 @and_zext_v8i8_i16(<8 x i8> %x) {
+; CHECK-LABEL: and_zext_v8i8_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    and w8, w8, w8, lsr #8
+; CHECK-NEXT:    and w0, w8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <8 x i8> %x to <8 x i16>
+  %z = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> %y)
+  ret i16 %z
+}
+
+define zeroext i16 @and_sext_v16i8_i16(<16 x i8> %x) {
+; CHECK-LABEL: and_sext_v16i8_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    and w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb w8, w8
+; CHECK-NEXT:    and w0, w8, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <16 x i8> %x to <16 x i16>
+  %z = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> %y)
+  ret i16 %z
+}
+
+define zeroext i16 @and_zext_v16i8_i16(<16 x i8> %x) {
+; CHECK-LABEL: and_zext_v16i8_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    and w8, w8, w8, lsr #8
+; CHECK-NEXT:    and w0, w8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <16 x i8> %x to <16 x i16>
+  %z = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> %y)
+  ret i16 %z
+}
+
+define i32 @and_sext_v8i8_i32(<8 x i8> %x) {
+; CHECK-LABEL: and_sext_v8i8_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    and w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <8 x i8> %x to <8 x i32>
+  %z = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @and_zext_v8i8_i32(<8 x i8> %x) {
+; CHECK-LABEL: and_zext_v8i8_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    and w8, w8, w8, lsr #8
+; CHECK-NEXT:    and w0, w8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <8 x i8> %x to <8 x i32>
+  %z = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @and_sext_v16i8_i32(<16 x i8> %x) {
+; CHECK-LABEL: and_sext_v16i8_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    and w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <16 x i8> %x to <16 x i32>
+  %z = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @and_zext_v16i8_i32(<16 x i8> %x) {
+; CHECK-LABEL: and_zext_v16i8_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    and w8, w8, w8, lsr #8
+; CHECK-NEXT:    and w0, w8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <16 x i8> %x to <16 x i32>
+  %z = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> %y)
+  ret i32 %z
+}
+
+define i64 @and_sext_v8i8_i64(<8 x i8> %x) {
+; CHECK-LABEL: and_sext_v8i8_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    and w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <8 x i8> %x to <8 x i64>
+  %z = call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @and_zext_v8i8_i64(<8 x i8> %x) {
+; CHECK-LABEL: and_zext_v8i8_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    and w8, w8, w8, lsr #8
+; CHECK-NEXT:    and x0, x8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <8 x i8> %x to <8 x i64>
+  %z = call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @and_sext_v16i8_i64(<16 x i8> %x) {
+; CHECK-LABEL: and_sext_v16i8_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    and w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <16 x i8> %x to <16 x i64>
+  %z = call i64 @llvm.vector.reduce.and.v16i64(<16 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @and_zext_v16i8_i64(<16 x i8> %x) {
+; CHECK-LABEL: and_zext_v16i8_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    and w8, w8, w8, lsr #8
+; CHECK-NEXT:    and x0, x8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <16 x i8> %x to <16 x i64>
+  %z = call i64 @llvm.vector.reduce.and.v16i64(<16 x i64> %y)
+  ret i64 %z
+}
+
+define i32 @and_sext_v4i16_i32(<4 x i16> %x) {
+; CHECK-LABEL: and_sext_v4i16_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    and w8, w8, w8, lsr #16
+; CHECK-NEXT:    sxth w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <4 x i16> %x to <4 x i32>
+  %z = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @and_zext_v4i16_i32(<4 x i16> %x) {
+; CHECK-LABEL: and_zext_v4i16_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w0, w8, w9
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <4 x i16> %x to <4 x i32>
+  %z = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @and_sext_v8i16_i32(<8 x i16> %x) {
+; CHECK-LABEL: and_sext_v8i16_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    and w8, w8, w8, lsr #16
+; CHECK-NEXT:    sxth w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <8 x i16> %x to <8 x i32>
+  %z = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @and_zext_v8i16_i32(<8 x i16> %x) {
+; CHECK-LABEL: and_zext_v8i16_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w0, w8, w9
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <8 x i16> %x to <8 x i32>
+  %z = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> %y)
+  ret i32 %z
+}
+
+define i64 @and_sext_v4i16_i64(<4 x i16> %x) {
+; CHECK-LABEL: and_sext_v4i16_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    and w8, w8, w8, lsr #16
+; CHECK-NEXT:    sxth x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <4 x i16> %x to <4 x i64>
+  %z = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @and_zext_v4i16_i64(<4 x i16> %x) {
+; CHECK-LABEL: and_zext_v4i16_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w0, w8, w9
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <4 x i16> %x to <4 x i64>
+  %z = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @and_sext_v8i16_i64(<8 x i16> %x) {
+; CHECK-LABEL: and_sext_v8i16_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    and w8, w8, w8, lsr #16
+; CHECK-NEXT:    sxth x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <8 x i16> %x to <8 x i64>
+  %z = call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @and_zext_v8i16_i64(<8 x i16> %x) {
+; CHECK-LABEL: and_zext_v8i16_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w0, w8, w9
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <8 x i16> %x to <8 x i64>
+  %z = call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @and_sext_v2i32_i64(<2 x i32> %x) {
+; CHECK-LABEL: and_sext_v2i32_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    sxtw x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <2 x i32> %x to <2 x i64>
+  %z = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @and_zext_v2i32_i64(<2 x i32> %x) {
+; CHECK-LABEL: and_zext_v2i32_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    and w0, w8, w9
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <2 x i32> %x to <2 x i64>
+  %z = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @and_sext_v4i32_i64(<4 x i32> %x) {
+; CHECK-LABEL: and_sext_v4i32_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    and w8, w8, w9
+; CHECK-NEXT:    sxtw x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <4 x i32> %x to <4 x i64>
+  %z = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @and_zext_v4i32_i64(<4 x i32> %x) {
+; CHECK-LABEL: and_zext_v4i32_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    and w0, w8, w9
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <4 x i32> %x to <4 x i64>
+  %z = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> %y)
+  ret i64 %z
+}
+
+define zeroext i16 @or_sext_v8i8_i16(<8 x i8> %x) {
+; CHECK-LABEL: or_sext_v8i8_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb w8, w8
+; CHECK-NEXT:    and w0, w8, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <8 x i8> %x to <8 x i16>
+  %z = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> %y)
+  ret i16 %z
+}
+
+define zeroext i16 @or_zext_v8i8_i16(<8 x i8> %x) {
+; CHECK-LABEL: or_zext_v8i8_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #8
+; CHECK-NEXT:    and w0, w8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <8 x i8> %x to <8 x i16>
+  %z = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> %y)
+  ret i16 %z
+}
+
+define zeroext i16 @or_sext_v16i8_i16(<16 x i8> %x) {
+; CHECK-LABEL: or_sext_v16i8_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb w8, w8
+; CHECK-NEXT:    and w0, w8, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <16 x i8> %x to <16 x i16>
+  %z = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> %y)
+  ret i16 %z
+}
+
+define zeroext i16 @or_zext_v16i8_i16(<16 x i8> %x) {
+; CHECK-LABEL: or_zext_v16i8_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #8
+; CHECK-NEXT:    and w0, w8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <16 x i8> %x to <16 x i16>
+  %z = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> %y)
+  ret i16 %z
+}
+
+define i32 @or_sext_v8i8_i32(<8 x i8> %x) {
+; CHECK-LABEL: or_sext_v8i8_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <8 x i8> %x to <8 x i32>
+  %z = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @or_zext_v8i8_i32(<8 x i8> %x) {
+; CHECK-LABEL: or_zext_v8i8_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #8
+; CHECK-NEXT:    and w0, w8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <8 x i8> %x to <8 x i32>
+  %z = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @or_sext_v16i8_i32(<16 x i8> %x) {
+; CHECK-LABEL: or_sext_v16i8_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <16 x i8> %x to <16 x i32>
+  %z = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @or_zext_v16i8_i32(<16 x i8> %x) {
+; CHECK-LABEL: or_zext_v16i8_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #8
+; CHECK-NEXT:    and w0, w8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <16 x i8> %x to <16 x i32>
+  %z = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> %y)
+  ret i32 %z
+}
+
+define i64 @or_sext_v8i8_i64(<8 x i8> %x) {
+; CHECK-LABEL: or_sext_v8i8_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <8 x i8> %x to <8 x i64>
+  %z = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @or_zext_v8i8_i64(<8 x i8> %x) {
+; CHECK-LABEL: or_zext_v8i8_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #8
+; CHECK-NEXT:    and x0, x8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <8 x i8> %x to <8 x i64>
+  %z = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @or_sext_v16i8_i64(<16 x i8> %x) {
+; CHECK-LABEL: or_sext_v16i8_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <16 x i8> %x to <16 x i64>
+  %z = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @or_zext_v16i8_i64(<16 x i8> %x) {
+; CHECK-LABEL: or_zext_v16i8_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #8
+; CHECK-NEXT:    and x0, x8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <16 x i8> %x to <16 x i64>
+  %z = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> %y)
+  ret i64 %z
+}
+
+define i32 @or_sext_v4i16_i32(<4 x i16> %x) {
+; CHECK-LABEL: or_sext_v4i16_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #16
+; CHECK-NEXT:    sxth w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <4 x i16> %x to <4 x i32>
+  %z = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @or_zext_v4i16_i32(<4 x i16> %x) {
+; CHECK-LABEL: or_zext_v4i16_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #16
+; CHECK-NEXT:    and w0, w8, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <4 x i16> %x to <4 x i32>
+  %z = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @or_sext_v8i16_i32(<8 x i16> %x) {
+; CHECK-LABEL: or_sext_v8i16_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #16
+; CHECK-NEXT:    sxth w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <8 x i16> %x to <8 x i32>
+  %z = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @or_zext_v8i16_i32(<8 x i16> %x) {
+; CHECK-LABEL: or_zext_v8i16_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #16
+; CHECK-NEXT:    and w0, w8, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <8 x i16> %x to <8 x i32>
+  %z = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> %y)
+  ret i32 %z
+}
+
+define i64 @or_sext_v4i16_i64(<4 x i16> %x) {
+; CHECK-LABEL: or_sext_v4i16_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #16
+; CHECK-NEXT:    sxth x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <4 x i16> %x to <4 x i64>
+  %z = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @or_zext_v4i16_i64(<4 x i16> %x) {
+; CHECK-LABEL: or_zext_v4i16_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #16
+; CHECK-NEXT:    and x0, x8, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <4 x i16> %x to <4 x i64>
+  %z = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @or_sext_v8i16_i64(<8 x i16> %x) {
+; CHECK-LABEL: or_sext_v8i16_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #16
+; CHECK-NEXT:    sxth x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <8 x i16> %x to <8 x i64>
+  %z = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @or_zext_v8i16_i64(<8 x i16> %x) {
+; CHECK-LABEL: or_zext_v8i16_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    orr w8, w8, w8, lsr #16
+; CHECK-NEXT:    and x0, x8, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <8 x i16> %x to <8 x i64>
+  %z = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @or_sext_v2i32_i64(<2 x i32> %x) {
+; CHECK-LABEL: or_sext_v2i32_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    sxtw x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <2 x i32> %x to <2 x i64>
+  %z = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @or_zext_v2i32_i64(<2 x i32> %x) {
+; CHECK-LABEL: or_zext_v2i32_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    orr w0, w8, w9
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <2 x i32> %x to <2 x i64>
+  %z = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @or_sext_v4i32_i64(<4 x i32> %x) {
+; CHECK-LABEL: or_sext_v4i32_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    sxtw x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <4 x i32> %x to <4 x i64>
+  %z = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @or_zext_v4i32_i64(<4 x i32> %x) {
+; CHECK-LABEL: or_zext_v4i32_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    orr w0, w8, w9
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <4 x i32> %x to <4 x i64>
+  %z = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> %y)
+  ret i64 %z
+}
+
+define zeroext i16 @xor_sext_v8i8_i16(<8 x i8> %x) {
+; CHECK-LABEL: xor_sext_v8i8_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb w8, w8
+; CHECK-NEXT:    and w0, w8, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <8 x i8> %x to <8 x i16>
+  %z = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> %y)
+  ret i16 %z
+}
+
+define zeroext i16 @xor_zext_v8i8_i16(<8 x i8> %x) {
+; CHECK-LABEL: xor_zext_v8i8_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #8
+; CHECK-NEXT:    and w0, w8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <8 x i8> %x to <8 x i16>
+  %z = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> %y)
+  ret i16 %z
+}
+
+define zeroext i16 @xor_sext_v16i8_i16(<16 x i8> %x) {
+; CHECK-LABEL: xor_sext_v16i8_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb w8, w8
+; CHECK-NEXT:    and w0, w8, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <16 x i8> %x to <16 x i16>
+  %z = call i16 @llvm.vector.reduce.xor.v16i16(<16 x i16> %y)
+  ret i16 %z
+}
+
+define zeroext i16 @xor_zext_v16i8_i16(<16 x i8> %x) {
+; CHECK-LABEL: xor_zext_v16i8_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #8
+; CHECK-NEXT:    and w0, w8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <16 x i8> %x to <16 x i16>
+  %z = call i16 @llvm.vector.reduce.xor.v16i16(<16 x i16> %y)
+  ret i16 %z
+}
+
+define i32 @xor_sext_v8i8_i32(<8 x i8> %x) {
+; CHECK-LABEL: xor_sext_v8i8_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <8 x i8> %x to <8 x i32>
+  %z = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @xor_zext_v8i8_i32(<8 x i8> %x) {
+; CHECK-LABEL: xor_zext_v8i8_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #8
+; CHECK-NEXT:    and w0, w8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <8 x i8> %x to <8 x i32>
+  %z = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @xor_sext_v16i8_i32(<16 x i8> %x) {
+; CHECK-LABEL: xor_sext_v16i8_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <16 x i8> %x to <16 x i32>
+  %z = call i32 @llvm.vector.reduce.xor.v16i32(<16 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @xor_zext_v16i8_i32(<16 x i8> %x) {
+; CHECK-LABEL: xor_zext_v16i8_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #8
+; CHECK-NEXT:    and w0, w8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <16 x i8> %x to <16 x i32>
+  %z = call i32 @llvm.vector.reduce.xor.v16i32(<16 x i32> %y)
+  ret i32 %z
+}
+
+define i64 @xor_sext_v8i8_i64(<8 x i8> %x) {
+; CHECK-LABEL: xor_sext_v8i8_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <8 x i8> %x to <8 x i64>
+  %z = call i64 @llvm.vector.reduce.xor.v8i64(<8 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @xor_zext_v8i8_i64(<8 x i8> %x) {
+; CHECK-LABEL: xor_zext_v8i8_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #8
+; CHECK-NEXT:    and x0, x8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <8 x i8> %x to <8 x i64>
+  %z = call i64 @llvm.vector.reduce.xor.v8i64(<8 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @xor_sext_v16i8_i64(<16 x i8> %x) {
+; CHECK-LABEL: xor_sext_v16i8_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #8
+; CHECK-NEXT:    sxtb x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <16 x i8> %x to <16 x i64>
+  %z = call i64 @llvm.vector.reduce.xor.v16i64(<16 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @xor_zext_v16i8_i64(<16 x i8> %x) {
+; CHECK-LABEL: xor_zext_v16i8_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #8
+; CHECK-NEXT:    and x0, x8, #0xff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <16 x i8> %x to <16 x i64>
+  %z = call i64 @llvm.vector.reduce.xor.v16i64(<16 x i64> %y)
+  ret i64 %z
+}
+
+define i32 @xor_sext_v4i16_i32(<4 x i16> %x) {
+; CHECK-LABEL: xor_sext_v4i16_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #16
+; CHECK-NEXT:    sxth w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <4 x i16> %x to <4 x i32>
+  %z = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @xor_zext_v4i16_i32(<4 x i16> %x) {
+; CHECK-LABEL: xor_zext_v4i16_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #16
+; CHECK-NEXT:    and w0, w8, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <4 x i16> %x to <4 x i32>
+  %z = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @xor_sext_v8i16_i32(<8 x i16> %x) {
+; CHECK-LABEL: xor_sext_v8i16_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #16
+; CHECK-NEXT:    sxth w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <8 x i16> %x to <8 x i32>
+  %z = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> %y)
+  ret i32 %z
+}
+
+define i32 @xor_zext_v8i16_i32(<8 x i16> %x) {
+; CHECK-LABEL: xor_zext_v8i16_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #16
+; CHECK-NEXT:    and w0, w8, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <8 x i16> %x to <8 x i32>
+  %z = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> %y)
+  ret i32 %z
+}
+
+define i64 @xor_sext_v4i16_i64(<4 x i16> %x) {
+; CHECK-LABEL: xor_sext_v4i16_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #16
+; CHECK-NEXT:    sxth x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <4 x i16> %x to <4 x i64>
+  %z = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @xor_zext_v4i16_i64(<4 x i16> %x) {
+; CHECK-LABEL: xor_zext_v4i16_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #16
+; CHECK-NEXT:    and x0, x8, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <4 x i16> %x to <4 x i64>
+  %z = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @xor_sext_v8i16_i64(<8 x i16> %x) {
+; CHECK-LABEL: xor_sext_v8i16_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #16
+; CHECK-NEXT:    sxth x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <8 x i16> %x to <8 x i64>
+  %z = call i64 @llvm.vector.reduce.xor.v8i64(<8 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @xor_zext_v8i16_i64(<8 x i16> %x) {
+; CHECK-LABEL: xor_zext_v8i16_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    eor w8, w8, w8, lsr #16
+; CHECK-NEXT:    and x0, x8, #0xffff
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <8 x i16> %x to <8 x i64>
+  %z = call i64 @llvm.vector.reduce.xor.v8i64(<8 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @xor_sext_v2i32_i64(<2 x i32> %x) {
+; CHECK-LABEL: xor_sext_v2i32_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    sxtw x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <2 x i32> %x to <2 x i64>
+  %z = call i64 @llvm.vector.reduce.xor.v2i64(<2 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @xor_zext_v2i32_i64(<2 x i32> %x) {
+; CHECK-LABEL: xor_zext_v2i32_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    eor w0, w8, w9
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <2 x i32> %x to <2 x i64>
+  %z = call i64 @llvm.vector.reduce.xor.v2i64(<2 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @xor_sext_v4i32_i64(<4 x i32> %x) {
+; CHECK-LABEL: xor_sext_v4i32_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    eor w8, w8, w9
+; CHECK-NEXT:    sxtw x0, w8
+; CHECK-NEXT:    ret
+entry:
+  %y = sext <4 x i32> %x to <4 x i64>
+  %z = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> %y)
+  ret i64 %z
+}
+
+define i64 @xor_zext_v4i32_i64(<4 x i32> %x) {
+; CHECK-LABEL: xor_zext_v4i32_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    eor w0, w8, w9
+; CHECK-NEXT:    ret
+entry:
+  %y = zext <4 x i32> %x to <4 x i64>
+  %z = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> %y)
+  ret i64 %z
+}
+
+declare i16 @llvm.vector.reduce.and.v16i16(<16 x i16>)
+declare i16 @llvm.vector.reduce.and.v8i16(<8 x i16>)
+declare i16 @llvm.vector.reduce.or.v16i16(<16 x i16>)
+declare i16 @llvm.vector.reduce.or.v8i16(<8 x i16>)
+declare i16 @llvm.vector.reduce.xor.v16i16(<16 x i16>)
+declare i16 @llvm.vector.reduce.xor.v8i16(<8 x i16>)
+declare i32 @llvm.vector.reduce.and.v16i32(<16 x i32>)
+declare i32 @llvm.vector.reduce.and.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.and.v8i32(<8 x i32>)
+declare i32 @llvm.vector.reduce.or.v16i32(<16 x i32>)
+declare i32 @llvm.vector.reduce.or.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.or.v8i32(<8 x i32>)
+declare i32 @llvm.vector.reduce.xor.v16i32(<16 x i32>)
+declare i32 @llvm.vector.reduce.xor.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.xor.v8i32(<8 x i32>)
+declare i64 @llvm.vector.reduce.and.v16i64(<16 x i64>)
+declare i64 @llvm.vector.reduce.and.v2i64(<2 x i64>)
+declare i64 @llvm.vector.reduce.and.v4i64(<4 x i64>)
+declare i64 @llvm.vector.reduce.and.v8i64(<8 x i64>)
+declare i64 @llvm.vector.reduce.or.v16i64(<16 x i64>)
+declare i64 @llvm.vector.reduce.or.v2i64(<2 x i64>)
+declare i64 @llvm.vector.reduce.or.v4i64(<4 x i64>)
+declare i64 @llvm.vector.reduce.or.v8i64(<8 x i64>)
+declare i64 @llvm.vector.reduce.xor.v16i64(<16 x i64>)
+declare i64 @llvm.vector.reduce.xor.v2i64(<2 x i64>)
+declare i64 @llvm.vector.reduce.xor.v4i64(<4 x i64>)
+declare i64 @llvm.vector.reduce.xor.v8i64(<8 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/xbfiz.ll b/llvm/test/CodeGen/AArch64/xbfiz.ll
index 3211cc3f2cedbf..b777ddcb7efcc4 100644
--- a/llvm/test/CodeGen/AArch64/xbfiz.ll
+++ b/llvm/test/CodeGen/AArch64/xbfiz.ll
@@ -61,3 +61,11 @@ define i32 @noubfiz32(i32 %v) {
   %add = add i32 %shl, %and
   ret i32 %add
 }
+
+define i64 @lsl32_not_ubfiz64(i64 %v) {
+; CHECK-LABEL: lsl32_not_ubfiz64:
+; CHECK: lsl	w0, w0, #6
+  %shl = shl i64 %v, 6
+  %and = and i64 %shl, 4294967295
+  ret i64 %and
+}
diff --git a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
index bb98a7c1dcb0e8..a583ee01dd6634 100644
--- a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
@@ -1259,7 +1259,7 @@ define void @zext_v16i4_to_v16i32_in_loop(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    mov.b v1[5], w10
 ; CHECK-NEXT:    ubfx w10, w9, #24, #4
 ; CHECK-NEXT:    mov.b v1[6], w10
-; CHECK-NEXT:    ubfx x10, x9, #28, #4
+; CHECK-NEXT:    lsr w10, w9, #28
 ; CHECK-NEXT:    mov.b v1[7], w10
 ; CHECK-NEXT:    ubfx x10, x9, #32, #4
 ; CHECK-NEXT:    mov.b v1[8], w10
@@ -1322,7 +1322,7 @@ define void @zext_v16i4_to_v16i32_in_loop(ptr %src, ptr %dst) {
 ; CHECK-BE-NEXT:    mov v1.b[6], w10
 ; CHECK-BE-NEXT:    ubfx x10, x9, #32, #4
 ; CHECK-BE-NEXT:    mov v1.b[7], w10
-; CHECK-BE-NEXT:    ubfx x10, x9, #28, #4
+; CHECK-BE-NEXT:    lsr w10, w9, #28
 ; CHECK-BE-NEXT:    mov v1.b[8], w10
 ; CHECK-BE-NEXT:    ubfx w10, w9, #24, #4
 ; CHECK-BE-NEXT:    mov v1.b[9], w10
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv_flags.f32.ll b/llvm/test/CodeGen/AMDGPU/fdiv_flags.f32.ll
index 5ae989603b31ab..2140f50611d711 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv_flags.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv_flags.f32.ll
@@ -342,7 +342,7 @@ define float @v_fdiv_recip_sqrt_f32(float %x) {
 ; CODEGEN-IEEE-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CODEGEN-IEEE-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; CODEGEN-IEEE-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; CODEGEN-IEEE-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; CODEGEN-IEEE-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; CODEGEN-IEEE-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CODEGEN-IEEE-GISEL-NEXT:    v_sqrt_f32_e32 v1, v0
 ; CODEGEN-IEEE-GISEL-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -410,7 +410,7 @@ define float @v_fdiv_recip_sqrt_f32(float %x) {
 ; IR-IEEE-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; IR-IEEE-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; IR-IEEE-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; IR-IEEE-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; IR-IEEE-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; IR-IEEE-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; IR-IEEE-GISEL-NEXT:    v_sqrt_f32_e32 v1, v0
 ; IR-IEEE-GISEL-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -479,7 +479,7 @@ define float @v_fdiv_recip_sqrt_f32(float %x) {
 ; CODEGEN-DAZ-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CODEGEN-DAZ-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; CODEGEN-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; CODEGEN-DAZ-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; CODEGEN-DAZ-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; CODEGEN-DAZ-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CODEGEN-DAZ-GISEL-NEXT:    v_rsq_f32_e32 v1, v0
 ; CODEGEN-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -549,7 +549,7 @@ define float @v_fdiv_recip_sqrt_f32(float %x) {
 ; IR-DAZ-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; IR-DAZ-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; IR-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; IR-DAZ-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; IR-DAZ-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; IR-DAZ-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; IR-DAZ-GISEL-NEXT:    v_rsq_f32_e32 v1, v0
 ; IR-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -607,7 +607,7 @@ define float @v_fdiv_recip_sqrt_f32_arcp(float %x) {
 ; CODEGEN-IEEE-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CODEGEN-IEEE-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; CODEGEN-IEEE-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; CODEGEN-IEEE-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; CODEGEN-IEEE-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; CODEGEN-IEEE-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CODEGEN-IEEE-GISEL-NEXT:    v_sqrt_f32_e32 v1, v0
 ; CODEGEN-IEEE-GISEL-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -647,7 +647,7 @@ define float @v_fdiv_recip_sqrt_f32_arcp(float %x) {
 ; IR-IEEE-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; IR-IEEE-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; IR-IEEE-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; IR-IEEE-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; IR-IEEE-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; IR-IEEE-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; IR-IEEE-GISEL-NEXT:    v_sqrt_f32_e32 v1, v0
 ; IR-IEEE-GISEL-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -687,7 +687,7 @@ define float @v_fdiv_recip_sqrt_f32_arcp(float %x) {
 ; CODEGEN-DAZ-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CODEGEN-DAZ-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; CODEGEN-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; CODEGEN-DAZ-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; CODEGEN-DAZ-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; CODEGEN-DAZ-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CODEGEN-DAZ-GISEL-NEXT:    v_rsq_f32_e32 v1, v0
 ; CODEGEN-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -728,7 +728,7 @@ define float @v_fdiv_recip_sqrt_f32_arcp(float %x) {
 ; IR-DAZ-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; IR-DAZ-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; IR-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; IR-DAZ-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; IR-DAZ-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; IR-DAZ-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; IR-DAZ-GISEL-NEXT:    v_rsq_f32_e32 v1, v0
 ; IR-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -831,7 +831,7 @@ define float @v_fdiv_recip_sqrt_f32_arcp_fdiv_only(float %x) {
 ; CODEGEN-IEEE-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CODEGEN-IEEE-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; CODEGEN-IEEE-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; CODEGEN-IEEE-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; CODEGEN-IEEE-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; CODEGEN-IEEE-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CODEGEN-IEEE-GISEL-NEXT:    v_sqrt_f32_e32 v1, v0
 ; CODEGEN-IEEE-GISEL-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -871,7 +871,7 @@ define float @v_fdiv_recip_sqrt_f32_arcp_fdiv_only(float %x) {
 ; IR-IEEE-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; IR-IEEE-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; IR-IEEE-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; IR-IEEE-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; IR-IEEE-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; IR-IEEE-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; IR-IEEE-GISEL-NEXT:    v_sqrt_f32_e32 v1, v0
 ; IR-IEEE-GISEL-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -911,7 +911,7 @@ define float @v_fdiv_recip_sqrt_f32_arcp_fdiv_only(float %x) {
 ; CODEGEN-DAZ-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CODEGEN-DAZ-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; CODEGEN-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; CODEGEN-DAZ-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; CODEGEN-DAZ-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; CODEGEN-DAZ-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CODEGEN-DAZ-GISEL-NEXT:    v_rsq_f32_e32 v1, v0
 ; CODEGEN-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -952,7 +952,7 @@ define float @v_fdiv_recip_sqrt_f32_arcp_fdiv_only(float %x) {
 ; IR-DAZ-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; IR-DAZ-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; IR-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; IR-DAZ-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; IR-DAZ-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; IR-DAZ-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; IR-DAZ-GISEL-NEXT:    v_rsq_f32_e32 v1, v0
 ; IR-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -1016,7 +1016,7 @@ define float @v_fdiv_recip_sqrt_f32_afn_fdiv_only(float %x) {
 ; CODEGEN-IEEE-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CODEGEN-IEEE-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; CODEGEN-IEEE-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; CODEGEN-IEEE-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; CODEGEN-IEEE-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; CODEGEN-IEEE-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CODEGEN-IEEE-GISEL-NEXT:    v_sqrt_f32_e32 v1, v0
 ; CODEGEN-IEEE-GISEL-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -1064,7 +1064,7 @@ define float @v_fdiv_recip_sqrt_f32_afn_fdiv_only(float %x) {
 ; IR-IEEE-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; IR-IEEE-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; IR-IEEE-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; IR-IEEE-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; IR-IEEE-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; IR-IEEE-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; IR-IEEE-GISEL-NEXT:    v_sqrt_f32_e32 v1, v0
 ; IR-IEEE-GISEL-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -1111,7 +1111,7 @@ define float @v_fdiv_recip_sqrt_f32_afn_fdiv_only(float %x) {
 ; CODEGEN-DAZ-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CODEGEN-DAZ-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; CODEGEN-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; CODEGEN-DAZ-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; CODEGEN-DAZ-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; CODEGEN-DAZ-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CODEGEN-DAZ-GISEL-NEXT:    v_rsq_f32_e32 v1, v0
 ; CODEGEN-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -1157,7 +1157,7 @@ define float @v_fdiv_recip_sqrt_f32_afn_fdiv_only(float %x) {
 ; IR-DAZ-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; IR-DAZ-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; IR-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; IR-DAZ-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; IR-DAZ-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; IR-DAZ-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; IR-DAZ-GISEL-NEXT:    v_rsq_f32_e32 v1, v0
 ; IR-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -1191,7 +1191,7 @@ define float @v_fdiv_recip_sqrt_f32_arcp_afn_fdiv_only(float %x) {
 ; CODEGEN-IEEE-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CODEGEN-IEEE-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; CODEGEN-IEEE-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; CODEGEN-IEEE-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; CODEGEN-IEEE-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; CODEGEN-IEEE-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CODEGEN-IEEE-GISEL-NEXT:    v_sqrt_f32_e32 v1, v0
 ; CODEGEN-IEEE-GISEL-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -1221,7 +1221,7 @@ define float @v_fdiv_recip_sqrt_f32_arcp_afn_fdiv_only(float %x) {
 ; IR-IEEE-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; IR-IEEE-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; IR-IEEE-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; IR-IEEE-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; IR-IEEE-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; IR-IEEE-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; IR-IEEE-GISEL-NEXT:    v_sqrt_f32_e32 v1, v0
 ; IR-IEEE-GISEL-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -1251,7 +1251,7 @@ define float @v_fdiv_recip_sqrt_f32_arcp_afn_fdiv_only(float %x) {
 ; CODEGEN-DAZ-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CODEGEN-DAZ-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; CODEGEN-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; CODEGEN-DAZ-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; CODEGEN-DAZ-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; CODEGEN-DAZ-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; CODEGEN-DAZ-GISEL-NEXT:    v_rsq_f32_e32 v1, v0
 ; CODEGEN-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -1280,7 +1280,7 @@ define float @v_fdiv_recip_sqrt_f32_arcp_afn_fdiv_only(float %x) {
 ; IR-DAZ-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; IR-DAZ-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; IR-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; IR-DAZ-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; IR-DAZ-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; IR-DAZ-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; IR-DAZ-GISEL-NEXT:    v_rsq_f32_e32 v1, v0
 ; IR-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -1676,7 +1676,7 @@ define float @v_recip_sqrt_f32_ulp25_contract(float %x) {
 ; IR-IEEE-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; IR-IEEE-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; IR-IEEE-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; IR-IEEE-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; IR-IEEE-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; IR-IEEE-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; IR-IEEE-GISEL-NEXT:    v_sqrt_f32_e32 v1, v0
 ; IR-IEEE-GISEL-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -1751,7 +1751,7 @@ define float @v_recip_sqrt_f32_ulp25_contract(float %x) {
 ; IR-DAZ-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; IR-DAZ-GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; IR-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; IR-DAZ-GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; IR-DAZ-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; IR-DAZ-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; IR-DAZ-GISEL-NEXT:    v_rsq_f32_e32 v1, v0
 ; IR-DAZ-GISEL-NEXT:    v_mul_f32_e32 v2, v0, v1
diff --git a/llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll b/llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll
index 8fd201038ad160..c6c145e090829c 100644
--- a/llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll
@@ -34,7 +34,7 @@ define float @v_sqrt_f32(float %x) {
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -79,7 +79,7 @@ define float @v_sqrt_f32(float %x) {
 ; GISEL-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -129,7 +129,7 @@ define float @v_sqrt_f32_fneg(float %x) {
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v2, 0x4f800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e64 v2, -v0, v2
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e64 vcc, v1, -v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e64 vcc, -v0, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -175,7 +175,7 @@ define float @v_sqrt_f32_fneg(float %x) {
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v2, 0x4f800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e64 v2, -v0, v2
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e64 vcc, v1, -v0
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e64 vcc, -v0, v1
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -227,7 +227,7 @@ define float @v_sqrt_f32_fabs(float %x) {
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v2, 0x4f800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e64 v2, |v0|, v2
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e64 vcc, v1, |v0|
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e64 v0, |v0|, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -274,7 +274,7 @@ define float @v_sqrt_f32_fabs(float %x) {
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v2, 0x4f800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e64 v2, |v0|, v2
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e64 vcc, v1, |v0|
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, v1
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e64 v0, |v0|, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -326,7 +326,7 @@ define float @v_sqrt_f32_fneg_fabs(float %x) {
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v2, 0x4f800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e64 v2, -|v0|, v2
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e64 vcc, v1, -|v0|
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e64 vcc, -|v0|, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e64 v0, -|v0|, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -373,7 +373,7 @@ define float @v_sqrt_f32_fneg_fabs(float %x) {
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v2, 0x4f800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e64 v2, -|v0|, v2
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e64 vcc, v1, -|v0|
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e64 vcc, -|v0|, v1
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e64 v0, -|v0|, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -424,7 +424,7 @@ define float @v_sqrt_f32_ninf(float %x) {
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -469,7 +469,7 @@ define float @v_sqrt_f32_ninf(float %x) {
 ; GISEL-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -518,7 +518,7 @@ define float @v_sqrt_f32_no_infs_attribute(float %x) #5 {
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -563,7 +563,7 @@ define float @v_sqrt_f32_no_infs_attribute(float %x) #5 {
 ; GISEL-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -612,7 +612,7 @@ define float @v_sqrt_f32_nnan(float %x) {
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -657,7 +657,7 @@ define float @v_sqrt_f32_nnan(float %x) {
 ; GISEL-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -705,12 +705,12 @@ define amdgpu_ps i32 @s_sqrt_f32(float inreg %x) {
 ;
 ; GISEL-IEEE-LABEL: s_sqrt_f32:
 ; GISEL-IEEE:       ; %bb.0:
-; GISEL-IEEE-NEXT:    v_mov_b32_e32 v0, 0xf800000
+; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, s0
+; GISEL-IEEE-NEXT:    v_mov_b32_e32 v0, s0
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v2, s0, v2
-; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v1
+; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[0:1], -1, v1
 ; GISEL-IEEE-NEXT:    v_fma_f32 v3, -v2, v1, v0
@@ -754,12 +754,12 @@ define amdgpu_ps i32 @s_sqrt_f32(float inreg %x) {
 ;
 ; GISEL-DAZ-LABEL: s_sqrt_f32:
 ; GISEL-DAZ:       ; %bb.0:
-; GISEL-DAZ-NEXT:    v_mov_b32_e32 v0, 0xf800000
+; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, s0
+; GISEL-DAZ-NEXT:    v_mov_b32_e32 v0, s0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, s0, v2
-; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v1
+; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v1, 0.5, v1
@@ -809,12 +809,12 @@ define amdgpu_ps i32 @s_sqrt_f32_ninf(float inreg %x) {
 ;
 ; GISEL-IEEE-LABEL: s_sqrt_f32_ninf:
 ; GISEL-IEEE:       ; %bb.0:
-; GISEL-IEEE-NEXT:    v_mov_b32_e32 v0, 0xf800000
+; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, s0
+; GISEL-IEEE-NEXT:    v_mov_b32_e32 v0, s0
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v2, s0, v2
-; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v1
+; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[0:1], -1, v1
 ; GISEL-IEEE-NEXT:    v_fma_f32 v3, -v2, v1, v0
@@ -858,12 +858,12 @@ define amdgpu_ps i32 @s_sqrt_f32_ninf(float inreg %x) {
 ;
 ; GISEL-DAZ-LABEL: s_sqrt_f32_ninf:
 ; GISEL-DAZ:       ; %bb.0:
-; GISEL-DAZ-NEXT:    v_mov_b32_e32 v0, 0xf800000
+; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, s0
+; GISEL-DAZ-NEXT:    v_mov_b32_e32 v0, s0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, s0, v2
-; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v1
+; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v1, 0.5, v1
@@ -938,7 +938,7 @@ define float @v_sqrt_f32_nsz(float %x) {
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -983,7 +983,7 @@ define float @v_sqrt_f32_nsz(float %x) {
 ; GISEL-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -1032,7 +1032,7 @@ define float @v_sqrt_f32_nnan_ninf(float %x) {
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -1077,7 +1077,7 @@ define float @v_sqrt_f32_nnan_ninf(float %x) {
 ; GISEL-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -1126,7 +1126,7 @@ define float @v_sqrt_f32_nnan_ninf_nsz(float %x) {
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -1171,7 +1171,7 @@ define float @v_sqrt_f32_nnan_ninf_nsz(float %x) {
 ; GISEL-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -1349,7 +1349,7 @@ define <2 x float> @v_sqrt_v2f32(<2 x float> %x) {
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v2, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v3, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v4, s[4:5], -1, v3
@@ -1363,7 +1363,7 @@ define <2 x float> @v_sqrt_v2f32(<2 x float> %x) {
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v3
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v1
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v1
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v2
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v2, v1
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v4, 0x260
@@ -1425,7 +1425,7 @@ define <2 x float> @v_sqrt_v2f32(<2 x float> %x) {
 ; GISEL-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v2, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v0
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v0
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v3, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v4, v0, v3
@@ -1438,7 +1438,7 @@ define <2 x float> @v_sqrt_v2f32(<2 x float> %x) {
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v4, 0x37800000, v3
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v1
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v1
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v2
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v2, v1
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v4, 0x260
@@ -1521,7 +1521,7 @@ define <3 x float> @v_sqrt_v3f32(<3 x float> %x) {
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v3, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v4, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v5, s[4:5], -1, v4
@@ -1535,7 +1535,7 @@ define <3 x float> @v_sqrt_v3f32(<3 x float> %x) {
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v6, 0x4f800000, v1
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v1
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v6, v1
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v5, 0x260
@@ -1552,7 +1552,7 @@ define <3 x float> @v_sqrt_v3f32(<3 x float> %x) {
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v6, 0x37800000, v4
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v6, 0x4f800000, v2
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v2
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v2, v3
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v3, v2
 ; GISEL-IEEE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v5
@@ -1628,7 +1628,7 @@ define <3 x float> @v_sqrt_v3f32(<3 x float> %x) {
 ; GISEL-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v3, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v0
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v0
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v3
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v4, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v5, v0, v4
@@ -1641,7 +1641,7 @@ define <3 x float> @v_sqrt_v3f32(<3 x float> %x) {
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v1
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v3, v1
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v3
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v5, v1
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v6, 0x260
@@ -1655,7 +1655,7 @@ define <3 x float> @v_sqrt_v3f32(<3 x float> %x) {
 ; GISEL-DAZ-NEXT:    v_fma_f32 v7, -v4, v4, v1
 ; GISEL-DAZ-NEXT:    v_fma_f32 v4, v7, v5, v4
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v7, 0x4f800000, v2
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e64 s[4:5], v3, v2
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e64 s[4:5], v2, v3
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e64 v2, v2, v7, s[4:5]
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v3, v2
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
@@ -1708,7 +1708,7 @@ define float @v_sqrt_f32_ulp05(float %x) {
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -1753,7 +1753,7 @@ define float @v_sqrt_f32_ulp05(float %x) {
 ; GISEL-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -1803,7 +1803,7 @@ define float @v_sqrt_f32_ulp1(float %x) {
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -2023,7 +2023,7 @@ define <2 x float> @v_sqrt_v2f32_ulp1(<2 x float> %x) {
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v2, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v2
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v3, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v4, s[4:5], -1, v3
@@ -2037,7 +2037,7 @@ define <2 x float> @v_sqrt_v2f32_ulp1(<2 x float> %x) {
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v3
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v1
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v2, v1
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v1, v2
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v2, v1
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v4, 0x260
@@ -2166,7 +2166,7 @@ define <2 x float> @v_sqrt_v2f32_ulp1_fabs(<2 x float> %x) {
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v2, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v3, 0x4f800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e64 v4, |v0|, v3
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e64 vcc, v2, |v0|
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, v2
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e64 v0, |v0|, v4, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v4, v0
 ; GISEL-IEEE-NEXT:    v_mul_f32_e64 v3, |v1|, v3
@@ -2180,7 +2180,7 @@ define <2 x float> @v_sqrt_v2f32_ulp1_fabs(<2 x float> %x) {
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[4:5]
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e64 vcc, v2, |v1|
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e64 vcc, |v1|, v2
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e64 v1, |v1|, v3, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v2, v1
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v5, 0x260
@@ -3175,7 +3175,7 @@ define float @v_sqrt_f32_ninf_known_never_zero(float nofpclass(zero) %x) {
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -3220,7 +3220,7 @@ define float @v_sqrt_f32_ninf_known_never_zero(float nofpclass(zero) %x) {
 ; GISEL-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -3269,7 +3269,7 @@ define float @v_sqrt_f32_known_never_zero(float nofpclass(zero) %x) {
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -3314,7 +3314,7 @@ define float @v_sqrt_f32_known_never_zero(float nofpclass(zero) %x) {
 ; GISEL-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -3363,7 +3363,7 @@ define float @v_sqrt_f32_known_never_zero_never_inf(float nofpclass(zero inf) %x
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -3408,7 +3408,7 @@ define float @v_sqrt_f32_known_never_zero_never_inf(float nofpclass(zero inf) %x
 ; GISEL-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -3457,7 +3457,7 @@ define float @v_sqrt_f32_known_never_zero_never_ninf(float nofpclass(zero ninf)
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -3502,7 +3502,7 @@ define float @v_sqrt_f32_known_never_zero_never_ninf(float nofpclass(zero ninf)
 ; GISEL-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -3551,7 +3551,7 @@ define float @v_sqrt_f32_known_never_zero_never_pinf(float nofpclass(zero pinf)
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v1, v0
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
@@ -3596,7 +3596,7 @@ define float @v_sqrt_f32_known_never_zero_never_pinf(float nofpclass(zero pinf)
 ; GISEL-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -3652,7 +3652,7 @@ define float @v_sqrt_f32_frexp_src(float %x) {
 ; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; GISEL-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GISEL-NEXT:    v_rsq_f32_e32 v1, v0
 ; GISEL-NEXT:    v_mul_f32_e32 v2, v0, v1
@@ -3837,7 +3837,7 @@ define float @v_elim_redun_check_ult_sqrt(float %in) {
 ; GISEL-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-IEEE-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-IEEE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-IEEE-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-IEEE-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-IEEE-NEXT:    v_cndmask_b32_e32 v1, v0, v2, vcc
 ; GISEL-IEEE-NEXT:    v_sqrt_f32_e32 v2, v1
 ; GISEL-IEEE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
@@ -3886,7 +3886,7 @@ define float @v_elim_redun_check_ult_sqrt(float %in) {
 ; GISEL-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GISEL-DAZ-NEXT:    v_mov_b32_e32 v1, 0xf800000
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GISEL-DAZ-NEXT:    v_cmp_gt_f32_e32 vcc, v1, v0
+; GISEL-DAZ-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; GISEL-DAZ-NEXT:    v_cndmask_b32_e32 v1, v0, v2, vcc
 ; GISEL-DAZ-NEXT:    v_rsq_f32_e32 v2, v1
 ; GISEL-DAZ-NEXT:    v_mul_f32_e32 v3, v1, v2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log.ll b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
index c2f6fbfe4667c0..7f4cf19e9b85b4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
@@ -5736,20 +5736,16 @@ define float @v_log_f32_0() {
 ; SI-GISEL:       ; %bb.0:
 ; SI-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-GISEL-NEXT:    v_log_f32_e32 v0, 0
-; SI-GISEL-NEXT:    v_mov_b32_e32 v2, 0x3f317217
-; SI-GISEL-NEXT:    v_mov_b32_e32 v3, 0x3377d1cf
-; SI-GISEL-NEXT:    v_mov_b32_e32 v1, 0x800000
-; SI-GISEL-NEXT:    v_mul_f32_e32 v4, 0x3f317217, v0
-; SI-GISEL-NEXT:    v_fma_f32 v2, v0, v2, -v4
-; SI-GISEL-NEXT:    v_fma_f32 v2, v0, v3, v2
+; SI-GISEL-NEXT:    v_mov_b32_e32 v1, 0x3f317217
+; SI-GISEL-NEXT:    v_mov_b32_e32 v2, 0x3377d1cf
 ; SI-GISEL-NEXT:    v_mov_b32_e32 v3, 0x7f800000
-; SI-GISEL-NEXT:    v_add_f32_e32 v2, v4, v2
+; SI-GISEL-NEXT:    v_mul_f32_e32 v4, 0x3f317217, v0
+; SI-GISEL-NEXT:    v_fma_f32 v1, v0, v1, -v4
+; SI-GISEL-NEXT:    v_fma_f32 v1, v0, v2, v1
+; SI-GISEL-NEXT:    v_add_f32_e32 v1, v4, v1
 ; SI-GISEL-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, v3
-; SI-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; SI-GISEL-NEXT:    v_mov_b32_e32 v2, 0x41b17218
-; SI-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v1
-; SI-GISEL-NEXT:    v_cndmask_b32_e32 v1, 0, v2, vcc
-; SI-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-GISEL-NEXT:    v_subrev_f32_e32 v0, 0x41b17218, v0
 ; SI-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-SDAG-LABEL: v_log_f32_0:
@@ -5775,23 +5771,19 @@ define float @v_log_f32_0() {
 ; VI-GISEL:       ; %bb.0:
 ; VI-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-GISEL-NEXT:    v_log_f32_e32 v0, 0
-; VI-GISEL-NEXT:    v_mov_b32_e32 v1, 0x800000
-; VI-GISEL-NEXT:    v_and_b32_e32 v2, 0xfffff000, v0
-; VI-GISEL-NEXT:    v_sub_f32_e32 v3, v0, v2
+; VI-GISEL-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
+; VI-GISEL-NEXT:    v_sub_f32_e32 v2, v0, v1
+; VI-GISEL-NEXT:    v_mul_f32_e32 v3, 0x3805fdf4, v1
 ; VI-GISEL-NEXT:    v_mul_f32_e32 v4, 0x3805fdf4, v2
-; VI-GISEL-NEXT:    v_mul_f32_e32 v5, 0x3805fdf4, v3
-; VI-GISEL-NEXT:    v_mul_f32_e32 v3, 0x3f317000, v3
-; VI-GISEL-NEXT:    v_add_f32_e32 v4, v4, v5
-; VI-GISEL-NEXT:    v_add_f32_e32 v3, v3, v4
 ; VI-GISEL-NEXT:    v_mul_f32_e32 v2, 0x3f317000, v2
+; VI-GISEL-NEXT:    v_add_f32_e32 v3, v3, v4
+; VI-GISEL-NEXT:    v_mul_f32_e32 v1, 0x3f317000, v1
 ; VI-GISEL-NEXT:    v_add_f32_e32 v2, v2, v3
-; VI-GISEL-NEXT:    v_mov_b32_e32 v3, 0x7f800000
-; VI-GISEL-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, v3
-; VI-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; VI-GISEL-NEXT:    v_mov_b32_e32 v2, 0x41b17218
-; VI-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v1
-; VI-GISEL-NEXT:    v_cndmask_b32_e32 v1, 0, v2, vcc
-; VI-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-GISEL-NEXT:    v_add_f32_e32 v1, v1, v2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, 0x7f800000
+; VI-GISEL-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, v2
+; VI-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; VI-GISEL-NEXT:    v_subrev_f32_e32 v0, 0x41b17218, v0
 ; VI-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-SDAG-LABEL: v_log_f32_0:
@@ -5814,20 +5806,16 @@ define float @v_log_f32_0() {
 ; GFX900-GISEL:       ; %bb.0:
 ; GFX900-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-GISEL-NEXT:    v_log_f32_e32 v0, 0
-; GFX900-GISEL-NEXT:    v_mov_b32_e32 v2, 0x3f317217
-; GFX900-GISEL-NEXT:    v_mov_b32_e32 v3, 0x3377d1cf
-; GFX900-GISEL-NEXT:    v_mov_b32_e32 v1, 0x800000
-; GFX900-GISEL-NEXT:    v_mul_f32_e32 v4, 0x3f317217, v0
-; GFX900-GISEL-NEXT:    v_fma_f32 v2, v0, v2, -v4
-; GFX900-GISEL-NEXT:    v_fma_f32 v2, v0, v3, v2
+; GFX900-GISEL-NEXT:    v_mov_b32_e32 v1, 0x3f317217
+; GFX900-GISEL-NEXT:    v_mov_b32_e32 v2, 0x3377d1cf
 ; GFX900-GISEL-NEXT:    v_mov_b32_e32 v3, 0x7f800000
-; GFX900-GISEL-NEXT:    v_add_f32_e32 v2, v4, v2
+; GFX900-GISEL-NEXT:    v_mul_f32_e32 v4, 0x3f317217, v0
+; GFX900-GISEL-NEXT:    v_fma_f32 v1, v0, v1, -v4
+; GFX900-GISEL-NEXT:    v_fma_f32 v1, v0, v2, v1
+; GFX900-GISEL-NEXT:    v_add_f32_e32 v1, v4, v1
 ; GFX900-GISEL-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, v3
-; GFX900-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX900-GISEL-NEXT:    v_mov_b32_e32 v2, 0x41b17218
-; GFX900-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v1
-; GFX900-GISEL-NEXT:    v_cndmask_b32_e32 v1, 0, v2, vcc
-; GFX900-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX900-GISEL-NEXT:    v_subrev_f32_e32 v0, 0x41b17218, v0
 ; GFX900-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-SDAG-LABEL: v_log_f32_0:
@@ -5851,7 +5839,6 @@ define float @v_log_f32_0() {
 ; GFX1100-GISEL:       ; %bb.0:
 ; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-GISEL-NEXT:    v_log_f32_e32 v0, 0
-; GFX1100-GISEL-NEXT:    v_cmp_lt_f32_e64 s0, 0, 0x800000
 ; GFX1100-GISEL-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-GISEL-NEXT:    v_mul_f32_e32 v1, 0x3f317217, v0
 ; GFX1100-GISEL-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
@@ -5861,9 +5848,8 @@ define float @v_log_f32_0() {
 ; GFX1100-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-GISEL-NEXT:    v_add_f32_e32 v1, v1, v2
 ; GFX1100-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX1100-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 0x41b17218, s0
 ; GFX1100-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX1100-GISEL-NEXT:    v_subrev_f32_e32 v0, 0x41b17218, v0
 ; GFX1100-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; R600-LABEL: v_log_f32_0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
index 0a1f7ab6fc0ae3..1c64e6b76c9577 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
@@ -5736,20 +5736,16 @@ define float @v_log10_f32_0() {
 ; SI-GISEL:       ; %bb.0:
 ; SI-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-GISEL-NEXT:    v_log_f32_e32 v0, 0
-; SI-GISEL-NEXT:    v_mov_b32_e32 v2, 0x3e9a209a
-; SI-GISEL-NEXT:    v_mov_b32_e32 v3, 0x3284fbcf
-; SI-GISEL-NEXT:    v_mov_b32_e32 v1, 0x800000
-; SI-GISEL-NEXT:    v_mul_f32_e32 v4, 0x3e9a209a, v0
-; SI-GISEL-NEXT:    v_fma_f32 v2, v0, v2, -v4
-; SI-GISEL-NEXT:    v_fma_f32 v2, v0, v3, v2
+; SI-GISEL-NEXT:    v_mov_b32_e32 v1, 0x3e9a209a
+; SI-GISEL-NEXT:    v_mov_b32_e32 v2, 0x3284fbcf
 ; SI-GISEL-NEXT:    v_mov_b32_e32 v3, 0x7f800000
-; SI-GISEL-NEXT:    v_add_f32_e32 v2, v4, v2
+; SI-GISEL-NEXT:    v_mul_f32_e32 v4, 0x3e9a209a, v0
+; SI-GISEL-NEXT:    v_fma_f32 v1, v0, v1, -v4
+; SI-GISEL-NEXT:    v_fma_f32 v1, v0, v2, v1
+; SI-GISEL-NEXT:    v_add_f32_e32 v1, v4, v1
 ; SI-GISEL-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, v3
-; SI-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; SI-GISEL-NEXT:    v_mov_b32_e32 v2, 0x411a209b
-; SI-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v1
-; SI-GISEL-NEXT:    v_cndmask_b32_e32 v1, 0, v2, vcc
-; SI-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
+; SI-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-GISEL-NEXT:    v_subrev_f32_e32 v0, 0x411a209b, v0
 ; SI-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-SDAG-LABEL: v_log10_f32_0:
@@ -5775,23 +5771,19 @@ define float @v_log10_f32_0() {
 ; VI-GISEL:       ; %bb.0:
 ; VI-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-GISEL-NEXT:    v_log_f32_e32 v0, 0
-; VI-GISEL-NEXT:    v_mov_b32_e32 v1, 0x800000
-; VI-GISEL-NEXT:    v_and_b32_e32 v2, 0xfffff000, v0
-; VI-GISEL-NEXT:    v_sub_f32_e32 v3, v0, v2
+; VI-GISEL-NEXT:    v_and_b32_e32 v1, 0xfffff000, v0
+; VI-GISEL-NEXT:    v_sub_f32_e32 v2, v0, v1
+; VI-GISEL-NEXT:    v_mul_f32_e32 v3, 0x369a84fb, v1
 ; VI-GISEL-NEXT:    v_mul_f32_e32 v4, 0x369a84fb, v2
-; VI-GISEL-NEXT:    v_mul_f32_e32 v5, 0x369a84fb, v3
-; VI-GISEL-NEXT:    v_mul_f32_e32 v3, 0x3e9a2000, v3
-; VI-GISEL-NEXT:    v_add_f32_e32 v4, v4, v5
-; VI-GISEL-NEXT:    v_add_f32_e32 v3, v3, v4
 ; VI-GISEL-NEXT:    v_mul_f32_e32 v2, 0x3e9a2000, v2
+; VI-GISEL-NEXT:    v_add_f32_e32 v3, v3, v4
+; VI-GISEL-NEXT:    v_mul_f32_e32 v1, 0x3e9a2000, v1
 ; VI-GISEL-NEXT:    v_add_f32_e32 v2, v2, v3
-; VI-GISEL-NEXT:    v_mov_b32_e32 v3, 0x7f800000
-; VI-GISEL-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, v3
-; VI-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; VI-GISEL-NEXT:    v_mov_b32_e32 v2, 0x411a209b
-; VI-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v1
-; VI-GISEL-NEXT:    v_cndmask_b32_e32 v1, 0, v2, vcc
-; VI-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
+; VI-GISEL-NEXT:    v_add_f32_e32 v1, v1, v2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, 0x7f800000
+; VI-GISEL-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, v2
+; VI-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; VI-GISEL-NEXT:    v_subrev_f32_e32 v0, 0x411a209b, v0
 ; VI-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-SDAG-LABEL: v_log10_f32_0:
@@ -5814,20 +5806,16 @@ define float @v_log10_f32_0() {
 ; GFX900-GISEL:       ; %bb.0:
 ; GFX900-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-GISEL-NEXT:    v_log_f32_e32 v0, 0
-; GFX900-GISEL-NEXT:    v_mov_b32_e32 v2, 0x3e9a209a
-; GFX900-GISEL-NEXT:    v_mov_b32_e32 v3, 0x3284fbcf
-; GFX900-GISEL-NEXT:    v_mov_b32_e32 v1, 0x800000
-; GFX900-GISEL-NEXT:    v_mul_f32_e32 v4, 0x3e9a209a, v0
-; GFX900-GISEL-NEXT:    v_fma_f32 v2, v0, v2, -v4
-; GFX900-GISEL-NEXT:    v_fma_f32 v2, v0, v3, v2
+; GFX900-GISEL-NEXT:    v_mov_b32_e32 v1, 0x3e9a209a
+; GFX900-GISEL-NEXT:    v_mov_b32_e32 v2, 0x3284fbcf
 ; GFX900-GISEL-NEXT:    v_mov_b32_e32 v3, 0x7f800000
-; GFX900-GISEL-NEXT:    v_add_f32_e32 v2, v4, v2
+; GFX900-GISEL-NEXT:    v_mul_f32_e32 v4, 0x3e9a209a, v0
+; GFX900-GISEL-NEXT:    v_fma_f32 v1, v0, v1, -v4
+; GFX900-GISEL-NEXT:    v_fma_f32 v1, v0, v2, v1
+; GFX900-GISEL-NEXT:    v_add_f32_e32 v1, v4, v1
 ; GFX900-GISEL-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, v3
-; GFX900-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX900-GISEL-NEXT:    v_mov_b32_e32 v2, 0x411a209b
-; GFX900-GISEL-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v1
-; GFX900-GISEL-NEXT:    v_cndmask_b32_e32 v1, 0, v2, vcc
-; GFX900-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX900-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX900-GISEL-NEXT:    v_subrev_f32_e32 v0, 0x411a209b, v0
 ; GFX900-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1100-SDAG-LABEL: v_log10_f32_0:
@@ -5851,7 +5839,6 @@ define float @v_log10_f32_0() {
 ; GFX1100-GISEL:       ; %bb.0:
 ; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX1100-GISEL-NEXT:    v_log_f32_e32 v0, 0
-; GFX1100-GISEL-NEXT:    v_cmp_lt_f32_e64 s0, 0, 0x800000
 ; GFX1100-GISEL-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX1100-GISEL-NEXT:    v_mul_f32_e32 v1, 0x3e9a209a, v0
 ; GFX1100-GISEL-NEXT:    v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
@@ -5861,9 +5848,8 @@ define float @v_log10_f32_0() {
 ; GFX1100-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-GISEL-NEXT:    v_add_f32_e32 v1, v1, v2
 ; GFX1100-GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX1100-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 0x411a209b, s0
 ; GFX1100-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-GISEL-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GFX1100-GISEL-NEXT:    v_subrev_f32_e32 v0, 0x411a209b, v0
 ; GFX1100-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; R600-LABEL: v_log10_f32_0:
diff --git a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
index a642543c3780db..6b097bd71c9f14 100644
--- a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
+++ b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
@@ -202,7 +202,7 @@ define amdgpu_cs float @v_s_sqrt_f32(float inreg %src) {
 ;
 ; GFX12-GISEL-LABEL: v_s_sqrt_f32:
 ; GFX12-GISEL:       ; %bb.0:
-; GFX12-GISEL-NEXT:    s_cmp_gt_f32 0xf800000, s0
+; GFX12-GISEL-NEXT:    s_cmp_lt_f32 s0, 0xf800000
 ; GFX12-GISEL-NEXT:    s_mul_f32 s2, s0, 0x4f800000
 ; GFX12-GISEL-NEXT:    s_cselect_b32 s1, 1, 0
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_2) | instskip(NEXT) | instid1(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/DirectX/frac.ll b/llvm/test/CodeGen/DirectX/frac.ll
index ae86fe06654da1..ef24527ce837b0 100644
--- a/llvm/test/CodeGen/DirectX/frac.ll
+++ b/llvm/test/CodeGen/DirectX/frac.ll
@@ -1,31 +1,55 @@
-; RUN: opt -S -dxil-op-lower < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -scalarizer -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
 
 ; Make sure dxil operation function calls for frac are generated for float and half.
-; CHECK:call float @dx.op.unary.f32(i32 22, float %{{.*}})
-; CHECK:call half @dx.op.unary.f16(i32 22, half %{{.*}})
 
-target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64"
-target triple = "dxil-pc-shadermodel6.7-library"
+define noundef half @frac_half(half noundef %a) {
+; CHECK-LABEL: define noundef half @frac_half(
+; CHECK-SAME: half noundef [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[DX_FRAC1:%.*]] = call half @dx.op.unary.f16(i32 22, half [[A]])
+; CHECK-NEXT:    ret half [[DX_FRAC1]]
+;
+entry:
+  %dx.frac = call half @llvm.dx.frac.f16(half %a)
+  ret half %dx.frac
+}
 
-; Function Attrs: noinline nounwind optnone
 define noundef float @frac_float(float noundef %a) #0 {
+; CHECK-LABEL: define noundef float @frac_float(
+; CHECK-SAME: float noundef [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[DX_FRAC1:%.*]] = call float @dx.op.unary.f32(i32 22, float [[A]])
+; CHECK-NEXT:    ret float [[DX_FRAC1]]
+;
 entry:
-  %a.addr = alloca float, align 4
-  store float %a, ptr %a.addr, align 4
-  %0 = load float, ptr %a.addr, align 4
-  %dx.frac = call float @llvm.dx.frac.f32(float %0)
+  %dx.frac = call float @llvm.dx.frac.f32(float %a)
   ret float %dx.frac
 }
 
-; Function Attrs: nocallback nofree nosync nounwind readnone speculatable willreturn
-declare float @llvm.dx.frac.f32(float) #1
-
-; Function Attrs: noinline nounwind optnone
-define noundef half @frac_half(half noundef %a) #0 {
+define noundef <4 x float> @frac_float4(<4 x float> noundef %a) #0 {
+; CHECK-LABEL: define noundef <4 x float> @frac_float4(
+; CHECK-SAME: <4 x float> noundef [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[A_I0:%.*]] = extractelement <4 x float> [[A]], i64 0
+; CHECK-NEXT:    [[DOTI04:%.*]] = call float @dx.op.unary.f32(i32 22, float [[A_I0]])
+; CHECK-NEXT:    [[A_I1:%.*]] = extractelement <4 x float> [[A]], i64 1
+; CHECK-NEXT:    [[DOTI13:%.*]] = call float @dx.op.unary.f32(i32 22, float [[A_I1]])
+; CHECK-NEXT:    [[A_I2:%.*]] = extractelement <4 x float> [[A]], i64 2
+; CHECK-NEXT:    [[DOTI22:%.*]] = call float @dx.op.unary.f32(i32 22, float [[A_I2]])
+; CHECK-NEXT:    [[A_I3:%.*]] = extractelement <4 x float> [[A]], i64 3
+; CHECK-NEXT:    [[DOTI31:%.*]] = call float @dx.op.unary.f32(i32 22, float [[A_I3]])
+; CHECK-NEXT:    [[DOTUPTO0:%.*]] = insertelement <4 x float> poison, float [[DOTI04]], i64 0
+; CHECK-NEXT:    [[DOTUPTO1:%.*]] = insertelement <4 x float> [[DOTUPTO0]], float [[DOTI13]], i64 1
+; CHECK-NEXT:    [[DOTUPTO2:%.*]] = insertelement <4 x float> [[DOTUPTO1]], float [[DOTI22]], i64 2
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x float> [[DOTUPTO2]], float [[DOTI31]], i64 3
+; CHECK-NEXT:    ret <4 x float> [[TMP0]]
+;
 entry:
-  %a.addr = alloca half, align 2
-  store half %a, ptr %a.addr, align 2
-  %0 = load half, ptr %a.addr, align 2
-  %dx.frac = call half @llvm.dx.frac.f16(half %0)
-  ret half %dx.frac
+  %2 = call <4 x float> @llvm.dx.frac.v4f32(<4 x float> %a)
+  ret <4 x float> %2
 }
+
+declare half  @llvm.dx.frac.f16(half)
+declare float @llvm.dx.frac.f32(float)
+declare <4 x float> @llvm.dx.frac.v4f32(<4 x float>)
diff --git a/llvm/test/CodeGen/DirectX/llc-pipeline.ll b/llvm/test/CodeGen/DirectX/llc-pipeline.ll
index 52bd891aee7d4e..46326d69175876 100644
--- a/llvm/test/CodeGen/DirectX/llc-pipeline.ll
+++ b/llvm/test/CodeGen/DirectX/llc-pipeline.ll
@@ -5,6 +5,7 @@
 
 ; CHECK-LABEL: Pass Arguments:
 ; CHECK-NEXT: Target Library Information
+; CHECK-NEXT: Target Transform Information
 ; CHECK-NEXT: ModulePass Manager
 ; CHECK-NEXT:   DXIL Intrinsic Expansion
 ; CHECK-NEXT:   FunctionPass Manager
diff --git a/llvm/test/CodeGen/DirectX/rsqrt.ll b/llvm/test/CodeGen/DirectX/rsqrt.ll
index 054c84483ef826..26b22e19635af2 100644
--- a/llvm/test/CodeGen/DirectX/rsqrt.ll
+++ b/llvm/test/CodeGen/DirectX/rsqrt.ll
@@ -1,28 +1,56 @@
-; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -scalarizer -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
 
 ; Make sure dxil operation function calls for rsqrt are generated for float and half.
 
 ; CHECK-LABEL: rsqrt_float
-; CHECK: call float @dx.op.unary.f32(i32 25, float %{{.*}})
 define noundef float @rsqrt_float(float noundef %a) {
+; CHECK-SAME: float noundef [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[DX_RSQRT1:%.*]] = call float @dx.op.unary.f32(i32 25, float [[A]])
+; CHECK-NEXT:    ret float [[DX_RSQRT1]]
+;
 entry:
-  %a.addr = alloca float, align 4
-  store float %a, ptr %a.addr, align 4
-  %0 = load float, ptr %a.addr, align 4
-  %dx.rsqrt = call float @llvm.dx.rsqrt.f32(float %0)
+  %dx.rsqrt = call float @llvm.dx.rsqrt.f32(float %a)
   ret float %dx.rsqrt
 }
 
 ; CHECK-LABEL: rsqrt_half
-; CHECK: call half @dx.op.unary.f16(i32 25, half %{{.*}})
 define noundef half @rsqrt_half(half noundef %a) {
+; CHECK-SAME: half noundef [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[DX_RSQRT1:%.*]] = call half @dx.op.unary.f16(i32 25, half [[A]])
+; CHECK-NEXT:    ret half [[DX_RSQRT1]]
+;
 entry:
-  %a.addr = alloca half, align 2
-  store half %a, ptr %a.addr, align 2
-  %0 = load half, ptr %a.addr, align 2
-  %dx.rsqrt = call half @llvm.dx.rsqrt.f16(half %0)
+  %dx.rsqrt = call half @llvm.dx.rsqrt.f16(half %a)
   ret half %dx.rsqrt
 }
 
+define noundef <4 x float> @rsqrt_float4(<4 x float> noundef %a) #0 {
+; CHECK-LABEL: define noundef <4 x float> @rsqrt_float4(
+; CHECK-SAME: <4 x float> noundef [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[A_I0:%.*]] = extractelement <4 x float> [[A]], i64 0
+; CHECK-NEXT:    [[DOTI04:%.*]] = call float @dx.op.unary.f32(i32 25, float [[A_I0]])
+; CHECK-NEXT:    [[A_I1:%.*]] = extractelement <4 x float> [[A]], i64 1
+; CHECK-NEXT:    [[DOTI13:%.*]] = call float @dx.op.unary.f32(i32 25, float [[A_I1]])
+; CHECK-NEXT:    [[A_I2:%.*]] = extractelement <4 x float> [[A]], i64 2
+; CHECK-NEXT:    [[DOTI22:%.*]] = call float @dx.op.unary.f32(i32 25, float [[A_I2]])
+; CHECK-NEXT:    [[A_I3:%.*]] = extractelement <4 x float> [[A]], i64 3
+; CHECK-NEXT:    [[DOTI31:%.*]] = call float @dx.op.unary.f32(i32 25, float [[A_I3]])
+; CHECK-NEXT:    [[DOTUPTO0:%.*]] = insertelement <4 x float> poison, float [[DOTI04]], i64 0
+; CHECK-NEXT:    [[DOTUPTO1:%.*]] = insertelement <4 x float> [[DOTUPTO0]], float [[DOTI13]], i64 1
+; CHECK-NEXT:    [[DOTUPTO2:%.*]] = insertelement <4 x float> [[DOTUPTO1]], float [[DOTI22]], i64 2
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x float> [[DOTUPTO2]], float [[DOTI31]], i64 3
+; CHECK-NEXT:    ret <4 x float> [[TMP0]]
+;
+entry:
+  %2 = call <4 x float> @llvm.dx.rsqrt.v4f32(<4 x float> %a)
+  ret <4 x float> %2
+}
+
+
 declare half @llvm.dx.rsqrt.f16(half)
 declare float @llvm.dx.rsqrt.f32(float)
+declare <4 x float> @llvm.dx.rsqrt.v4f32(<4 x float>)
diff --git a/llvm/test/CodeGen/NVPTX/convert-call-to-indirect.ll b/llvm/test/CodeGen/NVPTX/convert-call-to-indirect.ll
index 2602c3b0d041b5..71a46fa6d48208 100644
--- a/llvm/test/CodeGen/NVPTX/convert-call-to-indirect.ll
+++ b/llvm/test/CodeGen/NVPTX/convert-call-to-indirect.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 | FileCheck %s
-; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_90 | %ptxas-verify %}
+; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_90 | %ptxas-verify -arch=sm_90 %}
 
 %struct.64 = type <{ i64 }>
 declare i64 @callee(ptr %p);
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/alu-roundtrip.ll b/llvm/test/CodeGen/RISCV/GlobalISel/alu-roundtrip.ll
index 8745862112d0f8..c503d6541b0a57 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/alu-roundtrip.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/alu-roundtrip.ll
@@ -348,9 +348,7 @@ define i32 @sdiv_i32(i32 %a, i32 %b) {
 ;
 ; RV64IM-LABEL: sdiv_i32:
 ; RV64IM:       # %bb.0: # %entry
-; RV64IM-NEXT:    sext.w a0, a0
-; RV64IM-NEXT:    sext.w a1, a1
-; RV64IM-NEXT:    div a0, a0, a1
+; RV64IM-NEXT:    divw a0, a0, a1
 ; RV64IM-NEXT:    ret
 entry:
   %0 = sdiv i32 %a, %b
@@ -365,9 +363,7 @@ define i32 @srem_i32(i32 %a, i32 %b) {
 ;
 ; RV64IM-LABEL: srem_i32:
 ; RV64IM:       # %bb.0: # %entry
-; RV64IM-NEXT:    sext.w a0, a0
-; RV64IM-NEXT:    sext.w a1, a1
-; RV64IM-NEXT:    rem a0, a0, a1
+; RV64IM-NEXT:    remw a0, a0, a1
 ; RV64IM-NEXT:    ret
 entry:
   %0 = srem i32 %a, %b
@@ -382,11 +378,7 @@ define i32 @udiv_i32(i32 %a, i32 %b) {
 ;
 ; RV64IM-LABEL: udiv_i32:
 ; RV64IM:       # %bb.0: # %entry
-; RV64IM-NEXT:    slli a0, a0, 32
-; RV64IM-NEXT:    srli a0, a0, 32
-; RV64IM-NEXT:    slli a1, a1, 32
-; RV64IM-NEXT:    srli a1, a1, 32
-; RV64IM-NEXT:    divu a0, a0, a1
+; RV64IM-NEXT:    divuw a0, a0, a1
 ; RV64IM-NEXT:    ret
 entry:
   %0 = udiv i32 %a, %b
@@ -401,11 +393,7 @@ define i32 @urem_i32(i32 %a, i32 %b) {
 ;
 ; RV64IM-LABEL: urem_i32:
 ; RV64IM:       # %bb.0: # %entry
-; RV64IM-NEXT:    slli a0, a0, 32
-; RV64IM-NEXT:    srli a0, a0, 32
-; RV64IM-NEXT:    slli a1, a1, 32
-; RV64IM-NEXT:    srli a1, a1, 32
-; RV64IM-NEXT:    remu a0, a0, a1
+; RV64IM-NEXT:    remuw a0, a0, a1
 ; RV64IM-NEXT:    ret
 entry:
   %0 = urem i32 %a, %b
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/gisel-commandline-option.ll b/llvm/test/CodeGen/RISCV/GlobalISel/gisel-commandline-option.ll
index 82c66c23b26ba9..75607bb81ff513 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/gisel-commandline-option.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/gisel-commandline-option.ll
@@ -23,6 +23,7 @@
 ; ENABLED-NEXT:  Legalizer
 ; ENABLED-O1-NEXT:  MachineDominator Tree Construction
 ; ENABLED-O1-NEXT:  RISCVPostLegalizerCombiner
+; ENABLED-NEXT:  RISCVPostLegalizerLowering
 ; ENABLED-NEXT:  RegBankSelect
 ; ENABLED-NEXT:  Analysis for ComputingKnownBits
 ; ENABLED-O1-NEXT:  Lazy Branch Probability Analysis
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/iabs.ll b/llvm/test/CodeGen/RISCV/GlobalISel/iabs.ll
index 32593a74d307ef..54fc79216ec00f 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/iabs.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/iabs.ll
@@ -108,8 +108,8 @@ define i32 @abs32(i32 %x) {
 ;
 ; RV64ZBB-LABEL: abs32:
 ; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    negw a1, a0
 ; RV64ZBB-NEXT:    sext.w a0, a0
-; RV64ZBB-NEXT:    neg a1, a0
 ; RV64ZBB-NEXT:    max a0, a0, a1
 ; RV64ZBB-NEXT:    ret
   %abs = tail call i32 @llvm.abs.i32(i32 %x, i1 true)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/alu_m-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/alu_m-rv64.mir
index 3c54c0faeeb77a..f748f0811a99c6 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/alu_m-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/alu_m-rv64.mir
@@ -2,6 +2,141 @@
 # RUN: llc -mtriple=riscv64 -mattr=+m -run-pass=instruction-select -simplify-mir -verify-machineinstrs %s -o - \
 # RUN: | FileCheck -check-prefix=RV64I %s
 
+---
+name:            mul_i32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; RV64I-LABEL: name: mul_i32
+    ; RV64I: liveins: $x10, $x11
+    ; RV64I-NEXT: {{  $}}
+    ; RV64I-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; RV64I-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; RV64I-NEXT: [[MULW:%[0-9]+]]:gpr = MULW [[COPY]], [[COPY1]]
+    ; RV64I-NEXT: $x10 = COPY [[MULW]]
+    ; RV64I-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = COPY $x10
+    %1:gprb(s32) = G_TRUNC %0(s64)
+    %2:gprb(s64) = COPY $x11
+    %3:gprb(s32) = G_TRUNC %2(s64)
+    %4:gprb(s32) = G_MUL %1, %3
+    %5:gprb(s64) = G_ANYEXT %4(s32)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            sdiv_i32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; RV64I-LABEL: name: sdiv_i32
+    ; RV64I: liveins: $x10, $x11
+    ; RV64I-NEXT: {{  $}}
+    ; RV64I-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; RV64I-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; RV64I-NEXT: [[DIVW:%[0-9]+]]:gpr = DIVW [[COPY]], [[COPY1]]
+    ; RV64I-NEXT: $x10 = COPY [[DIVW]]
+    ; RV64I-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = COPY $x10
+    %1:gprb(s32) = G_TRUNC %0(s64)
+    %2:gprb(s64) = COPY $x11
+    %3:gprb(s32) = G_TRUNC %2(s64)
+    %4:gprb(s32) = G_SDIV %1, %3
+    %5:gprb(s64) = G_ANYEXT %4(s32)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            srem_i32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; RV64I-LABEL: name: srem_i32
+    ; RV64I: liveins: $x10, $x11
+    ; RV64I-NEXT: {{  $}}
+    ; RV64I-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; RV64I-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; RV64I-NEXT: [[REMW:%[0-9]+]]:gpr = REMW [[COPY]], [[COPY1]]
+    ; RV64I-NEXT: $x10 = COPY [[REMW]]
+    ; RV64I-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = COPY $x10
+    %1:gprb(s32) = G_TRUNC %0(s64)
+    %2:gprb(s64) = COPY $x11
+    %3:gprb(s32) = G_TRUNC %2(s64)
+    %4:gprb(s32) = G_SREM %1, %3
+    %5:gprb(s64) = G_ANYEXT %4(s32)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            udiv_i32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; RV64I-LABEL: name: udiv_i32
+    ; RV64I: liveins: $x10, $x11
+    ; RV64I-NEXT: {{  $}}
+    ; RV64I-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; RV64I-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; RV64I-NEXT: [[DIVUW:%[0-9]+]]:gpr = DIVUW [[COPY]], [[COPY1]]
+    ; RV64I-NEXT: $x10 = COPY [[DIVUW]]
+    ; RV64I-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = COPY $x10
+    %1:gprb(s32) = G_TRUNC %0(s64)
+    %2:gprb(s64) = COPY $x11
+    %3:gprb(s32) = G_TRUNC %2(s64)
+    %4:gprb(s32) = G_UDIV %1, %3
+    %5:gprb(s64) = G_ANYEXT %4(s32)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            urem_i32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; RV64I-LABEL: name: urem_i32
+    ; RV64I: liveins: $x10, $x11
+    ; RV64I-NEXT: {{  $}}
+    ; RV64I-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; RV64I-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; RV64I-NEXT: [[REMUW:%[0-9]+]]:gpr = REMUW [[COPY]], [[COPY1]]
+    ; RV64I-NEXT: $x10 = COPY [[REMUW]]
+    ; RV64I-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = COPY $x10
+    %1:gprb(s32) = G_TRUNC %0(s64)
+    %2:gprb(s64) = COPY $x11
+    %3:gprb(s32) = G_TRUNC %2(s64)
+    %4:gprb(s32) = G_UREM %1, %3
+    %5:gprb(s64) = G_ANYEXT %4(s32)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+
+...
 ---
 name:            mul_i64
 legalized:       true
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-abs-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-abs-rv64.mir
index 46fb98142bbb03..81da754b7ecc52 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-abs-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-abs-rv64.mir
@@ -102,9 +102,11 @@ body:             |
     ; RV64ZBB-LABEL: name: abs_i32
     ; RV64ZBB: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; RV64ZBB-NEXT: [[ASSERT_SEXT:%[0-9]+]]:_(s64) = G_ASSERT_SEXT [[COPY]], 32
-    ; RV64ZBB-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; RV64ZBB-NEXT: [[SUB:%[0-9]+]]:_(s64) = G_SUB [[C]], [[ASSERT_SEXT]]
-    ; RV64ZBB-NEXT: [[SMAX:%[0-9]+]]:_(s64) = G_SMAX [[ASSERT_SEXT]], [[SUB]]
+    ; RV64ZBB-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ASSERT_SEXT]](s64)
+    ; RV64ZBB-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV64ZBB-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; RV64ZBB-NEXT: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[SUB]](s32)
+    ; RV64ZBB-NEXT: [[SMAX:%[0-9]+]]:_(s64) = G_SMAX [[ASSERT_SEXT]], [[SEXT]]
     ; RV64ZBB-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[SMAX]], 32
     ; RV64ZBB-NEXT: $x10 = COPY [[SEXT_INREG]](s64)
     ; RV64ZBB-NEXT: PseudoRET implicit $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-div-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-div-rv64.mir
index 374b995b406601..bbbe38f695d2e4 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-div-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-div-rv64.mir
@@ -28,14 +28,19 @@ body:             |
     ; CHECK-M-LABEL: name: sdiv_i8
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
-    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
-    ; CHECK-M-NEXT: [[SDIV:%[0-9]+]]:_(s64) = G_SDIV [[ASHR]], [[ASHR1]]
-    ; CHECK-M-NEXT: $x10 = COPY [[SDIV]](s64)
+    ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
+    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
+    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
+    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-M-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C2]](s64)
+    ; CHECK-M-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C3]](s64)
+    ; CHECK-M-NEXT: [[SDIV:%[0-9]+]]:_(s32) = G_SDIV [[ASHR]], [[ASHR1]]
+    ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SDIV]](s32)
+    ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
     %1:_(s64) = COPY $x11
@@ -72,14 +77,19 @@ body:             |
     ; CHECK-M-LABEL: name: sdiv_i15
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
-    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
-    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
-    ; CHECK-M-NEXT: [[SDIV:%[0-9]+]]:_(s64) = G_SDIV [[ASHR]], [[ASHR1]]
-    ; CHECK-M-NEXT: $x10 = COPY [[SDIV]](s64)
+    ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
+    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
+    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
+    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-M-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C2]](s64)
+    ; CHECK-M-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C3]](s64)
+    ; CHECK-M-NEXT: [[SDIV:%[0-9]+]]:_(s32) = G_SDIV [[ASHR]], [[ASHR1]]
+    ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SDIV]](s32)
+    ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
     %1:_(s64) = COPY $x11
@@ -116,14 +126,19 @@ body:             |
     ; CHECK-M-LABEL: name: sdiv_i16
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
-    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
-    ; CHECK-M-NEXT: [[SDIV:%[0-9]+]]:_(s64) = G_SDIV [[ASHR]], [[ASHR1]]
-    ; CHECK-M-NEXT: $x10 = COPY [[SDIV]](s64)
+    ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
+    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-M-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C2]](s64)
+    ; CHECK-M-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C3]](s64)
+    ; CHECK-M-NEXT: [[SDIV:%[0-9]+]]:_(s32) = G_SDIV [[ASHR]], [[ASHR1]]
+    ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SDIV]](s32)
+    ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
     %1:_(s64) = COPY $x11
@@ -156,10 +171,11 @@ body:             |
     ; CHECK-M-LABEL: name: sdiv_i32
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; CHECK-M-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY]], 32
-    ; CHECK-M-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY1]], 32
-    ; CHECK-M-NEXT: [[SDIV:%[0-9]+]]:_(s64) = G_SDIV [[SEXT_INREG]], [[SEXT_INREG1]]
-    ; CHECK-M-NEXT: $x10 = COPY [[SDIV]](s64)
+    ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-M-NEXT: [[SDIV:%[0-9]+]]:_(s32) = G_SDIV [[TRUNC]], [[TRUNC1]]
+    ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SDIV]](s32)
+    ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
     %1:_(s64) = COPY $x11
@@ -342,12 +358,15 @@ body:             |
     ; CHECK-M-LABEL: name: udiv_i8
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
-    ; CHECK-M-NEXT: [[UDIV:%[0-9]+]]:_(s64) = G_UDIV [[AND]], [[AND1]]
-    ; CHECK-M-NEXT: $x10 = COPY [[UDIV]](s64)
+    ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
+    ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
+    ; CHECK-M-NEXT: [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[AND]], [[AND1]]
+    ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UDIV]](s32)
+    ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
     %1:_(s64) = COPY $x11
@@ -382,12 +401,15 @@ body:             |
     ; CHECK-M-LABEL: name: udiv_i15
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
-    ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
-    ; CHECK-M-NEXT: [[UDIV:%[0-9]+]]:_(s64) = G_UDIV [[AND]], [[AND1]]
-    ; CHECK-M-NEXT: $x10 = COPY [[UDIV]](s64)
+    ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
+    ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
+    ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
+    ; CHECK-M-NEXT: [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[AND]], [[AND1]]
+    ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UDIV]](s32)
+    ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
     %1:_(s64) = COPY $x11
@@ -422,12 +444,15 @@ body:             |
     ; CHECK-M-LABEL: name: udiv_i16
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
-    ; CHECK-M-NEXT: [[UDIV:%[0-9]+]]:_(s64) = G_UDIV [[AND]], [[AND1]]
-    ; CHECK-M-NEXT: $x10 = COPY [[UDIV]](s64)
+    ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
+    ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
+    ; CHECK-M-NEXT: [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[AND]], [[AND1]]
+    ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UDIV]](s32)
+    ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
     %1:_(s64) = COPY $x11
@@ -462,12 +487,11 @@ body:             |
     ; CHECK-M-LABEL: name: udiv_i32
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
-    ; CHECK-M-NEXT: [[UDIV:%[0-9]+]]:_(s64) = G_UDIV [[AND]], [[AND1]]
-    ; CHECK-M-NEXT: $x10 = COPY [[UDIV]](s64)
+    ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-M-NEXT: [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[TRUNC]], [[TRUNC1]]
+    ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UDIV]](s32)
+    ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
     %1:_(s64) = COPY $x11
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rem-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rem-rv64.mir
index bf5e120866142a..64458c40f44643 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rem-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rem-rv64.mir
@@ -28,14 +28,19 @@ body:             |
     ; CHECK-M-LABEL: name: srem_i8
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
-    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
-    ; CHECK-M-NEXT: [[SREM:%[0-9]+]]:_(s64) = G_SREM [[ASHR]], [[ASHR1]]
-    ; CHECK-M-NEXT: $x10 = COPY [[SREM]](s64)
+    ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
+    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
+    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
+    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-M-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C2]](s64)
+    ; CHECK-M-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C3]](s64)
+    ; CHECK-M-NEXT: [[SREM:%[0-9]+]]:_(s32) = G_SREM [[ASHR]], [[ASHR1]]
+    ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SREM]](s32)
+    ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
     %1:_(s64) = COPY $x11
@@ -72,14 +77,19 @@ body:             |
     ; CHECK-M-LABEL: name: srem_i15
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
-    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
-    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
-    ; CHECK-M-NEXT: [[SREM:%[0-9]+]]:_(s64) = G_SREM [[ASHR]], [[ASHR1]]
-    ; CHECK-M-NEXT: $x10 = COPY [[SREM]](s64)
+    ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
+    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
+    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
+    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-M-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C2]](s64)
+    ; CHECK-M-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C3]](s64)
+    ; CHECK-M-NEXT: [[SREM:%[0-9]+]]:_(s32) = G_SREM [[ASHR]], [[ASHR1]]
+    ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SREM]](s32)
+    ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
     %1:_(s64) = COPY $x11
@@ -116,14 +126,19 @@ body:             |
     ; CHECK-M-LABEL: name: srem_i16
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
-    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
-    ; CHECK-M-NEXT: [[SREM:%[0-9]+]]:_(s64) = G_SREM [[ASHR]], [[ASHR1]]
-    ; CHECK-M-NEXT: $x10 = COPY [[SREM]](s64)
+    ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
+    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-M-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C2]](s64)
+    ; CHECK-M-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C3]](s64)
+    ; CHECK-M-NEXT: [[SREM:%[0-9]+]]:_(s32) = G_SREM [[ASHR]], [[ASHR1]]
+    ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SREM]](s32)
+    ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
     %1:_(s64) = COPY $x11
@@ -156,10 +171,11 @@ body:             |
     ; CHECK-M-LABEL: name: srem_i32
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; CHECK-M-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY]], 32
-    ; CHECK-M-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY1]], 32
-    ; CHECK-M-NEXT: [[SREM:%[0-9]+]]:_(s64) = G_SREM [[SEXT_INREG]], [[SEXT_INREG1]]
-    ; CHECK-M-NEXT: $x10 = COPY [[SREM]](s64)
+    ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-M-NEXT: [[SREM:%[0-9]+]]:_(s32) = G_SREM [[TRUNC]], [[TRUNC1]]
+    ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SREM]](s32)
+    ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
     %1:_(s64) = COPY $x11
@@ -342,12 +358,15 @@ body:             |
     ; CHECK-M-LABEL: name: urem_i8
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
-    ; CHECK-M-NEXT: [[UREM:%[0-9]+]]:_(s64) = G_UREM [[AND]], [[AND1]]
-    ; CHECK-M-NEXT: $x10 = COPY [[UREM]](s64)
+    ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
+    ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
+    ; CHECK-M-NEXT: [[UREM:%[0-9]+]]:_(s32) = G_UREM [[AND]], [[AND1]]
+    ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UREM]](s32)
+    ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
     %1:_(s64) = COPY $x11
@@ -382,12 +401,15 @@ body:             |
     ; CHECK-M-LABEL: name: urem_i15
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
-    ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
-    ; CHECK-M-NEXT: [[UREM:%[0-9]+]]:_(s64) = G_UREM [[AND]], [[AND1]]
-    ; CHECK-M-NEXT: $x10 = COPY [[UREM]](s64)
+    ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
+    ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
+    ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
+    ; CHECK-M-NEXT: [[UREM:%[0-9]+]]:_(s32) = G_UREM [[AND]], [[AND1]]
+    ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UREM]](s32)
+    ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
     %1:_(s64) = COPY $x11
@@ -422,12 +444,15 @@ body:             |
     ; CHECK-M-LABEL: name: urem_i16
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
-    ; CHECK-M-NEXT: [[UREM:%[0-9]+]]:_(s64) = G_UREM [[AND]], [[AND1]]
-    ; CHECK-M-NEXT: $x10 = COPY [[UREM]](s64)
+    ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
+    ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
+    ; CHECK-M-NEXT: [[UREM:%[0-9]+]]:_(s32) = G_UREM [[AND]], [[AND1]]
+    ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UREM]](s32)
+    ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
     %1:_(s64) = COPY $x11
@@ -462,12 +487,11 @@ body:             |
     ; CHECK-M-LABEL: name: urem_i32
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
-    ; CHECK-M-NEXT: [[UREM:%[0-9]+]]:_(s64) = G_UREM [[AND]], [[AND1]]
-    ; CHECK-M-NEXT: $x10 = COPY [[UREM]](s64)
+    ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-M-NEXT: [[UREM:%[0-9]+]]:_(s32) = G_UREM [[TRUNC]], [[TRUNC1]]
+    ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UREM]](s32)
+    ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
     %0:_(s64) = COPY $x10
     %1:_(s64) = COPY $x11
diff --git a/llvm/test/CodeGen/RISCV/compress-opt-select.ll b/llvm/test/CodeGen/RISCV/compress-opt-select.ll
index cc27f326327df1..2667fde89e9354 100644
--- a/llvm/test/CodeGen/RISCV/compress-opt-select.ll
+++ b/llvm/test/CodeGen/RISCV/compress-opt-select.ll
@@ -1,201 +1,686 @@
-; This test is designed to run 4 times, once with function attribute +c,
-; once with function attribute -c for eq/ne in icmp
-; The optimization should appear only with +c, otherwise default isel should be
-; choosen.
-;
-; RUN: cat %s | sed 's/CMPCOND/eq/g' | sed 's/RESBRNORMAL/beq/g' | \
-; RUN: sed 's/RESBROPT/c.beqz/g' > %t.compress_eq
-; RUN: llc -mtriple=riscv32 -target-abi ilp32d -mattr=+c,+f,+d -filetype=obj \
-; RUN:   -disable-block-placement < %t.compress_eq \
-; RUN:   | llvm-objdump -d --triple=riscv32 --mattr=+c,+f,+d -M no-aliases - \
-; RUN:   | FileCheck -check-prefix=RV32IFDC %t.compress_eq
-;
-; RUN: cat %s | sed -e 's/CMPCOND/eq/g' | sed -e 's/RESBRNORMAL/beq/g'\
-; RUN:   | sed -e 's/RESBROPT/c.beqz/g' > %t.nocompr_eq
-; RUN: llc -mtriple=riscv32 -target-abi ilp32d -mattr=-c,+f,+d -filetype=obj \
-; RUN:   -disable-block-placement < %t.nocompr_eq \
-; RUN:   | llvm-objdump -d --triple=riscv32 --mattr=-c,+f,+d -M no-aliases - \
-; RUN:   | FileCheck -check-prefix=RV32IFD %t.nocompr_eq
-;
-; RUN: cat %s | sed 's/CMPCOND/ne/g' | sed 's/RESBRNORMAL/bne/g' | \
-; RUN: sed 's/RESBROPT/c.bnez/g' > %t.compress_neq
-; RUN: llc -mtriple=riscv32 -target-abi ilp32d -mattr=+c,+f,+d -filetype=obj \
-; RUN:   -disable-block-placement < %t.compress_neq \
-; RUN:   | llvm-objdump -d --triple=riscv32 --mattr=+c,+f,+d -M no-aliases - \
-; RUN:   | FileCheck -check-prefix=RV32IFDC %t.compress_neq
-;
-; RUN: cat %s | sed -e 's/CMPCOND/ne/g' | sed -e 's/RESBRNORMAL/bne/g'\
-; RUN:   | sed -e 's/RESBROPT/c.bnez/g' > %t.nocompr_neq
-; RUN: llc -mtriple=riscv32 -target-abi ilp32d -mattr=-c,+f,+d -filetype=obj \
-; RUN:   -disable-block-placement < %t.nocompr_neq \
-; RUN:   | llvm-objdump -d --triple=riscv32 --mattr=-c,+f,+d -M no-aliases - \
-; RUN:   | FileCheck -check-prefix=RV32IFD %t.nocompr_neq
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv32 -target-abi ilp32d -mattr=+c,+f,+d \
+; RUN:     -riscv-no-aliases < %s \
+; RUN:   | FileCheck -check-prefix=RV32IFDC %s
+; RUN: llc -mtriple=riscv32 -target-abi ilp32d -mattr=-c,+f,+d \
+; RUN:     -riscv-no-aliases < %s \
+; RUN:   | FileCheck -check-prefix=RV32IFD %s
 
+; constant is small and fit in 6 bit (compress imm)
+define i32 @ne_small_pos(i32 %in0) minsize {
+; RV32IFDC-LABEL: ne_small_pos:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    c.mv a1, a0
+; RV32IFDC-NEXT:    c.li a2, 20
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    bne a1, a2, .LBB0_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB0_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: ne_small_pos:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, 20
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    bne a1, a2, .LBB0_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB0_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp ne i32 %in0, 20
+  %toRet = select i1 %cmp, i32 -99, i32 42
+  ret i32 %toRet
+}
 
 ; constant is small and fit in 6 bit (compress imm)
-; RV32IFDC-LABEL: <f_small_pos>:
-; RV32IFDC: c.li [[REG:.*]], 0x14
-; RV32IFDC: RESBRNORMAL [[ANOTHER:.*]], [[REG]], [[PLACE:.*]]
-; --- no compress extension
-; RV32IFD-LABEL: <f_small_pos>:
-; RV32IFD: addi [[REG:.*]], zero, 0x14
-; RV32IFD: RESBRNORMAL [[ANOTHER:.*]], [[REG]], [[PLACE:.*]]
-define i32 @f_small_pos(i32 %in0) minsize {
-  %cmp = icmp CMPCOND i32 %in0, 20
+define i32 @ne_small_neg(i32 %in0) minsize {
+; RV32IFDC-LABEL: ne_small_neg:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    c.mv a1, a0
+; RV32IFDC-NEXT:    c.li a2, -20
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    bne a1, a2, .LBB1_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB1_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: ne_small_neg:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, -20
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    bne a1, a2, .LBB1_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB1_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp ne i32 %in0, -20
   %toRet = select i1 %cmp, i32 -99, i32 42
   ret i32 %toRet
 }
 
 ; constant is small and fit in 6 bit (compress imm)
-; RV32IFDC-LABEL: <f_small_neg>:
-; RV32IFDC: c.li [[REG:.*]], -0x14
-; RV32IFDC: RESBRNORMAL [[ANOTHER:.*]], [[REG]], [[PLACE:.*]]
-; --- no compress extension
-; RV32IFD-LABEL: <f_small_neg>:
-; RV32IFD: addi [[REG:.*]], zero, -0x14
-; RV32IFD: RESBRNORMAL [[ANOTHER:.*]], [[REG]], [[PLACE:.*]]
-define i32 @f_small_neg(i32 %in0) minsize {
-  %cmp = icmp CMPCOND i32 %in0, -20
+define i32 @ne_small_edge_pos(i32 %in0) minsize {
+; RV32IFDC-LABEL: ne_small_edge_pos:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    c.mv a1, a0
+; RV32IFDC-NEXT:    c.li a2, 31
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    bne a1, a2, .LBB2_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB2_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: ne_small_edge_pos:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, 31
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    bne a1, a2, .LBB2_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB2_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp ne i32 %in0, 31
   %toRet = select i1 %cmp, i32 -99, i32 42
   ret i32 %toRet
 }
 
 ; constant is small and fit in 6 bit (compress imm)
-; RV32IFDC-LABEL: <f_small_edge_pos>:
-; RV32IFDC: c.li [[REG:.*]], 0x1f
-; RV32IFDC: RESBRNORMAL [[ANOTHER:.*]], [[REG]], [[PLACE:.*]]
-; --- no compress extension
-; RV32IFD-LABEL: <f_small_edge_pos>:
-; RV32IFD: addi [[REG:.*]], zero, 0x1f
-; RV32IFD: RESBRNORMAL [[ANOTHER:.*]], [[REG]], [[PLACE:.*]]
-define i32 @f_small_edge_pos(i32 %in0) minsize {
-  %cmp = icmp CMPCOND i32 %in0, 31
+define i32 @ne_small_edge_neg(i32 %in0) minsize {
+; RV32IFDC-LABEL: ne_small_edge_neg:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    c.mv a1, a0
+; RV32IFDC-NEXT:    c.li a2, -32
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    bne a1, a2, .LBB3_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB3_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: ne_small_edge_neg:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, -32
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    bne a1, a2, .LBB3_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB3_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp ne i32 %in0, -32
+  %toRet = select i1 %cmp, i32 -99, i32 42
+  ret i32 %toRet
+}
+
+; constant is medium and not fit in 6 bit (compress imm),
+; but fit in 12 bit (imm)
+define i32 @ne_medium_ledge_pos(i32 %in0) minsize {
+; RV32IFDC-LABEL: ne_medium_ledge_pos:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    addi a1, a0, -33
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    c.bnez a1, .LBB4_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB4_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: ne_medium_ledge_pos:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, 33
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    bne a1, a2, .LBB4_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB4_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp ne i32 %in0, 33
+  %toRet = select i1 %cmp, i32 -99, i32 42
+  ret i32 %toRet
+}
+
+; constant is medium and not fit in 6 bit (compress imm),
+; but fit in 12 bit (imm)
+define i32 @ne_medium_ledge_neg(i32 %in0) minsize {
+; RV32IFDC-LABEL: ne_medium_ledge_neg:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    addi a1, a0, 33
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    c.bnez a1, .LBB5_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB5_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: ne_medium_ledge_neg:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, -33
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    bne a1, a2, .LBB5_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB5_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp ne i32 %in0, -33
+  %toRet = select i1 %cmp, i32 -99, i32 42
+  ret i32 %toRet
+}
+
+; constant is medium and not fit in 6 bit (compress imm),
+; but fit in 12 bit (imm)
+define i32 @ne_medium_pos(i32 %in0) minsize {
+; RV32IFDC-LABEL: ne_medium_pos:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    addi a1, a0, -63
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    c.bnez a1, .LBB6_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB6_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: ne_medium_pos:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, 63
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    bne a1, a2, .LBB6_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB6_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp ne i32 %in0, 63
   %toRet = select i1 %cmp, i32 -99, i32 42
   ret i32 %toRet
 }
 
+; constant is medium and not fit in 6 bit (compress imm),
+; but fit in 12 bit (imm)
+define i32 @ne_medium_neg(i32 %in0) minsize {
+; RV32IFDC-LABEL: ne_medium_neg:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    addi a1, a0, 63
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    c.bnez a1, .LBB7_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB7_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: ne_medium_neg:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, -63
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    bne a1, a2, .LBB7_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB7_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp ne i32 %in0, -63
+  %toRet = select i1 %cmp, i32 -99, i32 42
+  ret i32 %toRet
+}
+
+; constant is medium and not fit in 6 bit (compress imm),
+; but fit in 12 bit (imm)
+define i32 @ne_medium_bedge_pos(i32 %in0) minsize {
+; RV32IFDC-LABEL: ne_medium_bedge_pos:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    addi a1, a0, -2047
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    c.bnez a1, .LBB8_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB8_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: ne_medium_bedge_pos:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, 2047
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    bne a1, a2, .LBB8_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB8_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp ne i32 %in0, 2047
+  %toRet = select i1 %cmp, i32 -99, i32 42
+  ret i32 %toRet
+}
+
+; constant is medium and not fit in 6 bit (compress imm),
+; but fit in 12 bit (imm), negative value fit in 12 bit too.
+define i32 @ne_medium_bedge_neg(i32 %in0) minsize {
+; RV32IFDC-LABEL: ne_medium_bedge_neg:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    addi a1, a0, 2047
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    c.bnez a1, .LBB9_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB9_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: ne_medium_bedge_neg:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, -2047
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    bne a1, a2, .LBB9_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB9_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp ne i32 %in0, -2047
+  %toRet = select i1 %cmp, i32 -99, i32 42
+  ret i32 %toRet
+}
+
+; constant is big and do not fit in 12 bit (imm), fit in i32
+define i32 @ne_big_ledge_pos(i32 %in0) minsize {
+; RV32IFDC-LABEL: ne_big_ledge_pos:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    c.mv a1, a0
+; RV32IFDC-NEXT:    c.li a0, 1
+; RV32IFDC-NEXT:    slli a2, a0, 11
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    bne a1, a2, .LBB10_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB10_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: ne_big_ledge_pos:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a0, zero, 1
+; RV32IFD-NEXT:    slli a2, a0, 11
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    bne a1, a2, .LBB10_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB10_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp ne i32 %in0, 2048
+  %toRet = select i1 %cmp, i32 -99, i32 42
+  ret i32 %toRet
+}
+
+; constant is big and do not fit in 12 bit (imm), fit in i32
+define i32 @ne_big_ledge_neg(i32 %in0) minsize {
+; RV32IFDC-LABEL: ne_big_ledge_neg:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    c.mv a1, a0
+; RV32IFDC-NEXT:    addi a2, zero, -2048
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    bne a1, a2, .LBB11_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB11_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: ne_big_ledge_neg:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, -2048
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    bne a1, a2, .LBB11_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB11_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp ne i32 %in0, -2048
+  %toRet = select i1 %cmp, i32 -99, i32 42
+  ret i32 %toRet
+}
+
+
+;; Same as above, but for eq
+
 ; constant is small and fit in 6 bit (compress imm)
-; RV32IFDC-LABEL: <f_small_edge_neg>:
-; RV32IFDC: c.li [[REG:.*]], -0x20
-; RV32IFDC: RESBRNORMAL [[ANOTHER:.*]], [[REG]], [[PLACE:.*]]
-; --- no compress extension
-; RV32IFD-LABEL: <f_small_edge_neg>:
-; RV32IFD: addi [[REG:.*]], zero, -0x20
-; RV32IFD: RESBRNORMAL [[ANOTHER:.*]], [[REG]], [[PLACE:.*]]
-define i32 @f_small_edge_neg(i32 %in0) minsize {
-  %cmp = icmp CMPCOND i32 %in0, -32
+define i32 @eq_small_pos(i32 %in0) minsize {
+; RV32IFDC-LABEL: eq_small_pos:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    c.mv a1, a0
+; RV32IFDC-NEXT:    c.li a2, 20
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    beq a1, a2, .LBB12_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB12_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: eq_small_pos:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, 20
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    beq a1, a2, .LBB12_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB12_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp eq i32 %in0, 20
+  %toRet = select i1 %cmp, i32 -99, i32 42
+  ret i32 %toRet
+}
+
+; constant is small and fit in 6 bit (compress imm)
+define i32 @eq_small_neg(i32 %in0) minsize {
+; RV32IFDC-LABEL: eq_small_neg:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    c.mv a1, a0
+; RV32IFDC-NEXT:    c.li a2, -20
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    beq a1, a2, .LBB13_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB13_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: eq_small_neg:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, -20
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    beq a1, a2, .LBB13_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB13_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp eq i32 %in0, -20
+  %toRet = select i1 %cmp, i32 -99, i32 42
+  ret i32 %toRet
+}
+
+; constant is small and fit in 6 bit (compress imm)
+define i32 @eq_small_edge_pos(i32 %in0) minsize {
+; RV32IFDC-LABEL: eq_small_edge_pos:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    c.mv a1, a0
+; RV32IFDC-NEXT:    c.li a2, 31
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    beq a1, a2, .LBB14_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB14_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: eq_small_edge_pos:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, 31
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    beq a1, a2, .LBB14_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB14_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp eq i32 %in0, 31
+  %toRet = select i1 %cmp, i32 -99, i32 42
+  ret i32 %toRet
+}
+
+; constant is small and fit in 6 bit (compress imm)
+define i32 @eq_small_edge_neg(i32 %in0) minsize {
+; RV32IFDC-LABEL: eq_small_edge_neg:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    c.mv a1, a0
+; RV32IFDC-NEXT:    c.li a2, -32
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    beq a1, a2, .LBB15_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB15_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: eq_small_edge_neg:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, -32
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    beq a1, a2, .LBB15_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB15_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp eq i32 %in0, -32
   %toRet = select i1 %cmp, i32 -99, i32 42
   ret i32 %toRet
 }
 
 ; constant is medium and not fit in 6 bit (compress imm),
 ; but fit in 12 bit (imm)
-; RV32IFDC-LABEL: <f_medium_ledge_pos>:
-; RV32IFDC: addi [[MAYZEROREG:.*]], [[REG:.*]], -0x20
-; RV32IFDC: RESBROPT [[MAYZEROREG]], [[PLACE:.*]]
-; --- no compress extension
-; RV32IFD-LABEL: <f_medium_ledge_pos>:
-; RV32IFD: addi [[REG:.*]], zero, 0x20
-; RV32IFD: RESBRNORMAL [[ANOTHER:.*]], [[REG]], [[PLACE:.*]]
-define i32 @f_medium_ledge_pos(i32 %in0) minsize {
-  %cmp = icmp CMPCOND i32 %in0, 32
+define i32 @eq_medium_ledge_pos(i32 %in0) minsize {
+; RV32IFDC-LABEL: eq_medium_ledge_pos:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    addi a1, a0, -33
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    c.beqz a1, .LBB16_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB16_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: eq_medium_ledge_pos:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, 33
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    beq a1, a2, .LBB16_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB16_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp eq i32 %in0, 33
   %toRet = select i1 %cmp, i32 -99, i32 42
   ret i32 %toRet
 }
 
 ; constant is medium and not fit in 6 bit (compress imm),
 ; but fit in 12 bit (imm)
-; RV32IFDC-LABEL: <f_medium_ledge_neg>:
-; RV32IFDC: addi [[MAYZEROREG:.*]], [[REG:.*]], 0x21
-; RV32IFDC: RESBROPT [[MAYZEROREG]], [[PLACE:.*]]
-; --- no compress extension
-; RV32IFD-LABEL: <f_medium_ledge_neg>:
-; RV32IFD: addi [[REG:.*]], zero, -0x21
-; RV32IFD: RESBRNORMAL [[ANOTHER:.*]], [[REG]], [[PLACE:.*]]
-define i32 @f_medium_ledge_neg(i32 %in0) minsize {
-  %cmp = icmp CMPCOND i32 %in0, -33
+define i32 @eq_medium_ledge_neg(i32 %in0) minsize {
+; RV32IFDC-LABEL: eq_medium_ledge_neg:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    addi a1, a0, 33
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    c.beqz a1, .LBB17_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB17_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: eq_medium_ledge_neg:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, -33
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    beq a1, a2, .LBB17_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB17_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp eq i32 %in0, -33
   %toRet = select i1 %cmp, i32 -99, i32 42
   ret i32 %toRet
 }
 
 ; constant is medium and not fit in 6 bit (compress imm),
 ; but fit in 12 bit (imm)
-; RV32IFDC-LABEL: <f_medium_pos>:
-; RV32IFDC: addi [[MAYZEROREG:.*]], [[REG:.*]], -0x3f
-; RV32IFDC: RESBROPT [[MAYZEROREG]], [[PLACE:.*]]
-; --- no compress extension
-; RV32IFD-LABEL: <f_medium_pos>:
-; RV32IFD: addi [[REG:.*]], zero, 0x3f
-; RV32IFD: RESBRNORMAL [[ANOTHER:.*]], [[REG]], [[PLACE:.*]]
-define i32 @f_medium_pos(i32 %in0) minsize {
-  %cmp = icmp CMPCOND i32 %in0, 63
+define i32 @eq_medium_pos(i32 %in0) minsize {
+; RV32IFDC-LABEL: eq_medium_pos:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    addi a1, a0, -63
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    c.beqz a1, .LBB18_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB18_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: eq_medium_pos:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, 63
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    beq a1, a2, .LBB18_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB18_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp eq i32 %in0, 63
   %toRet = select i1 %cmp, i32 -99, i32 42
   ret i32 %toRet
 }
 
 ; constant is medium and not fit in 6 bit (compress imm),
 ; but fit in 12 bit (imm)
-; RV32IFDC-LABEL: <f_medium_neg>:
-; RV32IFDC: addi [[MAYZEROREG:.*]], [[REG:.*]], 0x3f
-; RV32IFDC: RESBROPT [[MAYZEROREG]], [[PLACE:.*]]
-; --- no compress extension
-; RV32IFD-LABEL: <f_medium_neg>:
-; RV32IFD: addi [[REG:.*]], zero, -0x3f
-; RV32IFD: RESBRNORMAL [[ANOTHER:.*]], [[REG]], [[PLACE:.*]]
-define i32 @f_medium_neg(i32 %in0) minsize {
-  %cmp = icmp CMPCOND i32 %in0, -63
+define i32 @eq_medium_neg(i32 %in0) minsize {
+; RV32IFDC-LABEL: eq_medium_neg:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    addi a1, a0, 63
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    c.beqz a1, .LBB19_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB19_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: eq_medium_neg:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, -63
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    beq a1, a2, .LBB19_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB19_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp eq i32 %in0, -63
   %toRet = select i1 %cmp, i32 -99, i32 42
   ret i32 %toRet
 }
 
 ; constant is medium and not fit in 6 bit (compress imm),
 ; but fit in 12 bit (imm)
-; RV32IFDC-LABEL: <f_medium_bedge_pos>:
-; RV32IFDC: addi [[MAYZEROREG:.*]], [[REG:.*]], -0x7ff
-; RV32IFDC: RESBROPT [[MAYZEROREG]], [[PLACE:.*]]
-; --- no compress extension
-; RV32IFD-LABEL: <f_medium_bedge_pos>:
-; RV32IFD: addi [[REG:.*]], zero, 0x7ff
-; RV32IFD: RESBRNORMAL [[ANOTHER:.*]], [[REG]], [[PLACE:.*]]
-define i32 @f_medium_bedge_pos(i32 %in0) minsize {
-  %cmp = icmp CMPCOND i32 %in0, 2047
+define i32 @eq_medium_bedge_pos(i32 %in0) minsize {
+; RV32IFDC-LABEL: eq_medium_bedge_pos:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    addi a1, a0, -2047
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    c.beqz a1, .LBB20_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB20_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: eq_medium_bedge_pos:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, 2047
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    beq a1, a2, .LBB20_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB20_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp eq i32 %in0, 2047
   %toRet = select i1 %cmp, i32 -99, i32 42
   ret i32 %toRet
 }
 
 ; constant is medium and not fit in 6 bit (compress imm),
 ; but fit in 12 bit (imm), negative value fit in 12 bit too.
-; RV32IFDC-LABEL: <f_medium_bedge_neg>:
-; RV32IFDC: addi [[MAYZEROREG:.*]], [[REG:.*]], 0x7ff
-; RV32IFDC: RESBROPT [[MAYZEROREG]], [[PLACE:.*]]
-; --- no compress extension
-; RV32IFD-LABEL: <f_medium_bedge_neg>:
-; RV32IFD: addi [[REG:.*]], zero, -0x7ff
-; RV32IFD: RESBRNORMAL [[ANOTHER:.*]], [[REG]], [[PLACE:.*]]
-define i32 @f_medium_bedge_neg(i32 %in0) minsize {
-  %cmp = icmp CMPCOND i32 %in0, -2047
+define i32 @eq_medium_bedge_neg(i32 %in0) minsize {
+; RV32IFDC-LABEL: eq_medium_bedge_neg:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    addi a1, a0, 2047
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    c.beqz a1, .LBB21_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB21_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: eq_medium_bedge_neg:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, -2047
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    beq a1, a2, .LBB21_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB21_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp eq i32 %in0, -2047
   %toRet = select i1 %cmp, i32 -99, i32 42
   ret i32 %toRet
 }
 
 ; constant is big and do not fit in 12 bit (imm), fit in i32
-; RV32IFDC-LABEL: <f_big_ledge_pos>:
-; RV32IFDC-NOT: RESBROPT
-; --- no compress extension
-; nothing to check.
-define i32 @f_big_ledge_pos(i32 %in0) minsize {
-  %cmp = icmp CMPCOND i32 %in0, 2048
+define i32 @eq_big_ledge_pos(i32 %in0) minsize {
+; RV32IFDC-LABEL: eq_big_ledge_pos:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    c.mv a1, a0
+; RV32IFDC-NEXT:    c.li a0, 1
+; RV32IFDC-NEXT:    slli a2, a0, 11
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    beq a1, a2, .LBB22_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB22_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: eq_big_ledge_pos:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a0, zero, 1
+; RV32IFD-NEXT:    slli a2, a0, 11
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    beq a1, a2, .LBB22_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB22_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp eq i32 %in0, 2048
   %toRet = select i1 %cmp, i32 -99, i32 42
   ret i32 %toRet
 }
 
 ; constant is big and do not fit in 12 bit (imm), fit in i32
-; RV32IFDC-LABEL: <f_big_ledge_neg>:
-; RV32IFDC-NOT: c.beqz
-; --- no compress extension
-; nothing to check.
-define i32 @f_big_ledge_neg(i32 %in0) minsize {
-  %cmp = icmp CMPCOND i32 %in0, -2048
+define i32 @eq_big_ledge_neg(i32 %in0) minsize {
+; RV32IFDC-LABEL: eq_big_ledge_neg:
+; RV32IFDC:       # %bb.0:
+; RV32IFDC-NEXT:    c.mv a1, a0
+; RV32IFDC-NEXT:    addi a2, zero, -2048
+; RV32IFDC-NEXT:    addi a0, zero, -99
+; RV32IFDC-NEXT:    beq a1, a2, .LBB23_2
+; RV32IFDC-NEXT:  # %bb.1:
+; RV32IFDC-NEXT:    addi a0, zero, 42
+; RV32IFDC-NEXT:  .LBB23_2:
+; RV32IFDC-NEXT:    c.jr ra
+;
+; RV32IFD-LABEL: eq_big_ledge_neg:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi a1, a0, 0
+; RV32IFD-NEXT:    addi a2, zero, -2048
+; RV32IFD-NEXT:    addi a0, zero, -99
+; RV32IFD-NEXT:    beq a1, a2, .LBB23_2
+; RV32IFD-NEXT:  # %bb.1:
+; RV32IFD-NEXT:    addi a0, zero, 42
+; RV32IFD-NEXT:  .LBB23_2:
+; RV32IFD-NEXT:    jalr zero, 0(ra)
+  %cmp = icmp eq i32 %in0, -2048
   %toRet = select i1 %cmp, i32 -99, i32 42
   ret i32 %toRet
 }
diff --git a/llvm/test/CodeGen/RISCV/half-convert-strict.ll b/llvm/test/CodeGen/RISCV/half-convert-strict.ll
index 6bd3ef775609ec..17164e8da562a5 100644
--- a/llvm/test/CodeGen/RISCV/half-convert-strict.ll
+++ b/llvm/test/CodeGen/RISCV/half-convert-strict.ll
@@ -47,6 +47,9 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+zdinx,+zhinxmin -verify-machineinstrs \
 ; RUN:   -target-abi lp64 -disable-strictnode-mutation < %s \
 ; RUN:   | FileCheck -check-prefixes=CHECK64-IZDINXZHINXMIN %s
+; RUN: llc -mtriple=riscv32 -mattr=+d -verify-machineinstrs \
+; RUN:   -target-abi ilp32d -disable-strictnode-mutation < %s \
+; RUN:   | FileCheck -check-prefixes=CHECK32-D %s
 
 ; NOTE: The rounding mode metadata does not effect which instruction is
 ; selected. Dynamic rounding mode is always used for operations that
@@ -128,6 +131,20 @@ define i16 @fcvt_si_h(half %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.l.s a0, a0, rtz
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_si_h:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    slli a0, a0, 16
+; CHECK32-D-NEXT:    srli a0, a0, 16
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    call __extendhfsf2
+; CHECK32-D-NEXT:    fcvt.w.s a0, fa0, rtz
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call i16 @llvm.experimental.constrained.fptosi.i16.f16(half %a, metadata !"fpexcept.strict")
   ret i16 %1
 }
@@ -209,6 +226,20 @@ define i16 @fcvt_ui_h(half %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.l.s a0, a0, rtz
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_ui_h:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    slli a0, a0, 16
+; CHECK32-D-NEXT:    srli a0, a0, 16
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    call __extendhfsf2
+; CHECK32-D-NEXT:    fcvt.w.s a0, fa0, rtz
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call i16 @llvm.experimental.constrained.fptoui.i16.f16(half %a, metadata !"fpexcept.strict")
   ret i16 %1
 }
@@ -280,6 +311,20 @@ define i32 @fcvt_w_h(half %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.w.s a0, a0, rtz
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_w_h:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    slli a0, a0, 16
+; CHECK32-D-NEXT:    srli a0, a0, 16
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    call __extendhfsf2
+; CHECK32-D-NEXT:    fcvt.w.s a0, fa0, rtz
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call i32 @llvm.experimental.constrained.fptosi.i32.f16(half %a, metadata !"fpexcept.strict")
   ret i32 %1
 }
@@ -351,6 +396,20 @@ define i32 @fcvt_wu_h(half %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.wu.s a0, a0, rtz
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_wu_h:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    slli a0, a0, 16
+; CHECK32-D-NEXT:    srli a0, a0, 16
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    call __extendhfsf2
+; CHECK32-D-NEXT:    fcvt.wu.s a0, fa0, rtz
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call i32 @llvm.experimental.constrained.fptoui.i32.f16(half %a, metadata !"fpexcept.strict")
   ret i32 %1
 }
@@ -449,6 +508,24 @@ define i32 @fcvt_wu_h_multiple_use(half %x, ptr %y) strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    seqz a1, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    add a0, a0, a1
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_wu_h_multiple_use:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    .cfi_def_cfa_offset 16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    .cfi_offset ra, -4
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    slli a0, a0, 16
+; CHECK32-D-NEXT:    srli a0, a0, 16
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    call __extendhfsf2
+; CHECK32-D-NEXT:    fcvt.wu.s a0, fa0, rtz
+; CHECK32-D-NEXT:    seqz a1, a0
+; CHECK32-D-NEXT:    add a0, a0, a1
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %a = call i32 @llvm.experimental.constrained.fptoui.i32.f16(half %x, metadata !"fpexcept.strict")
   %b = icmp eq i32 %a, 0
   %c = select i1 %b, i32 1, i32 %a
@@ -556,6 +633,20 @@ define i64 @fcvt_l_h(half %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.l.s a0, a0, rtz
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_l_h:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    slli a0, a0, 16
+; CHECK32-D-NEXT:    srli a0, a0, 16
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    call __extendhfsf2
+; CHECK32-D-NEXT:    call __fixsfdi
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call i64 @llvm.experimental.constrained.fptosi.i64.f16(half %a, metadata !"fpexcept.strict")
   ret i64 %1
 }
@@ -662,6 +753,20 @@ define i64 @fcvt_lu_h(half %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.lu.s a0, a0, rtz
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_lu_h:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    slli a0, a0, 16
+; CHECK32-D-NEXT:    srli a0, a0, 16
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    call __extendhfsf2
+; CHECK32-D-NEXT:    call __fixunssfdi
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call i64 @llvm.experimental.constrained.fptoui.i64.f16(half %a, metadata !"fpexcept.strict")
   ret i64 %1
 }
@@ -771,6 +876,22 @@ define half @fcvt_h_si(i16 %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.w a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_h_si:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    slli a0, a0, 16
+; CHECK32-D-NEXT:    srai a0, a0, 16
+; CHECK32-D-NEXT:    fcvt.s.w fa0, a0
+; CHECK32-D-NEXT:    call __truncsfhf2
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    lui a1, 1048560
+; CHECK32-D-NEXT:    or a0, a0, a1
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call half @llvm.experimental.constrained.sitofp.f16.i16(i16 %a, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret half %1
 }
@@ -842,6 +963,20 @@ define half @fcvt_h_si_signext(i16 signext %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.w a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_h_si_signext:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    fcvt.s.w fa0, a0
+; CHECK32-D-NEXT:    call __truncsfhf2
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    lui a1, 1048560
+; CHECK32-D-NEXT:    or a0, a0, a1
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call half @llvm.experimental.constrained.sitofp.f16.i16(i16 %a, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret half %1
 }
@@ -950,6 +1085,22 @@ define half @fcvt_h_ui(i16 %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.wu a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_h_ui:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    slli a0, a0, 16
+; CHECK32-D-NEXT:    srli a0, a0, 16
+; CHECK32-D-NEXT:    fcvt.s.wu fa0, a0
+; CHECK32-D-NEXT:    call __truncsfhf2
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    lui a1, 1048560
+; CHECK32-D-NEXT:    or a0, a0, a1
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call half @llvm.experimental.constrained.uitofp.f16.i16(i16 %a, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret half %1
 }
@@ -1021,6 +1172,20 @@ define half @fcvt_h_ui_zeroext(i16 zeroext %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.wu a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_h_ui_zeroext:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    fcvt.s.wu fa0, a0
+; CHECK32-D-NEXT:    call __truncsfhf2
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    lui a1, 1048560
+; CHECK32-D-NEXT:    or a0, a0, a1
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call half @llvm.experimental.constrained.uitofp.f16.i16(i16 %a, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret half %1
 }
@@ -1091,6 +1256,20 @@ define half @fcvt_h_w(i32 %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.w a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_h_w:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    fcvt.s.w fa0, a0
+; CHECK32-D-NEXT:    call __truncsfhf2
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    lui a1, 1048560
+; CHECK32-D-NEXT:    or a0, a0, a1
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %a, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret half %1
 }
@@ -1174,6 +1353,21 @@ define half @fcvt_h_w_load(ptr %p) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.w a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_h_w_load:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    lw a0, 0(a0)
+; CHECK32-D-NEXT:    fcvt.s.w fa0, a0
+; CHECK32-D-NEXT:    call __truncsfhf2
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    lui a1, 1048560
+; CHECK32-D-NEXT:    or a0, a0, a1
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %a = load i32, ptr %p
   %1 = call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %a, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret half %1
@@ -1245,6 +1439,20 @@ define half @fcvt_h_wu(i32 %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.wu a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_h_wu:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    fcvt.s.wu fa0, a0
+; CHECK32-D-NEXT:    call __truncsfhf2
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    lui a1, 1048560
+; CHECK32-D-NEXT:    or a0, a0, a1
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call half @llvm.experimental.constrained.uitofp.f16.i32(i32 %a, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret half %1
 }
@@ -1340,6 +1548,21 @@ define half @fcvt_h_wu_load(ptr %p) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.wu a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_h_wu_load:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    lw a0, 0(a0)
+; CHECK32-D-NEXT:    fcvt.s.wu fa0, a0
+; CHECK32-D-NEXT:    call __truncsfhf2
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    lui a1, 1048560
+; CHECK32-D-NEXT:    or a0, a0, a1
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %a = load i32, ptr %p
   %1 = call half @llvm.experimental.constrained.uitofp.f16.i32(i32 %a, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret half %1
@@ -1446,6 +1669,20 @@ define half @fcvt_h_l(i64 %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.l a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_h_l:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    call __floatdisf
+; CHECK32-D-NEXT:    call __truncsfhf2
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    lui a1, 1048560
+; CHECK32-D-NEXT:    or a0, a0, a1
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call half @llvm.experimental.constrained.sitofp.f16.i64(i64 %a, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret half %1
 }
@@ -1552,6 +1789,20 @@ define half @fcvt_h_lu(i64 %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.lu a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_h_lu:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    call __floatundisf
+; CHECK32-D-NEXT:    call __truncsfhf2
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    lui a1, 1048560
+; CHECK32-D-NEXT:    or a0, a0, a1
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call half @llvm.experimental.constrained.uitofp.f16.i64(i64 %a, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret half %1
 }
@@ -1617,6 +1868,19 @@ define half @fcvt_h_s(float %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN:       # %bb.0:
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_h_s:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    call __truncsfhf2
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    lui a1, 1048560
+; CHECK32-D-NEXT:    or a0, a0, a1
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call half @llvm.experimental.constrained.fptrunc.f16.f32(float %a, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret half %1
 }
@@ -1682,6 +1946,19 @@ define float @fcvt_s_h(half %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN:       # %bb.0:
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_s_h:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    slli a0, a0, 16
+; CHECK32-D-NEXT:    srli a0, a0, 16
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    call __extendhfsf2
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call float @llvm.experimental.constrained.fpext.f32.f16(half %a, metadata !"fpexcept.strict")
   ret float %1
 }
@@ -1799,6 +2076,19 @@ define half @fcvt_h_d(double %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN:       # %bb.0:
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.d a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_h_d:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    call __truncdfhf2
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    lui a1, 1048560
+; CHECK32-D-NEXT:    or a0, a0, a1
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call half @llvm.experimental.constrained.fptrunc.f16.f64(double %a, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret half %1
 }
@@ -1924,6 +2214,20 @@ define double @fcvt_d_h(half %a) nounwind strictfp {
 ; CHECK64-IZDINXZHINXMIN:       # %bb.0:
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.d.h a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_d_h:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    slli a0, a0, 16
+; CHECK32-D-NEXT:    srli a0, a0, 16
+; CHECK32-D-NEXT:    fmv.w.x fa0, a0
+; CHECK32-D-NEXT:    call __extendhfsf2
+; CHECK32-D-NEXT:    fcvt.d.s fa0, fa0
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %1 = call double @llvm.experimental.constrained.fpext.f64.f16(half %a, metadata !"fpexcept.strict")
   ret double %1
 }
@@ -2034,6 +2338,29 @@ define signext i32 @fcvt_h_w_demanded_bits(i32 signext %0, ptr %1) strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a2, a2
 ; CHECK64-IZDINXZHINXMIN-NEXT:    sh a2, 0(a1)
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_h_w_demanded_bits:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    .cfi_def_cfa_offset 16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    .cfi_offset ra, -4
+; CHECK32-D-NEXT:    .cfi_offset s0, -8
+; CHECK32-D-NEXT:    .cfi_offset s1, -12
+; CHECK32-D-NEXT:    mv s0, a1
+; CHECK32-D-NEXT:    addi s1, a0, 1
+; CHECK32-D-NEXT:    fcvt.s.w fa0, s1
+; CHECK32-D-NEXT:    call __truncsfhf2
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    sh a0, 0(s0)
+; CHECK32-D-NEXT:    mv a0, s1
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %3 = add i32 %0, 1
   %4 = call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %3, metadata !"round.dynamic", metadata !"fpexcept.strict")
   store half %4, ptr %1, align 2
@@ -2145,6 +2472,29 @@ define signext i32 @fcvt_h_wu_demanded_bits(i32 signext %0, ptr %1) strictfp {
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a2, a2
 ; CHECK64-IZDINXZHINXMIN-NEXT:    sh a2, 0(a1)
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+;
+; CHECK32-D-LABEL: fcvt_h_wu_demanded_bits:
+; CHECK32-D:       # %bb.0:
+; CHECK32-D-NEXT:    addi sp, sp, -16
+; CHECK32-D-NEXT:    .cfi_def_cfa_offset 16
+; CHECK32-D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; CHECK32-D-NEXT:    .cfi_offset ra, -4
+; CHECK32-D-NEXT:    .cfi_offset s0, -8
+; CHECK32-D-NEXT:    .cfi_offset s1, -12
+; CHECK32-D-NEXT:    mv s0, a1
+; CHECK32-D-NEXT:    addi s1, a0, 1
+; CHECK32-D-NEXT:    fcvt.s.wu fa0, s1
+; CHECK32-D-NEXT:    call __truncsfhf2
+; CHECK32-D-NEXT:    fmv.x.w a0, fa0
+; CHECK32-D-NEXT:    sh a0, 0(s0)
+; CHECK32-D-NEXT:    mv a0, s1
+; CHECK32-D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; CHECK32-D-NEXT:    addi sp, sp, 16
+; CHECK32-D-NEXT:    ret
   %3 = add i32 %0, 1
   %4 = call half @llvm.experimental.constrained.uitofp.f16.i32(i32 %3, metadata !"round.dynamic", metadata !"fpexcept.strict")
   store half %4, ptr %1, align 2
diff --git a/llvm/test/CodeGen/RISCV/machine-sink-load-immediate.ll b/llvm/test/CodeGen/RISCV/machine-sink-load-immediate.ll
index 6714991d091cfc..775ea8e820afe3 100644
--- a/llvm/test/CodeGen/RISCV/machine-sink-load-immediate.ll
+++ b/llvm/test/CodeGen/RISCV/machine-sink-load-immediate.ll
@@ -178,3 +178,30 @@ return:                                           ; preds = %strdup.exit, %while
 }
 
 declare i32 @toupper()
+
+; In this example, %arg and the return value (13) have overlapping live
+; intervals because the ABI mandidates they both be placed in a0.
+define signext i32 @overlap_live_ranges(ptr %arg, i32 signext %arg1) {
+; CHECK-LABEL: overlap_live_ranges:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    li a3, 1
+; CHECK-NEXT:    li a2, 13
+; CHECK-NEXT:    bne a1, a3, .LBB1_2
+; CHECK-NEXT:  # %bb.1: # %bb2
+; CHECK-NEXT:    lw a2, 4(a0)
+; CHECK-NEXT:  .LBB1_2: # %bb5
+; CHECK-NEXT:    mv a0, a2
+; CHECK-NEXT:    ret
+bb:
+  %i = icmp eq i32 %arg1, 1
+  br i1 %i, label %bb2, label %bb5
+
+bb2:                                              ; preds = %bb
+  %i3 = getelementptr inbounds nuw i8, ptr %arg, i64 4
+  %i4 = load i32, ptr %i3, align 4
+  br label %bb5
+
+bb5:                                              ; preds = %bb2, %bb
+  %i6 = phi i32 [ %i4, %bb2 ], [ 13, %bb ]
+  ret i32 %i6
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwadd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfwadd-vp.ll
index 1ef0ed858d80a9..f7297927db7174 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwadd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwadd-vp.ll
@@ -1,6 +1,44 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh | FileCheck %s --check-prefixes=ZVFH
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfhmin | FileCheck %s --check-prefixes=ZVFHMIN
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfhmin,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+
+define <vscale x 2 x float> @vfwadd_same_operand_nxv2bf16(<vscale x 2 x bfloat> %arg, i32 signext %vl) {
+; CHECK-LABEL: vfwadd_same_operand_nxv2bf16:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    slli a0, a0, 32
+; CHECK-NEXT:    srli a0, a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vfadd.vv v8, v9, v9
+; CHECK-NEXT:    ret
+bb:
+  %tmp = call <vscale x 2 x float> @llvm.vp.fpext.nxv2f32.nxv2bf16(<vscale x 2 x bfloat> %arg, <vscale x 2 x i1> splat (i1 true), i32 %vl)
+  %tmp2 = call <vscale x 2 x float> @llvm.vp.fadd.nxv2f32(<vscale x 2 x float> %tmp, <vscale x 2 x float> %tmp, <vscale x 2 x i1> splat (i1 true), i32 %vl)
+  ret <vscale x 2 x float> %tmp2
+}
+
+; Make sure we don't widen vfmadd.vv -> vfwmaccvbf16.vv if there's other
+; unwidenable uses
+define <vscale x 2 x float> @vfwadd_same_operand_nxv2bf16_multiuse(<vscale x 2 x bfloat> %arg, <vscale x 2 x float> %acc, i32 signext %vl, ptr %p) {
+; CHECK-LABEL: vfwadd_same_operand_nxv2bf16_multiuse:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    slli a0, a0, 32
+; CHECK-NEXT:    srli a0, a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vfadd.vv v8, v10, v10
+; CHECK-NEXT:    vfmadd.vv v10, v10, v9
+; CHECK-NEXT:    vs1r.v v10, (a1)
+; CHECK-NEXT:    ret
+bb:
+  %tmp = call <vscale x 2 x float> @llvm.vp.fpext.nxv2f32.nxv2bf16(<vscale x 2 x bfloat> %arg, <vscale x 2 x i1> splat (i1 true), i32 %vl)
+  %tmp2 = call <vscale x 2 x float> @llvm.vp.fadd.nxv2f32(<vscale x 2 x float> %tmp, <vscale x 2 x float> %tmp, <vscale x 2 x i1> splat (i1 true), i32 %vl)
+  %tmp3 = call <vscale x 2 x float> @llvm.vp.fma.nxv2f32(<vscale x 2 x float> %tmp, <vscale x 2 x float> %tmp, <vscale x 2 x float> %acc, <vscale x 2 x i1> splat (i1 true), i32 %vl)
+  store <vscale x 2 x float> %tmp3, ptr %p
+  ret <vscale x 2 x float> %tmp2
+}
 
 define <vscale x 2 x float> @vfwadd_same_operand(<vscale x 2 x half> %arg, i32 signext %vl) {
 ; ZVFH-LABEL: vfwadd_same_operand:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt-no-prop.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt-no-prop.ll
new file mode 100644
index 00000000000000..e1f641afd2cfe0
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt-no-prop.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs | FileCheck %s
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs | FileCheck %s
+
+declare <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, iXLen)
+declare <vscale x 4 x i32> @llvm.riscv.vrgather.vv.nxv4i32.iXLen(
+  <vscale x 4 x i32>,
+  <vscale x 4 x i32>,
+  <vscale x 4 x i32>,
+  iXLen)
+
+declare <vscale x 4 x i32> @llvm.riscv.vslidedown.nxv4i32(
+  <vscale x 4 x i32>,
+  <vscale x 4 x i32>,
+  iXLen,
+  iXLen,
+  iXLen);
+
+declare <vscale x 4 x i32> @llvm.riscv.vslide1down.nxv4i32.i32(
+  <vscale x 4 x i32>,
+  <vscale x 4 x i32>,
+  i32,
+  iXLen);
+
+define <vscale x 4 x i32> @vrgather(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1, iXLen %vl2) {
+; CHECK-LABEL: vrgather:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v12, v10, v12
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vrgather.vv v8, v12, v10
+; CHECK-NEXT:    ret
+  %v = add <vscale x 4 x i32> %a, %b
+  %w = call <vscale x 4 x i32> @llvm.riscv.vrgather.vv.nxv4i32.iXLen(
+    <vscale x 4 x i32> poison,
+    <vscale x 4 x i32> %v,
+    <vscale x 4 x i32> %a,
+    iXLen %vl1)
+
+  ret <vscale x 4 x i32> %w
+}
+
+define <vscale x 4 x i32> @vslidedown(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, iXLen %2, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) nounwind {
+; CHECK-LABEL: vslidedown:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v10, v12, v14
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v10, a0
+; CHECK-NEXT:    ret
+entry:
+  %v = add <vscale x 4 x i32> %a, %b
+  %w = call <vscale x 4 x i32> @llvm.riscv.vslidedown.nxv4i32(
+    <vscale x 4 x i32> %0,
+    <vscale x 4 x i32> %v,
+    iXLen %2,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 4 x i32> %w
+}
+
+define <vscale x 4 x i32> @vslide1down(<vscale x 4 x i32> %0, i32 %1, iXLen %2, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) nounwind {
+; CHECK-LABEL: vslide1down:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v10, v12
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    ret
+entry:
+  %v = add <vscale x 4 x i32> %a, %b
+  %w = call <vscale x 4 x i32> @llvm.riscv.vslide1down.nxv4i32.i32(
+    <vscale x 4 x i32> poison,
+    <vscale x 4 x i32> %v,
+    i32 %1,
+    iXLen %2)
+
+  ret <vscale x 4 x i32> %w
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.ll
new file mode 100644
index 00000000000000..2b3c5417b15b55
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt-op-info.ll
@@ -0,0 +1,83 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvl512b -verify-machineinstrs | FileCheck %s
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvl512b -verify-machineinstrs | FileCheck %s
+
+define <2 x i32> @vdot_lane_s32(<2 x i32> noundef %var_1, <8 x i8> noundef %var_3, <8 x i8> noundef %var_5, <8 x i16> %x) {
+; CHECK-LABEL: vdot_lane_s32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf4, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v11, 0
+; CHECK-NEXT:    vnsrl.wi v9, v11, 16
+; CHECK-NEXT:    vwadd.vv v10, v8, v9
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v10, 0
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vnsrl.wx v9, v10, a0
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    ret
+entry:
+  %a = shufflevector <8 x i16> %x, <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %b = shufflevector <8 x i16> %x, <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %c = sext <4 x i16> %a to <4 x i32>
+  %d = sext <4 x i16> %b to <4 x i32>
+  %e = add nsw <4 x i32> %c, %d
+  %z10 = shufflevector <4 x i32> %e, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+  %z11 = shufflevector <4 x i32> %e, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+  %y12 = add <2 x i32> %z10, %z11
+  ret <2 x i32> %y12
+}
+
+declare <vscale x 2 x i16> @llvm.riscv.vnsrl.nxv2i16.nxv2i32.nxv2i16(
+  <vscale x 2 x i16>,
+  <vscale x 2 x i32>,
+  <vscale x 2 x i16>,
+  iXLen);
+
+define <vscale x 2 x i16> @intrinsic_vnsrl_wv_nxv2i16_nxv2i32_nxv2i16(<vscale x 2 x i16> %a, <vscale x 2 x i16> %b, iXLen %2, <vscale x 2 x i32> %3, <vscale x 2 x i32> %4, <vscale x 2 x i16> %z) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_wv_nxv2i16_nxv2i32_nxv2i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vwadd.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wv v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <vscale x 2 x i16> %a to <vscale x 2 x i32>
+  %d = sext <vscale x 2 x i16> %b to <vscale x 2 x i32>
+  %v1 = add <vscale x 2 x i32> %c, %d
+  %x = call <vscale x 2 x i16> @llvm.riscv.vnsrl.nxv2i16.nxv2i32.nxv2i16(
+    <vscale x 2 x i16> undef,
+    <vscale x 2 x i32> %v1,
+    <vscale x 2 x i16> %z,
+    iXLen %2)
+
+  ret <vscale x 2 x i16> %x
+}
+
+declare <vscale x 2 x i16> @llvm.riscv.vnclip.nxv2i16.nxv2i32.nxv2i16(
+  <vscale x 2 x i16>,
+  <vscale x 2 x i32>,
+  <vscale x 2 x i16>,
+  iXLen, iXLen);
+
+define <vscale x 2 x i16> @vnclip(<vscale x 2 x i16> %a, <vscale x 2 x i16> %b, iXLen %2, <vscale x 2 x i32> %3, <vscale x 2 x i32> %4, <vscale x 2 x i16> %z) nounwind {
+; CHECK-LABEL: vnclip:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vwadd.vv v10, v8, v9
+; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vnclip.wv v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %c = sext <vscale x 2 x i16> %a to <vscale x 2 x i32>
+  %d = sext <vscale x 2 x i16> %b to <vscale x 2 x i32>
+  %v1 = add <vscale x 2 x i32> %c, %d
+  %x = call <vscale x 2 x i16> @llvm.riscv.vnclip.nxv2i16.nxv2i32.nxv2i16(
+    <vscale x 2 x i16> undef,
+    <vscale x 2 x i32> %v1,
+    <vscale x 2 x i16> %z,
+    iXLen 0, iXLen %2)
+
+  ret <vscale x 2 x i16> %x
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll
new file mode 100644
index 00000000000000..b03ba076059503
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll
@@ -0,0 +1,106 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs | FileCheck %s
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs | FileCheck %s
+
+declare <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, iXLen)
+
+define <vscale x 4 x i32> @different_imm_vl_with_ta(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1, iXLen %vl2) {
+; CHECK-LABEL: different_imm_vl_with_ta:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 5, e32, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v10, v12
+; CHECK-NEXT:    vsetivli zero, 4, e32, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v10
+; CHECK-NEXT:    ret
+  %v = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 5)
+  %w = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %v, <vscale x 4 x i32> %a, iXLen 4)
+  ret <vscale x 4 x i32> %w
+}
+
+; No benificial to propagate VL since VL is larger in the use side.
+define <vscale x 4 x i32> @different_imm_vl_with_ta_larger_vl(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1, iXLen %vl2) {
+; CHECK-LABEL: different_imm_vl_with_ta_larger_vl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v10, v12
+; CHECK-NEXT:    vsetivli zero, 5, e32, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v10
+; CHECK-NEXT:    ret
+  %v = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 4)
+  %w = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %v, <vscale x 4 x i32> %a, iXLen 5)
+  ret <vscale x 4 x i32> %w
+}
+
+define <vscale x 4 x i32> @different_imm_reg_vl_with_ta(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1, iXLen %vl2) {
+; CHECK-LABEL: different_imm_reg_vl_with_ta:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v10, v12
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v10
+; CHECK-NEXT:    ret
+  %v = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 4)
+  %w = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %v, <vscale x 4 x i32> %a, iXLen %vl1)
+  ret <vscale x 4 x i32> %w
+}
+
+
+; No benificial to propagate VL since VL is already one.
+define <vscale x 4 x i32> @different_imm_vl_with_ta_1(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1, iXLen %vl2) {
+; CHECK-LABEL: different_imm_vl_with_ta_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v10, v12
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v10
+; CHECK-NEXT:    ret
+  %v = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 1)
+  %w = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %v, <vscale x 4 x i32> %a, iXLen %vl1)
+  ret <vscale x 4 x i32> %w
+}
+
+; Propgate %vl2 to last instruction since it is may smaller than %vl1,
+; it's still safe even %vl2 is larger than %vl1, becuase rest of the vector are
+; undefined value.
+define <vscale x 4 x i32> @different_vl_with_ta(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1, iXLen %vl2) {
+; CHECK-LABEL: different_vl_with_ta:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v10, v8, v10
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1)
+  %w = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %v, <vscale x 4 x i32> %a,iXLen %vl2)
+  ret <vscale x 4 x i32> %w
+}
+
+; Test case to make sure VL won't propgate if using tail-undisturbed policy.
+define <vscale x 4 x i32> @different_vl_with_tu(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1, iXLen %vl2) {
+; CHECK-LABEL: different_vl_with_tu:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv2r.v v14, v10
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
+; CHECK-NEXT:    vadd.vv v14, v10, v12
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
+; CHECK-NEXT:    vadd.vv v8, v14, v10
+; CHECK-NEXT:    ret
+  %v = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1)
+  %w = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %v, <vscale x 4 x i32> %a,iXLen %vl2)
+  ret <vscale x 4 x i32> %w
+}
+
+; Test case to make sure VL won't propgate if using tail-undisturbed policy.
+define <vscale x 4 x i32> @different_imm_vl_with_tu(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1, iXLen %vl2) {
+; CHECK-LABEL: different_imm_vl_with_tu:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv2r.v v14, v10
+; CHECK-NEXT:    vsetivli zero, 5, e32, m2, tu, ma
+; CHECK-NEXT:    vadd.vv v14, v10, v12
+; CHECK-NEXT:    vsetivli zero, 4, e32, m2, tu, ma
+; CHECK-NEXT:    vadd.vv v8, v14, v10
+; CHECK-NEXT:    ret
+  %v = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen 5)
+  %w = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %v, <vscale x 4 x i32> %a,iXLen 4)
+  ret <vscale x 4 x i32> %w
+}
diff --git a/llvm/test/CodeGen/SPIRV/debug-info/basic-global-di.ll b/llvm/test/CodeGen/SPIRV/debug-info/debug-compilation-unit.ll
similarity index 94%
rename from llvm/test/CodeGen/SPIRV/debug-info/basic-global-di.ll
rename to llvm/test/CodeGen/SPIRV/debug-info/debug-compilation-unit.ll
index 139a8e11182bcc..bff4660559ab82 100644
--- a/llvm/test/CodeGen/SPIRV/debug-info/basic-global-di.ll
+++ b/llvm/test/CodeGen/SPIRV/debug-info/debug-compilation-unit.ll
@@ -1,7 +1,7 @@
 ; RUN: llc --verify-machineinstrs --spv-emit-nonsemantic-debug-info --spirv-ext=+SPV_KHR_non_semantic_info --print-after=spirv-nonsemantic-debug-info -O0 -mtriple=spirv64-unknown-unknown %s -o - 2>&1 | FileCheck %s --check-prefix=CHECK-MIR
 ; RUN: llc --verify-machineinstrs --spv-emit-nonsemantic-debug-info --spirv-ext=+SPV_KHR_non_semantic_info -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
 ; RUN: llc --verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_non_semantic_info %s -o - | FileCheck %s --check-prefix=CHECK-OPTION 
-; RUN: %if spirv-tools %{ llc --spirv-ext=+SPV_KHR_non_semantic_info -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc --verify-machineinstrs --spv-emit-nonsemantic-debug-info --spirv-ext=+SPV_KHR_non_semantic_info -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-MIR-DAG: [[type_void:%[0-9]+\:type]] = OpTypeVoid
 ; CHECK-MIR-DAG: [[type_i64:%[0-9]+\:type\(s64\)]] = OpTypeInt 32, 0
diff --git a/llvm/test/CodeGen/SPIRV/debug-info/debug-type-basic.ll b/llvm/test/CodeGen/SPIRV/debug-info/debug-type-basic.ll
new file mode 100644
index 00000000000000..e3c382d01c6c04
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/debug-info/debug-type-basic.ll
@@ -0,0 +1,233 @@
+; RUN: llc --verify-machineinstrs --spv-emit-nonsemantic-debug-info --spirv-ext=+SPV_KHR_non_semantic_info --print-after=spirv-nonsemantic-debug-info -O0 -mtriple=spirv64-unknown-unknown %s -o - 2>&1 | FileCheck %s --check-prefix=CHECK-MIR
+; RUN: llc --verify-machineinstrs --spv-emit-nonsemantic-debug-info --spirv-ext=+SPV_KHR_non_semantic_info -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: llc --verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_non_semantic_info %s -o - | FileCheck %s --check-prefix=CHECK-OPTION
+; RUN: %if spirv-tools %{ llc --verify-machineinstrs --spv-emit-nonsemantic-debug-info --spirv-ext=+SPV_KHR_non_semantic_info -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-MIR-DAG: [[type_void:%[0-9]+\:type]] = OpTypeVoid
+; CHECK-MIR-DAG: [[type_i32:%[0-9]+\:type]] = OpTypeInt 32, 0
+; CHECK-MIR-DAG: [[encoding_signedchar:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 5
+; CHECK-MIR-DAG: [[encoding_float:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 3
+; CHECK-MIR-DAG: [[flag_zero:%[0-9]+\:iid\(s32\)]] = OpConstantNull [[type_i32]]
+; CHECK-MIR-DAG: [[str_bool:%[0-9]+\:id\(s32\)]] = OpString 1819242338, 0
+; CHECK-MIR-DAG: [[size_8bits:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 8
+; CHECK-MIR-DAG: [[encoding_boolean:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 2
+; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_bool]], [[size_8bits]], [[encoding_boolean]], [[flag_zero]]
+; CHECK-MIR-DAG: [[str_int:%[0-9]+\:id\(s32\)]] = OpString 7630441
+; CHECK-MIR-DAG: [[size_32bits:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 32
+; CHECK-MIR-DAG: [[encoding_signed:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 4
+; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_int]], [[size_32bits]], [[encoding_signed]], [[flag_zero]]
+; CHECK-MIR-DAG: [[str_short:%[0-9]+\:id\(s32\)]] = OpString 1919903859, 116
+; CHECK-MIR-DAG: [[size_16bits:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 16
+; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_short]], [[size_16bits]], [[encoding_signed]], [[flag_zero]]
+; CHECK-MIR-DAG: [[str_char:%[0-9]+\:id\(s32\)]] = OpString 1918986339, 0
+; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_char]], [[size_8bits]], [[encoding_signedchar]], [[flag_zero]]
+; CHECK-MIR-DAG: [[str_long:%[0-9]+\:id\(s32\)]] = OpString 1735290732, 0
+; CHECK-MIR-DAG: [[size_64bits:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 64
+; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_long]], [[size_64bits]], [[encoding_signed]], [[flag_zero]]
+; CHECK-MIR-DAG: [[str_uint:%[0-9]+\:id\(s32\)]] = OpString 1769172597, 1684368999, 1953392928, 0
+; CHECK-MIR-DAG: [[encoding_unsigned:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 6
+; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_uint]], [[size_32bits]], [[encoding_unsigned]], [[flag_zero]]
+; CHECK-MIR-DAG: [[str_ushort:%[0-9]+\:id\(s32\)]] = OpString 1769172597, 1684368999, 1869116192, 29810
+; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_ushort]], [[size_16bits]], [[encoding_unsigned]], [[flag_zero]]
+; CHECK-MIR-DAG: [[str_uchar:%[0-9]+\:id\(s32\)]] = OpString 1769172597, 1684368999, 1634231072, 114
+; CHECK-MIR-DAG: [[encoding_unsignedchar:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i32]], 7
+; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_uchar]], [[size_8bits]], [[encoding_unsignedchar]], [[flag_zero]]
+; CHECK-MIR-DAG: [[str_ulong:%[0-9]+\:id\(s32\)]] = OpString 1769172597, 1684368999, 1852795936, 103
+; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_ulong]], [[size_64bits]], [[encoding_unsigned]], [[flag_zero]]
+; CHECK-MIR-DAG: [[str_float:%[0-9]+\:id\(s32\)]] = OpString 1634692198, 116
+; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_float]], [[size_32bits]], [[encoding_float]], [[flag_zero]]
+; CHECK-MIR-DAG: [[str_double:%[0-9]+\:id\(s32\)]] = OpString 1651863396, 25964
+; CHECK-MIR: OpExtInst [[type_void]], 3, 2, [[str_double]], [[size_64bits]], [[encoding_float]], [[flag_zero]]
+
+; CHECK-SPIRV: [[ext_inst_non_semantic:%[0-9]+]] = OpExtInstImport "NonSemantic.Shader.DebugInfo.100"
+; CHECK-SPIRV-DAG: [[str_bool:%[0-9]+]] = OpString "bool"
+; CHECK-SPIRV-DAG: [[str_int:%[0-9]+]] = OpString "int"
+; CHECK-SPIRV-DAG: [[str_short:%[0-9]+]] = OpString "short"
+; CHECK-SPIRV-DAG: [[str_char:%[0-9]+]] = OpString "char"
+; CHECK-SPIRV-DAG: [[str_long:%[0-9]+]] = OpString "long"
+; CHECK-SPIRV-DAG: [[str_uint:%[0-9]+]] = OpString "unsigned int"
+; CHECK-SPIRV-DAG: [[str_ushort:%[0-9]+]] = OpString "unsigned short"
+; CHECK-SPIRV-DAG: [[str_uchar:%[0-9]+]] = OpString "unsigned char"
+; CHECK-SPIRV-DAG: [[str_ulong:%[0-9]+]] = OpString "unsigned long"
+; CHECK-SPIRV-DAG: [[str_float:%[0-9]+]] = OpString "float"
+; CHECK-SPIRV-DAG: [[str_double:%[0-9]+]] = OpString "double"
+; CHECK-SPIRV-NOT: ----------------
+; CHECK-SPIRV-DAG: [[type_void:%[0-9]+]] = OpTypeVoid
+; CHECK-SPIRV-DAG: [[type_float64:%[0-9]+]] = OpTypeFloat 64
+; CHECK-SPIRV-DAG: [[type_float32:%[0-9]+]] = OpTypeFloat 32
+; CHECK-SPIRV-DAG: [[type_int64:%[0-9]+]] = OpTypeInt 64 0
+; CHECK-SPIRV-DAG: [[type_int8:%[0-9]+]] = OpTypeInt 8 0
+; CHECK-SPIRV-DAG: [[type_int16:%[0-9]+]] = OpTypeInt 16 0
+; CHECK-SPIRV-DAG: [[type_int32:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-SPIRV-DAG: [[encoding_signedchar:%[0-9]+]] = OpConstant [[type_int32]] 5
+; CHECK-SPIRV-DAG: [[flag_zero:%[0-9]+]] = OpConstantNull [[type_int32]]
+; CHECK-SPIRV-DAG: [[encoding_float:%[0-9]+]] = OpConstant [[type_int32]] 3
+; CHECK-SPIRV-DAG: [[size_8bit:%[0-9]+]] = OpConstant [[type_int32]] 8
+; CHECK-SPIRV-DAG: [[encoding_boolean:%[0-9]+]] = OpConstant [[type_int32]] 2
+; CHECK-SPIRV-DAG: [[size_32bit:%[0-9]+]] = OpConstant [[type_int32]] 32
+; CHECK-SPIRV-DAG: [[encoding_signed:%[0-9]+]] = OpConstant [[type_int32]] 4
+; CHECK-SPIRV-DAG: [[size_16bit:%[0-9]+]] = OpConstant [[type_int32]] 16
+; CHECK-SPIRV-DAG: [[size_64bit:%[0-9]+]] = OpConstant [[type_int32]] 64
+; CHECK-SPIRV-DAG: [[encoding_unsigned:%[0-9]+]] = OpConstant [[type_int32]] 6
+; CHECK-SPIRV-DAG: [[encoding_unsignedchar:%[0-9]+]] = OpConstant [[type_int32]] 7
+; CHECK-SPIRV-DAG: OpExtInst [[type_void]] [[ext_inst_non_semantic]] DebugTypeBasic [[str_bool]] [[size_8bit]] [[encoding_boolean]] [[flag_zero]]
+; CHECK-SPIRV-DAG: OpExtInst [[type_void]] [[ext_inst_non_semantic]] DebugTypeBasic [[str_int]] [[size_32bit]] [[encoding_signed]] [[flag_zero]]
+; CHECK-SPIRV-DAG: OpExtInst [[type_void]] [[ext_inst_non_semantic]] DebugTypeBasic [[str_short]] [[size_16bit]] [[encoding_signed]] [[flag_zero]]
+; CHECK-SPIRV-DAG: OpExtInst [[type_void]] [[ext_inst_non_semantic]] DebugTypeBasic [[str_char]] [[size_8bit]] [[encoding_signedchar]] [[flag_zero]]
+; CHECK-SPIRV-DAG: OpExtInst [[type_void]] [[ext_inst_non_semantic]] DebugTypeBasic [[str_long]] [[size_64bit]] [[encoding_signed]] [[flag_zero]]
+; CHECK-SPIRV-DAG: OpExtInst [[type_void]] [[ext_inst_non_semantic]] DebugTypeBasic [[str_uint]] [[size_32bit]] [[encoding_unsigned]] [[flag_zero]]
+; CHECK-SPIRV-DAG: OpExtInst [[type_void]] [[ext_inst_non_semantic]] DebugTypeBasic [[str_ushort]] [[size_16bit]] [[encoding_unsigned]] [[flag_zero]]
+; CHECK-SPIRV-DAG: OpExtInst [[type_void]] [[ext_inst_non_semantic]] DebugTypeBasic [[str_uchar]] [[size_8bit]] [[encoding_unsignedchar]] [[flag_zero]]
+; CHECK-SPIRV-DAG: OpExtInst [[type_void]] [[ext_inst_non_semantic]] DebugTypeBasic [[str_ulong]] [[size_64bit]] [[encoding_unsigned]] [[flag_zero]]
+; CHECK-SPIRV-DAG: OpExtInst [[type_void]] [[ext_inst_non_semantic]] DebugTypeBasic [[str_float]] [[size_32bit]] [[encoding_float]] [[flag_zero]]
+; CHECK-SPIRV-DAG: OpExtInst [[type_void]] [[ext_inst_non_semantic]] DebugTypeBasic [[str_double]] [[size_64bit]] [[encoding_float]] [[flag_zero]]
+
+; CHECK-OPTION-NOT: DebugTypeBasic
+
+define spir_func void  @test1() !dbg !9 {
+entry:
+  %b0 = alloca i8, align 1
+  %a0 = alloca i32, align 4
+  %s0 = alloca i16, align 2
+  %c0 = alloca i8, align 1
+  %l0 = alloca i64, align 8
+  %ua0 = alloca i32, align 4
+  %us0 = alloca i16, align 2
+  %uc0 = alloca i8, align 1
+  %ul0 = alloca i64, align 8
+  %f0 = alloca float, align 4
+  %d0 = alloca double, align 8
+    #dbg_declare(ptr %b0, !14, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !16)
+  store i8 0, ptr %b0, align 1, !dbg !16
+    #dbg_declare(ptr %a0, !17, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !18)
+  store i32 1, ptr %a0, align 4, !dbg !18
+    #dbg_declare(ptr %s0, !19, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !21)
+  store i16 2, ptr %s0, align 2, !dbg !21
+    #dbg_declare(ptr %c0, !22, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !24)
+  store i8 3, ptr %c0, align 1, !dbg !24
+    #dbg_declare(ptr %l0, !25, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !27)
+  store i64 4, ptr %l0, align 8, !dbg !27
+    #dbg_declare(ptr %ua0, !28, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !30)
+  store i32 1, ptr %ua0, align 4, !dbg !30
+    #dbg_declare(ptr %us0, !31, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !33)
+  store i16 2, ptr %us0, align 2, !dbg !33
+    #dbg_declare(ptr %uc0, !34, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !36)
+  store i8 3, ptr %uc0, align 1, !dbg !36
+    #dbg_declare(ptr %ul0, !37, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !39)
+  store i64 4, ptr %ul0, align 8, !dbg !39
+    #dbg_declare(ptr %f0, !40, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !42)
+  store float 1.000000e+00, ptr %f0, align 4, !dbg !42
+    #dbg_declare(ptr %d0, !43, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !45)
+  store double 2.000000e+00, ptr %d0, align 8, !dbg !45
+  ret void
+}
+
+define spir_func void @test2() !dbg !47 {
+entry:
+  %b1 = alloca i8, align 1
+  %a1 = alloca i32, align 4
+  %s1 = alloca i16, align 2
+  %c1 = alloca i8, align 1
+  %l1 = alloca i64, align 8
+  %ua1 = alloca i32, align 4
+  %us1 = alloca i16, align 2
+  %uc1 = alloca i8, align 1
+  %ul1 = alloca i64, align 8
+  %f1 = alloca float, align 4
+  %d1 = alloca double, align 8
+    #dbg_declare(ptr %b1, !48, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !49)
+  store i8 0, ptr %b1, align 1, !dbg !49
+    #dbg_declare(ptr %a1, !50, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !51)
+  store i32 1, ptr %a1, align 4, !dbg !51
+    #dbg_declare(ptr %s1, !52, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !53)
+  store i16 2, ptr %s1, align 2, !dbg !53
+    #dbg_declare(ptr %c1, !54, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !55)
+  store i8 3, ptr %c1, align 1, !dbg !55
+    #dbg_declare(ptr %l1, !56, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !57)
+  store i64 4, ptr %l1, align 8, !dbg !57
+    #dbg_declare(ptr %ua1, !58, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !59)
+  store i32 1, ptr %ua1, align 4, !dbg !59
+    #dbg_declare(ptr %us1, !60, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !61)
+  store i16 2, ptr %us1, align 2, !dbg !61
+    #dbg_declare(ptr %uc1, !62, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !63)
+  store i8 3, ptr %uc1, align 1, !dbg !63
+    #dbg_declare(ptr %ul1, !64, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !65)
+  store i64 4, ptr %ul1, align 8, !dbg !65
+    #dbg_declare(ptr %f1, !66, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !67)
+  store float 1.000000e+00, ptr %f1, align 4, !dbg !67
+    #dbg_declare(ptr %d1, !68, !DIExpression(DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef), !69)
+  store double 2.000000e+00, ptr %d1, align 8, !dbg !69
+  ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version XX.X.XXXX (FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "example.cpp", directory: "/AAAAAAAAAA/BBBBBBBB/CCCCCCCCC", checksumkind: CSK_MD5, checksum: "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF")
+!2 = !{i32 7, !"Dwarf Version", i32 5}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 7, !"frame-pointer", i32 2}
+!9 = distinct !DISubprogram(name: "test1", linkageName: "XXXX", scope: !1, file: !1, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !13)
+!10 = !DISubroutineType(cc: DW_CC_LLVM_SpirFunction, types: !11)
+!11 = !{!12}
+!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!13 = !{}
+!14 = !DILocalVariable(name: "b0", scope: !9, file: !1, line: 2, type: !15)
+!15 = !DIBasicType(name: "bool", size: 8, encoding: DW_ATE_boolean)
+!16 = !DILocation(line: 2, column: 8, scope: !9)
+!17 = !DILocalVariable(name: "a0", scope: !9, file: !1, line: 3, type: !12)
+!18 = !DILocation(line: 3, column: 7, scope: !9)
+!19 = !DILocalVariable(name: "s0", scope: !9, file: !1, line: 4, type: !20)
+!20 = !DIBasicType(name: "short", size: 16, encoding: DW_ATE_signed)
+!21 = !DILocation(line: 4, column: 9, scope: !9)
+!22 = !DILocalVariable(name: "c0", scope: !9, file: !1, line: 5, type: !23)
+!23 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!24 = !DILocation(line: 5, column: 8, scope: !9)
+!25 = !DILocalVariable(name: "l0", scope: !9, file: !1, line: 6, type: !26)
+!26 = !DIBasicType(name: "long", size: 64, encoding: DW_ATE_signed)
+!27 = !DILocation(line: 6, column: 8, scope: !9)
+!28 = !DILocalVariable(name: "ua0", scope: !9, file: !1, line: 7, type: !29)
+!29 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!30 = !DILocation(line: 7, column: 16, scope: !9)
+!31 = !DILocalVariable(name: "us0", scope: !9, file: !1, line: 8, type: !32)
+!32 = !DIBasicType(name: "unsigned short", size: 16, encoding: DW_ATE_unsigned)
+!33 = !DILocation(line: 8, column: 18, scope: !9)
+!34 = !DILocalVariable(name: "uc0", scope: !9, file: !1, line: 9, type: !35)
+!35 = !DIBasicType(name: "unsigned char", size: 8, encoding: DW_ATE_unsigned_char)
+!36 = !DILocation(line: 9, column: 17, scope: !9)
+!37 = !DILocalVariable(name: "ul0", scope: !9, file: !1, line: 10, type: !38)
+!38 = !DIBasicType(name: "unsigned long", size: 64, encoding: DW_ATE_unsigned)
+!39 = !DILocation(line: 10, column: 17, scope: !9)
+!40 = !DILocalVariable(name: "f0", scope: !9, file: !1, line: 11, type: !41)
+!41 = !DIBasicType(name: "float", size: 32, encoding: DW_ATE_float)
+!42 = !DILocation(line: 11, column: 9, scope: !9)
+!43 = !DILocalVariable(name: "d0", scope: !9, file: !1, line: 12, type: !44)
+!44 = !DIBasicType(name: "double", size: 64, encoding: DW_ATE_float)
+!45 = !DILocation(line: 12, column: 10, scope: !9)
+!46 = !DILocation(line: 13, column: 3, scope: !9)
+!47 = distinct !DISubprogram(name: "test2", linkageName: "YYYY", scope: !1, file: !1, line: 16, type: !10, scopeLine: 16, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !13)
+!48 = !DILocalVariable(name: "b1", scope: !47, file: !1, line: 17, type: !15)
+!49 = !DILocation(line: 17, column: 8, scope: !47)
+!50 = !DILocalVariable(name: "a1", scope: !47, file: !1, line: 18, type: !12)
+!51 = !DILocation(line: 18, column: 7, scope: !47)
+!52 = !DILocalVariable(name: "s1", scope: !47, file: !1, line: 19, type: !20)
+!53 = !DILocation(line: 19, column: 9, scope: !47)
+!54 = !DILocalVariable(name: "c1", scope: !47, file: !1, line: 20, type: !23)
+!55 = !DILocation(line: 20, column: 8, scope: !47)
+!56 = !DILocalVariable(name: "l1", scope: !47, file: !1, line: 21, type: !26)
+!57 = !DILocation(line: 21, column: 8, scope: !47)
+!58 = !DILocalVariable(name: "ua1", scope: !47, file: !1, line: 22, type: !29)
+!59 = !DILocation(line: 22, column: 16, scope: !47)
+!60 = !DILocalVariable(name: "us1", scope: !47, file: !1, line: 23, type: !32)
+!61 = !DILocation(line: 23, column: 18, scope: !47)
+!62 = !DILocalVariable(name: "uc1", scope: !47, file: !1, line: 24, type: !35)
+!63 = !DILocation(line: 24, column: 17, scope: !47)
+!64 = !DILocalVariable(name: "ul1", scope: !47, file: !1, line: 25, type: !38)
+!65 = !DILocation(line: 25, column: 17, scope: !47)
+!66 = !DILocalVariable(name: "f1", scope: !47, file: !1, line: 26, type: !41)
+!67 = !DILocation(line: 26, column: 9, scope: !47)
+!68 = !DILocalVariable(name: "d1", scope: !47, file: !1, line: 27, type: !44)
+!69 = !DILocation(line: 27, column: 10, scope: !47)
+!70 = !DILocation(line: 28, column: 3, scope: !47)
diff --git a/llvm/test/CodeGen/Thumb2/pr52817.ll b/llvm/test/CodeGen/Thumb2/pr52817.ll
index 12daeedf43d818..87615f0a1f7ef4 100644
--- a/llvm/test/CodeGen/Thumb2/pr52817.ll
+++ b/llvm/test/CodeGen/Thumb2/pr52817.ll
@@ -24,7 +24,6 @@ define i32 @test(ptr %arg, ptr %arg1, ptr %arg2) #0 !dbg !6 {
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:  LBB0_1: @ %bb3
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    .loc 1 0 0 is_stmt 0 @ :0:0
 ; CHECK-NEXT:    adds r5, r3, #1
 ; CHECK-NEXT:    str.w lr, [r2]
 ; CHECK-NEXT:    cmp.w lr, #0
@@ -37,7 +36,7 @@ define i32 @test(ptr %arg, ptr %arg1, ptr %arg2) #0 !dbg !6 {
 ; CHECK-NEXT:    movne r6, #0
 ; CHECK-NEXT:  Ltmp0:
 ; CHECK-NEXT:    @DEBUG_VALUE: test:this <- [DW_OP_LLVM_arg 0, DW_OP_plus_uconst 135168, DW_OP_LLVM_arg 1, DW_OP_constu 4, DW_OP_mul, DW_OP_plus, DW_OP_plus_uconst 4, DW_OP_stack_value] $r0, $r5
-; CHECK-NEXT:    .loc 1 28 24 prologue_end is_stmt 1 @ test.cpp:28:24
+; CHECK-NEXT:    .loc 1 28 24 prologue_end @ test.cpp:28:24
 ; CHECK-NEXT:    strne.w r6, [r8]
 ; CHECK-NEXT:    moveq r6, #1
 ; CHECK-NEXT:    ldr r4, [r4, #4]
diff --git a/llvm/test/CodeGen/WebAssembly/exception.ll b/llvm/test/CodeGen/WebAssembly/exception.ll
index 7259761d6313b6..1ad4c84f1c0237 100644
--- a/llvm/test/CodeGen/WebAssembly/exception.ll
+++ b/llvm/test/CodeGen/WebAssembly/exception.ll
@@ -1,6 +1,7 @@
 ; RUN: llc < %s -asm-verbose=false -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling -wasm-enable-exnref -verify-machineinstrs | FileCheck --implicit-check-not=ehgcr -allow-deprecated-dag-overlap %s
 ; RUN: llc < %s -asm-verbose=false -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling -wasm-enable-exnref -verify-machineinstrs -O0
 ; RUN: llc < %s -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling -wasm-enable-exnref
+; RUN: llc < %s -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling -wasm-enable-exnref -filetype=obj
 
 target triple = "wasm32-unknown-unknown"
 
diff --git a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
index c67049d5006ca3..a4e713602f673d 100644
--- a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
@@ -2030,9 +2030,9 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
 ; AVX512BW-FAST-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
 ; AVX512BW-FAST:       # %bb.0:
 ; AVX512BW-FAST-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512BW-FAST-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [0,25,0,27,0,29,0,31]
+; AVX512BW-FAST-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [0,9,0,11,0,13,0,15]
 ; AVX512BW-FAST-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT:    vpermt2d %zmm0, %zmm1, %zmm0
+; AVX512BW-FAST-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
 ; AVX512BW-FAST-NEXT:    vmovdqa64 %zmm0, (%rcx)
 ; AVX512BW-FAST-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/apx/domain-reassignment.mir b/llvm/test/CodeGen/X86/apx/domain-reassignment.mir
index 7352aa2b307f9a..49af7a6c949a12 100644
--- a/llvm/test/CodeGen/X86/apx/domain-reassignment.mir
+++ b/llvm/test/CodeGen/X86/apx/domain-reassignment.mir
@@ -273,15 +273,15 @@ body:             |
   ; CHECK:   [[VCMPPDZrri:%[0-9]+]]:vk8 = VCMPPDZrri [[COPY3]], [[COPY4]], 0
   ; CHECK:   [[COPY5:%[0-9]+]]:vk32 = COPY [[VCMPPDZrri]]
   ; CHECK:   [[COPY6:%[0-9]+]]:vk8 = COPY [[COPY5]]
-  ; CHECK:   [[KSHIFTRBri:%[0-9]+]]:vk8 = KSHIFTRBri [[COPY6]], 2
-  ; CHECK:   [[KSHIFTLBri:%[0-9]+]]:vk8 = KSHIFTLBri [[KSHIFTRBri]], 1
-  ; CHECK:   [[KNOTBrr:%[0-9]+]]:vk8 = KNOTBrr [[KSHIFTLBri]]
-  ; CHECK:   [[KORBrr:%[0-9]+]]:vk8 = KORBrr [[KNOTBrr]], [[KSHIFTRBri]]
-  ; CHECK:   [[KANDBrr:%[0-9]+]]:vk8 = KANDBrr [[KORBrr]], [[KSHIFTLBri]]
-  ; CHECK:   [[KXORBrr:%[0-9]+]]:vk8 = KXORBrr [[KANDBrr]], [[KSHIFTRBri]]
-  ; CHECK:   [[KADDBrr:%[0-9]+]]:vk8 = KADDBrr [[KXORBrr]], [[KNOTBrr]]
+  ; CHECK:   [[KSHIFTRBki:%[0-9]+]]:vk8 = KSHIFTRBki [[COPY6]], 2
+  ; CHECK:   [[KSHIFTLBki:%[0-9]+]]:vk8 = KSHIFTLBki [[KSHIFTRBki]], 1
+  ; CHECK:   [[KNOTBkk:%[0-9]+]]:vk8 = KNOTBkk [[KSHIFTLBki]]
+  ; CHECK:   [[KORBkk:%[0-9]+]]:vk8 = KORBkk [[KNOTBkk]], [[KSHIFTRBki]]
+  ; CHECK:   [[KANDBkk:%[0-9]+]]:vk8 = KANDBkk [[KORBkk]], [[KSHIFTLBki]]
+  ; CHECK:   [[KXORBkk:%[0-9]+]]:vk8 = KXORBkk [[KANDBkk]], [[KSHIFTRBki]]
+  ; CHECK:   [[KADDBkk:%[0-9]+]]:vk8 = KADDBkk [[KXORBkk]], [[KNOTBkk]]
   ; CHECK:   [[DEF:%[0-9]+]]:vk32 = IMPLICIT_DEF
-  ; CHECK:   [[COPY7:%[0-9]+]]:vk32 = COPY [[KADDBrr]]
+  ; CHECK:   [[COPY7:%[0-9]+]]:vk32 = COPY [[KADDBkk]]
   ; CHECK:   [[COPY8:%[0-9]+]]:vk8wm = COPY [[COPY7]]
   ; CHECK:   [[VMOVAPDZrrk:%[0-9]+]]:vr512 = VMOVAPDZrrk [[COPY2]], killed [[COPY8]], [[COPY1]]
   ; CHECK:   VMOVAPDZmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVAPDZrrk]]
@@ -393,14 +393,14 @@ body:             |
   ; CHECK:   [[VCMPPSZrri:%[0-9]+]]:vk16 = VCMPPSZrri [[COPY3]], [[COPY4]], 0
   ; CHECK:   [[COPY5:%[0-9]+]]:vk32 = COPY [[VCMPPSZrri]]
   ; CHECK:   [[COPY6:%[0-9]+]]:vk16 = COPY [[COPY5]]
-  ; CHECK:   [[KSHIFTRWri:%[0-9]+]]:vk16 = KSHIFTRWri [[COPY6]], 2
-  ; CHECK:   [[KSHIFTLWri:%[0-9]+]]:vk16 = KSHIFTLWri [[KSHIFTRWri]], 1
-  ; CHECK:   [[KNOTWrr:%[0-9]+]]:vk16 = KNOTWrr [[KSHIFTLWri]]
-  ; CHECK:   [[KORWrr:%[0-9]+]]:vk16 = KORWrr [[KNOTWrr]], [[KSHIFTRWri]]
-  ; CHECK:   [[KANDWrr:%[0-9]+]]:vk16 = KANDWrr [[KORWrr]], [[KSHIFTLWri]]
-  ; CHECK:   [[KXORWrr:%[0-9]+]]:vk16 = KXORWrr [[KANDWrr]], [[KSHIFTRWri]]
+  ; CHECK:   [[KSHIFTRWki:%[0-9]+]]:vk16 = KSHIFTRWki [[COPY6]], 2
+  ; CHECK:   [[KSHIFTLWki:%[0-9]+]]:vk16 = KSHIFTLWki [[KSHIFTRWki]], 1
+  ; CHECK:   [[KNOTWkk:%[0-9]+]]:vk16 = KNOTWkk [[KSHIFTLWki]]
+  ; CHECK:   [[KORWkk:%[0-9]+]]:vk16 = KORWkk [[KNOTWkk]], [[KSHIFTRWki]]
+  ; CHECK:   [[KANDWkk:%[0-9]+]]:vk16 = KANDWkk [[KORWkk]], [[KSHIFTLWki]]
+  ; CHECK:   [[KXORWkk:%[0-9]+]]:vk16 = KXORWkk [[KANDWkk]], [[KSHIFTRWki]]
   ; CHECK:   [[DEF:%[0-9]+]]:vk32 = IMPLICIT_DEF
-  ; CHECK:   [[COPY7:%[0-9]+]]:vk32 = COPY [[KXORWrr]]
+  ; CHECK:   [[COPY7:%[0-9]+]]:vk32 = COPY [[KXORWkk]]
   ; CHECK:   [[COPY8:%[0-9]+]]:vk16wm = COPY [[COPY7]]
   ; CHECK:   [[VMOVAPSZrrk:%[0-9]+]]:vr512 = VMOVAPSZrrk [[COPY2]], killed [[COPY8]], [[COPY1]]
   ; CHECK:   VMOVAPSZmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVAPSZrrk]]
@@ -501,15 +501,15 @@ body:             |
   ; CHECK:   [[COPY1:%[0-9]+]]:vr512 = COPY $zmm0
   ; CHECK:   [[COPY2:%[0-9]+]]:vr512 = COPY $zmm1
   ; CHECK:   [[KMOVDkm:%[0-9]+]]:vk32 = KMOVDkm [[COPY]], 1, $noreg, 0, $noreg
-  ; CHECK:   [[KSHIFTRDri:%[0-9]+]]:vk32 = KSHIFTRDri [[KMOVDkm]], 2
-  ; CHECK:   [[KSHIFTLDri:%[0-9]+]]:vk32 = KSHIFTLDri [[KSHIFTRDri]], 1
-  ; CHECK:   [[KNOTDrr:%[0-9]+]]:vk32 = KNOTDrr [[KSHIFTLDri]]
-  ; CHECK:   [[KORDrr:%[0-9]+]]:vk32 = KORDrr [[KNOTDrr]], [[KSHIFTRDri]]
-  ; CHECK:   [[KANDDrr:%[0-9]+]]:vk32 = KANDDrr [[KORDrr]], [[KSHIFTLDri]]
-  ; CHECK:   [[KXORDrr:%[0-9]+]]:vk32 = KXORDrr [[KANDDrr]], [[KSHIFTRDri]]
-  ; CHECK:   [[KANDNDrr:%[0-9]+]]:vk32 = KANDNDrr [[KXORDrr]], [[KORDrr]]
-  ; CHECK:   [[KADDDrr:%[0-9]+]]:vk32 = KADDDrr [[KANDNDrr]], [[KXORDrr]]
-  ; CHECK:   [[COPY3:%[0-9]+]]:vk32wm = COPY [[KADDDrr]]
+  ; CHECK:   [[KSHIFTRDki:%[0-9]+]]:vk32 = KSHIFTRDki [[KMOVDkm]], 2
+  ; CHECK:   [[KSHIFTLDki:%[0-9]+]]:vk32 = KSHIFTLDki [[KSHIFTRDki]], 1
+  ; CHECK:   [[KNOTDkk:%[0-9]+]]:vk32 = KNOTDkk [[KSHIFTLDki]]
+  ; CHECK:   [[KORDkk:%[0-9]+]]:vk32 = KORDkk [[KNOTDkk]], [[KSHIFTRDki]]
+  ; CHECK:   [[KANDDkk:%[0-9]+]]:vk32 = KANDDkk [[KORDkk]], [[KSHIFTLDki]]
+  ; CHECK:   [[KXORDkk:%[0-9]+]]:vk32 = KXORDkk [[KANDDkk]], [[KSHIFTRDki]]
+  ; CHECK:   [[KANDNDkk:%[0-9]+]]:vk32 = KANDNDkk [[KXORDkk]], [[KORDkk]]
+  ; CHECK:   [[KADDDkk:%[0-9]+]]:vk32 = KADDDkk [[KANDNDkk]], [[KXORDkk]]
+  ; CHECK:   [[COPY3:%[0-9]+]]:vk32wm = COPY [[KADDDkk]]
   ; CHECK:   [[VMOVDQU16Zrrk:%[0-9]+]]:vr512 = VMOVDQU16Zrrk [[COPY2]], killed [[COPY3]], [[COPY1]]
   ; CHECK:   VMOVDQA32Zmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVDQU16Zrrk]]
   ; CHECK: bb.1:
@@ -604,15 +604,15 @@ body:             |
   ; CHECK:   [[COPY1:%[0-9]+]]:vr512 = COPY $zmm0
   ; CHECK:   [[COPY2:%[0-9]+]]:vr512 = COPY $zmm1
   ; CHECK:   [[KMOVQkm:%[0-9]+]]:vk64 = KMOVQkm [[COPY]], 1, $noreg, 0, $noreg
-  ; CHECK:   [[KSHIFTRQri:%[0-9]+]]:vk64 = KSHIFTRQri [[KMOVQkm]], 2
-  ; CHECK:   [[KSHIFTLQri:%[0-9]+]]:vk64 = KSHIFTLQri [[KSHIFTRQri]], 1
-  ; CHECK:   [[KNOTQrr:%[0-9]+]]:vk64 = KNOTQrr [[KSHIFTLQri]]
-  ; CHECK:   [[KORQrr:%[0-9]+]]:vk64 = KORQrr [[KNOTQrr]], [[KSHIFTRQri]]
-  ; CHECK:   [[KANDQrr:%[0-9]+]]:vk64 = KANDQrr [[KORQrr]], [[KSHIFTLQri]]
-  ; CHECK:   [[KXORQrr:%[0-9]+]]:vk64 = KXORQrr [[KANDQrr]], [[KSHIFTRQri]]
-  ; CHECK:   [[KANDNQrr:%[0-9]+]]:vk64 = KANDNQrr [[KXORQrr]], [[KORQrr]]
-  ; CHECK:   [[KADDQrr:%[0-9]+]]:vk64 = KADDQrr [[KANDNQrr]], [[KXORQrr]]
-  ; CHECK:   [[COPY3:%[0-9]+]]:vk64wm = COPY [[KADDQrr]]
+  ; CHECK:   [[KSHIFTRQki:%[0-9]+]]:vk64 = KSHIFTRQki [[KMOVQkm]], 2
+  ; CHECK:   [[KSHIFTLQki:%[0-9]+]]:vk64 = KSHIFTLQki [[KSHIFTRQki]], 1
+  ; CHECK:   [[KNOTQkk:%[0-9]+]]:vk64 = KNOTQkk [[KSHIFTLQki]]
+  ; CHECK:   [[KORQkk:%[0-9]+]]:vk64 = KORQkk [[KNOTQkk]], [[KSHIFTRQki]]
+  ; CHECK:   [[KANDQkk:%[0-9]+]]:vk64 = KANDQkk [[KORQkk]], [[KSHIFTLQki]]
+  ; CHECK:   [[KXORQkk:%[0-9]+]]:vk64 = KXORQkk [[KANDQkk]], [[KSHIFTRQki]]
+  ; CHECK:   [[KANDNQkk:%[0-9]+]]:vk64 = KANDNQkk [[KXORQkk]], [[KORQkk]]
+  ; CHECK:   [[KADDQkk:%[0-9]+]]:vk64 = KADDQkk [[KANDNQkk]], [[KXORQkk]]
+  ; CHECK:   [[COPY3:%[0-9]+]]:vk64wm = COPY [[KADDQkk]]
   ; CHECK:   [[VMOVDQU8Zrrk:%[0-9]+]]:vr512 = VMOVDQU8Zrrk [[COPY2]], killed [[COPY3]], [[COPY1]]
   ; CHECK:   VMOVDQA32Zmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVDQU8Zrrk]]
   ; CHECK: bb.1:
@@ -702,8 +702,8 @@ body:             |
     ; CHECK: [[COPY2:%[0-9]+]]:vr512 = COPY $zmm1
     ; CHECK: [[KMOVBkm:%[0-9]+]]:vk8 = KMOVBkm [[COPY]], 1, $noreg, 0, $noreg
     ; CHECK: [[COPY3:%[0-9]+]]:vk16 = COPY [[KMOVBkm]]
-    ; CHECK: [[KNOTWrr:%[0-9]+]]:vk16 = KNOTWrr [[COPY3]]
-    ; CHECK: [[COPY4:%[0-9]+]]:vk16wm = COPY [[KNOTWrr]]
+    ; CHECK: [[KNOTWkk:%[0-9]+]]:vk16 = KNOTWkk [[COPY3]]
+    ; CHECK: [[COPY4:%[0-9]+]]:vk16wm = COPY [[KNOTWkk]]
     ; CHECK: [[VMOVAPSZrrk:%[0-9]+]]:vr512 = VMOVAPSZrrk [[COPY2]], killed [[COPY4]], [[COPY1]]
     ; CHECK: VMOVAPSZmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVAPSZrrk]]
     ; CHECK: RET 0
@@ -774,8 +774,8 @@ body:             |
     ; CHECK: [[COPY3:%[0-9]+]]:vk32 = COPY [[KMOVBkm]]
     ; CHECK: [[KMOVWkm:%[0-9]+]]:vk16 = KMOVWkm [[COPY]], 1, $noreg, 0, $noreg
     ; CHECK: [[COPY4:%[0-9]+]]:vk32 = COPY [[KMOVWkm]]
-    ; CHECK: [[KADDDrr:%[0-9]+]]:vk32 = KADDDrr [[COPY3]], [[COPY4]]
-    ; CHECK: [[COPY5:%[0-9]+]]:vk64wm = COPY [[KADDDrr]]
+    ; CHECK: [[KADDDkk:%[0-9]+]]:vk32 = KADDDkk [[COPY3]], [[COPY4]]
+    ; CHECK: [[COPY5:%[0-9]+]]:vk64wm = COPY [[KADDDkk]]
     ; CHECK: [[VMOVDQU16Zrrk:%[0-9]+]]:vr512 = VMOVDQU16Zrrk [[COPY2]], killed [[COPY5]], [[COPY1]]
     ; CHECK: VMOVDQA32Zmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVDQU16Zrrk]]
     ; CHECK: RET 0
@@ -847,8 +847,8 @@ body:             |
     ; CHECK: [[COPY3:%[0-9]+]]:vk64 = COPY [[KMOVBkm]]
     ; CHECK: [[KMOVWkm:%[0-9]+]]:vk16 = KMOVWkm [[COPY]], 1, $noreg, 0, $noreg
     ; CHECK: [[COPY4:%[0-9]+]]:vk64 = COPY [[KMOVWkm]]
-    ; CHECK: [[KADDQrr:%[0-9]+]]:vk64 = KADDQrr [[COPY3]], [[COPY4]]
-    ; CHECK: [[COPY5:%[0-9]+]]:vk64wm = COPY [[KADDQrr]]
+    ; CHECK: [[KADDQkk:%[0-9]+]]:vk64 = KADDQkk [[COPY3]], [[COPY4]]
+    ; CHECK: [[COPY5:%[0-9]+]]:vk64wm = COPY [[KADDQkk]]
     ; CHECK: [[VMOVDQU8Zrrk:%[0-9]+]]:vr512 = VMOVDQU8Zrrk [[COPY2]], killed [[COPY5]], [[COPY1]]
     ; CHECK: VMOVDQA32Zmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVDQU8Zrrk]]
     ; CHECK: RET 0
diff --git a/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll b/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
index add0592661db67..60d5f74c7a364d 100644
--- a/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
+++ b/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
@@ -2221,11 +2221,10 @@ define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask6(<8 x i64> %vec, <4 x i
 define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask7(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
 ; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mask7:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
-; CHECK-NEXT:    vpmovsxbq {{.*#+}} ymm4 = [2,0,3,4]
-; CHECK-NEXT:    vpermi2q %ymm3, %ymm0, %ymm4
+; CHECK-NEXT:    vpmovsxbq {{.*#+}} ymm3 = [2,0,3,4]
+; CHECK-NEXT:    vpermq %zmm0, %zmm3, %zmm0
 ; CHECK-NEXT:    vptestnmq %ymm2, %ymm2, %k1
-; CHECK-NEXT:    vpblendmq %ymm4, %ymm1, %ymm0 {%k1}
+; CHECK-NEXT:    vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 2, i32 0, i32 3, i32 4>
   %cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2235,10 +2234,9 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask7(<8 x i64> %vec, <4 x i64
 define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask7(<8 x i64> %vec, <4 x i64> %mask) {
 ; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask7:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextracti32x4 $2, %zmm0, %xmm2
-; CHECK-NEXT:    vpmovsxbq {{.*#+}} ymm3 = [2,0,3,4]
+; CHECK-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [2,0,3,4]
 ; CHECK-NEXT:    vptestnmq %ymm1, %ymm1, %k1
-; CHECK-NEXT:    vpermt2q %ymm2, %ymm3, %ymm0 {%k1} {z}
+; CHECK-NEXT:    vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
 ; CHECK-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 2, i32 0, i32 3, i32 4>
@@ -4092,8 +4090,8 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask5(<8 x double>
 define <4 x double> @test_8xdouble_to_4xdouble_perm_mask6(<8 x double> %vec) {
 ; CHECK-FAST-LABEL: test_8xdouble_to_4xdouble_perm_mask6:
 ; CHECK-FAST:       # %bb.0:
-; CHECK-FAST-NEXT:    vmovapd {{.*#+}} ymm1 = [5,8,7,8]
-; CHECK-FAST-NEXT:    vpermt2pd %zmm0, %zmm1, %zmm0
+; CHECK-FAST-NEXT:    vmovaps {{.*#+}} ymm1 = [5,0,7,0]
+; CHECK-FAST-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
 ; CHECK-FAST-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; CHECK-FAST-NEXT:    retq
 ;
@@ -4109,11 +4107,11 @@ define <4 x double> @test_8xdouble_to_4xdouble_perm_mask6(<8 x double> %vec) {
 define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask6(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {
 ; CHECK-FAST-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask6:
 ; CHECK-FAST:       # %bb.0:
-; CHECK-FAST-NEXT:    vmovapd {{.*#+}} ymm3 = [5,8,7,8]
-; CHECK-FAST-NEXT:    vpermi2pd %zmm0, %zmm0, %zmm3
-; CHECK-FAST-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
-; CHECK-FAST-NEXT:    vcmpeqpd %ymm0, %ymm2, %k1
-; CHECK-FAST-NEXT:    vblendmpd %ymm3, %ymm1, %ymm0 {%k1}
+; CHECK-FAST-NEXT:    vmovapd {{.*#+}} ymm3 = [5,0,7,0]
+; CHECK-FAST-NEXT:    vpermpd %zmm0, %zmm3, %zmm0
+; CHECK-FAST-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-FAST-NEXT:    vcmpeqpd %ymm3, %ymm2, %k1
+; CHECK-FAST-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1}
 ; CHECK-FAST-NEXT:    retq
 ;
 ; CHECK-FAST-PERLANE-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask6:
@@ -4134,10 +4132,10 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask6(<8 x double> %v
 define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask6(<8 x double> %vec, <4 x double> %mask) {
 ; CHECK-FAST-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask6:
 ; CHECK-FAST:       # %bb.0:
-; CHECK-FAST-NEXT:    vmovapd {{.*#+}} ymm2 = [5,8,7,8]
+; CHECK-FAST-NEXT:    vmovapd {{.*#+}} ymm2 = [5,0,7,0]
 ; CHECK-FAST-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
 ; CHECK-FAST-NEXT:    vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-FAST-NEXT:    vpermt2pd %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-FAST-NEXT:    vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}
 ; CHECK-FAST-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; CHECK-FAST-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/domain-reassignment.mir b/llvm/test/CodeGen/X86/domain-reassignment.mir
index dcd435619990cf..fc0f32c7d4a947 100644
--- a/llvm/test/CodeGen/X86/domain-reassignment.mir
+++ b/llvm/test/CodeGen/X86/domain-reassignment.mir
@@ -273,15 +273,15 @@ body:             |
   ; CHECK:   [[VCMPPDZrri:%[0-9]+]]:vk8 = VCMPPDZrri [[COPY3]], [[COPY4]], 0
   ; CHECK:   [[COPY5:%[0-9]+]]:vk32 = COPY [[VCMPPDZrri]]
   ; CHECK:   [[COPY6:%[0-9]+]]:vk8 = COPY [[COPY5]]
-  ; CHECK:   [[KSHIFTRBri:%[0-9]+]]:vk8 = KSHIFTRBri [[COPY6]], 2
-  ; CHECK:   [[KSHIFTLBri:%[0-9]+]]:vk8 = KSHIFTLBri [[KSHIFTRBri]], 1
-  ; CHECK:   [[KNOTBrr:%[0-9]+]]:vk8 = KNOTBrr [[KSHIFTLBri]]
-  ; CHECK:   [[KORBrr:%[0-9]+]]:vk8 = KORBrr [[KNOTBrr]], [[KSHIFTRBri]]
-  ; CHECK:   [[KANDBrr:%[0-9]+]]:vk8 = KANDBrr [[KORBrr]], [[KSHIFTLBri]]
-  ; CHECK:   [[KXORBrr:%[0-9]+]]:vk8 = KXORBrr [[KANDBrr]], [[KSHIFTRBri]]
-  ; CHECK:   [[KADDBrr:%[0-9]+]]:vk8 = KADDBrr [[KXORBrr]], [[KNOTBrr]]
+  ; CHECK:   [[KSHIFTRBki:%[0-9]+]]:vk8 = KSHIFTRBki [[COPY6]], 2
+  ; CHECK:   [[KSHIFTLBki:%[0-9]+]]:vk8 = KSHIFTLBki [[KSHIFTRBki]], 1
+  ; CHECK:   [[KNOTBkk:%[0-9]+]]:vk8 = KNOTBkk [[KSHIFTLBki]]
+  ; CHECK:   [[KORBkk:%[0-9]+]]:vk8 = KORBkk [[KNOTBkk]], [[KSHIFTRBki]]
+  ; CHECK:   [[KANDBkk:%[0-9]+]]:vk8 = KANDBkk [[KORBkk]], [[KSHIFTLBki]]
+  ; CHECK:   [[KXORBkk:%[0-9]+]]:vk8 = KXORBkk [[KANDBkk]], [[KSHIFTRBki]]
+  ; CHECK:   [[KADDBkk:%[0-9]+]]:vk8 = KADDBkk [[KXORBkk]], [[KNOTBkk]]
   ; CHECK:   [[DEF:%[0-9]+]]:vk32 = IMPLICIT_DEF
-  ; CHECK:   [[COPY7:%[0-9]+]]:vk32 = COPY [[KADDBrr]]
+  ; CHECK:   [[COPY7:%[0-9]+]]:vk32 = COPY [[KADDBkk]]
   ; CHECK:   [[COPY8:%[0-9]+]]:vk8wm = COPY [[COPY7]]
   ; CHECK:   [[VMOVAPDZrrk:%[0-9]+]]:vr512 = VMOVAPDZrrk [[COPY2]], killed [[COPY8]], [[COPY1]]
   ; CHECK:   VMOVAPDZmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVAPDZrrk]]
@@ -393,14 +393,14 @@ body:             |
   ; CHECK:   [[VCMPPSZrri:%[0-9]+]]:vk16 = VCMPPSZrri [[COPY3]], [[COPY4]], 0
   ; CHECK:   [[COPY5:%[0-9]+]]:vk32 = COPY [[VCMPPSZrri]]
   ; CHECK:   [[COPY6:%[0-9]+]]:vk16 = COPY [[COPY5]]
-  ; CHECK:   [[KSHIFTRWri:%[0-9]+]]:vk16 = KSHIFTRWri [[COPY6]], 2
-  ; CHECK:   [[KSHIFTLWri:%[0-9]+]]:vk16 = KSHIFTLWri [[KSHIFTRWri]], 1
-  ; CHECK:   [[KNOTWrr:%[0-9]+]]:vk16 = KNOTWrr [[KSHIFTLWri]]
-  ; CHECK:   [[KORWrr:%[0-9]+]]:vk16 = KORWrr [[KNOTWrr]], [[KSHIFTRWri]]
-  ; CHECK:   [[KANDWrr:%[0-9]+]]:vk16 = KANDWrr [[KORWrr]], [[KSHIFTLWri]]
-  ; CHECK:   [[KXORWrr:%[0-9]+]]:vk16 = KXORWrr [[KANDWrr]], [[KSHIFTRWri]]
+  ; CHECK:   [[KSHIFTRWki:%[0-9]+]]:vk16 = KSHIFTRWki [[COPY6]], 2
+  ; CHECK:   [[KSHIFTLWki:%[0-9]+]]:vk16 = KSHIFTLWki [[KSHIFTRWki]], 1
+  ; CHECK:   [[KNOTWkk:%[0-9]+]]:vk16 = KNOTWkk [[KSHIFTLWki]]
+  ; CHECK:   [[KORWkk:%[0-9]+]]:vk16 = KORWkk [[KNOTWkk]], [[KSHIFTRWki]]
+  ; CHECK:   [[KANDWkk:%[0-9]+]]:vk16 = KANDWkk [[KORWkk]], [[KSHIFTLWki]]
+  ; CHECK:   [[KXORWkk:%[0-9]+]]:vk16 = KXORWkk [[KANDWkk]], [[KSHIFTRWki]]
   ; CHECK:   [[DEF:%[0-9]+]]:vk32 = IMPLICIT_DEF
-  ; CHECK:   [[COPY7:%[0-9]+]]:vk32 = COPY [[KXORWrr]]
+  ; CHECK:   [[COPY7:%[0-9]+]]:vk32 = COPY [[KXORWkk]]
   ; CHECK:   [[COPY8:%[0-9]+]]:vk16wm = COPY [[COPY7]]
   ; CHECK:   [[VMOVAPSZrrk:%[0-9]+]]:vr512 = VMOVAPSZrrk [[COPY2]], killed [[COPY8]], [[COPY1]]
   ; CHECK:   VMOVAPSZmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVAPSZrrk]]
@@ -501,15 +501,15 @@ body:             |
   ; CHECK:   [[COPY1:%[0-9]+]]:vr512 = COPY $zmm0
   ; CHECK:   [[COPY2:%[0-9]+]]:vr512 = COPY $zmm1
   ; CHECK:   [[KMOVDkm:%[0-9]+]]:vk32 = KMOVDkm [[COPY]], 1, $noreg, 0, $noreg
-  ; CHECK:   [[KSHIFTRDri:%[0-9]+]]:vk32 = KSHIFTRDri [[KMOVDkm]], 2
-  ; CHECK:   [[KSHIFTLDri:%[0-9]+]]:vk32 = KSHIFTLDri [[KSHIFTRDri]], 1
-  ; CHECK:   [[KNOTDrr:%[0-9]+]]:vk32 = KNOTDrr [[KSHIFTLDri]]
-  ; CHECK:   [[KORDrr:%[0-9]+]]:vk32 = KORDrr [[KNOTDrr]], [[KSHIFTRDri]]
-  ; CHECK:   [[KANDDrr:%[0-9]+]]:vk32 = KANDDrr [[KORDrr]], [[KSHIFTLDri]]
-  ; CHECK:   [[KXORDrr:%[0-9]+]]:vk32 = KXORDrr [[KANDDrr]], [[KSHIFTRDri]]
-  ; CHECK:   [[KANDNDrr:%[0-9]+]]:vk32 = KANDNDrr [[KXORDrr]], [[KORDrr]]
-  ; CHECK:   [[KADDDrr:%[0-9]+]]:vk32 = KADDDrr [[KANDNDrr]], [[KXORDrr]]
-  ; CHECK:   [[COPY3:%[0-9]+]]:vk32wm = COPY [[KADDDrr]]
+  ; CHECK:   [[KSHIFTRDki:%[0-9]+]]:vk32 = KSHIFTRDki [[KMOVDkm]], 2
+  ; CHECK:   [[KSHIFTLDki:%[0-9]+]]:vk32 = KSHIFTLDki [[KSHIFTRDki]], 1
+  ; CHECK:   [[KNOTDkk:%[0-9]+]]:vk32 = KNOTDkk [[KSHIFTLDki]]
+  ; CHECK:   [[KORDkk:%[0-9]+]]:vk32 = KORDkk [[KNOTDkk]], [[KSHIFTRDki]]
+  ; CHECK:   [[KANDDkk:%[0-9]+]]:vk32 = KANDDkk [[KORDkk]], [[KSHIFTLDki]]
+  ; CHECK:   [[KXORDkk:%[0-9]+]]:vk32 = KXORDkk [[KANDDkk]], [[KSHIFTRDki]]
+  ; CHECK:   [[KANDNDkk:%[0-9]+]]:vk32 = KANDNDkk [[KXORDkk]], [[KORDkk]]
+  ; CHECK:   [[KADDDkk:%[0-9]+]]:vk32 = KADDDkk [[KANDNDkk]], [[KXORDkk]]
+  ; CHECK:   [[COPY3:%[0-9]+]]:vk32wm = COPY [[KADDDkk]]
   ; CHECK:   [[VMOVDQU16Zrrk:%[0-9]+]]:vr512 = VMOVDQU16Zrrk [[COPY2]], killed [[COPY3]], [[COPY1]]
   ; CHECK:   VMOVDQA32Zmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVDQU16Zrrk]]
   ; CHECK: bb.1:
@@ -604,15 +604,15 @@ body:             |
   ; CHECK:   [[COPY1:%[0-9]+]]:vr512 = COPY $zmm0
   ; CHECK:   [[COPY2:%[0-9]+]]:vr512 = COPY $zmm1
   ; CHECK:   [[KMOVQkm:%[0-9]+]]:vk64 = KMOVQkm [[COPY]], 1, $noreg, 0, $noreg
-  ; CHECK:   [[KSHIFTRQri:%[0-9]+]]:vk64 = KSHIFTRQri [[KMOVQkm]], 2
-  ; CHECK:   [[KSHIFTLQri:%[0-9]+]]:vk64 = KSHIFTLQri [[KSHIFTRQri]], 1
-  ; CHECK:   [[KNOTQrr:%[0-9]+]]:vk64 = KNOTQrr [[KSHIFTLQri]]
-  ; CHECK:   [[KORQrr:%[0-9]+]]:vk64 = KORQrr [[KNOTQrr]], [[KSHIFTRQri]]
-  ; CHECK:   [[KANDQrr:%[0-9]+]]:vk64 = KANDQrr [[KORQrr]], [[KSHIFTLQri]]
-  ; CHECK:   [[KXORQrr:%[0-9]+]]:vk64 = KXORQrr [[KANDQrr]], [[KSHIFTRQri]]
-  ; CHECK:   [[KANDNQrr:%[0-9]+]]:vk64 = KANDNQrr [[KXORQrr]], [[KORQrr]]
-  ; CHECK:   [[KADDQrr:%[0-9]+]]:vk64 = KADDQrr [[KANDNQrr]], [[KXORQrr]]
-  ; CHECK:   [[COPY3:%[0-9]+]]:vk64wm = COPY [[KADDQrr]]
+  ; CHECK:   [[KSHIFTRQki:%[0-9]+]]:vk64 = KSHIFTRQki [[KMOVQkm]], 2
+  ; CHECK:   [[KSHIFTLQki:%[0-9]+]]:vk64 = KSHIFTLQki [[KSHIFTRQki]], 1
+  ; CHECK:   [[KNOTQkk:%[0-9]+]]:vk64 = KNOTQkk [[KSHIFTLQki]]
+  ; CHECK:   [[KORQkk:%[0-9]+]]:vk64 = KORQkk [[KNOTQkk]], [[KSHIFTRQki]]
+  ; CHECK:   [[KANDQkk:%[0-9]+]]:vk64 = KANDQkk [[KORQkk]], [[KSHIFTLQki]]
+  ; CHECK:   [[KXORQkk:%[0-9]+]]:vk64 = KXORQkk [[KANDQkk]], [[KSHIFTRQki]]
+  ; CHECK:   [[KANDNQkk:%[0-9]+]]:vk64 = KANDNQkk [[KXORQkk]], [[KORQkk]]
+  ; CHECK:   [[KADDQkk:%[0-9]+]]:vk64 = KADDQkk [[KANDNQkk]], [[KXORQkk]]
+  ; CHECK:   [[COPY3:%[0-9]+]]:vk64wm = COPY [[KADDQkk]]
   ; CHECK:   [[VMOVDQU8Zrrk:%[0-9]+]]:vr512 = VMOVDQU8Zrrk [[COPY2]], killed [[COPY3]], [[COPY1]]
   ; CHECK:   VMOVDQA32Zmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVDQU8Zrrk]]
   ; CHECK: bb.1:
@@ -702,8 +702,8 @@ body:             |
     ; CHECK: [[COPY2:%[0-9]+]]:vr512 = COPY $zmm1
     ; CHECK: [[KMOVBkm:%[0-9]+]]:vk8 = KMOVBkm [[COPY]], 1, $noreg, 0, $noreg
     ; CHECK: [[COPY3:%[0-9]+]]:vk16 = COPY [[KMOVBkm]]
-    ; CHECK: [[KNOTWrr:%[0-9]+]]:vk16 = KNOTWrr [[COPY3]]
-    ; CHECK: [[COPY4:%[0-9]+]]:vk16wm = COPY [[KNOTWrr]]
+    ; CHECK: [[KNOTWkk:%[0-9]+]]:vk16 = KNOTWkk [[COPY3]]
+    ; CHECK: [[COPY4:%[0-9]+]]:vk16wm = COPY [[KNOTWkk]]
     ; CHECK: [[VMOVAPSZrrk:%[0-9]+]]:vr512 = VMOVAPSZrrk [[COPY2]], killed [[COPY4]], [[COPY1]]
     ; CHECK: VMOVAPSZmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVAPSZrrk]]
     ; CHECK: RET 0
@@ -774,8 +774,8 @@ body:             |
     ; CHECK: [[COPY3:%[0-9]+]]:vk32 = COPY [[KMOVBkm]]
     ; CHECK: [[KMOVWkm:%[0-9]+]]:vk16 = KMOVWkm [[COPY]], 1, $noreg, 0, $noreg
     ; CHECK: [[COPY4:%[0-9]+]]:vk32 = COPY [[KMOVWkm]]
-    ; CHECK: [[KADDDrr:%[0-9]+]]:vk32 = KADDDrr [[COPY3]], [[COPY4]]
-    ; CHECK: [[COPY5:%[0-9]+]]:vk64wm = COPY [[KADDDrr]]
+    ; CHECK: [[KADDDkk:%[0-9]+]]:vk32 = KADDDkk [[COPY3]], [[COPY4]]
+    ; CHECK: [[COPY5:%[0-9]+]]:vk64wm = COPY [[KADDDkk]]
     ; CHECK: [[VMOVDQU16Zrrk:%[0-9]+]]:vr512 = VMOVDQU16Zrrk [[COPY2]], killed [[COPY5]], [[COPY1]]
     ; CHECK: VMOVDQA32Zmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVDQU16Zrrk]]
     ; CHECK: RET 0
@@ -847,8 +847,8 @@ body:             |
     ; CHECK: [[COPY3:%[0-9]+]]:vk64 = COPY [[KMOVBkm]]
     ; CHECK: [[KMOVWkm:%[0-9]+]]:vk16 = KMOVWkm [[COPY]], 1, $noreg, 0, $noreg
     ; CHECK: [[COPY4:%[0-9]+]]:vk64 = COPY [[KMOVWkm]]
-    ; CHECK: [[KADDQrr:%[0-9]+]]:vk64 = KADDQrr [[COPY3]], [[COPY4]]
-    ; CHECK: [[COPY5:%[0-9]+]]:vk64wm = COPY [[KADDQrr]]
+    ; CHECK: [[KADDQkk:%[0-9]+]]:vk64 = KADDQkk [[COPY3]], [[COPY4]]
+    ; CHECK: [[COPY5:%[0-9]+]]:vk64wm = COPY [[KADDQkk]]
     ; CHECK: [[VMOVDQU8Zrrk:%[0-9]+]]:vr512 = VMOVDQU8Zrrk [[COPY2]], killed [[COPY5]], [[COPY1]]
     ; CHECK: VMOVDQA32Zmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVDQU8Zrrk]]
     ; CHECK: RET 0
diff --git a/llvm/test/CodeGen/X86/fsafdo_test1.ll b/llvm/test/CodeGen/X86/fsafdo_test1.ll
index 043d47e18e96df..e80a7f2f354f2d 100644
--- a/llvm/test/CodeGen/X86/fsafdo_test1.ll
+++ b/llvm/test/CodeGen/X86/fsafdo_test1.ll
@@ -4,9 +4,9 @@
 ; Check that fs-afdo discriminators are generated.
 ; V01: .loc    1 7 3 is_stmt 0 discriminator 2 # foo.c:7:3
 ; V01: .loc    1 9 5 is_stmt 1 discriminator 2 # foo.c:9:5
-; V0: .loc    1 9 5 is_stmt 1 discriminator 11266 # foo.c:9:5
+; V0: .loc    1 9 5 discriminator 11266 # foo.c:9:5
 ; V0: .loc    1 7 3 is_stmt 1 discriminator 11266 # foo.c:7:3
-; V1: .loc    1 9 5 is_stmt 1 discriminator 514 # foo.c:9:5
+; V1: .loc    1 9 5 discriminator 514 # foo.c:9:5
 ; V1: .loc    1 7 3 is_stmt 1 discriminator 258 # foo.c:7:3
 ; Check that variable __llvm_fs_discriminator__ is generated.
 ; V01: .type   __llvm_fs_discriminator__,@object # @__llvm_fs_discriminator__
diff --git a/llvm/test/CodeGen/X86/fsafdo_test4.ll b/llvm/test/CodeGen/X86/fsafdo_test4.ll
index c9b9ed5b9c919e..effc72b44ade80 100644
--- a/llvm/test/CodeGen/X86/fsafdo_test4.ll
+++ b/llvm/test/CodeGen/X86/fsafdo_test4.ll
@@ -7,8 +7,8 @@
 ; CHECK: .loc    1 0 3 # foo.c:0:3
 ; CHECK: .loc    1 9 5 is_stmt 1 discriminator 2 # foo.c:9:5
 ; CHECK: .loc    1 0 5 is_stmt 0 # :0:5
-; CHECK: .loc    1 9 5 is_stmt 1 discriminator 2 # foo.c:9:5
-; CHECK: .loc    1 0 5 is_stmt 0 # :0:5
+; CHECK: .loc    1 9 5 discriminator 2 # foo.c:9:5
+; CHECK: .loc    1 0 5 # :0:5
 ; CHECK: .loc	   1 7 3 is_stmt 1 discriminator 2 # foo.c:7:3
 ; CHECK: .loc    1 14 3 # foo.c:14:3
 ; Check that variable __llvm_fs_discriminator__ is NOT generated.
diff --git a/llvm/test/CodeGen/X86/masked_compressstore_isel.ll b/llvm/test/CodeGen/X86/masked_compressstore_isel.ll
index 2a557ac9b97b3a..b944712e4863d7 100644
--- a/llvm/test/CodeGen/X86/masked_compressstore_isel.ll
+++ b/llvm/test/CodeGen/X86/masked_compressstore_isel.ll
@@ -9,9 +9,9 @@ define void @_Z3fooiPiPs(<8 x i32> %gepload, <8 x i1> %0) #0 {
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vr128x = COPY $xmm1
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr256x = COPY $ymm0
   ; CHECK-NEXT:   [[VPSLLWZ128ri:%[0-9]+]]:vr128x = VPSLLWZ128ri [[COPY]], 15
-  ; CHECK-NEXT:   [[VPMOVW2MZ128rr:%[0-9]+]]:vk16wm = VPMOVW2MZ128rr killed [[VPSLLWZ128ri]]
+  ; CHECK-NEXT:   [[VPMOVW2MZ128kr:%[0-9]+]]:vk16wm = VPMOVW2MZ128kr killed [[VPSLLWZ128ri]]
   ; CHECK-NEXT:   [[VPMOVDWZ256rr:%[0-9]+]]:vr128x = VPMOVDWZ256rr [[COPY1]]
-  ; CHECK-NEXT:   VPCOMPRESSWZ128mrk $noreg, 1, $noreg, 0, $noreg, killed [[VPMOVW2MZ128rr]], killed [[VPMOVDWZ256rr]] :: (store unknown-size into `ptr null`, align 1)
+  ; CHECK-NEXT:   VPCOMPRESSWZ128mrk $noreg, 1, $noreg, 0, $noreg, killed [[VPMOVW2MZ128kr]], killed [[VPMOVDWZ256rr]] :: (store unknown-size into `ptr null`, align 1)
   ; CHECK-NEXT:   RET 0
 entry:
   %1 = trunc <8 x i32> %gepload to <8 x i16>
@@ -28,9 +28,9 @@ define void @_Z3foo2iPiPs(<8 x i32> %gepload, <8 x i1> %0) #0 {
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vr128x = COPY $xmm1
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr256x = COPY $ymm0
   ; CHECK-NEXT:   [[VPSLLWZ128ri:%[0-9]+]]:vr128x = VPSLLWZ128ri [[COPY]], 15
-  ; CHECK-NEXT:   [[VPMOVW2MZ128rr:%[0-9]+]]:vk16wm = VPMOVW2MZ128rr killed [[VPSLLWZ128ri]]
+  ; CHECK-NEXT:   [[VPMOVW2MZ128kr:%[0-9]+]]:vk16wm = VPMOVW2MZ128kr killed [[VPSLLWZ128ri]]
   ; CHECK-NEXT:   [[VPMOVDWZ256rr:%[0-9]+]]:vr128x = VPMOVDWZ256rr [[COPY1]]
-  ; CHECK-NEXT:   VPCOMPRESSWZ128mrk $noreg, 1, $noreg, 0, $noreg, killed [[VPMOVW2MZ128rr]], killed [[VPMOVDWZ256rr]] :: (store unknown-size into `ptr null`, align 16)
+  ; CHECK-NEXT:   VPCOMPRESSWZ128mrk $noreg, 1, $noreg, 0, $noreg, killed [[VPMOVW2MZ128kr]], killed [[VPMOVDWZ256rr]] :: (store unknown-size into `ptr null`, align 16)
   ; CHECK-NEXT:   RET 0
 entry:
   %1 = trunc <8 x i32> %gepload to <8 x i16>
diff --git a/llvm/test/CodeGen/X86/masked_expandload_isel.ll b/llvm/test/CodeGen/X86/masked_expandload_isel.ll
index b364625a1e6f15..a153def4a71a6c 100644
--- a/llvm/test/CodeGen/X86/masked_expandload_isel.ll
+++ b/llvm/test/CodeGen/X86/masked_expandload_isel.ll
@@ -9,8 +9,8 @@ define <8 x i16> @_Z3fooiPiPs(<8 x i16> %src, <8 x i1> %mask) #0 {
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vr128x = COPY $xmm1
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr128x = COPY $xmm0
   ; CHECK-NEXT:   [[VPSLLWZ128ri:%[0-9]+]]:vr128x = VPSLLWZ128ri [[COPY]], 15
-  ; CHECK-NEXT:   [[VPMOVW2MZ128rr:%[0-9]+]]:vk16wm = VPMOVW2MZ128rr killed [[VPSLLWZ128ri]]
-  ; CHECK-NEXT:   [[VPEXPANDWZ128rmk:%[0-9]+]]:vr128x = VPEXPANDWZ128rmk [[COPY1]], killed [[VPMOVW2MZ128rr]], $noreg, 1, $noreg, 0, $noreg :: (load unknown-size from `ptr null`, align 1)
+  ; CHECK-NEXT:   [[VPMOVW2MZ128kr:%[0-9]+]]:vk16wm = VPMOVW2MZ128kr killed [[VPSLLWZ128ri]]
+  ; CHECK-NEXT:   [[VPEXPANDWZ128rmk:%[0-9]+]]:vr128x = VPEXPANDWZ128rmk [[COPY1]], killed [[VPMOVW2MZ128kr]], $noreg, 1, $noreg, 0, $noreg :: (load unknown-size from `ptr null`, align 1)
   ; CHECK-NEXT:   $xmm0 = COPY [[VPEXPANDWZ128rmk]]
   ; CHECK-NEXT:   RET 0, $xmm0
 entry:
@@ -26,8 +26,8 @@ define <8 x i16> @_Z3foo2iPiPs(<8 x i16> %src, <8 x i1> %mask) #0 {
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vr128x = COPY $xmm1
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr128x = COPY $xmm0
   ; CHECK-NEXT:   [[VPSLLWZ128ri:%[0-9]+]]:vr128x = VPSLLWZ128ri [[COPY]], 15
-  ; CHECK-NEXT:   [[VPMOVW2MZ128rr:%[0-9]+]]:vk16wm = VPMOVW2MZ128rr killed [[VPSLLWZ128ri]]
-  ; CHECK-NEXT:   [[VPEXPANDWZ128rmk:%[0-9]+]]:vr128x = VPEXPANDWZ128rmk [[COPY1]], killed [[VPMOVW2MZ128rr]], $noreg, 1, $noreg, 0, $noreg :: (load unknown-size from `ptr null`, align 16)
+  ; CHECK-NEXT:   [[VPMOVW2MZ128kr:%[0-9]+]]:vk16wm = VPMOVW2MZ128kr killed [[VPSLLWZ128ri]]
+  ; CHECK-NEXT:   [[VPEXPANDWZ128rmk:%[0-9]+]]:vr128x = VPEXPANDWZ128rmk [[COPY1]], killed [[VPMOVW2MZ128kr]], $noreg, 1, $noreg, 0, $noreg :: (load unknown-size from `ptr null`, align 16)
   ; CHECK-NEXT:   $xmm0 = COPY [[VPEXPANDWZ128rmk]]
   ; CHECK-NEXT:   RET 0, $xmm0
 entry:
diff --git a/llvm/test/CodeGen/X86/pseudo_cmov_lower2.ll b/llvm/test/CodeGen/X86/pseudo_cmov_lower2.ll
index 19253d67c14945..ee3d31bb73dca4 100644
--- a/llvm/test/CodeGen/X86/pseudo_cmov_lower2.ll
+++ b/llvm/test/CodeGen/X86/pseudo_cmov_lower2.ll
@@ -198,9 +198,6 @@ declare void @llvm.dbg.value(metadata, metadata, metadata)
 ; Like the test for @foo1, but check that the inserted dbg.value does not
 ; affect codegen. The CHECK items below should always be identical to @foo1,
 ; minus the DEBUG_VALUE line and changes in labels..
-; We produce a scope-line source location for the entry block, and then
-; explicitly terminate it in the second block, as there are no other source
-; locations in the function.
 define double @foo1_g(float %p1, double %p2, double %p3) nounwind !dbg !4 {
 ; CHECK-LABEL: foo1_g:
 ; CHECK:       .file   1 "." "test.c"
@@ -211,7 +208,6 @@ define double @foo1_g(float %p1, double %p2, double %p3) nounwind !dbg !4 {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.25E+0,0.0E+0]
 ; CHECK-NEXT:    jae .LBB6_1
 ; CHECK-NEXT:  # %bb.2: # %entry
-; CHECK-NEXT:  .loc    1 0 0 is_stmt 0
 ; CHECK-NEXT:    addsd %xmm2, %xmm0
 ; CHECK-NEXT:    jmp .LBB6_3
 ; CHECK-NEXT:  .LBB6_1:
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
index 130ae31b37bfe9..1391e8e86869a7 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
@@ -13825,11 +13825,11 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,1,2,11,4,5,14,7,8,9,26,19,12,29,22,15]
 ; AVX512BW-FCP-NEXT:    vpermw %zmm26, %zmm1, %zmm18
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,17,10,3,4,21,14,7,24,9,2,11,28,13,6,31]
-; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm1, %zmm9
+; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm1, %zmm10
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,1,10,3,4,5,14,7,8,25,18,11,12,29,22,15]
-; AVX512BW-FCP-NEXT:    vpermw %zmm26, %zmm1, %zmm6
+; AVX512BW-FCP-NEXT:    vpermw %zmm26, %zmm1, %zmm4
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [16,17,10,3,20,13,6,23,24,25,18,27,28,21,30,31]
-; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm1, %zmm4
+; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm1, %zmm5
 ; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %ymm12
 ; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdi), %ymm3
 ; AVX512BW-FCP-NEXT:    movw $-28382, %ax # imm = 0x9122
@@ -13837,58 +13837,58 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512BW-FCP-NEXT:    vpblendmw %ymm3, %ymm12, %ymm1 {%k1}
 ; AVX512BW-FCP-NEXT:    kmovq %k1, %k2
 ; AVX512BW-FCP-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm5
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm5[5,12],zero,zero,xmm5[1,8,15,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm6
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm6[5,12],zero,zero,xmm6[1,8,15,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,7,14],zero,zero,xmm1[3,10],zero,zero,zero,xmm1[u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpor %xmm5, %xmm1, %xmm1
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,6,13,4,11,2,9,16,23,30,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpor %xmm6, %xmm1, %xmm1
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,u,u,6,13,4,11,2,9,16,23,30,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    movw $992, %ax # imm = 0x3E0
 ; AVX512BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm4, %ymm1 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm5, %ymm1 {%k1}
 ; AVX512BW-FCP-NEXT:    vmovdqa 128(%rdi), %ymm13
 ; AVX512BW-FCP-NEXT:    vmovdqa 160(%rdi), %ymm11
 ; AVX512BW-FCP-NEXT:    movw $8772, %ax # imm = 0x2244
 ; AVX512BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm13, %ymm11, %ymm4 {%k1}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm13, %ymm11, %ymm5 {%k1}
 ; AVX512BW-FCP-NEXT:    kmovq %k1, %k3
 ; AVX512BW-FCP-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm4, %xmm5
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u],zero,zero,xmm5[3,10],zero,zero,zero,xmm5[6,13,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,5,12],zero,zero,xmm4[1,8,15],zero,zero,xmm4[u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpor %xmm5, %xmm4, %xmm4
-; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm5 = [0,0,0,0,1,2,4,6]
+; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm5, %xmm6
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u],zero,zero,xmm6[3,10],zero,zero,zero,xmm6[6,13,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,5,12],zero,zero,xmm5[1,8,15],zero,zero,xmm5[u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpor %xmm6, %xmm5, %xmm5
+; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm6 = [0,0,0,0,1,2,4,6]
 ; AVX512BW-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm19
-; AVX512BW-FCP-NEXT:    vpermd %ymm19, %ymm5, %ymm5
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
-; AVX512BW-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5,6],ymm5[7]
-; AVX512BW-FCP-NEXT:    vmovdqa 240(%rdi), %xmm7
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm7[5,12,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpermd %ymm19, %ymm6, %ymm6
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
+; AVX512BW-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm6[7]
+; AVX512BW-FCP-NEXT:    vmovdqa 240(%rdi), %xmm6
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = zero,zero,zero,xmm6[5,12,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vmovdqa 224(%rdi), %xmm8
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm8[0,7,14],zero,zero,xmm8[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpor %xmm5, %xmm10, %xmm5
-; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm5, %zmm4, %zmm4
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm8[0,7,14],zero,zero,xmm8[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpor %xmm7, %xmm9, %xmm7
+; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm7, %zmm5, %zmm5
 ; AVX512BW-FCP-NEXT:    movabsq $137438429184, %rax # imm = 0x1FFFF80000
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k5
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm4, %zmm1 {%k5}
-; AVX512BW-FCP-NEXT:    vmovdqa 288(%rdi), %ymm5
-; AVX512BW-FCP-NEXT:    vmovdqa 256(%rdi), %ymm4
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm5, %zmm1 {%k5}
+; AVX512BW-FCP-NEXT:    vmovdqa 288(%rdi), %ymm7
+; AVX512BW-FCP-NEXT:    vmovdqa 256(%rdi), %ymm5
 ; AVX512BW-FCP-NEXT:    movw $9288, %ax # imm = 0x2448
 ; AVX512BW-FCP-NEXT:    kmovd %eax, %k6
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm5, %ymm4, %ymm10 {%k6}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm10[u,u,u,u,u,3,10],zero,zero,zero,xmm10[6,13],zero,zero,xmm10[u,u]
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm10, %xmm10
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[u,u,u,u,u],zero,zero,xmm10[1,8,15],zero,zero,xmm10[4,11,u,u]
-; AVX512BW-FCP-NEXT:    vporq %xmm20, %xmm10, %xmm21
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,2,9,16,23,30,21,28,19,26,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm7, %ymm5, %ymm9 {%k6}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm9[u,u,u,u,u,3,10],zero,zero,zero,xmm9[6,13],zero,zero,xmm9[u,u]
+; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm9, %xmm9
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[u,u,u,u,u],zero,zero,xmm9[1,8,15],zero,zero,xmm9[4,11,u,u]
+; AVX512BW-FCP-NEXT:    vporq %xmm20, %xmm9, %xmm21
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,2,9,16,23,30,21,28,19,26,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    movw $3968, %ax # imm = 0xF80
 ; AVX512BW-FCP-NEXT:    kmovd %eax, %k7
-; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm6, %ymm21 {%k7}
-; AVX512BW-FCP-NEXT:    vmovdqa 416(%rdi), %ymm10
-; AVX512BW-FCP-NEXT:    vmovdqa 384(%rdi), %ymm6
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm4, %ymm21 {%k7}
+; AVX512BW-FCP-NEXT:    vmovdqa 416(%rdi), %ymm9
+; AVX512BW-FCP-NEXT:    vmovdqa 384(%rdi), %ymm4
 ; AVX512BW-FCP-NEXT:    movw $4644, %ax # imm = 0x1224
 ; AVX512BW-FCP-NEXT:    kmovd %eax, %k4
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm10, %ymm6, %ymm20 {%k4}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm9, %ymm4, %ymm20 {%k4}
 ; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm20, %xmm22
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm22 = xmm22[u,u,u,u,u,u,u],zero,zero,zero,xmm22[6,13],zero,zero,xmm22[2,9]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm20[u,u,u,u,u,u,u,1,8,15],zero,zero,xmm20[4,11],zero,zero
@@ -13900,10 +13900,10 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm23 = zero,zero,zero,xmm23[6,13],zero,zero,xmm23[2,9,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm20[1,8,15],zero,zero,xmm20[4,11],zero,zero,xmm20[u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vporq %xmm23, %xmm20, %xmm20
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,0,7,14,5,12,3,10,17,24,31,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,0,7,14,5,12,3,10,17,24,31,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    movl $511, %r10d # imm = 0x1FF
 ; AVX512BW-FCP-NEXT:    kmovd %r10d, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm20, %ymm9 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm20, %ymm10 {%k1}
 ; AVX512BW-FCP-NEXT:    vpblendmw %ymm13, %ymm11, %ymm20 {%k6}
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm23 = xmm20[u,u,u,6,13],zero,zero,xmm20[2,9],zero,zero,zero,xmm20[u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm20, %xmm20
@@ -13914,11 +13914,11 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512BW-FCP-NEXT:    vpermd %ymm19, %ymm20, %ymm20
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm14 = ymm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,20,27,30]
 ; AVX512BW-FCP-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4,5,6],ymm14[7]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = zero,zero,zero,xmm7[6,13,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = zero,zero,zero,xmm6[6,13,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm8[1,8,15],zero,zero,xmm8[u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vporq %xmm15, %xmm20, %xmm15
 ; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm15, %zmm14, %zmm14
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm9 {%k5}
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm10 {%k5}
 ; AVX512BW-FCP-NEXT:    vpblendmw %ymm3, %ymm12, %ymm14 {%k3}
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm14[2,9],zero,zero,zero,xmm14[5,12],zero,zero,xmm14[u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm14, %xmm14
@@ -13938,7 +13938,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm19[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,21,24,31]
 ; AVX512BW-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm14[0,1,2,3,4,5,6],ymm0[7]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm8[2,9],zero,zero,zero,xmm8[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = zero,zero,xmm7[0,7,14,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = zero,zero,xmm6[0,7,14,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vporq %xmm14, %xmm19, %xmm14
 ; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm14, %zmm0, %zmm14
 ; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm15, %zmm14 {%k1}
@@ -13965,7 +13965,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512BW-FCP-NEXT:    kmovd %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm23, %ymm15 {%k2}
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm23 = xmm8[3,10],zero,zero,zero,xmm8[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm24 = zero,zero,xmm7[1,8,15,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm24 = zero,zero,xmm6[1,8,15,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vporq %xmm23, %xmm24, %xmm23
 ; AVX512BW-FCP-NEXT:    vinserti32x4 $2, %xmm23, %zmm15, %zmm15
 ; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm0, %zmm15 {%k1}
@@ -13975,7 +13975,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512BW-FCP-NEXT:    movabsq $-137438953472, %rax # imm = 0xFFFFFFE000000000
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k2}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm4, %ymm5, %ymm0 {%k4}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm5, %ymm7, %ymm0 {%k4}
 ; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm0, %xmm21
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm21[u,u,u,u,u],zero,zero,xmm21[2,9],zero,zero,zero,xmm21[5,12,u,u]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,4,11],zero,zero,xmm0[0,7,14],zero,zero,xmm0[u,u]
@@ -13983,7 +13983,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm18 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,3,10,17,24,31,22,29,20,27,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm18, %ymm0 {%k7}
 ; AVX512BW-FCP-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm10, %ymm6, %ymm18 {%k1}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm9, %ymm4, %ymm18 {%k1}
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm18[u,u,u,u,u,u,u,2,9],zero,zero,zero,xmm18[5,12],zero,zero
 ; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm18
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,u,u,u],zero,zero,xmm18[0,7,14],zero,zero,xmm18[3,10]
@@ -13991,15 +13991,15 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm18, %ymm0, %ymm18
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm18, %ymm0 {%k3}
 ; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm9 {%k2}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm4, %ymm5, %ymm0 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm10 {%k2}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm5, %ymm7, %ymm0 {%k1}
 ; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm0, %xmm18
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,u],zero,zero,xmm18[3,10],zero,zero,zero,xmm18[6,13,u,u]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,5,12],zero,zero,xmm0[1,8,15],zero,zero,xmm0[u,u]
 ; AVX512BW-FCP-NEXT:    vporq %xmm18, %xmm0, %xmm0
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm17 = ymm17[u,u,u,u,u,u,u,u,u,u,u,u,u,u,4,11,18,25,16,23,30,21,28,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm17, %ymm0 {%k7}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm10, %ymm6, %ymm17 {%k6}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm9, %ymm4, %ymm17 {%k6}
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm17[u,u,u,u,u,u,u,3,10],zero,zero,zero,xmm17[6,13],zero,zero
 ; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm17, %xmm17
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm17[u,u,u,u,u,u,u],zero,zero,xmm17[1,8,15],zero,zero,xmm17[4,11]
@@ -14008,14 +14008,14 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm17, %ymm0 {%k3}
 ; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm14 {%k2}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm4, %ymm5, %ymm0 {%k6}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm5, %ymm7, %ymm0 {%k6}
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm0[u,u,u,u,u,6,13],zero,zero,xmm0[2,9],zero,zero,zero,xmm0[u,u]
 ; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u],zero,zero,xmm0[4,11],zero,zero,xmm0[0,7,14,u,u]
 ; AVX512BW-FCP-NEXT:    vporq %xmm17, %xmm0, %xmm0
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm16 = ymm16[u,u,u,u,u,u,u,u,u,u,u,u,u,u,5,12,19,26,17,24,31,22,29,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm16, %ymm0 {%k7}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm6, %ymm10, %ymm16 {%k4}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm4, %ymm9, %ymm16 {%k4}
 ; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm16, %xmm17
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm17[u,u,u,u,u,u,u],zero,zero,xmm17[2,9],zero,zero,zero,xmm17[5,12]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[u,u,u,u,u,u,u,4,11],zero,zero,xmm16[0,7,14],zero,zero
@@ -14029,14 +14029,14 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm0 = [16,25,18,3,28,21,6,23,24,17,10,27,20,13,30,31]
 ; AVX512BW-FCP-NEXT:    vpermw %zmm26, %zmm0, %zmm0
 ; AVX512BW-FCP-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm5, %ymm4, %ymm16 {%k2}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm7, %ymm5, %ymm16 {%k2}
 ; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm16, %xmm18
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u],zero,zero,zero,xmm18[5,12],zero,zero,xmm18[1,8,15,u,u]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[u,u,u,u,0,7,14],zero,zero,xmm16[3,10],zero,zero,zero,xmm16[u,u]
 ; AVX512BW-FCP-NEXT:    vporq %xmm18, %xmm16, %xmm16
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm0, %ymm16 {%k7}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm5, %ymm4, %ymm0 {%k4}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm7, %ymm5, %ymm0 {%k4}
 ; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm0, %xmm18
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u],zero,zero,zero,xmm18[6,13],zero,zero,xmm18[2,9,u,u,u]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,1,8,15],zero,zero,xmm0[4,11],zero,zero,xmm0[u,u,u]
@@ -14046,14 +14046,14 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512BW-FCP-NEXT:    kmovd %eax, %k1
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm0, %ymm17 {%k1}
 ; AVX512BW-FCP-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm6, %ymm10, %ymm0 {%k1}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm4, %ymm9, %ymm0 {%k1}
 ; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm0, %xmm18
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,u,u,u],zero,zero,xmm18[3,10],zero,zero,zero,xmm18[6,13]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,5,12],zero,zero,xmm0[1,8,15],zero,zero
 ; AVX512BW-FCP-NEXT:    vporq %xmm18, %xmm0, %xmm0
 ; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm0, %ymm16 {%k3}
-; AVX512BW-FCP-NEXT:    vpblendmw %ymm6, %ymm10, %ymm0 {%k6}
+; AVX512BW-FCP-NEXT:    vpblendmw %ymm4, %ymm9, %ymm0 {%k6}
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm0[u,u,u,u,u,u,u,6,13],zero,zero,xmm0[2,9],zero,zero,zero
 ; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u],zero,zero,xmm0[4,11],zero,zero,xmm0[0,7,14]
@@ -14091,84 +14091,83 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm21[u,u],zero,zero,xmm21[2,9],zero,zero,zero,xmm21[5,12,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm11[u,u,4,11],zero,zero,xmm11[0,7,14],zero,zero,xmm11[u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vporq %xmm21, %xmm11, %xmm11
-; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm11, %ymm0, %ymm21
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm20[u,u,u,u,u,u,u,u,u,u,u,3,10],zero,zero,zero
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm19[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm19[1,8,15]
-; AVX512BW-FCP-NEXT:    vporq %xmm11, %xmm19, %xmm11
 ; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
-; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm11, %ymm21 {%k3}
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [16,9,2,19,20,13,6,23,24,17,26,27,28,21,30,31]
-; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm11, %zmm19
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [16,9,2,19,12,5,22,23,24,17,26,27,20,29,30,31]
-; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm11, %zmm20
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [8,1,2,19,12,5,22,15,0,9,26,11,4,29,14,7]
-; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm11, %zmm11
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm20[u,u,u,u,u,u,u,u,u,u,u,3,10],zero,zero,zero
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm19[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm19[1,8,15]
+; AVX512BW-FCP-NEXT:    vporq %xmm20, %xmm19, %xmm19
+; AVX512BW-FCP-NEXT:    vinserti32x4 $1, %xmm19, %ymm0, %ymm19
+; AVX512BW-FCP-NEXT:    vmovdqu8 %ymm19, %ymm11 {%k3}
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm19 = [16,9,2,19,20,13,6,23,24,17,26,27,28,21,30,31]
+; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm19, %zmm19
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm20 = [16,9,2,19,12,5,22,23,24,17,26,27,20,29,30,31]
+; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm20, %zmm20
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm21 = [8,1,2,19,12,5,22,15,0,9,26,11,4,29,14,7]
+; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm21, %zmm21
 ; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm2[2,9],zero,zero,zero,xmm2[5,12,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,11],zero,zero,xmm0[0,7,14],zero,zero,xmm0[u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vpor %xmm2, %xmm0, %xmm2
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 {%k5} = ymm11[u,u,u,u,u,u,u,u,u,3,10,1,8,15,6,13,20,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 {%k5} = ymm21[u,u,u,u,u,u,u,u,u,3,10,1,8,15,6,13,20,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [4,11,4,11,4,11,4,11,4,11,4,11,4,11,4,11]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm0, %xmm8, %xmm11
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm22 = xmm7[2,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm22[0],xmm11[1],xmm22[1],xmm11[2],xmm22[2],xmm11[3],xmm22[3]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm12, %zmm11
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm11, %zmm2 {%k5}
-; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm11
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = zero,zero,xmm11[3,10],zero,zero,zero,xmm11[6,13,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm18[5,12],zero,zero,xmm18[1,8,15],zero,zero,xmm18[u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpor %xmm11, %xmm12, %xmm11
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm11 {%k5} = ymm20[u,u,u,u,u,u,u,u,u,4,11,2,9,0,7,14,21,28,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm7[3,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm8[5,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm18[0],xmm12[0],xmm18[1],xmm12[1],xmm18[2],xmm12[2],xmm18[3],xmm12[3]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm12
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm12, %zmm11 {%k5}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm3[6,13],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm0, %xmm8, %xmm21
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm22 = xmm6[2,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm21 = xmm21[0],xmm22[0],xmm21[1],xmm22[1],xmm21[2],xmm22[2],xmm21[3],xmm22[3]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm21, %zmm12, %zmm12
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm12, %zmm2 {%k5}
+; AVX512BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm12
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = zero,zero,xmm12[3,10],zero,zero,zero,xmm12[6,13,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[5,12],zero,zero,xmm18[1,8,15],zero,zero,xmm18[u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vporq %xmm12, %xmm18, %xmm12
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm12 {%k5} = ymm20[u,u,u,u,u,u,u,u,u,4,11,2,9,0,7,14,21,28,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm6[3,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm8[5,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm18 = xmm20[0],xmm18[0],xmm20[1],xmm18[1],xmm20[2],xmm18[2],xmm20[3],xmm18[3]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm18, %zmm13, %zmm13
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm13, %zmm12 {%k5}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = xmm3[6,13],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[u,u,u,u,u,u,u]
 ; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm3, %xmm3
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,xmm3[4,11],zero,zero,xmm3[0,7,14,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpor %xmm3, %xmm12, %xmm3
+; AVX512BW-FCP-NEXT:    vpor %xmm3, %xmm13, %xmm3
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 {%k5} = ymm19[u,u,u,u,u,u,u,u,u,5,12,3,10,1,8,15,22,29,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm0, %xmm7, %xmm0
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm8[6,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm21, %zmm0
+; AVX512BW-FCP-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm8[6,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm11, %zmm0
 ; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm0, %zmm3 {%k5}
-; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm10, %ymm6 {%k2}
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm0 = [0,1,10,3,4,13,6,7,8,25,18,11,28,21,14,15]
 ; AVX512BW-FCP-NEXT:    vpermw %zmm26, %zmm0, %zmm0
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm16, %zmm0, %zmm7
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm7, %ymm5 {%k1}
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm5[u,u,u,u,2,9],zero,zero,zero,xmm5[5,12],zero,zero,xmm5[u,u,u]
+; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm5, %xmm5
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u],zero,zero,xmm5[0,7,14],zero,zero,xmm5[3,10,u,u,u]
+; AVX512BW-FCP-NEXT:    vpor %xmm6, %xmm5, %xmm5
+; AVX512BW-FCP-NEXT:    movl $4186112, %eax # imm = 0x3FE000
+; AVX512BW-FCP-NEXT:    kmovd %eax, %k1
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm5 {%k1} = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,1,8,15,22,29,20,27,18,25,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm16, %zmm0, %zmm0
 ; AVX512BW-FCP-NEXT:    movw $-512, %ax # imm = 0xFE00
-; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm5, %ymm4 {%k1}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm4[u,u,u,u,2,9],zero,zero,zero,xmm4[5,12],zero,zero,xmm4[u,u,u]
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm4, %xmm4
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u],zero,zero,xmm4[0,7,14],zero,zero,xmm4[3,10,u,u,u]
-; AVX512BW-FCP-NEXT:    vpor %xmm5, %xmm4, %xmm4
-; AVX512BW-FCP-NEXT:    movl $4186112, %edi # imm = 0x3FE000
-; AVX512BW-FCP-NEXT:    kmovd %edi, %k1
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm4 {%k1} = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,1,8,15,22,29,20,27,18,25,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm9, %ymm4 {%k2}
 ; AVX512BW-FCP-NEXT:    kmovd %eax, %k1
-; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm7, %zmm2 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k1}
 ; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm0, %zmm0
-; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm11 {%k1}
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm6, %xmm0
+; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm12 {%k1}
+; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm4, %xmm0
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u],zero,zero,zero,xmm0[5,12],zero,zero,xmm0[1,8,15]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm6[u,u,u,u,u,u,0,7,14],zero,zero,xmm6[3,10],zero,zero,zero
-; AVX512BW-FCP-NEXT:    vpor %xmm0, %xmm5, %xmm0
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u,u,0,7,14],zero,zero,xmm4[3,10],zero,zero,zero
+; AVX512BW-FCP-NEXT:    vpor %xmm0, %xmm4, %xmm0
 ; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm4
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm5 = [16,17,18,19,20,21,22,23,24,25,26,43,44,45,46,47]
-; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm4, %zmm5
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm0
+; AVX512BW-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0,1,2],ymm0[3,4,5,6,7],ymm5[8,9,10],ymm0[11,12,13,14,15]
+; AVX512BW-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm3 {%k1}
 ; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rdi
 ; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm1, (%rsi)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm9, (%rdx)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm10, (%rdx)
 ; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm14, (%rcx)
 ; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm15, (%r8)
 ; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm2, (%r9)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm11, (%rdi)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm12, (%rdi)
 ; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, (%rax)
 ; AVX512BW-FCP-NEXT:    vzeroupper
 ; AVX512BW-FCP-NEXT:    retq
@@ -14555,9 +14554,9 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,1,2,11,4,5,14,7,8,9,26,19,12,29,22,15]
 ; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm26, %zmm1, %zmm18
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,17,10,3,4,21,14,7,24,9,2,11,28,13,6,31]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm2, %zmm1, %zmm9
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm2, %zmm1, %zmm10
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,1,10,3,4,5,14,7,8,25,18,11,12,29,22,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm26, %zmm1, %zmm6
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm26, %zmm1, %zmm5
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [16,17,10,3,20,13,6,23,24,25,18,27,28,21,30,31]
 ; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm2, %zmm1, %zmm4
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %ymm12
@@ -14567,10 +14566,10 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm3, %ymm12, %ymm1 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %k1, %k2
 ; AVX512DQ-BW-FCP-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm5
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm5[5,12],zero,zero,xmm5[1,8,15,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm6
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm6[5,12],zero,zero,xmm6[1,8,15,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,7,14],zero,zero,xmm1[3,10],zero,zero,zero,xmm1[u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm5, %xmm1, %xmm1
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm6, %xmm1, %xmm1
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,6,13,4,11,2,9,16,23,30,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    movw $992, %ax # imm = 0x3E0
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k1
@@ -14582,43 +14581,43 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm13, %ymm11, %ymm4 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %k1, %k3
 ; AVX512DQ-BW-FCP-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm4, %xmm5
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u],zero,zero,xmm5[3,10],zero,zero,zero,xmm5[6,13,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm4, %xmm6
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u],zero,zero,xmm6[3,10],zero,zero,zero,xmm6[6,13,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,5,12],zero,zero,xmm4[1,8,15],zero,zero,xmm4[u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm5, %xmm4, %xmm4
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm6, %xmm4, %xmm4
 ; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm5 = [0,0,0,0,1,2,4,6]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm6 = [0,0,0,0,1,2,4,6]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm19
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm19, %ymm5, %ymm5
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
-; AVX512DQ-BW-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5,6],ymm5[7]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 240(%rdi), %xmm7
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm7[5,12,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm19, %ymm6, %ymm6
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
+; AVX512DQ-BW-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5,6],ymm6[7]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 240(%rdi), %xmm6
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = zero,zero,zero,xmm6[5,12,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa 224(%rdi), %xmm8
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm8[0,7,14],zero,zero,xmm8[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm5, %xmm10, %xmm5
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm5, %zmm4, %zmm4
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm8[0,7,14],zero,zero,xmm8[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm7, %xmm9, %xmm7
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm7, %zmm4, %zmm4
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $137438429184, %rax # imm = 0x1FFFF80000
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k5
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm4, %zmm1 {%k5}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 288(%rdi), %ymm5
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 288(%rdi), %ymm7
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa 256(%rdi), %ymm4
 ; AVX512DQ-BW-FCP-NEXT:    movw $9288, %ax # imm = 0x2448
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k6
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm5, %ymm4, %ymm10 {%k6}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm10[u,u,u,u,u,3,10],zero,zero,zero,xmm10[6,13],zero,zero,xmm10[u,u]
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm10, %xmm10
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[u,u,u,u,u],zero,zero,xmm10[1,8,15],zero,zero,xmm10[4,11,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm20, %xmm10, %xmm21
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,2,9,16,23,30,21,28,19,26,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm7, %ymm4, %ymm9 {%k6}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm9[u,u,u,u,u,3,10],zero,zero,zero,xmm9[6,13],zero,zero,xmm9[u,u]
+; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm9, %xmm9
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[u,u,u,u,u],zero,zero,xmm9[1,8,15],zero,zero,xmm9[4,11,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm20, %xmm9, %xmm21
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,2,9,16,23,30,21,28,19,26,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    movw $3968, %ax # imm = 0xF80
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k7
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm6, %ymm21 {%k7}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 416(%rdi), %ymm10
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 384(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm5, %ymm21 {%k7}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 416(%rdi), %ymm9
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 384(%rdi), %ymm5
 ; AVX512DQ-BW-FCP-NEXT:    movw $4644, %ax # imm = 0x1224
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k4
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm10, %ymm6, %ymm20 {%k4}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm9, %ymm5, %ymm20 {%k4}
 ; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm20, %xmm22
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm22 = xmm22[u,u,u,u,u,u,u],zero,zero,zero,xmm22[6,13],zero,zero,xmm22[2,9]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm20[u,u,u,u,u,u,u,1,8,15],zero,zero,xmm20[4,11],zero,zero
@@ -14630,10 +14629,10 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm23 = zero,zero,zero,xmm23[6,13],zero,zero,xmm23[2,9,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm20[1,8,15],zero,zero,xmm20[4,11],zero,zero,xmm20[u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vporq %xmm23, %xmm20, %xmm20
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,0,7,14,5,12,3,10,17,24,31,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,0,7,14,5,12,3,10,17,24,31,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    movl $511, %r10d # imm = 0x1FF
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %r10d, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm20, %ymm9 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm20, %ymm10 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm13, %ymm11, %ymm20 {%k6}
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm23 = xmm20[u,u,u,6,13],zero,zero,xmm20[2,9],zero,zero,zero,xmm20[u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm20, %xmm20
@@ -14644,11 +14643,11 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm19, %ymm20, %ymm20
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm14 = ymm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,20,27,30]
 ; AVX512DQ-BW-FCP-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4,5,6],ymm14[7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = zero,zero,zero,xmm7[6,13,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = zero,zero,zero,xmm6[6,13,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm20 = xmm8[1,8,15],zero,zero,xmm8[u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vporq %xmm15, %xmm20, %xmm15
 ; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm15, %zmm14, %zmm14
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm9 {%k5}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm14, %zmm10 {%k5}
 ; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm3, %ymm12, %ymm14 {%k3}
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm14[2,9],zero,zero,zero,xmm14[5,12],zero,zero,xmm14[u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm14, %xmm14
@@ -14668,7 +14667,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm19[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,21,24,31]
 ; AVX512DQ-BW-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm14[0,1,2,3,4,5,6],ymm0[7]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm8[2,9],zero,zero,zero,xmm8[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = zero,zero,xmm7[0,7,14,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = zero,zero,xmm6[0,7,14,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vporq %xmm14, %xmm19, %xmm14
 ; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm14, %zmm0, %zmm14
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm15, %zmm14 {%k1}
@@ -14695,7 +14694,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm23, %ymm15 {%k2}
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm23 = xmm8[3,10],zero,zero,zero,xmm8[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm24 = zero,zero,xmm7[1,8,15,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm24 = zero,zero,xmm6[1,8,15,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vporq %xmm23, %xmm24, %xmm23
 ; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $2, %xmm23, %zmm15, %zmm15
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm0, %zmm15 {%k1}
@@ -14705,7 +14704,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $-137438953472, %rax # imm = 0xFFFFFFE000000000
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm4, %ymm5, %ymm0 {%k4}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm4, %ymm7, %ymm0 {%k4}
 ; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm0, %xmm21
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm21[u,u,u,u,u],zero,zero,xmm21[2,9],zero,zero,zero,xmm21[5,12,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,4,11],zero,zero,xmm0[0,7,14],zero,zero,xmm0[u,u]
@@ -14713,7 +14712,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm18 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,3,10,17,24,31,22,29,20,27,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm18, %ymm0 {%k7}
 ; AVX512DQ-BW-FCP-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm10, %ymm6, %ymm18 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm9, %ymm5, %ymm18 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm18[u,u,u,u,u,u,u,2,9],zero,zero,zero,xmm18[5,12],zero,zero
 ; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm18
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,u,u,u],zero,zero,xmm18[0,7,14],zero,zero,xmm18[3,10]
@@ -14721,15 +14720,15 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-BW-FCP-NEXT:    vinserti32x4 $1, %xmm18, %ymm0, %ymm18
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm18, %ymm0 {%k3}
 ; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm9 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm4, %ymm5, %ymm0 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm10 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm4, %ymm7, %ymm0 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm0, %xmm18
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,u],zero,zero,xmm18[3,10],zero,zero,zero,xmm18[6,13,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,5,12],zero,zero,xmm0[1,8,15],zero,zero,xmm0[u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vporq %xmm18, %xmm0, %xmm0
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm17 = ymm17[u,u,u,u,u,u,u,u,u,u,u,u,u,u,4,11,18,25,16,23,30,21,28,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm17, %ymm0 {%k7}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm10, %ymm6, %ymm17 {%k6}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm9, %ymm5, %ymm17 {%k6}
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm17[u,u,u,u,u,u,u,3,10],zero,zero,zero,xmm17[6,13],zero,zero
 ; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm17, %xmm17
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm17[u,u,u,u,u,u,u],zero,zero,xmm17[1,8,15],zero,zero,xmm17[4,11]
@@ -14738,14 +14737,14 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm17, %ymm0 {%k3}
 ; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm14 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm4, %ymm5, %ymm0 {%k6}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm4, %ymm7, %ymm0 {%k6}
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm0[u,u,u,u,u,6,13],zero,zero,xmm0[2,9],zero,zero,zero,xmm0[u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u],zero,zero,xmm0[4,11],zero,zero,xmm0[0,7,14,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vporq %xmm17, %xmm0, %xmm0
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm16 = ymm16[u,u,u,u,u,u,u,u,u,u,u,u,u,u,5,12,19,26,17,24,31,22,29,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm16, %ymm0 {%k7}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm6, %ymm10, %ymm16 {%k4}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm5, %ymm9, %ymm16 {%k4}
 ; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm16, %xmm17
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm17 = xmm17[u,u,u,u,u,u,u],zero,zero,xmm17[2,9],zero,zero,zero,xmm17[5,12]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[u,u,u,u,u,u,u,4,11],zero,zero,xmm16[0,7,14],zero,zero
@@ -14759,14 +14758,14 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm0 = [16,25,18,3,28,21,6,23,24,17,10,27,20,13,30,31]
 ; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm26, %zmm0, %zmm0
 ; AVX512DQ-BW-FCP-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm5, %ymm4, %ymm16 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm7, %ymm4, %ymm16 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm16, %xmm18
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u],zero,zero,zero,xmm18[5,12],zero,zero,xmm18[1,8,15,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm16 = xmm16[u,u,u,u,0,7,14],zero,zero,xmm16[3,10],zero,zero,zero,xmm16[u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vporq %xmm18, %xmm16, %xmm16
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm0, %ymm16 {%k7}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm5, %ymm4, %ymm0 {%k4}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm7, %ymm4, %ymm0 {%k4}
 ; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm0, %xmm18
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u],zero,zero,zero,xmm18[6,13],zero,zero,xmm18[2,9,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,1,8,15],zero,zero,xmm0[4,11],zero,zero,xmm0[u,u,u]
@@ -14776,14 +14775,14 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k1
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm0, %ymm17 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm6, %ymm10, %ymm0 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm5, %ymm9, %ymm0 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm0, %xmm18
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[u,u,u,u,u,u,u],zero,zero,xmm18[3,10],zero,zero,zero,xmm18[6,13]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,5,12],zero,zero,xmm0[1,8,15],zero,zero
 ; AVX512DQ-BW-FCP-NEXT:    vporq %xmm18, %xmm0, %xmm0
 ; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm0, %ymm16 {%k3}
-; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm6, %ymm10, %ymm0 {%k6}
+; AVX512DQ-BW-FCP-NEXT:    vpblendmw %ymm5, %ymm9, %ymm0 {%k6}
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm0[u,u,u,u,u,u,u,6,13],zero,zero,xmm0[2,9],zero,zero,zero
 ; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u],zero,zero,xmm0[4,11],zero,zero,xmm0[0,7,14]
@@ -14821,82 +14820,81 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[u,u],zero,zero,xmm13[2,9],zero,zero,zero,xmm13[5,12,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm11[u,u,4,11],zero,zero,xmm11[0,7,14],zero,zero,xmm11[u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpor %xmm13, %xmm11, %xmm11
-; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm13
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm20[u,u,u,u,u,u,u,u,u,u,u,3,10],zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm19[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm19[1,8,15]
-; AVX512DQ-BW-FCP-NEXT:    vporq %xmm11, %xmm19, %xmm11
 ; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm11, %ymm13 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [16,9,2,19,20,13,6,23,24,17,26,27,28,21,30,31]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm2, %zmm11, %zmm19
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [16,9,2,19,12,5,22,23,24,17,26,27,20,29,30,31]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm2, %zmm11, %zmm20
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [8,1,2,19,12,5,22,15,0,9,26,11,4,29,14,7]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm2, %zmm11, %zmm11
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = xmm20[u,u,u,u,u,u,u,u,u,u,u,3,10],zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm19[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm19[1,8,15]
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm13, %xmm19, %xmm13
+; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %ymm13, %ymm11 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm13 = [16,9,2,19,20,13,6,23,24,17,26,27,28,21,30,31]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm2, %zmm13, %zmm13
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm19 = [16,9,2,19,12,5,22,23,24,17,26,27,20,29,30,31]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm2, %zmm19, %zmm19
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm20 = [8,1,2,19,12,5,22,15,0,9,26,11,4,29,14,7]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm2, %zmm20, %zmm20
 ; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm21, %xmm2
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm2[2,9],zero,zero,zero,xmm2[5,12,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm21 = xmm21[4,11],zero,zero,xmm21[0,7,14],zero,zero,xmm21[u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vporq %xmm2, %xmm21, %xmm2
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 {%k5} = ymm11[u,u,u,u,u,u,u,u,u,3,10,1,8,15,6,13,20,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpbroadcastw {{.*#+}} xmm21 = [4,11,4,11,4,11,4,11,4,11,4,11,4,11,4,11]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm21, %xmm8, %xmm11
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm22 = xmm7[2,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm22[0],xmm11[1],xmm22[1],xmm11[2],xmm22[2],xmm11[3],xmm22[3]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm12, %zmm11
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm11, %zmm2 {%k5}
-; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm11
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = zero,zero,xmm11[3,10],zero,zero,zero,xmm11[6,13,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm18[5,12],zero,zero,xmm18[1,8,15],zero,zero,xmm18[u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm11, %xmm12, %xmm11
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm11 {%k5} = ymm20[u,u,u,u,u,u,u,u,u,4,11,2,9,0,7,14,21,28,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm7[3,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm8[5,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm18[0],xmm12[0],xmm18[1],xmm12[1],xmm18[2],xmm12[2],xmm18[3],xmm12[3]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm0, %zmm0
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm0, %zmm11 {%k5}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 {%k5} = ymm20[u,u,u,u,u,u,u,u,u,3,10,1,8,15,6,13,20,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpbroadcastw {{.*#+}} xmm20 = [4,11,4,11,4,11,4,11,4,11,4,11,4,11,4,11]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm20, %xmm8, %xmm21
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm22 = xmm6[2,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm21 = xmm21[0],xmm22[0],xmm21[1],xmm22[1],xmm21[2],xmm22[2],xmm21[3],xmm22[3]
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm21, %zmm12, %zmm12
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm12, %zmm2 {%k5}
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $1, %ymm18, %xmm12
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = zero,zero,xmm12[3,10],zero,zero,zero,xmm12[6,13,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm18[5,12],zero,zero,xmm18[1,8,15],zero,zero,xmm18[u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vporq %xmm12, %xmm18, %xmm12
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm12 {%k5} = ymm19[u,u,u,u,u,u,u,u,u,4,11,2,9,0,7,14,21,28,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm18 = xmm6[3,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm19 = xmm8[5,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm18 = xmm19[0],xmm18[0],xmm19[1],xmm18[1],xmm19[2],xmm18[2],xmm19[3],xmm18[3]
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm18, %zmm0, %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm0, %zmm12 {%k5}
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm3[6,13],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm3, %xmm3
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,xmm3[4,11],zero,zero,xmm3[0,7,14,u,u,u,u,u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vpor %xmm0, %xmm3, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 {%k5} = ymm19[u,u,u,u,u,u,u,u,u,5,12,3,10,1,8,15,22,29,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm21, %xmm7, %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm8[6,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm13, %zmm3
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 {%k5} = ymm13[u,u,u,u,u,u,u,u,u,5,12,3,10,1,8,15,22,29,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm20, %xmm6, %xmm3
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm8[6,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm11, %zmm3
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm3, %zmm0 {%k5}
-; AVX512DQ-BW-FCP-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm10, %ymm6 {%k2}
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm3 = [0,1,10,3,4,13,6,7,8,25,18,11,28,21,14,15]
 ; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm26, %zmm3, %zmm3
-; AVX512DQ-BW-FCP-NEXT:    movw $-512, %ax # imm = 0xFE00
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm5, %ymm4 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm4[u,u,u,u,2,9],zero,zero,zero,xmm4[5,12],zero,zero,xmm4[u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm7, %ymm4 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm4[u,u,u,u,2,9],zero,zero,zero,xmm4[5,12],zero,zero,xmm4[u,u,u]
 ; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm4, %xmm4
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u],zero,zero,xmm4[0,7,14],zero,zero,xmm4[3,10,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm5, %xmm4, %xmm4
-; AVX512DQ-BW-FCP-NEXT:    movl $4186112, %edi # imm = 0x3FE000
-; AVX512DQ-BW-FCP-NEXT:    kmovd %edi, %k1
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm6, %xmm4, %xmm4
+; AVX512DQ-BW-FCP-NEXT:    movl $4186112, %eax # imm = 0x3FE000
+; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k1
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm4 {%k1} = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,1,8,15,22,29,20,27,18,25,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    movw $-512, %ax # imm = 0xFE00
+; AVX512DQ-BW-FCP-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm9, %ymm5 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k1
 ; AVX512DQ-BW-FCP-NEXT:    vinserti32x8 $1, %ymm16, %zmm0, %zmm2 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x8 $1, %ymm17, %zmm0, %zmm11 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm6, %xmm3
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x8 $1, %ymm17, %zmm0, %zmm12 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm5, %xmm3
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u],zero,zero,zero,xmm3[5,12],zero,zero,xmm3[1,8,15]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm6[u,u,u,u,u,u,0,7,14],zero,zero,xmm6[3,10],zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u,u,0,7,14],zero,zero,xmm5[3,10],zero,zero,zero
 ; AVX512DQ-BW-FCP-NEXT:    vpor %xmm3, %xmm5, %xmm3
 ; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm4
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm5 = [16,17,18,19,20,21,22,23,24,25,26,43,44,45,46,47]
-; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm4, %zmm5
-; AVX512DQ-BW-FCP-NEXT:    vinserti32x8 $1, %ymm5, %zmm0, %zmm0 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
+; AVX512DQ-BW-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vinserti32x8 $1, %ymm3, %zmm0, %zmm0 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rdi
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm1, (%rsi)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm9, (%rdx)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm10, (%rdx)
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm14, (%rcx)
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm15, (%r8)
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm2, (%r9)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm11, (%rdi)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm12, (%rdi)
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, (%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vzeroupper
 ; AVX512DQ-BW-FCP-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll b/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll
index 32d95540b036ab..4898ae98faea2d 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll
@@ -878,7 +878,7 @@ define i16 @test_v4i16_v4i8(<4 x i16> %a0) {
 ;
 ; AVX1-SLOW-LABEL: test_v4i16_v4i8:
 ; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vpmulhuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVX1-SLOW-NEXT:    vpmulhuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 # [u,32768,16384,8192,u,u,u,u]
 ; AVX1-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-SLOW-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -890,7 +890,7 @@ define i16 @test_v4i16_v4i8(<4 x i16> %a0) {
 ;
 ; AVX1-FAST-LABEL: test_v4i16_v4i8:
 ; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vpmulhuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVX1-FAST-NEXT:    vpmulhuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 # [u,32768,16384,8192,u,u,u,u]
 ; AVX1-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-FAST-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
@@ -901,7 +901,7 @@ define i16 @test_v4i16_v4i8(<4 x i16> %a0) {
 ;
 ; AVX2-LABEL: test_v4i16_v4i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmulhuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVX2-NEXT:    vpmulhuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 # [u,32768,16384,8192,u,u,u,u]
 ; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll
index 53de286cc5cf12..f7c29cba30bd50 100644
--- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll
+++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll
@@ -4754,10 +4754,7 @@ define void @vec384_v12i32_to_v6i64_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bi
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero
 ; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)
 ; AVX512BW-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
index 2ea01230ca02db..6fd1ba10f739b6 100644
--- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
@@ -2030,9 +2030,9 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
 ; AVX512BW-FAST-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
 ; AVX512BW-FAST:       # %bb.0:
 ; AVX512BW-FAST-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512BW-FAST-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [0,25,0,27,0,29,0,31]
+; AVX512BW-FAST-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [0,9,0,11,0,13,0,15]
 ; AVX512BW-FAST-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT:    vpermt2d %zmm0, %zmm1, %zmm0
+; AVX512BW-FAST-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
 ; AVX512BW-FAST-NEXT:    vmovdqa64 %zmm0, (%rcx)
 ; AVX512BW-FAST-NEXT:    vzeroupper
diff --git a/llvm/test/DebugInfo/MIR/X86/empty-inline.mir b/llvm/test/DebugInfo/MIR/X86/empty-inline.mir
index 58775e8cd852fb..695b7c60365b1d 100644
--- a/llvm/test/DebugInfo/MIR/X86/empty-inline.mir
+++ b/llvm/test/DebugInfo/MIR/X86/empty-inline.mir
@@ -14,9 +14,7 @@
 # CHECK: Address            Line   Column File   ISA Discriminator OpIndex Flags
 # CHECK-NEXT:                ---
 # CHECK-NEXT:                 25      0      1   0             0         0 is_stmt
-# CHECK-NEXT:                  0      0      1   0             0         0
 # CHECK-NEXT:                 29     28      1   0             0         0 is_stmt prologue_end
-# CHECK-NEXT:                 29     28      1   0             0         0 is_stmt
 # CHECK-NEXT:                 29     28      1   0             0         0 is_stmt end_sequence
 --- |
   source_filename = "t.ll"
diff --git a/llvm/test/DebugInfo/X86/discriminator.ll b/llvm/test/DebugInfo/X86/discriminator.ll
index 120d928b5676b0..ef89838acf0324 100644
--- a/llvm/test/DebugInfo/X86/discriminator.ll
+++ b/llvm/test/DebugInfo/X86/discriminator.ll
@@ -59,4 +59,4 @@ attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"=
 
 ; CHECK: Address            Line   Column File   ISA Discriminator OpIndex Flags
 ; CHECK: ------------------ ------ ------ ------ --- ------------- ------- -------------
-; CHECK: 0x000000000000000a      2      0      1   0            42       0 is_stmt{{$}}
+; CHECK: 0x000000000000000a      2      0      1   0            42       0 {{$}}
diff --git a/llvm/test/DebugInfo/X86/empty-line-info.ll b/llvm/test/DebugInfo/X86/empty-line-info.ll
index e726e5c882460c..232e1da1c901c0 100644
--- a/llvm/test/DebugInfo/X86/empty-line-info.ll
+++ b/llvm/test/DebugInfo/X86/empty-line-info.ll
@@ -19,9 +19,8 @@ entry:
 }
 
 ;; In a function with no source location, but multiple blocks, there will be
-;; an opening scope-line, but it'll be automagically terminated when we switch
-;; to a new block. Test for this behaviour, and preserve the unconditional
-;; branch by compiling -O0.
+;; an opening scope-line. Test for this behaviour, and preserve the
+;; unconditional branch by compiling -O0.
 
 ; UNOPT-LABEL: bar:
 ; UNOPT-NEXT:   .Lfunc_begin1:
@@ -30,7 +29,6 @@ entry:
 ; UNOPT-NEXT:    movq    %rdi, -8(%rsp)
 ; UNOPT-NEXT:    jmp     .LBB1_1
 ; UNOPT-LABEL: .LBB1_1:
-; UNOPT-NEXT:    .loc    1 0 0 is_stmt 0
 ; UNOPT-NEXT:    movq    -8(%rsp), %rax
 
 define dso_local noundef i32 @bar(ptr nocapture noundef writeonly %baz) local_unnamed_addr !dbg !20 {
diff --git a/llvm/test/DebugInfo/X86/is_stmt-at-block-start.ll b/llvm/test/DebugInfo/X86/is_stmt-at-block-start.ll
deleted file mode 100644
index f4e98512675d9d..00000000000000
--- a/llvm/test/DebugInfo/X86/is_stmt-at-block-start.ll
+++ /dev/null
@@ -1,37 +0,0 @@
-;; Checks that when an instruction at the start of a BasicBlock has the same
-;; DebugLoc as the instruction at the end of the previous BasicBlock, we add
-;; is_stmt to the new line, to ensure that we still step on it if we arrive from
-;; a BasicBlock other than the immediately preceding one.
-
-; RUN: %llc_dwarf -mtriple=x86_64-unknown-linux -O0 -filetype=obj < %s | llvm-dwarfdump --debug-line - | FileCheck %s
-
-; CHECK:      {{0x[0-9a-f]+}}     13      5 {{.+}} is_stmt
-; CHECK-NEXT: {{0x[0-9a-f]+}}     13      5 {{.+}} is_stmt
-
-define void @_Z1fi(i1 %cond) !dbg !21 {
-entry:
-  br i1 %cond, label %if.then2, label %if.else4
-
-if.then2:                                         ; preds = %entry
-  br label %if.end8, !dbg !24
-
-if.else4:                                         ; preds = %entry
-  %0 = load i32, ptr null, align 4, !dbg !24
-  %call5 = call i1 null(i32 %0)
-  ret void
-
-if.end8:                                          ; preds = %if.then2
-  ret void
-}
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!20}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 20.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "test.cpp", directory: "/home/gbtozers/dev/upstream-llvm")
-!20 = !{i32 2, !"Debug Info Version", i32 3}
-!21 = distinct !DISubprogram(name: "f", linkageName: "_Z1fi", scope: !1, file: !1, line: 7, type: !22, scopeLine: 7, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0)
-!22 = distinct !DISubroutineType(types: !23)
-!23 = !{null}
-!24 = !DILocation(line: 13, column: 5, scope: !25)
-!25 = distinct !DILexicalBlock(scope: !21, file: !1, line: 11, column: 27)
diff --git a/llvm/test/Demangle/ms-placeholder-return-type.test b/llvm/test/Demangle/ms-placeholder-return-type.test
new file mode 100644
index 00000000000000..18038e636c8d5a
--- /dev/null
+++ b/llvm/test/Demangle/ms-placeholder-return-type.test
@@ -0,0 +1,18 @@
+; RUN: llvm-undname < %s | FileCheck %s
+
+; CHECK-NOT: Invalid mangled name
+
+?TestNonTemplateAuto@@YA@XZ
+; CHECK: __cdecl TestNonTemplateAuto(void)
+
+??$AutoT@X@@YA?A_PXZ
+; CHECK: auto __cdecl AutoT<void>(void)
+
+??$AutoT@X@@YA?B_PXZ
+; CHECK: auto const __cdecl AutoT<void>(void)
+
+??$AutoT@X@@YA?A_TXZ
+; CHECK: decltype(auto) __cdecl AutoT<void>(void)
+
+??$AutoT@X@@YA?B_TXZ
+; CHECK: decltype(auto) const __cdecl AutoT<void>(void)
diff --git a/llvm/test/Instrumentation/sanitizers-naked.ll b/llvm/test/Instrumentation/sanitizers-naked.ll
new file mode 100644
index 00000000000000..7a2eee76e0aad5
--- /dev/null
+++ b/llvm/test/Instrumentation/sanitizers-naked.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=asan -S | FileCheck %s --check-prefixes=CHECK,CHECK-ASAN
+; RUN: opt < %s -passes=tsan -S | FileCheck %s --check-prefixes=CHECK,CHECK-TSAN
+; RUN: opt < %s -passes=msan -S | FileCheck %s --check-prefixes=CHECK,CHECK-MSAN
+; RUN: opt < %s -passes=hwasan -S | FileCheck %s --check-prefixes=CHECK,CHECK-HWASAN
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-control-flow -S | FileCheck %s --check-prefixes=CHECK,CHECK-SANCOV
+; RUN: opt < %s -passes='module(sanmd-module)' -sanitizer-metadata-atomics -S | FileCheck %s --check-prefixes=CHECK,CHECK-SANMD
+
+; REQUIRES: x86-registered-target
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @naked_function(ptr byval(i32) %p) naked {
+; CHECK-ASAN-LABEL: define void @naked_function(
+; CHECK-ASAN-SAME: ptr byval(i32) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-ASAN-NEXT:    call void asm sideeffect "nop", ""()
+; CHECK-ASAN-NEXT:    unreachable
+;
+; CHECK-TSAN-LABEL: define void @naked_function(
+; CHECK-TSAN-SAME: ptr byval(i32) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TSAN-NEXT:    call void asm sideeffect "nop", ""()
+; CHECK-TSAN-NEXT:    unreachable
+;
+; CHECK-MSAN-LABEL: define void @naked_function(
+; CHECK-MSAN-SAME: ptr byval(i32) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-MSAN-NEXT:    call void @llvm.donothing()
+; CHECK-MSAN-NEXT:    call void asm sideeffect "nop", ""()
+; CHECK-MSAN-NEXT:    unreachable
+;
+; CHECK-HWASAN-LABEL: define void @naked_function(
+; CHECK-HWASAN-SAME: ptr byval(i32) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-HWASAN-NEXT:    call void asm sideeffect "nop", ""()
+; CHECK-HWASAN-NEXT:    unreachable
+;
+; CHECK-SANCOV-LABEL: define void @naked_function(
+; CHECK-SANCOV-SAME: ptr byval(i32) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SANCOV-NEXT:    call void asm sideeffect "nop", ""()
+; CHECK-SANCOV-NEXT:    unreachable
+;
+; CHECK-SANMD-LABEL: define void @naked_function(
+; CHECK-SANMD-SAME: ptr byval(i32) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SANMD-NEXT:    call void asm sideeffect "nop", ""()
+; CHECK-SANMD-NEXT:    unreachable
+;
+  call void asm sideeffect "nop", ""()
+  unreachable
+}
+
+define void @naked_function_with_asan(ptr byval(i32) %p) sanitize_address naked {
+; CHECK-ASAN-LABEL: define void @naked_function_with_asan(
+; CHECK-ASAN-SAME: ptr byval(i32) [[P:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-ASAN-NEXT:    call void asm sideeffect "nop", ""()
+; CHECK-ASAN-NEXT:    unreachable
+;
+  call void asm sideeffect "nop", ""()
+  unreachable
+}
+
+define void @naked_function_with_tsan(ptr byval(i32) %p) sanitize_thread naked {
+; CHECK-TSAN-LABEL: define void @naked_function_with_tsan(
+; CHECK-TSAN-SAME: ptr byval(i32) [[P:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-TSAN-NEXT:    call void asm sideeffect "nop", ""()
+; CHECK-TSAN-NEXT:    unreachable
+;
+  call void asm sideeffect "nop", ""()
+  unreachable
+}
+
+define void @naked_function_with_msan(ptr byval(i32) %p) sanitize_memory naked {
+; CHECK-MSAN-LABEL: define void @naked_function_with_msan(
+; CHECK-MSAN-SAME: ptr byval(i32) [[P:%.*]]) #[[ATTR3:[0-9]+]] {
+; CHECK-MSAN-NEXT:    call void @llvm.donothing()
+; CHECK-MSAN-NEXT:    call void asm sideeffect "nop", ""()
+; CHECK-MSAN-NEXT:    unreachable
+;
+  call void asm sideeffect "nop", ""()
+  unreachable
+}
+
+define void @naked_function_with_hwasan(ptr byval(i32) %p) sanitize_hwaddress naked {
+; CHECK-HWASAN-LABEL: define void @naked_function_with_hwasan(
+; CHECK-HWASAN-SAME: ptr byval(i32) [[P:%.*]]) #[[ATTR4:[0-9]+]] personality ptr @__hwasan_personality_thunk {
+; CHECK-HWASAN-NEXT:    call void asm sideeffect "nop", ""()
+; CHECK-HWASAN-NEXT:    unreachable
+;
+  call void asm sideeffect "nop", ""()
+  unreachable
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/MC/Disassembler/WebAssembly/wasm.txt b/llvm/test/MC/Disassembler/WebAssembly/wasm.txt
index 0cbf584d9688e1..f6a3527fc5ed12 100644
--- a/llvm/test/MC/Disassembler/WebAssembly/wasm.txt
+++ b/llvm/test/MC/Disassembler/WebAssembly/wasm.txt
@@ -53,3 +53,10 @@
 # This can mean end_block/end_loop/end_if/end_function/end_try..
 # CHECK: end
 0x0B
+
+# CHECK:  try_table        (catch 0 0) (catch_ref 0 1) (catch_all 2) (catch_all_ref 3)
+0x1F 0x40 0x04 0x00 0x00 0x00 0x01 0x00 0x01 0x02 0x02 0x03 0x03
+# CHECK: throw 0
+0x08 0x00
+# CHECK: throw_ref
+0x0a
diff --git a/llvm/test/MC/WebAssembly/annotations.s b/llvm/test/MC/WebAssembly/annotations.s
index b1f97daccccd65..3e727591afa942 100644
--- a/llvm/test/MC/WebAssembly/annotations.s
+++ b/llvm/test/MC/WebAssembly/annotations.s
@@ -33,6 +33,22 @@ test_annotation:
       rethrow   0
     end_try
   end_try
+
+  block exnref
+    block
+      block () -> (i32, exnref)
+        block i32
+          try_table (catch __cpp_exception 0) (catch_ref __c_longjmp 1) (catch_all 2) (catch_all_ref 3)
+          end_try_table
+          return
+        end_block
+        return
+      end_block
+      return
+    end_block
+    return
+  end_block
+  drop
   end_function
 
 
@@ -61,5 +77,24 @@ test_annotation:
 # CHECK-NEXT:   rethrow   0               # to caller
 # CHECK-NEXT:   end_try                   # label3:
 # CHECK-NEXT:   end_try                   # label0:
+
+# CHECK:        block           exnref
+# CHECK-NEXT:   block
+# CHECK-NEXT:   block           () -> (i32, exnref)
+# CHECK-NEXT:   block           i32
+# CHECK-NEXT:   try_table        (catch __cpp_exception 0) (catch_ref __c_longjmp 1) (catch_all 2) (catch_all_ref 3) # 0: down to label10
+# CHECK-NEXT:                             # 1: down to label9
+# CHECK-NEXT:                             # 2: down to label8
+# CHECK-NEXT:                             # 3: down to label7
+# CHECK-NEXT:   end_try_table                           # label11:
+# CHECK-NEXT:   return
+# CHECK-NEXT:   end_block                               # label10:
+# CHECK-NEXT:   return
+# CHECK-NEXT:   end_block                               # label9:
+# CHECK-NEXT:   return
+# CHECK-NEXT:   end_block                               # label8:
+# CHECK-NEXT:   return
+# CHECK-NEXT:   end_block                               # label7:
+# CHECK-NEXT:   drop
 # CHECK-NEXT:   end_function
 
diff --git a/llvm/test/MC/WebAssembly/basic-assembly-errors.s b/llvm/test/MC/WebAssembly/basic-assembly-errors.s
index aab3b5b1a9028e..6699b66d04a150 100644
--- a/llvm/test/MC/WebAssembly/basic-assembly-errors.s
+++ b/llvm/test/MC/WebAssembly/basic-assembly-errors.s
@@ -26,11 +26,37 @@ test0:
     catch_all
 # CHECK: error: Block construct type mismatch, expected: end_try, instead got: catch_all
     end
-# CHECK: Block construct type mismatch, expected: end_try, instead got: end_function
+
+# CHECK: error: Expected integer constant, instead got: )
+    try_table (catch __cpp_exception)
+    end_try_table
+
+    block
+# CHECK: error: invalid operand for instruction
+    try_table (catch_all 0) i32
+    i32.const 0
+    end_try_table
+    drop
+    end_block
+
+    block
+# CHECK: error: Expected identifier, got: )
+    try_table (catch_all 0) () -> (i32, i32)
+    i32.const 0
+    i32.const 0
+    end_try_table
+    drop
+    drop
+    end_block
+
+# CHECK: error: unknown type: not_catch
+    try_table (not_catch 0)
+
+# CHECK: Block construct type mismatch, expected: end_try_table, instead got: end_function
+    end_function
+# CHECK: error: Unmatched block construct(s) at function end: try_table
 # CHECK: error: Unmatched block construct(s) at function end: catch_all
 # CHECK: error: Unmatched block construct(s) at function end: loop
 # CHECK: error: Unmatched block construct(s) at function end: try
 # CHECK: error: Unmatched block construct(s) at function end: block
 # CHECK: error: Unmatched block construct(s) at function end: function
-    end_function
-
diff --git a/llvm/test/MC/WebAssembly/eh-assembly-legacy.s b/llvm/test/MC/WebAssembly/eh-assembly-legacy.s
index cd33d198f8d9f5..deba6cc683f035 100644
--- a/llvm/test/MC/WebAssembly/eh-assembly-legacy.s
+++ b/llvm/test/MC/WebAssembly/eh-assembly-legacy.s
@@ -4,10 +4,11 @@
 
   .tagtype  __cpp_exception i32
   .tagtype  __c_longjmp i32
-  .functype  eh_legacy_test () -> ()
   .functype  foo () -> ()
 
 eh_legacy_test:
+  .functype  eh_legacy_test () -> ()
+
   # try-catch with catch, catch_all, throw, and rethrow
   try
     i32.const 3
@@ -55,7 +56,7 @@ eh_legacy_test:
   end_function
 
 # CHECK-LABEL: eh_legacy_test:
-# CHECK-NEXT:    try
+# CHECK:         try
 # CHECK-NEXT:    i32.const       3
 # CHECK-NEXT:    throw           __cpp_exception
 # CHECK-NEXT:    catch           __cpp_exception
diff --git a/llvm/test/MC/WebAssembly/eh-assembly.s b/llvm/test/MC/WebAssembly/eh-assembly.s
new file mode 100644
index 00000000000000..b4d6b324d96e3e
--- /dev/null
+++ b/llvm/test/MC/WebAssembly/eh-assembly.s
@@ -0,0 +1,157 @@
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -mattr=+exception-handling --no-type-check < %s | FileCheck %s
+# Check that it converts to .o without errors, but don't check any output:
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -filetype=obj -mattr=+exception-handling --no-type-check -o %t.o < %s
+
+  .tagtype  __cpp_exception i32
+  .tagtype  __c_longjmp i32
+  .functype  foo () -> ()
+
+eh_test:
+  .functype  eh_test () -> ()
+
+  # try_table with all four kinds of catch clauses
+  block exnref
+    block
+      block () -> (i32, exnref)
+        block i32
+          try_table (catch __cpp_exception 0) (catch_ref __c_longjmp 1) (catch_all 2) (catch_all_ref 3)
+            i32.const 0
+            throw     __cpp_exception
+          end_try_table
+          return
+        end_block
+        drop
+        return
+      end_block
+      throw_ref
+      drop
+    end_block
+    return
+  end_block
+  drop
+
+  # You can use the same kind of catch clause more than once
+  block
+    block exnref
+      block
+        try_table (catch_all 0) (catch_all_ref 1) (catch_all 2)
+          call  foo
+        end_try_table
+      end_block
+      return
+    end_block
+    drop
+  end_block
+
+  # Two catch clauses targeting the same block
+  block
+    try_table (catch_all 0) (catch_all 0)
+    end_try_table
+  end_block
+
+  # try_table with a return type
+  block
+    try_table f32 (catch_all 0)
+      f32.const 0.0
+    end_try_table
+    drop
+  end_block
+
+  # try_table with a multivalue type return
+  block
+    try_table () -> (i32, f32) (catch_all 0)
+      i32.const 0
+      f32.const 0.0
+    end_try_table
+    drop
+    drop
+  end_block
+
+  # catch-less try_tables
+  try_table
+    call  foo
+  end_try_table
+
+  try_table i32
+    i32.const 0
+  end_try_table
+  drop
+
+  try_table () -> (i32, f32)
+    i32.const 0
+    f32.const 0.0
+  end_try_table
+  drop
+  drop
+
+  end_function
+
+# CHECK-LABEL: eh_test:
+# CHECK:         block           exnref
+# CHECK-NEXT:    block
+# CHECK-NEXT:    block           () -> (i32, exnref)
+# CHECK-NEXT:    block           i32
+# CHECK-NEXT:    try_table        (catch __cpp_exception 0) (catch_ref __c_longjmp 1) (catch_all 2) (catch_all_ref 3)
+# CHECK:         i32.const       0
+# CHECK-NEXT:    throw           __cpp_exception
+# CHECK-NEXT:    end_try_table
+# CHECK-NEXT:    return
+# CHECK-NEXT:    end_block
+# CHECK-NEXT:    drop
+# CHECK-NEXT:    return
+# CHECK-NEXT:    end_block
+# CHECK-NEXT:    throw_ref
+# CHECK-NEXT:    drop
+# CHECK-NEXT:    end_block
+# CHECK-NEXT:    return
+# CHECK-NEXT:    end_block
+# CHECK-NEXT:    drop
+
+# CHECK:         block
+# CHECK-NEXT:    block           exnref
+# CHECK-NEXT:    block
+# CHECK-NEXT:    try_table        (catch_all 0) (catch_all_ref 1) (catch_all 2)
+# CHECK:         call    foo
+# CHECK-NEXT:    end_try_table
+# CHECK-NEXT:    end_block
+# CHECK-NEXT:    return
+# CHECK-NEXT:    end_block
+# CHECK-NEXT:    drop
+# CHECK-NEXT:    end_block
+
+# CHECK:         block
+# CHECK-NEXT:    try_table        (catch_all 0) (catch_all 0)
+# CHECK:         end_try_table
+# CHECK-NEXT:    end_block
+
+# CHECK:         block
+# CHECK-NEXT:    try_table       f32 (catch_all 0)
+# CHECK:         f32.const       0x0p0
+# CHECK-NEXT:    end_try_table
+# CHECK-NEXT:    drop
+# CHECK-NEXT:    end_block
+
+# CHECK:         block
+# CHECK-NEXT:    try_table       () -> (i32, f32) (catch_all 0)
+# CHECK:         i32.const       0
+# CHECK-NEXT:    f32.const       0x0p0
+# CHECK-NEXT:    end_try_table
+# CHECK-NEXT:    drop
+# CHECK-NEXT:    drop
+# CHECK-NEXT:    end_block
+
+# CHECK:         try_table
+# CHECK-NEXT:    call    foo
+# CHECK-NEXT:    end_try_table
+
+# CHECK:         try_table       i32
+# CHECK-NEXT:    i32.const       0
+# CHECK-NEXT:    end_try_table
+# CHECK-NEXT:    drop
+
+# CHECK:         try_table       () -> (i32, f32)
+# CHECK-NEXT:    i32.const       0
+# CHECK-NEXT:    f32.const       0x0p0
+# CHECK-NEXT:    end_try_table
+# CHECK-NEXT:    drop
+# CHECK-NEXT:    drop
diff --git a/llvm/test/MC/WebAssembly/func-end-errors.s b/llvm/test/MC/WebAssembly/func-end-errors.s
index dda91654c83c93..42d581fbccaa43 100644
--- a/llvm/test/MC/WebAssembly/func-end-errors.s
+++ b/llvm/test/MC/WebAssembly/func-end-errors.s
@@ -4,14 +4,14 @@
 # assembly. This causes the parser to properly wrap up function info when there
 # is no other instructions present.
 
-.functype test0 () -> ()
 test0:
+  .functype test0 () -> ()
 
-.functype test1 () -> ()
-# CHECK: [[@LINE+1]]:1: error: Unmatched block construct(s) at function end: function
 test1:
+# CHECK: [[@LINE+1]]:{{.*}}: error: Unmatched block construct(s) at function end: function
+  .functype test1 () -> ()
   end_function
 
-.functype test2 () -> ()
 test2:
-# CHECK: [[@LINE+1]]:1: error: Unmatched block construct(s) at function end: function
+  .functype test2 () -> ()
+# CHECK: [[@LINE+1]]:{{.*}}: error: Unmatched block construct(s) at function end: function
diff --git a/llvm/test/MC/WebAssembly/funcref-from-table.s b/llvm/test/MC/WebAssembly/funcref-from-table.s
index 62c817c8b76bca..3b6e0102698c3a 100644
--- a/llvm/test/MC/WebAssembly/funcref-from-table.s
+++ b/llvm/test/MC/WebAssembly/funcref-from-table.s
@@ -6,7 +6,7 @@
 .globl obtain_funcref_from_table_index
 
 obtain_funcref_from_table_index:
-  .functype obtain_funcref_from_table_index(i32) -> (funcref)
+  .functype obtain_funcref_from_table_index (i32) -> (funcref)
   local.get 0
   table.get __indirect_function_table
   end_function
diff --git a/llvm/test/MC/WebAssembly/type-checker-br.s b/llvm/test/MC/WebAssembly/type-checker-br.s
index df3f88f9685e1a..859e9c52547336 100644
--- a/llvm/test/MC/WebAssembly/type-checker-br.s
+++ b/llvm/test/MC/WebAssembly/type-checker-br.s
@@ -16,7 +16,7 @@ br_block:
   end_function
 
 br_func:
-  .functype br_block () -> ()
+  .functype br_func () -> ()
   block i32
     br 1
     i32.const 1
diff --git a/llvm/test/MachineVerifier/test_g_extract_subvector.mir b/llvm/test/MachineVerifier/test_g_extract_subvector.mir
index 5a441ff29c1721..6a0b7ebfb4b0b2 100644
--- a/llvm/test/MachineVerifier/test_g_extract_subvector.mir
+++ b/llvm/test/MachineVerifier/test_g_extract_subvector.mir
@@ -27,6 +27,34 @@ body:             |
     ; CHECK: Element type of vectors must be the same
     %8:_(<vscale x 2 x s32>) = G_EXTRACT_SUBVECTOR %7, 0
 
-    ; CHECK: Index must be a multiple of the source vector's minimum vector length
+    ; CHECK: Index must be a multiple of the destination vector's minimum vector length
     %9:_(<vscale x 4 x s32>) = G_EXTRACT_SUBVECTOR  %1, 3
+
+    ; CHECK: Index must be a multiple of the destination vector's minimum vector length
+    %10:_(<vscale x 4 x s32>) = G_EXTRACT_SUBVECTOR  %1, 2
+
+    ; CHECK: Source type and index must not cause extract to overrun to the destination type
+    %11:_(<vscale x 1 x s32>) = G_EXTRACT_SUBVECTOR  %1, 4
+
+    %12:_(<vscale x 4 x s32>) = G_IMPLICIT_DEF
+
+    ; CHECK: Source type and index must not cause extract to overrun to the destination type
+    %13:_(<vscale x 3 x s32>) = G_EXTRACT_SUBVECTOR  %12, 3
+
+    %14:_(<2 x s32>) = G_IMPLICIT_DEF
+    %15:_(<4 x s32>) = G_IMPLICIT_DEF
+
+    ; CHECK: Source type and index must not cause extract to overrun to the destination type
+    %16:_(<2 x s32>) = G_EXTRACT_SUBVECTOR  %14, 4
+
+    ; CHECK: Source type and index must not cause extract to overrun to the destination type
+    %17:_(<3 x s32>) = G_EXTRACT_SUBVECTOR  %15, 3
+
+    ; CHECK: Vector types must both be fixed or both be scalable
+    %18:_(<vscale x 2 x s32>) = G_EXTRACT_SUBVECTOR %15, 0
+
+    ; CHECK: Vector types must both be fixed or both be scalable
+    %19:_(<2 x s32>) = G_EXTRACT_SUBVECTOR %12, 0
+
+
 ...
diff --git a/llvm/test/MachineVerifier/test_g_insert_subvector.mir b/llvm/test/MachineVerifier/test_g_insert_subvector.mir
index 9fce3c3e842d40..62ddd28919b205 100644
--- a/llvm/test/MachineVerifier/test_g_insert_subvector.mir
+++ b/llvm/test/MachineVerifier/test_g_insert_subvector.mir
@@ -41,4 +41,7 @@ body:             |
 
     ; CHECK: Index must be a multiple of the second source vector's minimum vector length
     %13:_(<vscale x 4 x s32>) = G_INSERT_SUBVECTOR %12, %1, 3
+
+    ; CHECK: Index must be a multiple of the second source vector's minimum vector length
+    %13:_(<vscale x 4 x s32>) = G_INSERT_SUBVECTOR %12, %1, 1
 ...
diff --git a/llvm/test/ThinLTO/X86/ctxprof.ll b/llvm/test/ThinLTO/X86/ctxprof.ll
index 1e30b90ec23d58..4baea3b25890eb 100644
--- a/llvm/test/ThinLTO/X86/ctxprof.ll
+++ b/llvm/test/ThinLTO/X86/ctxprof.ll
@@ -47,18 +47,21 @@
 ; NOPROF-1-NOT: m2_f1()
 ; NOPROF-2-NOT: m1_f1()
 ;
-; The run with workload definitions - same other options.
+; The run with workload definitions - same other options. We do need to re-generate the .bc
+; files, to include instrumentation.
+; RUN: opt -module-summary -passes=assign-guid,ctx-instr-gen %t/m1.ll -o %t/m1-instr.bc
+; RUN: opt -module-summary -passes=assign-guid,ctx-instr-gen %t/m2.ll -o %t/m2-instr.bc
 ;
 ; RUN: echo '[ \
 ; RUN:        {"Guid": 6019442868614718803, "Counters": [1], "Callsites": [[{"Guid": 15593096274670919754, "Counters": [1]}]]}, \
 ; RUN:        {"Guid": 15593096274670919754, "Counters": [1], "Callsites": [[{"Guid": 6019442868614718803, "Counters": [1]}]]} \
 ; RUN:  ]' > %t_exp/ctxprof.json
 ; RUN: llvm-ctxprof-util fromJSON --input %t_exp/ctxprof.json --output %t_exp/ctxprof.bitstream
-; RUN: llvm-lto2 run %t/m1.bc %t/m2.bc \
+; RUN: llvm-lto2 run %t/m1-instr.bc %t/m2-instr.bc \
 ; RUN:  -o %t_exp/result.o -save-temps \
 ; RUN:  -use-ctx-profile=%t_exp/ctxprof.bitstream \
-; RUN:  -r %t/m1.bc,m1_f1,plx \
-; RUN:  -r %t/m2.bc,m2_f1,plx
+; RUN:  -r %t/m1-instr.bc,m1_f1,plx \
+; RUN:  -r %t/m2-instr.bc,m2_f1,plx
 ; RUN: llvm-dis %t_exp/result.o.1.3.import.bc -o - | FileCheck %s --check-prefix=FIRST
 ; RUN: llvm-dis %t_exp/result.o.2.3.import.bc -o - | FileCheck %s --check-prefix=SECOND
 ;
diff --git a/llvm/test/Transforms/InstCombine/assume-align.ll b/llvm/test/Transforms/InstCombine/assume-align.ll
index ce3195d50be7ca..2b8ca5d25fd1a8 100644
--- a/llvm/test/Transforms/InstCombine/assume-align.ll
+++ b/llvm/test/Transforms/InstCombine/assume-align.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals smart
 ; RUN: opt -S -passes=instcombine,simplifycfg < %s 2>&1 | FileCheck %s
 
 declare void @llvm.assume(i1 noundef)
@@ -122,3 +122,52 @@ define i8 @assume_align_non_pow2(ptr %p) {
   %v = load i8, ptr %p
   ret i8 %v
 }
+
+; TODO: Can fold alignment assumption into !align metadata on load.
+define ptr @fold_assume_align_pow2_of_loaded_pointer_into_align_metadata(ptr %p) {
+; CHECK-LABEL: @fold_assume_align_pow2_of_loaded_pointer_into_align_metadata(
+; CHECK-NEXT:    [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 8) ]
+; CHECK-NEXT:    ret ptr [[P2]]
+;
+  %p2 = load ptr, ptr %p
+  call void @llvm.assume(i1 true) [ "align"(ptr %p2, i64 8) ]
+  ret ptr %p2
+}
+
+define ptr @dont_fold_assume_align_pow2_of_loaded_pointer_into_align_metadata_due_to_call(ptr %p) {
+; CHECK-LABEL: @dont_fold_assume_align_pow2_of_loaded_pointer_into_align_metadata_due_to_call(
+; CHECK-NEXT:    [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
+; CHECK-NEXT:    call void @g(i64 0)
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 8) ]
+; CHECK-NEXT:    ret ptr [[P2]]
+;
+  %p2 = load ptr, ptr %p
+  call void @g(i64 0)
+  call void @llvm.assume(i1 true) [ "align"(ptr %p2, i64 8) ]
+  ret ptr %p2
+}
+
+; !align must have a power-of-2 alignment.
+define ptr @dont_fold_assume_align_non_pow2_of_loaded_pointer_into_align_metadata(ptr %p) {
+; CHECK-LABEL: @dont_fold_assume_align_non_pow2_of_loaded_pointer_into_align_metadata(
+; CHECK-NEXT:    [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 13) ]
+; CHECK-NEXT:    ret ptr [[P2]]
+;
+  %p2 = load ptr, ptr %p
+  call void @llvm.assume(i1 true) [ "align"(ptr %p2, i64 13) ]
+  ret ptr %p2
+}
+
+; !align must have a power-of-2 alignment.
+define ptr @dont_fold_assume_align_zero_of_loaded_pointer_into_align_metadata(ptr %p) {
+; CHECK-LABEL: @dont_fold_assume_align_zero_of_loaded_pointer_into_align_metadata(
+; CHECK-NEXT:    [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 0) ]
+; CHECK-NEXT:    ret ptr [[P2]]
+;
+  %p2 = load ptr, ptr %p
+  call void @llvm.assume(i1 true) [ "align"(ptr %p2, i64 0) ]
+  ret ptr %p2
+}
diff --git a/llvm/test/Transforms/InstCombine/bitcast.ll b/llvm/test/Transforms/InstCombine/bitcast.ll
index 4ab24ce7b925dc..79e6370b7242f5 100644
--- a/llvm/test/Transforms/InstCombine/bitcast.ll
+++ b/llvm/test/Transforms/InstCombine/bitcast.ll
@@ -879,3 +879,26 @@ define half @copysign_idiom_constant_wrong_type2(bfloat %x, i16 %mag) {
   %y = bitcast i16 %res to half
   ret half %y
 }
+
+define i16 @bitcast_undef_to_vector() {
+; CHECK-LABEL: @bitcast_undef_to_vector(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       unreachable:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret i16 undef
+;
+entry:
+  br label %end
+
+unreachable:                                 ; No predecessors!
+  %0 = extractvalue { i32, i32 } zeroinitializer, 1
+  br label %end
+
+end:                                        ; preds = %unreachable, %entry
+  %1 = phi i32 [ %0, %unreachable ], [ undef, %entry ]
+  %2 = bitcast i32 %1 to <2 x i16>
+  %3 = extractelement <2 x i16> %2, i64 0
+  ret i16 %3
+}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll
new file mode 100644
index 00000000000000..71ee00fbf3ec5f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll
@@ -0,0 +1,32 @@
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfbfmin -debug-only=loop-vectorize -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s
+
+define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i32 signext %size, ptr noalias nocapture writeonly %result) {
+; CHECK-LABEL: add
+; CHECK:       LV(REG): Found max usage: 2 item
+; CHECK-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 2 registers
+; CHECK-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 4 registers
+; CHECK-NEXT:  LV(REG): Found invariant usage: 1 item
+; CHECK-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+
+entry:
+  %conv = zext i32 %size to i64
+  %cmp10.not = icmp eq i32 %size, 0
+  br i1 %cmp10.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.011 = phi i64 [ %add4, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds bfloat, ptr %src1, i64 %i.011
+  %0 = load bfloat, ptr %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds bfloat, ptr %src2, i64 %i.011
+  %1 = load bfloat, ptr %arrayidx2, align 4
+  %add = fadd bfloat %0, %1
+  %arrayidx3 = getelementptr inbounds bfloat, ptr %result, i64 %i.011
+  store bfloat %add, ptr %arrayidx3, align 4
+  %add4 = add nuw nsw i64 %i.011, 1
+  %exitcond.not = icmp eq i64 %add4, %conv
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll
new file mode 100644
index 00000000000000..c58307d9607869
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll
@@ -0,0 +1,38 @@
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfh -debug-only=loop-vectorize -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s --check-prefix=ZVFH
+; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfhmin -debug-only=loop-vectorize -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s --check-prefix=ZVFHMIN
+
+define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i32 signext %size, ptr noalias nocapture writeonly %result) {
+; CHECK-LABEL: add
+; ZVFH:       LV(REG): Found max usage: 2 item
+; ZVFH-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 2 registers
+; ZVFH-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 2 registers
+; ZVFH-NEXT:  LV(REG): Found invariant usage: 1 item
+; ZVFH-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; ZVFHMIN:       LV(REG): Found max usage: 2 item
+; ZVFHMIN-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 2 registers
+; ZVFHMIN-NEXT:  LV(REG): RegisterClass: RISCV::VRRC, 4 registers
+; ZVFHMIN-NEXT:  LV(REG): Found invariant usage: 1 item
+; ZVFHMIN-NEXT:  LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+
+entry:
+  %conv = zext i32 %size to i64
+  %cmp10.not = icmp eq i32 %size, 0
+  br i1 %cmp10.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.011 = phi i64 [ %add4, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds half, ptr %src1, i64 %i.011
+  %0 = load half, ptr %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds half, ptr %src2, i64 %i.011
+  %1 = load half, ptr %arrayidx2, align 4
+  %add = fadd half %0, %1
+  %arrayidx3 = getelementptr inbounds half, ptr %result, i64 %i.011
+  store half %add, ptr %arrayidx3, align 4
+  %add4 = add nuw nsw i64 %i.011, 1
+  %exitcond.not = icmp eq i64 %add4, %conv
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
index 621888c144852f..a51ef66951f0e8 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=loop-vectorize -force-vector-width=16 -force-vector-interleave=2 < %s -S -o - | FileCheck %s
+; RUN: opt -passes=loop-vectorize -S -o - %s  | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc_linux"
@@ -59,11 +59,11 @@ define void @firstorderrec(ptr nocapture noundef readonly %x, ptr noalias nocapt
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i8 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP16:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP16:%.*]] = phi i8 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP17:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP16]] = load i8, ptr [[ARRAYIDX4]], align 1
-; CHECK-NEXT:    [[ADD7:%.*]] = add i8 [[TMP16]], [[SCALAR_RECUR]]
+; CHECK-NEXT:    [[TMP17]] = load i8, ptr [[ARRAYIDX4]], align 1
+; CHECK-NEXT:    [[ADD7:%.*]] = add i8 [[TMP17]], [[TMP16]]
 ; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i8, ptr [[Y]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store i8 [[ADD7]], ptr [[ARRAYIDX10]], align 1
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -163,22 +163,22 @@ define void @thirdorderrec(ptr nocapture noundef readonly %x, ptr noalias nocapt
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 3, [[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i8 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ [[DOTPRE45]], [[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    [[SCALAR_RECUR_INIT8:%.*]] = phi i8 [ [[VECTOR_RECUR_EXTRACT6]], [[MIDDLE_BLOCK]] ], [ [[DOTPRE44]], [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT10:%.*]] = phi i8 [ [[VECTOR_RECUR_EXTRACT7]], [[MIDDLE_BLOCK]] ], [ [[DOTPRE]], [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT9:%.*]] = phi i8 [ [[VECTOR_RECUR_EXTRACT7]], [[MIDDLE_BLOCK]] ], [ [[DOTPRE]], [[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i8 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP24:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR9:%.*]] = phi i8 [ [[SCALAR_RECUR_INIT8]], [[SCALAR_PH]] ], [ [[SCALAR_RECUR]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR11:%.*]] = phi i8 [ [[SCALAR_RECUR_INIT10]], [[SCALAR_PH]] ], [ [[SCALAR_RECUR9]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP24:%.*]] = phi i8 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP27:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP25:%.*]] = phi i8 [ [[SCALAR_RECUR_INIT8]], [[SCALAR_PH]] ], [ [[TMP24]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP26:%.*]] = phi i8 [ [[SCALAR_RECUR_INIT9]], [[SCALAR_PH]] ], [ [[TMP25]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ADD8:%.*]] = add i8 [[SCALAR_RECUR9]], [[SCALAR_RECUR11]]
-; CHECK-NEXT:    [[ADD15:%.*]] = add i8 [[ADD8]], [[SCALAR_RECUR]]
+; CHECK-NEXT:    [[ADD8:%.*]] = add i8 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[ADD15:%.*]] = add i8 [[ADD8]], [[TMP24]]
 ; CHECK-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP24]] = load i8, ptr [[ARRAYIDX18]], align 1
-; CHECK-NEXT:    [[ADD21:%.*]] = add i8 [[ADD15]], [[TMP24]]
+; CHECK-NEXT:    [[TMP27]] = load i8, ptr [[ARRAYIDX18]], align 1
+; CHECK-NEXT:    [[ADD21:%.*]] = add i8 [[ADD15]], [[TMP27]]
 ; CHECK-NEXT:    [[ARRAYIDX24:%.*]] = getelementptr inbounds i8, ptr [[Y]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store i8 [[ADD21]], ptr [[ARRAYIDX24]], align 1
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -225,31 +225,31 @@ define i64 @test_pr62954_scalar_epilogue_required(ptr %A, ptr noalias %B, ptr %C
 ; CHECK-NEXT:    [[REC_START:%.*]] = load i64, ptr [[GEP]], align 8
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <16 x i64> poison, i64 [[REC_START]], i32 15
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <2 x i64> poison, i64 [[REC_START]], i32 1
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 1, i64 3, i64 5, i64 7, i64 9, i64 11, i64 13, i64 15, i64 17, i64 19, i64 21, i64 23, i64 25, i64 27, i64 29, i64 31>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <16 x i64> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <16 x i64> [[VEC_IND]], <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
-; CHECK-NEXT:    [[TMP0:%.*]] = sub nsw <16 x i64> zeroinitializer, [[VEC_IND]]
-; CHECK-NEXT:    [[TMP1]] = sub nsw <16 x i64> zeroinitializer, [[STEP_ADD]]
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x i64> [[TMP1]], i32 15
-; CHECK-NEXT:    store i64 [[TMP4]], ptr [[GEP]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <16 x i64> [[STEP_ADD]], <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
-; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <2 x i64> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <2 x i64> [[VEC_IND]], <i64 4, i64 4>
+; CHECK-NEXT:    [[TMP0:%.*]] = sub nsw <2 x i64> zeroinitializer, [[VEC_IND]]
+; CHECK-NEXT:    [[TMP1]] = sub nsw <2 x i64> zeroinitializer, [[STEP_ADD]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT:    store i64 [[TMP2]], ptr [[GEP]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[STEP_ADD]], <i64 4, i64 4>
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 36
+; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i64> [[TMP1]], i32 15
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
 ; CHECK-NEXT:    br label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 65, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 73, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ [[REC_START]], [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[NEG_IV:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[FOR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[NEG_IV:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr double, ptr [[B:%.*]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_B:%.*]] = load double, ptr [[GEP_B]], align 8
 ; CHECK-NEXT:    [[NEG_IV]] = sub nsw i64 0, [[IV]]
@@ -258,7 +258,7 @@ define i64 @test_pr62954_scalar_epilogue_required(ptr %A, ptr noalias %B, ptr %C
 ; CHECK-NEXT:    [[EC:%.*]] = icmp ugt i64 [[IV]], 74
 ; CHECK-NEXT:    br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[DOTIN_LCSSA:%.*]] = phi i64 [ [[SCALAR_RECUR]], [[LOOP]] ]
+; CHECK-NEXT:    [[DOTIN_LCSSA:%.*]] = phi i64 [ [[FOR]], [[LOOP]] ]
 ; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi double [ [[L_B]], [[LOOP]] ]
 ; CHECK-NEXT:    store double [[DOTLCSSA]], ptr [[C:%.*]], align 8
 ; CHECK-NEXT:    ret i64 [[DOTIN_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll
index 779836ee9e91e1..dbe373b46cce21 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll
@@ -381,3 +381,42 @@ loop:
 exit:
   ret void
 }
+
+define void @hoist_previous_value_and_operands(ptr %dst, i64 %mask) {
+; CHECK-LABEL: @hoist_previous_value_and_operands(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[ADD:%.*]], [[LOOP]] ], [ 1, [[BB:%.*]] ]
+; CHECK-NEXT:    [[FOR_1:%.*]] = phi i32 [ [[TRUNC:%.*]], [[LOOP]] ], [ 1, [[BB]] ]
+; CHECK-NEXT:    [[FOR_2:%.*]] = phi i32 [ [[OR:%.*]], [[LOOP]] ], [ 0, [[BB]] ]
+; CHECK-NEXT:    [[OR]] = or i32 [[FOR_1]], 0
+; CHECK-NEXT:    [[ADD]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[IV]]
+; CHECK-NEXT:    store i32 [[FOR_2]], ptr [[GEP]], align 4
+; CHECK-NEXT:    [[ICMP:%.*]] = icmp ult i64 [[IV]], 337
+; CHECK-NEXT:    [[A:%.*]] = and i64 [[IV]], [[MASK:%.*]]
+; CHECK-NEXT:    [[TRUNC]] = trunc i64 [[A]] to i32
+; CHECK-NEXT:    br i1 [[ICMP]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+bb:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %add, %loop ], [ 1, %bb ]
+  %for.1 = phi i32 [ %trunc, %loop ], [ 1, %bb ]
+  %for.2 = phi i32 [ %or, %loop ], [ 0, %bb ]
+  %or = or i32 %for.1, 0
+  %add = add i64 %iv, 1
+  %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+  store i32 %for.2, ptr %gep, align 4
+  %icmp = icmp ult i64 %iv, 337
+  %a = and i64 %iv, %mask
+  %trunc = trunc i64 %a to i32
+  br i1 %icmp, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/hoist-loads-stores-with-cf.ll b/llvm/test/Transforms/SimplifyCFG/X86/hoist-loads-stores-with-cf.ll
index 0507e4e29a68a2..6ea0cf290ffc82 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/hoist-loads-stores-with-cf.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/hoist-loads-stores-with-cf.ll
@@ -206,12 +206,12 @@ define void @nondebug_metadata(i1 %cond, ptr %p, ptr %q) {
 ; CHECK-LABEL: @nondebug_metadata(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i1 [[COND:%.*]] to <1 x i1>
-; CHECK-NEXT:    [[TMP1:%.*]] = call <1 x i16> @llvm.masked.load.v1i16.p0(ptr [[P:%.*]], i32 2, <1 x i1> [[TMP0]], <1 x i16> poison), !range [[RNG5:![0-9]+]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call range(i16 0, 10) <1 x i16> @llvm.masked.load.v1i16.p0(ptr [[P:%.*]], i32 2, <1 x i1> [[TMP0]], <1 x i16> poison)
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <1 x i16> [[TMP1]] to i16
-; CHECK-NEXT:    [[TMP3:%.*]] = call <1 x i32> @llvm.masked.load.v1i32.p0(ptr [[Q:%.*]], i32 4, <1 x i1> [[TMP0]], <1 x i32> poison), !annotation [[META6:![0-9]+]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <1 x i32> @llvm.masked.load.v1i32.p0(ptr [[Q:%.*]], i32 4, <1 x i1> [[TMP0]], <1 x i32> poison), !annotation [[META5:![0-9]+]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i32> [[TMP3]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP2]] to <1 x i16>
-; CHECK-NEXT:    call void @llvm.masked.store.v1i16.p0(<1 x i16> [[TMP5]], ptr [[Q]], i32 4, <1 x i1> [[TMP0]]), !annotation [[META6]]
+; CHECK-NEXT:    call void @llvm.masked.store.v1i16.p0(<1 x i16> [[TMP5]], ptr [[Q]], i32 4, <1 x i1> [[TMP0]]), !annotation [[META5]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP4]] to <1 x i32>
 ; CHECK-NEXT:    call void @llvm.masked.store.v1i32.p0(<1 x i32> [[TMP6]], ptr [[P]], i32 2, <1 x i1> [[TMP0]])
 ; CHECK-NEXT:    ret void
@@ -583,7 +583,7 @@ define void @not_likely_to_execute(ptr %p, ptr %q, i32 %a) {
 ; CHECK-LABEL: @not_likely_to_execute(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[A:%.*]], 0
-; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]], !prof [[PROF7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]], !prof [[PROF6:![0-9]+]]
 ; CHECK:       if.end:
 ; CHECK-NEXT:    ret void
 ; CHECK:       if.then:
diff --git a/llvm/test/Transforms/SimplifyCFG/patchpoint-invalid-sink.ll b/llvm/test/Transforms/SimplifyCFG/patchpoint-invalid-sink.ll
new file mode 100644
index 00000000000000..eeb57102340654
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/patchpoint-invalid-sink.ll
@@ -0,0 +1,35 @@
+; RUN: opt -passes='simplifycfg<sink-common-insts>' -S %s | FileCheck %s
+
+declare void @personalityFn()
+
+define void @test(i1 %c) personality ptr @personalityFn {
+; CHECK-LABEL: define void @test
+; CHECK-LABEL: entry:
+; CHECK-NEXT:    br i1 %c, label %taken, label %untaken
+; CHECK-LABEL: taken:
+; CHECK-NEXT:    invoke void (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.void(i64 1, i32 0, ptr null, i32 0)
+; CHECK-LABEL: untaken:
+; CHECK-NEXT:    invoke void (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.void(i64 2, i32 0, ptr null, i32 0)
+; CHECK-LABEL: end:
+; CHECK-NEXT:    ret void
+entry:
+  br i1 %c, label %taken, label %untaken
+
+taken:
+  invoke void (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.void(i64 1, i32 0, ptr null, i32 0)
+          to label %end unwind label %unwind
+
+untaken:
+  invoke void (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.void(i64 2, i32 0, ptr null, i32 0)
+          to label %end unwind label %unwind
+
+end:
+  ret void
+
+unwind:
+  %0 = landingpad { ptr, i32 }
+          cleanup
+  br label %end
+}
+
+declare void @llvm.experimental.patchpoint.void(i64 immarg, i32 immarg, ptr, i32 immarg, ...)
diff --git a/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll b/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll
index 4c93837f1422a9..87d64932ef0935 100644
--- a/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll
+++ b/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll
@@ -44,3 +44,104 @@ bb5:
   call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %y)
   ret void
 }
+
+define i64 @dont_make_div_variable(i64 noundef %x, i64 noundef %i) {
+; CHECK-LABEL: define i64 @dont_make_div_variable(
+; CHECK-SAME: i64 noundef [[X:%.*]], i64 noundef [[I:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    switch i64 [[I]], label %[[SW_DEFAULT:.*]] [
+; CHECK-NEXT:      i64 9, label %[[SW_BB:.*]]
+; CHECK-NEXT:      i64 10, label %[[SW_BB1:.*]]
+; CHECK-NEXT:      i64 11, label %[[SW_BB3:.*]]
+; CHECK-NEXT:      i64 12, label %[[SW_BB5:.*]]
+; CHECK-NEXT:    ]
+; CHECK:       [[SW_BB]]:
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[X]], 9
+; CHECK-NEXT:    br label %[[RETURN:.*]]
+; CHECK:       [[SW_BB1]]:
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[X]], 10
+; CHECK-NEXT:    br label %[[RETURN]]
+; CHECK:       [[SW_BB3]]:
+; CHECK-NEXT:    [[DIV4:%.*]] = udiv i64 [[X]], 11
+; CHECK-NEXT:    br label %[[RETURN]]
+; CHECK:       [[SW_BB5]]:
+; CHECK-NEXT:    [[DIV7:%.*]] = udiv i64 [[X]], 12
+; CHECK-NEXT:    br label %[[RETURN]]
+; CHECK:       [[SW_DEFAULT]]:
+; CHECK-NEXT:    unreachable
+; CHECK:       [[RETURN]]:
+; CHECK-NEXT:    [[DIV6:%.*]] = phi i64 [ [[DIV7]], %[[SW_BB5]] ], [ [[DIV4]], %[[SW_BB3]] ], [ [[DIV2]], %[[SW_BB1]] ], [ [[DIV]], %[[SW_BB]] ]
+; CHECK-NEXT:    ret i64 [[DIV6]]
+;
+entry:
+  switch i64 %i, label %sw.default [
+  i64 9, label %sw.bb
+  i64 10, label %sw.bb1
+  i64 11, label %sw.bb3
+  i64 12, label %sw.bb5
+  ]
+
+sw.bb:
+  %div = udiv i64 %x, 9
+  br label %return
+
+sw.bb1:
+  %div2 = udiv i64 %x, 10
+  br label %return
+
+sw.bb3:
+  %div4 = udiv i64 %x, 11
+  br label %return
+
+sw.bb5:
+  %div6 = udiv i64 %x, 12
+  br label %return
+
+sw.default:
+  unreachable
+
+return:
+  %retval.0 = phi i64 [ %div6, %sw.bb5 ], [ %div4, %sw.bb3 ], [ %div2, %sw.bb1 ], [ %div, %sw.bb ]
+  ret i64 %retval.0
+}
+
+define i64 @okay_to_make_div_variable(i64 noundef %x, i64 noundef %i) {
+; CHECK-LABEL: define i64 @okay_to_make_div_variable(
+; CHECK-SAME: i64 noundef [[X:%.*]], i64 noundef [[I:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SWITCH_TABLEIDX:%.*]] = sub nsw i64 [[I]], 9
+; CHECK-NEXT:    [[SWITCH_OFFSET:%.*]] = add nsw i64 [[SWITCH_TABLEIDX]], 9
+; CHECK-NEXT:    [[DIV6:%.*]] = udiv i64 [[SWITCH_OFFSET]], [[X]]
+; CHECK-NEXT:    ret i64 [[DIV6]]
+;
+entry:
+  switch i64 %i, label %sw.default [
+  i64 9, label %sw.bb
+  i64 10, label %sw.bb1
+  i64 11, label %sw.bb3
+  i64 12, label %sw.bb5
+  ]
+
+sw.bb:
+  %div = udiv i64 9, %x
+  br label %return
+
+sw.bb1:
+  %div2 = udiv i64 10, %x
+  br label %return
+
+sw.bb3:
+  %div4 = udiv i64 11, %x
+  br label %return
+
+sw.bb5:
+  %div6 = udiv i64 12, %x
+  br label %return
+
+sw.default:
+  unreachable
+
+return:
+  %retval.0 = phi i64 [ %div6, %sw.bb5 ], [ %div4, %sw.bb3 ], [ %div2, %sw.bb1 ], [ %div, %sw.bb ]
+  ret i64 %retval.0
+}
diff --git a/llvm/test/Verifier/intrinsic-immarg.ll b/llvm/test/Verifier/intrinsic-immarg.ll
index ad70b17e5fb727..e310cd2e0b7819 100644
--- a/llvm/test/Verifier/intrinsic-immarg.ll
+++ b/llvm/test/Verifier/intrinsic-immarg.ll
@@ -238,6 +238,37 @@ define void @calls_statepoint(ptr addrspace(1) %arg0, i64 %arg1, i32 %arg2, i32
   ret void
 }
 
+declare void @llvm.experimental.patchpoint.void(i64, i32, ptr, i32, ...)
+declare i64 @llvm.experimental.patchpoint.i64(i64, i32, ptr, i32, ...)
+
+define void @test_patchpoint(i64 %arg0, i32 %arg1, i32 %arg2) {
+  ; CHECK: immarg operand has non-immediate parameter
+  ; CHECK-NEXT: i64 %arg0
+  ; CHECK-NEXT: call void (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.void(i64 %arg0, i32 4, ptr null, i32 0)
+  ; CHECK: immarg operand has non-immediate parameter
+  ; CHECK-NEXT: i32 %arg1
+  ; CHECK-NEXT: call void (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.void(i64 0, i32 %arg1, ptr null, i32 0)
+  ; CHECK: immarg operand has non-immediate parameter
+  ; CHECK-NEXT: i32 %arg2
+  ; CHECK-NEXT: call void (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.void(i64 0, i32 4, ptr null, i32 %arg2)
+  ; CHECK: immarg operand has non-immediate parameter
+  ; CHECK-NEXT: i64 %arg0
+  ; CHECK-NEXT: %patchpoint0 = call i64 (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.i64(i64 %arg0, i32 4, ptr null, i32 0)
+  ; CHECK: immarg operand has non-immediate parameter
+  ; CHECK-NEXT: i32 %arg1
+  ; CHECK-NEXT: %patchpoint1 = call i64 (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.i64(i64 0, i32 %arg1, ptr null, i32 0)
+  ; CHECK: immarg operand has non-immediate parameter
+  ; CHECK-NEXT: i32 %arg2
+  ; CHECK-NEXT: %patchpoint2 = call i64 (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.i64(i64 0, i32 4, ptr null, i32 %arg2)
+  call void (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.void(i64 %arg0, i32 4, ptr null, i32 0)
+  call void (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.void(i64 0, i32 %arg1, ptr null, i32 0)
+  call void (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.void(i64 0, i32 4, ptr null, i32 %arg2)
+  %patchpoint0 = call i64 (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.i64(i64 %arg0, i32 4, ptr null, i32 0)
+  %patchpoint1 = call i64 (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.i64(i64 0, i32 %arg1, ptr null, i32 0)
+  %patchpoint2 = call i64 (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.i64(i64 0, i32 4, ptr null, i32 %arg2)
+  ret void
+}
+
 declare void @llvm.hwasan.check.memaccess(ptr, ptr, i32)
 
 define void @hwasan_check_memaccess(ptr %arg0,ptr %arg1, i32 %arg2) {
diff --git a/llvm/test/tools/llvm-mca/RISCV/SyntacoreSCR/SCR7-ALU.s b/llvm/test/tools/llvm-mca/RISCV/SyntacoreSCR/SCR7-ALU.s
new file mode 100644
index 00000000000000..5448ba85c95440
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SyntacoreSCR/SCR7-ALU.s
@@ -0,0 +1,65 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64-unknown-unknown -mcpu=syntacore-scr7 --iterations=1 < %s | FileCheck %s --check-prefixes=CHECK
+
+div a0, a0, a0
+mul t0, a0, t0
+add t1, a0, t0
+add t2, t2, t2
+div a1, a1, a1
+mul s0, a1, s0
+add s1, s0, s1
+add s2, s2, s2
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      8
+# CHECK-NEXT: Total Cycles:      77
+# CHECK-NEXT: Total uOps:        8
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    0.10
+# CHECK-NEXT: IPC:               0.10
+# CHECK-NEXT: Block RThroughput: 70.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      35    35.00                       div	a0, a0, a0
+# CHECK-NEXT:  1      3     1.00                        mul	t0, a0, t0
+# CHECK-NEXT:  1      1     0.50                        add	t1, a0, t0
+# CHECK-NEXT:  1      1     0.50                        add	t2, t2, t2
+# CHECK-NEXT:  1      35    35.00                       div	a1, a1, a1
+# CHECK-NEXT:  1      3     1.00                        mul	s0, a1, s0
+# CHECK-NEXT:  1      1     0.50                        add	s1, s1, s0
+# CHECK-NEXT:  1      1     0.50                        add	s2, s2, s2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SCR7_ALU_DIV_IS
+# CHECK-NEXT: [1]   - SCR7_ALU_MUL_IS
+# CHECK-NEXT: [2]   - SCR7_DIV
+# CHECK-NEXT: [3]   - SCR7_FALU
+# CHECK-NEXT: [4]   - SCR7_FDIVSQRT
+# CHECK-NEXT: [5]   - SCR7_FMA
+# CHECK-NEXT: [6]   - SCR7_FPU_IS
+# CHECK-NEXT: [7]   - SCR7_LSU
+# CHECK-NEXT: [8]   - SCR7_MUL
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]
+# CHECK-NEXT: 4.00   4.00   70.00   -      -      -      -      -     2.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    Instructions:
+# CHECK-NEXT: 1.00    -     35.00   -      -      -      -      -      -     div	a0, a0, a0
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -     1.00   mul	t0, a0, t0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	t1, a0, t0
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     add	t2, t2, t2
+# CHECK-NEXT: 1.00    -     35.00   -      -      -      -      -      -     div	a1, a1, a1
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -     1.00   mul	s0, a1, s0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	s1, s1, s0
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     add	s2, s2, s2
diff --git a/llvm/test/tools/llvm-mca/RISCV/SyntacoreSCR/SCR7-FPALU_D.s b/llvm/test/tools/llvm-mca/RISCV/SyntacoreSCR/SCR7-FPALU_D.s
new file mode 100644
index 00000000000000..938d6569190720
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SyntacoreSCR/SCR7-FPALU_D.s
@@ -0,0 +1,75 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64-unknown-unknown -mcpu=syntacore-scr7 --iterations=1 < %s | FileCheck %s --check-prefixes=CHECK
+
+fmul.d f1, f1, f2
+
+fadd.d f2, f2, f5
+fadd.d f3, f3, f5
+fadd.d f4, f4, f5
+
+fmul.d f6, f6, f7
+
+fadd.d f7, f7, f5
+fadd.d f8, f8, f5
+fadd.d f9, f9, f5
+
+fmadd.d f1, f1, f2, f4
+fdiv.d f3, f3, f5
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      10
+# CHECK-NEXT: Total Cycles:      42
+# CHECK-NEXT: Total uOps:        10
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    0.24
+# CHECK-NEXT: IPC:               0.24
+# CHECK-NEXT: Block RThroughput: 29.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      6     1.00                        fmul.d	ft1, ft1, ft2
+# CHECK-NEXT:  1      4     1.00                        fadd.d	ft2, ft2, ft5
+# CHECK-NEXT:  1      4     1.00                        fadd.d	ft3, ft3, ft5
+# CHECK-NEXT:  1      4     1.00                        fadd.d	ft4, ft4, ft5
+# CHECK-NEXT:  1      6     1.00                        fmul.d	ft6, ft6, ft7
+# CHECK-NEXT:  1      4     1.00                        fadd.d	ft7, ft7, ft5
+# CHECK-NEXT:  1      4     1.00                        fadd.d	fs0, fs0, ft5
+# CHECK-NEXT:  1      4     1.00                        fadd.d	fs1, fs1, ft5
+# CHECK-NEXT:  1      6     1.00                        fmadd.d	ft1, ft1, ft2, ft4
+# CHECK-NEXT:  1      30    29.00                       fdiv.d	ft3, ft3, ft5
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SCR7_ALU_DIV_IS
+# CHECK-NEXT: [1]   - SCR7_ALU_MUL_IS
+# CHECK-NEXT: [2]   - SCR7_DIV
+# CHECK-NEXT: [3]   - SCR7_FALU
+# CHECK-NEXT: [4]   - SCR7_FDIVSQRT
+# CHECK-NEXT: [5]   - SCR7_FMA
+# CHECK-NEXT: [6]   - SCR7_FPU_IS
+# CHECK-NEXT: [7]   - SCR7_LSU
+# CHECK-NEXT: [8]   - SCR7_MUL
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]
+# CHECK-NEXT:  -      -      -     6.00   29.00  3.00   10.00   -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    Instructions:
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -     fmul.d	ft1, ft1, ft2
+# CHECK-NEXT:  -      -      -     1.00    -      -     1.00    -      -     fadd.d	ft2, ft2, ft5
+# CHECK-NEXT:  -      -      -     1.00    -      -     1.00    -      -     fadd.d	ft3, ft3, ft5
+# CHECK-NEXT:  -      -      -     1.00    -      -     1.00    -      -     fadd.d	ft4, ft4, ft5
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -     fmul.d	ft6, ft6, ft7
+# CHECK-NEXT:  -      -      -     1.00    -      -     1.00    -      -     fadd.d	ft7, ft7, ft5
+# CHECK-NEXT:  -      -      -     1.00    -      -     1.00    -      -     fadd.d	fs0, fs0, ft5
+# CHECK-NEXT:  -      -      -     1.00    -      -     1.00    -      -     fadd.d	fs1, fs1, ft5
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -     fmadd.d	ft1, ft1, ft2, ft4
+# CHECK-NEXT:  -      -      -      -     29.00   -     1.00    -      -     fdiv.d	ft3, ft3, ft5
diff --git a/llvm/test/tools/llvm-mca/RISCV/SyntacoreSCR/SCR7-FPALU_S.s b/llvm/test/tools/llvm-mca/RISCV/SyntacoreSCR/SCR7-FPALU_S.s
new file mode 100644
index 00000000000000..2c82129f7ce631
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SyntacoreSCR/SCR7-FPALU_S.s
@@ -0,0 +1,75 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64-unknown-unknown -mcpu=syntacore-scr7 --iterations=1 < %s | FileCheck %s --check-prefixes=CHECK
+
+fmul.s f1, f1, f2
+
+fadd.s f2, f2, f5
+fadd.s f3, f3, f5
+fadd.s f4, f4, f5
+
+fmul.s f6, f6, f7
+
+fadd.s f7, f7, f5
+fadd.s f8, f8, f5
+fadd.s f9, f9, f5
+
+fmadd.s f1, f1, f2, f4
+fdiv.s f3, f3, f5
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      10
+# CHECK-NEXT: Total Cycles:      28
+# CHECK-NEXT: Total uOps:        10
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    0.36
+# CHECK-NEXT: IPC:               0.36
+# CHECK-NEXT: Block RThroughput: 15.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      6     1.00                        fmul.s	ft1, ft1, ft2
+# CHECK-NEXT:  1      4     1.00                        fadd.s	ft2, ft2, ft5
+# CHECK-NEXT:  1      4     1.00                        fadd.s	ft3, ft3, ft5
+# CHECK-NEXT:  1      4     1.00                        fadd.s	ft4, ft4, ft5
+# CHECK-NEXT:  1      6     1.00                        fmul.s	ft6, ft6, ft7
+# CHECK-NEXT:  1      4     1.00                        fadd.s	ft7, ft7, ft5
+# CHECK-NEXT:  1      4     1.00                        fadd.s	fs0, fs0, ft5
+# CHECK-NEXT:  1      4     1.00                        fadd.s	fs1, fs1, ft5
+# CHECK-NEXT:  1      6     1.00                        fmadd.s	ft1, ft1, ft2, ft4
+# CHECK-NEXT:  1      16    15.00                       fdiv.s	ft3, ft3, ft5
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SCR7_ALU_DIV_IS
+# CHECK-NEXT: [1]   - SCR7_ALU_MUL_IS
+# CHECK-NEXT: [2]   - SCR7_DIV
+# CHECK-NEXT: [3]   - SCR7_FALU
+# CHECK-NEXT: [4]   - SCR7_FDIVSQRT
+# CHECK-NEXT: [5]   - SCR7_FMA
+# CHECK-NEXT: [6]   - SCR7_FPU_IS
+# CHECK-NEXT: [7]   - SCR7_LSU
+# CHECK-NEXT: [8]   - SCR7_MUL
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]
+# CHECK-NEXT:  -      -      -     6.00   15.00  3.00   10.00   -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    Instructions:
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -     fmul.s	ft1, ft1, ft2
+# CHECK-NEXT:  -      -      -     1.00    -      -     1.00    -      -     fadd.s	ft2, ft2, ft5
+# CHECK-NEXT:  -      -      -     1.00    -      -     1.00    -      -     fadd.s	ft3, ft3, ft5
+# CHECK-NEXT:  -      -      -     1.00    -      -     1.00    -      -     fadd.s	ft4, ft4, ft5
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -     fmul.s	ft6, ft6, ft7
+# CHECK-NEXT:  -      -      -     1.00    -      -     1.00    -      -     fadd.s	ft7, ft7, ft5
+# CHECK-NEXT:  -      -      -     1.00    -      -     1.00    -      -     fadd.s	fs0, fs0, ft5
+# CHECK-NEXT:  -      -      -     1.00    -      -     1.00    -      -     fadd.s	fs1, fs1, ft5
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -     fmadd.s	ft1, ft1, ft2, ft4
+# CHECK-NEXT:  -      -      -      -     15.00   -     1.00    -      -     fdiv.s	ft3, ft3, ft5
diff --git a/llvm/test/tools/llvm-mca/RISCV/SyntacoreSCR/SCR7-LSU.s b/llvm/test/tools/llvm-mca/RISCV/SyntacoreSCR/SCR7-LSU.s
new file mode 100644
index 00000000000000..2b3ab2ba4192f0
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SyntacoreSCR/SCR7-LSU.s
@@ -0,0 +1,53 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64-unknown-unknown -mcpu=syntacore-scr7 --iterations=1 < %s | FileCheck %s --check-prefixes=CHECK
+
+lw a0, 0(s0)
+lw a1, 0(s0)
+lw a2, 0(s0)
+lw a3, 0(s0)
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      4
+# CHECK-NEXT: Total Cycles:      9
+# CHECK-NEXT: Total uOps:        4
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    0.44
+# CHECK-NEXT: IPC:               0.44
+# CHECK-NEXT: Block RThroughput: 4.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     1.00    *                   lw	a0, 0(s0)
+# CHECK-NEXT:  1      3     1.00    *                   lw	a1, 0(s0)
+# CHECK-NEXT:  1      3     1.00    *                   lw	a2, 0(s0)
+# CHECK-NEXT:  1      3     1.00    *                   lw	a3, 0(s0)
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SCR7_ALU_DIV_IS
+# CHECK-NEXT: [1]   - SCR7_ALU_MUL_IS
+# CHECK-NEXT: [2]   - SCR7_DIV
+# CHECK-NEXT: [3]   - SCR7_FALU
+# CHECK-NEXT: [4]   - SCR7_FDIVSQRT
+# CHECK-NEXT: [5]   - SCR7_FMA
+# CHECK-NEXT: [6]   - SCR7_FPU_IS
+# CHECK-NEXT: [7]   - SCR7_LSU
+# CHECK-NEXT: [8]   - SCR7_MUL
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]
+# CHECK-NEXT:  -      -      -      -      -      -      -     4.00    -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     lw	a0, 0(s0)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     lw	a1, 0(s0)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     lw	a2, 0(s0)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     lw	a3, 0(s0)
diff --git a/llvm/test/tools/llvm-mca/X86/Znver4/resources-avx512.s b/llvm/test/tools/llvm-mca/X86/Znver4/resources-avx512.s
index 6e52eddd9a8f5e..0c4c6567680017 100644
--- a/llvm/test/tools/llvm-mca/X86/Znver4/resources-avx512.s
+++ b/llvm/test/tools/llvm-mca/X86/Znver4/resources-avx512.s
@@ -1609,13 +1609,13 @@ vunpcklps         (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  1      1     0.50                        vpcmpeqq	%zmm0, %zmm1, %k2 {%k3}
 # CHECK-NEXT:  1      8     0.50    *                   vpcmpeqq	(%rax), %zmm1, %k2 {%k3}
 # CHECK-NEXT:  1      8     0.50    *                   vpcmpeqq	(%rax){1to8}, %zmm1, %k2 {%k3}
-# CHECK-NEXT:  1      1     0.50                        vpcmpgtd	%zmm0, %zmm1, %k2
+# CHECK-NEXT:  1      1     0.25                        vpcmpgtd	%zmm0, %zmm1, %k2
 # CHECK-NEXT:  1      8     0.50    *                   vpcmpgtd	(%rax), %zmm1, %k2
 # CHECK-NEXT:  1      8     0.50    *                   vpcmpgtd	(%rax){1to16}, %zmm1, %k2
 # CHECK-NEXT:  1      1     0.50                        vpcmpgtd	%zmm0, %zmm1, %k2 {%k3}
 # CHECK-NEXT:  1      8     0.50    *                   vpcmpgtd	(%rax), %zmm1, %k2 {%k3}
 # CHECK-NEXT:  1      8     0.50    *                   vpcmpgtd	(%rax){1to16}, %zmm1, %k2 {%k3}
-# CHECK-NEXT:  1      1     0.50                        vpcmpgtq	%zmm0, %zmm1, %k2
+# CHECK-NEXT:  1      1     0.25                        vpcmpgtq	%zmm0, %zmm1, %k2
 # CHECK-NEXT:  1      8     0.50    *                   vpcmpgtq	(%rax), %zmm1, %k2
 # CHECK-NEXT:  1      8     0.50    *                   vpcmpgtq	(%rax){1to8}, %zmm1, %k2
 # CHECK-NEXT:  1      1     0.50                        vpcmpgtq	%zmm0, %zmm1, %k2 {%k3}
@@ -1815,7 +1815,7 @@ vunpcklps         (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  1      1     1.00                        vpshufd	$0, %zmm16, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      8     1.00    *                   vpshufd	$0, (%rax), %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      8     1.00    *                   vpshufd	$0, (%rax){1to16}, %zmm19 {%k1} {z}
-# CHECK-NEXT:  1      1     0.50                        vpsubd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.25                        vpsubd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  1      8     0.50    *                   vpsubd	(%rax), %zmm17, %zmm19
 # CHECK-NEXT:  1      8     0.50    *                   vpsubd	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  1      1     0.50                        vpsubd	%zmm16, %zmm17, %zmm19 {%k1}
@@ -1824,7 +1824,7 @@ vunpcklps         (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  1      1     0.50                        vpsubd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      8     0.50    *                   vpsubd	(%rax), %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      8     0.50    *                   vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  1      1     0.50                        vpsubq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.25                        vpsubq	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  1      8     0.50    *                   vpsubq	(%rax), %zmm17, %zmm19
 # CHECK-NEXT:  1      8     0.50    *                   vpsubq	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  1      1     0.50                        vpsubq	%zmm16, %zmm17, %zmm19 {%k1}
@@ -1939,7 +1939,7 @@ vunpcklps         (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  1      22    5.00    *                   vsqrtss	(%rax), %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  1      15    5.00                        vsqrtss	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  1      22    5.00    *                   vsqrtss	(%rax), %xmm17, %xmm19 {%k1} {z}
-# CHECK-NEXT:  1      1     0.50                        vpsubd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.25                        vpsubd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  1      8     0.50    *                   vpsubd	(%rax), %zmm17, %zmm19
 # CHECK-NEXT:  1      8     0.50    *                   vpsubd	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  1      1     0.50                        vpsubd	%zmm16, %zmm17, %zmm19 {%k1}
@@ -1948,7 +1948,7 @@ vunpcklps         (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  1      1     0.50                        vpsubd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      8     0.50    *                   vpsubd	(%rax), %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      8     0.50    *                   vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  1      1     0.50                        vpsubq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.25                        vpsubq	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  1      8     0.50    *                   vpsubq	(%rax), %zmm17, %zmm19
 # CHECK-NEXT:  1      8     0.50    *                   vpsubq	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  1      1     0.50                        vpsubq	%zmm16, %zmm17, %zmm19 {%k1}
@@ -2065,7 +2065,7 @@ vunpcklps         (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12.0] [12.1] [13]   [14.0] [14.1] [14.2] [15.0] [15.1] [15.2] [16.0] [16.1]
-# CHECK-NEXT: 5.33   5.33   5.33    -      -      -      -      -     221.00 1060.50 618.00 352.50 297.00 297.00 17.00 205.33 205.33 205.33 194.33 194.33 194.33 16.50  16.50
+# CHECK-NEXT: 5.33   5.33   5.33    -      -      -      -      -     219.50 1059.00 616.50 351.00 297.00 297.00 17.00 205.33 205.33 205.33 194.33 194.33 194.33 16.50  16.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12.0] [12.1] [13]   [14.0] [14.1] [14.2] [15.0] [15.1] [15.2] [16.0] [16.1] Instructions:
@@ -2590,13 +2590,13 @@ vunpcklps         (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpcmpeqq	%zmm0, %zmm1, %k2 {%k3}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpcmpeqq	(%rax), %zmm1, %k2 {%k3}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpcmpeqq	(%rax){1to8}, %zmm1, %k2 {%k3}
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpcmpgtd	%zmm0, %zmm1, %k2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -      -      -      -      -      -     vpcmpgtd	%zmm0, %zmm1, %k2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpcmpgtd	(%rax), %zmm1, %k2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpcmpgtd	(%rax){1to16}, %zmm1, %k2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpcmpgtd	%zmm0, %zmm1, %k2 {%k3}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpcmpgtd	(%rax), %zmm1, %k2 {%k3}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpcmpgtd	(%rax){1to16}, %zmm1, %k2 {%k3}
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpcmpgtq	%zmm0, %zmm1, %k2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -      -      -      -      -      -     vpcmpgtq	%zmm0, %zmm1, %k2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpcmpgtq	(%rax), %zmm1, %k2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpcmpgtq	(%rax){1to8}, %zmm1, %k2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpcmpgtq	%zmm0, %zmm1, %k2 {%k3}
@@ -2796,7 +2796,7 @@ vunpcklps         (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -      -      -      -      -     vpshufd	$0, %zmm16, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -     0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpshufd	$0, (%rax), %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -     0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpshufd	$0, (%rax){1to16}, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpsubd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -      -      -      -      -      -     vpsubd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubd	(%rax), %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubd	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpsubd	%zmm16, %zmm17, %zmm19 {%k1}
@@ -2805,7 +2805,7 @@ vunpcklps         (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpsubd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubd	(%rax), %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpsubq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -      -      -      -      -      -     vpsubq	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubq	(%rax), %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubq	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpsubq	%zmm16, %zmm17, %zmm19 {%k1}
@@ -2920,7 +2920,7 @@ vunpcklps         (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     5.00    -      -     0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vsqrtss	(%rax), %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     5.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vsqrtss	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     5.00    -      -     0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vsqrtss	(%rax), %xmm17, %xmm19 {%k1} {z}
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpsubd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -      -      -      -      -      -     vpsubd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubd	(%rax), %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubd	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpsubd	%zmm16, %zmm17, %zmm19 {%k1}
@@ -2929,7 +2929,7 @@ vunpcklps         (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpsubd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubd	(%rax), %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpsubq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -      -      -      -      -      -     vpsubq	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubq	(%rax), %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubq	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpsubq	%zmm16, %zmm17, %zmm19 {%k1}
diff --git a/llvm/test/tools/llvm-mca/X86/Znver4/resources-avx512bw.s b/llvm/test/tools/llvm-mca/X86/Znver4/resources-avx512bw.s
index d2f32608efb42d..4585ed0f2ed742 100644
--- a/llvm/test/tools/llvm-mca/X86/Znver4/resources-avx512bw.s
+++ b/llvm/test/tools/llvm-mca/X86/Znver4/resources-avx512bw.s
@@ -799,11 +799,11 @@ vpunpcklwd        (%rax), %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  1      8     0.50    *                   vpcmpeqw	(%rax), %zmm1, %k2
 # CHECK-NEXT:  1      1     0.50                        vpcmpeqw	%zmm0, %zmm1, %k2 {%k3}
 # CHECK-NEXT:  1      8     0.50    *                   vpcmpeqw	(%rax), %zmm1, %k2 {%k3}
-# CHECK-NEXT:  1      1     0.50                        vpcmpgtb	%zmm0, %zmm1, %k2
+# CHECK-NEXT:  1      1     0.25                        vpcmpgtb	%zmm0, %zmm1, %k2
 # CHECK-NEXT:  1      8     0.50    *                   vpcmpgtb	(%rax), %zmm1, %k2
 # CHECK-NEXT:  1      1     0.50                        vpcmpgtb	%zmm0, %zmm1, %k2 {%k3}
 # CHECK-NEXT:  1      8     0.50    *                   vpcmpgtb	(%rax), %zmm1, %k2 {%k3}
-# CHECK-NEXT:  1      1     0.50                        vpcmpgtw	%zmm0, %zmm1, %k2
+# CHECK-NEXT:  1      1     0.25                        vpcmpgtw	%zmm0, %zmm1, %k2
 # CHECK-NEXT:  1      8     0.50    *                   vpcmpgtw	(%rax), %zmm1, %k2
 # CHECK-NEXT:  1      1     0.50                        vpcmpgtw	%zmm0, %zmm1, %k2 {%k3}
 # CHECK-NEXT:  1      8     0.50    *                   vpcmpgtw	(%rax), %zmm1, %k2 {%k3}
@@ -1035,7 +1035,7 @@ vpunpcklwd        (%rax), %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  1      8     1.00    *                   vpsrlw	(%rax), %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      2     1.00                        vpsrlw	%xmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      8     1.00    *                   vpsrlw	(%rax), %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  1      1     0.50                        vpsubb	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.25                        vpsubb	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  1      8     0.50    *                   vpsubb	(%rax), %zmm17, %zmm19
 # CHECK-NEXT:  1      1     0.50                        vpsubb	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      8     0.50    *                   vpsubb	(%rax), %zmm17, %zmm19 {%k1}
@@ -1065,7 +1065,7 @@ vpunpcklwd        (%rax), %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  1      8     0.50    *                   vpsubusw	(%rax), %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      1     0.50                        vpsubusw	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      8     0.50    *                   vpsubusw	(%rax), %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  1      1     0.50                        vpsubw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.25                        vpsubw	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  1      8     0.50    *                   vpsubw	(%rax), %zmm17, %zmm19
 # CHECK-NEXT:  1      1     0.50                        vpsubw	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      8     0.50    *                   vpsubw	(%rax), %zmm17, %zmm19 {%k1}
@@ -1139,7 +1139,7 @@ vpunpcklwd        (%rax), %zmm17, %zmm19 {z}{k1}
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12.0] [12.1] [13]   [14.0] [14.1] [14.2] [15.0] [15.1] [15.2] [16.0] [16.1]
-# CHECK-NEXT: 0.67   0.67   0.67   2.00   2.00   2.00   2.00    -     178.50 310.00 275.50 162.00 123.00 123.00 6.00   77.33  77.33  77.33  75.33  75.33  75.33  3.00   3.00
+# CHECK-NEXT: 0.67   0.67   0.67   2.00   2.00   2.00   2.00    -     177.50 309.00 274.50 161.00 123.00 123.00 6.00   77.33  77.33  77.33  75.33  75.33  75.33  3.00   3.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12.0] [12.1] [13]   [14.0] [14.1] [14.2] [15.0] [15.1] [15.2] [16.0] [16.1] Instructions:
@@ -1331,11 +1331,11 @@ vpunpcklwd        (%rax), %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpcmpeqw	(%rax), %zmm1, %k2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpcmpeqw	%zmm0, %zmm1, %k2 {%k3}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpcmpeqw	(%rax), %zmm1, %k2 {%k3}
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpcmpgtb	%zmm0, %zmm1, %k2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -      -      -      -      -      -     vpcmpgtb	%zmm0, %zmm1, %k2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpcmpgtb	(%rax), %zmm1, %k2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpcmpgtb	%zmm0, %zmm1, %k2 {%k3}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpcmpgtb	(%rax), %zmm1, %k2 {%k3}
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpcmpgtw	%zmm0, %zmm1, %k2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -      -      -      -      -      -     vpcmpgtw	%zmm0, %zmm1, %k2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpcmpgtw	(%rax), %zmm1, %k2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpcmpgtw	%zmm0, %zmm1, %k2 {%k3}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpcmpgtw	(%rax), %zmm1, %k2 {%k3}
@@ -1567,7 +1567,7 @@ vpunpcklwd        (%rax), %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -     0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsrlw	(%rax), %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vpsrlw	%xmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -     0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsrlw	(%rax), %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpsubb	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -      -      -      -      -      -     vpsubb	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubb	(%rax), %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpsubb	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubb	(%rax), %zmm17, %zmm19 {%k1}
@@ -1597,7 +1597,7 @@ vpunpcklwd        (%rax), %zmm17, %zmm19 {z}{k1}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubusw	(%rax), %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpsubusw	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubusw	(%rax), %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpsubw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -      -      -      -      -      -     vpsubw	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubw	(%rax), %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -     vpsubw	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   0.50   0.50    -     0.33   0.33   0.33   0.33   0.33   0.33    -      -     vpsubw	(%rax), %zmm17, %zmm19 {%k1}
diff --git a/llvm/test/tools/llvm-mca/X86/Znver4/zero-idioms.s b/llvm/test/tools/llvm-mca/X86/Znver4/zero-idioms.s
index b6ebd93f681fe3..30df2ed0a926d6 100644
--- a/llvm/test/tools/llvm-mca/X86/Znver4/zero-idioms.s
+++ b/llvm/test/tools/llvm-mca/X86/Znver4/zero-idioms.s
@@ -161,13 +161,13 @@ vpxorq  %zmm19, %zmm19, %zmm21
 
 # CHECK:      Iterations:        1
 # CHECK-NEXT: Instructions:      139
-# CHECK-NEXT: Total Cycles:      40
+# CHECK-NEXT: Total Cycles:      25
 # CHECK-NEXT: Total uOps:        139
 
 # CHECK:      Dispatch Width:    6
-# CHECK-NEXT: uOps Per Cycle:    3.48
-# CHECK-NEXT: IPC:               3.48
-# CHECK-NEXT: Block RThroughput: 24.8
+# CHECK-NEXT: uOps Per Cycle:    5.56
+# CHECK-NEXT: IPC:               5.56
+# CHECK-NEXT: Block RThroughput: 23.2
 
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
@@ -229,103 +229,103 @@ vpxorq  %zmm19, %zmm19, %zmm21
 # CHECK-NEXT:  1      0     0.17                        vpsubd	%ymm3, %ymm3, %ymm5
 # CHECK-NEXT:  1      0     0.17                        vpsubq	%ymm3, %ymm3, %ymm5
 # CHECK-NEXT:  1      0     0.17                        vpsubw	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT:  1      1     0.25                        vpsubb	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT:  1      1     0.25                        vpsubd	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT:  1      1     0.25                        vpsubq	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT:  1      1     0.25                        vpsubw	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT:  1      1     0.25                        vpsubb	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT:  1      1     0.25                        vpsubd	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT:  1      1     0.25                        vpsubq	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT:  1      1     0.25                        vpsubw	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT:  1      1     0.50                        vpsubb	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT:  1      1     0.50                        vpsubd	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT:  1      1     0.50                        vpsubq	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT:  1      1     0.50                        vpsubw	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT:  1      1     0.25                        vpsubb	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT:  1      1     0.25                        vpsubd	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT:  1      1     0.25                        vpsubq	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT:  1      1     0.25                        vpsubw	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT:  1      1     0.25                        vpsubb	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT:  1      1     0.25                        vpsubd	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT:  1      1     0.25                        vpsubq	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT:  1      1     0.25                        vpsubw	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT:  1      1     0.50                        vpsubb	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT:  1      1     0.50                        vpsubd	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT:  1      1     0.50                        vpsubq	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT:  1      1     0.50                        vpsubw	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:  1      0     0.17                        vpsubb	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT:  1      0     0.17                        vpsubd	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT:  1      0     0.17                        vpsubq	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT:  1      0     0.17                        vpsubw	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT:  1      0     0.17                        vpsubb	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT:  1      0     0.17                        vpsubd	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT:  1      0     0.17                        vpsubq	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT:  1      0     0.17                        vpsubw	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT:  1      0     0.17                        vpsubb	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT:  1      0     0.17                        vpsubd	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT:  1      0     0.17                        vpsubq	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT:  1      0     0.17                        vpsubw	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT:  1      0     0.17                        vpsubb	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT:  1      0     0.17                        vpsubd	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT:  1      0     0.17                        vpsubq	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT:  1      0     0.17                        vpsubw	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT:  1      0     0.17                        vpsubb	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT:  1      0     0.17                        vpsubd	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT:  1      0     0.17                        vpsubq	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT:  1      0     0.17                        vpsubw	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT:  1      0     0.17                        vpsubb	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:  1      0     0.17                        vpsubd	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:  1      0     0.17                        vpsubq	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:  1      0     0.17                        vpsubw	%zmm19, %zmm19, %zmm21
 # CHECK-NEXT:  1      1     0.25                        andnps	%xmm0, %xmm0
 # CHECK-NEXT:  1      1     0.25                        andnpd	%xmm1, %xmm1
 # CHECK-NEXT:  1      0     0.17                        vandnps	%xmm2, %xmm2, %xmm2
 # CHECK-NEXT:  1      0     0.17                        vandnpd	%xmm1, %xmm1, %xmm1
 # CHECK-NEXT:  1      0     0.17                        vandnps	%ymm2, %ymm2, %ymm2
 # CHECK-NEXT:  1      0     0.17                        vandnpd	%ymm1, %ymm1, %ymm1
-# CHECK-NEXT:  1      1     0.50                        vandnps	%zmm2, %zmm2, %zmm2
-# CHECK-NEXT:  1      1     0.50                        vandnpd	%zmm1, %zmm1, %zmm1
+# CHECK-NEXT:  1      0     0.17                        vandnps	%zmm2, %zmm2, %zmm2
+# CHECK-NEXT:  1      0     0.17                        vandnpd	%zmm1, %zmm1, %zmm1
 # CHECK-NEXT:  1      1     0.25                        pandn	%mm2, %mm2
 # CHECK-NEXT:  1      1     0.25                        pandn	%xmm2, %xmm2
 # CHECK-NEXT:  1      0     0.17                        vpandn	%xmm3, %xmm3, %xmm3
 # CHECK-NEXT:  1      0     0.17                        vpandn	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT:  1      1     0.25                        vpandnd	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT:  1      1     0.25                        vpandnq	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT:  1      1     0.25                        vpandnd	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT:  1      1     0.25                        vpandnq	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT:  1      1     0.50                        vpandnd	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT:  1      1     0.50                        vpandnq	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT:  1      0     0.17                        vpandnd	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT:  1      0     0.17                        vpandnq	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT:  1      0     0.17                        vpandnd	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT:  1      0     0.17                        vpandnq	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT:  1      0     0.17                        vpandnd	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT:  1      0     0.17                        vpandnq	%zmm19, %zmm19, %zmm19
 # CHECK-NEXT:  1      0     0.17                        vandnps	%xmm2, %xmm2, %xmm5
 # CHECK-NEXT:  1      0     0.17                        vandnpd	%xmm1, %xmm1, %xmm5
 # CHECK-NEXT:  1      0     0.17                        vpandn	%xmm3, %xmm3, %xmm5
 # CHECK-NEXT:  1      0     0.17                        vandnps	%ymm2, %ymm2, %ymm5
 # CHECK-NEXT:  1      0     0.17                        vandnpd	%ymm1, %ymm1, %ymm5
 # CHECK-NEXT:  1      0     0.17                        vpandn	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT:  1      1     0.50                        vandnps	%zmm2, %zmm2, %zmm5
-# CHECK-NEXT:  1      1     0.50                        vandnpd	%zmm1, %zmm1, %zmm5
-# CHECK-NEXT:  1      1     0.25                        vpandnd	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT:  1      1     0.25                        vpandnq	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT:  1      1     0.25                        vpandnd	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT:  1      1     0.25                        vpandnq	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT:  1      1     0.50                        vpandnd	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT:  1      1     0.50                        vpandnq	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:  1      0     0.17                        vandnps	%zmm2, %zmm2, %zmm5
+# CHECK-NEXT:  1      0     0.17                        vandnpd	%zmm1, %zmm1, %zmm5
+# CHECK-NEXT:  1      0     0.17                        vpandnd	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT:  1      0     0.17                        vpandnq	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT:  1      0     0.17                        vpandnd	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT:  1      0     0.17                        vpandnq	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT:  1      0     0.17                        vpandnd	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:  1      0     0.17                        vpandnq	%zmm19, %zmm19, %zmm21
 # CHECK-NEXT:  1      1     0.25                        xorps	%xmm0, %xmm0
 # CHECK-NEXT:  1      1     0.25                        xorpd	%xmm1, %xmm1
 # CHECK-NEXT:  1      0     0.17                        vxorps	%xmm2, %xmm2, %xmm2
 # CHECK-NEXT:  1      0     0.17                        vxorpd	%xmm1, %xmm1, %xmm1
 # CHECK-NEXT:  1      0     0.17                        vxorps	%ymm2, %ymm2, %ymm2
 # CHECK-NEXT:  1      0     0.17                        vxorpd	%ymm1, %ymm1, %ymm1
-# CHECK-NEXT:  1      1     0.50                        vxorps	%zmm2, %zmm2, %zmm2
-# CHECK-NEXT:  1      1     0.50                        vxorpd	%zmm1, %zmm1, %zmm1
+# CHECK-NEXT:  1      0     0.17                        vxorps	%zmm2, %zmm2, %zmm2
+# CHECK-NEXT:  1      0     0.17                        vxorpd	%zmm1, %zmm1, %zmm1
 # CHECK-NEXT:  1      1     0.25                        pxor	%mm2, %mm2
 # CHECK-NEXT:  1      1     0.25                        pxor	%xmm2, %xmm2
 # CHECK-NEXT:  1      0     0.17                        vpxor	%xmm3, %xmm3, %xmm3
 # CHECK-NEXT:  1      0     0.17                        vpxor	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT:  1      1     0.25                        vpxord	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT:  1      1     0.25                        vpxorq	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT:  1      1     0.25                        vpxord	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT:  1      1     0.25                        vpxorq	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT:  1      0     0.17                        vpxord	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT:  1      0     0.17                        vpxorq	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT:  1      0     0.17                        vpxord	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT:  1      0     0.17                        vpxorq	%ymm19, %ymm19, %ymm19
 # CHECK-NEXT:  1      0     0.17                        vpxord	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT:  1      1     0.50                        vpxorq	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT:  1      0     0.17                        vpxorq	%zmm19, %zmm19, %zmm19
 # CHECK-NEXT:  1      0     0.17                        vxorps	%xmm4, %xmm4, %xmm5
 # CHECK-NEXT:  1      0     0.17                        vxorpd	%xmm1, %xmm1, %xmm3
 # CHECK-NEXT:  1      0     0.17                        vxorps	%ymm4, %ymm4, %ymm5
 # CHECK-NEXT:  1      0     0.17                        vxorpd	%ymm1, %ymm1, %ymm3
-# CHECK-NEXT:  1      1     0.50                        vxorps	%zmm4, %zmm4, %zmm5
-# CHECK-NEXT:  1      1     0.50                        vxorpd	%zmm1, %zmm1, %zmm3
+# CHECK-NEXT:  1      0     0.17                        vxorps	%zmm4, %zmm4, %zmm5
+# CHECK-NEXT:  1      0     0.17                        vxorpd	%zmm1, %zmm1, %zmm3
 # CHECK-NEXT:  1      0     0.17                        vpxor	%xmm3, %xmm3, %xmm5
 # CHECK-NEXT:  1      0     0.17                        vpxor	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT:  1      1     0.25                        vpxord	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT:  1      1     0.25                        vpxorq	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT:  1      1     0.25                        vpxord	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT:  1      1     0.25                        vpxorq	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT:  1      0     0.17                        vpxord	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT:  1      0     0.17                        vpxorq	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT:  1      0     0.17                        vpxord	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT:  1      0     0.17                        vpxorq	%ymm19, %ymm19, %ymm21
 # CHECK-NEXT:  1      0     0.17                        vpxord	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT:  1      1     0.50                        vpxorq	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:  1      0     0.17                        vpxorq	%zmm19, %zmm19, %zmm21
 
 # CHECK:      Register File statistics:
-# CHECK-NEXT: Total number of mappings created:    63
-# CHECK-NEXT: Max number of mappings used:         43
+# CHECK-NEXT: Total number of mappings created:    9
+# CHECK-NEXT: Max number of mappings used:         4
 
 # CHECK:      *  Register File #1 -- Zn4FpPRF:
 # CHECK-NEXT:    Number of physical registers:     192
-# CHECK-NEXT:    Total number of mappings created: 63
-# CHECK-NEXT:    Max number of mappings used:      43
+# CHECK-NEXT:    Total number of mappings created: 9
+# CHECK-NEXT:    Max number of mappings used:      4
 
 # CHECK:      *  Register File #2 -- Zn4IntegerPRF:
 # CHECK-NEXT:    Number of physical registers:     224
@@ -359,7 +359,7 @@ vpxorq  %zmm19, %zmm19, %zmm21
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12.0] [12.1] [13]   [14.0] [14.1] [14.2] [15.0] [15.1] [15.2] [16.0] [16.1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     24.00  25.00  25.00  25.00   -      -      -      -      -      -      -      -      -      -      -
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     5.00   6.00   6.00   6.00    -      -      -      -      -      -      -      -      -      -      -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12.0] [12.1] [13]   [14.0] [14.1] [14.2] [15.0] [15.1] [15.2] [16.0] [16.1] Instructions:
@@ -414,238 +414,238 @@ vpxorq  %zmm19, %zmm19, %zmm21
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubd	%ymm3, %ymm3, %ymm5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubq	%ymm3, %ymm3, %ymm5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubw	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubb	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -     vpsubd	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -     vpsubq	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubw	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubb	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubd	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubq	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubw	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubb	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubd	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubq	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubw	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubb	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -     vpsubd	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -     vpsubq	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubw	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubb	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -     vpsubd	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -     vpsubq	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubw	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubb	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -     vpsubd	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -      -     vpsubq	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubw	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -     andnps	%xmm0, %xmm0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -     andnpd	%xmm1, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubb	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubd	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubq	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubw	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubb	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubd	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubq	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubw	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubb	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubd	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubq	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubw	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubb	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubd	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubq	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubw	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubb	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubd	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubq	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubw	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubb	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubd	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubq	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpsubw	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     andnps	%xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -     andnpd	%xmm1, %xmm1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vandnps	%xmm2, %xmm2, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vandnpd	%xmm1, %xmm1, %xmm1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vandnps	%ymm2, %ymm2, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vandnpd	%ymm1, %ymm1, %ymm1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     vandnps	%zmm2, %zmm2, %zmm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -     vandnpd	%zmm1, %zmm1, %zmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vandnps	%zmm2, %zmm2, %zmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vandnpd	%zmm1, %zmm1, %zmm1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -     pandn	%mm2, %mm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -     pandn	%xmm2, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     pandn	%xmm2, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandn	%xmm3, %xmm3, %xmm3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandn	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnd	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -     vpandnq	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -     vpandnd	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -     vpandnq	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnd	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnq	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnd	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnq	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnd	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnq	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnd	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnq	%zmm19, %zmm19, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vandnps	%xmm2, %xmm2, %xmm5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vandnpd	%xmm1, %xmm1, %xmm5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandn	%xmm3, %xmm3, %xmm5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vandnps	%ymm2, %ymm2, %ymm5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vandnpd	%ymm1, %ymm1, %ymm5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandn	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -     vandnps	%zmm2, %zmm2, %zmm5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -      -     vandnpd	%zmm1, %zmm1, %zmm5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -     vpandnd	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnq	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnd	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -     vpandnq	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -      -     vpandnd	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnq	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -     xorps	%xmm0, %xmm0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -     xorpd	%xmm1, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vandnps	%zmm2, %zmm2, %zmm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vandnpd	%zmm1, %zmm1, %zmm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnd	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnq	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnd	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnq	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnd	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpandnq	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     xorps	%xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -     xorpd	%xmm1, %xmm1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vxorps	%xmm2, %xmm2, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vxorpd	%xmm1, %xmm1, %xmm1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vxorps	%ymm2, %ymm2, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vxorpd	%ymm1, %ymm1, %ymm1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -     vxorps	%zmm2, %zmm2, %zmm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -      -     vxorpd	%zmm1, %zmm1, %zmm1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     pxor	%mm2, %mm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     pxor	%xmm2, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vxorps	%zmm2, %zmm2, %zmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vxorpd	%zmm1, %zmm1, %zmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -     pxor	%mm2, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     pxor	%xmm2, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxor	%xmm3, %xmm3, %xmm3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxor	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxord	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -     vpxorq	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -     vpxord	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vpxorq	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxord	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxorq	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxord	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxorq	%ymm19, %ymm19, %ymm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxord	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -      -     vpxorq	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxorq	%zmm19, %zmm19, %zmm19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vxorps	%xmm4, %xmm4, %xmm5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vxorpd	%xmm1, %xmm1, %xmm3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vxorps	%ymm4, %ymm4, %ymm5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vxorpd	%ymm1, %ymm1, %ymm3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vxorps	%zmm4, %zmm4, %zmm5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     vxorpd	%zmm1, %zmm1, %zmm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vxorps	%zmm4, %zmm4, %zmm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vxorpd	%zmm1, %zmm1, %zmm3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxor	%xmm3, %xmm3, %xmm5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxor	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -     vpxord	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxorq	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -     vpxord	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -     vpxorq	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxord	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxorq	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxord	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxorq	%ymm19, %ymm19, %ymm21
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxord	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -      -      -      -      -     vpxorq	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpxorq	%zmm19, %zmm19, %zmm21
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DR   .    .    .    .    .    .    .   .   subl	%eax, %eax
-# CHECK-NEXT: [0,1]     DR   .    .    .    .    .    .    .   .   subq	%rax, %rax
-# CHECK-NEXT: [0,2]     DR   .    .    .    .    .    .    .   .   xorl	%eax, %eax
-# CHECK-NEXT: [0,3]     DR   .    .    .    .    .    .    .   .   xorq	%rax, %rax
-# CHECK-NEXT: [0,4]     DeER .    .    .    .    .    .    .   .   pcmpgtb	%mm2, %mm2
-# CHECK-NEXT: [0,5]     D=eER.    .    .    .    .    .    .   .   pcmpgtd	%mm2, %mm2
-# CHECK-NEXT: [0,6]     .D=eER    .    .    .    .    .    .   .   pcmpgtw	%mm2, %mm2
-# CHECK-NEXT: [0,7]     .DeE-R    .    .    .    .    .    .   .   pcmpgtb	%xmm2, %xmm2
-# CHECK-NEXT: [0,8]     .DeE-R    .    .    .    .    .    .   .   pcmpgtd	%xmm2, %xmm2
-# CHECK-NEXT: [0,9]     .DeE-R    .    .    .    .    .    .   .   pcmpgtq	%xmm2, %xmm2
-# CHECK-NEXT: [0,10]    .D=eER    .    .    .    .    .    .   .   pcmpgtw	%xmm2, %xmm2
-# CHECK-NEXT: [0,11]    .D---R    .    .    .    .    .    .   .   vpcmpgtb	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,12]    . D--R    .    .    .    .    .    .   .   vpcmpgtd	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,13]    . D--R    .    .    .    .    .    .   .   vpcmpgtq	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,14]    . D--R    .    .    .    .    .    .   .   vpcmpgtw	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,15]    . D---R   .    .    .    .    .    .   .   vpcmpgtb	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,16]    . D---R   .    .    .    .    .    .   .   vpcmpgtd	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,17]    . D---R   .    .    .    .    .    .   .   vpcmpgtq	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,18]    .  D--R   .    .    .    .    .    .   .   vpcmpgtw	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,19]    .  D--R   .    .    .    .    .    .   .   vpcmpgtb	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,20]    .  D--R   .    .    .    .    .    .   .   vpcmpgtd	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,21]    .  D--R   .    .    .    .    .    .   .   vpcmpgtq	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,22]    .  D--R   .    .    .    .    .    .   .   vpcmpgtw	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,23]    .  D--R   .    .    .    .    .    .   .   vpcmpgtb	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,24]    .   D--R  .    .    .    .    .    .   .   vpcmpgtd	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,25]    .   D--R  .    .    .    .    .    .   .   vpcmpgtq	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,26]    .   D--R  .    .    .    .    .    .   .   vpcmpgtw	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,27]    .   DeER  .    .    .    .    .    .   .   psubb	%mm2, %mm2
-# CHECK-NEXT: [0,28]    .   D=eER .    .    .    .    .    .   .   psubd	%mm2, %mm2
-# CHECK-NEXT: [0,29]    .   D==eER.    .    .    .    .    .   .   psubq	%mm2, %mm2
-# CHECK-NEXT: [0,30]    .    D==eER    .    .    .    .    .   .   psubw	%mm2, %mm2
-# CHECK-NEXT: [0,31]    .    DeE--R    .    .    .    .    .   .   psubb	%xmm2, %xmm2
-# CHECK-NEXT: [0,32]    .    DeE--R    .    .    .    .    .   .   psubd	%xmm2, %xmm2
-# CHECK-NEXT: [0,33]    .    DeE--R    .    .    .    .    .   .   psubq	%xmm2, %xmm2
-# CHECK-NEXT: [0,34]    .    D=eE-R    .    .    .    .    .   .   psubw	%xmm2, %xmm2
-# CHECK-NEXT: [0,35]    .    D----R    .    .    .    .    .   .   vpsubb	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,36]    .    .D---R    .    .    .    .    .   .   vpsubd	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,37]    .    .D---R    .    .    .    .    .   .   vpsubq	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,38]    .    .D---R    .    .    .    .    .   .   vpsubw	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,39]    .    .D----R   .    .    .    .    .   .   vpsubb	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,40]    .    .D----R   .    .    .    .    .   .   vpsubd	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,41]    .    .D----R   .    .    .    .    .   .   vpsubq	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,42]    .    . D---R   .    .    .    .    .   .   vpsubw	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,43]    .    . D---R   .    .    .    .    .   .   vpsubb	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,44]    .    . D---R   .    .    .    .    .   .   vpsubd	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,45]    .    . D---R   .    .    .    .    .   .   vpsubq	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,46]    .    . D---R   .    .    .    .    .   .   vpsubw	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,47]    .    . D---R   .    .    .    .    .   .   vpsubb	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,48]    .    .  D---R  .    .    .    .    .   .   vpsubd	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,49]    .    .  D---R  .    .    .    .    .   .   vpsubq	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,50]    .    .  D---R  .    .    .    .    .   .   vpsubw	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,51]    .    .  DeE-R  .    .    .    .    .   .   vpsubb	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: [0,52]    .    .  D=eER  .    .    .    .    .   .   vpsubd	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: [0,53]    .    .  D==eER .    .    .    .    .   .   vpsubq	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: [0,54]    .    .   D==eER.    .    .    .    .   .   vpsubw	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: [0,55]    .    .   D===eER    .    .    .    .   .   vpsubb	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: [0,56]    .    .   D====eER   .    .    .    .   .   vpsubd	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: [0,57]    .    .   D=====eER  .    .    .    .   .   vpsubq	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: [0,58]    .    .   D======eER .    .    .    .   .   vpsubw	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: [0,59]    .    .   D=======eER.    .    .    .   .   vpsubb	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: [0,60]    .    .    D=======eER    .    .    .   .   vpsubd	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: [0,61]    .    .    D========eER   .    .    .   .   vpsubq	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: [0,62]    .    .    D=========eER  .    .    .   .   vpsubw	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: [0,63]    .    .    D==========eER .    .    .   .   vpsubb	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: [0,64]    .    .    D===========eER.    .    .   .   vpsubd	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: [0,65]    .    .    D===========eER.    .    .   .   vpsubq	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: [0,66]    .    .    .D==========eER.    .    .   .   vpsubw	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: [0,67]    .    .    .D==========eER.    .    .   .   vpsubb	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: [0,68]    .    .    .D===========eER    .    .   .   vpsubd	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: [0,69]    .    .    .D===========eER    .    .   .   vpsubq	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: [0,70]    .    .    .D===========eER    .    .   .   vpsubw	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: [0,71]    .    .    .D===========eER    .    .   .   vpsubb	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: [0,72]    .    .    . D===========eER   .    .   .   vpsubd	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: [0,73]    .    .    . D===========eER   .    .   .   vpsubq	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: [0,74]    .    .    . D===========eER   .    .   .   vpsubw	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: [0,75]    .    .    . DeE-----------R   .    .   .   andnps	%xmm0, %xmm0
-# CHECK-NEXT: [0,76]    .    .    . DeE-----------R   .    .   .   andnpd	%xmm1, %xmm1
-# CHECK-NEXT: [0,77]    .    .    . D-------------R   .    .   .   vandnps	%xmm2, %xmm2, %xmm2
-# CHECK-NEXT: [0,78]    .    .    .  D------------R   .    .   .   vandnpd	%xmm1, %xmm1, %xmm1
-# CHECK-NEXT: [0,79]    .    .    .  D------------R   .    .   .   vandnps	%ymm2, %ymm2, %ymm2
-# CHECK-NEXT: [0,80]    .    .    .  D------------R   .    .   .   vandnpd	%ymm1, %ymm1, %ymm1
-# CHECK-NEXT: [0,81]    .    .    .  DeE-----------R  .    .   .   vandnps	%zmm2, %zmm2, %zmm2
-# CHECK-NEXT: [0,82]    .    .    .  DeE-----------R  .    .   .   vandnpd	%zmm1, %zmm1, %zmm1
-# CHECK-NEXT: [0,83]    .    .    .  DeE-----------R  .    .   .   pandn	%mm2, %mm2
-# CHECK-NEXT: [0,84]    .    .    .   DeE----------R  .    .   .   pandn	%xmm2, %xmm2
-# CHECK-NEXT: [0,85]    .    .    .   D------------R  .    .   .   vpandn	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,86]    .    .    .   D------------R  .    .   .   vpandn	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,87]    .    .    .   D==========eER  .    .   .   vpandnd	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: [0,88]    .    .    .   D===========eER .    .   .   vpandnq	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: [0,89]    .    .    .   D============eER.    .   .   vpandnd	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: [0,90]    .    .    .    D============eER    .   .   vpandnq	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: [0,91]    .    .    .    D=============eER   .   .   vpandnd	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: [0,92]    .    .    .    D==============eER  .   .   vpandnq	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: [0,93]    .    .    .    D----------------R  .   .   vandnps	%xmm2, %xmm2, %xmm5
-# CHECK-NEXT: [0,94]    .    .    .    D----------------R  .   .   vandnpd	%xmm1, %xmm1, %xmm5
-# CHECK-NEXT: [0,95]    .    .    .    D----------------R  .   .   vpandn	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,96]    .    .    .    .D---------------R  .   .   vandnps	%ymm2, %ymm2, %ymm5
-# CHECK-NEXT: [0,97]    .    .    .    .D---------------R  .   .   vandnpd	%ymm1, %ymm1, %ymm5
-# CHECK-NEXT: [0,98]    .    .    .    .D---------------R  .   .   vpandn	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,99]    .    .    .    .DeE-------------R  .   .   vandnps	%zmm2, %zmm2, %zmm5
-# CHECK-NEXT: [0,100]   .    .    .    .DeE-------------R  .   .   vandnpd	%zmm1, %zmm1, %zmm5
-# CHECK-NEXT: [0,101]   .    .    .    .D==============eER .   .   vpandnd	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: [0,102]   .    .    .    . D=============eER .   .   vpandnq	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: [0,103]   .    .    .    . D==============eER.   .   vpandnd	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: [0,104]   .    .    .    . D==============eER.   .   vpandnq	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: [0,105]   .    .    .    . D==============eER.   .   vpandnd	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: [0,106]   .    .    .    . D==============eER.   .   vpandnq	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: [0,107]   .    .    .    . D=eE-------------R.   .   xorps	%xmm0, %xmm0
-# CHECK-NEXT: [0,108]   .    .    .    .  DeE-------------R.   .   xorpd	%xmm1, %xmm1
-# CHECK-NEXT: [0,109]   .    .    .    .  D---------------R.   .   vxorps	%xmm2, %xmm2, %xmm2
-# CHECK-NEXT: [0,110]   .    .    .    .  D---------------R.   .   vxorpd	%xmm1, %xmm1, %xmm1
-# CHECK-NEXT: [0,111]   .    .    .    .  D---------------R.   .   vxorps	%ymm2, %ymm2, %ymm2
-# CHECK-NEXT: [0,112]   .    .    .    .  D----------------R   .   vxorpd	%ymm1, %ymm1, %ymm1
-# CHECK-NEXT: [0,113]   .    .    .    .  D=eE-------------R   .   vxorps	%zmm2, %zmm2, %zmm2
-# CHECK-NEXT: [0,114]   .    .    .    .   DeE-------------R   .   vxorpd	%zmm1, %zmm1, %zmm1
-# CHECK-NEXT: [0,115]   .    .    .    .   D======eE-------R   .   pxor	%mm2, %mm2
-# CHECK-NEXT: [0,116]   .    .    .    .   D======eE-------R   .   pxor	%xmm2, %xmm2
-# CHECK-NEXT: [0,117]   .    .    .    .   D---------------R   .   vpxor	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,118]   .    .    .    .   D---------------R   .   vpxor	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,119]   .    .    .    .   D=============eER   .   vpxord	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: [0,120]   .    .    .    .    D=============eER  .   vpxorq	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: [0,121]   .    .    .    .    D==============eER .   vpxord	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: [0,122]   .    .    .    .    D===============eER.   vpxorq	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: [0,123]   .    .    .    .    D-----------------R.   vpxord	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: [0,124]   .    .    .    .    D=====eE----------R.   vpxorq	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: [0,125]   .    .    .    .    D-----------------R.   vxorps	%xmm4, %xmm4, %xmm5
-# CHECK-NEXT: [0,126]   .    .    .    .    .D----------------R.   vxorpd	%xmm1, %xmm1, %xmm3
-# CHECK-NEXT: [0,127]   .    .    .    .    .D----------------R.   vxorps	%ymm4, %ymm4, %ymm5
-# CHECK-NEXT: [0,128]   .    .    .    .    .D----------------R.   vxorpd	%ymm1, %ymm1, %ymm3
-# CHECK-NEXT: [0,129]   .    .    .    .    .D=====eE---------R.   vxorps	%zmm4, %zmm4, %zmm5
-# CHECK-NEXT: [0,130]   .    .    .    .    .D=====eE---------R.   vxorpd	%zmm1, %zmm1, %zmm3
-# CHECK-NEXT: [0,131]   .    .    .    .    .D-----------------R   vpxor	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,132]   .    .    .    .    . D----------------R   vpxor	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,133]   .    .    .    .    . D=====eE---------R   vpxord	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: [0,134]   .    .    .    .    . D======eE--------R   vpxorq	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: [0,135]   .    .    .    .    . D======eE--------R   vpxord	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: [0,136]   .    .    .    .    . D======eE--------R   vpxorq	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: [0,137]   .    .    .    .    . D----------------R   vpxord	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: [0,138]   .    .    .    .    .  D======eE-------R   vpxorq	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          01234
+
+# CHECK:      [0,0]     DR   .    .    .    .   .   subl	%eax, %eax
+# CHECK-NEXT: [0,1]     DR   .    .    .    .   .   subq	%rax, %rax
+# CHECK-NEXT: [0,2]     DR   .    .    .    .   .   xorl	%eax, %eax
+# CHECK-NEXT: [0,3]     DR   .    .    .    .   .   xorq	%rax, %rax
+# CHECK-NEXT: [0,4]     DeER .    .    .    .   .   pcmpgtb	%mm2, %mm2
+# CHECK-NEXT: [0,5]     D=eER.    .    .    .   .   pcmpgtd	%mm2, %mm2
+# CHECK-NEXT: [0,6]     .D=eER    .    .    .   .   pcmpgtw	%mm2, %mm2
+# CHECK-NEXT: [0,7]     .DeE-R    .    .    .   .   pcmpgtb	%xmm2, %xmm2
+# CHECK-NEXT: [0,8]     .DeE-R    .    .    .   .   pcmpgtd	%xmm2, %xmm2
+# CHECK-NEXT: [0,9]     .DeE-R    .    .    .   .   pcmpgtq	%xmm2, %xmm2
+# CHECK-NEXT: [0,10]    .D=eER    .    .    .   .   pcmpgtw	%xmm2, %xmm2
+# CHECK-NEXT: [0,11]    .D---R    .    .    .   .   vpcmpgtb	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,12]    . D--R    .    .    .   .   vpcmpgtd	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,13]    . D--R    .    .    .   .   vpcmpgtq	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,14]    . D--R    .    .    .   .   vpcmpgtw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,15]    . D---R   .    .    .   .   vpcmpgtb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,16]    . D---R   .    .    .   .   vpcmpgtd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,17]    . D---R   .    .    .   .   vpcmpgtq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,18]    .  D--R   .    .    .   .   vpcmpgtw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,19]    .  D--R   .    .    .   .   vpcmpgtb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,20]    .  D--R   .    .    .   .   vpcmpgtd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,21]    .  D--R   .    .    .   .   vpcmpgtq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,22]    .  D--R   .    .    .   .   vpcmpgtw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,23]    .  D--R   .    .    .   .   vpcmpgtb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,24]    .   D--R  .    .    .   .   vpcmpgtd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,25]    .   D--R  .    .    .   .   vpcmpgtq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,26]    .   D--R  .    .    .   .   vpcmpgtw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,27]    .   DeER  .    .    .   .   psubb	%mm2, %mm2
+# CHECK-NEXT: [0,28]    .   D=eER .    .    .   .   psubd	%mm2, %mm2
+# CHECK-NEXT: [0,29]    .   D==eER.    .    .   .   psubq	%mm2, %mm2
+# CHECK-NEXT: [0,30]    .    D==eER    .    .   .   psubw	%mm2, %mm2
+# CHECK-NEXT: [0,31]    .    DeE--R    .    .   .   psubb	%xmm2, %xmm2
+# CHECK-NEXT: [0,32]    .    DeE--R    .    .   .   psubd	%xmm2, %xmm2
+# CHECK-NEXT: [0,33]    .    DeE--R    .    .   .   psubq	%xmm2, %xmm2
+# CHECK-NEXT: [0,34]    .    D=eE-R    .    .   .   psubw	%xmm2, %xmm2
+# CHECK-NEXT: [0,35]    .    D----R    .    .   .   vpsubb	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,36]    .    .D---R    .    .   .   vpsubd	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,37]    .    .D---R    .    .   .   vpsubq	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,38]    .    .D---R    .    .   .   vpsubw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,39]    .    .D----R   .    .   .   vpsubb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,40]    .    .D----R   .    .   .   vpsubd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,41]    .    .D----R   .    .   .   vpsubq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,42]    .    . D---R   .    .   .   vpsubw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,43]    .    . D---R   .    .   .   vpsubb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,44]    .    . D---R   .    .   .   vpsubd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,45]    .    . D---R   .    .   .   vpsubq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,46]    .    . D---R   .    .   .   vpsubw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,47]    .    . D---R   .    .   .   vpsubb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,48]    .    .  D---R  .    .   .   vpsubd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,49]    .    .  D---R  .    .   .   vpsubq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,50]    .    .  D---R  .    .   .   vpsubw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,51]    .    .  D---R  .    .   .   vpsubb	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: [0,52]    .    .  D---R  .    .   .   vpsubd	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: [0,53]    .    .  D---R  .    .   .   vpsubq	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: [0,54]    .    .   D--R  .    .   .   vpsubw	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: [0,55]    .    .   D--R  .    .   .   vpsubb	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: [0,56]    .    .   D--R  .    .   .   vpsubd	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: [0,57]    .    .   D---R .    .   .   vpsubq	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: [0,58]    .    .   D---R .    .   .   vpsubw	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: [0,59]    .    .   D---R .    .   .   vpsubb	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: [0,60]    .    .    D--R .    .   .   vpsubd	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: [0,61]    .    .    D--R .    .   .   vpsubq	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: [0,62]    .    .    D--R .    .   .   vpsubw	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: [0,63]    .    .    D--R .    .   .   vpsubb	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: [0,64]    .    .    D--R .    .   .   vpsubd	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: [0,65]    .    .    D--R .    .   .   vpsubq	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: [0,66]    .    .    .D--R.    .   .   vpsubw	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: [0,67]    .    .    .D--R.    .   .   vpsubb	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: [0,68]    .    .    .D--R.    .   .   vpsubd	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: [0,69]    .    .    .D--R.    .   .   vpsubq	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: [0,70]    .    .    .D--R.    .   .   vpsubw	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: [0,71]    .    .    .D--R.    .   .   vpsubb	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: [0,72]    .    .    . D-R.    .   .   vpsubd	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: [0,73]    .    .    . D-R.    .   .   vpsubq	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: [0,74]    .    .    . D-R.    .   .   vpsubw	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: [0,75]    .    .    . DeER    .   .   andnps	%xmm0, %xmm0
+# CHECK-NEXT: [0,76]    .    .    . DeER    .   .   andnpd	%xmm1, %xmm1
+# CHECK-NEXT: [0,77]    .    .    . D--R    .   .   vandnps	%xmm2, %xmm2, %xmm2
+# CHECK-NEXT: [0,78]    .    .    .  D-R    .   .   vandnpd	%xmm1, %xmm1, %xmm1
+# CHECK-NEXT: [0,79]    .    .    .  D-R    .   .   vandnps	%ymm2, %ymm2, %ymm2
+# CHECK-NEXT: [0,80]    .    .    .  D-R    .   .   vandnpd	%ymm1, %ymm1, %ymm1
+# CHECK-NEXT: [0,81]    .    .    .  D-R    .   .   vandnps	%zmm2, %zmm2, %zmm2
+# CHECK-NEXT: [0,82]    .    .    .  D-R    .   .   vandnpd	%zmm1, %zmm1, %zmm1
+# CHECK-NEXT: [0,83]    .    .    .  DeER   .   .   pandn	%mm2, %mm2
+# CHECK-NEXT: [0,84]    .    .    .   DeER  .   .   pandn	%xmm2, %xmm2
+# CHECK-NEXT: [0,85]    .    .    .   D--R  .   .   vpandn	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,86]    .    .    .   D--R  .   .   vpandn	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,87]    .    .    .   D--R  .   .   vpandnd	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: [0,88]    .    .    .   D--R  .   .   vpandnq	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: [0,89]    .    .    .   D--R  .   .   vpandnd	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: [0,90]    .    .    .    D-R  .   .   vpandnq	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: [0,91]    .    .    .    D-R  .   .   vpandnd	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: [0,92]    .    .    .    D-R  .   .   vpandnq	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: [0,93]    .    .    .    D--R .   .   vandnps	%xmm2, %xmm2, %xmm5
+# CHECK-NEXT: [0,94]    .    .    .    D--R .   .   vandnpd	%xmm1, %xmm1, %xmm5
+# CHECK-NEXT: [0,95]    .    .    .    D--R .   .   vpandn	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,96]    .    .    .    .D-R .   .   vandnps	%ymm2, %ymm2, %ymm5
+# CHECK-NEXT: [0,97]    .    .    .    .D-R .   .   vandnpd	%ymm1, %ymm1, %ymm5
+# CHECK-NEXT: [0,98]    .    .    .    .D-R .   .   vpandn	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,99]    .    .    .    .D-R .   .   vandnps	%zmm2, %zmm2, %zmm5
+# CHECK-NEXT: [0,100]   .    .    .    .D-R .   .   vandnpd	%zmm1, %zmm1, %zmm5
+# CHECK-NEXT: [0,101]   .    .    .    .D-R .   .   vpandnd	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: [0,102]   .    .    .    . D-R.   .   vpandnq	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: [0,103]   .    .    .    . D-R.   .   vpandnd	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: [0,104]   .    .    .    . D-R.   .   vpandnq	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: [0,105]   .    .    .    . D-R.   .   vpandnd	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: [0,106]   .    .    .    . D-R.   .   vpandnq	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: [0,107]   .    .    .    . DeER   .   xorps	%xmm0, %xmm0
+# CHECK-NEXT: [0,108]   .    .    .    .  DeER  .   xorpd	%xmm1, %xmm1
+# CHECK-NEXT: [0,109]   .    .    .    .  D--R  .   vxorps	%xmm2, %xmm2, %xmm2
+# CHECK-NEXT: [0,110]   .    .    .    .  D--R  .   vxorpd	%xmm1, %xmm1, %xmm1
+# CHECK-NEXT: [0,111]   .    .    .    .  D--R  .   vxorps	%ymm2, %ymm2, %ymm2
+# CHECK-NEXT: [0,112]   .    .    .    .  D--R  .   vxorpd	%ymm1, %ymm1, %ymm1
+# CHECK-NEXT: [0,113]   .    .    .    .  D--R  .   vxorps	%zmm2, %zmm2, %zmm2
+# CHECK-NEXT: [0,114]   .    .    .    .   D-R  .   vxorpd	%zmm1, %zmm1, %zmm1
+# CHECK-NEXT: [0,115]   .    .    .    .   DeER .   pxor	%mm2, %mm2
+# CHECK-NEXT: [0,116]   .    .    .    .   DeER .   pxor	%xmm2, %xmm2
+# CHECK-NEXT: [0,117]   .    .    .    .   D--R .   vpxor	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,118]   .    .    .    .   D--R .   vpxor	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,119]   .    .    .    .   D--R .   vpxord	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: [0,120]   .    .    .    .    D-R .   vpxorq	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: [0,121]   .    .    .    .    D-R .   vpxord	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: [0,122]   .    .    .    .    D-R .   vpxorq	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: [0,123]   .    .    .    .    D-R .   vpxord	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: [0,124]   .    .    .    .    D--R.   vpxorq	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: [0,125]   .    .    .    .    D--R.   vxorps	%xmm4, %xmm4, %xmm5
+# CHECK-NEXT: [0,126]   .    .    .    .    .D-R.   vxorpd	%xmm1, %xmm1, %xmm3
+# CHECK-NEXT: [0,127]   .    .    .    .    .D-R.   vxorps	%ymm4, %ymm4, %ymm5
+# CHECK-NEXT: [0,128]   .    .    .    .    .D-R.   vxorpd	%ymm1, %ymm1, %ymm3
+# CHECK-NEXT: [0,129]   .    .    .    .    .D-R.   vxorps	%zmm4, %zmm4, %zmm5
+# CHECK-NEXT: [0,130]   .    .    .    .    .D-R.   vxorpd	%zmm1, %zmm1, %zmm3
+# CHECK-NEXT: [0,131]   .    .    .    .    .D-R.   vpxor	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,132]   .    .    .    .    . DR.   vpxor	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,133]   .    .    .    .    . D-R   vpxord	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: [0,134]   .    .    .    .    . D-R   vpxorq	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: [0,135]   .    .    .    .    . D-R   vpxord	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: [0,136]   .    .    .    .    . D-R   vpxorq	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: [0,137]   .    .    .    .    . D-R   vpxord	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: [0,138]   .    .    .    .    .  DR   vpxorq	%zmm19, %zmm19, %zmm21
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -705,92 +705,92 @@ vpxorq  %zmm19, %zmm19, %zmm21
 # CHECK-NEXT: 48.    1     0.0    0.0    3.0       vpsubd	%ymm3, %ymm3, %ymm5
 # CHECK-NEXT: 49.    1     0.0    0.0    3.0       vpsubq	%ymm3, %ymm3, %ymm5
 # CHECK-NEXT: 50.    1     0.0    0.0    3.0       vpsubw	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 51.    1     1.0    1.0    1.0       vpsubb	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: 52.    1     2.0    0.0    0.0       vpsubd	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: 53.    1     3.0    0.0    0.0       vpsubq	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: 54.    1     3.0    0.0    0.0       vpsubw	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: 55.    1     4.0    0.0    0.0       vpsubb	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: 56.    1     5.0    0.0    0.0       vpsubd	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: 57.    1     6.0    0.0    0.0       vpsubq	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: 58.    1     7.0    0.0    0.0       vpsubw	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: 59.    1     8.0    0.0    0.0       vpsubb	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: 60.    1     8.0    0.0    0.0       vpsubd	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: 61.    1     9.0    0.0    0.0       vpsubq	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: 62.    1     10.0   0.0    0.0       vpsubw	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: 63.    1     11.0   0.0    0.0       vpsubb	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: 64.    1     12.0   1.0    0.0       vpsubd	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: 65.    1     12.0   1.0    0.0       vpsubq	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: 66.    1     11.0   1.0    0.0       vpsubw	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: 67.    1     11.0   1.0    0.0       vpsubb	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: 68.    1     12.0   2.0    0.0       vpsubd	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: 69.    1     12.0   2.0    0.0       vpsubq	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: 70.    1     12.0   2.0    0.0       vpsubw	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: 71.    1     12.0   2.0    0.0       vpsubb	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: 72.    1     12.0   3.0    0.0       vpsubd	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: 73.    1     12.0   3.0    0.0       vpsubq	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: 74.    1     12.0   3.0    0.0       vpsubw	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: 75.    1     1.0    1.0    11.0      andnps	%xmm0, %xmm0
-# CHECK-NEXT: 76.    1     1.0    1.0    11.0      andnpd	%xmm1, %xmm1
-# CHECK-NEXT: 77.    1     0.0    0.0    13.0      vandnps	%xmm2, %xmm2, %xmm2
-# CHECK-NEXT: 78.    1     0.0    0.0    12.0      vandnpd	%xmm1, %xmm1, %xmm1
-# CHECK-NEXT: 79.    1     0.0    0.0    12.0      vandnps	%ymm2, %ymm2, %ymm2
-# CHECK-NEXT: 80.    1     0.0    0.0    12.0      vandnpd	%ymm1, %ymm1, %ymm1
-# CHECK-NEXT: 81.    1     1.0    1.0    11.0      vandnps	%zmm2, %zmm2, %zmm2
-# CHECK-NEXT: 82.    1     1.0    1.0    11.0      vandnpd	%zmm1, %zmm1, %zmm1
-# CHECK-NEXT: 83.    1     1.0    1.0    11.0      pandn	%mm2, %mm2
-# CHECK-NEXT: 84.    1     1.0    1.0    10.0      pandn	%xmm2, %xmm2
-# CHECK-NEXT: 85.    1     0.0    0.0    12.0      vpandn	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: 86.    1     0.0    0.0    12.0      vpandn	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 87.    1     11.0   4.0    0.0       vpandnd	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: 88.    1     12.0   0.0    0.0       vpandnq	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: 89.    1     13.0   0.0    0.0       vpandnd	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: 90.    1     13.0   0.0    0.0       vpandnq	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: 91.    1     14.0   0.0    0.0       vpandnd	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: 92.    1     15.0   0.0    0.0       vpandnq	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: 93.    1     0.0    0.0    16.0      vandnps	%xmm2, %xmm2, %xmm5
-# CHECK-NEXT: 94.    1     0.0    0.0    16.0      vandnpd	%xmm1, %xmm1, %xmm5
-# CHECK-NEXT: 95.    1     0.0    0.0    16.0      vpandn	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 96.    1     0.0    0.0    15.0      vandnps	%ymm2, %ymm2, %ymm5
-# CHECK-NEXT: 97.    1     0.0    0.0    15.0      vandnpd	%ymm1, %ymm1, %ymm5
-# CHECK-NEXT: 98.    1     0.0    0.0    15.0      vpandn	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 99.    1     1.0    1.0    13.0      vandnps	%zmm2, %zmm2, %zmm5
-# CHECK-NEXT: 100.   1     1.0    1.0    13.0      vandnpd	%zmm1, %zmm1, %zmm5
-# CHECK-NEXT: 101.   1     15.0   0.0    0.0       vpandnd	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: 102.   1     14.0   0.0    0.0       vpandnq	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: 103.   1     15.0   1.0    0.0       vpandnd	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: 104.   1     15.0   1.0    0.0       vpandnq	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: 105.   1     15.0   1.0    0.0       vpandnd	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: 106.   1     15.0   1.0    0.0       vpandnq	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: 107.   1     2.0    2.0    13.0      xorps	%xmm0, %xmm0
-# CHECK-NEXT: 108.   1     1.0    1.0    13.0      xorpd	%xmm1, %xmm1
-# CHECK-NEXT: 109.   1     0.0    0.0    15.0      vxorps	%xmm2, %xmm2, %xmm2
-# CHECK-NEXT: 110.   1     0.0    0.0    15.0      vxorpd	%xmm1, %xmm1, %xmm1
-# CHECK-NEXT: 111.   1     0.0    0.0    15.0      vxorps	%ymm2, %ymm2, %ymm2
-# CHECK-NEXT: 112.   1     0.0    0.0    16.0      vxorpd	%ymm1, %ymm1, %ymm1
-# CHECK-NEXT: 113.   1     2.0    2.0    13.0      vxorps	%zmm2, %zmm2, %zmm2
-# CHECK-NEXT: 114.   1     1.0    1.0    13.0      vxorpd	%zmm1, %zmm1, %zmm1
-# CHECK-NEXT: 115.   1     7.0    7.0    7.0       pxor	%mm2, %mm2
-# CHECK-NEXT: 116.   1     7.0    7.0    7.0       pxor	%xmm2, %xmm2
-# CHECK-NEXT: 117.   1     0.0    0.0    15.0      vpxor	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: 118.   1     0.0    0.0    15.0      vpxor	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 119.   1     14.0   2.0    0.0       vpxord	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: 120.   1     14.0   0.0    0.0       vpxorq	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: 121.   1     15.0   0.0    0.0       vpxord	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: 122.   1     16.0   0.0    0.0       vpxorq	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: 123.   1     0.0    0.0    17.0      vpxord	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: 124.   1     6.0    6.0    10.0      vpxorq	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: 125.   1     0.0    0.0    17.0      vxorps	%xmm4, %xmm4, %xmm5
-# CHECK-NEXT: 126.   1     0.0    0.0    16.0      vxorpd	%xmm1, %xmm1, %xmm3
-# CHECK-NEXT: 127.   1     0.0    0.0    16.0      vxorps	%ymm4, %ymm4, %ymm5
-# CHECK-NEXT: 128.   1     0.0    0.0    16.0      vxorpd	%ymm1, %ymm1, %ymm3
-# CHECK-NEXT: 129.   1     6.0    6.0    9.0       vxorps	%zmm4, %zmm4, %zmm5
-# CHECK-NEXT: 130.   1     6.0    6.0    9.0       vxorpd	%zmm1, %zmm1, %zmm3
-# CHECK-NEXT: 131.   1     0.0    0.0    17.0      vpxor	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 132.   1     0.0    0.0    16.0      vpxor	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 133.   1     6.0    1.0    9.0       vpxord	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: 134.   1     7.0    2.0    8.0       vpxorq	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: 135.   1     7.0    2.0    8.0       vpxord	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: 136.   1     7.0    2.0    8.0       vpxorq	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: 137.   1     0.0    0.0    16.0      vpxord	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: 138.   1     7.0    3.0    7.0       vpxorq	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT:        1     3.9    0.7    5.1       <total>
+# CHECK-NEXT: 51.    1     0.0    0.0    3.0       vpsubb	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: 52.    1     0.0    0.0    3.0       vpsubd	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: 53.    1     0.0    0.0    3.0       vpsubq	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: 54.    1     0.0    0.0    2.0       vpsubw	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: 55.    1     0.0    0.0    2.0       vpsubb	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: 56.    1     0.0    0.0    2.0       vpsubd	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: 57.    1     0.0    0.0    3.0       vpsubq	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: 58.    1     0.0    0.0    3.0       vpsubw	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: 59.    1     0.0    0.0    3.0       vpsubb	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: 60.    1     0.0    0.0    2.0       vpsubd	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: 61.    1     0.0    0.0    2.0       vpsubq	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: 62.    1     0.0    0.0    2.0       vpsubw	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: 63.    1     0.0    0.0    2.0       vpsubb	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: 64.    1     0.0    0.0    2.0       vpsubd	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: 65.    1     0.0    0.0    2.0       vpsubq	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: 66.    1     0.0    0.0    2.0       vpsubw	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: 67.    1     0.0    0.0    2.0       vpsubb	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: 68.    1     0.0    0.0    2.0       vpsubd	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: 69.    1     0.0    0.0    2.0       vpsubq	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: 70.    1     0.0    0.0    2.0       vpsubw	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: 71.    1     0.0    0.0    2.0       vpsubb	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: 72.    1     0.0    0.0    1.0       vpsubd	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: 73.    1     0.0    0.0    1.0       vpsubq	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: 74.    1     0.0    0.0    1.0       vpsubw	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: 75.    1     1.0    1.0    0.0       andnps	%xmm0, %xmm0
+# CHECK-NEXT: 76.    1     1.0    1.0    0.0       andnpd	%xmm1, %xmm1
+# CHECK-NEXT: 77.    1     0.0    0.0    2.0       vandnps	%xmm2, %xmm2, %xmm2
+# CHECK-NEXT: 78.    1     0.0    0.0    1.0       vandnpd	%xmm1, %xmm1, %xmm1
+# CHECK-NEXT: 79.    1     0.0    0.0    1.0       vandnps	%ymm2, %ymm2, %ymm2
+# CHECK-NEXT: 80.    1     0.0    0.0    1.0       vandnpd	%ymm1, %ymm1, %ymm1
+# CHECK-NEXT: 81.    1     0.0    0.0    1.0       vandnps	%zmm2, %zmm2, %zmm2
+# CHECK-NEXT: 82.    1     0.0    0.0    1.0       vandnpd	%zmm1, %zmm1, %zmm1
+# CHECK-NEXT: 83.    1     1.0    1.0    0.0       pandn	%mm2, %mm2
+# CHECK-NEXT: 84.    1     1.0    1.0    0.0       pandn	%xmm2, %xmm2
+# CHECK-NEXT: 85.    1     0.0    0.0    2.0       vpandn	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 86.    1     0.0    0.0    2.0       vpandn	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 87.    1     0.0    0.0    2.0       vpandnd	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: 88.    1     0.0    0.0    2.0       vpandnq	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: 89.    1     0.0    0.0    2.0       vpandnd	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: 90.    1     0.0    0.0    1.0       vpandnq	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: 91.    1     0.0    0.0    1.0       vpandnd	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: 92.    1     0.0    0.0    1.0       vpandnq	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: 93.    1     0.0    0.0    2.0       vandnps	%xmm2, %xmm2, %xmm5
+# CHECK-NEXT: 94.    1     0.0    0.0    2.0       vandnpd	%xmm1, %xmm1, %xmm5
+# CHECK-NEXT: 95.    1     0.0    0.0    2.0       vpandn	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 96.    1     0.0    0.0    1.0       vandnps	%ymm2, %ymm2, %ymm5
+# CHECK-NEXT: 97.    1     0.0    0.0    1.0       vandnpd	%ymm1, %ymm1, %ymm5
+# CHECK-NEXT: 98.    1     0.0    0.0    1.0       vpandn	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 99.    1     0.0    0.0    1.0       vandnps	%zmm2, %zmm2, %zmm5
+# CHECK-NEXT: 100.   1     0.0    0.0    1.0       vandnpd	%zmm1, %zmm1, %zmm5
+# CHECK-NEXT: 101.   1     0.0    0.0    1.0       vpandnd	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: 102.   1     0.0    0.0    1.0       vpandnq	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: 103.   1     0.0    0.0    1.0       vpandnd	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: 104.   1     0.0    0.0    1.0       vpandnq	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: 105.   1     0.0    0.0    1.0       vpandnd	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: 106.   1     0.0    0.0    1.0       vpandnq	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: 107.   1     1.0    1.0    0.0       xorps	%xmm0, %xmm0
+# CHECK-NEXT: 108.   1     1.0    1.0    0.0       xorpd	%xmm1, %xmm1
+# CHECK-NEXT: 109.   1     0.0    0.0    2.0       vxorps	%xmm2, %xmm2, %xmm2
+# CHECK-NEXT: 110.   1     0.0    0.0    2.0       vxorpd	%xmm1, %xmm1, %xmm1
+# CHECK-NEXT: 111.   1     0.0    0.0    2.0       vxorps	%ymm2, %ymm2, %ymm2
+# CHECK-NEXT: 112.   1     0.0    0.0    2.0       vxorpd	%ymm1, %ymm1, %ymm1
+# CHECK-NEXT: 113.   1     0.0    0.0    2.0       vxorps	%zmm2, %zmm2, %zmm2
+# CHECK-NEXT: 114.   1     0.0    0.0    1.0       vxorpd	%zmm1, %zmm1, %zmm1
+# CHECK-NEXT: 115.   1     1.0    1.0    0.0       pxor	%mm2, %mm2
+# CHECK-NEXT: 116.   1     1.0    1.0    0.0       pxor	%xmm2, %xmm2
+# CHECK-NEXT: 117.   1     0.0    0.0    2.0       vpxor	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 118.   1     0.0    0.0    2.0       vpxor	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 119.   1     0.0    0.0    2.0       vpxord	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: 120.   1     0.0    0.0    1.0       vpxorq	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: 121.   1     0.0    0.0    1.0       vpxord	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: 122.   1     0.0    0.0    1.0       vpxorq	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: 123.   1     0.0    0.0    1.0       vpxord	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: 124.   1     0.0    0.0    2.0       vpxorq	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: 125.   1     0.0    0.0    2.0       vxorps	%xmm4, %xmm4, %xmm5
+# CHECK-NEXT: 126.   1     0.0    0.0    1.0       vxorpd	%xmm1, %xmm1, %xmm3
+# CHECK-NEXT: 127.   1     0.0    0.0    1.0       vxorps	%ymm4, %ymm4, %ymm5
+# CHECK-NEXT: 128.   1     0.0    0.0    1.0       vxorpd	%ymm1, %ymm1, %ymm3
+# CHECK-NEXT: 129.   1     0.0    0.0    1.0       vxorps	%zmm4, %zmm4, %zmm5
+# CHECK-NEXT: 130.   1     0.0    0.0    1.0       vxorpd	%zmm1, %zmm1, %zmm3
+# CHECK-NEXT: 131.   1     0.0    0.0    1.0       vpxor	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 132.   1     0.0    0.0    0.0       vpxor	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 133.   1     0.0    0.0    1.0       vpxord	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: 134.   1     0.0    0.0    1.0       vpxorq	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: 135.   1     0.0    0.0    1.0       vpxord	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: 136.   1     0.0    0.0    1.0       vpxorq	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: 137.   1     0.0    0.0    1.0       vpxord	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: 138.   1     0.0    0.0    0.0       vpxorq	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:        1     0.2    0.1    1.6       <total>
diff --git a/llvm/tools/llvm-as/llvm-as.cpp b/llvm/tools/llvm-as/llvm-as.cpp
index f28ad00184420f..d8e36de59bcb4f 100644
--- a/llvm/tools/llvm-as/llvm-as.cpp
+++ b/llvm/tools/llvm-as/llvm-as.cpp
@@ -32,9 +32,8 @@ using namespace llvm;
 
 cl::OptionCategory AsCat("llvm-as Options");
 
-static cl::opt<std::string> InputFilename(cl::Positional,
-                                          cl::desc("<input .llvm file>"),
-                                          cl::init("-"));
+static cl::opt<std::string>
+    InputFilename(cl::Positional, cl::desc("<input .ll file>"), cl::init("-"));
 
 static cl::opt<std::string> OutputFilename("o",
                                            cl::desc("Override output filename"),
diff --git a/llvm/tools/llvm-cat/llvm-cat.cpp b/llvm/tools/llvm-cat/llvm-cat.cpp
index 039f63ad9465fb..f19f66d5621654 100644
--- a/llvm/tools/llvm-cat/llvm-cat.cpp
+++ b/llvm/tools/llvm-cat/llvm-cat.cpp
@@ -44,7 +44,7 @@ static cl::opt<std::string> OutputFilename("o", cl::Required,
                                            cl::cat(CatCategory));
 
 static cl::list<std::string> InputFilenames(cl::Positional,
-                                            cl::desc("<input  files>"),
+                                            cl::desc("<input files>"),
                                             cl::cat(CatCategory));
 
 int main(int argc, char **argv) {
diff --git a/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp b/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
index 2c068ba7f88fe6..c85e6d486e0acf 100644
--- a/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
@@ -480,3 +480,28 @@ TEST_F(AArch64GISelMITest, BuildFPEnv) {
 
   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
+
+TEST_F(AArch64GISelMITest, BuildExtractSubvector) {
+  setUp();
+  if (!TM)
+    GTEST_SKIP();
+
+  LLT VecTy = LLT::fixed_vector(4, 32);
+  LLT SubVecTy = LLT::fixed_vector(2, 32);
+  auto Vec = B.buildUndef(VecTy);
+  B.buildExtractSubvector(SubVecTy, Vec, 0);
+
+  VecTy = LLT::scalable_vector(4, 32);
+  SubVecTy = LLT::scalable_vector(2, 32);
+  Vec = B.buildUndef(VecTy);
+  B.buildExtractSubvector(SubVecTy, Vec, 0);
+
+  auto CheckStr = R"(
+  ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+  ; CHECK: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<2 x s32>) = G_EXTRACT_SUBVECTOR [[DEF]]:_(<4 x s32>), 0
+  ; CHECK: [[DEF1:%[0-9]+]]:_(<vscale x 4 x s32>) = G_IMPLICIT_DEF
+  ; CHECK: [[EXTRACT_SUBVECTOR1:%[0-9]+]]:_(<vscale x 2 x s32>) = G_EXTRACT_SUBVECTOR [[DEF1]]:_(<vscale x 4 x s32>), 0
+  )";
+
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
+}
diff --git a/llvm/unittests/SandboxIR/PassTest.cpp b/llvm/unittests/SandboxIR/PassTest.cpp
index ed226d57655861..2eaf369caf084b 100644
--- a/llvm/unittests/SandboxIR/PassTest.cpp
+++ b/llvm/unittests/SandboxIR/PassTest.cpp
@@ -179,7 +179,7 @@ TEST_F(PassTest, ParsePassPipeline) {
   Registry.registerPass(std::make_unique<TestPass1>());
   Registry.registerPass(std::make_unique<TestPass2>());
 
-  auto &FPM =
+  [[maybe_unused]] auto &FPM =
       Registry.parseAndCreatePassPipeline("test-pass1,test-pass2,test-pass1");
 #ifndef NDEBUG
   std::string Buff;
diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
index 92118bfb4cfd98..8807716a52738f 100644
--- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp
+++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
@@ -937,6 +937,186 @@ define void @foo() {
   }
 }
 
+TEST_F(SandboxIRTest, GlobalVariable) {
+  parseIR(C, R"IR(
+@glob0 = global i32 42
+@glob1 = global i32 43
+define void @foo() {
+  %ld0 = load i32, ptr @glob0
+  %ld1 = load i32, ptr @glob1
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  auto *LLVMBB = &*LLVMF.begin();
+  auto LLVMIt = LLVMBB->begin();
+  auto *LLVMLd0 = cast<llvm::LoadInst>(&*LLVMIt++);
+  auto *LLVMGV0 = cast<llvm::GlobalVariable>(LLVMLd0->getPointerOperand());
+  sandboxir::Context Ctx(C);
+
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto *BB = &*F.begin();
+  auto It = BB->begin();
+  auto *Ld0 = cast<sandboxir::LoadInst>(&*It++);
+  auto *Ld1 = cast<sandboxir::LoadInst>(&*It++);
+  // Check classof(), creation.
+  auto *GV0 = cast<sandboxir::GlobalVariable>(Ld0->getPointerOperand());
+  auto *GV1 = cast<sandboxir::GlobalVariable>(Ld1->getPointerOperand());
+  // Check getIterator().
+  {
+    auto It0 = GV0->getIterator();
+    auto It1 = GV1->getIterator();
+    EXPECT_EQ(&*It0, GV0);
+    EXPECT_EQ(&*It1, GV1);
+    EXPECT_EQ(std::next(It0), It1);
+    EXPECT_EQ(std::prev(It1), It0);
+    EXPECT_EQ(&*std::next(It0), GV1);
+    EXPECT_EQ(&*std::prev(It1), GV0);
+  }
+  // Check getReverseIterator().
+  {
+    auto RevIt0 = GV0->getReverseIterator();
+    auto RevIt1 = GV1->getReverseIterator();
+    EXPECT_EQ(&*RevIt0, GV0);
+    EXPECT_EQ(&*RevIt1, GV1);
+    EXPECT_EQ(std::prev(RevIt0), RevIt1);
+    EXPECT_EQ(std::next(RevIt1), RevIt0);
+    EXPECT_EQ(&*std::prev(RevIt0), GV1);
+    EXPECT_EQ(&*std::next(RevIt1), GV0);
+  }
+  // Check hasInitializer().
+  EXPECT_EQ(GV0->hasInitializer(), LLVMGV0->hasInitializer());
+  // Check hasDefinitiveInitializer().
+  EXPECT_EQ(GV0->hasDefinitiveInitializer(),
+            LLVMGV0->hasDefinitiveInitializer());
+  // Check hasUniqueInitializer().
+  EXPECT_EQ(GV0->hasUniqueInitializer(), LLVMGV0->hasUniqueInitializer());
+  // Check getInitializer().
+  EXPECT_EQ(GV0->getInitializer(), Ctx.getValue(LLVMGV0->getInitializer()));
+  // Check setInitializer().
+  auto *OrigInitializer = GV0->getInitializer();
+  auto *NewInitializer = GV1->getInitializer();
+  EXPECT_NE(NewInitializer, OrigInitializer);
+  GV0->setInitializer(NewInitializer);
+  EXPECT_EQ(GV0->getInitializer(), NewInitializer);
+  GV0->setInitializer(OrigInitializer);
+  EXPECT_EQ(GV0->getInitializer(), OrigInitializer);
+  // Check isConstant().
+  EXPECT_EQ(GV0->isConstant(), LLVMGV0->isConstant());
+  // Check setConstant().
+  bool OrigIsConstant = GV0->isConstant();
+  bool NewIsConstant = !OrigIsConstant;
+  GV0->setConstant(NewIsConstant);
+  EXPECT_EQ(GV0->isConstant(), NewIsConstant);
+  GV0->setConstant(OrigIsConstant);
+  EXPECT_EQ(GV0->isConstant(), OrigIsConstant);
+  // Check isExternallyInitialized().
+  EXPECT_EQ(GV0->isExternallyInitialized(), LLVMGV0->isExternallyInitialized());
+  // Check setExternallyInitialized().
+  bool OrigIsExtInit = GV0->isExternallyInitialized();
+  bool NewIsExtInit = !OrigIsExtInit;
+  GV0->setExternallyInitialized(NewIsExtInit);
+  EXPECT_EQ(GV0->isExternallyInitialized(), NewIsExtInit);
+  GV0->setExternallyInitialized(OrigIsExtInit);
+  EXPECT_EQ(GV0->isExternallyInitialized(), OrigIsExtInit);
+  for (auto KindIdx : seq<int>(0, Attribute::AttrKind::EndAttrKinds)) {
+    // Check hasAttribute(AttrKind).
+    auto Kind = static_cast<Attribute::AttrKind>(KindIdx);
+    EXPECT_EQ(GV0->hasAttribute(Kind), LLVMGV0->hasAttribute(Kind));
+    // Check hasAttribute(StringRef).
+    StringRef KindStr = Attribute::getNameFromAttrKind(Kind);
+    EXPECT_EQ(GV0->hasAttribute(KindStr), LLVMGV0->hasAttribute(KindStr));
+  }
+  // Check hasAttributes().
+  EXPECT_EQ(GV0->hasAttributes(), LLVMGV0->hasAttributes());
+
+  for (auto KindIdx : seq<int>(0, Attribute::AttrKind::EndAttrKinds)) {
+    // Check getAttribute(AttrKind).
+    auto Kind = static_cast<Attribute::AttrKind>(KindIdx);
+    EXPECT_EQ(GV0->getAttribute(Kind), LLVMGV0->getAttribute(Kind));
+    // Check getAttribute(StringRef).
+    StringRef KindStr = Attribute::getNameFromAttrKind(Kind);
+    EXPECT_EQ(GV0->getAttribute(KindStr), LLVMGV0->getAttribute(KindStr));
+  }
+  // Check getAttributes().
+  EXPECT_EQ(GV0->getAttributes(), LLVMGV0->getAttributes());
+  // Check getAttributesAsList().
+  EXPECT_THAT(GV0->getAttributesAsList(0u),
+              testing::ContainerEq(LLVMGV0->getAttributesAsList(0u)));
+  // Check hasImplicitSection().
+  EXPECT_EQ(GV0->hasImplicitSection(), LLVMGV0->hasImplicitSection());
+  // Check getCodeModelRaw().
+  EXPECT_EQ(GV0->getCodeModelRaw(), LLVMGV0->getCodeModelRaw());
+  // Check getCodeModel().
+  EXPECT_EQ(GV0->getCodeModel(), LLVMGV0->getCodeModel());
+}
+
+TEST_F(SandboxIRTest, GlobalAlias) {
+  parseIR(C, R"IR(
+@alias0 = dso_local alias void(), ptr @foo
+@alias1 = dso_local alias void(), ptr @foo
+declare void @bar();
+define void @foo() {
+  call void @alias0()
+  call void @alias1()
+  call void @bar()
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  auto *LLVMBB = &*LLVMF.begin();
+  auto LLVMIt = LLVMBB->begin();
+  auto *LLVMCall0 = cast<llvm::CallInst>(&*LLVMIt++);
+  auto *LLVMAlias0 = cast<llvm::GlobalAlias>(LLVMCall0->getCalledOperand());
+  sandboxir::Context Ctx(C);
+
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto *BB = &*F.begin();
+  auto It = BB->begin();
+  auto *Call0 = cast<sandboxir::CallInst>(&*It++);
+  auto *Call1 = cast<sandboxir::CallInst>(&*It++);
+  auto *CallBar = cast<sandboxir::CallInst>(&*It++);
+  auto *CalleeBar = cast<sandboxir::Constant>(CallBar->getCalledOperand());
+  // Check classof(), creation.
+  auto *Alias0 = cast<sandboxir::GlobalAlias>(Call0->getCalledOperand());
+  auto *Alias1 = cast<sandboxir::GlobalAlias>(Call1->getCalledOperand());
+  // Check getIterator().
+  {
+    auto It0 = Alias0->getIterator();
+    auto It1 = Alias1->getIterator();
+    EXPECT_EQ(&*It0, Alias0);
+    EXPECT_EQ(&*It1, Alias1);
+    EXPECT_EQ(std::next(It0), It1);
+    EXPECT_EQ(std::prev(It1), It0);
+    EXPECT_EQ(&*std::next(It0), Alias1);
+    EXPECT_EQ(&*std::prev(It1), Alias0);
+  }
+  // Check getReverseIterator().
+  {
+    auto RevIt0 = Alias0->getReverseIterator();
+    auto RevIt1 = Alias1->getReverseIterator();
+    EXPECT_EQ(&*RevIt0, Alias0);
+    EXPECT_EQ(&*RevIt1, Alias1);
+    EXPECT_EQ(std::prev(RevIt0), RevIt1);
+    EXPECT_EQ(std::next(RevIt1), RevIt0);
+    EXPECT_EQ(&*std::prev(RevIt0), Alias1);
+    EXPECT_EQ(&*std::next(RevIt1), Alias0);
+  }
+  // Check getAliasee().
+  EXPECT_EQ(Alias0->getAliasee(), Ctx.getValue(LLVMAlias0->getAliasee()));
+  // Check setAliasee().
+  auto *OrigAliasee = Alias0->getAliasee();
+  auto *NewAliasee = CalleeBar;
+  EXPECT_NE(NewAliasee, OrigAliasee);
+  Alias0->setAliasee(NewAliasee);
+  EXPECT_EQ(Alias0->getAliasee(), NewAliasee);
+  Alias0->setAliasee(OrigAliasee);
+  EXPECT_EQ(Alias0->getAliasee(), OrigAliasee);
+  // Check getAliaseeObject().
+  EXPECT_EQ(Alias0->getAliaseeObject(),
+            Ctx.getValue(LLVMAlias0->getAliaseeObject()));
+}
+
 TEST_F(SandboxIRTest, BlockAddress) {
   parseIR(C, R"IR(
 define void @foo(ptr %ptr) {
@@ -1395,12 +1575,16 @@ define void @foo(i32 %v1) {
   for (sandboxir::Instruction &I : BB0) {
     EXPECT_EQ(&I, Ctx.getValue(LLVMI));
     LLVMI = LLVMI->getNextNode();
+    // Check getNodeParent().
+    EXPECT_EQ(I.getIterator().getNodeParent(), &BB0);
   }
   LLVMI = &*LLVMBB1->begin();
   for (sandboxir::Instruction &I : BB1) {
     EXPECT_EQ(&I, Ctx.getValue(LLVMI));
     LLVMI = LLVMI->getNextNode();
   }
+  // Check NodeParent() for BB::end().
+  EXPECT_EQ(BB0.end().getNodeParent(), &BB0);
 
   // Check BB.getTerminator()
   EXPECT_EQ(BB0.getTerminator(), Ctx.getValue(LLVMBB0->getTerminator()));
diff --git a/llvm/unittests/SandboxIR/TrackerTest.cpp b/llvm/unittests/SandboxIR/TrackerTest.cpp
index d4ff4fd6464e5c..da5416395ec42f 100644
--- a/llvm/unittests/SandboxIR/TrackerTest.cpp
+++ b/llvm/unittests/SandboxIR/TrackerTest.cpp
@@ -1590,6 +1590,85 @@ define void @foo() {
   EXPECT_EQ(IFunc->getResolver(), OrigResolver);
 }
 
+TEST_F(TrackerTest, GlobalVariableSetters) {
+  parseIR(C, R"IR(
+@glob0 = global i32 42
+@glob1 = global i32 43
+define void @foo() {
+  %ld0 = load i32, ptr @glob0
+  %ld1 = load i32, ptr @glob1
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto *BB = &*F.begin();
+  auto It = BB->begin();
+  auto *Ld0 = cast<sandboxir::LoadInst>(&*It++);
+  auto *Ld1 = cast<sandboxir::LoadInst>(&*It++);
+  // Check classof(), creation.
+  auto *GV0 = cast<sandboxir::GlobalVariable>(Ld0->getPointerOperand());
+  auto *GV1 = cast<sandboxir::GlobalVariable>(Ld1->getPointerOperand());
+  // Check setInitializer().
+  auto *OrigInitializer = GV0->getInitializer();
+  auto *NewInitializer = GV1->getInitializer();
+  EXPECT_NE(NewInitializer, OrigInitializer);
+  Ctx.save();
+  GV0->setInitializer(NewInitializer);
+  EXPECT_EQ(GV0->getInitializer(), NewInitializer);
+  Ctx.revert();
+  EXPECT_EQ(GV0->getInitializer(), OrigInitializer);
+  // Check setConstant().
+  bool OrigIsConstant = GV0->isConstant();
+  bool NewIsConstant = !OrigIsConstant;
+  Ctx.save();
+  GV0->setConstant(NewIsConstant);
+  EXPECT_EQ(GV0->isConstant(), NewIsConstant);
+  Ctx.revert();
+  EXPECT_EQ(GV0->isConstant(), OrigIsConstant);
+  // Check setExternallyInitialized().
+  bool OrigIsExtInit = GV0->isExternallyInitialized();
+  bool NewIsExtInit = !OrigIsExtInit;
+  Ctx.save();
+  GV0->setExternallyInitialized(NewIsExtInit);
+  EXPECT_EQ(GV0->isExternallyInitialized(), NewIsExtInit);
+  Ctx.revert();
+  EXPECT_EQ(GV0->isExternallyInitialized(), OrigIsExtInit);
+}
+
+TEST_F(TrackerTest, GlobalAliasSetters) {
+  parseIR(C, R"IR(
+@alias = dso_local alias void(), ptr @foo
+declare void @bar();
+define void @foo() {
+  call void @alias()
+  call void @bar()
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto *BB = &*F.begin();
+  auto It = BB->begin();
+  auto *Call0 = cast<sandboxir::CallInst>(&*It++);
+  auto *Call1 = cast<sandboxir::CallInst>(&*It++);
+  auto *Callee1 = cast<sandboxir::Constant>(Call1->getCalledOperand());
+  auto *Alias = cast<sandboxir::GlobalAlias>(Call0->getCalledOperand());
+  // Check setAliasee().
+  auto *OrigAliasee = Alias->getAliasee();
+  auto *NewAliasee = Callee1;
+  EXPECT_NE(NewAliasee, OrigAliasee);
+  Ctx.save();
+  Alias->setAliasee(NewAliasee);
+  EXPECT_EQ(Alias->getAliasee(), NewAliasee);
+  Ctx.revert();
+  EXPECT_EQ(Alias->getAliasee(), OrigAliasee);
+}
+
 TEST_F(TrackerTest, SetVolatile) {
   parseIR(C, R"IR(
 define void @foo(ptr %arg0, i8 %val) {
diff --git a/llvm/unittests/Support/FormatVariadicTest.cpp b/llvm/unittests/Support/FormatVariadicTest.cpp
index 8e326e1f2b4396..e745f99a5a6c4e 100644
--- a/llvm/unittests/Support/FormatVariadicTest.cpp
+++ b/llvm/unittests/Support/FormatVariadicTest.cpp
@@ -283,6 +283,15 @@ TEST(FormatVariadicTest, AutomaticIndices) {
   EXPECT_EQ(ReplacementType::Format, Replacements[0].Type);
   EXPECT_EQ(0u, Replacements[0].Index);
 
+  Replacements = parseFormatString("{}bar{}");
+  ASSERT_EQ(3u, Replacements.size());
+  EXPECT_EQ(ReplacementType::Format, Replacements[0].Type);
+  EXPECT_EQ(0u, Replacements[0].Index);
+  EXPECT_EQ(ReplacementType::Literal, Replacements[1].Type);
+  EXPECT_EQ("bar", Replacements[1].Spec);
+  EXPECT_EQ(ReplacementType::Format, Replacements[2].Type);
+  EXPECT_EQ(1u, Replacements[2].Index);
+
   Replacements = parseFormatString("{}{}");
   ASSERT_EQ(2u, Replacements.size());
   EXPECT_EQ(ReplacementType::Format, Replacements[0].Type);
@@ -760,6 +769,8 @@ TEST(FormatVariadicTest, Validate) {
                "Replacement field indices cannot have holes");
   EXPECT_DEATH(formatv("{}{1}", 0, 1).str(),
                "Cannot mix automatic and explicit indices");
+  EXPECT_DEATH(formatv("{}{0}{}", 0, 1).str(),
+               "Cannot mix automatic and explicit indices");
 #else  // GTEST_HAS_DEATH_TEST
   GTEST_SKIP() << "No support for EXPECT_DEATH";
 #endif // GTEST_HAS_DEATH_TEST
@@ -768,6 +779,7 @@ TEST(FormatVariadicTest, Validate) {
   EXPECT_EQ(formatv("{0}", 1, 2).str(), "1");
   EXPECT_EQ(formatv("{0} {2}", 1, 2, 3).str(), "1 3");
   EXPECT_EQ(formatv("{}{1}", 0, 1).str(), "01");
+  EXPECT_EQ(formatv("{}{0}{}", 0, 1).str(), "001");
 #endif // NDEBUG
 }
 
diff --git a/llvm/unittests/Transforms/Vectorize/CMakeLists.txt b/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
index 1354558a94f0d5..0df39c41a90414 100644
--- a/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
@@ -1,3 +1,5 @@
+add_subdirectory(SandboxVectorizer)
+
 set(LLVM_LINK_COMPONENTS
   Analysis
   Core
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/CMakeLists.txt b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/CMakeLists.txt
new file mode 100644
index 00000000000000..2c7bf7d7e87541
--- /dev/null
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/CMakeLists.txt
@@ -0,0 +1,13 @@
+set(LLVM_LINK_COMPONENTS
+  Analysis
+  Core
+  Vectorize
+  AsmParser
+  TargetParser
+  SandboxIR
+  )
+
+add_llvm_unittest(SandboxVectorizerTests
+  DependencyGraphTest.cpp
+  LegalityTest.cpp
+  )
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
new file mode 100644
index 00000000000000..dc85a22f7f4832
--- /dev/null
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
@@ -0,0 +1,63 @@
+//===- DependencyGraphTest.cpp --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/SandboxIR/SandboxIR.h"
+#include "llvm/Support/SourceMgr.h"
+#include "gmock/gmock-matchers.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+struct DependencyGraphTest : public testing::Test {
+  LLVMContext C;
+  std::unique_ptr<Module> M;
+
+  void parseIR(LLVMContext &C, const char *IR) {
+    SMDiagnostic Err;
+    M = parseAssemblyString(IR, Err, C);
+    if (!M)
+      Err.print("DependencyGraphTest", errs());
+  }
+};
+
+TEST_F(DependencyGraphTest, Basic) {
+  parseIR(C, R"IR(
+define void @foo(ptr %ptr, i8 %v0, i8 %v1) {
+  store i8 %v0, ptr %ptr
+  store i8 %v1, ptr %ptr
+  ret void
+}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  auto *F = Ctx.createFunction(LLVMF);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  auto *S0 = cast<sandboxir::StoreInst>(&*It++);
+  auto *S1 = cast<sandboxir::StoreInst>(&*It++);
+  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
+  sandboxir::DependencyGraph DAG;
+  DAG.extend(BB);
+
+  sandboxir::DGNode *N0 = DAG.getNode(S0);
+  sandboxir::DGNode *N1 = DAG.getNode(S1);
+  sandboxir::DGNode *N2 = DAG.getNode(Ret);
+  // Check getInstruction().
+  EXPECT_EQ(N0->getInstruction(), S0);
+  EXPECT_EQ(N1->getInstruction(), S1);
+  // Check hasMemPred()
+  EXPECT_TRUE(N1->hasMemPred(N0));
+  EXPECT_FALSE(N0->hasMemPred(N1));
+
+  // Check memPreds().
+  EXPECT_TRUE(N0->memPreds().empty());
+  EXPECT_THAT(N1->memPreds(), testing::ElementsAre(N0));
+  EXPECT_THAT(N2->memPreds(), testing::ElementsAre(N1));
+}
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
new file mode 100644
index 00000000000000..a136be41ae363b
--- /dev/null
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
@@ -0,0 +1,56 @@
+//===- LegalityTest.cpp ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/SandboxIR/SandboxIR.h"
+#include "llvm/Support/SourceMgr.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+struct LegalityTest : public testing::Test {
+  LLVMContext C;
+  std::unique_ptr<Module> M;
+
+  void parseIR(LLVMContext &C, const char *IR) {
+    SMDiagnostic Err;
+    M = parseAssemblyString(IR, Err, C);
+    if (!M)
+      Err.print("LegalityTest", errs());
+  }
+};
+
+TEST_F(LegalityTest, Legality) {
+  parseIR(C, R"IR(
+define void @foo(ptr %ptr) {
+  %gep0 = getelementptr float, ptr %ptr, i32 0
+  %gep1 = getelementptr float, ptr %ptr, i32 1
+  %ld0 = load float, ptr %gep0
+  %ld1 = load float, ptr %gep0
+  store float %ld0, ptr %gep0
+  store float %ld1, ptr %gep1
+  ret void
+}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  auto *F = Ctx.createFunction(LLVMF);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  [[maybe_unused]] auto *Gep0 = cast<sandboxir::GetElementPtrInst>(&*It++);
+  [[maybe_unused]] auto *Gep1 = cast<sandboxir::GetElementPtrInst>(&*It++);
+  [[maybe_unused]] auto *Ld0 = cast<sandboxir::LoadInst>(&*It++);
+  [[maybe_unused]] auto *Ld1 = cast<sandboxir::LoadInst>(&*It++);
+  auto *St0 = cast<sandboxir::StoreInst>(&*It++);
+  auto *St1 = cast<sandboxir::StoreInst>(&*It++);
+
+  sandboxir::LegalityAnalysis Legality;
+  auto Result = Legality.canVectorize({St0, St1});
+  EXPECT_TRUE(isa<sandboxir::Widen>(Result));
+}
diff --git a/llvm/utils/TableGen/ARMTargetDefEmitter.cpp b/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
index 71ca331461c00c..6b8ebf96cdf383 100644
--- a/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
+++ b/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
@@ -25,24 +25,26 @@
 using namespace llvm;
 
 /// Collect the full set of implied features for a SubtargetFeature.
-static void CollectImpliedFeatures(std::set<Record *> &SeenFeats, Record *Rec) {
+static void CollectImpliedFeatures(std::set<const Record *> &SeenFeats,
+                                   const Record *Rec) {
   assert(Rec->isSubClassOf("SubtargetFeature") &&
          "Rec is not a SubtargetFeature");
 
   SeenFeats.insert(Rec);
-  for (Record *Implied : Rec->getValueAsListOfDefs("Implies"))
+  for (const Record *Implied : Rec->getValueAsListOfDefs("Implies"))
     CollectImpliedFeatures(SeenFeats, Implied);
 }
 
-static void CheckFeatureTree(Record *Root) {
-  std::set<Record *> SeenFeats;
+static void CheckFeatureTree(const Record *Root) {
+  std::set<const Record *> SeenFeats;
   CollectImpliedFeatures(SeenFeats, Root);
 
   // Check that each of the mandatory (implied) features which is an
   // ExtensionWithMArch is also enabled by default.
   auto DefaultExtsVec = Root->getValueAsListOfDefs("DefaultExts");
-  std::set<Record *> DefaultExts{DefaultExtsVec.begin(), DefaultExtsVec.end()};
-  for (auto *Feat : SeenFeats) {
+  std::set<const Record *> DefaultExts{DefaultExtsVec.begin(),
+                                       DefaultExtsVec.end()};
+  for (const Record *Feat : SeenFeats) {
     if (Feat->isSubClassOf("ExtensionWithMArch") && !DefaultExts.count(Feat))
       PrintFatalError(Root->getLoc(),
                       "ExtensionWithMArch " + Feat->getName() +
@@ -51,7 +53,7 @@ static void CheckFeatureTree(Record *Root) {
   }
 }
 
-static void EmitARMTargetDef(RecordKeeper &RK, raw_ostream &OS) {
+static void EmitARMTargetDef(const RecordKeeper &RK, raw_ostream &OS) {
   OS << "// Autogenerated by ARMTargetDefEmitter.cpp\n\n";
 
   // Look through all SubtargetFeature defs with the given FieldName, and
@@ -67,14 +69,14 @@ static void EmitARMTargetDef(RecordKeeper &RK, raw_ostream &OS) {
   };
 
   // Sort the extensions alphabetically, so they don't appear in tablegen order.
-  std::vector<Record *> SortedExtensions =
+  std::vector<const Record *> SortedExtensions =
       RK.getAllDerivedDefinitions("Extension");
-  auto Alphabetical = [](Record *A, Record *B) -> bool {
+  auto Alphabetical = [](const Record *A, const Record *B) -> bool {
     const auto NameA = A->getValueAsString("Name");
     const auto NameB = B->getValueAsString("Name");
     return NameA.compare(NameB) < 0; // A lexographically less than B
   };
-  std::sort(SortedExtensions.begin(), SortedExtensions.end(), Alphabetical);
+  sort(SortedExtensions, Alphabetical);
 
   // The ARMProcFamilyEnum values are initialised by SubtargetFeature defs
   // which set the ARMProcFamily field. We can generate the enum from these defs
@@ -287,7 +289,7 @@ static void EmitARMTargetDef(RecordKeeper &RK, raw_ostream &OS) {
     if (Name == "apple-latest")
       continue;
 
-    Record *Arch;
+    const Record *Arch;
     if (Name == "generic") {
       // "generic" is an exception. It does not have an architecture, and there
       // are tests that depend on e.g. -mattr=-v8.4a meaning HasV8_0aOps==false.
@@ -295,7 +297,9 @@ static void EmitARMTargetDef(RecordKeeper &RK, raw_ostream &OS) {
       Arch = RK.getDef("HasV8_0aOps");
     } else {
       // Search for an Architecture64 in the list of features.
-      auto IsArch = [](Record *F) { return F->isSubClassOf("Architecture64"); };
+      auto IsArch = [](const Record *F) {
+        return F->isSubClassOf("Architecture64");
+      };
       auto ArchIter = llvm::find_if(Features, IsArch);
       if (ArchIter == Features.end())
         PrintFatalError(Rec, "Features must include an Architecture64.");
@@ -320,7 +324,7 @@ static void EmitARMTargetDef(RecordKeeper &RK, raw_ostream &OS) {
 
     // Keep track of extensions we have seen
     StringSet<> SeenExts;
-    for (auto *E : Rec->getValueAsListOfDefs("Features"))
+    for (const Record *E : Rec->getValueAsListOfDefs("Features"))
       // Only process subclasses of Extension
       if (E->isSubClassOf("Extension")) {
         const auto AEK = E->getValueAsString("ArchExtKindSpelling").upper();
diff --git a/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
index ce4cd354540d04..9b454cf7944620 100644
--- a/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
+++ b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
@@ -67,7 +67,7 @@ using namespace llvm;
 namespace {
 
 class VarLenCodeEmitterGen {
-  RecordKeeper &Records;
+  const RecordKeeper &Records;
 
   // Representaton of alternative encodings used for HwModes.
   using AltEncodingTy = int;
@@ -83,15 +83,16 @@ class VarLenCodeEmitterGen {
   void emitInstructionBaseValues(
       raw_ostream &OS,
       ArrayRef<const CodeGenInstruction *> NumberedInstructions,
-      CodeGenTarget &Target, AltEncodingTy Mode);
+      const CodeGenTarget &Target, AltEncodingTy Mode);
 
-  std::string getInstructionCases(Record *R, CodeGenTarget &Target);
-  std::string getInstructionCaseForEncoding(Record *R, AltEncodingTy Mode,
+  std::string getInstructionCases(const Record *R, const CodeGenTarget &Target);
+  std::string getInstructionCaseForEncoding(const Record *R, AltEncodingTy Mode,
                                             const VarLenInst &VLI,
-                                            CodeGenTarget &Target, int I);
+                                            const CodeGenTarget &Target,
+                                            int Indent);
 
 public:
-  explicit VarLenCodeEmitterGen(RecordKeeper &R) : Records(R) {}
+  explicit VarLenCodeEmitterGen(const RecordKeeper &R) : Records(R) {}
 
   void run(raw_ostream &OS);
 };
@@ -222,7 +223,6 @@ void VarLenInst::buildRec(const DagInit *DI) {
 
 void VarLenCodeEmitterGen::run(raw_ostream &OS) {
   CodeGenTarget Target(Records);
-  auto Insts = Records.getAllDerivedDefinitions("Instruction");
 
   auto NumberedInstructions = Target.getInstructionsByEnumValue();
 
@@ -238,10 +238,8 @@ void VarLenCodeEmitterGen::run(raw_ostream &OS) {
       if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
         const CodeGenHwModes &HWM = Target.getHwModes();
         EncodingInfoByHwMode EBM(DI->getDef(), HWM);
-        for (auto &KV : EBM) {
-          AltEncodingTy Mode = KV.first;
+        for (const auto [Mode, EncodingDef] : EBM) {
           Modes.insert({Mode, "_" + HWM.getMode(Mode).Name.str()});
-          const Record *EncodingDef = KV.second;
           const RecordVal *RV = EncodingDef->getValue("Inst");
           DagInit *DI = cast<DagInit>(RV->getValue());
           VarLenInsts[R].insert({Mode, VarLenInst(DI, RV)});
@@ -250,7 +248,7 @@ void VarLenCodeEmitterGen::run(raw_ostream &OS) {
       }
     }
     const RecordVal *RV = R->getValue("Inst");
-    DagInit *DI = cast<DagInit>(RV->getValue());
+    const DagInit *DI = cast<DagInit>(RV->getValue());
     VarLenInsts[R].insert({Universal, VarLenInst(DI, RV)});
   }
 
@@ -291,7 +289,7 @@ void VarLenCodeEmitterGen::run(raw_ostream &OS) {
   std::map<std::string, std::vector<std::string>> CaseMap;
 
   // Construct all cases statement for each opcode
-  for (Record *R : Insts) {
+  for (const Record *R : Records.getAllDerivedDefinitions("Instruction")) {
     if (R->getValueAsString("Namespace") == "TargetOpcode" ||
         R->getValueAsBit("isPseudo"))
       continue;
@@ -347,7 +345,7 @@ static void emitInstBits(raw_ostream &IS, raw_ostream &SS, const APInt &Bits,
 
 void VarLenCodeEmitterGen::emitInstructionBaseValues(
     raw_ostream &OS, ArrayRef<const CodeGenInstruction *> NumberedInstructions,
-    CodeGenTarget &Target, AltEncodingTy Mode) {
+    const CodeGenTarget &Target, AltEncodingTy Mode) {
   std::string IndexArray, StorageArray;
   raw_string_ostream IS(IndexArray), SS(StorageArray);
 
@@ -408,8 +406,9 @@ void VarLenCodeEmitterGen::emitInstructionBaseValues(
   OS << IndexArray << StorageArray;
 }
 
-std::string VarLenCodeEmitterGen::getInstructionCases(Record *R,
-                                                      CodeGenTarget &Target) {
+std::string
+VarLenCodeEmitterGen::getInstructionCases(const Record *R,
+                                          const CodeGenTarget &Target) {
   auto It = VarLenInsts.find(R);
   if (It == VarLenInsts.end())
     PrintFatalError(R, "Parsed encoding record not found");
@@ -421,7 +420,8 @@ std::string VarLenCodeEmitterGen::getInstructionCases(Record *R,
     // Universal, just pick the first mode.
     AltEncodingTy Mode = Modes.begin()->first;
     const auto &Encoding = Map.begin()->second;
-    return getInstructionCaseForEncoding(R, Mode, Encoding, Target, 6);
+    return getInstructionCaseForEncoding(R, Mode, Encoding, Target,
+                                         /*Indent=*/6);
   }
 
   std::string Case;
@@ -434,8 +434,8 @@ std::string VarLenCodeEmitterGen::getInstructionCases(Record *R,
       Case +=
           "        llvm_unreachable(\"Undefined encoding in this mode\");\n";
     } else {
-      Case +=
-          getInstructionCaseForEncoding(R, It->first, It->second, Target, 8);
+      Case += getInstructionCaseForEncoding(R, It->first, It->second, Target,
+                                            /*Indent=*/8);
     }
     Case += "        break;\n";
     Case += "      }\n";
@@ -445,15 +445,14 @@ std::string VarLenCodeEmitterGen::getInstructionCases(Record *R,
 }
 
 std::string VarLenCodeEmitterGen::getInstructionCaseForEncoding(
-    Record *R, AltEncodingTy Mode, const VarLenInst &VLI, CodeGenTarget &Target,
-    int I) {
-
+    const Record *R, AltEncodingTy Mode, const VarLenInst &VLI,
+    const CodeGenTarget &Target, int Indent) {
   CodeGenInstruction &CGI = Target.getInstruction(R);
 
   std::string Case;
   raw_string_ostream SS(Case);
   // Populate based value.
-  SS.indent(I) << "Inst = getInstBits" << Modes[Mode] << "(opcode);\n";
+  SS.indent(Indent) << "Inst = getInstBits" << Modes[Mode] << "(opcode);\n";
 
   // Process each segment in VLI.
   size_t Offset = 0U;
@@ -482,19 +481,21 @@ std::string VarLenCodeEmitterGen::getInstructionCaseForEncoding(
       if (ES.CustomEncoder.size())
         CustomEncoder = ES.CustomEncoder;
 
-      SS.indent(I) << "Scratch.clearAllBits();\n";
-      SS.indent(I) << "// op: " << OperandName.drop_front(1) << "\n";
+      SS.indent(Indent) << "Scratch.clearAllBits();\n";
+      SS.indent(Indent) << "// op: " << OperandName.drop_front(1) << "\n";
       if (CustomEncoder.empty())
-        SS.indent(I) << "getMachineOpValue(MI, MI.getOperand("
-                     << utostr(FlatOpIdx) << ")";
+        SS.indent(Indent) << "getMachineOpValue(MI, MI.getOperand("
+                          << utostr(FlatOpIdx) << ")";
       else
-        SS.indent(I) << CustomEncoder << "(MI, /*OpIdx=*/" << utostr(FlatOpIdx);
+        SS.indent(Indent) << CustomEncoder << "(MI, /*OpIdx=*/"
+                          << utostr(FlatOpIdx);
 
       SS << ", /*Pos=*/" << utostr(Offset) << ", Scratch, Fixups, STI);\n";
 
-      SS.indent(I) << "Inst.insertBits(" << "Scratch.extractBits("
-                   << utostr(NumBits) << ", " << utostr(LoBit) << ")" << ", "
-                   << Offset << ");\n";
+      SS.indent(Indent) << "Inst.insertBits("
+                        << "Scratch.extractBits(" << utostr(NumBits) << ", "
+                        << utostr(LoBit) << ")"
+                        << ", " << Offset << ");\n";
 
       HighScratchAccess = std::max(HighScratchAccess, NumBits + LoBit);
     }
@@ -503,24 +504,20 @@ std::string VarLenCodeEmitterGen::getInstructionCaseForEncoding(
 
   StringRef PostEmitter = R->getValueAsString("PostEncoderMethod");
   if (!PostEmitter.empty())
-    SS.indent(I) << "Inst = " << PostEmitter << "(MI, Inst, STI);\n";
+    SS.indent(Indent) << "Inst = " << PostEmitter << "(MI, Inst, STI);\n";
 
   // Resize the scratch buffer if it's to small.
   std::string ScratchResizeStr;
   if (VLI.size() && !VLI.isFixedValueOnly()) {
     raw_string_ostream RS(ScratchResizeStr);
-    RS.indent(I) << "if (Scratch.getBitWidth() < " << HighScratchAccess
-                 << ") { Scratch = Scratch.zext(" << HighScratchAccess
-                 << "); }\n";
+    RS.indent(Indent) << "if (Scratch.getBitWidth() < " << HighScratchAccess
+                      << ") { Scratch = Scratch.zext(" << HighScratchAccess
+                      << "); }\n";
   }
 
   return ScratchResizeStr + Case;
 }
 
-namespace llvm {
-
-void emitVarLenCodeEmitter(RecordKeeper &R, raw_ostream &OS) {
+void llvm::emitVarLenCodeEmitter(const RecordKeeper &R, raw_ostream &OS) {
   VarLenCodeEmitterGen(R).run(OS);
 }
-
-} // end namespace llvm
diff --git a/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.h b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.h
index 48b317be88d018..ca0f9c3f8b4f3f 100644
--- a/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.h
+++ b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.h
@@ -53,7 +53,7 @@ class VarLenInst {
   bool isFixedValueOnly() const { return !HasDynamicSegment; }
 };
 
-void emitVarLenCodeEmitter(RecordKeeper &R, raw_ostream &OS);
+void emitVarLenCodeEmitter(const RecordKeeper &R, raw_ostream &OS);
 
 } // end namespace llvm
 
diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp
index 71a6ecd0b4882a..51c2e9a12e00cf 100644
--- a/llvm/utils/TableGen/IntrinsicEmitter.cpp
+++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp
@@ -146,10 +146,10 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
     OS << "#ifdef GET_INTRINSIC_ENUM_VALUES\n";
   } else {
     std::string UpperPrefix = StringRef(IntrinsicPrefix).upper();
-    OS << formatv("#ifndef LLVM_IR_INTRINSIC_{0}_ENUMS_H\n", UpperPrefix);
-    OS << formatv("#define LLVM_IR_INTRINSIC_{0}_ENUMS_H\n", UpperPrefix);
+    OS << formatv("#ifndef LLVM_IR_INTRINSIC_{}_ENUMS_H\n", UpperPrefix);
+    OS << formatv("#define LLVM_IR_INTRINSIC_{}_ENUMS_H\n", UpperPrefix);
     OS << "namespace llvm::Intrinsic {\n";
-    OS << formatv("enum {0}Intrinsics : unsigned {{\n", UpperPrefix);
+    OS << formatv("enum {}Intrinsics : unsigned {{\n", UpperPrefix);
   }
 
   OS << "// Enum values for intrinsics.\n";
@@ -167,12 +167,12 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
     OS << ", ";
     if (Int.EnumName.size() < 40)
       OS.indent(40 - Int.EnumName.size());
-    OS << formatv(" // {0}\n", Int.Name);
+    OS << formatv(" // {}\n", Int.Name);
   }
 
   // Emit num_intrinsics into the target neutral enum.
   if (IntrinsicPrefix.empty()) {
-    OS << formatv("    num_intrinsics = {0}\n", Ints.size() + 1);
+    OS << formatv("    num_intrinsics = {}\n", Ints.size() + 1);
     OS << "#endif\n\n";
   } else {
     OS << R"(}; // enum
@@ -230,7 +230,7 @@ struct IntrinsicTargetInfo {
 static constexpr IntrinsicTargetInfo TargetInfos[] = {
 )";
   for (const auto [Name, Offset, Count] : Ints.Targets)
-    OS << formatv("  {{\"{0}\", {1}, {2}},\n", Name, Offset, Count);
+    OS << formatv("  {{\"{}\", {}, {}},\n", Name, Offset, Count);
   OS << R"(};
 #endif
 
@@ -342,7 +342,7 @@ void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints,
 
   OS << formatv(R"(// Global intrinsic function declaration type table.
 #ifdef GET_INTRINSIC_GENERATOR_GLOBAL
-static constexpr {0} IIT_Table[] = {{
+static constexpr {} IIT_Table[] = {{
   )",
                 FixedEncodingTypeName);
 
@@ -363,7 +363,7 @@ static constexpr {0} IIT_Table[] = {{
 
     // Otherwise, emit the offset into the long encoding table.  We emit it this
     // way so that it is easier to read the offset in the .def file.
-    OS << formatv("(1U<<{0}) | {1}, ", MSBPostion, Offset);
+    OS << formatv("(1U<<{}) | {}, ", MSBPostion, Offset);
   }
 
   OS << "0\n};\n\n";
@@ -494,7 +494,7 @@ static AttributeSet getIntrinsicArgAttributeSet(LLVMContext &C, unsigned ID) {
       assert(is_sorted(Attrs) && "Argument attributes are not sorted");
 
       OS << formatv(R"(
-  case {0}:
+  case {}:
     return AttributeSet::get(C, {{
 )",
                     ID);
@@ -502,10 +502,10 @@ static AttributeSet getIntrinsicArgAttributeSet(LLVMContext &C, unsigned ID) {
         StringRef AttrName = getArgAttrEnumName(Attr.Kind);
         if (Attr.Kind == CodeGenIntrinsic::Alignment ||
             Attr.Kind == CodeGenIntrinsic::Dereferenceable)
-          OS << formatv("      Attribute::get(C, Attribute::{0}, {1}),\n",
+          OS << formatv("      Attribute::get(C, Attribute::{}, {}),\n",
                         AttrName, Attr.Value);
         else
-          OS << formatv("      Attribute::get(C, Attribute::{0}),\n", AttrName);
+          OS << formatv("      Attribute::get(C, Attribute::{}),\n", AttrName);
       }
       OS << "    });";
     }
@@ -530,12 +530,12 @@ static AttributeSet getIntrinsicFnAttributeSet(LLVMContext &C, unsigned ID) {
     if (!UniqFnAttributes.try_emplace(&Int, ID).second)
       continue;
     OS << formatv(R"(
-  case {0}:
+  case {}:
     return AttributeSet::get(C, {{
 )",
                   ID);
     auto addAttribute = [&OS](StringRef Attr) {
-      OS << formatv("      Attribute::get(C, Attribute::{0}),\n", Attr);
+      OS << formatv("      Attribute::get(C, Attribute::{}),\n", Attr);
     };
     if (!Int.canThrow)
       addAttribute("NoUnwind");
@@ -564,9 +564,9 @@ static AttributeSet getIntrinsicFnAttributeSet(LLVMContext &C, unsigned ID) {
 
     const MemoryEffects ME = getEffectiveME(Int);
     if (ME != MemoryEffects::unknown()) {
-      OS << formatv("      // {0}\n", ME);
+      OS << formatv("      // {}\n", ME);
       OS << formatv("      Attribute::getWithMemoryEffects(C, "
-                    "MemoryEffects::createFromIntValue({0})),\n",
+                    "MemoryEffects::createFromIntValue({})),\n",
                     ME.toIntValue());
     }
     OS << "    });";
@@ -598,11 +598,11 @@ AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id) {
   // entry, for the function itself (index ~1), which is usually nounwind.
   OS << "  static constexpr uint16_t IntrinsicsToAttributesMap[] = {";
   for (const CodeGenIntrinsic &Int : Ints)
-    OS << formatv("\n    {0}, // {1}", UniqAttributes[&Int], Int.Name);
+    OS << formatv("\n    {}, // {}", UniqAttributes[&Int], Int.Name);
 
   OS << formatv(R"(
   };
-  std::pair<unsigned, AttributeSet> AS[{0}];
+  std::pair<unsigned, AttributeSet> AS[{}];
   unsigned NumAttrs = 0;
   if (id != 0) {{
     switch(IntrinsicsToAttributesMap[id - 1]) {{
@@ -611,7 +611,7 @@ AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id) {
                 MaxArgAttrs + 1);
 
   for (const auto [IntPtr, UniqueID] : UniqAttributes) {
-    OS << formatv("    case {0}:\n", UniqueID);
+    OS << formatv("    case {}:\n", UniqueID);
     const CodeGenIntrinsic &Int = *IntPtr;
 
     // Keep track of the number of attributes we're writing out.
@@ -623,19 +623,19 @@ AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id) {
 
       unsigned ArgAttrID = UniqArgAttributes.find(Attrs)->second;
       OS << formatv(
-          "      AS[{0}] = {{{1}, getIntrinsicArgAttributeSet(C, {2})};\n",
+          "      AS[{}] = {{{}, getIntrinsicArgAttributeSet(C, {})};\n",
           NumAttrs++, AttrIdx, ArgAttrID);
     }
 
     if (hasFnAttributes(Int)) {
       unsigned FnAttrID = UniqFnAttributes.find(&Int)->second;
-      OS << formatv("      AS[{0}] = {{AttributeList::FunctionIndex, "
-                    "getIntrinsicFnAttributeSet(C, {1})};\n",
+      OS << formatv("      AS[{}] = {{AttributeList::FunctionIndex, "
+                    "getIntrinsicFnAttributeSet(C, {})};\n",
                     NumAttrs++, FnAttrID);
     }
 
     if (NumAttrs) {
-      OS << formatv(R"(      NumAttrs = {0};
+      OS << formatv(R"(      NumAttrs = {};
       break;
 )",
                     NumAttrs);
@@ -709,9 +709,9 @@ void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
 // Get the LLVM intrinsic that corresponds to a builtin. This is used by the
 // C front-end. The builtin name is passed in as BuiltinName, and a target
 // prefix (e.g. 'ppc') is passed in as TargetPrefix.
-#ifdef GET_LLVM_INTRINSIC_FOR_{0}_BUILTIN
+#ifdef GET_LLVM_INTRINSIC_FOR_{}_BUILTIN
 Intrinsic::ID
-Intrinsic::getIntrinsicFor{1}Builtin(StringRef TargetPrefix, 
+Intrinsic::getIntrinsicFor{}Builtin(StringRef TargetPrefix, 
                                       StringRef BuiltinName) {{
   using namespace Intrinsic;
 )",
@@ -721,7 +721,7 @@ Intrinsic::getIntrinsicFor{1}Builtin(StringRef TargetPrefix,
     OS << formatv(R"(
   return not_intrinsic;
   }
-#endif  // GET_LLVM_INTRINSIC_FOR_{0}_BUILTIN
+#endif  // GET_LLVM_INTRINSIC_FOR_{}_BUILTIN
 )",
                   UpperCompilerName);
     return;
@@ -757,14 +757,14 @@ Intrinsic::getIntrinsicFor{1}Builtin(StringRef TargetPrefix,
     }
 
     // Emit the builtin table for this target prefix.
-    OS << formatv("  static constexpr BuiltinEntry {0}Names[] = {{\n",
+    OS << formatv("  static constexpr BuiltinEntry {}Names[] = {{\n",
                   TargetPrefix);
     for (const auto &[BuiltinName, EnumName] : Map) {
       StringRef Suffix = BuiltinName.substr(CommonPrefix->size());
-      OS << formatv("    {{{0}, {1}}, // {2}\n", EnumName,
+      OS << formatv("    {{{}, {}}, // {}\n", EnumName,
                     *Table.GetStringOffset(Suffix), BuiltinName);
     }
-    OS << formatv("  }; // {0}Names\n\n", TargetPrefix);
+    OS << formatv("  }; // {}Names\n\n", TargetPrefix);
   }
 
   // After emitting the builtin tables for all targets, emit a lookup table for
@@ -800,7 +800,7 @@ Intrinsic::getIntrinsicFor{1}Builtin(StringRef TargetPrefix,
   // Copy the builtin name so we can use it in consume_front without clobbering
   // if for the lookup in the target specific table.
   StringRef Suffix = BuiltinName;
-  if (Suffix.consume_front("{0}")) {{
+  if (Suffix.consume_front("{}")) {{
     auto II = lower_bound(Names, Suffix);
     if (II != std::end(Names) && II->getName() == Suffix)
       return II->IntrinsicID;
@@ -823,7 +823,7 @@ Intrinsic::getIntrinsicFor{1}Builtin(StringRef TargetPrefix,
     return not_intrinsic;
   return II->IntrinsicID;
 }
-#endif // GET_LLVM_INTRINSIC_FOR_{0}_BUILTIN
+#endif // GET_LLVM_INTRINSIC_FOR_{}_BUILTIN
 
 )",
                 UpperCompilerName);
diff --git a/llvm/utils/git/requirements.txt b/llvm/utils/git/requirements.txt
index 0fa6b31d860473..9ed52610c53435 100644
--- a/llvm/utils/git/requirements.txt
+++ b/llvm/utils/git/requirements.txt
@@ -4,80 +4,80 @@
 #
 #    pip-compile --generate-hashes --output-file=requirements.txt requirements.txt.in
 #
-certifi==2024.7.4 \
-    --hash=sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b \
-    --hash=sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90
+certifi==2024.8.30 \
+    --hash=sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8 \
+    --hash=sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9
     # via
     #   -r requirements.txt.in
     #   requests
-cffi==1.17.0 \
-    --hash=sha256:011aff3524d578a9412c8b3cfaa50f2c0bd78e03eb7af7aa5e0df59b158efb2f \
-    --hash=sha256:0a048d4f6630113e54bb4b77e315e1ba32a5a31512c31a273807d0027a7e69ab \
-    --hash=sha256:0bb15e7acf8ab35ca8b24b90af52c8b391690ef5c4aec3d31f38f0d37d2cc499 \
-    --hash=sha256:0d46ee4764b88b91f16661a8befc6bfb24806d885e27436fdc292ed7e6f6d058 \
-    --hash=sha256:0e60821d312f99d3e1569202518dddf10ae547e799d75aef3bca3a2d9e8ee693 \
-    --hash=sha256:0fdacad9e0d9fc23e519efd5ea24a70348305e8d7d85ecbb1a5fa66dc834e7fb \
-    --hash=sha256:14b9cbc8f7ac98a739558eb86fabc283d4d564dafed50216e7f7ee62d0d25377 \
-    --hash=sha256:17c6d6d3260c7f2d94f657e6872591fe8733872a86ed1345bda872cfc8c74885 \
-    --hash=sha256:1a2ddbac59dc3716bc79f27906c010406155031a1c801410f1bafff17ea304d2 \
-    --hash=sha256:2404f3de742f47cb62d023f0ba7c5a916c9c653d5b368cc966382ae4e57da401 \
-    --hash=sha256:24658baf6224d8f280e827f0a50c46ad819ec8ba380a42448e24459daf809cf4 \
-    --hash=sha256:24aa705a5f5bd3a8bcfa4d123f03413de5d86e497435693b638cbffb7d5d8a1b \
-    --hash=sha256:2770bb0d5e3cc0e31e7318db06efcbcdb7b31bcb1a70086d3177692a02256f59 \
-    --hash=sha256:331ad15c39c9fe9186ceaf87203a9ecf5ae0ba2538c9e898e3a6967e8ad3db6f \
-    --hash=sha256:3aa9d43b02a0c681f0bfbc12d476d47b2b2b6a3f9287f11ee42989a268a1833c \
-    --hash=sha256:41f4915e09218744d8bae14759f983e466ab69b178de38066f7579892ff2a555 \
-    --hash=sha256:4304d4416ff032ed50ad6bb87416d802e67139e31c0bde4628f36a47a3164bfa \
-    --hash=sha256:435a22d00ec7d7ea533db494da8581b05977f9c37338c80bc86314bec2619424 \
-    --hash=sha256:45f7cd36186db767d803b1473b3c659d57a23b5fa491ad83c6d40f2af58e4dbb \
-    --hash=sha256:48b389b1fd5144603d61d752afd7167dfd205973a43151ae5045b35793232aa2 \
-    --hash=sha256:4e67d26532bfd8b7f7c05d5a766d6f437b362c1bf203a3a5ce3593a645e870b8 \
-    --hash=sha256:516a405f174fd3b88829eabfe4bb296ac602d6a0f68e0d64d5ac9456194a5b7e \
-    --hash=sha256:5ba5c243f4004c750836f81606a9fcb7841f8874ad8f3bf204ff5e56332b72b9 \
-    --hash=sha256:5bdc0f1f610d067c70aa3737ed06e2726fd9d6f7bfee4a351f4c40b6831f4e82 \
-    --hash=sha256:6107e445faf057c118d5050560695e46d272e5301feffda3c41849641222a828 \
-    --hash=sha256:6327b572f5770293fc062a7ec04160e89741e8552bf1c358d1a23eba68166759 \
-    --hash=sha256:669b29a9eca6146465cc574659058ed949748f0809a2582d1f1a324eb91054dc \
-    --hash=sha256:6ce01337d23884b21c03869d2f68c5523d43174d4fc405490eb0091057943118 \
-    --hash=sha256:6d872186c1617d143969defeadac5a904e6e374183e07977eedef9c07c8953bf \
-    --hash=sha256:6f76a90c345796c01d85e6332e81cab6d70de83b829cf1d9762d0a3da59c7932 \
-    --hash=sha256:70d2aa9fb00cf52034feac4b913181a6e10356019b18ef89bc7c12a283bf5f5a \
-    --hash=sha256:7cbc78dc018596315d4e7841c8c3a7ae31cc4d638c9b627f87d52e8abaaf2d29 \
-    --hash=sha256:856bf0924d24e7f93b8aee12a3a1095c34085600aa805693fb7f5d1962393206 \
-    --hash=sha256:8a98748ed1a1df4ee1d6f927e151ed6c1a09d5ec21684de879c7ea6aa96f58f2 \
-    --hash=sha256:93a7350f6706b31f457c1457d3a3259ff9071a66f312ae64dc024f049055f72c \
-    --hash=sha256:964823b2fc77b55355999ade496c54dde161c621cb1f6eac61dc30ed1b63cd4c \
-    --hash=sha256:a003ac9edc22d99ae1286b0875c460351f4e101f8c9d9d2576e78d7e048f64e0 \
-    --hash=sha256:a0ce71725cacc9ebf839630772b07eeec220cbb5f03be1399e0457a1464f8e1a \
-    --hash=sha256:a47eef975d2b8b721775a0fa286f50eab535b9d56c70a6e62842134cf7841195 \
-    --hash=sha256:a8b5b9712783415695663bd463990e2f00c6750562e6ad1d28e072a611c5f2a6 \
-    --hash=sha256:a9015f5b8af1bb6837a3fcb0cdf3b874fe3385ff6274e8b7925d81ccaec3c5c9 \
-    --hash=sha256:aec510255ce690d240f7cb23d7114f6b351c733a74c279a84def763660a2c3bc \
-    --hash=sha256:b00e7bcd71caa0282cbe3c90966f738e2db91e64092a877c3ff7f19a1628fdcb \
-    --hash=sha256:b50aaac7d05c2c26dfd50c3321199f019ba76bb650e346a6ef3616306eed67b0 \
-    --hash=sha256:b7b6ea9e36d32582cda3465f54c4b454f62f23cb083ebc7a94e2ca6ef011c3a7 \
-    --hash=sha256:bb9333f58fc3a2296fb1d54576138d4cf5d496a2cc118422bd77835e6ae0b9cb \
-    --hash=sha256:c1c13185b90bbd3f8b5963cd8ce7ad4ff441924c31e23c975cb150e27c2bf67a \
-    --hash=sha256:c3b8bd3133cd50f6b637bb4322822c94c5ce4bf0d724ed5ae70afce62187c492 \
-    --hash=sha256:c5d97162c196ce54af6700949ddf9409e9833ef1003b4741c2b39ef46f1d9720 \
-    --hash=sha256:c815270206f983309915a6844fe994b2fa47e5d05c4c4cef267c3b30e34dbe42 \
-    --hash=sha256:cab2eba3830bf4f6d91e2d6718e0e1c14a2f5ad1af68a89d24ace0c6b17cced7 \
-    --hash=sha256:d1df34588123fcc88c872f5acb6f74ae59e9d182a2707097f9e28275ec26a12d \
-    --hash=sha256:d6bdcd415ba87846fd317bee0774e412e8792832e7805938987e4ede1d13046d \
-    --hash=sha256:db9a30ec064129d605d0f1aedc93e00894b9334ec74ba9c6bdd08147434b33eb \
-    --hash=sha256:dbc183e7bef690c9abe5ea67b7b60fdbca81aa8da43468287dae7b5c046107d4 \
-    --hash=sha256:dca802c8db0720ce1c49cce1149ff7b06e91ba15fa84b1d59144fef1a1bc7ac2 \
-    --hash=sha256:dec6b307ce928e8e112a6bb9921a1cb00a0e14979bf28b98e084a4b8a742bd9b \
-    --hash=sha256:df8bb0010fdd0a743b7542589223a2816bdde4d94bb5ad67884348fa2c1c67e8 \
-    --hash=sha256:e4094c7b464cf0a858e75cd14b03509e84789abf7b79f8537e6a72152109c76e \
-    --hash=sha256:e4760a68cab57bfaa628938e9c2971137e05ce48e762a9cb53b76c9b569f1204 \
-    --hash=sha256:eb09b82377233b902d4c3fbeeb7ad731cdab579c6c6fda1f763cd779139e47c3 \
-    --hash=sha256:eb862356ee9391dc5a0b3cbc00f416b48c1b9a52d252d898e5b7696a5f9fe150 \
-    --hash=sha256:ef9528915df81b8f4c7612b19b8628214c65c9b7f74db2e34a646a0a2a0da2d4 \
-    --hash=sha256:f3157624b7558b914cb039fd1af735e5e8049a87c817cc215109ad1c8779df76 \
-    --hash=sha256:f3e0992f23bbb0be00a921eae5363329253c3b86287db27092461c887b791e5e \
-    --hash=sha256:f9338cc05451f1942d0d8203ec2c346c830f8e86469903d5126c1f0a13a2bcbb \
-    --hash=sha256:ffef8fd58a36fb5f1196919638f73dd3ae0db1a878982b27a9a5a176ede4ba91
+cffi==1.17.1 \
+    --hash=sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8 \
+    --hash=sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2 \
+    --hash=sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1 \
+    --hash=sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15 \
+    --hash=sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36 \
+    --hash=sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824 \
+    --hash=sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8 \
+    --hash=sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36 \
+    --hash=sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17 \
+    --hash=sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf \
+    --hash=sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc \
+    --hash=sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3 \
+    --hash=sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed \
+    --hash=sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702 \
+    --hash=sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1 \
+    --hash=sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8 \
+    --hash=sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903 \
+    --hash=sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6 \
+    --hash=sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d \
+    --hash=sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b \
+    --hash=sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e \
+    --hash=sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be \
+    --hash=sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c \
+    --hash=sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683 \
+    --hash=sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9 \
+    --hash=sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c \
+    --hash=sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8 \
+    --hash=sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1 \
+    --hash=sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4 \
+    --hash=sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655 \
+    --hash=sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67 \
+    --hash=sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595 \
+    --hash=sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0 \
+    --hash=sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65 \
+    --hash=sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41 \
+    --hash=sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6 \
+    --hash=sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401 \
+    --hash=sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6 \
+    --hash=sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3 \
+    --hash=sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16 \
+    --hash=sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93 \
+    --hash=sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e \
+    --hash=sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4 \
+    --hash=sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964 \
+    --hash=sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c \
+    --hash=sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576 \
+    --hash=sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0 \
+    --hash=sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3 \
+    --hash=sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662 \
+    --hash=sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3 \
+    --hash=sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff \
+    --hash=sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5 \
+    --hash=sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd \
+    --hash=sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f \
+    --hash=sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5 \
+    --hash=sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14 \
+    --hash=sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d \
+    --hash=sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9 \
+    --hash=sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7 \
+    --hash=sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382 \
+    --hash=sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a \
+    --hash=sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e \
+    --hash=sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a \
+    --hash=sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4 \
+    --hash=sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99 \
+    --hash=sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87 \
+    --hash=sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b
     # via
     #   cryptography
     #   pynacl
@@ -173,34 +173,34 @@ charset-normalizer==3.3.2 \
     --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \
     --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561
     # via requests
-cryptography==43.0.0 \
-    --hash=sha256:0663585d02f76929792470451a5ba64424acc3cd5227b03921dab0e2f27b1709 \
-    --hash=sha256:08a24a7070b2b6804c1940ff0f910ff728932a9d0e80e7814234269f9d46d069 \
-    --hash=sha256:232ce02943a579095a339ac4b390fbbe97f5b5d5d107f8a08260ea2768be8cc2 \
-    --hash=sha256:2905ccf93a8a2a416f3ec01b1a7911c3fe4073ef35640e7ee5296754e30b762b \
-    --hash=sha256:299d3da8e00b7e2b54bb02ef58d73cd5f55fb31f33ebbf33bd00d9aa6807df7e \
-    --hash=sha256:2c6d112bf61c5ef44042c253e4859b3cbbb50df2f78fa8fae6747a7814484a70 \
-    --hash=sha256:31e44a986ceccec3d0498e16f3d27b2ee5fdf69ce2ab89b52eaad1d2f33d8778 \
-    --hash=sha256:3d9a1eca329405219b605fac09ecfc09ac09e595d6def650a437523fcd08dd22 \
-    --hash=sha256:3dcdedae5c7710b9f97ac6bba7e1052b95c7083c9d0e9df96e02a1932e777895 \
-    --hash=sha256:47ca71115e545954e6c1d207dd13461ab81f4eccfcb1345eac874828b5e3eaaf \
-    --hash=sha256:4a997df8c1c2aae1e1e5ac49c2e4f610ad037fc5a3aadc7b64e39dea42249431 \
-    --hash=sha256:51956cf8730665e2bdf8ddb8da0056f699c1a5715648c1b0144670c1ba00b48f \
-    --hash=sha256:5bcb8a5620008a8034d39bce21dc3e23735dfdb6a33a06974739bfa04f853947 \
-    --hash=sha256:64c3f16e2a4fc51c0d06af28441881f98c5d91009b8caaff40cf3548089e9c74 \
-    --hash=sha256:6e2b11c55d260d03a8cf29ac9b5e0608d35f08077d8c087be96287f43af3ccdc \
-    --hash=sha256:7b3f5fe74a5ca32d4d0f302ffe6680fcc5c28f8ef0dc0ae8f40c0f3a1b4fca66 \
-    --hash=sha256:844b6d608374e7d08f4f6e6f9f7b951f9256db41421917dfb2d003dde4cd6b66 \
-    --hash=sha256:9a8d6802e0825767476f62aafed40532bd435e8a5f7d23bd8b4f5fd04cc80ecf \
-    --hash=sha256:aae4d918f6b180a8ab8bf6511a419473d107df4dbb4225c7b48c5c9602c38c7f \
-    --hash=sha256:ac1955ce000cb29ab40def14fd1bbfa7af2017cca696ee696925615cafd0dce5 \
-    --hash=sha256:b88075ada2d51aa9f18283532c9f60e72170041bba88d7f37e49cbb10275299e \
-    --hash=sha256:cb013933d4c127349b3948aa8aaf2f12c0353ad0eccd715ca789c8a0f671646f \
-    --hash=sha256:cc70b4b581f28d0a254d006f26949245e3657d40d8857066c2ae22a61222ef55 \
-    --hash=sha256:e9c5266c432a1e23738d178e51c2c7a5e2ddf790f248be939448c0ba2021f9d1 \
-    --hash=sha256:ea9e57f8ea880eeea38ab5abf9fbe39f923544d7884228ec67d666abd60f5a47 \
-    --hash=sha256:ee0c405832ade84d4de74b9029bedb7b31200600fa524d218fc29bfa371e97f5 \
-    --hash=sha256:fdcb265de28585de5b859ae13e3846a8e805268a823a12a4da2597f1f5afc9f0
+cryptography==43.0.1 \
+    --hash=sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494 \
+    --hash=sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806 \
+    --hash=sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d \
+    --hash=sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062 \
+    --hash=sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2 \
+    --hash=sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4 \
+    --hash=sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1 \
+    --hash=sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85 \
+    --hash=sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84 \
+    --hash=sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042 \
+    --hash=sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d \
+    --hash=sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962 \
+    --hash=sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2 \
+    --hash=sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa \
+    --hash=sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d \
+    --hash=sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365 \
+    --hash=sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96 \
+    --hash=sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47 \
+    --hash=sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d \
+    --hash=sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d \
+    --hash=sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c \
+    --hash=sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb \
+    --hash=sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277 \
+    --hash=sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172 \
+    --hash=sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034 \
+    --hash=sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a \
+    --hash=sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289
     # via pyjwt
 deprecated==1.2.14 \
     --hash=sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c \
@@ -222,9 +222,9 @@ pycparser==2.22 \
     --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \
     --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc
     # via cffi
-pygithub==1.59.1 \
-    --hash=sha256:3d87a822e6c868142f0c2c4bf16cce4696b5a7a4d142a7bd160e1bdf75bc54a9 \
-    --hash=sha256:c44e3a121c15bf9d3a5cc98d94c9a047a5132a9b01d22264627f58ade9ddc217
+pygithub==2.2.0 \
+    --hash=sha256:41042ea53e4c372219db708c38d2ca1fd4fadab75475bac27d89d339596cfad1 \
+    --hash=sha256:e39be7c4dc39418bdd6e3ecab5931c636170b8b21b4d26f9ecf7e6102a3b51c3
     # via -r requirements.txt.in
 pyjwt[crypto]==2.9.0 \
     --hash=sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850 \
@@ -250,10 +250,16 @@ smmap==5.0.1 \
     --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \
     --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da
     # via gitdb
+typing-extensions==4.12.2 \
+    --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \
+    --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8
+    # via pygithub
 urllib3==2.2.2 \
     --hash=sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472 \
     --hash=sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168
-    # via requests
+    # via
+    #   pygithub
+    #   requests
 wrapt==1.16.0 \
     --hash=sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc \
     --hash=sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81 \
diff --git a/llvm/utils/git/requirements.txt.in b/llvm/utils/git/requirements.txt.in
index a8bda5c8114058..512b80b60e1d2d 100644
--- a/llvm/utils/git/requirements.txt.in
+++ b/llvm/utils/git/requirements.txt.in
@@ -4,5 +4,6 @@
 # pip-compile -o requirements.txt  requirements.txt.in
 
 certifi>=2023.7.22  # https://security.snyk.io/vuln/SNYK-PYTHON-CERTIFI-5805047
-PyGithub==1.59.1 # For WorkflowRun.name
+PyGithub==2.2.0 # >=1.59.1 For WorkflowRun.name
+                # >= 2.2.0 for permission arg to Repository.get_collaborators
 GitPython>=3.1.32  # https://security.snyk.io/vuln/SNYK-PYTHON-GITPYTHON-5840584
diff --git a/llvm/utils/gn/secondary/lld/test/BUILD.gn b/llvm/utils/gn/secondary/lld/test/BUILD.gn
index d3836172c56936..dabc578a863cd0 100644
--- a/llvm/utils/gn/secondary/lld/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/test/BUILD.gn
@@ -129,6 +129,7 @@ group("test") {
     "//llvm/tools/llvm-ar:symlinks",
     "//llvm/tools/llvm-as",
     "//llvm/tools/llvm-bcanalyzer",
+    "//llvm/tools/llvm-cgdata",
     "//llvm/tools/llvm-cvtres",
     "//llvm/tools/llvm-dis",
     "//llvm/tools/llvm-dwarfdump",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
index 67451c04dcef62..1bc8ebc9ded1f3 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
@@ -68,6 +68,15 @@ tablegen("RISCVGenPostLegalizeGICombiner") {
   td_file = "RISCVGISel.td"
 }
 
+tablegen("RISCVGenPostLegalizeGILowering") {
+  visibility = [ ":LLVMRISCVCodeGen" ]
+  args = [
+    "-gen-global-isel-combiner",
+    "-combiners=RISCVPostLegalizerLowering",
+  ]
+  td_file = "RISCVGISel.td"
+}
+
 tablegen("RISCVGenRegisterBank") {
   visibility = [ ":LLVMRISCVCodeGen" ]
   args = [ "-gen-register-bank" ]
@@ -83,6 +92,7 @@ static_library("LLVMRISCVCodeGen") {
     ":RISCVGenMacroFusion",
     ":RISCVGenO0PreLegalizeGICombiner",
     ":RISCVGenPostLegalizeGICombiner",
+    ":RISCVGenPostLegalizeGILowering",
     ":RISCVGenPreLegalizeGICombiner",
     ":RISCVGenRegisterBank",
 
@@ -109,6 +119,7 @@ static_library("LLVMRISCVCodeGen") {
     "GISel/RISCVLegalizerInfo.cpp",
     "GISel/RISCVO0PreLegalizerCombiner.cpp",
     "GISel/RISCVPostLegalizerCombiner.cpp",
+    "GISel/RISCVPostLegalizerLowering.cpp",
     "GISel/RISCVPreLegalizerCombiner.cpp",
     "GISel/RISCVRegisterBankInfo.cpp",
     "RISCVAsmPrinter.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
index 24413165eec57c..0900872d444939 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
@@ -13,6 +13,7 @@ static_library("Vectorize") {
     "LoopVectorizationLegality.cpp",
     "LoopVectorize.cpp",
     "SLPVectorizer.cpp",
+    "SandboxVectorizer/DependencyGraph.cpp",
     "SandboxVectorizer/Passes/BottomUpVec.cpp",
     "SandboxVectorizer/SandboxVectorizer.cpp",
     "VPlan.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn
index 02c913ab53bc70..95279099fa53b4 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn
@@ -59,6 +59,7 @@ group("unittests") {
     "Transforms/Scalar:ScalarTests",
     "Transforms/Utils:UtilsTests",
     "Transforms/Vectorize:VectorizeTests",
+    "Transforms/Vectorize/SandboxVectorizer:SandboxVectorizerTests",
     "XRay:XRayTests",
     "tools/llvm-cfi-verify:CFIVerifyTests",
     "tools/llvm-exegesis:LLVMExegesisTests",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/BUILD.gn
new file mode 100644
index 00000000000000..c990fb313f557a
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/BUILD.gn
@@ -0,0 +1,16 @@
+import("//third-party/unittest/unittest.gni")
+
+unittest("SandboxVectorizerTests") {
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/AsmParser",
+    "//llvm/lib/IR",
+    "//llvm/lib/SandboxIR",
+    "//llvm/lib/TargetParser",
+    "//llvm/lib/Transforms/Vectorize",
+  ]
+  sources = [
+    "DependencyGraphTest.cpp",
+    "LegalityTest.cpp",
+  ]
+}
diff --git a/mlir/docs/Dialects/OpenMPDialect/_index.md b/mlir/docs/Dialects/OpenMPDialect/_index.md
index 7a3837a661c592..88437b8cf828cc 100644
--- a/mlir/docs/Dialects/OpenMPDialect/_index.md
+++ b/mlir/docs/Dialects/OpenMPDialect/_index.md
@@ -238,6 +238,55 @@ corresponding operation, except if it is explicitly skipped as described
 tablegen failure while processing OpenMP dialect operations, earlier messages
 triggered by that pass can point to a likely solution.
 
+### Operand Structures
+
+One consequence of basing the representation of operations on the set of values
+and attributes defined for each clause applicable to the corresponding OpenMP
+directive is that operation argument lists tend to be long. This has the effect
+of making C++ operation builders difficult to work with and easy to mistakenly
+pass arguments in the wrong order, which may sometimes introduce hard to detect
+problems.
+
+A solution provided to this issue are operand structures. The main idea behind
+them is that there is one defined for each clause, holding a set of fields that
+contain the data needed to initialize each of the arguments associated with that
+clause. Clause operand structures are aggregated into operation operand
+structures via class inheritance. Then, a custom builder is defined for each
+operation taking the corresponding operand structure as a parameter. Since each
+argument is a named member of the structure, it becomes much simpler to set up
+the desired arguments to create a new operation.
+
+Ad-hoc operand structures available for use within the ODS definition of custom
+operation builders might be defined in
+[OpenMPClauseOperands.h](https://github.com/llvm/llvm-project/blob/main/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h).
+However, this is generally not needed for clause-based operation definitions.
+The `-gen-openmp-clause-ops` tablegen backend, triggered when building the 'omp'
+dialect, will automatically produce structures in the following way:
+
+- It will create a `<Name>ClauseOps` structure for each `OpenMP_Clause`
+definition with one field per argument.
+- The name of each field will match the tablegen name of the corresponding
+argument, except for replacing snake case with camel case.
+- The type of the field will be obtained from the corresponding tablegen
+argument's type:
+  - Values are represented with `mlir::Value`, except for `Variadic`, which
+  makes it an `llvm::SmallVector<mlir::Value>`.
+  - `OptionalAttr` is represented by the translation of its `baseAttr`.
+  - `TypedArrayAttrBase`-based attribute types are represented by wrapping the
+  translation of their `elementAttr` in an `llvm::SmallVector`. The only
+  exception for this case is if the `elementAttr` is a "scalar" (i.e. non
+  array-like) attribute type, in which case the more generic `mlir::Attribute`
+  will be used in place of its `storageType`.
+  - For `ElementsAttrBase`-based attribute types a best effort is attempted to
+  obtain an element type (`llvm::APInt`, `llvm::APFloat` or
+  `DenseArrayAttrBase`'s `returnType`) to be wrapped in an `llvm::SmallVector`.
+  If it cannot be obtained, which will happen with non-builtin direct subclasses
+  of `ElementsAttrBase`, a warning will be emitted and the `storageType` (i.e.
+  specific `mlir::Attribute` subclass) will be used instead.
+  - Other attribute types will be represented with their `storageType`.
+- It will create `<Name>Operands` structure for each operation, which is an
+empty structure subclassing all operand structures defined for the corresponding `OpenMP_Op`'s clauses.
+
 ## Loop-Associated Directives
 
 Loop-associated OpenMP constructs are represented in the dialect as loop wrapper
diff --git a/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h b/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h
index a20bae86ace286..67baa8777a6fcc 100644
--- a/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h
@@ -66,14 +66,14 @@ void populateGpuAllReducePatterns(RewritePatternSet &patterns);
 /// Collect a set of patterns to break down subgroup_reduce ops into smaller
 /// ones supported by the target of `size <= maxShuffleBitwidth`, where `size`
 /// is the subgroup_reduce value bitwidth.
-void populateGpuBreakDownSubgrupReducePatterns(RewritePatternSet &patterns,
-                                               unsigned maxShuffleBitwidth = 32,
-                                               PatternBenefit benefit = 1);
+void populateGpuBreakDownSubgroupReducePatterns(
+    RewritePatternSet &patterns, unsigned maxShuffleBitwidth = 32,
+    PatternBenefit benefit = 1);
 
 /// Collect a set of patterns to lower `gpu.subgroup_reduce` into `gpu.shuffle`
 /// ops over `shuffleBitwidth` scalar types. Assumes that the subgroup has
 /// `subgroupSize` lanes. Uses the butterfly shuffle algorithm.
-void populateGpuLowerSubgroupReduceToShufflePattenrs(
+void populateGpuLowerSubgroupReduceToShufflePatterns(
     RewritePatternSet &patterns, unsigned subgroupSize,
     unsigned shuffleBitwidth = 32, PatternBenefit benefit = 1);
 
diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index 164622d77e6b62..4be330b0bb26bb 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -377,7 +377,7 @@ void mlir::populateGpuToNVVMConversionPatterns(LLVMTypeConverter &converter,
       NVVM::ClusterDimZOp>>(converter, IndexKind::Other, IntrType::Dim);
   patterns.add<gpu::index_lowering::OpLowering<
       gpu::BlockIdOp, NVVM::BlockIdXOp, NVVM::BlockIdYOp, NVVM::BlockIdZOp>>(
-      converter, IndexKind::Block, IntrType::Id);
+      converter, IndexKind::Grid, IntrType::Id);
   patterns.add<gpu::index_lowering::OpLowering<
       gpu::GridDimOp, NVVM::GridDimXOp, NVVM::GridDimYOp, NVVM::GridDimZOp>>(
       converter, IndexKind::Grid, IntrType::Dim);
diff --git a/mlir/lib/Debug/Observers/ActionProfiler.cpp b/mlir/lib/Debug/Observers/ActionProfiler.cpp
index 07bf9fd0ccc7db..a6b7d5e18aa077 100644
--- a/mlir/lib/Debug/Observers/ActionProfiler.cpp
+++ b/mlir/lib/Debug/Observers/ActionProfiler.cpp
@@ -58,6 +58,6 @@ void ActionProfiler::print(const ActionActiveStack *action,
   if (printComma)
     os << ",\n";
   printComma = true;
-  os << event.str();
+  os << str;
   os.flush();
 }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp
index 975bfb4d41e0bc..829e954d53b259 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp
@@ -284,9 +284,8 @@ static bool isReachable(Block *from, Block *to, ArrayRef<Block *> except) {
       continue;
     if (next == to)
       return true;
-    if (visited.contains(next))
+    if (!visited.insert(next).second)
       continue;
-    visited.insert(next);
     for (Block *succ : next->getSuccessors())
       worklist.push_back(succ);
   }
@@ -1299,7 +1298,7 @@ static void annotateOpsWithAliasSets(Operation *op,
       std::string buffer;
       llvm::raw_string_ostream stream(buffer);
       alias.printAsOperand(stream, asmState);
-      aliases.push_back(b.getStringAttr(stream.str()));
+      aliases.push_back(b.getStringAttr(buffer));
     });
     return b.getArrayAttr(aliases);
   };
diff --git a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
index 2cf5d12588a73c..b166f1cd469a4d 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
@@ -346,7 +346,7 @@ struct VectorSubgroupReduceToShuffles final
 };
 } // namespace
 
-void mlir::populateGpuBreakDownSubgrupReducePatterns(
+void mlir::populateGpuBreakDownSubgroupReducePatterns(
     RewritePatternSet &patterns, unsigned maxShuffleBitwidth,
     PatternBenefit benefit) {
   patterns.add<BreakDownSubgroupReduce>(patterns.getContext(),
@@ -354,7 +354,7 @@ void mlir::populateGpuBreakDownSubgrupReducePatterns(
   patterns.add<ScalarizeSingleElementReduce>(patterns.getContext(), benefit);
 }
 
-void mlir::populateGpuLowerSubgroupReduceToShufflePattenrs(
+void mlir::populateGpuLowerSubgroupReduceToShufflePatterns(
     RewritePatternSet &patterns, unsigned subgroupSize,
     unsigned shuffleBitwidth, PatternBenefit benefit) {
   patterns.add<ScalarSubgroupReduceToShuffles, VectorSubgroupReduceToShuffles>(
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp
index 6047c4a7ef5150..bd2164e640e7b8 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp
@@ -326,7 +326,7 @@ std::string TargetFeaturesAttr::getFeaturesString() const {
   llvm::raw_string_ostream ss(featuresString);
   llvm::interleave(
       getFeatures(), ss, [&](auto &feature) { ss << feature.strref(); }, ",");
-  return ss.str();
+  return featuresString;
 }
 
 TargetFeaturesAttr TargetFeaturesAttr::featuresAt(Operation *op) {
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index b4ef5d76770856..205d7494d4378c 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -3247,7 +3247,7 @@ LogicalResult LLVMDialect::verifyDataLayoutString(
   std::string message;
   llvm::raw_string_ostream messageStream(message);
   llvm::logAllUnhandledErrors(maybeDataLayout.takeError(), messageStream);
-  reportError("invalid data layout descriptor: " + messageStream.str());
+  reportError("invalid data layout descriptor: " + message);
   return failure();
 }
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index 2c7c3e9d535f7d..5ab64ea1b2097a 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -521,7 +521,7 @@ LogicalResult MmaOp::verify() {
       }
       errorStream << "but got ";
       llvm::interleaveComma(operandTySeg, errorStream);
-      return emitOpError(errorStream.str());
+      return emitOpError(errorMessage);
     }
   }
 
@@ -533,7 +533,7 @@ LogicalResult MmaOp::verify() {
         << "Could not match allowed types for the result; expected one of ";
     llvm::interleaveComma(expectedResult, errorStream);
     errorStream << " but got " << getResult().getType();
-    return emitOpError(errorStream.str());
+    return emitOpError(errorMessage);
   }
 
   // Ensure that binary MMA variants have a b1 MMA operation defined.
@@ -967,7 +967,6 @@ std::string NVVM::WgmmaMmaAsyncOp::getPtx() {
   }
   ss << ";\n"
      << "}\n";
-  ss.flush();
   return ptx;
 }
 
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index b888005625eda7..730c478c2883ef 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -2334,9 +2334,8 @@ std::string mlir::linalg::generateLibraryCallName(Operation *op) {
       return std::string();
     ss << "_";
   }
-  std::string res = ss.str();
-  res.pop_back();
-  return res;
+  name.pop_back();
+  return name;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp
index be4ab361083b9a..5460f50328a452 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp
@@ -209,7 +209,7 @@ DiagnosedSilenceableFailure transform::MatchStructuredBodyOp::matchOperation(
         os);
     if (result)
       return DiagnosedSilenceableFailure::success();
-    return emitSilenceableError() << "contraction: " << os.str();
+    return emitSilenceableError() << "contraction: " << message;
   }
   return emitDefiniteFailure() << "unknown body condition";
 }
@@ -226,7 +226,7 @@ LogicalResult transform::MatchStructuredBodyOp::verify() {
                                                getElementwiseAttrName(),
                                                getContractionAttrName()},
                           os);
-    return emitOpError() << "only one of {" << os.str() << "} is allowed";
+    return emitOpError() << "only one of {" << attributeNames << "} is allowed";
   }
 
   if (std::optional<ArrayAttr> contractionAttr = getContraction()) {
diff --git a/mlir/lib/Dialect/MLProgram/Transforms/PipelineGlobalOps.cpp b/mlir/lib/Dialect/MLProgram/Transforms/PipelineGlobalOps.cpp
index bc7fce95b86a24..40c83487fd47d5 100644
--- a/mlir/lib/Dialect/MLProgram/Transforms/PipelineGlobalOps.cpp
+++ b/mlir/lib/Dialect/MLProgram/Transforms/PipelineGlobalOps.cpp
@@ -100,10 +100,8 @@ LogicalResult MLProgramPipelineGlobals::buildGlobalMap(ModuleOp module) {
     for (size_t i = 0; i < work.size(); ++i) {
       callableMap[work[i]]->walk([&](CallOpInterface call) {
         auto symbol = dyn_cast<SymbolRefAttr>(call.getCallableForCallee());
-        if (!visited.contains(symbol)) {
-          visited.insert(symbol);
+        if (visited.insert(symbol).second)
           work.push_back(symbol);
-        }
       });
 
       for (auto load : opLoadSymbols[work[i]])
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index e4ed58f26016a5..db47276dcefe95 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -1258,7 +1258,7 @@ static void printCaptureType(OpAsmPrinter &p, Operation *op,
     typeCap << "VLAType";
   if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::This)
     typeCap << "This";
-  p << typeCap.str();
+  p << typeCapStr;
 }
 
 static ParseResult parseCaptureType(OpAsmParser &parser,
diff --git a/mlir/lib/Dialect/PDL/IR/PDL.cpp b/mlir/lib/Dialect/PDL/IR/PDL.cpp
index abfa66489cb843..120e963b92cc23 100644
--- a/mlir/lib/Dialect/PDL/IR/PDL.cpp
+++ b/mlir/lib/Dialect/PDL/IR/PDL.cpp
@@ -65,13 +65,10 @@ static void visit(Operation *op, DenseSet<Operation *> &visited) {
   if (!isa<PatternOp>(op->getParentOp()) || isa<RewriteOp>(op))
     return;
 
-  // Ignore if already visited.
-  if (visited.contains(op))
+  // Ignore if already visited.  Otherwise, mark as visited.
+  if (!visited.insert(op).second)
     return;
 
-  // Mark as visited.
-  visited.insert(op);
-
   // Traverse the operands / parent.
   TypeSwitch<Operation *>(op)
       .Case<OperationOp>([&visited](auto operation) {
diff --git a/mlir/lib/Dialect/SCF/Transforms/ForallToFor.cpp b/mlir/lib/Dialect/SCF/Transforms/ForallToFor.cpp
index 5da1b76e929be4..a2f03f1e1056ed 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ForallToFor.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ForallToFor.cpp
@@ -23,8 +23,6 @@ namespace mlir {
 
 using namespace llvm;
 using namespace mlir;
-using scf::ForallOp;
-using scf::ForOp;
 using scf::LoopNest;
 
 LogicalResult
diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
index 5934d85373b03d..e172f02c00f516 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
@@ -85,7 +85,7 @@ static bool haveNoReadsAfterWriteExceptSameIndex(
       return WalkResult::advance();
 
     // Check that at last one store was retrieved
-    if (!write->second.size())
+    if (write->second.empty())
       return WalkResult::interrupt();
 
     auto storeIndices = write->second.front();
diff --git a/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp b/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp
index f467123f9b33de..3297740a5f3ae6 100644
--- a/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/OutlineShapeComputation.cpp
@@ -292,10 +292,8 @@ void OutlineShapeComputationPass::getClusterFromValue(
       cluster.insert(op);
       for (Value inp : op->getOperands()) {
         Operation *inpDefOp = inp.getDefiningOp();
-        if (nullptr != inpDefOp && !visited.contains(inpDefOp)) {
-          visited.insert(inpDefOp);
+        if (nullptr != inpDefOp && visited.insert(inpDefOp).second)
           queue.push(inpDefOp);
-        }
       }
     }
   }
diff --git a/mlir/lib/Dialect/SparseTensor/IR/Detail/Var.cpp b/mlir/lib/Dialect/SparseTensor/IR/Detail/Var.cpp
index 187b3b71e3458b..9e2e6ab3374d41 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/Detail/Var.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/Detail/Var.cpp
@@ -29,7 +29,7 @@ std::string Var::str() const {
   std::string str;
   llvm::raw_string_ostream os(str);
   print(os);
-  return os.str();
+  return str;
 }
 
 void Var::print(AsmPrinter &printer) const { print(printer.getStream()); }
diff --git a/mlir/lib/Dialect/Traits.cpp b/mlir/lib/Dialect/Traits.cpp
index 2efc157ce79617..a7aa25eae2644a 100644
--- a/mlir/lib/Dialect/Traits.cpp
+++ b/mlir/lib/Dialect/Traits.cpp
@@ -223,7 +223,7 @@ static std::string getShapeString(ArrayRef<int64_t> shape) {
       },
       "x");
   ss << '\'';
-  return ss.str();
+  return ret;
 }
 
 LogicalResult OpTrait::impl::verifyCompatibleOperandBroadcast(Operation *op) {
diff --git a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
index 94d4a96a07ad64..7ad017af485f95 100644
--- a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
@@ -60,10 +60,10 @@ DiagnosedSilenceableFailure transform::DebugEmitParamAsRemarkOp::apply(
     os << *getMessage() << " ";
   llvm::interleaveComma(state.getParams(getParam()), os);
   if (!getAnchor()) {
-    emitRemark() << os.str();
+    emitRemark() << str;
     return DiagnosedSilenceableFailure::success();
   }
   for (Operation *payload : state.getPayloadOps(getAnchor()))
-    ::mlir::emitRemark(payload->getLoc()) << os.str();
+    ::mlir::emitRemark(payload->getLoc()) << str;
   return DiagnosedSilenceableFailure::success();
 }
diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
index 14d97964964bb9..590cae9aa0d667 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
@@ -1967,7 +1967,7 @@ transform::MatchParamCmpIOp::apply(transform::TransformRewriter &rewriter,
     std::string str;
     llvm::raw_string_ostream os(str);
     value.print(os, /*isSigned=*/true);
-    return os.str();
+    return str;
   };
 
   ArrayRef<Attribute> params = state.getParams(getParam());
diff --git a/mlir/lib/ExecutionEngine/ExecutionEngine.cpp b/mlir/lib/ExecutionEngine/ExecutionEngine.cpp
index 19917ea4ff1dab..2ecc80400b3db4 100644
--- a/mlir/lib/ExecutionEngine/ExecutionEngine.cpp
+++ b/mlir/lib/ExecutionEngine/ExecutionEngine.cpp
@@ -432,7 +432,7 @@ Expected<void *> ExecutionEngine::lookup(StringRef name) const {
     llvm::raw_string_ostream os(errorMessage);
     llvm::handleAllErrors(expectedSymbol.takeError(),
                           [&os](llvm::ErrorInfoBase &ei) { ei.log(os); });
-    return makeStringError(os.str());
+    return makeStringError(errorMessage);
   }
 
   if (void *fptr = expectedSymbol->toPtr<void *>())
diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp
index c8a30a9dc6f010..f82ea917f3aa25 100644
--- a/mlir/lib/IR/AsmPrinter.cpp
+++ b/mlir/lib/IR/AsmPrinter.cpp
@@ -614,9 +614,8 @@ class AliasInitializer {
   struct InProgressAliasInfo {
     InProgressAliasInfo()
         : aliasDepth(0), isType(false), canBeDeferred(false) {}
-    InProgressAliasInfo(StringRef alias, bool isType, bool canBeDeferred)
-        : alias(alias), aliasDepth(1), isType(isType),
-          canBeDeferred(canBeDeferred) {}
+    InProgressAliasInfo(StringRef alias)
+        : alias(alias), aliasDepth(1), isType(false), canBeDeferred(false) {}
 
     bool operator<(const InProgressAliasInfo &rhs) const {
       // Order first by depth, then by attr/type kind, and then by name.
@@ -1127,6 +1126,8 @@ std::pair<size_t, size_t> AliasInitializer::visitImpl(
 
   // Try to generate an alias for this value.
   generateAlias(value, it->second, canBeDeferred);
+  it->second.isType = std::is_base_of_v<Type, T>;
+  it->second.canBeDeferred = canBeDeferred;
 
   // Print the value, capturing any nested elements that require aliases.
   SmallVector<size_t> childAliases;
@@ -1184,8 +1185,7 @@ void AliasInitializer::generateAlias(T symbol, InProgressAliasInfo &alias,
       sanitizeIdentifier(nameBuffer, tempBuffer, /*allowedPunctChars=*/"$_-",
                          /*allowTrailingDigit=*/false);
   name = name.copy(aliasAllocator);
-  alias = InProgressAliasInfo(name, /*isType=*/std::is_base_of_v<Type, T>,
-                              canBeDeferred);
+  alias = InProgressAliasInfo(name);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/IR/Diagnostics.cpp b/mlir/lib/IR/Diagnostics.cpp
index 6b311a90e0de59..7eb3d5bcd07f19 100644
--- a/mlir/lib/IR/Diagnostics.cpp
+++ b/mlir/lib/IR/Diagnostics.cpp
@@ -146,7 +146,7 @@ Diagnostic &Diagnostic::appendOp(Operation &op, const OpPrintingFlags &flags) {
   // multiple lines.
   if (str.find('\n') != std::string::npos)
     *this << '\n';
-  return *this << os.str();
+  return *this << str;
 }
 
 /// Stream in a Value.
@@ -154,7 +154,7 @@ Diagnostic &Diagnostic::operator<<(Value val) {
   std::string str;
   llvm::raw_string_ostream os(str);
   val.print(os, adjustPrintingFlags(OpPrintingFlags(), severity));
-  return *this << os.str();
+  return *this << str;
 }
 
 /// Outputs this diagnostic to a stream.
@@ -168,7 +168,7 @@ std::string Diagnostic::str() const {
   std::string str;
   llvm::raw_string_ostream os(str);
   print(os);
-  return os.str();
+  return str;
 }
 
 /// Attaches a note to this diagnostic. A new location may be optionally
@@ -451,7 +451,7 @@ void SourceMgrDiagnosticHandler::emitDiagnostic(Location loc, Twine message,
     if (!llvm::isa<UnknownLoc>(loc))
       strOS << loc << ": ";
     strOS << message;
-    return mgr.PrintMessage(os, SMLoc(), getDiagKind(kind), strOS.str());
+    return mgr.PrintMessage(os, SMLoc(), getDiagKind(kind), str);
   }
 
   // Otherwise if we are displaying the source line, try to convert the file
@@ -469,7 +469,7 @@ void SourceMgrDiagnosticHandler::emitDiagnostic(Location loc, Twine message,
   llvm::raw_string_ostream locOS(locStr);
   locOS << fileLoc.getFilename().getValue() << ":" << fileLoc.getLine() << ":"
         << fileLoc.getColumn();
-  llvm::SMDiagnostic diag(locOS.str(), getDiagKind(kind), message.str());
+  llvm::SMDiagnostic diag(locStr, getDiagKind(kind), message.str());
   diag.print(nullptr, os);
 }
 
@@ -637,7 +637,7 @@ struct ExpectedDiag {
       regexOS << '(' << regexStr << ')';
       strToProcess = strToProcess.drop_front(regexEndIt + 2);
     }
-    substringRegex = llvm::Regex(regexOS.str());
+    substringRegex = llvm::Regex(regexStr);
     return success();
   }
 
diff --git a/mlir/lib/Interfaces/DataLayoutInterfaces.cpp b/mlir/lib/Interfaces/DataLayoutInterfaces.cpp
index 2634245a4b7b1e..2158953c07110a 100644
--- a/mlir/lib/Interfaces/DataLayoutInterfaces.cpp
+++ b/mlir/lib/Interfaces/DataLayoutInterfaces.cpp
@@ -29,7 +29,7 @@ using namespace mlir;
   os << "neither the scoping op nor the type class provide data layout "
         "information for "
      << type;
-  llvm::report_fatal_error(Twine(os.str()));
+  llvm::report_fatal_error(Twine(message));
 }
 
 /// Returns the bitwidth of the index type if specified in the param list.
diff --git a/mlir/lib/Interfaces/RuntimeVerifiableOpInterface.cpp b/mlir/lib/Interfaces/RuntimeVerifiableOpInterface.cpp
index 561e8d33868748..8aa194befb4205 100644
--- a/mlir/lib/Interfaces/RuntimeVerifiableOpInterface.cpp
+++ b/mlir/lib/Interfaces/RuntimeVerifiableOpInterface.cpp
@@ -30,7 +30,7 @@ RuntimeVerifiableOpInterface::generateErrorMessage(Operation *op,
   stream << "\n^ " << msg;
   stream << "\nLocation: ";
   op->getLoc().print(stream);
-  return stream.str();
+  return buffer;
 }
 } // namespace mlir
 
diff --git a/mlir/lib/Pass/Pass.cpp b/mlir/lib/Pass/Pass.cpp
index deca03ae137f4e..83b46e8f5680a0 100644
--- a/mlir/lib/Pass/Pass.cpp
+++ b/mlir/lib/Pass/Pass.cpp
@@ -66,7 +66,6 @@ LogicalResult Pass::initializeOptions(
   std::string errStr;
   llvm::raw_string_ostream os(errStr);
   if (failed(passOptions.parseFromString(options, os))) {
-    os.flush();
     return errorHandler(errStr);
   }
   return success();
@@ -700,7 +699,7 @@ std::string OpToOpPassAdaptor::getAdaptorName() {
     os << '\'' << pm.getOpAnchorName() << '\'';
   });
   os << ']';
-  return os.str();
+  return name;
 }
 
 void OpToOpPassAdaptor::runOnOperation() {
diff --git a/mlir/lib/Pass/PassCrashRecovery.cpp b/mlir/lib/Pass/PassCrashRecovery.cpp
index af3071e2566018..8c6d865cb31dd9 100644
--- a/mlir/lib/Pass/PassCrashRecovery.cpp
+++ b/mlir/lib/Pass/PassCrashRecovery.cpp
@@ -321,7 +321,7 @@ void PassCrashReproducerGenerator::prepareReproducerFor(Pass *pass,
     passOS << ")";
 
   impl->activeContexts.push_back(std::make_unique<RecoveryReproducerContext>(
-      passOS.str(), op, impl->streamFactory, impl->pmFlagVerifyPasses));
+      passStr, op, impl->streamFactory, impl->pmFlagVerifyPasses));
 }
 void PassCrashReproducerGenerator::prepareReproducerFor(
     iterator_range<PassManager::pass_iterator> passes, Operation *op) {
@@ -331,7 +331,7 @@ void PassCrashReproducerGenerator::prepareReproducerFor(
       passes, passOS, [&](Pass &pass) { pass.printAsTextualPipeline(passOS); });
 
   impl->activeContexts.push_back(std::make_unique<RecoveryReproducerContext>(
-      passOS.str(), op, impl->streamFactory, impl->pmFlagVerifyPasses));
+      passStr, op, impl->streamFactory, impl->pmFlagVerifyPasses));
 }
 
 void PassCrashReproducerGenerator::removeLastReproducerFor(Pass *pass,
diff --git a/mlir/lib/Query/Matcher/RegistryManager.cpp b/mlir/lib/Query/Matcher/RegistryManager.cpp
index 8c9197f4d00981..645db7109c2deb 100644
--- a/mlir/lib/Query/Matcher/RegistryManager.cpp
+++ b/mlir/lib/Query/Matcher/RegistryManager.cpp
@@ -124,7 +124,7 @@ RegistryManager::getMatcherCompletions(llvm::ArrayRef<ArgKind> acceptedTypes,
     else if (argKinds[0][0] == ArgKind::String)
       typedText += "\"";
 
-    completions.emplace_back(typedText, os.str());
+    completions.emplace_back(typedText, decl);
   }
 
   return completions;
diff --git a/mlir/lib/Query/QueryParser.cpp b/mlir/lib/Query/QueryParser.cpp
index 8a034634c5b89a..13ee931cc5227f 100644
--- a/mlir/lib/Query/QueryParser.cpp
+++ b/mlir/lib/Query/QueryParser.cpp
@@ -123,7 +123,7 @@ makeInvalidQueryFromDiagnostics(const matcher::internal::Diagnostics &diag) {
   std::string errStr;
   llvm::raw_string_ostream os(errStr);
   diag.print(os);
-  return new InvalidQuery(os.str());
+  return new InvalidQuery(errStr);
 }
 } // namespace
 
diff --git a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
index 66ad1e307fc3a5..ad4e9ec1791a77 100644
--- a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
+++ b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
@@ -943,6 +943,25 @@ gpu.module @test_module_49 {
   }
 }
 
+gpu.module @test_module_50 {
+// CHECK-LABEL: func @kernel_with_grid_size(
+  gpu.func @kernel_with_grid_size(%arg0: !llvm.ptr) kernel attributes {known_grid_size = array<i32: 32, 4, 2>} {
+    // CHECK: = nvvm.read.ptx.sreg.ctaid.x range <i32, 0, 32> : i32
+    %0 = gpu.block_id x
+    // CHECK: = nvvm.read.ptx.sreg.ctaid.y range <i32, 0, 4> : i32
+    %1 = gpu.block_id y
+    // CHECK: = nvvm.read.ptx.sreg.ctaid.z range <i32, 0, 2> : i32
+    %2 = gpu.block_id z
+
+    // Fake usage to prevent dead code elimination
+    %3 = arith.addi %0, %1 : index
+    %4 = arith.addi %3, %2 : index
+    %5 = arith.index_cast %4 : index to i64
+    llvm.store %5, %arg0 : i64, !llvm.ptr
+    gpu.return
+  }
+}
+
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%toplevel_module: !transform.any_op {transform.readonly}) {
     %gpu_module = transform.structured.match ops{["gpu.module"]} in %toplevel_module
diff --git a/mlir/test/IR/print-attr-type-aliases.mlir b/mlir/test/IR/print-attr-type-aliases.mlir
index 27c5a75addbb59..e878d862076c90 100644
--- a/mlir/test/IR/print-attr-type-aliases.mlir
+++ b/mlir/test/IR/print-attr-type-aliases.mlir
@@ -1,6 +1,6 @@
-// RUN: mlir-opt %s -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -split-input-file -mlir-print-debuginfo | FileCheck %s
 // Verify printer of type & attr aliases.
-// RUN: mlir-opt %s -split-input-file | mlir-opt -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -split-input-file -mlir-print-debuginfo | mlir-opt -split-input-file -mlir-print-debuginfo | FileCheck %s
 
 // CHECK-DAG: #test2Ealias = "alias_test:dot_in_name"
 "test.op"() {alias_test = "alias_test:dot_in_name"} : () -> ()
@@ -32,16 +32,16 @@
 // CHECK-DAG: tensor<32x!test_ui8_>
 "test.op"() : () -> tensor<32x!test.int<unsigned, 8>>
 
-// CHECK-DAG: #loc = loc("nested")
-// CHECK-DAG: #loc1 = loc("test.mlir":10:8)
-// CHECK-DAG: #loc2 = loc(fused<#loc>[#loc1])
+// CHECK-DAG: #[[LOC_NESTED:.+]] = loc("nested")
+// CHECK-DAG: #[[LOC_RAW:.+]] = loc("test.mlir":10:8)
+// CHECK-DAG: = loc(fused<#[[LOC_NESTED]]>[#[[LOC_RAW]]])
 "test.op"() {alias_test = loc(fused<loc("nested")>["test.mlir":10:8])} : () -> ()
 
 // -----
 
 // Check proper ordering of intermixed attribute/type aliases.
 // CHECK: !tuple = tuple<
-// CHECK: #loc1 = loc(fused<!tuple
+// CHECK: = loc(fused<!tuple
 "test.op"() {alias_test = loc(fused<tuple<i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32>>["test.mlir":10:8])} : () -> ()
 
 // -----
@@ -54,7 +54,7 @@
 // -----
 
 // Check that we don't print aliases for things that aren't printed.
-// CHECK: #loc1 = loc(fused<memref<1xi32>
+// CHECK: = loc(fused<memref<1xi32>
 // CHECK-NOT: #map
 "test.op"() {alias_test = loc(fused<memref<1xi32, affine_map<(d0) -> (d0)>>>["test.mlir":10:8])} : () -> ()
 
@@ -71,3 +71,16 @@
 "test.op"() {attr = #test.conditional_alias<#unalias_me>} : () -> ()
 // CHECK-NEXT: #test.conditional_alias<#test2Ealias>
 "test.op"() {attr = #test.conditional_alias<#keep_aliased>} : () -> ()
+
+// -----
+
+// Check that a deferred no_alias attr can be un-deferred.
+
+#keep_aliased = "alias_test:dot_in_name"
+#cond_alias = #test.conditional_alias<#keep_aliased>
+#no_alias = loc(fused<#cond_alias>["test.mlir":1:1])
+
+// CHECK: #[[TEST_ALIAS:.+]] = "alias_test:dot_in_name"
+// CHECK: fused<#test.conditional_alias<#[[TEST_ALIAS]]>
+// CHECK: "test.op"
+"test.op"() {attr = #no_alias} : () -> () loc(fused<#no_alias>["test.mlir":0:0])
diff --git a/mlir/test/lib/Dialect/GPU/TestGpuRewrite.cpp b/mlir/test/lib/Dialect/GPU/TestGpuRewrite.cpp
index 8c13c0e2575c9c..99a914506b011a 100644
--- a/mlir/test/lib/Dialect/GPU/TestGpuRewrite.cpp
+++ b/mlir/test/lib/Dialect/GPU/TestGpuRewrite.cpp
@@ -75,11 +75,11 @@ struct TestGpuSubgroupReduceLoweringPass
 
     // Since both pattern sets match on the same ops, set higher benefit to
     // perform fewer failing matches.
-    populateGpuBreakDownSubgrupReducePatterns(patterns,
-                                              /*maxShuffleBitwidth=*/32,
-                                              PatternBenefit(2));
+    populateGpuBreakDownSubgroupReducePatterns(patterns,
+                                               /*maxShuffleBitwidth=*/32,
+                                               PatternBenefit(2));
     if (expandToShuffles)
-      populateGpuLowerSubgroupReduceToShufflePattenrs(
+      populateGpuLowerSubgroupReduceToShufflePatterns(
           patterns, /*subgroupSize=*/32, /*shuffleBitwidth=*/32);
 
     (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns));
diff --git a/mlir/test/tblgen-to-irdl/CMathDialect.td b/mlir/test/tblgen-to-irdl/CMathDialect.td
index 454543e074c489..abda7ca41e9db9 100644
--- a/mlir/test/tblgen-to-irdl/CMathDialect.td
+++ b/mlir/test/tblgen-to-irdl/CMathDialect.td
@@ -19,12 +19,14 @@ class CMath_Op<string mnemonic, list<Trait> traits = []>
 def f32Orf64Type : Or<[CPred<"::llvm::isa<::mlir::F32>">,
                        CPred<"::llvm::isa<::mlir::F64>">]>;
 
+// CHECK: irdl.type @"!complex"
 def CMath_ComplexType : CMath_Type<"ComplexType", "complex"> {
   let parameters = (ins f32Orf64Type:$elementType);
+  let assemblyFormat = "`<` $elementType `>`";
 }
 
 // CHECK:      irdl.operation @identity {
-// CHECK-NEXT:   %0 = irdl.base "!cmath.complex"
+// CHECK-NEXT:   %0 = irdl.base @cmath::@"!complex"
 // CHECK-NEXT:   irdl.results(%0)
 // CHECK-NEXT: }
 def CMath_IdentityOp : CMath_Op<"identity"> {
@@ -32,9 +34,9 @@ def CMath_IdentityOp : CMath_Op<"identity"> {
 }
 
 // CHECK:      irdl.operation @mul {
-// CHECK-NEXT:   %0 = irdl.base "!cmath.complex"
-// CHECK-NEXT:   %1 = irdl.base "!cmath.complex"
-// CHECK-NEXT:   %2 = irdl.base "!cmath.complex"
+// CHECK-NEXT:   %0 = irdl.base @cmath::@"!complex"
+// CHECK-NEXT:   %1 = irdl.base @cmath::@"!complex"
+// CHECK-NEXT:   %2 = irdl.base @cmath::@"!complex"
 // CHECK-NEXT:   irdl.operands(%0, %1)
 // CHECK-NEXT:   irdl.results(%2)
 // CHECK-NEXT: }
@@ -45,7 +47,7 @@ def CMath_MulOp : CMath_Op<"mul"> {
 
 // CHECK:      irdl.operation @norm {
 // CHECK-NEXT:   %0 = irdl.any
-// CHECK-NEXT:   %1 = irdl.base "!cmath.complex"
+// CHECK-NEXT:   %1 = irdl.base @cmath::@"!complex"
 // CHECK-NEXT:   irdl.operands(%0)
 // CHECK-NEXT:   irdl.results(%1)
 // CHECK-NEXT: }
diff --git a/mlir/test/tblgen-to-irdl/TestDialect.td b/mlir/test/tblgen-to-irdl/TestDialect.td
index 2622c817760762..4fea3d8576e9ab 100644
--- a/mlir/test/tblgen-to-irdl/TestDialect.td
+++ b/mlir/test/tblgen-to-irdl/TestDialect.td
@@ -16,8 +16,11 @@ class Test_Type<string name, string typeMnemonic, list<Trait> traits = []>
 class Test_Op<string mnemonic, list<Trait> traits = []>
     : Op<Test_Dialect, mnemonic, traits>;
 
+// CHECK: irdl.type @"!singleton_a"
 def Test_SingletonAType : Test_Type<"SingletonAType", "singleton_a"> {}
+// CHECK: irdl.type @"!singleton_b"
 def Test_SingletonBType : Test_Type<"SingletonBType", "singleton_b"> {}
+// CHECK: irdl.type @"!singleton_c"
 def Test_SingletonCType : Test_Type<"SingletonCType", "singleton_c"> {}
 
 
@@ -26,7 +29,7 @@ def Test_AndOp : Test_Op<"and"> {
   let arguments = (ins AllOfType<[Test_SingletonAType, AnyType]>:$in);
 }
 // CHECK-LABEL: irdl.operation @and {
-// CHECK-NEXT:    %[[v0:[^ ]*]] = irdl.base "!test.singleton_a"
+// CHECK-NEXT:    %[[v0:[^ ]*]] = irdl.base @test::@"!singleton_a"
 // CHECK-NEXT:    %[[v1:[^ ]*]] = irdl.any
 // CHECK-NEXT:    %[[v2:[^ ]*]] = irdl.all_of(%[[v0]], %[[v1]])
 // CHECK-NEXT:    irdl.operands(%[[v2]])
@@ -79,9 +82,9 @@ def Test_OrOp : Test_Op<"or"> {
   let arguments = (ins AnyTypeOf<[Test_SingletonAType, Test_SingletonBType, Test_SingletonCType]>:$in);
 }
 // CHECK-LABEL: irdl.operation @or {
-// CHECK-NEXT:    %[[v0:[^ ]*]] = irdl.base "!test.singleton_a"
-// CHECK-NEXT:    %[[v1:[^ ]*]] = irdl.base "!test.singleton_b"
-// CHECK-NEXT:    %[[v2:[^ ]*]] = irdl.base "!test.singleton_c"
+// CHECK-NEXT:    %[[v0:[^ ]*]] = irdl.base @test::@"!singleton_a"
+// CHECK-NEXT:    %[[v1:[^ ]*]] = irdl.base @test::@"!singleton_b"
+// CHECK-NEXT:    %[[v2:[^ ]*]] = irdl.base @test::@"!singleton_c"
 // CHECK-NEXT:    %[[v3:[^ ]*]] = irdl.any_of(%[[v0]], %[[v1]], %[[v2]])
 // CHECK-NEXT:    irdl.operands(%[[v3]])
 // CHECK-NEXT:  }
@@ -114,8 +117,8 @@ def Test_VariadicityOp : Test_Op<"variadicity"> {
                        Test_SingletonCType:$required);
 }
 // CHECK-LABEL: irdl.operation @variadicity {
-// CHECK-NEXT:    %[[v0:[^ ]*]] = irdl.base "!test.singleton_a"
-// CHECK-NEXT:    %[[v1:[^ ]*]] = irdl.base "!test.singleton_b"
-// CHECK-NEXT:    %[[v2:[^ ]*]] = irdl.base "!test.singleton_c"
+// CHECK-NEXT:    %[[v0:[^ ]*]] = irdl.base @test::@"!singleton_a"
+// CHECK-NEXT:    %[[v1:[^ ]*]] = irdl.base @test::@"!singleton_b"
+// CHECK-NEXT:    %[[v2:[^ ]*]] = irdl.base @test::@"!singleton_c"
 // CHECK-NEXT:    irdl.operands(variadic %[[v0]], optional %[[v1]], %[[v2]])
 // CHECK-NEXT:  }
diff --git a/mlir/tools/tblgen-to-irdl/OpDefinitionsGen.cpp b/mlir/tools/tblgen-to-irdl/OpDefinitionsGen.cpp
index dd0d98de496e86..45957bafc378e3 100644
--- a/mlir/tools/tblgen-to-irdl/OpDefinitionsGen.cpp
+++ b/mlir/tools/tblgen-to-irdl/OpDefinitionsGen.cpp
@@ -177,6 +177,15 @@ Value createConstraint(OpBuilder &builder, tblgen::Constraint constraint) {
   }
 
   if (predRec.isSubClassOf("TypeDef")) {
+    auto dialect = predRec.getValueAsDef("dialect")->getValueAsString("name");
+    if (dialect == selectedDialect) {
+      std::string combined = ("!" + predRec.getValueAsString("mnemonic")).str();
+      SmallVector<FlatSymbolRefAttr> nested = {
+          SymbolRefAttr::get(ctx, combined)};
+      auto typeSymbol = SymbolRefAttr::get(ctx, dialect, nested);
+      auto op = builder.create<irdl::BaseOp>(UnknownLoc::get(ctx), typeSymbol);
+      return op.getOutput();
+    }
     std::string typeName = ("!" + predRec.getValueAsString("typeName")).str();
     auto op = builder.create<irdl::BaseOp>(UnknownLoc::get(ctx),
                                            StringAttr::get(ctx, typeName));
@@ -250,6 +259,12 @@ static StringRef getOperatorName(tblgen::Operator &tblgenOp) {
   return opName;
 }
 
+/// Returns the name of the type without the dialect prefix.
+static StringRef getTypeName(tblgen::TypeDef &tblgenType) {
+  StringRef opName = tblgenType.getDef()->getValueAsString("mnemonic");
+  return opName;
+}
+
 /// Extract an operation to IRDL.
 irdl::OperationOp createIRDLOperation(OpBuilder &builder,
                                       tblgen::Operator &tblgenOp) {
@@ -300,6 +315,19 @@ irdl::OperationOp createIRDLOperation(OpBuilder &builder,
   return op;
 }
 
+irdl::TypeOp createIRDLType(OpBuilder &builder, tblgen::TypeDef &tblgenType) {
+  MLIRContext *ctx = builder.getContext();
+  StringRef typeName = getTypeName(tblgenType);
+  std::string combined = ("!" + typeName).str();
+
+  irdl::TypeOp op = builder.create<irdl::TypeOp>(
+      UnknownLoc::get(ctx), StringAttr::get(ctx, combined));
+
+  op.getBody().emplaceBlock();
+
+  return op;
+}
+
 static irdl::DialectOp createIRDLDialect(OpBuilder &builder) {
   MLIRContext *ctx = builder.getContext();
   return builder.create<irdl::DialectOp>(UnknownLoc::get(ctx),
@@ -322,6 +350,14 @@ static bool emitDialectIRDLDefs(const RecordKeeper &recordKeeper,
   // Set insertion point to start of DialectOp.
   builder = builder.atBlockBegin(&dialect.getBody().emplaceBlock());
 
+  for (const Record *type :
+       recordKeeper.getAllDerivedDefinitionsIfDefined("TypeDef")) {
+    tblgen::TypeDef tblgenType(type);
+    if (tblgenType.getDialect().getName() != selectedDialect)
+      continue;
+    createIRDLType(builder, tblgenType);
+  }
+
   for (const Record *def :
        recordKeeper.getAllDerivedDefinitionsIfDefined("Op")) {
     tblgen::Operator tblgenOp(def);
diff --git a/polly/lib/Analysis/ScopDetectionDiagnostic.cpp b/polly/lib/Analysis/ScopDetectionDiagnostic.cpp
index d2fbcf7319856a..14a6f074454f76 100644
--- a/polly/lib/Analysis/ScopDetectionDiagnostic.cpp
+++ b/polly/lib/Analysis/ScopDetectionDiagnostic.cpp
@@ -89,7 +89,6 @@ template <typename T> std::string operator+(Twine LHS, const T &RHS) {
   std::string Buf;
   raw_string_ostream fmt(Buf);
   fmt << RHS;
-  fmt.flush();
 
   return LHS.concat(Buf).str();
 }
@@ -669,7 +668,7 @@ std::string ReportAlias::formatInvalidAlias(std::string Prefix,
 
   OS << Suffix;
 
-  return OS.str();
+  return Message;
 }
 
 std::string ReportAlias::getRemarkName() const { return "Alias"; }
diff --git a/polly/lib/Analysis/ScopInfo.cpp b/polly/lib/Analysis/ScopInfo.cpp
index 044d3573793f01..56ffb990faf1c2 100644
--- a/polly/lib/Analysis/ScopInfo.cpp
+++ b/polly/lib/Analysis/ScopInfo.cpp
@@ -1818,11 +1818,9 @@ std::pair<std::string, std::string> Scop::getEntryExitStr() const {
   raw_string_ostream EntryStr(EntryName);
 
   R.getEntry()->printAsOperand(EntryStr, false);
-  EntryStr.str();
 
   if (R.getExit()) {
     R.getExit()->printAsOperand(ExitStr, false);
-    ExitStr.str();
   } else
     ExitName = "FunctionExit";
 
diff --git a/polly/lib/CodeGen/BlockGenerators.cpp b/polly/lib/CodeGen/BlockGenerators.cpp
index 004fa64c82db2b..c58763603cfa22 100644
--- a/polly/lib/CodeGen/BlockGenerators.cpp
+++ b/polly/lib/CodeGen/BlockGenerators.cpp
@@ -645,7 +645,7 @@ static std::string getInstName(Value *Val) {
   std::string Result;
   raw_string_ostream OS(Result);
   Val->printAsOperand(OS, false);
-  return OS.str();
+  return Result;
 }
 
 void BlockGenerator::generateBeginStmtTrace(ScopStmt &Stmt, LoopToScevMapT &LTS,
diff --git a/polly/lib/Exchange/JSONExporter.cpp b/polly/lib/Exchange/JSONExporter.cpp
index 63fb06a634cc12..dfd63146edb5e2 100644
--- a/polly/lib/Exchange/JSONExporter.cpp
+++ b/polly/lib/Exchange/JSONExporter.cpp
@@ -116,12 +116,12 @@ static json::Array exportArrays(const Scop &S) {
     }
     for (; i < SAI->getNumberOfDimensions(); i++) {
       SAI->getDimensionSize(i)->print(RawStringOstream);
-      Sizes.push_back(RawStringOstream.str());
+      Sizes.push_back(Buffer);
       Buffer.clear();
     }
     Array["sizes"] = std::move(Sizes);
     SAI->getElementType()->print(RawStringOstream);
-    Array["type"] = RawStringOstream.str();
+    Array["type"] = Buffer;
     Buffer.clear();
     Arrays.push_back(std::move(Array));
   }
@@ -575,14 +575,14 @@ static bool areArraysEqual(ScopArrayInfo *SAI, const json::Object &Array) {
   for (unsigned i = 1; i < Array.getArray("sizes")->size(); i++) {
     SAI->getDimensionSize(i)->print(RawStringOstream);
     const json::Array &SizesArray = *Array.getArray("sizes");
-    if (RawStringOstream.str() != SizesArray[i].getAsString().value())
+    if (Buffer != SizesArray[i].getAsString().value())
       return false;
     Buffer.clear();
   }
 
   // Check if key 'type' differs from the current one or is not valid.
   SAI->getElementType()->print(RawStringOstream);
-  if (RawStringOstream.str() != Array.getString("type").value()) {
+  if (Buffer != Array.getString("type").value()) {
     errs() << "Array has not a valid type.\n";
     return false;
   }
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 664bb623f5f55c..b6ccf1a659caf4 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -1084,25 +1084,26 @@ libc_support_library(
     name = "__support_osutil_vdso",
     hdrs = [
         "src/__support/OSUtil/linux/vdso.h",
-        "src/__support/OSUtil/linux/vdso_sym.h"
+        "src/__support/OSUtil/linux/vdso_sym.h",
     ],
+    target_compatible_with = select({
+        "@platforms//os:linux": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
     textual_hdrs = [
         "src/__support/OSUtil/linux/riscv/vdso.h",
         "src/__support/OSUtil/linux/arm/vdso.h",
         "src/__support/OSUtil/linux/x86_64/vdso.h",
         "src/__support/OSUtil/linux/aarch64/vdso.h",
     ],
-    target_compatible_with = select({
-        "@platforms//os:linux": [],
-        "//conditions:default": ["@platforms//:incompatible"],
-    }),
     deps = [
         ":__support_cpp_array",
         ":__support_cpp_optional",
         ":__support_cpp_string_view",
         ":__support_threads_callonce",
         ":types_clock_t",
-        ":types_struct_timeval"
+        ":types_clockid_t",
+        ":types_struct_timeval",
     ],
 )
 
@@ -1190,7 +1191,7 @@ libc_support_library(
         ":__support_cpp_optional",
         ":__support_osutil_syscall",
         ":__support_threads_linux_futex_word_type",
-        ":__support_time_linux",
+        ":__support_time_linux_abs_timeout",
         ":types_struct_timespec",
     ],
 )
@@ -1218,10 +1219,12 @@ libc_support_library(
         "//conditions:default": ["@platforms//:incompatible"],
     }),
     deps = [
+        ":__support_cpp_expected",
         ":__support_cpp_optional",
         ":__support_threads_linux_futex_utils",
         ":__support_threads_sleep",
-        ":__support_time_linux",
+        ":__support_time_linux_abs_timeout",
+        ":__support_time_linux_monotonicity",
         ":types_pid_t",
     ],
 )
@@ -1248,7 +1251,6 @@ libc_support_library(
     ],
 )
 
-
 libc_support_library(
     name = "__support_threads_callonce",
     hdrs = [
@@ -1263,7 +1265,8 @@ libc_support_library(
     ],
     deps = [
         ":__support_macros_config",
-        ":__support_macros_optimization"
+        ":__support_macros_optimization",
+        ":__support_threads_linux_futex_utils",
     ],
 )
 
@@ -1278,25 +1281,62 @@ libc_support_library(
 )
 
 libc_support_library(
-    name = "__support_time_linux",
-    hdrs = glob(["src/__support/time/linux/**/*.h"]),
+    name = "__support_time_linux_abs_timeout",
+    hdrs = ["src/__support/time/linux/abs_timeout.h"],
     target_compatible_with = select({
         "@platforms//os:linux": [],
         "//conditions:default": ["@platforms//:incompatible"],
     }),
     deps = [
-        ":__support_common",
         ":__support_cpp_expected",
-        ":__support_error_or",
-        ":__support_libc_assert",
-        ":__support_osutil_vdso",
         ":__support_time",
         ":hdr_time_macros",
-        ":types_clockid_t",
         ":types_struct_timespec",
     ],
 )
 
+libc_support_library(
+    name = "__support_time_linux_clock_conversion",
+    hdrs = ["src/__support/time/linux/clock_conversion.h"],
+    target_compatible_with = select({
+        "@platforms//os:linux": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    deps = [
+        ":__support_time",
+        ":__support_time_linux_clock_gettime",
+    ],
+)
+
+libc_support_library(
+    name = "__support_time_linux_clock_gettime",
+    hdrs = ["src/__support/time/linux/clock_gettime.h"],
+    target_compatible_with = select({
+        "@platforms//os:linux": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    deps = [
+        ":__support_common",
+        ":__support_error_or",
+        ":__support_osutil_vdso",
+    ],
+)
+
+libc_support_library(
+    name = "__support_time_linux_monotonicity",
+    hdrs = ["src/__support/time/linux/monotonicity.h"],
+    target_compatible_with = select({
+        "@platforms//os:linux": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    deps = [
+        ":__support_libc_assert",
+        ":__support_time_linux_abs_timeout",
+        ":__support_time_linux_clock_conversion",
+        ":hdr_time_macros",
+    ],
+)
+
 ############################### errno targets ################################
 
 libc_function(
diff --git a/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl b/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl
index f780c323d9a996..f65da9e98226b6 100644
--- a/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl
@@ -25,7 +25,7 @@ LIBC_CONFIGURE_OPTIONS = [
     # "LIBC_COPT_MEMCPY_USE_EMBEDDED_TINY",
     # "LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE",
     # "LIBC_COPT_MEMCPY_X86_USE_SOFTWARE_PREFETCHING",
-    # "LIBC_COPT_MEMSET_X86_USE_SOFTWARE_PREFETCHING",
+    "LIBC_COPT_MEMSET_X86_USE_SOFTWARE_PREFETCHING",
 
     # Documentation in libc/docs/dev/printf_behavior.rst
     # "LIBC_COPT_PRINTF_CONV_ATLAS",
diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index bbb0435837e415..4b458e07e0b3a2 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -2362,6 +2362,7 @@ gentbl(
         ("-gen-global-isel-combiner -combiners=RISCVO0PreLegalizerCombiner", "lib/Target/RISCV/RISCVGenO0PreLegalizeGICombiner.inc"),
         ("-gen-global-isel-combiner -combiners=RISCVPostLegalizerCombiner", "lib/Target/RISCV/RISCVGenPostLegalizeGICombiner.inc"),
         ("-gen-global-isel-combiner -combiners=RISCVPreLegalizerCombiner", "lib/Target/RISCV/RISCVGenPreLegalizeGICombiner.inc"),
+        ("-gen-global-isel-combiner -combiners=RISCVPostLegalizerLowering", "lib/Target/RISCV/RISCVGenPostLegalizeGILowering.inc"),
     ],
     tblgen = ":llvm-tblgen",
     td_file = "lib/Target/RISCV/RISCVGISel.td",